From 0897fcb1c1e7316375166e0a665237bce2391a09 Mon Sep 17 00:00:00 2001 From: Quanfa Fu Date: Sun, 15 Jan 2023 00:49:52 +0800 Subject: [PATCH 001/475] apparmor: make aa_set_current_onexec return void Change the return type to void since it always return 0, and no need to do the checking in aa_set_current_onexec. Signed-off-by: Quanfa Fu Reviewed-by: "Tyler Hicks (Microsoft)" Signed-off-by: John Johansen --- security/apparmor/domain.c | 2 +- security/apparmor/include/task.h | 2 +- security/apparmor/task.c | 5 +---- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/security/apparmor/domain.c b/security/apparmor/domain.c index f3715cda59c5..68598557aef5 100644 --- a/security/apparmor/domain.c +++ b/security/apparmor/domain.c @@ -1446,7 +1446,7 @@ check: } /* full transition will be built in exec path */ - error = aa_set_current_onexec(target, stack); + aa_set_current_onexec(target, stack); } audit: diff --git a/security/apparmor/include/task.h b/security/apparmor/include/task.h index 13437d62c70f..01717fe432c3 100644 --- a/security/apparmor/include/task.h +++ b/security/apparmor/include/task.h @@ -30,7 +30,7 @@ struct aa_task_ctx { }; int aa_replace_current_label(struct aa_label *label); -int aa_set_current_onexec(struct aa_label *label, bool stack); +void aa_set_current_onexec(struct aa_label *label, bool stack); int aa_set_current_hat(struct aa_label *label, u64 token); int aa_restore_previous_label(u64 cookie); struct aa_label *aa_get_task_label(struct task_struct *task); diff --git a/security/apparmor/task.c b/security/apparmor/task.c index 84d16a29bfcb..5671a716fcd2 100644 --- a/security/apparmor/task.c +++ b/security/apparmor/task.c @@ -93,9 +93,8 @@ int aa_replace_current_label(struct aa_label *label) * aa_set_current_onexec - set the tasks change_profile to happen onexec * @label: system label to set at exec (MAYBE NULL to clear value) * @stack: whether stacking should be done - * Returns: 0 or error on failure */ -int aa_set_current_onexec(struct aa_label *label, bool stack) +void aa_set_current_onexec(struct aa_label *label, bool stack) { struct aa_task_ctx *ctx = task_ctx(current); @@ -103,8 +102,6 @@ int aa_set_current_onexec(struct aa_label *label, bool stack) aa_put_label(ctx->onexec); ctx->onexec = label; ctx->token = stack; - - return 0; } /** From 8de4a7de1950e88c233b105faf24666db348e65a Mon Sep 17 00:00:00 2001 From: "GONG, Ruiqi" Date: Wed, 31 May 2023 19:18:33 +0800 Subject: [PATCH 002/475] apparmor: remove unused macro SOCK_ctx() doesn't seem to be used anywhere in the code, so remove it. Signed-off-by: GONG, Ruiqi Signed-off-by: John Johansen --- security/apparmor/include/net.h | 1 - 1 file changed, 1 deletion(-) diff --git a/security/apparmor/include/net.h b/security/apparmor/include/net.h index 6fa440b5daed..64a0112bf62d 100644 --- a/security/apparmor/include/net.h +++ b/security/apparmor/include/net.h @@ -52,7 +52,6 @@ struct aa_sk_ctx { }; #define SK_CTX(X) ((X)->sk_security) -#define SOCK_ctx(X) SOCK_INODE(X)->i_security #define DEFINE_AUDIT_NET(NAME, OP, SK, F, T, P) \ struct lsm_network_audit NAME ## _net = { .sk = (SK), \ .family = (F)}; \ From 180cf257998c5f136f76b8899ef6ec57b410680b Mon Sep 17 00:00:00 2001 From: John Johansen Date: Thu, 16 Mar 2023 16:04:17 -0700 Subject: [PATCH 003/475] apparmor: advertise availability of exended perms Userspace won't load policy using extended perms unless it knows the kernel can handle them. Advertise that extended perms are supported in the feature set. Signed-off-by: John Johansen Reviewed-by: Jon Tourville --- security/apparmor/apparmorfs.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/security/apparmor/apparmorfs.c b/security/apparmor/apparmorfs.c index db7a51acf9db..0e8d690c911b 100644 --- a/security/apparmor/apparmorfs.c +++ b/security/apparmor/apparmorfs.c @@ -2348,11 +2348,14 @@ static struct aa_sfs_entry aa_sfs_entry_versions[] = { { } }; +#define PERMS32STR "allow deny subtree cond kill complain prompt audit quiet hide xindex tag label" static struct aa_sfs_entry aa_sfs_entry_policy[] = { AA_SFS_DIR("versions", aa_sfs_entry_versions), AA_SFS_FILE_BOOLEAN("set_load", 1), /* number of out of band transitions supported */ AA_SFS_FILE_U64("outofband", MAX_OOB_SUPPORTED), + AA_SFS_FILE_U64("permstable32_version", 1), + AA_SFS_FILE_STRING("permstable32", PERMS32STR), { } }; From afad53575a938ceb557227ecfeb0dda59d668d4e Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Wed, 14 Jun 2023 16:02:04 +0300 Subject: [PATCH 004/475] apparmor: use passed in gfp flags in aa_alloc_null() These allocations should use the gfp flags from the caller instead of GFP_KERNEL. But from what I can see, all the callers pass in GFP_KERNEL so this does not affect runtime. Fixes: e31dd6e412f7 ("apparmor: fix: kzalloc perms tables for shared dfas") Signed-off-by: Dan Carpenter Signed-off-by: John Johansen --- security/apparmor/policy.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/security/apparmor/policy.c b/security/apparmor/policy.c index b38f7b2a5e1d..715fe1b66d12 100644 --- a/security/apparmor/policy.c +++ b/security/apparmor/policy.c @@ -589,12 +589,12 @@ struct aa_profile *aa_alloc_null(struct aa_profile *parent, const char *name, profile->label.flags |= FLAG_NULL; rules = list_first_entry(&profile->rules, typeof(*rules), list); rules->file.dfa = aa_get_dfa(nulldfa); - rules->file.perms = kcalloc(2, sizeof(struct aa_perms), GFP_KERNEL); + rules->file.perms = kcalloc(2, sizeof(struct aa_perms), gfp); if (!rules->file.perms) goto fail; rules->file.size = 2; rules->policy.dfa = aa_get_dfa(nulldfa); - rules->policy.perms = kcalloc(2, sizeof(struct aa_perms), GFP_KERNEL); + rules->policy.perms = kcalloc(2, sizeof(struct aa_perms), gfp); if (!rules->policy.perms) goto fail; rules->policy.size = 2; From 26c9ecb34f5f5fa43c041a220de01d7cbea97dd0 Mon Sep 17 00:00:00 2001 From: Gaosheng Cui Date: Sun, 25 Jun 2023 09:13:39 +0800 Subject: [PATCH 005/475] apparmor: Fix kernel-doc warnings in apparmor/audit.c Fix kernel-doc warnings: security/apparmor/audit.c:150: warning: Function parameter or member 'type' not described in 'aa_audit_msg' Signed-off-by: Gaosheng Cui Signed-off-by: John Johansen --- security/apparmor/audit.c | 1 + 1 file changed, 1 insertion(+) diff --git a/security/apparmor/audit.c b/security/apparmor/audit.c index 5a7978aa4b19..a3db0f8bd4f8 100644 --- a/security/apparmor/audit.c +++ b/security/apparmor/audit.c @@ -142,6 +142,7 @@ static void audit_pre(struct audit_buffer *ab, void *ca) /** * aa_audit_msg - Log a message to the audit subsystem + * @type: audit type for the message * @sa: audit event structure (NOT NULL) * @cb: optional callback fn for type specific fields (MAYBE NULL) */ From c98c8972feb3bdf2e197e63403305176217c78a7 Mon Sep 17 00:00:00 2001 From: Gaosheng Cui Date: Sun, 25 Jun 2023 09:13:40 +0800 Subject: [PATCH 006/475] apparmor: Fix kernel-doc warnings in apparmor/capability.c Fix kernel-doc warnings: security/apparmor/capability.c:45: warning: Function parameter or member 'ab' not described in 'audit_cb' security/apparmor/capability.c:45: warning: Function parameter or member 'va' not described in 'audit_cb' Signed-off-by: Gaosheng Cui Signed-off-by: John Johansen --- security/apparmor/capability.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/security/apparmor/capability.c b/security/apparmor/capability.c index 326a51838ef2..8ebf0fcb796c 100644 --- a/security/apparmor/capability.c +++ b/security/apparmor/capability.c @@ -38,8 +38,8 @@ static DEFINE_PER_CPU(struct audit_cache, audit_cache); /** * audit_cb - call back for capability components of audit struct - * @ab - audit buffer (NOT NULL) - * @va - audit struct to audit data from (NOT NULL) + * @ab: audit buffer (NOT NULL) + * @va: audit struct to audit data from (NOT NULL) */ static void audit_cb(struct audit_buffer *ab, void *va) { From 76426c9d92fdbd3dd88b1c3553bc4a4d98e6d184 Mon Sep 17 00:00:00 2001 From: Gaosheng Cui Date: Sun, 25 Jun 2023 09:13:41 +0800 Subject: [PATCH 007/475] apparmor: Fix kernel-doc warnings in apparmor/domain.c Fix kernel-doc warnings: security/apparmor/domain.c:279: warning: Function parameter or member 'perms' not described in 'change_profile_perms' security/apparmor/domain.c:380: warning: Function parameter or member 'bprm' not described in 'find_attach' security/apparmor/domain.c:380: warning: Function parameter or member 'head' not described in 'find_attach' security/apparmor/domain.c:380: warning: Function parameter or member 'info' not described in 'find_attach' security/apparmor/domain.c:380: warning: Function parameter or member 'name' not described in 'find_attach' security/apparmor/domain.c:558: warning: Function parameter or member 'info' not described in 'x_to_label' Signed-off-by: Gaosheng Cui Signed-off-by: John Johansen --- security/apparmor/domain.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/security/apparmor/domain.c b/security/apparmor/domain.c index 68598557aef5..7f81502019da 100644 --- a/security/apparmor/domain.c +++ b/security/apparmor/domain.c @@ -265,6 +265,7 @@ static int label_match(struct aa_profile *profile, struct aa_label *label, * @stack: whether this is a stacking request * @request: requested perms * @start: state to start matching in + * @perms: Returns computed perms (NOT NULL) * * * Returns: permission set @@ -359,11 +360,11 @@ out: /** * find_attach - do attachment search for unconfined processes - * @bprm - binprm structure of transitioning task + * @bprm: binprm structure of transitioning task * @ns: the current namespace (NOT NULL) - * @head - profile list to walk (NOT NULL) - * @name - to match against (NOT NULL) - * @info - info message if there was an error (NOT NULL) + * @head: profile list to walk (NOT NULL) + * @name: to match against (NOT NULL) + * @info: info message if there was an error (NOT NULL) * * Do a linear search on the profiles in the list. There is a matching * preference where an exact match is preferred over a name which uses @@ -545,6 +546,7 @@ struct aa_label *x_table_lookup(struct aa_profile *profile, u32 xindex, * @name: name to lookup (NOT NULL) * @xindex: index into x transition table * @lookupname: returns: name used in lookup if one was specified (NOT NULL) + * @info: info message if there was an error (NOT NULL) * * find label for a transition index * From 3175df8032d9486c6d6875256144aa027d905d3f Mon Sep 17 00:00:00 2001 From: Gaosheng Cui Date: Sun, 25 Jun 2023 09:13:42 +0800 Subject: [PATCH 008/475] apparmor: Fix kernel-doc warnings in apparmor/file.c Fix kernel-doc warnings: security/apparmor/file.c:177: warning: Excess function parameter 'dfa' description in 'aa_lookup_fperms' security/apparmor/file.c:177: warning: Function parameter or member 'file_rules' not described in 'aa_lookup_fperms' security/apparmor/file.c:202: warning: Excess function parameter 'dfa' description in 'aa_str_perms' security/apparmor/file.c:202: warning: Excess function parameter 'state' description in 'aa_str_perms' security/apparmor/file.c:202: warning: Function parameter or member 'file_rules' not described in 'aa_str_perms' security/apparmor/file.c:202: warning: Function parameter or member 'start' not described in 'aa_str_perms' Signed-off-by: Gaosheng Cui Signed-off-by: John Johansen --- security/apparmor/file.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/security/apparmor/file.c b/security/apparmor/file.c index 698b124e649f..c0fc5e77e889 100644 --- a/security/apparmor/file.c +++ b/security/apparmor/file.c @@ -164,7 +164,7 @@ static int path_name(const char *op, struct aa_label *label, struct aa_perms default_perms = {}; /** * aa_lookup_fperms - convert dfa compressed perms to internal perms - * @dfa: dfa to lookup perms for (NOT NULL) + * @file_rules: the aa_policydb to lookup perms for (NOT NULL) * @state: state in dfa * @cond: conditions to consider (NOT NULL) * @@ -188,8 +188,8 @@ struct aa_perms *aa_lookup_fperms(struct aa_policydb *file_rules, /** * aa_str_perms - find permission that match @name - * @dfa: to match against (MAYBE NULL) - * @state: state to start matching in + * @file_rules: the aa_policydb to match against (NOT NULL) + * @start: state to start matching in * @name: string to match against dfa (NOT NULL) * @cond: conditions to consider for permission set computation (NOT NULL) * @perms: Returns - the permissions found when matching @name From e18573dd2bacead6faa8aa09df8536e7c1b86e57 Mon Sep 17 00:00:00 2001 From: Gaosheng Cui Date: Sun, 25 Jun 2023 09:13:43 +0800 Subject: [PATCH 009/475] apparmor: Fix kernel-doc warnings in apparmor/label.c Fix kernel-doc warnings: security/apparmor/label.c:166: warning: Excess function parameter 'n' description in 'vec_cmp' security/apparmor/label.c:166: warning: Excess function parameter 'vec' description in 'vec_cmp' security/apparmor/label.c:166: warning: Function parameter or member 'an' not described in 'vec_cmp' security/apparmor/label.c:166: warning: Function parameter or member 'bn' not described in 'vec_cmp' security/apparmor/label.c:166: warning: Function parameter or member 'b' not described in 'vec_cmp' security/apparmor/label.c:2051: warning: Function parameter or member 'label' not described in '__label_update' security/apparmor/label.c:266: warning: Function parameter or member 'flags' not described in 'aa_vec_unique' security/apparmor/label.c:594: warning: Excess function parameter 'l' description in '__label_remove' security/apparmor/label.c:594: warning: Function parameter or member 'label' not described in '__label_remove' security/apparmor/label.c:929: warning: Function parameter or member 'label' not described in 'aa_label_insert' security/apparmor/label.c:929: warning: Function parameter or member 'ls' not described in 'aa_label_insert' security/apparmor/label.c:1221: warning: Excess function parameter 'ls' description in 'aa_label_merge' security/apparmor/label.c:1302: warning: Excess function parameter 'start' description in 'label_compound_match' security/apparmor/label.c:1302: warning: Function parameter or member 'rules' not described in 'label_compound_match' security/apparmor/label.c:1302: warning: Function parameter or member 'state' not described in 'label_compound_match' security/apparmor/label.c:2051: warning: Function parameter or member 'label' not described in '__label_update' Signed-off-by: Gaosheng Cui Signed-off-by: John Johansen --- security/apparmor/label.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/security/apparmor/label.c b/security/apparmor/label.c index 8a2af96f4da5..ce9c39102a93 100644 --- a/security/apparmor/label.c +++ b/security/apparmor/label.c @@ -154,9 +154,10 @@ static int profile_cmp(struct aa_profile *a, struct aa_profile *b) /** * vec_cmp - label comparison for set ordering - * @a: label to compare (NOT NULL) - * @vec: vector of profiles to compare (NOT NULL) - * @n: length of @vec + * @a: aa_profile to compare (NOT NULL) + * @an: length of @a + * @b: aa_profile to compare (NOT NULL) + * @bn: length of @b * * Returns: <0 if a < vec * ==0 if a == vec @@ -256,6 +257,7 @@ static inline int unique(struct aa_profile **vec, int n) * aa_vec_unique - canonical sort and unique a list of profiles * @n: number of refcounted profiles in the list (@n > 0) * @vec: list of profiles to sort and merge + * @flags: null terminator flags of @vec * * Returns: the number of duplicates eliminated == references put * @@ -584,7 +586,7 @@ bool aa_label_is_unconfined_subset(struct aa_label *set, struct aa_label *sub) /** * __label_remove - remove @label from the label set - * @l: label to remove + * @label: label to remove * @new: label to redirect to * * Requires: labels_set(@label)->lock write_lock @@ -917,8 +919,8 @@ struct aa_label *aa_label_find(struct aa_label *label) /** * aa_label_insert - insert label @label into @ls or return existing label - * @ls - labelset to insert @label into - * @label - label to insert + * @ls: labelset to insert @label into + * @label: label to insert * * Requires: caller to hold a valid ref on @label * @@ -1204,7 +1206,6 @@ struct aa_label *aa_label_find_merge(struct aa_label *a, struct aa_label *b) /** * aa_label_merge - attempt to insert new merged label of @a and @b - * @ls: set of labels to insert label into (NOT NULL) * @a: label to merge with @b (NOT NULL) * @b: label to merge with @a (NOT NULL) * @gfp: memory allocation type @@ -1282,8 +1283,9 @@ static inline aa_state_t match_component(struct aa_profile *profile, /** * label_compound_match - find perms for full compound label * @profile: profile to find perms for + * @rules: ruleset to search * @label: label to check access permissions for - * @start: state to start match in + * @state: state to start match in * @subns: whether to do permission checks on components in a subns * @request: permissions to request * @perms: perms struct to set @@ -2037,7 +2039,7 @@ out: /** * __label_update - insert updated version of @label into labelset - * @label - the label to update/replace + * @label: the label to update/replace * * Returns: new label that is up to date * else NULL on failure From 8921482286116af193980f04f2f2755775a410a5 Mon Sep 17 00:00:00 2001 From: Gaosheng Cui Date: Sun, 25 Jun 2023 09:13:44 +0800 Subject: [PATCH 010/475] apparmor: Fix kernel-doc warnings in apparmor/lib.c Fix kernel-doc warnings: security/apparmor/lib.c:33: warning: Excess function parameter 'str' description in 'aa_free_str_table' security/apparmor/lib.c:33: warning: Function parameter or member 't' not described in 'aa_free_str_table' security/apparmor/lib.c:94: warning: Function parameter or member 'n' not described in 'skipn_spaces' security/apparmor/lib.c:390: warning: Excess function parameter 'deny' description in 'aa_check_perms' Signed-off-by: Gaosheng Cui Signed-off-by: John Johansen --- security/apparmor/lib.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/security/apparmor/lib.c b/security/apparmor/lib.c index a630c951bb3b..8e1073477c09 100644 --- a/security/apparmor/lib.c +++ b/security/apparmor/lib.c @@ -27,7 +27,7 @@ struct aa_perms allperms = { .allow = ALL_PERMS_MASK, /** * aa_free_str_table - free entries str table - * @str: the string table to free (MAYBE NULL) + * @t: the string table to free (MAYBE NULL) */ void aa_free_str_table(struct aa_str_table *t) { @@ -85,6 +85,7 @@ char *aa_split_fqname(char *fqname, char **ns_name) /** * skipn_spaces - Removes leading whitespace from @str. * @str: The string to be stripped. + * @n: length of str to parse, will stop at \0 if encountered before n * * Returns a pointer to the first non-whitespace character in @str. * if all whitespace will return NULL @@ -371,7 +372,6 @@ int aa_profile_label_perm(struct aa_profile *profile, struct aa_profile *target, * @profile: profile being checked * @perms: perms computed for the request * @request: requested perms - * @deny: Returns: explicit deny set * @sa: initialized audit structure (MAY BE NULL if not auditing) * @cb: callback fn for type specific fields (MAY BE NULL) * From 7abbbd573cbd27158137a7c36cdf2f7fe1051644 Mon Sep 17 00:00:00 2001 From: Gaosheng Cui Date: Sun, 25 Jun 2023 09:13:45 +0800 Subject: [PATCH 011/475] apparmor: Fix kernel-doc warnings in apparmor/match.c Fix kernel-doc warnings: security/apparmor/match.c:148: warning: Function parameter or member 'tables' not described in 'verify_table_headers' security/apparmor/match.c:289: warning: Excess function parameter 'kr' description in 'aa_dfa_free_kref' security/apparmor/match.c:289: warning: Function parameter or member 'kref' not described in 'aa_dfa_free_kref' Signed-off-by: Gaosheng Cui Signed-off-by: John Johansen --- security/apparmor/match.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/security/apparmor/match.c b/security/apparmor/match.c index b97ef5e1db73..7bdcca2aed7d 100644 --- a/security/apparmor/match.c +++ b/security/apparmor/match.c @@ -136,7 +136,7 @@ fail: /** * verify_table_headers - verify that the tables headers are as expected - * @tables - array of dfa tables to check (NOT NULL) + * @tables: array of dfa tables to check (NOT NULL) * @flags: flags controlling what type of accept table are acceptable * * Assumes dfa has gone through the first pass verification done by unpacking @@ -283,7 +283,7 @@ static void dfa_free(struct aa_dfa *dfa) /** * aa_dfa_free_kref - free aa_dfa by kref (called by aa_put_dfa) - * @kr: kref callback for freeing of a dfa (NOT NULL) + * @kref: kref callback for freeing of a dfa (NOT NULL) */ void aa_dfa_free_kref(struct kref *kref) { From 13c1748e217078d437727eef333cb0387d13bc0e Mon Sep 17 00:00:00 2001 From: Gaosheng Cui Date: Sun, 25 Jun 2023 09:13:46 +0800 Subject: [PATCH 012/475] apparmor: Fix kernel-doc warnings in apparmor/resource.c Fix kernel-doc warnings: security/apparmor/resource.c:111: warning: Function parameter or member 'label' not described in 'aa_task_setrlimit' security/apparmor/resource.c:111: warning: Function parameter or member 'new_rlim' not described in 'aa_task_setrlimit' security/apparmor/resource.c:111: warning: Function parameter or member 'resource' not described in 'aa_task_setrlimit' security/apparmor/resource.c:111: warning: Function parameter or member 'task' not described in 'aa_task_setrlimit' Signed-off-by: Gaosheng Cui Signed-off-by: John Johansen --- security/apparmor/resource.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/security/apparmor/resource.c b/security/apparmor/resource.c index e85948164896..2bebc5d9e741 100644 --- a/security/apparmor/resource.c +++ b/security/apparmor/resource.c @@ -97,10 +97,10 @@ static int profile_setrlimit(struct aa_profile *profile, unsigned int resource, /** * aa_task_setrlimit - test permission to set an rlimit - * @label - label confining the task (NOT NULL) - * @task - task the resource is being set on - * @resource - the resource being set - * @new_rlim - the new resource limit (NOT NULL) + * @label: label confining the task (NOT NULL) + * @task: task the resource is being set on + * @resource: the resource being set + * @new_rlim: the new resource limit (NOT NULL) * * Control raising the processes hard limit. * From f8fce898e5571846aa1e74385f18691ccfa3ed28 Mon Sep 17 00:00:00 2001 From: Gaosheng Cui Date: Sun, 25 Jun 2023 09:13:47 +0800 Subject: [PATCH 013/475] apparmor: Fix kernel-doc warnings in apparmor/policy_unpack.c Fix kernel-doc warnings: security/apparmor/policy_unpack.c:1173: warning: Function parameter or member 'table_size' not described in 'verify_dfa_accept_index' Signed-off-by: Gaosheng Cui Signed-off-by: John Johansen --- security/apparmor/policy_unpack.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/security/apparmor/policy_unpack.c b/security/apparmor/policy_unpack.c index 694fb7a09962..d2d740bb5618 100644 --- a/security/apparmor/policy_unpack.c +++ b/security/apparmor/policy_unpack.c @@ -1167,7 +1167,7 @@ static int verify_header(struct aa_ext *e, int required, const char **ns) /** * verify_dfa_accept_index - verify accept indexes are in range of perms table * @dfa: the dfa to check accept indexes are in range - * table_size: the permission table size the indexes should be within + * @table_size: the permission table size the indexes should be within */ static bool verify_dfa_accept_index(struct aa_dfa *dfa, int table_size) { From 2520d61c500b26a4adc0af430c0316e99e1000b9 Mon Sep 17 00:00:00 2001 From: Gaosheng Cui Date: Sun, 25 Jun 2023 09:13:48 +0800 Subject: [PATCH 014/475] apparmor: Fix kernel-doc warnings in apparmor/policy_compat.c Fix kernel-doc warnings: security/apparmor/policy_compat.c:151: warning: Function parameter or member 'size' not described in 'compute_fperms' Signed-off-by: Gaosheng Cui Signed-off-by: John Johansen --- security/apparmor/policy_compat.c | 1 + 1 file changed, 1 insertion(+) diff --git a/security/apparmor/policy_compat.c b/security/apparmor/policy_compat.c index 0cb02da8a319..18e4ddccf623 100644 --- a/security/apparmor/policy_compat.c +++ b/security/apparmor/policy_compat.c @@ -143,6 +143,7 @@ static struct aa_perms compute_fperms_other(struct aa_dfa *dfa, * compute_fperms - convert dfa compressed perms to internal perms and store * them so they can be retrieved later. * @dfa: a dfa using fperms to remap to internal permissions + * @size: the permission table size * * Returns: remapped perm table */ From 25ff0ff2d6286928dc516c74b879809c691c2dd8 Mon Sep 17 00:00:00 2001 From: Gaosheng Cui Date: Sun, 25 Jun 2023 09:13:49 +0800 Subject: [PATCH 015/475] apparmor: Fix kernel-doc warnings in apparmor/policy.c Fix kernel-doc warnings: security/apparmor/policy.c:294: warning: Function parameter or member 'proxy' not described in 'aa_alloc_profile' security/apparmor/policy.c:785: warning: Function parameter or member 'label' not described in 'aa_policy_view_capable' security/apparmor/policy.c:785: warning: Function parameter or member 'ns' not described in 'aa_policy_view_capable' security/apparmor/policy.c:847: warning: Function parameter or member 'ns' not described in 'aa_may_manage_policy' security/apparmor/policy.c:964: warning: Function parameter or member 'hname' not described in '__lookup_replace' security/apparmor/policy.c:964: warning: Function parameter or member 'info' not described in '__lookup_replace' security/apparmor/policy.c:964: warning: Function parameter or member 'noreplace' not described in '__lookup_replace' security/apparmor/policy.c:964: warning: Function parameter or member 'ns' not described in '__lookup_replace' security/apparmor/policy.c:964: warning: Function parameter or member 'p' not described in '__lookup_replace' Signed-off-by: Gaosheng Cui Signed-off-by: John Johansen --- security/apparmor/policy.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/security/apparmor/policy.c b/security/apparmor/policy.c index 715fe1b66d12..4817c1b30105 100644 --- a/security/apparmor/policy.c +++ b/security/apparmor/policy.c @@ -285,6 +285,7 @@ void aa_free_profile(struct aa_profile *profile) /** * aa_alloc_profile - allocate, initialize and return a new profile * @hname: name of the profile (NOT NULL) + * @proxy: proxy to use OR null if to allocate a new one * @gfp: allocation type * * Returns: refcount profile or NULL on failure @@ -774,8 +775,9 @@ static int policy_ns_capable(struct aa_label *label, /** * aa_policy_view_capable - check if viewing policy in at @ns is allowed - * label: label that is trying to view policy in ns - * ns: namespace being viewed by @label (may be NULL if @label's ns) + * @label: label that is trying to view policy in ns + * @ns: namespace being viewed by @label (may be NULL if @label's ns) + * * Returns: true if viewing policy is allowed * * If @ns is NULL then the namespace being viewed is assumed to be the @@ -839,6 +841,7 @@ bool aa_current_policy_admin_capable(struct aa_ns *ns) /** * aa_may_manage_policy - can the current task manage policy * @label: label to check if it can manage policy + * @ns: namespace being managed by @label (may be NULL if @label's ns) * @mask: contains the policy manipulation operation being done * * Returns: 0 if the task is allowed to manipulate policy else error @@ -950,11 +953,11 @@ static void __replace_profile(struct aa_profile *old, struct aa_profile *new) /** * __lookup_replace - lookup replacement information for a profile - * @ns - namespace the lookup occurs in - * @hname - name of profile to lookup - * @noreplace - true if not replacing an existing profile - * @p - Returns: profile to be replaced - * @info - Returns: info string on why lookup failed + * @ns: namespace the lookup occurs in + * @hname: name of profile to lookup + * @noreplace: true if not replacing an existing profile + * @p: Returns - profile to be replaced + * @info: Returns - info string on why lookup failed * * Returns: profile to replace (no ref) on success else ptr error */ From d2fe16e94cc30c0af5327d5a673dd1761b9edfae Mon Sep 17 00:00:00 2001 From: John Johansen Date: Tue, 8 Aug 2023 13:12:19 -0700 Subject: [PATCH 016/475] apparmor: fixup return comments for kernel doc cleanups by Gaosheng Cui [PATCH -next 05/11] apparmor: Fix kernel-doc warnings in apparmor/label.c missed updating the Returns comment for the new parameter names [PATCH -next 05/11] apparmor: Fix kernel-doc warnings in apparmor/label.c Added the @size parameter comment without mentioning it is a return value. Signed-off-by: John Johansen --- security/apparmor/label.c | 6 +++--- security/apparmor/policy_compat.c | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/security/apparmor/label.c b/security/apparmor/label.c index ce9c39102a93..8a74e89b05f4 100644 --- a/security/apparmor/label.c +++ b/security/apparmor/label.c @@ -159,9 +159,9 @@ static int profile_cmp(struct aa_profile *a, struct aa_profile *b) * @b: aa_profile to compare (NOT NULL) * @bn: length of @b * - * Returns: <0 if a < vec - * ==0 if a == vec - * >0 if a > vec + * Returns: <0 if @a < @b + * ==0 if @a == @b + * >0 if @a > @b */ static int vec_cmp(struct aa_profile **a, int an, struct aa_profile **b, int bn) { diff --git a/security/apparmor/policy_compat.c b/security/apparmor/policy_compat.c index 18e4ddccf623..423227670e68 100644 --- a/security/apparmor/policy_compat.c +++ b/security/apparmor/policy_compat.c @@ -143,7 +143,7 @@ static struct aa_perms compute_fperms_other(struct aa_dfa *dfa, * compute_fperms - convert dfa compressed perms to internal perms and store * them so they can be retrieved later. * @dfa: a dfa using fperms to remap to internal permissions - * @size: the permission table size + * @size: Returns the permission table size * * Returns: remapped perm table */ From 9a0dbdbff01818a4790f6768612e72c222add906 Mon Sep 17 00:00:00 2001 From: Xiu Jianfeng Date: Tue, 25 Jul 2023 14:12:39 +0000 Subject: [PATCH 017/475] apparmor: cleanup unused declarations in policy.h The implementions of these declarations do not exist, remove them all. Signed-off-by: Xiu Jianfeng Signed-off-by: John Johansen --- security/apparmor/include/policy.h | 6 ------ 1 file changed, 6 deletions(-) diff --git a/security/apparmor/include/policy.h b/security/apparmor/include/policy.h index 545f791cabda..6e5ec9c37b48 100644 --- a/security/apparmor/include/policy.h +++ b/security/apparmor/include/policy.h @@ -227,10 +227,6 @@ extern enum profile_mode aa_g_profile_mode; #define profiles_ns(P) ((P)->ns) #define name_is_shared(A, B) ((A)->hname && (A)->hname == (B)->hname) -void aa_add_profile(struct aa_policy *common, struct aa_profile *profile); - - -void aa_free_proxy_kref(struct kref *kref); struct aa_ruleset *aa_alloc_ruleset(gfp_t gfp); struct aa_profile *aa_alloc_profile(const char *name, struct aa_proxy *proxy, gfp_t gfp); @@ -239,14 +235,12 @@ struct aa_profile *aa_alloc_null(struct aa_profile *parent, const char *name, struct aa_profile *aa_new_learning_profile(struct aa_profile *parent, bool hat, const char *base, gfp_t gfp); void aa_free_profile(struct aa_profile *profile); -void aa_free_profile_kref(struct kref *kref); struct aa_profile *aa_find_child(struct aa_profile *parent, const char *name); struct aa_profile *aa_lookupn_profile(struct aa_ns *ns, const char *hname, size_t n); struct aa_profile *aa_lookup_profile(struct aa_ns *ns, const char *name); struct aa_profile *aa_fqlookupn_profile(struct aa_label *base, const char *fqname, size_t n); -struct aa_profile *aa_match_profile(struct aa_ns *ns, const char *name); ssize_t aa_replace_profiles(struct aa_ns *view, struct aa_label *label, u32 mask, struct aa_loaddata *udata); From 980a5808686e29f4e2b3cd412a9e0c9ebaac2910 Mon Sep 17 00:00:00 2001 From: Xiu Jianfeng Date: Tue, 25 Jul 2023 14:12:59 +0000 Subject: [PATCH 018/475] apparmor: cleanup unused functions in file.h After changes in commit 33bf60cabcc7 ("LSM: Infrastructure management of the file security"), aa_alloc_file_ctx() and aa_free_file_ctx() are no longer used, so remove them, and also remove aa_get_file_label() because it seems that it's never been used before. Signed-off-by: Xiu Jianfeng Signed-off-by: John Johansen --- security/apparmor/include/file.h | 37 -------------------------------- 1 file changed, 37 deletions(-) diff --git a/security/apparmor/include/file.h b/security/apparmor/include/file.h index 5be620af33ba..23cb6f9dbe6e 100644 --- a/security/apparmor/include/file.h +++ b/security/apparmor/include/file.h @@ -45,43 +45,6 @@ struct aa_file_ctx { u32 allow; }; -/** - * aa_alloc_file_ctx - allocate file_ctx - * @label: initial label of task creating the file - * @gfp: gfp flags for allocation - * - * Returns: file_ctx or NULL on failure - */ -static inline struct aa_file_ctx *aa_alloc_file_ctx(struct aa_label *label, - gfp_t gfp) -{ - struct aa_file_ctx *ctx; - - ctx = kzalloc(sizeof(struct aa_file_ctx), gfp); - if (ctx) { - spin_lock_init(&ctx->lock); - rcu_assign_pointer(ctx->label, aa_get_label(label)); - } - return ctx; -} - -/** - * aa_free_file_ctx - free a file_ctx - * @ctx: file_ctx to free (MAYBE_NULL) - */ -static inline void aa_free_file_ctx(struct aa_file_ctx *ctx) -{ - if (ctx) { - aa_put_label(rcu_access_pointer(ctx->label)); - kfree_sensitive(ctx); - } -} - -static inline struct aa_label *aa_get_file_label(struct aa_file_ctx *ctx) -{ - return aa_get_label_rcu(&ctx->label); -} - /* * The xindex is broken into 3 parts * - index - an index into either the exec name table or the variable table From efea5b0dcc433df6e7d74605d37813d69f99bef0 Mon Sep 17 00:00:00 2001 From: "GONG, Ruiqi" Date: Mon, 7 Aug 2023 17:39:04 +0800 Subject: [PATCH 019/475] apparmor: remove unused PROF_* macros The last usage of PROF_{ADD,REPLACE} were removed by commit 18e99f191a8e ("apparmor: provide finer control over policy management"). So remove these two unused macros. Signed-off-by: GONG, Ruiqi Signed-off-by: John Johansen --- security/apparmor/include/policy.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/security/apparmor/include/policy.h b/security/apparmor/include/policy.h index 6e5ec9c37b48..062a4cd3c367 100644 --- a/security/apparmor/include/policy.h +++ b/security/apparmor/include/policy.h @@ -248,9 +248,6 @@ ssize_t aa_remove_profiles(struct aa_ns *view, struct aa_label *label, char *name, size_t size); void __aa_profile_list_release(struct list_head *head); -#define PROF_ADD 1 -#define PROF_REPLACE 0 - #define profile_unconfined(X) ((X)->mode == APPARMOR_UNCONFINED) /** From 8884ba07786c718771cf7b78cb3024924b27ec2b Mon Sep 17 00:00:00 2001 From: Georgia Garcia Date: Mon, 21 Aug 2023 15:37:24 -0300 Subject: [PATCH 020/475] apparmor: fix invalid reference on profile->disconnected profile->disconnected was storing an invalid reference to the disconnected path. Fix it by duplicating the string using aa_unpack_strdup and freeing accordingly. Fixes: 72c8a768641d ("apparmor: allow profiles to provide info to disconnected paths") Signed-off-by: Georgia Garcia Signed-off-by: John Johansen --- security/apparmor/policy.c | 1 + security/apparmor/policy_unpack.c | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/security/apparmor/policy.c b/security/apparmor/policy.c index 4817c1b30105..d7525f228c43 100644 --- a/security/apparmor/policy.c +++ b/security/apparmor/policy.c @@ -255,6 +255,7 @@ void aa_free_profile(struct aa_profile *profile) aa_put_ns(profile->ns); kfree_sensitive(profile->rename); + kfree_sensitive(profile->disconnected); free_attachment(&profile->attach); diff --git a/security/apparmor/policy_unpack.c b/security/apparmor/policy_unpack.c index d2d740bb5618..8b2ad42c80ba 100644 --- a/security/apparmor/policy_unpack.c +++ b/security/apparmor/policy_unpack.c @@ -804,7 +804,7 @@ static struct aa_profile *unpack_profile(struct aa_ext *e, char **ns_name) const char *info = "failed to unpack profile"; size_t ns_len; struct rhashtable_params params = { 0 }; - char *key = NULL; + char *key = NULL, *disconnected = NULL; struct aa_data *data; int error = -EPROTO; kernel_cap_t tmpcap; @@ -870,7 +870,8 @@ static struct aa_profile *unpack_profile(struct aa_ext *e, char **ns_name) } /* disconnected attachment string is optional */ - (void) aa_unpack_str(e, &profile->disconnected, "disconnected"); + (void) aa_unpack_strdup(e, &disconnected, "disconnected"); + profile->disconnected = disconnected; /* per profile debug flags (complain, audit) */ if (!aa_unpack_nameX(e, AA_STRUCT, "flags")) { From 6066622c97cc0b25d287dc859dbe5f185740e0b4 Mon Sep 17 00:00:00 2001 From: Jiapeng Chong Date: Thu, 25 Feb 2021 18:04:43 +0800 Subject: [PATCH 021/475] perf machine: Use true and false for bool variable Fix the following coccicheck warnings: ./tools/perf/util/machine.c:2000:9-10: WARNING: return of 0/1 in function 'symbol__match_regex' with return type bool. Committer notes: Found this in the pile, it was already returning bool, but this patch simplifies it further, from 3 lines to just 1. Reported-by: Abaci Robot Signed-off-by: Jiapeng Chong Suggested-by: David Laight Cc: Alexander Shishkin Cc: Alexei Starovoitov Cc: Andrii Nakryiko Cc: Daniel Borkmann Cc: Ingo Molnar Cc: Jiri Olsa Cc: John Fastabend Cc: KP Singh Cc: Mark Rutland Cc: Martin KaFai Lau Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Song Liu Cc: Yonghong Song Cc: bpf@vger.kernel.org Cc: netdev@vger.kernel.org Link: https://lore.kernel.org/r/1614247483-102665-1-git-send-email-jiapeng.chong@linux.alibaba.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/util/machine.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tools/perf/util/machine.c b/tools/perf/util/machine.c index 88f31b3a63ac..addfae2f63ef 100644 --- a/tools/perf/util/machine.c +++ b/tools/perf/util/machine.c @@ -2213,9 +2213,7 @@ int machine__process_event(struct machine *machine, union perf_event *event, static bool symbol__match_regex(struct symbol *sym, regex_t *regex) { - if (!regexec(regex, sym->name, 0, NULL, 0)) - return true; - return false; + return regexec(regex, sym->name, 0, NULL, 0) == 0; } static void ip__resolve_ams(struct thread *thread, From fc143580753348c62068f30958fede937aa471ad Mon Sep 17 00:00:00 2001 From: Kajol Jain Date: Tue, 5 Sep 2023 17:10:37 +0530 Subject: [PATCH 022/475] perf vendor events power10: Update JSON/events Update JSON/Events list with data-source events for power10 platform. Signed-off-by: Kajol Jain Cc: Athira Rajeev Cc: Disha Goel Cc: Madhavan Srinivasan Cc: Namhyung Kim Cc: linuxppc-dev@lists.ozlabs.org Link: https://lore.kernel.org/r/20230905114039.176645-1-kjain@linux.ibm.com Signed-off-by: Arnaldo Carvalho de Melo --- .../arch/powerpc/power10/datasource.json | 1282 +++++++++++++++++ .../arch/powerpc/power10/others.json | 10 - .../arch/powerpc/power10/translation.json | 5 - 3 files changed, 1282 insertions(+), 15 deletions(-) create mode 100644 tools/perf/pmu-events/arch/powerpc/power10/datasource.json diff --git a/tools/perf/pmu-events/arch/powerpc/power10/datasource.json b/tools/perf/pmu-events/arch/powerpc/power10/datasource.json new file mode 100644 index 000000000000..12cfb9785433 --- /dev/null +++ b/tools/perf/pmu-events/arch/powerpc/power10/datasource.json @@ -0,0 +1,1282 @@ +[ + { + "EventCode": "0x200FE", + "EventName": "PM_DATA_FROM_L2MISS", + "BriefDescription": "The processor's L1 data cache was reloaded from a source beyond the local core's L2 due to a demand miss." + }, + { + "EventCode": "0x300FE", + "EventName": "PM_DATA_FROM_L3MISS", + "BriefDescription": "The processor's L1 data cache was reloaded from beyond the local core's L3 due to a demand miss." + }, + { + "EventCode": "0x400FE", + "EventName": "PM_DATA_FROM_MEMORY", + "BriefDescription": "The processor's data cache was reloaded from local, remote, or distant memory due to a demand miss." + }, + { + "EventCode": "0x000300000000C040", + "EventName": "PM_INST_FROM_L2", + "BriefDescription": "The processor's instruction cache was reloaded from the local core's L2 due to a demand miss." + }, + { + "EventCode": "0x000340000000C040", + "EventName": "PM_DATA_FROM_L2", + "BriefDescription": "The processor's L1 data cache was reloaded from the local core's L2 due to a demand miss." + }, + { + "EventCode": "0x000300000010C040", + "EventName": "PM_INST_FROM_L2_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from the local core's L2 due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x000340000020C040", + "EventName": "PM_DATA_FROM_L2_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from the local core's L2 due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x003F00000000C040", + "EventName": "PM_INST_FROM_L1MISS", + "BriefDescription": "The processor's instruction cache was reloaded from a source beyond the local core's L1 due to a demand miss." + }, + { + "EventCode": "0x003F40000000C040", + "EventName": "PM_DATA_FROM_L1MISS", + "BriefDescription": "The processor's L1 data cache was reloaded from a source beyond the local core's L1 due to a demand miss." + }, + { + "EventCode": "0x003F00000010C040", + "EventName": "PM_INST_FROM_L1MISS_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from a source beyond the local core's L1 due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x003F40000020C040", + "EventName": "PM_DATA_FROM_L1MISS_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from a source beyond the local core's L1 due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x000040000000C040", + "EventName": "PM_DATA_FROM_L2_NO_CONFLICT", + "BriefDescription": "The processor's L1 data cache was reloaded without dispatch conflicts with data NOT in the MEPF state from the local core's L2 due to a demand miss." + }, + { + "EventCode": "0x000040000020C040", + "EventName": "PM_DATA_FROM_L2_NO_CONFLICT_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded without dispatch conflicts with data NOT in the MEPF state from the local core's L2 due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x004040000000C040", + "EventName": "PM_DATA_FROM_L2_MEPF", + "BriefDescription": "The processor's L1 data cache was reloaded with data in the MEPF state without dispatch conflicts from the local core's L2 due to a demand miss." + }, + { + "EventCode": "0x004040000020C040", + "EventName": "PM_DATA_FROM_L2_MEPF_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with data in the MEPF state without dispatch conflicts from the local core's L2 due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x008040000000C040", + "EventName": "PM_DATA_FROM_L2_LDHITST_CONFLICT", + "BriefDescription": "The processor's L1 data cache was reloaded with data that had a dispatch conflict on ld-hit-store from the local core's L2 due to a demand miss." + }, + { + "EventCode": "0x008040000020C040", + "EventName": "PM_DATA_FROM_L2_LDHITST_CONFLICT_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with data that had a dispatch conflict on ld-hit-store from the local core's L2 due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x00C040000000C040", + "EventName": "PM_DATA_FROM_L2_OTHER_CONFLICT", + "BriefDescription": "The processor's L1 data cache was reloaded with data that had a dispatch conflict other than ld-hit-store from the local core's L2 due to a demand miss." + }, + { + "EventCode": "0x00C040000020C040", + "EventName": "PM_DATA_FROM_L2_OTHER_CONFLICT_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with data that had a dispatch conflict other than ld-hit-store from the local core's L2 due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x000380000000C040", + "EventName": "PM_INST_FROM_L2MISS", + "BriefDescription": "The processor's instruction cache was reloaded from a source beyond the local core's L2 due to a demand miss." + }, + { + "EventCode": "0x000380000010C040", + "EventName": "PM_INST_FROM_L2MISS_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from a source beyond the local core's L2 due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0003C0000020C040", + "EventName": "PM_DATA_FROM_L2MISS_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from a source beyond the local core's L2 due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x010300000000C040", + "EventName": "PM_INST_FROM_L3", + "BriefDescription": "The processor's instruction cache was reloaded from the local core's L3 due to a demand miss." + }, + { + "EventCode": "0x010340000000C040", + "EventName": "PM_DATA_FROM_L3", + "BriefDescription": "The processor's L1 data cache was reloaded from the local core's L3 due to a demand miss." + }, + { + "EventCode": "0x010300000010C040", + "EventName": "PM_INST_FROM_L3_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from the local core's L3 due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x010340000020C040", + "EventName": "PM_DATA_FROM_L3_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from the local core's L3 due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x010040000000C040", + "EventName": "PM_DATA_FROM_L3_NO_CONFLICT", + "BriefDescription": "The processor's L1 data cache was reloaded without dispatch conflicts with data NOT in the MEPF state from the local core's L3 due to a demand miss." + }, + { + "EventCode": "0x010040000020C040", + "EventName": "PM_DATA_FROM_L3_NO_CONFLICT_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded without dispatch conflicts with data NOT in the MEPF state from the local core's L3 due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x014040000000C040", + "EventName": "PM_DATA_FROM_L3_MEPF", + "BriefDescription": "The processor's L1 data cache was reloaded with data in the MEPF state without dispatch conflicts from the local core's L3 due to a demand miss." + }, + { + "EventCode": "0x014040000020C040", + "EventName": "PM_DATA_FROM_L3_MEPF_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with data in the MEPF state without dispatch conflicts from the local core's L3 due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x01C040000000C040", + "EventName": "PM_DATA_FROM_L3_CONFLICT", + "BriefDescription": "The processor's L1 data cache was reloaded from the local core's L3 due to a demand miss." + }, + { + "EventCode": "0x01C040000020C040", + "EventName": "PM_DATA_FROM_L3_CONFLICT_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from the local core's L3 due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x000780000000C040", + "EventName": "PM_INST_FROM_L3MISS", + "BriefDescription": "The processor's instruction cache was reloaded from beyond the local core's L3 due to a demand miss." + }, + { + "EventCode": "0x000780000010C040", + "EventName": "PM_INST_FROM_L3MISS_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from beyond the local core's L3 due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0007C0000020C040", + "EventName": "PM_DATA_FROM_L3MISS_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from beyond the local core's L3 due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x080040000000C040", + "EventName": "PM_DATA_FROM_L21_REGENT_SHR", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L2 on the same chip in the same regent due to a demand miss." + }, + { + "EventCode": "0x080040000020C040", + "EventName": "PM_DATA_FROM_L21_REGENT_SHR_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L2 on the same chip in the same regent due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x084040000000C040", + "EventName": "PM_DATA_FROM_L21_REGENT_MOD", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L2 on the same chip in the same regent due to a demand miss." + }, + { + "EventCode": "0x084040000020C040", + "EventName": "PM_DATA_FROM_L21_REGENT_MOD_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L2 on the same chip in the same regent due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x080100000000C040", + "EventName": "PM_INST_FROM_L21_REGENT", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 on the same chip in the same regent due to a demand miss." + }, + { + "EventCode": "0x080140000000C040", + "EventName": "PM_DATA_FROM_L21_REGENT", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 on the same chip in the same regent due to a demand miss." + }, + { + "EventCode": "0x080100000010C040", + "EventName": "PM_INST_FROM_L21_REGENT_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 on the same chip in the same regent due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x080140000020C040", + "EventName": "PM_DATA_FROM_L21_REGENT_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 on the same chip in the same regent due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x088040000000C040", + "EventName": "PM_DATA_FROM_L31_REGENT_SHR", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L3 on the same chip in the same regent due to a demand miss." + }, + { + "EventCode": "0x088040000020C040", + "EventName": "PM_DATA_FROM_L31_REGENT_SHR_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L3 on the same chip in the same regent due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x08C040000000C040", + "EventName": "PM_DATA_FROM_L31_REGENT_MOD", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L3 on the same chip in the same regent due to a demand miss." + }, + { + "EventCode": "0x08C040000020C040", + "EventName": "PM_DATA_FROM_L31_REGENT_MOD_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L3 on the same chip in the same regent due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x088100000000C040", + "EventName": "PM_INST_FROM_L31_REGENT", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L3 on the same chip in the same regent due to a demand miss." + }, + { + "EventCode": "0x088140000000C040", + "EventName": "PM_DATA_FROM_L31_REGENT", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L3 on the same chip in the same regent due to a demand miss." + }, + { + "EventCode": "0x088100000010C040", + "EventName": "PM_INST_FROM_L31_REGENT_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L3 on the same chip in the same regent due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x088140000020C040", + "EventName": "PM_DATA_FROM_L31_REGENT_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L3 on the same chip in the same regent due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x080240000000C040", + "EventName": "PM_DATA_FROM_REGENT_L2L3_SHR", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L2 or L3 on the same chip in the same regent due to a demand miss." + }, + { + "EventCode": "0x080240000020C040", + "EventName": "PM_DATA_FROM_REGENT_L2L3_SHR_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L2 or L3 on the same chip in the same regent due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x084240000000C040", + "EventName": "PM_DATA_FROM_REGENT_L2L3_MOD", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L2 or L3 on the same chip in the same regent due to a demand miss." + }, + { + "EventCode": "0x084240000020C040", + "EventName": "PM_DATA_FROM_REGENT_L2L3_MOD_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L2 or L3 on the same chip in the same regent due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x080300000000C040", + "EventName": "PM_INST_FROM_REGENT_L2L3", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 or L3 on the same chip in the same regent due to a demand miss." + }, + { + "EventCode": "0x080340000000C040", + "EventName": "PM_DATA_FROM_REGENT_L2L3", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 or L3 on the same chip in the same regent due to a demand miss." + }, + { + "EventCode": "0x080300000010C040", + "EventName": "PM_INST_FROM_REGENT_L2L3_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 or L3 on the same chip in the same regent due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x080340000020C040", + "EventName": "PM_DATA_FROM_REGENT_L2L3_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 or L3 on the same chip in the same regent due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0A0040000000C040", + "EventName": "PM_DATA_FROM_L21_NON_REGENT_SHR", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L2 on the same chip in a different regent due to a demand miss." + }, + { + "EventCode": "0x0A0040000020C040", + "EventName": "PM_DATA_FROM_L21_NON_REGENT_SHR_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L2 on the same chip in a different regent due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0A4040000000C040", + "EventName": "PM_DATA_FROM_L21_NON_REGENT_MOD", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L2 on the same chip in a different regent due to a demand miss." + }, + { + "EventCode": "0x0A4040000020C040", + "EventName": "PM_DATA_FROM_L21_NON_REGENT_MOD_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L2 on the same chip in a different regent due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0A0100000000C040", + "EventName": "PM_INST_FROM_L21_NON_REGENT", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 on the same chip in a different regent due to a demand miss." + }, + { + "EventCode": "0x0A0140000000C040", + "EventName": "PM_DATA_FROM_L21_NON_REGENT", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 on the same chip in a different regent due to a demand miss." + }, + { + "EventCode": "0x0A0100000010C040", + "EventName": "PM_INST_FROM_L21_NON_REGENT_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 on the same chip in a different regent due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0A0140000020C040", + "EventName": "PM_DATA_FROM_L21_NON_REGENT_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 on the same chip in a different regent due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0A8040000000C040", + "EventName": "PM_DATA_FROM_L31_NON_REGENT_SHR", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L3 on the same chip in a different regent due to a demand miss." + }, + { + "EventCode": "0x0A8040000020C040", + "EventName": "PM_DATA_FROM_L31_NON_REGENT_SHR_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L3 on the same chip in a different regent due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0AC040000000C040", + "EventName": "PM_DATA_FROM_L31_NON_REGENT_MOD", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L3 on the same chip in a different regent due to a demand miss." + }, + { + "EventCode": "0x0AC040000020C040", + "EventName": "PM_DATA_FROM_L31_NON_REGENT_MOD_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L3 on the same chip in a different regent due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0A8100000000C040", + "EventName": "PM_INST_FROM_L31_NON_REGENT", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L3 on the same chip in a different regent due to a demand miss." + }, + { + "EventCode": "0x0A8140000000C040", + "EventName": "PM_DATA_FROM_L31_NON_REGENT", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L3 on the same chip in a different regent due to a demand miss." + }, + { + "EventCode": "0x0A8100000010C040", + "EventName": "PM_INST_FROM_L31_NON_REGENT_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L3 on the same chip in a different regent due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0A8140000020C040", + "EventName": "PM_DATA_FROM_L31_NON_REGENT_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L3 on the same chip in a different regent due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0A0240000000C040", + "EventName": "PM_DATA_FROM_NON_REGENT_L2L3_SHR", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L2 or L3 on the same chip in a different regent due to a demand miss." + }, + { + "EventCode": "0x0A0240000020C040", + "EventName": "PM_DATA_FROM_NON_REGENT_L2L3_SHR_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L2 or L3 on the same chip in a different regent due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0A4240000000C040", + "EventName": "PM_DATA_FROM_NON_REGENT_L2L3_MOD", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L2 or L3 on the same chip in a different regent due to a demand miss." + }, + { + "EventCode": "0x0A4240000020C040", + "EventName": "PM_DATA_FROM_NON_REGENT_L2L3_MOD_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L2 or L3 on the same chip in a different regent due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0A0300000000C040", + "EventName": "PM_INST_FROM_NON_REGENT_L2L3", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 or L3 on the same chip in a different regent due to a demand miss." + }, + { + "EventCode": "0x0A0340000000C040", + "EventName": "PM_DATA_FROM_NON_REGENT_L2L3", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 or L3 on the same chip in a different regent due to a demand miss." + }, + { + "EventCode": "0x0A0300000010C040", + "EventName": "PM_INST_FROM_NON_REGENT_L2L3_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 or L3 on the same chip in a different regent due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0A0340000020C040", + "EventName": "PM_DATA_FROM_NON_REGENT_L2L3_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 or L3 on the same chip in a different regent due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x094100000000C040", + "EventName": "PM_INST_FROM_LMEM", + "BriefDescription": "The processor's instruction cache was reloaded from the local chip's memory due to a demand miss." + }, + { + "EventCode": "0x094040000000C040", + "EventName": "PM_DATA_FROM_LMEM", + "BriefDescription": "The processor's L1 data cache was reloaded from the local chip's memory due to a demand miss." + }, + { + "EventCode": "0x094100000010C040", + "EventName": "PM_INST_FROM_LMEM_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from the local chip's memory due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x094040000020C040", + "EventName": "PM_DATA_FROM_LMEM_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from the local chip's memory due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x098040000000C040", + "EventName": "PM_DATA_FROM_L_OC_CACHE", + "BriefDescription": "The processor's L1 data cache was reloaded from the local chip's OpenCAPI cache due to a demand miss." + }, + { + "EventCode": "0x098040000020C040", + "EventName": "PM_DATA_FROM_L_OC_CACHE_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from the local chip's OpenCAPI cache due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x09C040000000C040", + "EventName": "PM_DATA_FROM_L_OC_MEM", + "BriefDescription": "The processor's L1 data cache was reloaded from the local chip's OpenCAPI memory due to a demand miss." + }, + { + "EventCode": "0x09C040000020C040", + "EventName": "PM_DATA_FROM_L_OC_MEM_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from the local chip's OpenCAPI memory due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x098100000000C040", + "EventName": "PM_INST_FROM_L_OC_ANY", + "BriefDescription": "The processor's instruction cache was reloaded from the local chip's OpenCAPI cache or memory due to a demand miss." + }, + { + "EventCode": "0x098140000000C040", + "EventName": "PM_DATA_FROM_L_OC_ANY", + "BriefDescription": "The processor's L1 data cache was reloaded from the local chip's OpenCAPI cache or memory due to a demand miss." + }, + { + "EventCode": "0x098100000010C040", + "EventName": "PM_INST_FROM_L_OC_ANY_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from the local chip's OpenCAPI cache or memory due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x098140000020C040", + "EventName": "PM_DATA_FROM_L_OC_ANY_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from the local chip's OpenCAPI cache or memory due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0C0040000000C040", + "EventName": "PM_DATA_FROM_RL2_SHR", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L2 from a remote chip due to a demand miss." + }, + { + "EventCode": "0x0C0040000020C040", + "EventName": "PM_DATA_FROM_RL2_SHR_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L2 from a remote chip due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0C4040000000C040", + "EventName": "PM_DATA_FROM_RL2_MOD", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L2 from a remote chip due to a demand miss." + }, + { + "EventCode": "0x0C4040000020C040", + "EventName": "PM_DATA_FROM_RL2_MOD_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L2 from a remote chip due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0C0100000000C040", + "EventName": "PM_INST_FROM_RL2", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 from a remote chip due to a demand miss." + }, + { + "EventCode": "0x0C0140000000C040", + "EventName": "PM_DATA_FROM_RL2", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 from a remote chip due to a demand miss." + }, + { + "EventCode": "0x0C0100000010C040", + "EventName": "PM_INST_FROM_RL2_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 from a remote chip due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0C0140000020C040", + "EventName": "PM_DATA_FROM_RL2_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 from a remote chip due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0C8040000000C040", + "EventName": "PM_DATA_FROM_RL3_SHR", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L3 from a remote chip due to a demand miss." + }, + { + "EventCode": "0x0C8040000020C040", + "EventName": "PM_DATA_FROM_RL3_SHR_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L3 from a remote chip due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0CC040000000C040", + "EventName": "PM_DATA_FROM_RL3_MOD", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L3 from a remote chip due to a demand miss." + }, + { + "EventCode": "0x0CC040000020C040", + "EventName": "PM_DATA_FROM_RL3_MOD_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L3 from a remote chip due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0C8100000000C040", + "EventName": "PM_INST_FROM_RL3", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L3 from a remote chip due to a demand miss." + }, + { + "EventCode": "0x0C8140000000C040", + "EventName": "PM_DATA_FROM_RL3", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L3 from a remote chip due to a demand miss." + }, + { + "EventCode": "0x0C8100000010C040", + "EventName": "PM_INST_FROM_RL3_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L3 from a remote chip due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0C8140000020C040", + "EventName": "PM_DATA_FROM_RL3_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L3 from a remote chip due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0C0240000000C040", + "EventName": "PM_DATA_FROM_RL2L3_SHR", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L2 or L3 from a remote chip due to a demand miss." + }, + { + "EventCode": "0x0C0240000020C040", + "EventName": "PM_DATA_FROM_RL2L3_SHR_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L2 or L3 from a remote chip due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0C4240000000C040", + "EventName": "PM_DATA_FROM_RL2L3_MOD", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L2 or L3 from a remote chip due to a demand miss." + }, + { + "EventCode": "0x0C4240000020C040", + "EventName": "PM_DATA_FROM_RL2L3_MOD_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L2 or L3 from a remote chip due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0C0300000000C040", + "EventName": "PM_INST_FROM_RL2L3", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 or L3 from a remote chip due to a demand miss." + }, + { + "EventCode": "0x0C0340000000C040", + "EventName": "PM_DATA_FROM_RL2L3", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 or L3 from a remote chip due to a demand miss." + }, + { + "EventCode": "0x0C0300000010C040", + "EventName": "PM_INST_FROM_RL2L3_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 or L3 from a remote chip due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0C0340000020C040", + "EventName": "PM_DATA_FROM_RL2L3_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 or L3 from a remote chip due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0D4100000000C040", + "EventName": "PM_INST_FROM_RMEM", + "BriefDescription": "The processor's instruction cache was reloaded from remote memory (MC slow) due to a demand miss." + }, + { + "EventCode": "0x0D4040000000C040", + "EventName": "PM_DATA_FROM_RMEM", + "BriefDescription": "The processor's L1 data cache was reloaded from remote memory (MC slow) due to a demand miss." + }, + { + "EventCode": "0x0D4100000010C040", + "EventName": "PM_INST_FROM_RMEM_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from remote memory (MC slow) due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0D4040000020C040", + "EventName": "PM_DATA_FROM_RMEM_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from remote memory (MC slow) due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0D8040000000C040", + "EventName": "PM_DATA_FROM_R_OC_CACHE", + "BriefDescription": "The processor's L1 data cache was reloaded from a remote chip's OpenCAPI cache due to a demand miss." + }, + { + "EventCode": "0x0D8040000020C040", + "EventName": "PM_DATA_FROM_R_OC_CACHE_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from a remote chip's OpenCAPI cache due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0DC040000000C040", + "EventName": "PM_DATA_FROM_R_OC_MEM", + "BriefDescription": "The processor's L1 data cache was reloaded from a remote chip's OpenCAPI memory due to a demand miss." + }, + { + "EventCode": "0x0DC040000020C040", + "EventName": "PM_DATA_FROM_R_OC_MEM_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from a remote chip's OpenCAPI memory due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0D8100000000C040", + "EventName": "PM_INST_FROM_R_OC_ANY", + "BriefDescription": "The processor's instruction cache was reloaded from a remote chip's OpenCAPI cache or memory due to a demand miss." + }, + { + "EventCode": "0x0D8140000000C040", + "EventName": "PM_DATA_FROM_R_OC_ANY", + "BriefDescription": "The processor's L1 data cache was reloaded from a remote chip's OpenCAPI cache or memory due to a demand miss." + }, + { + "EventCode": "0x0D8100000010C040", + "EventName": "PM_INST_FROM_R_OC_ANY_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from a remote chip's OpenCAPI cache or memory due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0D8140000020C040", + "EventName": "PM_DATA_FROM_R_OC_ANY_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from a remote chip's OpenCAPI cache or memory due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0E0040000000C040", + "EventName": "PM_DATA_FROM_DL2_SHR", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L2 from a distant chip due to a demand miss." + }, + { + "EventCode": "0x0E0040000020C040", + "EventName": "PM_DATA_FROM_DL2_SHR_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L2 from a distant chip due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0E4040000000C040", + "EventName": "PM_DATA_FROM_DL2_MOD", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L2 from a distant chip due to a demand miss." + }, + { + "EventCode": "0x0E4040000020C040", + "EventName": "PM_DATA_FROM_DL2_MOD_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L2 from a distant chip due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0E0100000000C040", + "EventName": "PM_INST_FROM_DL2", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 from a distant chip due to a demand miss." + }, + { + "EventCode": "0x0E0140000000C040", + "EventName": "PM_DATA_FROM_DL2", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 from a distant chip due to a demand miss." + }, + { + "EventCode": "0x0E0100000010C040", + "EventName": "PM_INST_FROM_DL2_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 from a distant chip due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0E0140000020C040", + "EventName": "PM_DATA_FROM_DL2_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 from a distant chip due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0E8040000000C040", + "EventName": "PM_DATA_FROM_DL3_SHR", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L3 from a distant chip due to a demand miss." + }, + { + "EventCode": "0x0E8040000020C040", + "EventName": "PM_DATA_FROM_DL3_SHR_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L3 from a distant chip due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0EC040000000C040", + "EventName": "PM_DATA_FROM_DL3_MOD", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L3 from a distant chip due to a demand miss." + }, + { + "EventCode": "0x0EC040000020C040", + "EventName": "PM_DATA_FROM_DL3_MOD_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L3 from a distant chip due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0E8100000000C040", + "EventName": "PM_INST_FROM_DL3", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L3 from a distant chip due to a demand miss." + }, + { + "EventCode": "0x0E8140000000C040", + "EventName": "PM_DATA_FROM_DL3", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L3 from a distant chip due to a demand miss." + }, + { + "EventCode": "0x0E8100000010C040", + "EventName": "PM_INST_FROM_DL3_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L3 from a distant chip due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0E8140000020C040", + "EventName": "PM_DATA_FROM_DL3_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L3 from a distant chip due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0E0240000000C040", + "EventName": "PM_DATA_FROM_DL2L3_SHR", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L2 or L3 from a distant chip due to a demand miss." + }, + { + "EventCode": "0x0E0240000020C040", + "EventName": "PM_DATA_FROM_DL2L3_SHR_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L2 or L3 from a distant chip due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0E4240000000C040", + "EventName": "PM_DATA_FROM_DL2L3_MOD", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L2 or L3 from a distant chip due to a demand miss." + }, + { + "EventCode": "0x0E4240000020C040", + "EventName": "PM_DATA_FROM_DL2L3_MOD_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L2 or L3 from a distant chip due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0E0300000000C040", + "EventName": "PM_INST_FROM_DL2L3", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 or L3 from a distant chip due to a demand miss." + }, + { + "EventCode": "0x0E0340000000C040", + "EventName": "PM_DATA_FROM_DL2L3", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 or L3 from a distant chip due to a demand miss." + }, + { + "EventCode": "0x0E0300000010C040", + "EventName": "PM_INST_FROM_DL2L3_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 or L3 from a distant chip due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0E0340000020C040", + "EventName": "PM_DATA_FROM_DL2L3_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 or L3 from a distant chip due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0F4100000000C040", + "EventName": "PM_INST_FROM_DMEM", + "BriefDescription": "The processor's instruction cache was reloaded from distant memory (MC slow) due to a demand miss." + }, + { + "EventCode": "0x0F4040000000C040", + "EventName": "PM_DATA_FROM_DMEM", + "BriefDescription": "The processor's L1 data cache was reloaded from distant memory (MC slow) due to a demand miss." + }, + { + "EventCode": "0x0F4100000010C040", + "EventName": "PM_INST_FROM_DMEM_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from distant memory (MC slow) due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0F4040000020C040", + "EventName": "PM_DATA_FROM_DMEM_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from distant memory (MC slow) due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0F8040000000C040", + "EventName": "PM_DATA_FROM_D_OC_CACHE", + "BriefDescription": "The processor's L1 data cache was reloaded from a distant chip's OpenCAPI cache due to a demand miss." + }, + { + "EventCode": "0x0F8040000020C040", + "EventName": "PM_DATA_FROM_D_OC_CACHE_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from a distant chip's OpenCAPI cache due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0FC040000000C040", + "EventName": "PM_DATA_FROM_D_OC_MEM", + "BriefDescription": "The processor's L1 data cache was reloaded from a distant chip's OpenCAPI memory due to a demand miss." + }, + { + "EventCode": "0x0FC040000020C040", + "EventName": "PM_DATA_FROM_D_OC_MEM_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from a distant chip's OpenCAPI memory due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0F8100000000C040", + "EventName": "PM_INST_FROM_D_OC_ANY", + "BriefDescription": "The processor's instruction cache was reloaded from a distant chip's OpenCAPI cache or memory due to a demand miss." + }, + { + "EventCode": "0x0F8140000000C040", + "EventName": "PM_DATA_FROM_D_OC_ANY", + "BriefDescription": "The processor's L1 data cache was reloaded from a distant chip's OpenCAPI cache or memory due to a demand miss." + }, + { + "EventCode": "0x0F8100000010C040", + "EventName": "PM_INST_FROM_D_OC_ANY_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from a distant chip's OpenCAPI cache or memory due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0F8140000020C040", + "EventName": "PM_DATA_FROM_D_OC_ANY_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from a distant chip's OpenCAPI cache or memory due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x080B00000000C040", + "EventName": "PM_INST_FROM_ONCHIP_CACHE", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 or L3 from the same chip due to a demand miss." + }, + { + "EventCode": "0x080B40000000C040", + "EventName": "PM_DATA_FROM_ONCHIP_CACHE", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 or L3 from the same chip due to a demand miss." + }, + { + "EventCode": "0x080B00000010C040", + "EventName": "PM_INST_FROM_ONCHIP_CACHE_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 or L3 from the same chip due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x080B40000020C040", + "EventName": "PM_DATA_FROM_ONCHIP_CACHE_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 or L3 from the same chip due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0C0B00000000C040", + "EventName": "PM_INST_FROM_OFFCHIP_CACHE", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 or L3 from a different chip due to a demand miss." + }, + { + "EventCode": "0x0C0B40000000C040", + "EventName": "PM_DATA_FROM_OFFCHIP_CACHE", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 or L3 from a different chip due to a demand miss." + }, + { + "EventCode": "0x0C0B00000010C040", + "EventName": "PM_INST_FROM_OFFCHIP_CACHE_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 or L3 from a different chip due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x0C0B40000020C040", + "EventName": "PM_DATA_FROM_OFFCHIP_CACHE_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 or L3 from a different chip due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x095900000000C040", + "EventName": "PM_INST_FROM_ANY_MEMORY", + "BriefDescription": "The processor's instruction cache was reloaded from any chip's memory (MC slow) due to a demand miss." + }, + { + "EventCode": "0x095840000000C040", + "EventName": "PM_DATA_FROM_ANY_MEMORY", + "BriefDescription": "The processor's L1 data cache was reloaded from any chip's memory (MC slow) due to a demand miss." + }, + { + "EventCode": "0x095900000010C040", + "EventName": "PM_INST_FROM_ANY_MEMORY_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from any chip's memory (MC slow) due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x095840000020C040", + "EventName": "PM_DATA_FROM_ANY_MEMORY_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from any chip's memory (MC slow) due to a demand miss or prefetch reload." + }, + { + "EventCode": "0x000300000000C142", + "EventName": "PM_MRK_INST_FROM_L2", + "BriefDescription": "The processor's instruction cache was reloaded from the local core's L2 due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x000340000000C142", + "EventName": "PM_MRK_DATA_FROM_L2", + "BriefDescription": "The processor's L1 data cache was reloaded from the local core's L2 due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x000300000010C142", + "EventName": "PM_MRK_INST_FROM_L2_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from the local core's L2 due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x000340000020C142", + "EventName": "PM_MRK_DATA_FROM_L2_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from the local core's L2 due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x003F00000000C142", + "EventName": "PM_MRK_INST_FROM_L1MISS", + "BriefDescription": "The processor's instruction cache was reloaded from a source beyond the local core's L1 due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x003F40000000C142", + "EventName": "PM_MRK_DATA_FROM_L1MISS", + "BriefDescription": "The processor's L1 data cache was reloaded from a source beyond the local core's L1 due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x003F00000010C142", + "EventName": "PM_MRK_INST_FROM_L1MISS_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from a source beyond the local core's L1 due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x003F40000020C142", + "EventName": "PM_MRK_DATA_FROM_L1MISS_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from a source beyond the local core's L1 due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x000040000000C142", + "EventName": "PM_MRK_DATA_FROM_L2_NO_CONFLICT", + "BriefDescription": "The processor's L1 data cache was reloaded without dispatch conflicts with data NOT in the MEPF state from the local core's L2 due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x000040000020C142", + "EventName": "PM_MRK_DATA_FROM_L2_NO_CONFLICT_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded without dispatch conflicts with data NOT in the MEPF state from the local core's L2 due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x004040000000C142", + "EventName": "PM_MRK_DATA_FROM_L2_MEPF", + "BriefDescription": "The processor's L1 data cache was reloaded with data in the MEPF state without dispatch conflicts from the local core's L2 due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x004040000020C142", + "EventName": "PM_MRK_DATA_FROM_L2_MEPF_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with data in the MEPF state without dispatch conflicts from the local core's L2 due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x008040000000C142", + "EventName": "PM_MRK_DATA_FROM_L2_LDHITST_CONFLICT", + "BriefDescription": "The processor's L1 data cache was reloaded with data that had a dispatch conflict on ld-hit-store from the local core's L2 due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x008040000020C142", + "EventName": "PM_MRK_DATA_FROM_L2_LDHITST_CONFLICT_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with data that had a dispatch conflict on ld-hit-store from the local core's L2 due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x00C040000000C142", + "EventName": "PM_MRK_DATA_FROM_L2_OTHER_CONFLICT", + "BriefDescription": "The processor's L1 data cache was reloaded with data that had a dispatch conflict other than ld-hit-store from the local core's L2 due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x00C040000020C142", + "EventName": "PM_MRK_DATA_FROM_L2_OTHER_CONFLICT_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with data that had a dispatch conflict other than ld-hit-store from the local core's L2 due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x000380000000C142", + "EventName": "PM_MRK_INST_FROM_L2MISS", + "BriefDescription": "The processor's instruction cache was reloaded from a source beyond the local core's L2 due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0003C0000000C142", + "EventName": "PM_MRK_DATA_FROM_L2MISS", + "BriefDescription": "The processor's L1 data cache was reloaded from a source beyond the local core's L2 due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x000380000010C142", + "EventName": "PM_MRK_INST_FROM_L2MISS_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from a source beyond the local core's L2 due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0003C0000020C142", + "EventName": "PM_MRK_DATA_FROM_L2MISS_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from a source beyond the local core's L2 due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x010300000000C142", + "EventName": "PM_MRK_INST_FROM_L3", + "BriefDescription": "The processor's instruction cache was reloaded from the local core's L3 due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x010340000000C142", + "EventName": "PM_MRK_DATA_FROM_L3", + "BriefDescription": "The processor's L1 data cache was reloaded from the local core's L3 due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x010300000010C142", + "EventName": "PM_MRK_INST_FROM_L3_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from the local core's L3 due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x010340000020C142", + "EventName": "PM_MRK_DATA_FROM_L3_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from the local core's L3 due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x010040000000C142", + "EventName": "PM_MRK_DATA_FROM_L3_NO_CONFLICT", + "BriefDescription": "The processor's L1 data cache was reloaded without dispatch conflicts with data NOT in the MEPF state from the local core's L3 due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x010040000020C142", + "EventName": "PM_MRK_DATA_FROM_L3_NO_CONFLICT_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded without dispatch conflicts with data NOT in the MEPF state from the local core's L3 due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x014040000000C142", + "EventName": "PM_MRK_DATA_FROM_L3_MEPF", + "BriefDescription": "The processor's L1 data cache was reloaded with data in the MEPF state without dispatch conflicts from the local core's L3 due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x014040000020C142", + "EventName": "PM_MRK_DATA_FROM_L3_MEPF_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with data in the MEPF state without dispatch conflicts from the local core's L3 due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x01C040000000C142", + "EventName": "PM_MRK_DATA_FROM_L3_CONFLICT", + "BriefDescription": "The processor's L1 data cache was reloaded from the local core's L3 due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x01C040000020C142", + "EventName": "PM_MRK_DATA_FROM_L3_CONFLICT_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from the local core's L3 due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x000780000000C142", + "EventName": "PM_MRK_INST_FROM_L3MISS", + "BriefDescription": "The processor's instruction cache was reloaded from beyond the local core's L3 due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0007C0000000C142", + "EventName": "PM_MRK_DATA_FROM_L3MISS", + "BriefDescription": "The processor's L1 data cache was reloaded from beyond the local core's L3 due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x000780000010C142", + "EventName": "PM_MRK_INST_FROM_L3MISS_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from beyond the local core's L3 due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0007C0000020C142", + "EventName": "PM_MRK_DATA_FROM_L3MISS_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from beyond the local core's L3 due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x080040000000C142", + "EventName": "PM_MRK_DATA_FROM_L21_REGENT_SHR", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L2 on the same chip in the same regent due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x080040000020C142", + "EventName": "PM_MRK_DATA_FROM_L21_REGENT_SHR_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L2 on the same chip in the same regent due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x084040000000C142", + "EventName": "PM_MRK_DATA_FROM_L21_REGENT_MOD", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L2 on the same chip in the same regent due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x084040000020C142", + "EventName": "PM_MRK_DATA_FROM_L21_REGENT_MOD_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L2 on the same chip in the same regent due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x080100000000C142", + "EventName": "PM_MRK_INST_FROM_L21_REGENT", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 on the same chip in the same regent due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x080140000000C142", + "EventName": "PM_MRK_DATA_FROM_L21_REGENT", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 on the same chip in the same regent due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x080100000010C142", + "EventName": "PM_MRK_INST_FROM_L21_REGENT_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 on the same chip in the same regent due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x080140000020C142", + "EventName": "PM_MRK_DATA_FROM_L21_REGENT_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 on the same chip in the same regent due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x088040000000C142", + "EventName": "PM_MRK_DATA_FROM_L31_REGENT_SHR", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L3 on the same chip in the same regent due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x088040000020C142", + "EventName": "PM_MRK_DATA_FROM_L31_REGENT_SHR_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L3 on the same chip in the same regent due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x08C040000000C142", + "EventName": "PM_MRK_DATA_FROM_L31_REGENT_MOD", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L3 on the same chip in the same regent due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x08C040000020C142", + "EventName": "PM_MRK_DATA_FROM_L31_REGENT_MOD_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L3 on the same chip in the same regent due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x088100000000C142", + "EventName": "PM_MRK_INST_FROM_L31_REGENT", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L3 on the same chip in the same regent due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x088140000000C142", + "EventName": "PM_MRK_DATA_FROM_L31_REGENT", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L3 on the same chip in the same regent due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x088100000010C142", + "EventName": "PM_MRK_INST_FROM_L31_REGENT_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L3 on the same chip in the same regent due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x088140000020C142", + "EventName": "PM_MRK_DATA_FROM_L31_REGENT_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L3 on the same chip in the same regent due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x080240000000C142", + "EventName": "PM_MRK_DATA_FROM_REGENT_L2L3_SHR", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L2 or L3 on the same chip in the same regent due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x080240000020C142", + "EventName": "PM_MRK_DATA_FROM_REGENT_L2L3_SHR_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L2 or L3 on the same chip in the same regent due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x084240000000C142", + "EventName": "PM_MRK_DATA_FROM_REGENT_L2L3_MOD", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L2 or L3 on the same chip in the same regent due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x084240000020C142", + "EventName": "PM_MRK_DATA_FROM_REGENT_L2L3_MOD_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L2 or L3 on the same chip in the same regent due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x080300000000C142", + "EventName": "PM_MRK_INST_FROM_REGENT_L2L3", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 or L3 on the same chip in the same regent due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x080340000000C142", + "EventName": "PM_MRK_DATA_FROM_REGENT_L2L3", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 or L3 on the same chip in the same regent due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x080300000010C142", + "EventName": "PM_MRK_INST_FROM_REGENT_L2L3_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 or L3 on the same chip in the same regent due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x080340000020C142", + "EventName": "PM_MRK_DATA_FROM_REGENT_L2L3_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 or L3 on the same chip in the same regent due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0A0040000000C142", + "EventName": "PM_MRK_DATA_FROM_L21_NON_REGENT_SHR", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L2 on the same chip in a different regent due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0A0040000020C142", + "EventName": "PM_MRK_DATA_FROM_L21_NON_REGENT_SHR_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L2 on the same chip in a different regent due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0A4040000000C142", + "EventName": "PM_MRK_DATA_FROM_L21_NON_REGENT_MOD", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L2 on the same chip in a different regent due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0A4040000020C142", + "EventName": "PM_MRK_DATA_FROM_L21_NON_REGENT_MOD_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L2 on the same chip in a different regent due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0A0100000000C142", + "EventName": "PM_MRK_INST_FROM_L21_NON_REGENT", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 on the same chip in a different regent due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0A0140000000C142", + "EventName": "PM_MRK_DATA_FROM_L21_NON_REGENT", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 on the same chip in a different regent due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0A0100000010C142", + "EventName": "PM_MRK_INST_FROM_L21_NON_REGENT_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 on the same chip in a different regent due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0A0140000020C142", + "EventName": "PM_MRK_DATA_FROM_L21_NON_REGENT_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 on the same chip in a different regent due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0A8040000000C142", + "EventName": "PM_MRK_DATA_FROM_L31_NON_REGENT_SHR", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L3 on the same chip in a different regent due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0A8040000020C142", + "EventName": "PM_MRK_DATA_FROM_L31_NON_REGENT_SHR_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L3 on the same chip in a different regent due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0AC040000000C142", + "EventName": "PM_MRK_DATA_FROM_L31_NON_REGENT_MOD", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L3 on the same chip in a different regent due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0AC040000020C142", + "EventName": "PM_MRK_DATA_FROM_L31_NON_REGENT_MOD_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L3 on the same chip in a different regent due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0A8100000000C142", + "EventName": "PM_MRK_INST_FROM_L31_NON_REGENT", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L3 on the same chip in a different regent due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0A8140000000C142", + "EventName": "PM_MRK_DATA_FROM_L31_NON_REGENT", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L3 on the same chip in a different regent due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0A8100000010C142", + "EventName": "PM_MRK_INST_FROM_L31_NON_REGENT_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L3 on the same chip in a different regent due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0A8140000020C142", + "EventName": "PM_MRK_DATA_FROM_L31_NON_REGENT_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L3 on the same chip in a different regent due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0A0240000000C142", + "EventName": "PM_MRK_DATA_FROM_NON_REGENT_L2L3_SHR", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L2 or L3 on the same chip in a different regent due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0A0240000020C142", + "EventName": "PM_MRK_DATA_FROM_NON_REGENT_L2L3_SHR_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L2 or L3 on the same chip in a different regent due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0A4240000000C142", + "EventName": "PM_MRK_DATA_FROM_NON_REGENT_L2L3_MOD", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L2 or L3 on the same chip in a different regent due to a demand miss for a marked instruction." + } +] diff --git a/tools/perf/pmu-events/arch/powerpc/power10/others.json b/tools/perf/pmu-events/arch/powerpc/power10/others.json index 0e21e7ba1959..fcf8a8ebe7bd 100644 --- a/tools/perf/pmu-events/arch/powerpc/power10/others.json +++ b/tools/perf/pmu-events/arch/powerpc/power10/others.json @@ -89,11 +89,6 @@ "EventName": "PM_LD_DEMAND_MISS_L1", "BriefDescription": "The L1 cache was reloaded with a line that fulfills a demand miss request. Counted at reload time, before finish." }, - { - "EventCode": "0x300FE", - "EventName": "PM_DATA_FROM_L3MISS", - "BriefDescription": "The processor's data cache was reloaded from a source other than the local core's L1, L2, or L3 due to a demand miss." - }, { "EventCode": "0x40012", "EventName": "PM_L1_ICACHE_RELOADED_ALL", @@ -113,10 +108,5 @@ "EventCode": "0x400F0", "EventName": "PM_LD_DEMAND_MISS_L1_FIN", "BriefDescription": "Load missed L1, counted at finish time." - }, - { - "EventCode": "0x400FE", - "EventName": "PM_DATA_FROM_MEMORY", - "BriefDescription": "The processor's data cache was reloaded from local, remote, or distant memory due to a demand miss." } ] diff --git a/tools/perf/pmu-events/arch/powerpc/power10/translation.json b/tools/perf/pmu-events/arch/powerpc/power10/translation.json index ea73900d248a..a96f76797da0 100644 --- a/tools/perf/pmu-events/arch/powerpc/power10/translation.json +++ b/tools/perf/pmu-events/arch/powerpc/power10/translation.json @@ -9,11 +9,6 @@ "EventName": "PM_ST_CMPL", "BriefDescription": "Stores completed from S2Q (2nd-level store queue). This event includes regular stores, stcx and cache inhibited stores. The following operations are excluded (pteupdate, snoop tlbie complete, store atomics, miso, load atomic payloads, tlbie, tlbsync, slbieg, isync, msgsnd, slbiag, cpabort, copy, tcheck, tend, stsync, dcbst, icbi, dcbf, hwsync, lwsync, ptesync, eieio, msgsync)." }, - { - "EventCode": "0x200FE", - "EventName": "PM_DATA_FROM_L2MISS", - "BriefDescription": "The processor's L1 data cache was reloaded from a source beyond the local core's L2 due to a demand miss." - }, { "EventCode": "0x300F0", "EventName": "PM_ST_MISS_L1", From 23ba30b23b761a2c7d6af87eb44e376459ecce3e Mon Sep 17 00:00:00 2001 From: Kajol Jain Date: Tue, 5 Sep 2023 17:10:38 +0530 Subject: [PATCH 023/475] perf vendor events power10: Add extra data-source events Update JSON/Events list with additional data-source events for power10 platform. Signed-off-by: Kajol Jain Cc: Athira Rajeev Cc: Disha Goel Cc: Madhavan Srinivasan Cc: Namhyung Kim Cc: linuxppc-dev@lists.ozlabs.org Link: https://lore.kernel.org/r/20230905114039.176645-2-kjain@linux.ibm.com Signed-off-by: Arnaldo Carvalho de Melo --- .../arch/powerpc/power10/datasource.json | 505 ++++++++++++++++++ 1 file changed, 505 insertions(+) diff --git a/tools/perf/pmu-events/arch/powerpc/power10/datasource.json b/tools/perf/pmu-events/arch/powerpc/power10/datasource.json index 12cfb9785433..6b0356f2d301 100644 --- a/tools/perf/pmu-events/arch/powerpc/power10/datasource.json +++ b/tools/perf/pmu-events/arch/powerpc/power10/datasource.json @@ -1278,5 +1278,510 @@ "EventCode": "0x0A4240000000C142", "EventName": "PM_MRK_DATA_FROM_NON_REGENT_L2L3_MOD", "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L2 or L3 on the same chip in a different regent due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0A4240000020C142", + "EventName": "PM_MRK_DATA_FROM_NON_REGENT_L2L3_MOD_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L2 or L3 on the same chip in a different regent due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0A0300000000C142", + "EventName": "PM_MRK_INST_FROM_NON_REGENT_L2L3", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 or L3 on the same chip in a different regent due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0A0340000000C142", + "EventName": "PM_MRK_DATA_FROM_NON_REGENT_L2L3", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 or L3 on the same chip in a different regent due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0A0300000010C142", + "EventName": "PM_MRK_INST_FROM_NON_REGENT_L2L3_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 or L3 on the same chip in a different regent due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0A0340000020C142", + "EventName": "PM_MRK_DATA_FROM_NON_REGENT_L2L3_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 or L3 on the same chip in a different regent due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x094100000000C142", + "EventName": "PM_MRK_INST_FROM_LMEM", + "BriefDescription": "The processor's instruction cache was reloaded from the local chip's memory due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x094040000000C142", + "EventName": "PM_MRK_DATA_FROM_LMEM", + "BriefDescription": "The processor's L1 data cache was reloaded from the local chip's memory due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x094100000010C142", + "EventName": "PM_MRK_INST_FROM_LMEM_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from the local chip's memory due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x094040000020C142", + "EventName": "PM_MRK_DATA_FROM_LMEM_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from the local chip's memory due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x098040000000C142", + "EventName": "PM_MRK_DATA_FROM_L_OC_CACHE", + "BriefDescription": "The processor's L1 data cache was reloaded from the local chip's OpenCAPI cache due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x098040000020C142", + "EventName": "PM_MRK_DATA_FROM_L_OC_CACHE_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from the local chip's OpenCAPI cache due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x09C040000000C142", + "EventName": "PM_MRK_DATA_FROM_L_OC_MEM", + "BriefDescription": "The processor's L1 data cache was reloaded from the local chip's OpenCAPI memory due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x09C040000020C142", + "EventName": "PM_MRK_DATA_FROM_L_OC_MEM_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from the local chip's OpenCAPI memory due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x098100000000C142", + "EventName": "PM_MRK_INST_FROM_L_OC_ANY", + "BriefDescription": "The processor's instruction cache was reloaded from the local chip's OpenCAPI cache or memory due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x098140000000C142", + "EventName": "PM_MRK_DATA_FROM_L_OC_ANY", + "BriefDescription": "The processor's L1 data cache was reloaded from the local chip's OpenCAPI cache or memory due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x098100000010C142", + "EventName": "PM_MRK_INST_FROM_L_OC_ANY_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from the local chip's OpenCAPI cache or memory due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x098140000020C142", + "EventName": "PM_MRK_DATA_FROM_L_OC_ANY_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from the local chip's OpenCAPI cache or memory due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0C0040000000C142", + "EventName": "PM_MRK_DATA_FROM_RL2_SHR", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L2 from a remote chip due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0C0040000020C142", + "EventName": "PM_MRK_DATA_FROM_RL2_SHR_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L2 from a remote chip due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0C4040000000C142", + "EventName": "PM_MRK_DATA_FROM_RL2_MOD", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L2 from a remote chip due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0C4040000020C142", + "EventName": "PM_MRK_DATA_FROM_RL2_MOD_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L2 from a remote chip due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0C0100000000C142", + "EventName": "PM_MRK_INST_FROM_RL2", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 from a remote chip due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0C0140000000C142", + "EventName": "PM_MRK_DATA_FROM_RL2", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 from a remote chip due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0C0100000010C142", + "EventName": "PM_MRK_INST_FROM_RL2_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 from a remote chip due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0C0140000020C142", + "EventName": "PM_MRK_DATA_FROM_RL2_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 from a remote chip due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0C8040000000C142", + "EventName": "PM_MRK_DATA_FROM_RL3_SHR", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L3 from a remote chip due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0C8040000020C142", + "EventName": "PM_MRK_DATA_FROM_RL3_SHR_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L3 from a remote chip due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0CC040000000C142", + "EventName": "PM_MRK_DATA_FROM_RL3_MOD", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L3 from a remote chip due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0CC040000020C142", + "EventName": "PM_MRK_DATA_FROM_RL3_MOD_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L3 from a remote chip due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0C8100000000C142", + "EventName": "PM_MRK_INST_FROM_RL3", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L3 from a remote chip due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0C8140000000C142", + "EventName": "PM_MRK_DATA_FROM_RL3", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L3 from a remote chip due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0C8100000010C142", + "EventName": "PM_MRK_INST_FROM_RL3_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L3 from a remote chip due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0C8140000020C142", + "EventName": "PM_MRK_DATA_FROM_RL3_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L3 from a remote chip due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0C0240000000C142", + "EventName": "PM_MRK_DATA_FROM_RL2L3_SHR", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L2 or L3 from a remote chip due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0C0240000020C142", + "EventName": "PM_MRK_DATA_FROM_RL2L3_SHR_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L2 or L3 from a remote chip due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0C4240000000C142", + "EventName": "PM_MRK_DATA_FROM_RL2L3_MOD", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L2 or L3 from a remote chip due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0C4240000020C142", + "EventName": "PM_MRK_DATA_FROM_RL2L3_MOD_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L2 or L3 from a remote chip due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0C0300000000C142", + "EventName": "PM_MRK_INST_FROM_RL2L3", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 or L3 from a remote chip due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0C0340000000C142", + "EventName": "PM_MRK_DATA_FROM_RL2L3", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 or L3 from a remote chip due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0C0300000010C142", + "EventName": "PM_MRK_INST_FROM_RL2L3_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 or L3 from a remote chip due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0C0340000020C142", + "EventName": "PM_MRK_DATA_FROM_RL2L3_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 or L3 from a remote chip due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0D4100000000C142", + "EventName": "PM_MRK_INST_FROM_RMEM", + "BriefDescription": "The processor's instruction cache was reloaded from remote memory (MC slow) due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0D4040000000C142", + "EventName": "PM_MRK_DATA_FROM_RMEM", + "BriefDescription": "The processor's L1 data cache was reloaded from remote memory (MC slow) due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0D4100000010C142", + "EventName": "PM_MRK_INST_FROM_RMEM_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from remote memory (MC slow) due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0D4040000020C142", + "EventName": "PM_MRK_DATA_FROM_RMEM_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from remote memory (MC slow) due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0D8040000000C142", + "EventName": "PM_MRK_DATA_FROM_R_OC_CACHE", + "BriefDescription": "The processor's L1 data cache was reloaded from a remote chip's OpenCAPI cache due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0D8040000020C142", + "EventName": "PM_MRK_DATA_FROM_R_OC_CACHE_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from a remote chip's OpenCAPI cache due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0DC040000000C142", + "EventName": "PM_MRK_DATA_FROM_R_OC_MEM", + "BriefDescription": "The processor's L1 data cache was reloaded from a remote chip's OpenCAPI memory due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0DC040000020C142", + "EventName": "PM_MRK_DATA_FROM_R_OC_MEM_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from a remote chip's OpenCAPI memory due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0D8100000000C142", + "EventName": "PM_MRK_INST_FROM_R_OC_ANY", + "BriefDescription": "The processor's instruction cache was reloaded from a remote chip's OpenCAPI cache or memory due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0D8140000000C142", + "EventName": "PM_MRK_DATA_FROM_R_OC_ANY", + "BriefDescription": "The processor's L1 data cache was reloaded from a remote chip's OpenCAPI cache or memory due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0D8100000010C142", + "EventName": "PM_MRK_INST_FROM_R_OC_ANY_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from a remote chip's OpenCAPI cache or memory due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0D8140000020C142", + "EventName": "PM_MRK_DATA_FROM_R_OC_ANY_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from a remote chip's OpenCAPI cache or memory due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0E0040000000C142", + "EventName": "PM_MRK_DATA_FROM_DL2_SHR", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L2 from a distant chip due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0E0040000020C142", + "EventName": "PM_MRK_DATA_FROM_DL2_SHR_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L2 from a distant chip due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0E4040000000C142", + "EventName": "PM_MRK_DATA_FROM_DL2_MOD", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L2 from a distant chip due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0E4040000020C142", + "EventName": "PM_MRK_DATA_FROM_DL2_MOD_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L2 from a distant chip due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0E0100000000C142", + "EventName": "PM_MRK_INST_FROM_DL2", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 from a distant chip due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0E0140000000C142", + "EventName": "PM_MRK_DATA_FROM_DL2", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 from a distant chip due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0E0100000010C142", + "EventName": "PM_MRK_INST_FROM_DL2_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 from a distant chip due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0E0140000020C142", + "EventName": "PM_MRK_DATA_FROM_DL2_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 from a distant chip due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0E8040000000C142", + "EventName": "PM_MRK_DATA_FROM_DL3_SHR", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L3 from a distant chip due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0E8040000020C142", + "EventName": "PM_MRK_DATA_FROM_DL3_SHR_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L3 from a distant chip due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0EC040000000C142", + "EventName": "PM_MRK_DATA_FROM_DL3_MOD", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L3 from a distant chip due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0EC040000020C142", + "EventName": "PM_MRK_DATA_FROM_DL3_MOD_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L3 from a distant chip due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0E8100000000C142", + "EventName": "PM_MRK_INST_FROM_DL3", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L3 from a distant chip due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0E8140000000C142", + "EventName": "PM_MRK_DATA_FROM_DL3", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L3 from a distant chip due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0E8100000010C142", + "EventName": "PM_MRK_INST_FROM_DL3_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L3 from a distant chip due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0E8140000020C142", + "EventName": "PM_MRK_DATA_FROM_DL3_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L3 from a distant chip due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0E0240000000C142", + "EventName": "PM_MRK_DATA_FROM_DL2L3_SHR", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L2 or L3 from a distant chip due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0E0240000020C142", + "EventName": "PM_MRK_DATA_FROM_DL2L3_SHR_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a valid line that was not in the M (exclusive) state from another core's L2 or L3 from a distant chip due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0E4240000000C142", + "EventName": "PM_MRK_DATA_FROM_DL2L3_MOD", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L2 or L3 from a distant chip due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0E4240000020C142", + "EventName": "PM_MRK_DATA_FROM_DL2L3_MOD_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded with a line in the M (exclusive) state from another core's L2 or L3 from a distant chip due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0E0300000000C142", + "EventName": "PM_MRK_INST_FROM_DL2L3", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 or L3 from a distant chip due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0E0340000000C142", + "EventName": "PM_MRK_DATA_FROM_DL2L3", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 or L3 from a distant chip due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0E0300000010C142", + "EventName": "PM_MRK_INST_FROM_DL2L3_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 or L3 from a distant chip due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0E0340000020C142", + "EventName": "PM_MRK_DATA_FROM_DL2L3_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 or L3 from a distant chip due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0F4100000000C142", + "EventName": "PM_MRK_INST_FROM_DMEM", + "BriefDescription": "The processor's instruction cache was reloaded from distant memory (MC slow) due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0F4040000000C142", + "EventName": "PM_MRK_DATA_FROM_DMEM", + "BriefDescription": "The processor's L1 data cache was reloaded from distant memory (MC slow) due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0F4100000010C142", + "EventName": "PM_MRK_INST_FROM_DMEM_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from distant memory (MC slow) due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0F4040000020C142", + "EventName": "PM_MRK_DATA_FROM_DMEM_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from distant memory (MC slow) due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0F8040000000C142", + "EventName": "PM_MRK_DATA_FROM_D_OC_CACHE", + "BriefDescription": "The processor's L1 data cache was reloaded from a distant chip's OpenCAPI cache due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0F8040000020C142", + "EventName": "PM_MRK_DATA_FROM_D_OC_CACHE_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from a distant chip's OpenCAPI cache due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0FC040000000C142", + "EventName": "PM_MRK_DATA_FROM_D_OC_MEM", + "BriefDescription": "The processor's L1 data cache was reloaded from a distant chip's OpenCAPI memory due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0FC040000020C142", + "EventName": "PM_MRK_DATA_FROM_D_OC_MEM_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from a distant chip's OpenCAPI memory due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0F8100000000C142", + "EventName": "PM_MRK_INST_FROM_D_OC_ANY", + "BriefDescription": "The processor's instruction cache was reloaded from a distant chip's OpenCAPI cache or memory due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0F8140000000C142", + "EventName": "PM_MRK_DATA_FROM_D_OC_ANY", + "BriefDescription": "The processor's L1 data cache was reloaded from a distant chip's OpenCAPI cache or memory due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0F8100000010C142", + "EventName": "PM_MRK_INST_FROM_D_OC_ANY_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from a distant chip's OpenCAPI cache or memory due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0F8140000020C142", + "EventName": "PM_MRK_DATA_FROM_D_OC_ANY_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from a distant chip's OpenCAPI cache or memory due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x080B00000000C142", + "EventName": "PM_MRK_INST_FROM_ONCHIP_CACHE", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 or L3 from the same chip due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x080B40000000C142", + "EventName": "PM_MRK_DATA_FROM_ONCHIP_CACHE", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 or L3 from the same chip due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x080B00000010C142", + "EventName": "PM_MRK_INST_FROM_ONCHIP_CACHE_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 or L3 from the same chip due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x080B40000020C142", + "EventName": "PM_MRK_DATA_FROM_ONCHIP_CACHE_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 or L3 from the same chip due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0C0B00000000C142", + "EventName": "PM_MRK_INST_FROM_OFFCHIP_CACHE", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 or L3 from a different chip due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0C0B40000000C142", + "EventName": "PM_MRK_DATA_FROM_OFFCHIP_CACHE", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 or L3 from a different chip due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x0C0B00000010C142", + "EventName": "PM_MRK_INST_FROM_OFFCHIP_CACHE_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from another core's L2 or L3 from a different chip due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x0C0B40000020C142", + "EventName": "PM_MRK_DATA_FROM_OFFCHIP_CACHE_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from another core's L2 or L3 from a different chip due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x095900000000C142", + "EventName": "PM_MRK_INST_FROM_ANY_MEMORY", + "BriefDescription": "The processor's instruction cache was reloaded from any chip's memory (MC slow) due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x095840000000C142", + "EventName": "PM_MRK_DATA_FROM_ANY_MEMORY", + "BriefDescription": "The processor's L1 data cache was reloaded from any chip's memory (MC slow) due to a demand miss for a marked instruction." + }, + { + "EventCode": "0x095900000010C142", + "EventName": "PM_MRK_INST_FROM_ANY_MEMORY_ALL", + "BriefDescription": "The processor's instruction cache was reloaded from any chip's memory (MC slow) due to a demand miss or prefetch reload for a marked instruction." + }, + { + "EventCode": "0x095840000020C142", + "EventName": "PM_MRK_DATA_FROM_ANY_MEMORY_ALL", + "BriefDescription": "The processor's L1 data cache was reloaded from any chip's memory (MC slow) due to a demand miss or prefetch reload for a marked instruction." } ] From 1bd69b4bf1dde99ac1ba2fdbcd5408d7f82c406a Mon Sep 17 00:00:00 2001 From: Kajol Jain Date: Tue, 5 Sep 2023 17:10:39 +0530 Subject: [PATCH 024/475] perf vendor events: Update metric events for power10 platform Update JSON/events for power10 platform with additional metrics. Signed-off-by: Kajol Jain Cc: Athira Rajeev Cc: Disha Goel Cc: Madhavan Srinivasan Cc: Namhyung Kim Cc: linuxppc-dev@lists.ozlabs.org Link: https://lore.kernel.org/r/20230905114039.176645-3-kjain@linux.ibm.com Signed-off-by: Arnaldo Carvalho de Melo --- .../arch/powerpc/power10/metrics.json | 388 ++++++++++++++++++ 1 file changed, 388 insertions(+) diff --git a/tools/perf/pmu-events/arch/powerpc/power10/metrics.json b/tools/perf/pmu-events/arch/powerpc/power10/metrics.json index 4d66b75c6ad5..a36621858ea3 100644 --- a/tools/perf/pmu-events/arch/powerpc/power10/metrics.json +++ b/tools/perf/pmu-events/arch/powerpc/power10/metrics.json @@ -434,6 +434,13 @@ "MetricName": "L1_INST_MISS_RATE", "ScaleUnit": "1%" }, + { + "BriefDescription": "Percentage of completed instructions that were demand fetches that missed the L1 and L2 instruction cache", + "MetricExpr": "PM_INST_FROM_L2MISS * 100 / PM_RUN_INST_CMPL", + "MetricGroup": "General", + "MetricName": "L2_INST_MISS_RATE", + "ScaleUnit": "1%" + }, { "BriefDescription": "Percentage of completed instructions that were demand fetches that reloaded from beyond the L3 icache", "MetricExpr": "PM_INST_FROM_L3MISS / PM_RUN_INST_CMPL * 100", @@ -466,6 +473,13 @@ "MetricGroup": "General", "MetricName": "LOADS_PER_INST" }, + { + "BriefDescription": "Percentage of demand loads that reloaded from the L2 per completed instruction", + "MetricExpr": "PM_DATA_FROM_L2 * 100 / PM_RUN_INST_CMPL", + "MetricGroup": "dL1_Reloads", + "MetricName": "DL1_RELOAD_FROM_L2_RATE", + "ScaleUnit": "1%" + }, { "BriefDescription": "Percentage of demand loads that reloaded from beyond the L2 per completed instruction", "MetricExpr": "PM_DATA_FROM_L2MISS / PM_RUN_INST_CMPL * 100", @@ -473,6 +487,34 @@ "MetricName": "DL1_RELOAD_FROM_L2_MISS_RATE", "ScaleUnit": "1%" }, + { + "BriefDescription": "Percentage of demand loads that reloaded using modified data from another core's L2 or L3 on a remote chip, per completed instruction", + "MetricExpr": "PM_DATA_FROM_RL2L3_MOD * 100 / PM_RUN_INST_CMPL", + "MetricGroup": "dL1_Reloads", + "MetricName": "DL1_RELOAD_FROM_RL2L3_MOD_RATE", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of demand loads that reloaded using shared data from another core's L2 or L3 on a remote chip, per completed instruction", + "MetricExpr": "PM_DATA_FROM_RL2L3_SHR * 100 / PM_RUN_INST_CMPL", + "MetricGroup": "dL1_Reloads", + "MetricName": "DL1_RELOAD_FROM_RL2L3_SHR_RATE", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of demand loads that reloaded from the L3 per completed instruction", + "MetricExpr": "PM_DATA_FROM_L3 * 100 / PM_RUN_INST_CMPL", + "MetricGroup": "dL1_Reloads", + "MetricName": "DL1_RELOAD_FROM_L3_RATE", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of demand loads that reloaded with data brought into the L3 by prefetch per completed instruction", + "MetricExpr": "PM_DATA_FROM_L3_MEPF * 100 / PM_RUN_INST_CMPL", + "MetricGroup": "dL1_Reloads", + "MetricName": "DL1_RELOAD_FROM_L3_MEPF_RATE", + "ScaleUnit": "1%" + }, { "BriefDescription": "Percentage of demand loads that reloaded from beyond the L3 per completed instruction", "MetricExpr": "PM_DATA_FROM_L3MISS / PM_RUN_INST_CMPL * 100", @@ -480,6 +522,66 @@ "MetricName": "DL1_RELOAD_FROM_L3_MISS_RATE", "ScaleUnit": "1%" }, + { + "BriefDescription": "Percentage of demand loads that reloaded using modified data from another core's L2 or L3 on a distant chip, per completed instruction", + "MetricExpr": "PM_DATA_FROM_DL2L3_MOD * 100 / PM_RUN_INST_CMPL", + "MetricGroup": "dL1_Reloads", + "MetricName": "DL1_RELOAD_FROM_DL2L3_MOD_RATE", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of demand loads that reloaded using shared data from another core's L2 or L3 on a distant chip, per completed instruction", + "MetricExpr": "PM_DATA_FROM_DL2L3_SHR * 100 / PM_RUN_INST_CMPL", + "MetricGroup": "dL1_Reloads", + "MetricName": "DL1_RELOAD_FROM_DL2L3_SHR_RATE", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of demand loads that reloaded from local memory per completed instruction", + "MetricExpr": "PM_DATA_FROM_LMEM * 100 / PM_RUN_INST_CMPL", + "MetricGroup": "dL1_Reloads", + "MetricName": "DL1_RELOAD_FROM_LMEM_RATE", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of demand loads that reloaded from remote memory per completed instruction", + "MetricExpr": "PM_DATA_FROM_RMEM * 100 / PM_RUN_INST_CMPL", + "MetricGroup": "dL1_Reloads", + "MetricName": "DL1_RELOAD_FROM_RMEM_RATE", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of demand loads that reloaded from distant memory per completed instruction", + "MetricExpr": "PM_DATA_FROM_DMEM * 100 / PM_RUN_INST_CMPL", + "MetricGroup": "dL1_Reloads", + "MetricName": "DL1_RELOAD_FROM_DMEM_RATE", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of data reloads from local memory per data reloads from any memory", + "MetricExpr": "PM_DATA_FROM_LMEM * 100 / (PM_DATA_FROM_LMEM + PM_DATA_FROM_RMEM + PM_DATA_FROM_DMEM)", + "MetricGroup": "Memory", + "MetricName": "MEM_LOCALITY", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Number of data reloads from local memory per data reloads from remote memory", + "MetricExpr": "PM_DATA_FROM_LMEM / PM_DATA_FROM_RMEM", + "MetricGroup": "Memory", + "MetricName": "LD_LMEM_PER_LD_RMEM" + }, + { + "BriefDescription": "Number of data reloads from local memory per data reloads from distant memory", + "MetricExpr": "PM_DATA_FROM_LMEM / PM_DATA_FROM_DMEM", + "MetricGroup": "Memory", + "MetricName": "LD_LMEM_PER_LD_DMEM" + }, + { + "BriefDescription": "Number of data reloads from local memory per data reloads from distant and remote memory", + "MetricExpr": "PM_DATA_FROM_LMEM / (PM_DATA_FROM_DMEM + PM_DATA_FROM_RMEM)", + "MetricGroup": "Memory", + "MetricName": "LD_LMEM_PER_LD_MEM" + }, { "BriefDescription": "Percentage of ITLB misses per completed run instruction", "MetricExpr": "PM_ITLB_MISS / PM_RUN_INST_CMPL * 100", @@ -487,6 +589,12 @@ "MetricName": "ITLB_MISS_RATE", "ScaleUnit": "1%" }, + { + "BriefDescription": "Number of data reloads from remote memory per data reloads from distant memory", + "MetricExpr": "PM_DATA_FROM_RMEM / PM_DATA_FROM_DMEM", + "MetricGroup": "Memory", + "MetricName": "LD_RMEM_PER_LD_DMEM" + }, { "BriefDescription": "Percentage of DERAT misses with 4k page size per completed instruction", "MetricExpr": "PM_DERAT_MISS_4K / PM_RUN_INST_CMPL * 100", @@ -501,6 +609,76 @@ "MetricName": "DERAT_64K_MISS_RATE", "ScaleUnit": "1%" }, + { + "BriefDescription": "Percentage of ICache misses that were reloaded from the L2", + "MetricExpr": "PM_INST_FROM_L2 * 100 / PM_L1_ICACHE_MISS", + "MetricGroup": "Instruction_Stats", + "MetricName": "INST_FROM_L2", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of ICache misses that were reloaded from the L3", + "MetricExpr": "PM_INST_FROM_L3 * 100 / PM_L1_ICACHE_MISS", + "MetricGroup": "Instruction_Stats", + "MetricName": "INST_FROM_L3", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of ICache misses that were reloaded from local memory", + "MetricExpr": "PM_INST_FROM_LMEM * 100 / PM_L1_ICACHE_MISS", + "MetricGroup": "Instruction_Stats", + "MetricName": "INST_FROM_LMEM", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of ICache misses that were reloaded from remote memory", + "MetricExpr": "PM_INST_FROM_RMEM * 100 / PM_L1_ICACHE_MISS", + "MetricGroup": "Instruction_Stats", + "MetricName": "INST_FROM_RMEM", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of ICache misses that were reloaded from distant memory", + "MetricExpr": "PM_INST_FROM_DMEM * 100 / PM_L1_ICACHE_MISS", + "MetricGroup": "Instruction_Stats", + "MetricName": "INST_FROM_DMEM", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of ICache reloads from the L2 per completed instruction", + "MetricExpr": "PM_INST_FROM_L2 * 100 / PM_RUN_INST_CMPL", + "MetricGroup": "Instruction_Misses", + "MetricName": "INST_FROM_L2_RATE", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of ICache reloads from the L3 per completed instruction", + "MetricExpr": "PM_INST_FROM_L3 * 100 / PM_RUN_INST_CMPL", + "MetricGroup": "Instruction_Misses", + "MetricName": "INST_FROM_L3_RATE", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of ICache reloads from local memory per completed instruction", + "MetricExpr": "PM_INST_FROM_LMEM * 100 / PM_RUN_INST_CMPL", + "MetricGroup": "Instruction_Misses", + "MetricName": "INST_FROM_LMEM_RATE", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of ICache reloads from remote memory per completed instruction", + "MetricExpr": "PM_INST_FROM_RMEM * 100 / PM_RUN_INST_CMPL", + "MetricGroup": "Instruction_Misses", + "MetricName": "INST_FROM_RMEM_RATE", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of ICache reloads from distant memory per completed instruction", + "MetricExpr": "PM_INST_FROM_DMEM * 100 / PM_RUN_INST_CMPL", + "MetricGroup": "Instruction_Misses", + "MetricName": "INST_FROM_DMEM_RATE", + "ScaleUnit": "1%" + }, { "BriefDescription": "Average number of run cycles per completed instruction", "MetricExpr": "PM_RUN_CYC / PM_RUN_INST_CMPL", @@ -607,6 +785,13 @@ "MetricName": "DL1_MISS_RELOADS", "ScaleUnit": "1%" }, + { + "BriefDescription": "Percentage of demand load misses that reloaded from the local L2", + "MetricExpr": "PM_DATA_FROM_L2 * 100 / PM_LD_DEMAND_MISS_L1", + "MetricGroup": "dL1_Reloads", + "MetricName": "DL1_RELOAD_FROM_L2", + "ScaleUnit": "1%" + }, { "BriefDescription": "Percentage of demand load misses that reloaded from beyond the local L2", "MetricExpr": "PM_DATA_FROM_L2MISS / PM_LD_DEMAND_MISS_L1 * 100", @@ -614,6 +799,13 @@ "MetricName": "DL1_RELOAD_FROM_L2_MISS", "ScaleUnit": "1%" }, + { + "BriefDescription": "Percentage of demand load misses that reloaded from the local L3", + "MetricExpr": "PM_DATA_FROM_L3 * 100 / PM_LD_DEMAND_MISS_L1", + "MetricGroup": "dL1_Reloads", + "MetricName": "DL1_RELOAD_FROM_L3", + "ScaleUnit": "1%" + }, { "BriefDescription": "Percentage of demand load misses that reloaded from beyond the local L3", "MetricExpr": "PM_DATA_FROM_L3MISS / PM_LD_DEMAND_MISS_L1 * 100", @@ -621,6 +813,188 @@ "MetricName": "DL1_RELOAD_FROM_L3_MISS", "ScaleUnit": "1%" }, + { + "BriefDescription": "Percentage of demand load misses that reloaded from the local L3 with modified data", + "MetricExpr": "PM_DATA_FROM_L3_MEPF * 100 / PM_LD_DEMAND_MISS_L1", + "MetricGroup": "dL1_Reloads", + "MetricName": "DL1_RELOAD_FROM_L3_MEPF", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of demand load misses that reloaded from another core's L2 on the same regent with modified data", + "MetricExpr": "PM_DATA_FROM_L21_REGENT_MOD * 100 / PM_LD_DEMAND_MISS_L1", + "MetricGroup": "dL1_Reloads", + "MetricName": "DL1_RELOAD_FROM_L21_REGENT_MOD", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of demand load misses that reloaded from another core's L2 on the same regent with shared data", + "MetricExpr": "PM_DATA_FROM_L21_REGENT_SHR * 100 / PM_LD_DEMAND_MISS_L1", + "MetricGroup": "dL1_Reloads", + "MetricName": "DL1_RELOAD_FROM_L21_REGENT_SHR", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of demand load misses that reloaded from another core's L2 on the same chip in a different regent with modified data", + "MetricExpr": "PM_DATA_FROM_L21_NON_REGENT_MOD * 100 / PM_LD_DEMAND_MISS_L1", + "MetricGroup": "dL1_Reloads", + "MetricName": "DL1_RELOAD_FROM_L21_NON_REGENT_MOD", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of demand load misses that reloaded from another core's L2 on the same chip in a different regent with shared data", + "MetricExpr": "PM_DATA_FROM_L21_NON_REGENT_SHR * 100 / PM_LD_DEMAND_MISS_L1", + "MetricGroup": "dL1_Reloads", + "MetricName": "DL1_RELOAD_FROM_L21_NON_REGENT_SHR", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of demand load misses that reloaded from another core's L3 on the same regent with modified data", + "MetricExpr": "PM_DATA_FROM_L31_REGENT_MOD * 100 / PM_LD_DEMAND_MISS_L1", + "MetricGroup": "dL1_Reloads", + "MetricName": "DL1_RELOAD_FROM_L31_REGENT_MOD", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of demand load misses that reloaded from another core's L3 on the same regent with shared data", + "MetricExpr": "PM_DATA_FROM_L31_REGENT_SHR * 100 / PM_LD_DEMAND_MISS_L1", + "MetricGroup": "dL1_Reloads", + "MetricName": "DL1_RELOAD_FROM_L31_REGENT_SHR", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of demand load misses that reloaded from another core's L3 on the same chip in a different regent with modified data", + "MetricExpr": "PM_DATA_FROM_L31_NON_REGENT_MOD * 100 / PM_LD_DEMAND_MISS_L1", + "MetricGroup": "dL1_Reloads", + "MetricName": "DL1_RELOAD_FROM_L31_NON_REGENT_MOD", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of demand load misses that reloaded from another core's L3 on the same chip in a different regent with shared data", + "MetricExpr": "PM_DATA_FROM_L31_NON_REGENT_SHR * 100 / PM_LD_DEMAND_MISS_L1", + "MetricGroup": "dL1_Reloads", + "MetricName": "DL1_RELOAD_FROM_L31_NON_REGENT_SHR", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of demand load misses that reloaded from another core's L2 on a remote chip with modified data", + "MetricExpr": "PM_DATA_FROM_RL2_MOD * 100 / PM_LD_DEMAND_MISS_L1", + "MetricGroup": "dL1_Reloads", + "MetricName": "DL1_RELOAD_FROM_RL2_MOD", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of demand load misses that reloaded from another core's L2 on a remote chip with shared data", + "MetricExpr": "PM_DATA_FROM_RL2_SHR * 100 / PM_LD_DEMAND_MISS_L1", + "MetricGroup": "dL1_Reloads", + "MetricName": "DL1_RELOAD_FROM_RL2_SHR", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of demand load misses that reloaded from another core's L3 on a remote chip with modified data", + "MetricExpr": "PM_DATA_FROM_RL3_MOD * 100 / PM_LD_DEMAND_MISS_L1", + "MetricGroup": "dL1_Reloads", + "MetricName": "DL1_RELOAD_FROM_RL3_MOD", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of demand load misses that reloaded from another core's L3 on a remote chip with shared data", + "MetricExpr": "PM_DATA_FROM_RL3_SHR * 100 / PM_LD_DEMAND_MISS_L1", + "MetricGroup": "dL1_Reloads", + "MetricName": "DL1_RELOAD_FROM_RL3_SHR", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of demand load misses that reloaded from another core's L2 on a distant chip with modified data", + "MetricExpr": "PM_DATA_FROM_DL2_MOD * 100 / PM_LD_DEMAND_MISS_L1", + "MetricGroup": "dL1_Reloads", + "MetricName": "DL1_RELOAD_FROM_DL2_MOD", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of demand load misses that reloaded from another core's L2 on a distant chip with shared data", + "MetricExpr": "PM_DATA_FROM_DL2_SHR * 100 / PM_LD_DEMAND_MISS_L1", + "MetricGroup": "dL1_Reloads", + "MetricName": "DL1_RELOAD_FROM_DL2_SHR", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of demand load misses that reloaded from another core's L3 on a distant chip with modified data", + "MetricExpr": "PM_DATA_FROM_DL3_MOD * 100 / PM_LD_DEMAND_MISS_L1", + "MetricGroup": "dL1_Reloads", + "MetricName": "DL1_RELOAD_FROM_DL3_MOD", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of demand load misses that reloaded from another core's L3 on a distant chip with shared data", + "MetricExpr": "PM_DATA_FROM_DL3_SHR * 100 / PM_LD_DEMAND_MISS_L1", + "MetricGroup": "dL1_Reloads", + "MetricName": "DL1_RELOAD_FROM_DL3_SHR", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of demand load misses that reloaded from the local chip's memory", + "MetricExpr": "PM_DATA_FROM_LMEM * 100 / PM_LD_DEMAND_MISS_L1", + "MetricGroup": "dL1_Reloads", + "MetricName": "DL1_RELOAD_FROM_LMEM", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of demand load misses that reloaded from the local chip's OpenCAPI Cache", + "MetricExpr": "PM_DATA_FROM_L_OC_CACHE * 100 / PM_LD_DEMAND_MISS_L1", + "MetricGroup": "dL1_Reloads", + "MetricName": "DL1_RELOAD_FROM_L_OC_CACHE", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of demand load misses that reloaded from the local chip's OpenCAPI memory", + "MetricExpr": "PM_DATA_FROM_L_OC_MEM * 100 / PM_LD_DEMAND_MISS_L1", + "MetricGroup": "dL1_Reloads", + "MetricName": "DL1_RELOAD_FROM_L_OC_MEM", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of demand load misses that reloaded from a remote chip's memory", + "MetricExpr": "PM_DATA_FROM_RMEM * 100 / PM_LD_DEMAND_MISS_L1", + "MetricGroup": "dL1_Reloads", + "MetricName": "DL1_RELOAD_FROM_RMEM", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of demand load misses that reloaded from a remote chip's OpenCAPI Cache", + "MetricExpr": "PM_DATA_FROM_R_OC_CACHE * 100 / PM_LD_DEMAND_MISS_L1", + "MetricGroup": "dL1_Reloads", + "MetricName": "DL1_RELOAD_FROM_R_OC_CACHE", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of demand load misses that reloaded from a remote chip's OpenCAPI memory", + "MetricExpr": "PM_DATA_FROM_R_OC_MEM * 100 / PM_LD_DEMAND_MISS_L1", + "MetricGroup": "dL1_Reloads", + "MetricName": "DL1_RELOAD_FROM_R_OC_MEM", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of demand load misses that reloaded from a distant chip's memory", + "MetricExpr": "PM_DATA_FROM_DMEM * 100 / PM_LD_DEMAND_MISS_L1", + "MetricGroup": "dL1_Reloads", + "MetricName": "DL1_RELOAD_FROM_DMEM", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of demand load misses that reloaded from a distant chip's OpenCAPI Cache", + "MetricExpr": "PM_DATA_FROM_D_OC_CACHE * 100 / PM_LD_DEMAND_MISS_L1", + "MetricGroup": "dL1_Reloads", + "MetricName": "DL1_RELOAD_FROM_D_OC_CACHE", + "ScaleUnit": "1%" + }, + { + "BriefDescription": "Percentage of demand load misses that reloaded from a distant chip's OpenCAPI memory", + "MetricExpr": "PM_DATA_FROM_D_OC_MEM * 100 / PM_LD_DEMAND_MISS_L1", + "MetricGroup": "dL1_Reloads", + "MetricName": "DL1_RELOAD_FROM_D_OC_MEM", + "ScaleUnit": "1%" + }, { "BriefDescription": "Percentage of cycles stalled due to the NTC instruction waiting for a load miss to resolve from a source beyond the local L2 and local L3", "MetricExpr": "DMISS_L3MISS_STALL_CPI / RUN_CPI * 100", @@ -686,6 +1060,13 @@ "MetricName": "DERAT_MISS_RELOAD", "ScaleUnit": "1%" }, + { + "BriefDescription": "Percentage of ICache misses that were reloaded from beyond the local L2", + "MetricExpr": "PM_INST_FROM_L2MISS * 100 / PM_L1_ICACHE_MISS", + "MetricGroup": "Instruction_Misses", + "MetricName": "INST_FROM_L2_MISS", + "ScaleUnit": "1%" + }, { "BriefDescription": "Percentage of icache misses that were reloaded from beyond the local L3", "MetricExpr": "PM_INST_FROM_L3MISS / PM_L1_ICACHE_MISS * 100", @@ -693,6 +1074,13 @@ "MetricName": "INST_FROM_L3_MISS", "ScaleUnit": "1%" }, + { + "BriefDescription": "Percentage of ICache reloads from beyond the L2 per completed instruction", + "MetricExpr": "PM_INST_FROM_L2MISS * 100 / PM_RUN_INST_CMPL", + "MetricGroup": "Instruction_Misses", + "MetricName": "INST_FROM_L2_MISS_RATE", + "ScaleUnit": "1%" + }, { "BriefDescription": "Percentage of icache reloads from the beyond the L3 per completed instruction", "MetricExpr": "PM_INST_FROM_L3MISS / PM_RUN_INST_CMPL * 100", From a84fbf205609313594b86065c67e823f09ebe29b Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Tue, 5 Sep 2023 17:39:12 -0700 Subject: [PATCH 025/475] perf stat: Fix aggr mode initialization Generating metrics llc_code_read_mpi_demand_plus_prefetch, llc_data_read_mpi_demand_plus_prefetch, llc_miss_local_memory_bandwidth_read, llc_miss_local_memory_bandwidth_write, nllc_miss_remote_memory_bandwidth_read, memory_bandwidth_read, memory_bandwidth_write, uncore_frequency, upi_data_transmit_bw, C2_Pkg_Residency, C3_Core_Residency, C3_Pkg_Residency, C6_Core_Residency, C6_Pkg_Residency, C7_Core_Residency, C7_Pkg_Residency, UNCORE_FREQ and tma_info_system_socket_clks would trigger an address sanitizer heap-buffer-overflows on a SkylakeX. ``` ==2567752==ERROR: AddressSanitizer: heap-buffer-overflow on address 0x5020003ed098 at pc 0x5621a816654e bp 0x7fffb55d4da0 sp 0x7fffb55d4d98 READ of size 4 at 0x5020003eee78 thread T0 #0 0x558265d6654d in aggr_cpu_id__is_empty tools/perf/util/cpumap.c:694:12 #1 0x558265c914da in perf_stat__get_aggr tools/perf/builtin-stat.c:1490:6 #2 0x558265c914da in perf_stat__get_global_cached tools/perf/builtin-stat.c:1530:9 #3 0x558265e53290 in should_skip_zero_counter tools/perf/util/stat-display.c:947:31 #4 0x558265e53290 in print_counter_aggrdata tools/perf/util/stat-display.c:985:18 #5 0x558265e51931 in print_counter tools/perf/util/stat-display.c:1110:3 #6 0x558265e51931 in evlist__print_counters tools/perf/util/stat-display.c:1571:5 #7 0x558265c8ec87 in print_counters tools/perf/builtin-stat.c:981:2 #8 0x558265c8cc71 in cmd_stat tools/perf/builtin-stat.c:2837:3 #9 0x558265bb9bd4 in run_builtin tools/perf/perf.c:323:11 #10 0x558265bb98eb in handle_internal_command tools/perf/perf.c:377:8 #11 0x558265bb9389 in run_argv tools/perf/perf.c:421:2 #12 0x558265bb9389 in main tools/perf/perf.c:537:3 ``` The issue was the use of testing a cpumap with NULL rather than using empty, as a map containing the dummy value isn't NULL and the -1 results in an empty aggr map being allocated which legitimately overflows when any member is accessed. Fixes: 8a96f454f5668572 ("perf stat: Avoid SEGV if core.cpus isn't set") Signed-off-by: Ian Rogers Acked-by: Namhyung Kim Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ian Rogers Cc: Ingo Molnar Cc: Jiri Olsa Cc: Mark Rutland Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20230906003912.3317462-1-irogers@google.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/builtin-stat.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/perf/builtin-stat.c b/tools/perf/builtin-stat.c index 07b48f6df48e..a3af805a1d57 100644 --- a/tools/perf/builtin-stat.c +++ b/tools/perf/builtin-stat.c @@ -1622,7 +1622,7 @@ static int perf_stat_init_aggr_mode(void) * taking the highest cpu number to be the size of * the aggregation translate cpumap. */ - if (evsel_list->core.user_requested_cpus) + if (!perf_cpu_map__empty(evsel_list->core.user_requested_cpus)) nr = perf_cpu_map__max(evsel_list->core.user_requested_cpus).cpu; else nr = 0; From 10864594d8dedc63183f164c9cb139a521c7c64d Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Tue, 5 Sep 2023 11:15:52 -0700 Subject: [PATCH 026/475] perf shell completion: Restrict completion of events to events 'perf list' will list libpfm4 events and metrics which aren't valid options to the '-e' option. Restrict the events gathered so that invalid ones aren't shown. Before: $ perf stat -e Display all 633 possibilities? (y or n) After: $ perf stat -e Display all 375 possibilities? (y or n) Committer testing: # grep -m1 'model name' /proc/cpuinfo model name : AMD Ryzen 9 5950X 16-Core Processor # Before: # . tools/perf/perf-completion.sh # perf stat -e Display all 2672 possibilities? (y or n) After: # . tools/perf/perf-completion.sh # perf stat -e Display all 2648 possibilities? (y or n) Reported-by: Arnaldo Carvalho de Melo Signed-off-by: Ian Rogers Tested-by: Arnaldo Carvalho de Melo Acked-by: Namhyung Kim Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ingo Molnar Cc: Jiri Olsa Cc: Mark Rutland Cc: Peter Zijlstra Cc: Stephane Eranian Link: https://lore.kernel.org/r/20230905181554.3202873-1-irogers@google.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/perf-completion.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/perf/perf-completion.sh b/tools/perf/perf-completion.sh index 978249d7868c..40cfc30ad7ad 100644 --- a/tools/perf/perf-completion.sh +++ b/tools/perf/perf-completion.sh @@ -164,7 +164,7 @@ __perf_main () $prev_skip_opts == @(record|stat|top) ]]; then local cur1=${COMP_WORDS[COMP_CWORD]} - local raw_evts=$($cmd list --raw-dump) + local raw_evts=$($cmd list --raw-dump hw sw cache tracepoint pmu sdt) local arr s tmp result cpu_evts # aarch64 doesn't have /sys/bus/event_source/devices/cpu/events From 493902fcbdd608a6ece49c1670ffc76fed1e617f Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Tue, 5 Sep 2023 11:15:53 -0700 Subject: [PATCH 027/475] perf completion: Support completion of libpfm4 events Use `perf list --raw-dump pfm` to support completion of libpfm4 events. Committer testing: # grep -m1 'model name' /proc/cpuinfo model name : AMD Ryzen 9 5950X 16-Core Processor Before: Files in the current directory are expanded when After: Only the PFM events are: # . tools/perf/perf-completion.sh # perf stat --pfm-events Becomes: # perf stat --pfm-events perf_raw::r0000 As apparently there are no other PFM events for this Ryzen 9 5950X machine. Signed-off-by: Ian Rogers Acked-by: Namhyung Kim Tested-by: Arnaldo Carvalho de Melo Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ingo Molnar Cc: Jiri Olsa Cc: Mark Rutland Cc: Peter Zijlstra Cc: Stephane Eranian Link: https://lore.kernel.org/r/20230905181554.3202873-2-irogers@google.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/perf-completion.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/perf/perf-completion.sh b/tools/perf/perf-completion.sh index 40cfc30ad7ad..319ad724b97a 100644 --- a/tools/perf/perf-completion.sh +++ b/tools/perf/perf-completion.sh @@ -198,6 +198,10 @@ __perf_main () else __perfcomp_colon "$evts" "$cur1" fi + elif [[ $prev == @("--pfm-events") && + $prev_skip_opts == @(record|stat|top) ]]; then + local evts=$($cmd list --raw-dump pfm) + __perfcomp "$evts" "$cur" else # List subcommands for perf commands if [[ $prev_skip_opts == @(kvm|kmem|mem|lock|sched| From fa88095856dfbad439129324a0fb6b4716e17ab1 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Tue, 5 Sep 2023 11:15:54 -0700 Subject: [PATCH 028/475] perf shell completion: Support completion of metrics/metricgroups Allow metrics to expand for -M or --metrics options. Committer testing: # grep -m1 'model name' /proc/cpuinfo model name : AMD Ryzen 9 5950X 16-Core Processor # Before: Just expansion of files/directories in the pwd are expanded: # . tools/perf/perf-completion.sh # perf stat -M b block/ build/ # perf stat -M b After: # . tools/perf/perf-completion.sh # perf stat -M all_l2_cache_accesses all_remote_links_outbound data_fabric l1_itlb_misses l2_cache_misses_from_l2_hwpf macro_ops_dispatched tlb all_l2_cache_hits branch_misprediction_ratio decoder l2_cache l3_cache nps1_die_to_dram all_l2_cache_misses branch_prediction ic_fetch_miss_ratio l2_cache_accesses_from_l2_hwpf l3_read_miss_latency op_cache_fetch_miss_ratio # perf stat -M branch_ branch_misprediction_ratio branch_prediction # perf stat -M branch_prediction -a sleep 1 Performance counter stats for 'system wide': 115,079,765 ex_ret_brn # 4.0 % branch_misprediction_ratio 4,561,456 ex_ret_brn_misp 1.015925106 seconds time elapsed # Suggested-by: Arnaldo Carvalho de Melo Signed-off-by: Ian Rogers Tested-by: Arnaldo Carvalho de Melo Acked-by: Namhyung Kim Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ingo Molnar Cc: Jiri Olsa Cc: Mark Rutland Cc: Peter Zijlstra Cc: Stephane Eranian Link: https://lore.kernel.org/r/20230905181554.3202873-3-irogers@google.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/perf-completion.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/perf/perf-completion.sh b/tools/perf/perf-completion.sh index 319ad724b97a..f224d79b89e6 100644 --- a/tools/perf/perf-completion.sh +++ b/tools/perf/perf-completion.sh @@ -202,6 +202,10 @@ __perf_main () $prev_skip_opts == @(record|stat|top) ]]; then local evts=$($cmd list --raw-dump pfm) __perfcomp "$evts" "$cur" + elif [[ $prev == @("-M"|"--metrics") && + $prev_skip_opts == @(stat) ]]; then + local metrics=$($cmd list --raw-dump metric metricgroup) + __perfcomp "$metrics" "$cur" else # List subcommands for perf commands if [[ $prev_skip_opts == @(kvm|kmem|mem|lock|sched| From 6fcfe54d2c91925ec3054cf25e68064913ca7948 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Fri, 1 Sep 2023 16:39:45 -0700 Subject: [PATCH 029/475] perf parse-events: Remove unnecessary __maybe_unused The parameter head_terms is always used in get_config_terms. Reviewed-by: James Clark Signed-off-by: Ian Rogers Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ingo Molnar Cc: Jiri Olsa Cc: Kan Liang Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Rob Herring Link: https://lore.kernel.org/r/20230901233949.2930562-2-irogers@google.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/util/parse-events.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tools/perf/util/parse-events.c b/tools/perf/util/parse-events.c index 65608a3cba81..e9e3623f3fed 100644 --- a/tools/perf/util/parse-events.c +++ b/tools/perf/util/parse-events.c @@ -34,8 +34,7 @@ #ifdef PARSER_DEBUG extern int parse_events_debug; #endif -static int get_config_terms(struct list_head *head_config, - struct list_head *head_terms __maybe_unused); +static int get_config_terms(struct list_head *head_config, struct list_head *head_terms); struct event_symbol event_symbols_hw[PERF_COUNT_HW_MAX] = { [PERF_COUNT_HW_CPU_CYCLES] = { @@ -1079,8 +1078,7 @@ static int config_attr(struct perf_event_attr *attr, return 0; } -static int get_config_terms(struct list_head *head_config, - struct list_head *head_terms __maybe_unused) +static int get_config_terms(struct list_head *head_config, struct list_head *head_terms) { #define ADD_CONFIG_TERM(__type, __weak) \ struct evsel_config_term *__t; \ From 8f91662ef8be473fb025e170601b0dd75838f7d4 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Fri, 1 Sep 2023 16:39:46 -0700 Subject: [PATCH 030/475] perf parse-events: Tidy up str parameter Add a const and rename str to event_name. Reviewed-by: James Clark Signed-off-by: Ian Rogers Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ingo Molnar Cc: Jiri Olsa Cc: Kan Liang Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Rob Herring Link: https://lore.kernel.org/r/20230901233949.2930562-3-irogers@google.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/util/parse-events.c | 13 +++++++------ tools/perf/util/parse-events.h | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/tools/perf/util/parse-events.c b/tools/perf/util/parse-events.c index e9e3623f3fed..283c559a35b4 100644 --- a/tools/perf/util/parse-events.c +++ b/tools/perf/util/parse-events.c @@ -1482,7 +1482,7 @@ int parse_events_add_pmu(struct parse_events_state *parse_state, } int parse_events_multi_pmu_add(struct parse_events_state *parse_state, - char *str, struct list_head *head, + const char *event_name, struct list_head *head, struct list_head **listp, void *loc_) { struct parse_events_term *term; @@ -1502,7 +1502,8 @@ int parse_events_multi_pmu_add(struct parse_events_state *parse_state, INIT_LIST_HEAD(head); } - config = strdup(str); + + config = strdup(event_name); if (!config) goto out_err; @@ -1528,7 +1529,7 @@ int parse_events_multi_pmu_add(struct parse_events_state *parse_state, if (parse_events__filter_pmu(parse_state, pmu)) continue; - if (!perf_pmu__have_event(pmu, str)) + if (!perf_pmu__have_event(pmu, event_name)) continue; auto_merge_stats = perf_pmu__auto_merge_stats(pmu); @@ -1539,7 +1540,7 @@ int parse_events_multi_pmu_add(struct parse_events_state *parse_state, strbuf_init(&sb, /*hint=*/ 0); parse_events_term__to_strbuf(orig_head, &sb); - pr_debug("%s -> %s/%s/\n", str, pmu->name, sb.buf); + pr_debug("%s -> %s/%s/\n", event_name, pmu->name, sb.buf); strbuf_release(&sb); ok++; } @@ -1547,13 +1548,13 @@ int parse_events_multi_pmu_add(struct parse_events_state *parse_state, } if (parse_state->fake_pmu) { - if (!parse_events_add_pmu(parse_state, list, str, head, + if (!parse_events_add_pmu(parse_state, list, event_name, head, /*auto_merge_stats=*/true, loc)) { struct strbuf sb; strbuf_init(&sb, /*hint=*/ 0); parse_events_term__to_strbuf(head, &sb); - pr_debug("%s -> %s/%s/\n", str, "fake_pmu", sb.buf); + pr_debug("%s -> %s/%s/\n", event_name, "fake_pmu", sb.buf); strbuf_release(&sb); ok++; } diff --git a/tools/perf/util/parse-events.h b/tools/perf/util/parse-events.h index 594e5d2dc67f..36a67ef7b35a 100644 --- a/tools/perf/util/parse-events.h +++ b/tools/perf/util/parse-events.h @@ -217,7 +217,7 @@ struct evsel *parse_events__add_event(int idx, struct perf_event_attr *attr, struct perf_pmu *pmu); int parse_events_multi_pmu_add(struct parse_events_state *parse_state, - char *str, + const char *event_name, struct list_head *head_config, struct list_head **listp, void *loc); From 4163644818e95ea6b0afb3982b34c4d59ed50bb2 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Fri, 1 Sep 2023 16:39:47 -0700 Subject: [PATCH 031/475] perf parse-events: Avoid enum casts Add term_type to union of values returned by the lexer to avoid casts to and from an integer. Reviewed-by: James Clark Signed-off-by: Ian Rogers Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ingo Molnar Cc: Jiri Olsa Cc: Kan Liang Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Rob Herring Link: https://lore.kernel.org/r/20230901233949.2930562-4-irogers@google.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/util/parse-events.l | 2 +- tools/perf/util/parse-events.y | 25 +++++++++++-------------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/tools/perf/util/parse-events.l b/tools/perf/util/parse-events.l index 4ef4b6f171a0..7bdf0565a92c 100644 --- a/tools/perf/util/parse-events.l +++ b/tools/perf/util/parse-events.l @@ -120,7 +120,7 @@ static int term(yyscan_t scanner, enum parse_events__term_type type) { YYSTYPE *yylval = parse_events_get_lval(scanner); - yylval->num = type; + yylval->term_type = type; return PE_TERM; } diff --git a/tools/perf/util/parse-events.y b/tools/perf/util/parse-events.y index 21bfe7e0d944..286ead08065e 100644 --- a/tools/perf/util/parse-events.y +++ b/tools/perf/util/parse-events.y @@ -70,7 +70,7 @@ static void free_list_evsel(struct list_head* list_evsel) %type PE_VALUE_SYM_HW %type PE_VALUE_SYM_SW %type PE_VALUE_SYM_TOOL -%type PE_TERM +%type PE_TERM %type value_sym %type PE_RAW %type PE_NAME @@ -111,6 +111,7 @@ static void free_list_evsel(struct list_head* list_evsel) { char *str; u64 num; + enum parse_events__term_type term_type; struct list_head *list_evsel; struct list_head *list_terms; struct parse_events_term *term; @@ -778,8 +779,7 @@ PE_TERM_HW PE_TERM '=' name_or_legacy { struct parse_events_term *term; - int err = parse_events_term__str(&term, (enum parse_events__term_type)$1, - /*config=*/NULL, $3, &@1, &@3); + int err = parse_events_term__str(&term, $1, /*config=*/NULL, $3, &@1, &@3); if (err) { free($3); @@ -791,8 +791,7 @@ PE_TERM '=' name_or_legacy PE_TERM '=' PE_TERM_HW { struct parse_events_term *term; - int err = parse_events_term__str(&term, (enum parse_events__term_type)$1, - /*config=*/NULL, $3.str, &@1, &@3); + int err = parse_events_term__str(&term, $1, /*config=*/NULL, $3.str, &@1, &@3); if (err) { free($3.str); @@ -804,10 +803,7 @@ PE_TERM '=' PE_TERM_HW PE_TERM '=' PE_TERM { struct parse_events_term *term; - int err = parse_events_term__term(&term, - (enum parse_events__term_type)$1, - (enum parse_events__term_type)$3, - &@1, &@3); + int err = parse_events_term__term(&term, $1, $3, &@1, &@3); if (err) PE_ABORT(err); @@ -818,8 +814,9 @@ PE_TERM '=' PE_TERM PE_TERM '=' PE_VALUE { struct parse_events_term *term; - int err = parse_events_term__num(&term, (enum parse_events__term_type)$1, - /*config=*/NULL, $3, /*novalue=*/false, &@1, &@3); + int err = parse_events_term__num(&term, $1, + /*config=*/NULL, $3, /*novalue=*/false, + &@1, &@3); if (err) PE_ABORT(err); @@ -830,9 +827,9 @@ PE_TERM '=' PE_VALUE PE_TERM { struct parse_events_term *term; - int err = parse_events_term__num(&term, (enum parse_events__term_type)$1, - /*config=*/NULL, /*num=*/1, /*novalue=*/true, - &@1, /*loc_val=*/NULL); + int err = parse_events_term__num(&term, $1, + /*config=*/NULL, /*num=*/1, /*novalue=*/true, + &@1, /*loc_val=*/NULL); if (err) PE_ABORT(err); From 727adeed06e82915841e121762eb329881ae0107 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Fri, 1 Sep 2023 16:39:48 -0700 Subject: [PATCH 032/475] perf parse-events: Copy fewer term lists When trying to add events to multiple PMUs the term list is copied first as adding the event will rewrite the event's name term into the sysfs and/or json encoding terms (see perf_pmu__check_alias). Change the parse events add API so the passed in term list is const, then copy the list when modification is necessary. Reviewed-by: James Clark Signed-off-by: Ian Rogers Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ingo Molnar Cc: Jiri Olsa Cc: Kan Liang Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Rob Herring Link: https://lore.kernel.org/r/20230901233949.2930562-5-irogers@google.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/util/parse-events.c | 108 ++++++++++++++++++--------------- tools/perf/util/parse-events.h | 7 +-- tools/perf/util/parse-events.y | 17 +----- 3 files changed, 65 insertions(+), 67 deletions(-) diff --git a/tools/perf/util/parse-events.c b/tools/perf/util/parse-events.c index 283c559a35b4..06a844bcce4a 100644 --- a/tools/perf/util/parse-events.c +++ b/tools/perf/util/parse-events.c @@ -35,6 +35,7 @@ extern int parse_events_debug; #endif static int get_config_terms(struct list_head *head_config, struct list_head *head_terms); +static int parse_events_terms__copy(const struct list_head *src, struct list_head *dest); struct event_symbol event_symbols_hw[PERF_COUNT_HW_MAX] = { [PERF_COUNT_HW_CPU_CYCLES] = { @@ -1367,7 +1368,7 @@ static bool config_term_percore(struct list_head *config_terms) int parse_events_add_pmu(struct parse_events_state *parse_state, struct list_head *list, const char *name, - struct list_head *head_config, + const struct list_head *const_head_terms, bool auto_merge_stats, void *loc_) { struct perf_event_attr attr; @@ -1377,6 +1378,7 @@ int parse_events_add_pmu(struct parse_events_state *parse_state, struct parse_events_error *err = parse_state->error; YYLTYPE *loc = loc_; LIST_HEAD(config_terms); + LIST_HEAD(head_terms); pmu = parse_state->fake_pmu ?: perf_pmus__find(name); @@ -1390,32 +1392,37 @@ int parse_events_add_pmu(struct parse_events_state *parse_state, return -EINVAL; } + if (const_head_terms) { + int ret = parse_events_terms__copy(const_head_terms, &head_terms); + + if (ret) + return ret; + } + if (verbose > 1) { struct strbuf sb; strbuf_init(&sb, /*hint=*/ 0); - if (pmu->selectable && !head_config) { + if (pmu->selectable && list_empty(&head_terms)) { strbuf_addf(&sb, "%s//", name); } else { strbuf_addf(&sb, "%s/", name); - parse_events_term__to_strbuf(head_config, &sb); + parse_events_term__to_strbuf(&head_terms, &sb); strbuf_addch(&sb, '/'); } fprintf(stderr, "Attempt to add: %s\n", sb.buf); strbuf_release(&sb); } - if (head_config) - fix_raw(head_config, pmu); + fix_raw(&head_terms, pmu); if (pmu->default_config) { - memcpy(&attr, pmu->default_config, - sizeof(struct perf_event_attr)); + memcpy(&attr, pmu->default_config, sizeof(struct perf_event_attr)); } else { memset(&attr, 0, sizeof(attr)); } attr.type = pmu->type; - if (!head_config) { + if (list_empty(&head_terms)) { evsel = __add_event(list, &parse_state->idx, &attr, /*init_attr=*/true, /*name=*/NULL, /*metric_id=*/NULL, pmu, @@ -1424,14 +1431,16 @@ int parse_events_add_pmu(struct parse_events_state *parse_state, return evsel ? 0 : -ENOMEM; } - if (!parse_state->fake_pmu && perf_pmu__check_alias(pmu, head_config, &info, err)) + if (!parse_state->fake_pmu && perf_pmu__check_alias(pmu, &head_terms, &info, err)) { + parse_events_terms__purge(&head_terms); return -EINVAL; + } if (verbose > 1) { struct strbuf sb; strbuf_init(&sb, /*hint=*/ 0); - parse_events_term__to_strbuf(head_config, &sb); + parse_events_term__to_strbuf(&head_terms, &sb); fprintf(stderr, "..after resolving event: %s/%s/\n", name, sb.buf); strbuf_release(&sb); } @@ -1440,39 +1449,52 @@ int parse_events_add_pmu(struct parse_events_state *parse_state, * Configure hardcoded terms first, no need to check * return value when called with fail == 0 ;) */ - if (config_attr(&attr, head_config, parse_state->error, config_term_pmu)) + if (config_attr(&attr, &head_terms, parse_state->error, config_term_pmu)) { + parse_events_terms__purge(&head_terms); return -EINVAL; + } - if (get_config_terms(head_config, &config_terms)) + if (get_config_terms(&head_terms, &config_terms)) { + parse_events_terms__purge(&head_terms); return -ENOMEM; + } /* * When using default config, record which bits of attr->config were * changed by the user. */ - if (pmu->default_config && get_config_chgs(pmu, head_config, &config_terms)) + if (pmu->default_config && get_config_chgs(pmu, &head_terms, &config_terms)) { + parse_events_terms__purge(&head_terms); return -ENOMEM; + } - if (!parse_state->fake_pmu && perf_pmu__config(pmu, &attr, head_config, parse_state->error)) { + if (!parse_state->fake_pmu && + perf_pmu__config(pmu, &attr, &head_terms, parse_state->error)) { free_config_terms(&config_terms); + parse_events_terms__purge(&head_terms); return -EINVAL; } evsel = __add_event(list, &parse_state->idx, &attr, /*init_attr=*/true, - get_config_name(head_config), - get_config_metric_id(head_config), pmu, + get_config_name(&head_terms), + get_config_metric_id(&head_terms), pmu, &config_terms, auto_merge_stats, /*cpu_list=*/NULL); - if (!evsel) + if (!evsel) { + parse_events_terms__purge(&head_terms); return -ENOMEM; + } if (evsel->name) evsel->use_config_name = true; evsel->percore = config_term_percore(&evsel->config_terms); - if (parse_state->fake_pmu) + if (parse_state->fake_pmu) { + parse_events_terms__purge(&head_terms); return 0; + } + parse_events_terms__purge(&head_terms); free((char *)evsel->unit); evsel->unit = strdup(info.unit); evsel->scale = info.scale; @@ -1482,25 +1504,25 @@ int parse_events_add_pmu(struct parse_events_state *parse_state, } int parse_events_multi_pmu_add(struct parse_events_state *parse_state, - const char *event_name, struct list_head *head, + const char *event_name, + const struct list_head *const_head_terms, struct list_head **listp, void *loc_) { struct parse_events_term *term; struct list_head *list = NULL; - struct list_head *orig_head = NULL; struct perf_pmu *pmu = NULL; YYLTYPE *loc = loc_; int ok = 0; const char *config; + LIST_HEAD(head_terms); *listp = NULL; - if (!head) { - head = malloc(sizeof(struct list_head)); - if (!head) - goto out_err; + if (const_head_terms) { + int ret = parse_events_terms__copy(const_head_terms, &head_terms); - INIT_LIST_HEAD(head); + if (ret) + return ret; } config = strdup(event_name); @@ -1514,7 +1536,7 @@ int parse_events_multi_pmu_add(struct parse_events_state *parse_state, zfree(&config); goto out_err; } - list_add_tail(&term->list, head); + list_add_tail(&term->list, &head_terms); /* Add it for all PMUs that support the alias */ list = malloc(sizeof(struct list_head)); @@ -1533,27 +1555,25 @@ int parse_events_multi_pmu_add(struct parse_events_state *parse_state, continue; auto_merge_stats = perf_pmu__auto_merge_stats(pmu); - parse_events_copy_term_list(head, &orig_head); if (!parse_events_add_pmu(parse_state, list, pmu->name, - orig_head, auto_merge_stats, loc)) { + &head_terms, auto_merge_stats, loc)) { struct strbuf sb; strbuf_init(&sb, /*hint=*/ 0); - parse_events_term__to_strbuf(orig_head, &sb); + parse_events_term__to_strbuf(&head_terms, &sb); pr_debug("%s -> %s/%s/\n", event_name, pmu->name, sb.buf); strbuf_release(&sb); ok++; } - parse_events_terms__delete(orig_head); } if (parse_state->fake_pmu) { - if (!parse_events_add_pmu(parse_state, list, event_name, head, + if (!parse_events_add_pmu(parse_state, list, event_name, &head_terms, /*auto_merge_stats=*/true, loc)) { struct strbuf sb; strbuf_init(&sb, /*hint=*/ 0); - parse_events_term__to_strbuf(head, &sb); + parse_events_term__to_strbuf(&head_terms, &sb); pr_debug("%s -> %s/%s/\n", event_name, "fake_pmu", sb.buf); strbuf_release(&sb); ok++; @@ -1561,12 +1581,12 @@ int parse_events_multi_pmu_add(struct parse_events_state *parse_state, } out_err: + parse_events_terms__purge(&head_terms); if (ok) *listp = list; else free(list); - parse_events_terms__delete(head); return ok ? 0 : -1; } @@ -2543,27 +2563,19 @@ void parse_events_term__delete(struct parse_events_term *term) free(term); } -int parse_events_copy_term_list(struct list_head *old, - struct list_head **new) +static int parse_events_terms__copy(const struct list_head *src, struct list_head *dest) { - struct parse_events_term *term, *n; - int ret; + struct parse_events_term *term; - if (!old) { - *new = NULL; - return 0; - } + list_for_each_entry (term, src, list) { + struct parse_events_term *n; + int ret; - *new = malloc(sizeof(struct list_head)); - if (!*new) - return -ENOMEM; - INIT_LIST_HEAD(*new); - - list_for_each_entry (term, old, list) { ret = parse_events_term__clone(&n, term); if (ret) return ret; - list_add_tail(&n->list, *new); + + list_add_tail(&n->list, dest); } return 0; } diff --git a/tools/perf/util/parse-events.h b/tools/perf/util/parse-events.h index 36a67ef7b35a..e6612856e881 100644 --- a/tools/perf/util/parse-events.h +++ b/tools/perf/util/parse-events.h @@ -209,7 +209,7 @@ int parse_events_add_breakpoint(struct parse_events_state *parse_state, struct list_head *head_config); int parse_events_add_pmu(struct parse_events_state *parse_state, struct list_head *list, const char *name, - struct list_head *head_config, + const struct list_head *const_head_terms, bool auto_merge_stats, void *loc); struct evsel *parse_events__add_event(int idx, struct perf_event_attr *attr, @@ -218,12 +218,9 @@ struct evsel *parse_events__add_event(int idx, struct perf_event_attr *attr, int parse_events_multi_pmu_add(struct parse_events_state *parse_state, const char *event_name, - struct list_head *head_config, + const struct list_head *head_terms, struct list_head **listp, void *loc); -int parse_events_copy_term_list(struct list_head *old, - struct list_head **new); - void parse_events__set_leader(char *name, struct list_head *list); void parse_events_update_lists(struct list_head *list_event, struct list_head *list_all); diff --git a/tools/perf/util/parse-events.y b/tools/perf/util/parse-events.y index 286ead08065e..4d7947fb4a98 100644 --- a/tools/perf/util/parse-events.y +++ b/tools/perf/util/parse-events.y @@ -274,23 +274,18 @@ event_pmu: PE_NAME opt_pmu_config { struct parse_events_state *parse_state = _parse_state; - struct list_head *list = NULL, *orig_terms = NULL, *terms= NULL; + /* List of created evsels. */ + struct list_head *list = NULL; char *pattern = NULL; #define CLEANUP \ do { \ parse_events_terms__delete($2); \ - parse_events_terms__delete(orig_terms); \ free(list); \ free($1); \ free(pattern); \ } while(0) - if (parse_events_copy_term_list($2, &orig_terms)) { - CLEANUP; - YYNOMEM; - } - list = alloc_list(); if (!list) { CLEANUP; @@ -320,16 +315,11 @@ PE_NAME opt_pmu_config !perf_pmu__match(pattern, pmu->alias_name, $1)) { bool auto_merge_stats = perf_pmu__auto_merge_stats(pmu); - if (parse_events_copy_term_list(orig_terms, &terms)) { - CLEANUP; - YYNOMEM; - } - if (!parse_events_add_pmu(parse_state, list, pmu->name, terms, + if (!parse_events_add_pmu(parse_state, list, pmu->name, $2, auto_merge_stats, &@1)) { ok++; parse_state->wild_card_pmus = true; } - parse_events_terms__delete(terms); } } @@ -337,7 +327,6 @@ PE_NAME opt_pmu_config /* Failure to add, assume $1 is an event name. */ zfree(&list); ok = !parse_events_multi_pmu_add(parse_state, $1, $2, &list, &@1); - $2 = NULL; } if (!ok) { struct parse_events_error *error = parse_state->error; From 0d3f0e6f94ef58d5532e23b6d153b0890cf0014c Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Fri, 1 Sep 2023 16:39:49 -0700 Subject: [PATCH 033/475] perf parse-events: Introduce 'struct parse_events_terms' parse_events_terms() existed in function names but was passed a 'struct list_head'. As many parse_events functions take an evsel_config list as well as a parse_event_term list, and the naming head_terms and head_config is inconsistent, there's a potential to switch the lists and get errors. Introduce a 'struct parse_events_terms', that just wraps a list_head, to avoid this. Add the regular init/exit functions and transition the code to use them. Reviewed-by: James Clark Signed-off-by: Ian Rogers Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ingo Molnar Cc: Jiri Olsa Cc: Kan Liang Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Rob Herring Link: https://lore.kernel.org/r/20230901233949.2930562-6-irogers@google.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/arch/x86/util/intel-pt.c | 15 +-- tools/perf/tests/parse-events.c | 12 +-- tools/perf/tests/pmu.c | 23 ++-- tools/perf/util/parse-events.c | 158 +++++++++++++++------------- tools/perf/util/parse-events.h | 29 +++-- tools/perf/util/parse-events.y | 12 +-- tools/perf/util/pmu.c | 49 ++++----- tools/perf/util/pmu.h | 6 +- 8 files changed, 160 insertions(+), 144 deletions(-) diff --git a/tools/perf/arch/x86/util/intel-pt.c b/tools/perf/arch/x86/util/intel-pt.c index 31807791589e..b923bca939d9 100644 --- a/tools/perf/arch/x86/util/intel-pt.c +++ b/tools/perf/arch/x86/util/intel-pt.c @@ -64,28 +64,23 @@ static int intel_pt_parse_terms_with_default(struct perf_pmu *pmu, const char *str, u64 *config) { - struct list_head *terms; + struct parse_events_terms terms; struct perf_event_attr attr = { .size = 0, }; int err; - terms = malloc(sizeof(struct list_head)); - if (!terms) - return -ENOMEM; - - INIT_LIST_HEAD(terms); - - err = parse_events_terms(terms, str, /*input=*/ NULL); + parse_events_terms__init(&terms); + err = parse_events_terms(&terms, str, /*input=*/ NULL); if (err) goto out_free; attr.config = *config; - err = perf_pmu__config_terms(pmu, &attr, terms, /*zero=*/true, /*err=*/NULL); + err = perf_pmu__config_terms(pmu, &attr, &terms, /*zero=*/true, /*err=*/NULL); if (err) goto out_free; *config = attr.config; out_free: - parse_events_terms__delete(terms); + parse_events_terms__exit(&terms); return err; } diff --git a/tools/perf/tests/parse-events.c b/tools/perf/tests/parse-events.c index d47f1f871164..93796e885cef 100644 --- a/tools/perf/tests/parse-events.c +++ b/tools/perf/tests/parse-events.c @@ -771,12 +771,12 @@ static int test__checkevent_pmu_events_mix(struct evlist *evlist) return TEST_OK; } -static int test__checkterms_simple(struct list_head *terms) +static int test__checkterms_simple(struct parse_events_terms *terms) { struct parse_events_term *term; /* config=10 */ - term = list_entry(terms->next, struct parse_events_term, list); + term = list_entry(terms->terms.next, struct parse_events_term, list); TEST_ASSERT_VAL("wrong type term", term->type_term == PARSE_EVENTS__TERM_TYPE_CONFIG); TEST_ASSERT_VAL("wrong type val", @@ -2363,7 +2363,7 @@ static const struct evlist_test test__events_pmu[] = { struct terms_test { const char *str; - int (*check)(struct list_head *terms); + int (*check)(struct parse_events_terms *terms); }; static const struct terms_test test__terms[] = { @@ -2467,11 +2467,11 @@ static int test__events2(struct test_suite *test __maybe_unused, int subtest __m static int test_term(const struct terms_test *t) { - struct list_head terms; + struct parse_events_terms terms; int ret; - INIT_LIST_HEAD(&terms); + parse_events_terms__init(&terms); ret = parse_events_terms(&terms, t->str, /*input=*/ NULL); if (ret) { pr_debug("failed to parse terms '%s', err %d\n", @@ -2480,7 +2480,7 @@ static int test_term(const struct terms_test *t) } ret = t->check(&terms); - parse_events_terms__purge(&terms); + parse_events_terms__exit(&terms); return ret; } diff --git a/tools/perf/tests/pmu.c b/tools/perf/tests/pmu.c index eb60e5f66859..8f18127d876a 100644 --- a/tools/perf/tests/pmu.c +++ b/tools/perf/tests/pmu.c @@ -128,30 +128,35 @@ static int test_format_dir_put(char *dir) return system(buf); } -static struct list_head *test_terms_list(void) +static void add_test_terms(struct parse_events_terms *terms) { - static LIST_HEAD(terms); unsigned int i; - for (i = 0; i < ARRAY_SIZE(test_terms); i++) - list_add_tail(&test_terms[i].list, &terms); + for (i = 0; i < ARRAY_SIZE(test_terms); i++) { + struct parse_events_term *clone; - return &terms; + parse_events_term__clone(&clone, &test_terms[i]); + list_add_tail(&clone->list, &terms->terms); + } } static int test__pmu(struct test_suite *test __maybe_unused, int subtest __maybe_unused) { char dir[PATH_MAX]; char *format; - struct list_head *terms = test_terms_list(); + struct parse_events_terms terms; struct perf_event_attr attr; struct perf_pmu *pmu; int fd; int ret; + parse_events_terms__init(&terms); + add_test_terms(&terms); pmu = zalloc(sizeof(*pmu)); - if (!pmu) + if (!pmu) { + parse_events_terms__exit(&terms); return -ENOMEM; + } INIT_LIST_HEAD(&pmu->format); INIT_LIST_HEAD(&pmu->aliases); @@ -159,6 +164,7 @@ static int test__pmu(struct test_suite *test __maybe_unused, int subtest __maybe format = test_format_dir_get(dir, sizeof(dir)); if (!format) { free(pmu); + parse_events_terms__exit(&terms); return -EINVAL; } @@ -175,7 +181,7 @@ static int test__pmu(struct test_suite *test __maybe_unused, int subtest __maybe if (ret) goto out; - ret = perf_pmu__config_terms(pmu, &attr, terms, /*zero=*/false, /*err=*/NULL); + ret = perf_pmu__config_terms(pmu, &attr, &terms, /*zero=*/false, /*err=*/NULL); if (ret) goto out; @@ -191,6 +197,7 @@ static int test__pmu(struct test_suite *test __maybe_unused, int subtest __maybe out: test_format_dir_put(format); perf_pmu__delete(pmu); + parse_events_terms__exit(&terms); return ret; } diff --git a/tools/perf/util/parse-events.c b/tools/perf/util/parse-events.c index 06a844bcce4a..c56e07bd7dd6 100644 --- a/tools/perf/util/parse-events.c +++ b/tools/perf/util/parse-events.c @@ -34,8 +34,9 @@ #ifdef PARSER_DEBUG extern int parse_events_debug; #endif -static int get_config_terms(struct list_head *head_config, struct list_head *head_terms); -static int parse_events_terms__copy(const struct list_head *src, struct list_head *dest); +static int get_config_terms(struct parse_events_terms *head_config, struct list_head *head_terms); +static int parse_events_terms__copy(const struct parse_events_terms *src, + struct parse_events_terms *dest); struct event_symbol event_symbols_hw[PERF_COUNT_HW_MAX] = { [PERF_COUNT_HW_CPU_CYCLES] = { @@ -153,26 +154,27 @@ const char *event_type(int type) return "unknown"; } -static char *get_config_str(struct list_head *head_terms, enum parse_events__term_type type_term) +static char *get_config_str(struct parse_events_terms *head_terms, + enum parse_events__term_type type_term) { struct parse_events_term *term; if (!head_terms) return NULL; - list_for_each_entry(term, head_terms, list) + list_for_each_entry(term, &head_terms->terms, list) if (term->type_term == type_term) return term->val.str; return NULL; } -static char *get_config_metric_id(struct list_head *head_terms) +static char *get_config_metric_id(struct parse_events_terms *head_terms) { return get_config_str(head_terms, PARSE_EVENTS__TERM_TYPE_METRIC_ID); } -static char *get_config_name(struct list_head *head_terms) +static char *get_config_name(struct parse_events_terms *head_terms) { return get_config_str(head_terms, PARSE_EVENTS__TERM_TYPE_NAME); } @@ -188,11 +190,11 @@ static char *get_config_name(struct list_head *head_terms) * @config_terms: the list of terms that may contain a raw term. * @pmu: the PMU to scan for events from. */ -static void fix_raw(struct list_head *config_terms, struct perf_pmu *pmu) +static void fix_raw(struct parse_events_terms *config_terms, struct perf_pmu *pmu) { struct parse_events_term *term; - list_for_each_entry(term, config_terms, list) { + list_for_each_entry(term, &config_terms->terms, list) { u64 num; if (term->type_term != PARSE_EVENTS__TERM_TYPE_RAW) @@ -356,7 +358,7 @@ static int config_term_common(struct perf_event_attr *attr, struct parse_events_term *term, struct parse_events_error *err); static int config_attr(struct perf_event_attr *attr, - struct list_head *head, + struct parse_events_terms *head, struct parse_events_error *err, config_term_func_t config_term); @@ -442,7 +444,7 @@ bool parse_events__filter_pmu(const struct parse_events_state *parse_state, int parse_events_add_cache(struct list_head *list, int *idx, const char *name, struct parse_events_state *parse_state, - struct list_head *head_config) + struct parse_events_terms *head_config) { struct perf_pmu *pmu = NULL; bool found_supported = false; @@ -520,7 +522,7 @@ static void tracepoint_error(struct parse_events_error *e, int err, static int add_tracepoint(struct list_head *list, int *idx, const char *sys_name, const char *evt_name, struct parse_events_error *err, - struct list_head *head_config, void *loc_) + struct parse_events_terms *head_config, void *loc_) { YYLTYPE *loc = loc_; struct evsel *evsel = evsel__newtp_idx(sys_name, evt_name, (*idx)++); @@ -545,7 +547,7 @@ static int add_tracepoint(struct list_head *list, int *idx, static int add_tracepoint_multi_event(struct list_head *list, int *idx, const char *sys_name, const char *evt_name, struct parse_events_error *err, - struct list_head *head_config, YYLTYPE *loc) + struct parse_events_terms *head_config, YYLTYPE *loc) { char *evt_path; struct dirent *evt_ent; @@ -593,7 +595,7 @@ static int add_tracepoint_multi_event(struct list_head *list, int *idx, static int add_tracepoint_event(struct list_head *list, int *idx, const char *sys_name, const char *evt_name, struct parse_events_error *err, - struct list_head *head_config, YYLTYPE *loc) + struct parse_events_terms *head_config, YYLTYPE *loc) { return strpbrk(evt_name, "*?") ? add_tracepoint_multi_event(list, idx, sys_name, evt_name, @@ -605,7 +607,7 @@ static int add_tracepoint_event(struct list_head *list, int *idx, static int add_tracepoint_multi_sys(struct list_head *list, int *idx, const char *sys_name, const char *evt_name, struct parse_events_error *err, - struct list_head *head_config, YYLTYPE *loc) + struct parse_events_terms *head_config, YYLTYPE *loc) { struct dirent *events_ent; DIR *events_dir; @@ -680,7 +682,7 @@ do { \ int parse_events_add_breakpoint(struct parse_events_state *parse_state, struct list_head *list, u64 addr, char *type, u64 len, - struct list_head *head_config __maybe_unused) + struct parse_events_terms *head_config) { struct perf_event_attr attr; LIST_HEAD(config_terms); @@ -1066,20 +1068,20 @@ static int config_term_tracepoint(struct perf_event_attr *attr, #endif static int config_attr(struct perf_event_attr *attr, - struct list_head *head, + struct parse_events_terms *head, struct parse_events_error *err, config_term_func_t config_term) { struct parse_events_term *term; - list_for_each_entry(term, head, list) + list_for_each_entry(term, &head->terms, list) if (config_term(attr, term, err)) return -EINVAL; return 0; } -static int get_config_terms(struct list_head *head_config, struct list_head *head_terms) +static int get_config_terms(struct parse_events_terms *head_config, struct list_head *head_terms) { #define ADD_CONFIG_TERM(__type, __weak) \ struct evsel_config_term *__t; \ @@ -1112,7 +1114,7 @@ do { \ struct parse_events_term *term; - list_for_each_entry(term, head_config, list) { + list_for_each_entry(term, &head_config->terms, list) { switch (term->type_term) { case PARSE_EVENTS__TERM_TYPE_SAMPLE_PERIOD: ADD_CONFIG_TERM_VAL(PERIOD, period, term->val.num, term->weak); @@ -1193,14 +1195,14 @@ do { \ * Add EVSEL__CONFIG_TERM_CFG_CHG where cfg_chg will have a bit set for * each bit of attr->config that the user has changed. */ -static int get_config_chgs(struct perf_pmu *pmu, struct list_head *head_config, +static int get_config_chgs(struct perf_pmu *pmu, struct parse_events_terms *head_config, struct list_head *head_terms) { struct parse_events_term *term; u64 bits = 0; int type; - list_for_each_entry(term, head_config, list) { + list_for_each_entry(term, &head_config->terms, list) { switch (term->type_term) { case PARSE_EVENTS__TERM_TYPE_USER: type = perf_pmu__format_type(pmu, term->config); @@ -1250,7 +1252,7 @@ static int get_config_chgs(struct perf_pmu *pmu, struct list_head *head_config, int parse_events_add_tracepoint(struct list_head *list, int *idx, const char *sys, const char *event, struct parse_events_error *err, - struct list_head *head_config, void *loc_) + struct parse_events_terms *head_config, void *loc_) { YYLTYPE *loc = loc_; #ifdef HAVE_LIBTRACEEVENT @@ -1283,7 +1285,7 @@ int parse_events_add_tracepoint(struct list_head *list, int *idx, static int __parse_events_add_numeric(struct parse_events_state *parse_state, struct list_head *list, struct perf_pmu *pmu, u32 type, u32 extended_type, - u64 config, struct list_head *head_config) + u64 config, struct parse_events_terms *head_config) { struct perf_event_attr attr; LIST_HEAD(config_terms); @@ -1319,7 +1321,7 @@ static int __parse_events_add_numeric(struct parse_events_state *parse_state, int parse_events_add_numeric(struct parse_events_state *parse_state, struct list_head *list, u32 type, u64 config, - struct list_head *head_config, + struct parse_events_terms *head_config, bool wildcard) { struct perf_pmu *pmu = NULL; @@ -1368,7 +1370,7 @@ static bool config_term_percore(struct list_head *config_terms) int parse_events_add_pmu(struct parse_events_state *parse_state, struct list_head *list, const char *name, - const struct list_head *const_head_terms, + const struct parse_events_terms *const_parsed_terms, bool auto_merge_stats, void *loc_) { struct perf_event_attr attr; @@ -1378,7 +1380,7 @@ int parse_events_add_pmu(struct parse_events_state *parse_state, struct parse_events_error *err = parse_state->error; YYLTYPE *loc = loc_; LIST_HEAD(config_terms); - LIST_HEAD(head_terms); + struct parse_events_terms parsed_terms; pmu = parse_state->fake_pmu ?: perf_pmus__find(name); @@ -1392,8 +1394,9 @@ int parse_events_add_pmu(struct parse_events_state *parse_state, return -EINVAL; } - if (const_head_terms) { - int ret = parse_events_terms__copy(const_head_terms, &head_terms); + parse_events_terms__init(&parsed_terms); + if (const_parsed_terms) { + int ret = parse_events_terms__copy(const_parsed_terms, &parsed_terms); if (ret) return ret; @@ -1403,17 +1406,17 @@ int parse_events_add_pmu(struct parse_events_state *parse_state, struct strbuf sb; strbuf_init(&sb, /*hint=*/ 0); - if (pmu->selectable && list_empty(&head_terms)) { + if (pmu->selectable && list_empty(&parsed_terms.terms)) { strbuf_addf(&sb, "%s//", name); } else { strbuf_addf(&sb, "%s/", name); - parse_events_term__to_strbuf(&head_terms, &sb); + parse_events_terms__to_strbuf(&parsed_terms, &sb); strbuf_addch(&sb, '/'); } fprintf(stderr, "Attempt to add: %s\n", sb.buf); strbuf_release(&sb); } - fix_raw(&head_terms, pmu); + fix_raw(&parsed_terms, pmu); if (pmu->default_config) { memcpy(&attr, pmu->default_config, sizeof(struct perf_event_attr)); @@ -1422,7 +1425,7 @@ int parse_events_add_pmu(struct parse_events_state *parse_state, } attr.type = pmu->type; - if (list_empty(&head_terms)) { + if (list_empty(&parsed_terms.terms)) { evsel = __add_event(list, &parse_state->idx, &attr, /*init_attr=*/true, /*name=*/NULL, /*metric_id=*/NULL, pmu, @@ -1431,8 +1434,8 @@ int parse_events_add_pmu(struct parse_events_state *parse_state, return evsel ? 0 : -ENOMEM; } - if (!parse_state->fake_pmu && perf_pmu__check_alias(pmu, &head_terms, &info, err)) { - parse_events_terms__purge(&head_terms); + if (!parse_state->fake_pmu && perf_pmu__check_alias(pmu, &parsed_terms, &info, err)) { + parse_events_terms__exit(&parsed_terms); return -EINVAL; } @@ -1440,7 +1443,7 @@ int parse_events_add_pmu(struct parse_events_state *parse_state, struct strbuf sb; strbuf_init(&sb, /*hint=*/ 0); - parse_events_term__to_strbuf(&head_terms, &sb); + parse_events_terms__to_strbuf(&parsed_terms, &sb); fprintf(stderr, "..after resolving event: %s/%s/\n", name, sb.buf); strbuf_release(&sb); } @@ -1449,13 +1452,13 @@ int parse_events_add_pmu(struct parse_events_state *parse_state, * Configure hardcoded terms first, no need to check * return value when called with fail == 0 ;) */ - if (config_attr(&attr, &head_terms, parse_state->error, config_term_pmu)) { - parse_events_terms__purge(&head_terms); + if (config_attr(&attr, &parsed_terms, parse_state->error, config_term_pmu)) { + parse_events_terms__exit(&parsed_terms); return -EINVAL; } - if (get_config_terms(&head_terms, &config_terms)) { - parse_events_terms__purge(&head_terms); + if (get_config_terms(&parsed_terms, &config_terms)) { + parse_events_terms__exit(&parsed_terms); return -ENOMEM; } @@ -1463,24 +1466,24 @@ int parse_events_add_pmu(struct parse_events_state *parse_state, * When using default config, record which bits of attr->config were * changed by the user. */ - if (pmu->default_config && get_config_chgs(pmu, &head_terms, &config_terms)) { - parse_events_terms__purge(&head_terms); + if (pmu->default_config && get_config_chgs(pmu, &parsed_terms, &config_terms)) { + parse_events_terms__exit(&parsed_terms); return -ENOMEM; } if (!parse_state->fake_pmu && - perf_pmu__config(pmu, &attr, &head_terms, parse_state->error)) { + perf_pmu__config(pmu, &attr, &parsed_terms, parse_state->error)) { free_config_terms(&config_terms); - parse_events_terms__purge(&head_terms); + parse_events_terms__exit(&parsed_terms); return -EINVAL; } evsel = __add_event(list, &parse_state->idx, &attr, /*init_attr=*/true, - get_config_name(&head_terms), - get_config_metric_id(&head_terms), pmu, + get_config_name(&parsed_terms), + get_config_metric_id(&parsed_terms), pmu, &config_terms, auto_merge_stats, /*cpu_list=*/NULL); if (!evsel) { - parse_events_terms__purge(&head_terms); + parse_events_terms__exit(&parsed_terms); return -ENOMEM; } @@ -1490,11 +1493,11 @@ int parse_events_add_pmu(struct parse_events_state *parse_state, evsel->percore = config_term_percore(&evsel->config_terms); if (parse_state->fake_pmu) { - parse_events_terms__purge(&head_terms); + parse_events_terms__exit(&parsed_terms); return 0; } - parse_events_terms__purge(&head_terms); + parse_events_terms__exit(&parsed_terms); free((char *)evsel->unit); evsel->unit = strdup(info.unit); evsel->scale = info.scale; @@ -1505,7 +1508,7 @@ int parse_events_add_pmu(struct parse_events_state *parse_state, int parse_events_multi_pmu_add(struct parse_events_state *parse_state, const char *event_name, - const struct list_head *const_head_terms, + const struct parse_events_terms *const_parsed_terms, struct list_head **listp, void *loc_) { struct parse_events_term *term; @@ -1514,12 +1517,13 @@ int parse_events_multi_pmu_add(struct parse_events_state *parse_state, YYLTYPE *loc = loc_; int ok = 0; const char *config; - LIST_HEAD(head_terms); + struct parse_events_terms parsed_terms; *listp = NULL; - if (const_head_terms) { - int ret = parse_events_terms__copy(const_head_terms, &head_terms); + parse_events_terms__init(&parsed_terms); + if (const_parsed_terms) { + int ret = parse_events_terms__copy(const_parsed_terms, &parsed_terms); if (ret) return ret; @@ -1536,7 +1540,7 @@ int parse_events_multi_pmu_add(struct parse_events_state *parse_state, zfree(&config); goto out_err; } - list_add_tail(&term->list, &head_terms); + list_add_tail(&term->list, &parsed_terms.terms); /* Add it for all PMUs that support the alias */ list = malloc(sizeof(struct list_head)); @@ -1556,11 +1560,11 @@ int parse_events_multi_pmu_add(struct parse_events_state *parse_state, auto_merge_stats = perf_pmu__auto_merge_stats(pmu); if (!parse_events_add_pmu(parse_state, list, pmu->name, - &head_terms, auto_merge_stats, loc)) { + &parsed_terms, auto_merge_stats, loc)) { struct strbuf sb; strbuf_init(&sb, /*hint=*/ 0); - parse_events_term__to_strbuf(&head_terms, &sb); + parse_events_terms__to_strbuf(&parsed_terms, &sb); pr_debug("%s -> %s/%s/\n", event_name, pmu->name, sb.buf); strbuf_release(&sb); ok++; @@ -1568,12 +1572,12 @@ int parse_events_multi_pmu_add(struct parse_events_state *parse_state, } if (parse_state->fake_pmu) { - if (!parse_events_add_pmu(parse_state, list, event_name, &head_terms, + if (!parse_events_add_pmu(parse_state, list, event_name, &parsed_terms, /*auto_merge_stats=*/true, loc)) { struct strbuf sb; strbuf_init(&sb, /*hint=*/ 0); - parse_events_term__to_strbuf(&head_terms, &sb); + parse_events_terms__to_strbuf(&parsed_terms, &sb); pr_debug("%s -> %s/%s/\n", event_name, "fake_pmu", sb.buf); strbuf_release(&sb); ok++; @@ -1581,7 +1585,7 @@ int parse_events_multi_pmu_add(struct parse_events_state *parse_state, } out_err: - parse_events_terms__purge(&head_terms); + parse_events_terms__exit(&parsed_terms); if (ok) *listp = list; else @@ -1851,7 +1855,7 @@ static int parse_events__scanner(const char *str, /* * parse event config string, return a list of event terms. */ -int parse_events_terms(struct list_head *terms, const char *str, FILE *input) +int parse_events_terms(struct parse_events_terms *terms, const char *str, FILE *input) { struct parse_events_state parse_state = { .terms = NULL, @@ -1860,14 +1864,10 @@ int parse_events_terms(struct list_head *terms, const char *str, FILE *input) int ret; ret = parse_events__scanner(str, input, &parse_state); + if (!ret) + list_splice(&parse_state.terms->terms, &terms->terms); - if (!ret) { - list_splice(parse_state.terms, terms); - zfree(&parse_state.terms); - return 0; - } - - parse_events_terms__delete(parse_state.terms); + zfree(&parse_state.terms); return ret; } @@ -2563,11 +2563,12 @@ void parse_events_term__delete(struct parse_events_term *term) free(term); } -static int parse_events_terms__copy(const struct list_head *src, struct list_head *dest) +static int parse_events_terms__copy(const struct parse_events_terms *src, + struct parse_events_terms *dest) { struct parse_events_term *term; - list_for_each_entry (term, src, list) { + list_for_each_entry (term, &src->terms, list) { struct parse_events_term *n; int ret; @@ -2575,38 +2576,43 @@ static int parse_events_terms__copy(const struct list_head *src, struct list_hea if (ret) return ret; - list_add_tail(&n->list, dest); + list_add_tail(&n->list, &dest->terms); } return 0; } -void parse_events_terms__purge(struct list_head *terms) +void parse_events_terms__init(struct parse_events_terms *terms) +{ + INIT_LIST_HEAD(&terms->terms); +} + +void parse_events_terms__exit(struct parse_events_terms *terms) { struct parse_events_term *term, *h; - list_for_each_entry_safe(term, h, terms, list) { + list_for_each_entry_safe(term, h, &terms->terms, list) { list_del_init(&term->list); parse_events_term__delete(term); } } -void parse_events_terms__delete(struct list_head *terms) +void parse_events_terms__delete(struct parse_events_terms *terms) { if (!terms) return; - parse_events_terms__purge(terms); + parse_events_terms__exit(terms); free(terms); } -int parse_events_term__to_strbuf(struct list_head *term_list, struct strbuf *sb) +int parse_events_terms__to_strbuf(const struct parse_events_terms *terms, struct strbuf *sb) { struct parse_events_term *term; bool first = true; - if (!term_list) + if (!terms) return 0; - list_for_each_entry(term, term_list, list) { + list_for_each_entry(term, &terms->terms, list) { int ret; if (!first) { diff --git a/tools/perf/util/parse-events.h b/tools/perf/util/parse-events.h index e6612856e881..63c0a36a4bf1 100644 --- a/tools/perf/util/parse-events.h +++ b/tools/perf/util/parse-events.h @@ -44,7 +44,6 @@ static inline int parse_events(struct evlist *evlist, const char *str, int parse_event(struct evlist *evlist, const char *str); -int parse_events_terms(struct list_head *terms, const char *str, FILE *input); int parse_filter(const struct option *opt, const char *str, int unset); int exclude_perf(const struct option *opt, const char *arg, int unset); @@ -140,6 +139,11 @@ struct parse_events_error { char *first_help; }; +/* A wrapper around a list of terms for the sake of better type safety. */ +struct parse_events_terms { + struct list_head terms; +}; + struct parse_events_state { /* The list parsed events are placed on. */ struct list_head list; @@ -148,7 +152,7 @@ struct parse_events_state { /* Error information. */ struct parse_events_error *error; /* Holds returned terms for term parsing. */ - struct list_head *terms; + struct parse_events_terms *terms; /* Start token. */ int stoken; /* Special fake PMU marker for testing. */ @@ -181,35 +185,38 @@ int parse_events_term__term(struct parse_events_term **term, int parse_events_term__clone(struct parse_events_term **new, struct parse_events_term *term); void parse_events_term__delete(struct parse_events_term *term); -void parse_events_terms__delete(struct list_head *terms); -void parse_events_terms__purge(struct list_head *terms); -int parse_events_term__to_strbuf(struct list_head *term_list, struct strbuf *sb); + +void parse_events_terms__delete(struct parse_events_terms *terms); +void parse_events_terms__init(struct parse_events_terms *terms); +void parse_events_terms__exit(struct parse_events_terms *terms); +int parse_events_terms(struct parse_events_terms *terms, const char *str, FILE *input); +int parse_events_terms__to_strbuf(const struct parse_events_terms *terms, struct strbuf *sb); int parse_events__modifier_event(struct list_head *list, char *str, bool add); int parse_events__modifier_group(struct list_head *list, char *event_mod); int parse_events_name(struct list_head *list, const char *name); int parse_events_add_tracepoint(struct list_head *list, int *idx, const char *sys, const char *event, struct parse_events_error *error, - struct list_head *head_config, void *loc); + struct parse_events_terms *head_config, void *loc); int parse_events_add_numeric(struct parse_events_state *parse_state, struct list_head *list, u32 type, u64 config, - struct list_head *head_config, + struct parse_events_terms *head_config, bool wildcard); int parse_events_add_tool(struct parse_events_state *parse_state, struct list_head *list, int tool_event); int parse_events_add_cache(struct list_head *list, int *idx, const char *name, struct parse_events_state *parse_state, - struct list_head *head_config); + struct parse_events_terms *head_config); int parse_events__decode_legacy_cache(const char *name, int pmu_type, __u64 *config); int parse_events_add_breakpoint(struct parse_events_state *parse_state, struct list_head *list, u64 addr, char *type, u64 len, - struct list_head *head_config); + struct parse_events_terms *head_config); int parse_events_add_pmu(struct parse_events_state *parse_state, struct list_head *list, const char *name, - const struct list_head *const_head_terms, + const struct parse_events_terms *const_parsed_terms, bool auto_merge_stats, void *loc); struct evsel *parse_events__add_event(int idx, struct perf_event_attr *attr, @@ -218,7 +225,7 @@ struct evsel *parse_events__add_event(int idx, struct perf_event_attr *attr, int parse_events_multi_pmu_add(struct parse_events_state *parse_state, const char *event_name, - const struct list_head *head_terms, + const struct parse_events_terms *const_parsed_terms, struct list_head **listp, void *loc); void parse_events__set_leader(char *name, struct list_head *list); diff --git a/tools/perf/util/parse-events.y b/tools/perf/util/parse-events.y index 4d7947fb4a98..bf74d6564eba 100644 --- a/tools/perf/util/parse-events.y +++ b/tools/perf/util/parse-events.y @@ -113,7 +113,7 @@ static void free_list_evsel(struct list_head* list_evsel) u64 num; enum parse_events__term_type term_type; struct list_head *list_evsel; - struct list_head *list_terms; + struct parse_events_terms *list_terms; struct parse_events_term *term; struct tracepoint_name { char *sys; @@ -644,26 +644,26 @@ start_terms: event_config event_config: event_config ',' event_term { - struct list_head *head = $1; + struct parse_events_terms *head = $1; struct parse_events_term *term = $3; if (!head) { parse_events_term__delete(term); YYABORT; } - list_add_tail(&term->list, head); + list_add_tail(&term->list, &head->terms); $$ = $1; } | event_term { - struct list_head *head = malloc(sizeof(*head)); + struct parse_events_terms *head = malloc(sizeof(*head)); struct parse_events_term *term = $1; if (!head) YYNOMEM; - INIT_LIST_HEAD(head); - list_add_tail(&term->list, head); + parse_events_terms__init(head); + list_add_tail(&term->list, &head->terms); $$ = head; } diff --git a/tools/perf/util/pmu.c b/tools/perf/util/pmu.c index d85602aa4b9f..e2159854ab26 100644 --- a/tools/perf/util/pmu.c +++ b/tools/perf/util/pmu.c @@ -52,7 +52,7 @@ struct perf_pmu_alias { */ char *topic; /** @terms: Owned list of the original parsed parameters. */ - struct list_head terms; + struct parse_events_terms terms; /** @list: List element of struct perf_pmu aliases. */ struct list_head list; /** @@ -404,7 +404,7 @@ static void perf_pmu_free_alias(struct perf_pmu_alias *newalias) zfree(&newalias->long_desc); zfree(&newalias->topic); zfree(&newalias->pmu_name); - parse_events_terms__purge(&newalias->terms); + parse_events_terms__exit(&newalias->terms); free(newalias); } @@ -484,7 +484,7 @@ static int update_alias(const struct pmu_event *pe, assign_str(pe->name, "topic", &data->alias->topic, pe->topic); data->alias->per_pkg = pe->perpkg; if (pe->event) { - parse_events_terms__purge(&data->alias->terms); + parse_events_terms__exit(&data->alias->terms); ret = parse_events_terms(&data->alias->terms, pe->event, /*input=*/NULL); } if (!ret && pe->unit) { @@ -524,7 +524,7 @@ static int perf_pmu__new_alias(struct perf_pmu *pmu, const char *name, if (!alias) return -ENOMEM; - INIT_LIST_HEAD(&alias->terms); + parse_events_terms__init(&alias->terms); alias->scale = 1.0; alias->unit[0] = '\0'; alias->per_pkg = perpkg; @@ -656,17 +656,17 @@ static int pmu_aliases_parse(struct perf_pmu *pmu) return 0; } -static int pmu_alias_terms(struct perf_pmu_alias *alias, - struct list_head *terms) +static int pmu_alias_terms(struct perf_pmu_alias *alias, struct list_head *terms) { struct parse_events_term *term, *cloned; - LIST_HEAD(list); - int ret; + struct parse_events_terms clone_terms; + + parse_events_terms__init(&clone_terms); + list_for_each_entry(term, &alias->terms.terms, list) { + int ret = parse_events_term__clone(&cloned, term); - list_for_each_entry(term, &alias->terms, list) { - ret = parse_events_term__clone(&cloned, term); if (ret) { - parse_events_terms__purge(&list); + parse_events_terms__exit(&clone_terms); return ret; } /* @@ -674,9 +674,10 @@ static int pmu_alias_terms(struct perf_pmu_alias *alias, * which we don't want for implicit terms in aliases. */ cloned->weak = true; - list_add_tail(&cloned->list, &list); + list_add_tail(&cloned->list, &clone_terms.terms); } - list_splice(&list, terms); + list_splice_init(&clone_terms.terms, terms); + parse_events_terms__exit(&clone_terms); return 0; } @@ -1188,12 +1189,12 @@ static __u64 pmu_format_max_value(const unsigned long *format) * in a config string) later on in the term list. */ static int pmu_resolve_param_term(struct parse_events_term *term, - struct list_head *head_terms, + struct parse_events_terms *head_terms, __u64 *value) { struct parse_events_term *t; - list_for_each_entry(t, head_terms, list) { + list_for_each_entry(t, &head_terms->terms, list) { if (t->type_val == PARSE_EVENTS__TERM_TYPE_NUM && t->config && !strcmp(t->config, term->config)) { t->used = true; @@ -1237,7 +1238,7 @@ error: static int pmu_config_term(struct perf_pmu *pmu, struct perf_event_attr *attr, struct parse_events_term *term, - struct list_head *head_terms, + struct parse_events_terms *head_terms, bool zero, struct parse_events_error *err) { struct perf_pmu_format *format; @@ -1359,13 +1360,13 @@ static int pmu_config_term(struct perf_pmu *pmu, int perf_pmu__config_terms(struct perf_pmu *pmu, struct perf_event_attr *attr, - struct list_head *head_terms, + struct parse_events_terms *terms, bool zero, struct parse_events_error *err) { struct parse_events_term *term; - list_for_each_entry(term, head_terms, list) { - if (pmu_config_term(pmu, attr, term, head_terms, zero, err)) + list_for_each_entry(term, &terms->terms, list) { + if (pmu_config_term(pmu, attr, term, terms, zero, err)) return -EINVAL; } @@ -1378,7 +1379,7 @@ int perf_pmu__config_terms(struct perf_pmu *pmu, * 2) pmu format definitions - specified by pmu parameter */ int perf_pmu__config(struct perf_pmu *pmu, struct perf_event_attr *attr, - struct list_head *head_terms, + struct parse_events_terms *head_terms, struct parse_events_error *err) { bool zero = !!pmu->default_config; @@ -1472,7 +1473,7 @@ static int check_info_data(struct perf_pmu *pmu, * Find alias in the terms list and replace it with the terms * defined for the alias */ -int perf_pmu__check_alias(struct perf_pmu *pmu, struct list_head *head_terms, +int perf_pmu__check_alias(struct perf_pmu *pmu, struct parse_events_terms *head_terms, struct perf_pmu_info *info, struct parse_events_error *err) { struct parse_events_term *term, *h; @@ -1489,7 +1490,7 @@ int perf_pmu__check_alias(struct perf_pmu *pmu, struct list_head *head_terms, info->scale = 0.0; info->snapshot = false; - list_for_each_entry_safe(term, h, head_terms, list) { + list_for_each_entry_safe(term, h, &head_terms->terms, list) { alias = pmu_find_alias(pmu, term); if (!alias) continue; @@ -1634,7 +1635,7 @@ static char *format_alias(char *buf, int len, const struct perf_pmu *pmu, : (int)strlen(pmu->name); int used = snprintf(buf, len, "%.*s/%s", pmu_name_len, pmu->name, alias->name); - list_for_each_entry(term, &alias->terms, list) { + list_for_each_entry(term, &alias->terms.terms, list) { if (term->type_val == PARSE_EVENTS__TERM_TYPE_STR) used += snprintf(buf + used, sub_non_neg(len, used), ",%s=%s", term->config, @@ -1693,7 +1694,7 @@ int perf_pmu__for_each_event(struct perf_pmu *pmu, bool skip_duplicate_pmus, info.desc = event->desc; info.long_desc = event->long_desc; info.encoding_desc = buf + buf_used; - parse_events_term__to_strbuf(&event->terms, &sb); + parse_events_terms__to_strbuf(&event->terms, &sb); buf_used += snprintf(buf + buf_used, sizeof(buf) - buf_used, "%s/%s/", info.pmu_name, sb.buf) + 1; info.topic = event->topic; diff --git a/tools/perf/util/pmu.h b/tools/perf/util/pmu.h index 6a4e170c61d6..bd5d804a6736 100644 --- a/tools/perf/util/pmu.h +++ b/tools/perf/util/pmu.h @@ -191,15 +191,15 @@ typedef int (*pmu_event_callback)(void *state, struct pmu_event_info *info); void pmu_add_sys_aliases(struct perf_pmu *pmu); int perf_pmu__config(struct perf_pmu *pmu, struct perf_event_attr *attr, - struct list_head *head_terms, + struct parse_events_terms *head_terms, struct parse_events_error *error); int perf_pmu__config_terms(struct perf_pmu *pmu, struct perf_event_attr *attr, - struct list_head *head_terms, + struct parse_events_terms *terms, bool zero, struct parse_events_error *error); __u64 perf_pmu__format_bits(struct perf_pmu *pmu, const char *name); int perf_pmu__format_type(struct perf_pmu *pmu, const char *name); -int perf_pmu__check_alias(struct perf_pmu *pmu, struct list_head *head_terms, +int perf_pmu__check_alias(struct perf_pmu *pmu, struct parse_events_terms *head_terms, struct perf_pmu_info *info, struct parse_events_error *err); int perf_pmu__find_event(struct perf_pmu *pmu, const char *event, void *state, pmu_event_callback cb); From 5d6151531a373a71c6ff02eafacd5e3aca610f2d Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Mon, 28 Aug 2023 17:17:28 -0700 Subject: [PATCH 034/475] perf vendor events intel: Add lunarlake v1.0 Add lunarlake events that were added at intel's perfmon site in: https://github.com/intel/perfmon/pull/97 "LNL: Release initial events" Reviewed-by: Kan Liang Signed-off-by: Ian Rogers Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Alexandre Torgue Cc: Colin Ian King Cc: Edward Baker Cc: Ingo Molnar Cc: Jiri Olsa Cc: Mark Rutland Cc: Maxime Coquelin Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Xing Zhengjun Link: https://lore.kernel.org/r/20230829001730.1352769-1-irogers@google.com Signed-off-by: Arnaldo Carvalho de Melo --- .../pmu-events/arch/x86/lunarlake/cache.json | 219 ++++++++++++++++++ .../arch/x86/lunarlake/frontend.json | 27 +++ .../pmu-events/arch/x86/lunarlake/memory.json | 183 +++++++++++++++ .../pmu-events/arch/x86/lunarlake/other.json | 62 +++++ .../arch/x86/lunarlake/pipeline.json | 217 +++++++++++++++++ .../arch/x86/lunarlake/virtual-memory.json | 56 +++++ tools/perf/pmu-events/arch/x86/mapfile.csv | 1 + 7 files changed, 765 insertions(+) create mode 100644 tools/perf/pmu-events/arch/x86/lunarlake/cache.json create mode 100644 tools/perf/pmu-events/arch/x86/lunarlake/frontend.json create mode 100644 tools/perf/pmu-events/arch/x86/lunarlake/memory.json create mode 100644 tools/perf/pmu-events/arch/x86/lunarlake/other.json create mode 100644 tools/perf/pmu-events/arch/x86/lunarlake/pipeline.json create mode 100644 tools/perf/pmu-events/arch/x86/lunarlake/virtual-memory.json diff --git a/tools/perf/pmu-events/arch/x86/lunarlake/cache.json b/tools/perf/pmu-events/arch/x86/lunarlake/cache.json new file mode 100644 index 000000000000..1823149067b5 --- /dev/null +++ b/tools/perf/pmu-events/arch/x86/lunarlake/cache.json @@ -0,0 +1,219 @@ +[ + { + "BriefDescription": "Counts the number of L2 Cache Accesses Counts the total number of L2 Cache Accesses - sum of hits, misses, rejects front door requests for CRd/DRd/RFO/ItoM/L2 Prefetches only, per core event", + "EventCode": "0x24", + "EventName": "L2_REQUEST.ALL", + "PublicDescription": "Counts the number of L2 Cache Accesses Counts the total number of L2 Cache Accesses - sum of hits, misses, rejects front door requests for CRd/DRd/RFO/ItoM/L2 Prefetches only.", + "SampleAfterValue": "1000003", + "UMask": "0x7", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of cacheable memory requests that miss in the LLC. Counts on a per core basis.", + "EventCode": "0x2e", + "EventName": "LONGEST_LAT_CACHE.MISS", + "PublicDescription": "Counts the number of cacheable memory requests that miss in the Last Level Cache (LLC). Requests include demand loads, reads for ownership (RFO), instruction fetches and L1 HW prefetches. If the platform has an L3 cache, the LLC is the L3 cache, otherwise it is the L2 cache. Counts on a per core basis.", + "SampleAfterValue": "200003", + "UMask": "0x41", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Core-originated cacheable requests that missed L3 (Except hardware prefetches to the L3)", + "EventCode": "0x2e", + "EventName": "LONGEST_LAT_CACHE.MISS", + "PublicDescription": "Counts core-originated cacheable requests that miss the L3 cache (Longest Latency cache). Requests include data and code reads, Reads-for-Ownership (RFOs), speculative accesses and hardware prefetches to the L1 and L2. It does not include hardware prefetches to the L3, and may not count other types of requests to the L3.", + "SampleAfterValue": "100003", + "UMask": "0x41", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Counts the number of cacheable memory requests that access the LLC. Counts on a per core basis.", + "EventCode": "0x2e", + "EventName": "LONGEST_LAT_CACHE.REFERENCE", + "PublicDescription": "Counts the number of cacheable memory requests that access the Last Level Cache (LLC). Requests include demand loads, reads for ownership (RFO), instruction fetches and L1 HW prefetches. If the platform has an L3 cache, the LLC is the L3 cache, otherwise it is the L2 cache. Counts on a per core basis.", + "SampleAfterValue": "1000003", + "UMask": "0x4f", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Core-originated cacheable requests that refer to L3 (Except hardware prefetches to the L3)", + "EventCode": "0x2e", + "EventName": "LONGEST_LAT_CACHE.REFERENCE", + "PublicDescription": "Counts core-originated cacheable requests to the L3 cache (Longest Latency cache). Requests include data and code reads, Reads-for-Ownership (RFOs), speculative accesses and hardware prefetches to the L1 and L2. It does not include hardware prefetches to the L3, and may not count other types of requests to the L3.", + "SampleAfterValue": "100003", + "UMask": "0x4f", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Retired load instructions.", + "Data_LA": "1", + "EventCode": "0xd0", + "EventName": "MEM_INST_RETIRED.ALL_LOADS", + "PEBS": "1", + "PublicDescription": "Counts all retired load instructions. This event accounts for SW prefetch instructions of PREFETCHNTA or PREFETCHT0/1/2 or PREFETCHW.", + "SampleAfterValue": "1000003", + "UMask": "0x81", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Retired store instructions.", + "Data_LA": "1", + "EventCode": "0xd0", + "EventName": "MEM_INST_RETIRED.ALL_STORES", + "PEBS": "1", + "PublicDescription": "Counts all retired store instructions.", + "SampleAfterValue": "1000003", + "UMask": "0x82", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Counts the number of load uops retired.", + "Data_LA": "1", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.ALL_LOADS", + "PEBS": "1", + "SampleAfterValue": "200003", + "UMask": "0x81", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of store uops retired.", + "Data_LA": "1", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.ALL_STORES", + "PEBS": "1", + "SampleAfterValue": "200003", + "UMask": "0x82", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled", + "Data_LA": "1", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_1024", + "MSRIndex": "0x3F6", + "MSRValue": "0x400", + "PEBS": "2", + "SampleAfterValue": "1000003", + "UMask": "0x5", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled", + "Data_LA": "1", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_128", + "MSRIndex": "0x3F6", + "MSRValue": "0x80", + "PEBS": "2", + "SampleAfterValue": "1000003", + "UMask": "0x5", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled", + "Data_LA": "1", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_16", + "MSRIndex": "0x3F6", + "MSRValue": "0x10", + "PEBS": "2", + "SampleAfterValue": "1000003", + "UMask": "0x5", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled", + "Data_LA": "1", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_2048", + "MSRIndex": "0x3F6", + "MSRValue": "0x800", + "PEBS": "2", + "SampleAfterValue": "1000003", + "UMask": "0x5", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled", + "Data_LA": "1", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_256", + "MSRIndex": "0x3F6", + "MSRValue": "0x100", + "PEBS": "2", + "SampleAfterValue": "1000003", + "UMask": "0x5", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled", + "Data_LA": "1", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_32", + "MSRIndex": "0x3F6", + "MSRValue": "0x20", + "PEBS": "2", + "SampleAfterValue": "1000003", + "UMask": "0x5", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled", + "Data_LA": "1", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_4", + "MSRIndex": "0x3F6", + "MSRValue": "0x4", + "PEBS": "2", + "SampleAfterValue": "1000003", + "UMask": "0x5", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled", + "Data_LA": "1", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_512", + "MSRIndex": "0x3F6", + "MSRValue": "0x200", + "PEBS": "2", + "SampleAfterValue": "1000003", + "UMask": "0x5", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled", + "Data_LA": "1", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_64", + "MSRIndex": "0x3F6", + "MSRValue": "0x40", + "PEBS": "2", + "SampleAfterValue": "1000003", + "UMask": "0x5", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of tagged load uops retired that exceed the latency threshold defined in MEC_CR_PEBS_LD_LAT_THRESHOLD - Only counts with PEBS enabled", + "Data_LA": "1", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_8", + "MSRIndex": "0x3F6", + "MSRValue": "0x8", + "PEBS": "2", + "SampleAfterValue": "1000003", + "UMask": "0x5", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of stores uops retired same as MEM_UOPS_RETIRED.ALL_STORES", + "Data_LA": "1", + "EventCode": "0xd0", + "EventName": "MEM_UOPS_RETIRED.STORE_LATENCY", + "PEBS": "2", + "SampleAfterValue": "1000003", + "UMask": "0x6", + "Unit": "cpu_atom" + } +] diff --git a/tools/perf/pmu-events/arch/x86/lunarlake/frontend.json b/tools/perf/pmu-events/arch/x86/lunarlake/frontend.json new file mode 100644 index 000000000000..5e4ef81b43d6 --- /dev/null +++ b/tools/perf/pmu-events/arch/x86/lunarlake/frontend.json @@ -0,0 +1,27 @@ +[ + { + "BriefDescription": "Counts every time the code stream enters into a new cache line by walking sequential from the previous line or being redirected by a jump.", + "EventCode": "0x80", + "EventName": "ICACHE.ACCESSES", + "SampleAfterValue": "200003", + "UMask": "0x3", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts every time the code stream enters into a new cache line by walking sequential from the previous line or being redirected by a jump and the instruction cache registers bytes are not present. -", + "EventCode": "0x80", + "EventName": "ICACHE.MISSES", + "SampleAfterValue": "200003", + "UMask": "0x2", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "This event counts a subset of the Topdown Slots event that were no operation was delivered to the back-end pipeline due to instruction fetch limitations when the back-end could have accepted more operations. Common examples include instruction cache misses or x86 instruction decode limitations.", + "EventCode": "0x9c", + "EventName": "IDQ_BUBBLES.CORE", + "PublicDescription": "This event counts a subset of the Topdown Slots event that were no operation was delivered to the back-end pipeline due to instruction fetch limitations when the back-end could have accepted more operations. Common examples include instruction cache misses or x86 instruction decode limitations.\nSoftware can use this event as the numerator for the Frontend Bound metric (or top-level category) of the Top-down Microarchitecture Analysis method.", + "SampleAfterValue": "1000003", + "UMask": "0x1", + "Unit": "cpu_core" + } +] diff --git a/tools/perf/pmu-events/arch/x86/lunarlake/memory.json b/tools/perf/pmu-events/arch/x86/lunarlake/memory.json new file mode 100644 index 000000000000..51d70ba00bd4 --- /dev/null +++ b/tools/perf/pmu-events/arch/x86/lunarlake/memory.json @@ -0,0 +1,183 @@ +[ + { + "BriefDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 1024 cycles.", + "Data_LA": "1", + "EventCode": "0xcd", + "EventName": "MEM_TRANS_RETIRED.LOAD_LATENCY_GT_1024", + "MSRIndex": "0x3F6", + "MSRValue": "0x400", + "PEBS": "2", + "PublicDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 1024 cycles. Reported latency may be longer than just the memory latency.", + "SampleAfterValue": "53", + "UMask": "0x1", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 128 cycles.", + "Data_LA": "1", + "EventCode": "0xcd", + "EventName": "MEM_TRANS_RETIRED.LOAD_LATENCY_GT_128", + "MSRIndex": "0x3F6", + "MSRValue": "0x80", + "PEBS": "2", + "PublicDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 128 cycles. Reported latency may be longer than just the memory latency.", + "SampleAfterValue": "1009", + "UMask": "0x1", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 16 cycles.", + "Data_LA": "1", + "EventCode": "0xcd", + "EventName": "MEM_TRANS_RETIRED.LOAD_LATENCY_GT_16", + "MSRIndex": "0x3F6", + "MSRValue": "0x10", + "PEBS": "2", + "PublicDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 16 cycles. Reported latency may be longer than just the memory latency.", + "SampleAfterValue": "20011", + "UMask": "0x1", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 2048 cycles.", + "Data_LA": "1", + "EventCode": "0xcd", + "EventName": "MEM_TRANS_RETIRED.LOAD_LATENCY_GT_2048", + "MSRIndex": "0x3F6", + "MSRValue": "0x800", + "PEBS": "2", + "PublicDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 2048 cycles. Reported latency may be longer than just the memory latency.", + "SampleAfterValue": "23", + "UMask": "0x1", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 256 cycles.", + "Data_LA": "1", + "EventCode": "0xcd", + "EventName": "MEM_TRANS_RETIRED.LOAD_LATENCY_GT_256", + "MSRIndex": "0x3F6", + "MSRValue": "0x100", + "PEBS": "2", + "PublicDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 256 cycles. Reported latency may be longer than just the memory latency.", + "SampleAfterValue": "503", + "UMask": "0x1", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 32 cycles.", + "Data_LA": "1", + "EventCode": "0xcd", + "EventName": "MEM_TRANS_RETIRED.LOAD_LATENCY_GT_32", + "MSRIndex": "0x3F6", + "MSRValue": "0x20", + "PEBS": "2", + "PublicDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 32 cycles. Reported latency may be longer than just the memory latency.", + "SampleAfterValue": "100007", + "UMask": "0x1", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 4 cycles.", + "Data_LA": "1", + "EventCode": "0xcd", + "EventName": "MEM_TRANS_RETIRED.LOAD_LATENCY_GT_4", + "MSRIndex": "0x3F6", + "MSRValue": "0x4", + "PEBS": "2", + "PublicDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 4 cycles. Reported latency may be longer than just the memory latency.", + "SampleAfterValue": "100003", + "UMask": "0x1", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 512 cycles.", + "Data_LA": "1", + "EventCode": "0xcd", + "EventName": "MEM_TRANS_RETIRED.LOAD_LATENCY_GT_512", + "MSRIndex": "0x3F6", + "MSRValue": "0x200", + "PEBS": "2", + "PublicDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 512 cycles. Reported latency may be longer than just the memory latency.", + "SampleAfterValue": "101", + "UMask": "0x1", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 64 cycles.", + "Data_LA": "1", + "EventCode": "0xcd", + "EventName": "MEM_TRANS_RETIRED.LOAD_LATENCY_GT_64", + "MSRIndex": "0x3F6", + "MSRValue": "0x40", + "PEBS": "2", + "PublicDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 64 cycles. Reported latency may be longer than just the memory latency.", + "SampleAfterValue": "2003", + "UMask": "0x1", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 8 cycles.", + "Data_LA": "1", + "EventCode": "0xcd", + "EventName": "MEM_TRANS_RETIRED.LOAD_LATENCY_GT_8", + "MSRIndex": "0x3F6", + "MSRValue": "0x8", + "PEBS": "2", + "PublicDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 8 cycles. Reported latency may be longer than just the memory latency.", + "SampleAfterValue": "50021", + "UMask": "0x1", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Retired memory store access operations. A PDist event for PEBS Store Latency Facility.", + "Data_LA": "1", + "EventCode": "0xcd", + "EventName": "MEM_TRANS_RETIRED.STORE_SAMPLE", + "PEBS": "2", + "PublicDescription": "Counts Retired memory accesses with at least 1 store operation. This PEBS event is the precisely-distributed (PDist) trigger covering all stores uops for sampling by the PEBS Store Latency Facility. The facility is described in Intel SDM Volume 3 section 19.9.8", + "SampleAfterValue": "1000003", + "UMask": "0x2", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Counts cacheable demand data reads were not supplied by the L3 cache.", + "EventCode": "0xB7", + "EventName": "OCR.DEMAND_DATA_RD.L3_MISS", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x3FBFC00001", + "SampleAfterValue": "100003", + "UMask": "0x1", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts demand data reads that were not supplied by the L3 cache.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_DATA_RD.L3_MISS", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x3FBFC00001", + "SampleAfterValue": "100003", + "UMask": "0x1", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Counts demand reads for ownership, including SWPREFETCHW which is an RFO were not supplied by the L3 cache.", + "EventCode": "0xB7", + "EventName": "OCR.DEMAND_RFO.L3_MISS", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x3FBFC00002", + "SampleAfterValue": "100003", + "UMask": "0x1", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts demand read for ownership (RFO) requests and software prefetches for exclusive ownership (PREFETCHW) that were not supplied by the L3 cache.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_RFO.L3_MISS", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x3FBFC00002", + "SampleAfterValue": "100003", + "UMask": "0x1", + "Unit": "cpu_core" + } +] diff --git a/tools/perf/pmu-events/arch/x86/lunarlake/other.json b/tools/perf/pmu-events/arch/x86/lunarlake/other.json new file mode 100644 index 000000000000..69adaed5686d --- /dev/null +++ b/tools/perf/pmu-events/arch/x86/lunarlake/other.json @@ -0,0 +1,62 @@ +[ + { + "BriefDescription": "Counts cacheable demand data reads Catch all value for any response types - this includes response types not define in the OCR. If this is set all other response types will be ignored", + "EventCode": "0xB7", + "EventName": "OCR.DEMAND_DATA_RD.ANY_RESPONSE", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x10001", + "SampleAfterValue": "100003", + "UMask": "0x1", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts demand data reads that have any type of response.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_DATA_RD.ANY_RESPONSE", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x10001", + "SampleAfterValue": "100003", + "UMask": "0x1", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Counts cacheable demand data reads were supplied by DRAM.", + "EventCode": "0xB7", + "EventName": "OCR.DEMAND_DATA_RD.DRAM", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x184000001", + "SampleAfterValue": "100003", + "UMask": "0x1", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts demand data reads that were supplied by DRAM.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_DATA_RD.DRAM", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x184000001", + "SampleAfterValue": "100003", + "UMask": "0x1", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Counts demand reads for ownership, including SWPREFETCHW which is an RFO Catch all value for any response types - this includes response types not define in the OCR. If this is set all other response types will be ignored", + "EventCode": "0xB7", + "EventName": "OCR.DEMAND_RFO.ANY_RESPONSE", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x10002", + "SampleAfterValue": "100003", + "UMask": "0x1", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts demand read for ownership (RFO) requests and software prefetches for exclusive ownership (PREFETCHW) that have any type of response.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_RFO.ANY_RESPONSE", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x10002", + "SampleAfterValue": "100003", + "UMask": "0x1", + "Unit": "cpu_core" + } +] diff --git a/tools/perf/pmu-events/arch/x86/lunarlake/pipeline.json b/tools/perf/pmu-events/arch/x86/lunarlake/pipeline.json new file mode 100644 index 000000000000..2bde664fdc0f --- /dev/null +++ b/tools/perf/pmu-events/arch/x86/lunarlake/pipeline.json @@ -0,0 +1,217 @@ +[ + { + "BriefDescription": "Counts the total number of branch instructions retired for all branch types.", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.ALL_BRANCHES", + "PEBS": "1", + "PublicDescription": "Counts the total number of instructions in which the instruction pointer (IP) of the processor is resteered due to a branch instruction and the branch instruction successfully retires. All branch type instructions are accounted for.", + "SampleAfterValue": "200003", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "All branch instructions retired.", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.ALL_BRANCHES", + "PEBS": "1", + "PublicDescription": "Counts all branch instructions retired.", + "SampleAfterValue": "400009", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Counts the total number of mispredicted branch instructions retired for all branch types.", + "EventCode": "0xc5", + "EventName": "BR_MISP_RETIRED.ALL_BRANCHES", + "PEBS": "1", + "PublicDescription": "Counts the total number of mispredicted branch instructions retired. All branch type instructions are accounted for. Prediction of the branch target address enables the processor to begin executing instructions before the non-speculative execution path is known. The branch prediction unit (BPU) predicts the target address based on the instruction pointer (IP) of the branch and on the execution path through which execution reached this IP. A branch misprediction occurs when the prediction is wrong, and results in discarding all instructions executed in the speculative path and re-fetching from the correct path.", + "SampleAfterValue": "200003", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "All mispredicted branch instructions retired.", + "EventCode": "0xc5", + "EventName": "BR_MISP_RETIRED.ALL_BRANCHES", + "PEBS": "1", + "PublicDescription": "Counts all the retired branch instructions that were mispredicted by the processor. A branch misprediction occurs when the processor incorrectly predicts the destination of the branch. When the misprediction is discovered at execution, all the instructions executed in the wrong (speculative) path must be discarded, and the processor must start fetching from the correct path.", + "SampleAfterValue": "400009", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Fixed Counter: Counts the number of unhalted core clock cycles", + "EventName": "CPU_CLK_UNHALTED.CORE", + "SampleAfterValue": "1000003", + "UMask": "0x2", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of unhalted core clock cycles [This event is alias to CPU_CLK_UNHALTED.THREAD_P]", + "EventCode": "0x3c", + "EventName": "CPU_CLK_UNHALTED.CORE_P", + "SampleAfterValue": "2000003", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Fixed Counter: Counts the number of unhalted reference clock cycles", + "EventName": "CPU_CLK_UNHALTED.REF_TSC", + "SampleAfterValue": "1000003", + "UMask": "0x3", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Reference cycles when the core is not in halt state.", + "EventName": "CPU_CLK_UNHALTED.REF_TSC", + "PublicDescription": "Counts the number of reference cycles when the core is not in a halt state. The core enters the halt state when it is running the HLT instruction or the MWAIT instruction. This event is not affected by core frequency changes (for example, P states, TM2 transitions) but has the same incrementing frequency as the time stamp counter. This event can approximate elapsed time while the core was not in a halt state. Note: On all current platforms this event stops counting during 'throttling (TM)' states duty off periods the processor is 'halted'. The counter update is done at a lower clock rate then the core clock the overflow status bit for this counter may appear 'sticky'. After the counter has overflowed and software clears the overflow status bit and resets the counter to less than MAX. The reset value to the counter is not clocked immediately so the overflow status bit will flip 'high (1)' and generate another PMI (if enabled) after which the reset value gets clocked into the counter. Therefore, software will get the interrupt, read the overflow status bit '1 for bit 34 while the counter value is less than MAX. Software should ignore this case.", + "SampleAfterValue": "2000003", + "UMask": "0x3", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Reference cycles when the core is not in halt state.", + "EventCode": "0x3c", + "EventName": "CPU_CLK_UNHALTED.REF_TSC_P", + "PublicDescription": "Counts the number of reference cycles when the core is not in a halt state. The core enters the halt state when it is running the HLT instruction or the MWAIT instruction. This event is not affected by core frequency changes (for example, P states, TM2 transitions) but has the same incrementing frequency as the time stamp counter. This event can approximate elapsed time while the core was not in a halt state. Note: On all current platforms this event stops counting during 'throttling (TM)' states duty off periods the processor is 'halted'. The counter update is done at a lower clock rate then the core clock the overflow status bit for this counter may appear 'sticky'. After the counter has overflowed and software clears the overflow status bit and resets the counter to less than MAX. The reset value to the counter is not clocked immediately so the overflow status bit will flip 'high (1)' and generate another PMI (if enabled) after which the reset value gets clocked into the counter. Therefore, software will get the interrupt, read the overflow status bit '1 for bit 34 while the counter value is less than MAX. Software should ignore this case.", + "SampleAfterValue": "2000003", + "UMask": "0x1", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Core cycles when the thread is not in halt state", + "EventName": "CPU_CLK_UNHALTED.THREAD", + "PublicDescription": "Counts the number of core cycles while the thread is not in a halt state. The thread enters the halt state when it is running the HLT instruction. This event is a component in many key event ratios. The core frequency may change from time to time due to transitions associated with Enhanced Intel SpeedStep Technology or TM2. For this reason this event may have a changing ratio with regards to time. When the core frequency is constant, this event can approximate elapsed time while the core was not in the halt state. It is counted on a dedicated fixed counter, leaving the eight programmable counters available for other events.", + "SampleAfterValue": "2000003", + "UMask": "0x2", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Counts the number of unhalted core clock cycles [This event is alias to CPU_CLK_UNHALTED.CORE_P]", + "EventCode": "0x3c", + "EventName": "CPU_CLK_UNHALTED.THREAD_P", + "SampleAfterValue": "2000003", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Thread cycles when thread is not in halt state", + "EventCode": "0x3c", + "EventName": "CPU_CLK_UNHALTED.THREAD_P", + "PublicDescription": "This is an architectural event that counts the number of thread cycles while the thread is not in a halt state. The thread enters the halt state when it is running the HLT instruction. The core frequency may change from time to time due to power or thermal throttling. For this reason, this event may have a changing ratio with regards to wall clock time.", + "SampleAfterValue": "2000003", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Fixed Counter: Counts the number of instructions retired", + "EventName": "INST_RETIRED.ANY", + "PEBS": "1", + "SampleAfterValue": "1000003", + "UMask": "0x1", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Number of instructions retired. Fixed Counter - architectural event", + "EventName": "INST_RETIRED.ANY", + "PEBS": "1", + "PublicDescription": "Counts the number of X86 instructions retired - an Architectural PerfMon event. Counting continues during hardware interrupts, traps, and inside interrupt handlers. Notes: INST_RETIRED.ANY is counted by a designated fixed counter freeing up programmable counters to count other events. INST_RETIRED.ANY_P is counted by a programmable counter.", + "SampleAfterValue": "2000003", + "UMask": "0x1", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Counts the number of instructions retired", + "EventCode": "0xc0", + "EventName": "INST_RETIRED.ANY_P", + "PEBS": "1", + "SampleAfterValue": "2000003", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Number of instructions retired. General Counter - architectural event", + "EventCode": "0xc0", + "EventName": "INST_RETIRED.ANY_P", + "PEBS": "1", + "PublicDescription": "Counts the number of X86 instructions retired - an Architectural PerfMon event. Counting continues during hardware interrupts, traps, and inside interrupt handlers. Notes: INST_RETIRED.ANY is counted by a designated fixed counter freeing up programmable counters to count other events. INST_RETIRED.ANY_P is counted by a programmable counter.", + "SampleAfterValue": "2000003", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Counts the number of occurrences a retired load gets blocked because its address partially overlaps with an older store (size mismatch) - unknown_sta/bad_forward", + "EventCode": "0x03", + "EventName": "LD_BLOCKS.STORE_FORWARD", + "PEBS": "1", + "SampleAfterValue": "1000003", + "UMask": "0x2", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Loads blocked due to overlapping with a preceding store that cannot be forwarded.", + "EventCode": "0x03", + "EventName": "LD_BLOCKS.STORE_FORWARD", + "PublicDescription": "Counts the number of times where store forwarding was prevented for a load operation. The most common case is a load blocked due to the address of memory access (partially) overlapping with a preceding uncompleted store. Note: See the table of not supported store forwards in the Optimization Guide.", + "SampleAfterValue": "100003", + "UMask": "0x82", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This event counts a subset of the Topdown Slots event that were not consumed by the back-end pipeline due to lack of back-end resources, as a result of memory subsystem delays, execution units limitations, or other conditions.", + "EventCode": "0xa4", + "EventName": "TOPDOWN.BACKEND_BOUND_SLOTS", + "PublicDescription": "This event counts a subset of the Topdown Slots event that were not consumed by the back-end pipeline due to lack of back-end resources, as a result of memory subsystem delays, execution units limitations, or other conditions.\nSoftware can use this event as the numerator for the Backend Bound metric (or top-level category) of the Top-down Microarchitecture Analysis method.", + "SampleAfterValue": "10000003", + "UMask": "0x2", + "Unit": "cpu_core" + }, + { + "BriefDescription": "TMA slots available for an unhalted logical processor. Fixed counter - architectural event", + "EventName": "TOPDOWN.SLOTS", + "PublicDescription": "Number of available slots for an unhalted logical processor. The event increments by machine-width of the narrowest pipeline as employed by the Top-down Microarchitecture Analysis method (TMA). Software can use this event as the denominator for the top-level metrics of the TMA method. This architectural event is counted on a designated fixed counter (Fixed Counter 3).", + "SampleAfterValue": "10000003", + "UMask": "0x4", + "Unit": "cpu_core" + }, + { + "BriefDescription": "TMA slots available for an unhalted logical processor. General counter - architectural event", + "EventCode": "0xa4", + "EventName": "TOPDOWN.SLOTS_P", + "PublicDescription": "Counts the number of available slots for an unhalted logical processor. The event increments by machine-width of the narrowest pipeline as employed by the Top-down Microarchitecture Analysis method.", + "SampleAfterValue": "10000003", + "UMask": "0x1", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Fixed Counter: Counts the number of issue slots that were not consumed by the backend because allocation is stalled due to a mispredicted jump or a machine clear.", + "EventName": "TOPDOWN_BAD_SPECULATION.ALL", + "PublicDescription": "Fixed Counter: Counts the number of issue slots that were not consumed by the backend because allocation is stalled due to a mispredicted jump or a machine clear. Counts all issue slots blocked during this recovery window including relevant microcode flows and while uops are not yet available in the IQ. Also, includes the issue slots that were consumed by the backend but were thrown away because they were younger than the mispredict or machine clear.", + "SampleAfterValue": "1000003", + "UMask": "0x5", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of retirement slots not consumed due to backend stalls", + "EventCode": "0xa4", + "EventName": "TOPDOWN_BE_BOUND.ALL", + "SampleAfterValue": "1000003", + "UMask": "0x2", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Fixed Counter: Counts the number of retirement slots not consumed due to front end stalls", + "EventName": "TOPDOWN_FE_BOUND.ALL", + "SampleAfterValue": "1000003", + "UMask": "0x6", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Fixed Counter: Counts the number of consumed retirement slots. Similar to UOPS_RETIRED.ALL", + "EventName": "TOPDOWN_RETIRING.ALL", + "PEBS": "1", + "SampleAfterValue": "1000003", + "UMask": "0x7", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "This event counts a subset of the Topdown Slots event that are utilized by operations that eventually get retired (committed) by the processor pipeline. Usually, this event positively correlates with higher performance for example, as measured by the instructions-per-cycle metric.", + "EventCode": "0xc2", + "EventName": "UOPS_RETIRED.SLOTS", + "PublicDescription": "This event counts a subset of the Topdown Slots event that are utilized by operations that eventually get retired (committed) by the processor pipeline. Usually, this event positively correlates with higher performance for example, as measured by the instructions-per-cycle metric.\nSoftware can use this event as the numerator for the Retiring metric (or top-level category) of the Top-down Microarchitecture Analysis method.", + "SampleAfterValue": "2000003", + "UMask": "0x2", + "Unit": "cpu_core" + } +] diff --git a/tools/perf/pmu-events/arch/x86/lunarlake/virtual-memory.json b/tools/perf/pmu-events/arch/x86/lunarlake/virtual-memory.json new file mode 100644 index 000000000000..bb9458799f1c --- /dev/null +++ b/tools/perf/pmu-events/arch/x86/lunarlake/virtual-memory.json @@ -0,0 +1,56 @@ +[ + { + "BriefDescription": "Counts the number of page walks completed due to load DTLB misses to any page size.", + "EventCode": "0x08", + "EventName": "DTLB_LOAD_MISSES.WALK_COMPLETED", + "PublicDescription": "Counts the number of page walks completed due to loads (including SW prefetches) whose address translations missed in all Translation Lookaside Buffer (TLB) levels and were mapped to any page size. Includes page walks that page fault.", + "SampleAfterValue": "200003", + "UMask": "0xe", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Load miss in all TLB levels causes a page walk that completes. (All page sizes)", + "EventCode": "0x12", + "EventName": "DTLB_LOAD_MISSES.WALK_COMPLETED", + "PublicDescription": "Counts completed page walks (all page sizes) caused by demand data loads. This implies it missed in the DTLB and further levels of TLB. The page walk can end with or without a fault.", + "SampleAfterValue": "100003", + "UMask": "0xe", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Counts the number of page walks completed due to store DTLB misses to any page size.", + "EventCode": "0x49", + "EventName": "DTLB_STORE_MISSES.WALK_COMPLETED", + "PublicDescription": "Counts the number of page walks completed due to stores whose address translations missed in all Translation Lookaside Buffer (TLB) levels and were mapped to any page size. Includes page walks that page fault.", + "SampleAfterValue": "2000003", + "UMask": "0xe", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Store misses in all TLB levels causes a page walk that completes. (All page sizes)", + "EventCode": "0x13", + "EventName": "DTLB_STORE_MISSES.WALK_COMPLETED", + "PublicDescription": "Counts completed page walks (all page sizes) caused by demand data stores. This implies it missed in the DTLB and further levels of TLB. The page walk can end with or without a fault.", + "SampleAfterValue": "100003", + "UMask": "0xe", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Counts the number of page walks completed due to instruction fetch misses to any page size.", + "EventCode": "0x85", + "EventName": "ITLB_MISSES.WALK_COMPLETED", + "PublicDescription": "Counts the number of page walks completed due to instruction fetches whose address translations missed in all Translation Lookaside Buffer (TLB) levels and were mapped to any page size. Includes page walks that page fault.", + "SampleAfterValue": "2000003", + "UMask": "0xe", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Code miss in all TLB levels causes a page walk that completes. (All page sizes)", + "EventCode": "0x11", + "EventName": "ITLB_MISSES.WALK_COMPLETED", + "PublicDescription": "Counts completed page walks (all page sizes) caused by a code fetch. This implies it missed in the ITLB (Instruction TLB) and further levels of TLB. The page walk can end with or without a fault.", + "SampleAfterValue": "100003", + "UMask": "0xe", + "Unit": "cpu_core" + } +] diff --git a/tools/perf/pmu-events/arch/x86/mapfile.csv b/tools/perf/pmu-events/arch/x86/mapfile.csv index 3a8770e29fe8..de9582f183cb 100644 --- a/tools/perf/pmu-events/arch/x86/mapfile.csv +++ b/tools/perf/pmu-events/arch/x86/mapfile.csv @@ -19,6 +19,7 @@ GenuineIntel-6-3A,v24,ivybridge,core GenuineIntel-6-3E,v23,ivytown,core GenuineIntel-6-2D,v23,jaketown,core GenuineIntel-6-(57|85),v10,knightslanding,core +GenuineIntel-6-BD,v1.00,lunarlake,core GenuineIntel-6-A[AC],v1.04,meteorlake,core GenuineIntel-6-1[AEF],v3,nehalemep,core GenuineIntel-6-2E,v3,nehalemex,core From 8c994eff8fcfe8ecb1f1dbebed25b4d7bb75be12 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Mon, 28 Aug 2023 17:17:29 -0700 Subject: [PATCH 035/475] perf vendor events intel: Add emeraldrapids, update sapphirerapids to v1.16 Add emeraldrapids events that were added at intel's perfmon site in: https://github.com/intel/perfmon/pull/98 "EMR, SPR, CLX, SKX, BDX, HSX, BDW-DE, WSM-EP*, NHM-*, JKT, IVT : Release event updates" "Emerald Rapids (0xCF) was previously pointing to SPR core. In this pull request dedicated EMR files are introduced." Signed-off-by: Ian Rogers Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Alexandre Torgue Cc: Colin Ian King Cc: Edward Baker Cc: Ingo Molnar Cc: Jiri Olsa Cc: Kan Liang Cc: Mark Rutland Cc: Maxime Coquelin Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Xing Zhengjun Link: https://lore.kernel.org/r/20230829001730.1352769-2-irogers@google.com Signed-off-by: Arnaldo Carvalho de Melo --- .../arch/x86/emeraldrapids/cache.json | 888 +++ .../x86/emeraldrapids/floating-point.json | 193 + .../arch/x86/emeraldrapids/frontend.json | 389 ++ .../arch/x86/emeraldrapids/memory.json | 343 + .../arch/x86/emeraldrapids/other.json | 315 + .../arch/x86/emeraldrapids/pipeline.json | 975 +++ .../arch/x86/emeraldrapids/uncore-cache.json | 5952 ++++++++++++++++ .../arch/x86/emeraldrapids/uncore-cxl.json | 450 ++ .../emeraldrapids/uncore-interconnect.json | 6199 +++++++++++++++++ .../arch/x86/emeraldrapids/uncore-io.json | 3587 ++++++++++ .../arch/x86/emeraldrapids/uncore-memory.json | 3308 +++++++++ .../arch/x86/emeraldrapids/uncore-power.json | 197 + .../x86/emeraldrapids/virtual-memory.json | 165 + tools/perf/pmu-events/arch/x86/mapfile.csv | 3 +- .../arch/x86/sapphirerapids/frontend.json | 41 +- .../arch/x86/sapphirerapids/uncore-cache.json | 20 +- 16 files changed, 23007 insertions(+), 18 deletions(-) create mode 100644 tools/perf/pmu-events/arch/x86/emeraldrapids/cache.json create mode 100644 tools/perf/pmu-events/arch/x86/emeraldrapids/floating-point.json create mode 100644 tools/perf/pmu-events/arch/x86/emeraldrapids/frontend.json create mode 100644 tools/perf/pmu-events/arch/x86/emeraldrapids/memory.json create mode 100644 tools/perf/pmu-events/arch/x86/emeraldrapids/other.json create mode 100644 tools/perf/pmu-events/arch/x86/emeraldrapids/pipeline.json create mode 100644 tools/perf/pmu-events/arch/x86/emeraldrapids/uncore-cache.json create mode 100644 tools/perf/pmu-events/arch/x86/emeraldrapids/uncore-cxl.json create mode 100644 tools/perf/pmu-events/arch/x86/emeraldrapids/uncore-interconnect.json create mode 100644 tools/perf/pmu-events/arch/x86/emeraldrapids/uncore-io.json create mode 100644 tools/perf/pmu-events/arch/x86/emeraldrapids/uncore-memory.json create mode 100644 tools/perf/pmu-events/arch/x86/emeraldrapids/uncore-power.json create mode 100644 tools/perf/pmu-events/arch/x86/emeraldrapids/virtual-memory.json diff --git a/tools/perf/pmu-events/arch/x86/emeraldrapids/cache.json b/tools/perf/pmu-events/arch/x86/emeraldrapids/cache.json new file mode 100644 index 000000000000..ab09bd9fb409 --- /dev/null +++ b/tools/perf/pmu-events/arch/x86/emeraldrapids/cache.json @@ -0,0 +1,888 @@ +[ + { + "BriefDescription": "L1D.HWPF_MISS", + "EventCode": "0x51", + "EventName": "L1D.HWPF_MISS", + "SampleAfterValue": "1000003", + "UMask": "0x20" + }, + { + "BriefDescription": "Counts the number of cache lines replaced in L1 data cache.", + "EventCode": "0x51", + "EventName": "L1D.REPLACEMENT", + "PublicDescription": "Counts L1D data line replacements including opportunistic replacements, and replacements that require stall-for-replace or block-for-replace.", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Number of cycles a demand request has waited due to L1D Fill Buffer (FB) unavailability.", + "EventCode": "0x48", + "EventName": "L1D_PEND_MISS.FB_FULL", + "PublicDescription": "Counts number of cycles a demand request has waited due to L1D Fill Buffer (FB) unavailability. Demand requests include cacheable/uncacheable demand load, store, lock or SW prefetch accesses.", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Number of phases a demand request has waited due to L1D Fill Buffer (FB) unavailability.", + "CounterMask": "1", + "EdgeDetect": "1", + "EventCode": "0x48", + "EventName": "L1D_PEND_MISS.FB_FULL_PERIODS", + "PublicDescription": "Counts number of phases a demand request has waited due to L1D Fill Buffer (FB) unavailability. Demand requests include cacheable/uncacheable demand load, store, lock or SW prefetch accesses.", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "This event is deprecated. Refer to new event L1D_PEND_MISS.L2_STALLS", + "Deprecated": "1", + "EventCode": "0x48", + "EventName": "L1D_PEND_MISS.L2_STALL", + "SampleAfterValue": "1000003", + "UMask": "0x4" + }, + { + "BriefDescription": "Number of cycles a demand request has waited due to L1D due to lack of L2 resources.", + "EventCode": "0x48", + "EventName": "L1D_PEND_MISS.L2_STALLS", + "PublicDescription": "Counts number of cycles a demand request has waited due to L1D due to lack of L2 resources. Demand requests include cacheable/uncacheable demand load, store, lock or SW prefetch accesses.", + "SampleAfterValue": "1000003", + "UMask": "0x4" + }, + { + "BriefDescription": "Number of L1D misses that are outstanding", + "EventCode": "0x48", + "EventName": "L1D_PEND_MISS.PENDING", + "PublicDescription": "Counts number of L1D misses that are outstanding in each cycle, that is each cycle the number of Fill Buffers (FB) outstanding required by Demand Reads. FB either is held by demand loads, or it is held by non-demand loads and gets hit at least once by demand. The valid outstanding interval is defined until the FB deallocation by one of the following ways: from FB allocation, if FB is allocated by demand from the demand Hit FB, if it is allocated by hardware or software prefetch. Note: In the L1D, a Demand Read contains cacheable or noncacheable demand loads, including ones causing cache-line splits and reads due to page walks resulted from any request type.", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Cycles with L1D load Misses outstanding.", + "CounterMask": "1", + "EventCode": "0x48", + "EventName": "L1D_PEND_MISS.PENDING_CYCLES", + "PublicDescription": "Counts duration of L1D miss outstanding in cycles.", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "L2 cache lines filling L2", + "EventCode": "0x25", + "EventName": "L2_LINES_IN.ALL", + "PublicDescription": "Counts the number of L2 cache lines filling the L2. Counting does not cover rejects.", + "SampleAfterValue": "100003", + "UMask": "0x1f" + }, + { + "BriefDescription": "L2_LINES_OUT.NON_SILENT", + "EventCode": "0x26", + "EventName": "L2_LINES_OUT.NON_SILENT", + "SampleAfterValue": "200003", + "UMask": "0x2" + }, + { + "BriefDescription": "Non-modified cache lines that are silently dropped by L2 cache when triggered by an L2 cache fill.", + "EventCode": "0x26", + "EventName": "L2_LINES_OUT.SILENT", + "PublicDescription": "Counts the number of lines that are silently dropped by L2 cache when triggered by an L2 cache fill. These lines are typically in Shared or Exclusive state. A non-threaded event.", + "SampleAfterValue": "200003", + "UMask": "0x1" + }, + { + "BriefDescription": "Cache lines that have been L2 hardware prefetched but not used by demand accesses", + "EventCode": "0x26", + "EventName": "L2_LINES_OUT.USELESS_HWPF", + "PublicDescription": "Counts the number of cache lines that have been prefetched by the L2 hardware prefetcher but not used by demand access when evicted from the L2 cache", + "SampleAfterValue": "200003", + "UMask": "0x4" + }, + { + "BriefDescription": "All accesses to L2 cache [This event is alias to L2_RQSTS.REFERENCES]", + "EventCode": "0x24", + "EventName": "L2_REQUEST.ALL", + "PublicDescription": "Counts all requests that were hit or true misses in L2 cache. True-miss excludes misses that were merged with ongoing L2 misses. [This event is alias to L2_RQSTS.REFERENCES]", + "SampleAfterValue": "200003", + "UMask": "0xff" + }, + { + "BriefDescription": "Read requests with true-miss in L2 cache. [This event is alias to L2_RQSTS.MISS]", + "EventCode": "0x24", + "EventName": "L2_REQUEST.MISS", + "PublicDescription": "Counts read requests of any type with true-miss in the L2 cache. True-miss excludes L2 misses that were merged with ongoing L2 misses. [This event is alias to L2_RQSTS.MISS]", + "SampleAfterValue": "200003", + "UMask": "0x3f" + }, + { + "BriefDescription": "L2 code requests", + "EventCode": "0x24", + "EventName": "L2_RQSTS.ALL_CODE_RD", + "PublicDescription": "Counts the total number of L2 code requests.", + "SampleAfterValue": "200003", + "UMask": "0xe4" + }, + { + "BriefDescription": "Demand Data Read access L2 cache", + "EventCode": "0x24", + "EventName": "L2_RQSTS.ALL_DEMAND_DATA_RD", + "PublicDescription": "Counts Demand Data Read requests accessing the L2 cache. These requests may hit or miss L2 cache. True-miss exclude misses that were merged with ongoing L2 misses. An access is counted once.", + "SampleAfterValue": "200003", + "UMask": "0xe1" + }, + { + "BriefDescription": "Demand requests that miss L2 cache", + "EventCode": "0x24", + "EventName": "L2_RQSTS.ALL_DEMAND_MISS", + "PublicDescription": "Counts demand requests that miss L2 cache.", + "SampleAfterValue": "200003", + "UMask": "0x27" + }, + { + "BriefDescription": "Demand requests to L2 cache", + "EventCode": "0x24", + "EventName": "L2_RQSTS.ALL_DEMAND_REFERENCES", + "PublicDescription": "Counts demand requests to L2 cache.", + "SampleAfterValue": "200003", + "UMask": "0xe7" + }, + { + "BriefDescription": "L2_RQSTS.ALL_HWPF", + "EventCode": "0x24", + "EventName": "L2_RQSTS.ALL_HWPF", + "SampleAfterValue": "200003", + "UMask": "0xf0" + }, + { + "BriefDescription": "RFO requests to L2 cache", + "EventCode": "0x24", + "EventName": "L2_RQSTS.ALL_RFO", + "PublicDescription": "Counts the total number of RFO (read for ownership) requests to L2 cache. L2 RFO requests include both L1D demand RFO misses as well as L1D RFO prefetches.", + "SampleAfterValue": "200003", + "UMask": "0xe2" + }, + { + "BriefDescription": "L2 cache hits when fetching instructions, code reads.", + "EventCode": "0x24", + "EventName": "L2_RQSTS.CODE_RD_HIT", + "PublicDescription": "Counts L2 cache hits when fetching instructions, code reads.", + "SampleAfterValue": "200003", + "UMask": "0xc4" + }, + { + "BriefDescription": "L2 cache misses when fetching instructions", + "EventCode": "0x24", + "EventName": "L2_RQSTS.CODE_RD_MISS", + "PublicDescription": "Counts L2 cache misses when fetching instructions.", + "SampleAfterValue": "200003", + "UMask": "0x24" + }, + { + "BriefDescription": "Demand Data Read requests that hit L2 cache", + "EventCode": "0x24", + "EventName": "L2_RQSTS.DEMAND_DATA_RD_HIT", + "PublicDescription": "Counts the number of demand Data Read requests initiated by load instructions that hit L2 cache.", + "SampleAfterValue": "200003", + "UMask": "0xc1" + }, + { + "BriefDescription": "Demand Data Read miss L2 cache", + "EventCode": "0x24", + "EventName": "L2_RQSTS.DEMAND_DATA_RD_MISS", + "PublicDescription": "Counts demand Data Read requests with true-miss in the L2 cache. True-miss excludes misses that were merged with ongoing L2 misses. An access is counted once.", + "SampleAfterValue": "200003", + "UMask": "0x21" + }, + { + "BriefDescription": "L2_RQSTS.HWPF_MISS", + "EventCode": "0x24", + "EventName": "L2_RQSTS.HWPF_MISS", + "SampleAfterValue": "200003", + "UMask": "0x30" + }, + { + "BriefDescription": "Read requests with true-miss in L2 cache. [This event is alias to L2_REQUEST.MISS]", + "EventCode": "0x24", + "EventName": "L2_RQSTS.MISS", + "PublicDescription": "Counts read requests of any type with true-miss in the L2 cache. True-miss excludes L2 misses that were merged with ongoing L2 misses. [This event is alias to L2_REQUEST.MISS]", + "SampleAfterValue": "200003", + "UMask": "0x3f" + }, + { + "BriefDescription": "All accesses to L2 cache [This event is alias to L2_REQUEST.ALL]", + "EventCode": "0x24", + "EventName": "L2_RQSTS.REFERENCES", + "PublicDescription": "Counts all requests that were hit or true misses in L2 cache. True-miss excludes misses that were merged with ongoing L2 misses. [This event is alias to L2_REQUEST.ALL]", + "SampleAfterValue": "200003", + "UMask": "0xff" + }, + { + "BriefDescription": "RFO requests that hit L2 cache", + "EventCode": "0x24", + "EventName": "L2_RQSTS.RFO_HIT", + "PublicDescription": "Counts the RFO (Read-for-Ownership) requests that hit L2 cache.", + "SampleAfterValue": "200003", + "UMask": "0xc2" + }, + { + "BriefDescription": "RFO requests that miss L2 cache", + "EventCode": "0x24", + "EventName": "L2_RQSTS.RFO_MISS", + "PublicDescription": "Counts the RFO (Read-for-Ownership) requests that miss L2 cache.", + "SampleAfterValue": "200003", + "UMask": "0x22" + }, + { + "BriefDescription": "SW prefetch requests that hit L2 cache.", + "EventCode": "0x24", + "EventName": "L2_RQSTS.SWPF_HIT", + "PublicDescription": "Counts Software prefetch requests that hit the L2 cache. Accounts for PREFETCHNTA and PREFETCHT0/1/2 instructions when FB is not full.", + "SampleAfterValue": "200003", + "UMask": "0xc8" + }, + { + "BriefDescription": "SW prefetch requests that miss L2 cache.", + "EventCode": "0x24", + "EventName": "L2_RQSTS.SWPF_MISS", + "PublicDescription": "Counts Software prefetch requests that miss the L2 cache. Accounts for PREFETCHNTA and PREFETCHT0/1/2 instructions when FB is not full.", + "SampleAfterValue": "200003", + "UMask": "0x28" + }, + { + "BriefDescription": "Core-originated cacheable requests that missed L3 (Except hardware prefetches to the L3)", + "EventCode": "0x2e", + "EventName": "LONGEST_LAT_CACHE.MISS", + "PublicDescription": "Counts core-originated cacheable requests that miss the L3 cache (Longest Latency cache). Requests include data and code reads, Reads-for-Ownership (RFOs), speculative accesses and hardware prefetches to the L1 and L2. It does not include hardware prefetches to the L3, and may not count other types of requests to the L3.", + "SampleAfterValue": "100003", + "UMask": "0x41" + }, + { + "BriefDescription": "Core-originated cacheable requests that refer to L3 (Except hardware prefetches to the L3)", + "EventCode": "0x2e", + "EventName": "LONGEST_LAT_CACHE.REFERENCE", + "PublicDescription": "Counts core-originated cacheable requests to the L3 cache (Longest Latency cache). Requests include data and code reads, Reads-for-Ownership (RFOs), speculative accesses and hardware prefetches to the L1 and L2. It does not include hardware prefetches to the L3, and may not count other types of requests to the L3.", + "SampleAfterValue": "100003", + "UMask": "0x4f" + }, + { + "BriefDescription": "Retired load instructions.", + "Data_LA": "1", + "EventCode": "0xd0", + "EventName": "MEM_INST_RETIRED.ALL_LOADS", + "PEBS": "1", + "PublicDescription": "Counts all retired load instructions. This event accounts for SW prefetch instructions of PREFETCHNTA or PREFETCHT0/1/2 or PREFETCHW.", + "SampleAfterValue": "1000003", + "UMask": "0x81" + }, + { + "BriefDescription": "Retired store instructions.", + "Data_LA": "1", + "EventCode": "0xd0", + "EventName": "MEM_INST_RETIRED.ALL_STORES", + "PEBS": "1", + "PublicDescription": "Counts all retired store instructions.", + "SampleAfterValue": "1000003", + "UMask": "0x82" + }, + { + "BriefDescription": "All retired memory instructions.", + "Data_LA": "1", + "EventCode": "0xd0", + "EventName": "MEM_INST_RETIRED.ANY", + "PEBS": "1", + "PublicDescription": "Counts all retired memory instructions - loads and stores.", + "SampleAfterValue": "1000003", + "UMask": "0x83" + }, + { + "BriefDescription": "Retired load instructions with locked access.", + "Data_LA": "1", + "EventCode": "0xd0", + "EventName": "MEM_INST_RETIRED.LOCK_LOADS", + "PEBS": "1", + "PublicDescription": "Counts retired load instructions with locked access.", + "SampleAfterValue": "100007", + "UMask": "0x21" + }, + { + "BriefDescription": "Retired load instructions that split across a cacheline boundary.", + "Data_LA": "1", + "EventCode": "0xd0", + "EventName": "MEM_INST_RETIRED.SPLIT_LOADS", + "PEBS": "1", + "PublicDescription": "Counts retired load instructions that split across a cacheline boundary.", + "SampleAfterValue": "100003", + "UMask": "0x41" + }, + { + "BriefDescription": "Retired store instructions that split across a cacheline boundary.", + "Data_LA": "1", + "EventCode": "0xd0", + "EventName": "MEM_INST_RETIRED.SPLIT_STORES", + "PEBS": "1", + "PublicDescription": "Counts retired store instructions that split across a cacheline boundary.", + "SampleAfterValue": "100003", + "UMask": "0x42" + }, + { + "BriefDescription": "Retired load instructions that miss the STLB.", + "Data_LA": "1", + "EventCode": "0xd0", + "EventName": "MEM_INST_RETIRED.STLB_MISS_LOADS", + "PEBS": "1", + "PublicDescription": "Number of retired load instructions that (start a) miss in the 2nd-level TLB (STLB).", + "SampleAfterValue": "100003", + "UMask": "0x11" + }, + { + "BriefDescription": "Retired store instructions that miss the STLB.", + "Data_LA": "1", + "EventCode": "0xd0", + "EventName": "MEM_INST_RETIRED.STLB_MISS_STORES", + "PEBS": "1", + "PublicDescription": "Number of retired store instructions that (start a) miss in the 2nd-level TLB (STLB).", + "SampleAfterValue": "100003", + "UMask": "0x12" + }, + { + "BriefDescription": "Completed demand load uops that miss the L1 d-cache.", + "EventCode": "0x43", + "EventName": "MEM_LOAD_COMPLETED.L1_MISS_ANY", + "PublicDescription": "Number of completed demand load requests that missed the L1 data cache including shadow misses (FB hits, merge to an ongoing L1D miss)", + "SampleAfterValue": "1000003", + "UMask": "0xfd" + }, + { + "BriefDescription": "Retired load instructions whose data sources were HitM responses from shared L3", + "Data_LA": "1", + "EventCode": "0xd2", + "EventName": "MEM_LOAD_L3_HIT_RETIRED.XSNP_FWD", + "PEBS": "1", + "PublicDescription": "Counts retired load instructions whose data sources were HitM responses from shared L3.", + "SampleAfterValue": "20011", + "UMask": "0x4" + }, + { + "BriefDescription": "Retired load instructions whose data sources were L3 hit and cross-core snoop missed in on-pkg core cache.", + "Data_LA": "1", + "EventCode": "0xd2", + "EventName": "MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS", + "PEBS": "1", + "PublicDescription": "Counts the retired load instructions whose data sources were L3 hit and cross-core snoop missed in on-pkg core cache.", + "SampleAfterValue": "20011", + "UMask": "0x1" + }, + { + "BriefDescription": "Retired load instructions whose data sources were hits in L3 without snoops required", + "Data_LA": "1", + "EventCode": "0xd2", + "EventName": "MEM_LOAD_L3_HIT_RETIRED.XSNP_NONE", + "PEBS": "1", + "PublicDescription": "Counts retired load instructions whose data sources were hits in L3 without snoops required.", + "SampleAfterValue": "100003", + "UMask": "0x8" + }, + { + "BriefDescription": "Retired load instructions whose data sources were L3 and cross-core snoop hits in on-pkg core cache", + "Data_LA": "1", + "EventCode": "0xd2", + "EventName": "MEM_LOAD_L3_HIT_RETIRED.XSNP_NO_FWD", + "PEBS": "1", + "PublicDescription": "Counts retired load instructions whose data sources were L3 and cross-core snoop hits in on-pkg core cache.", + "SampleAfterValue": "20011", + "UMask": "0x2" + }, + { + "BriefDescription": "Retired load instructions which data sources missed L3 but serviced from local dram", + "Data_LA": "1", + "EventCode": "0xd3", + "EventName": "MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM", + "PEBS": "1", + "PublicDescription": "Retired load instructions which data sources missed L3 but serviced from local DRAM.", + "SampleAfterValue": "100007", + "UMask": "0x1" + }, + { + "BriefDescription": "MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM", + "Data_LA": "1", + "EventCode": "0xd3", + "EventName": "MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM", + "PEBS": "1", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Retired load instructions whose data sources was forwarded from a remote cache", + "Data_LA": "1", + "EventCode": "0xd3", + "EventName": "MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD", + "PEBS": "1", + "PublicDescription": "Retired load instructions whose data sources was forwarded from a remote cache.", + "SampleAfterValue": "100007", + "UMask": "0x8" + }, + { + "BriefDescription": "MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM", + "Data_LA": "1", + "EventCode": "0xd3", + "EventName": "MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM", + "PEBS": "1", + "SampleAfterValue": "1000003", + "UMask": "0x4" + }, + { + "BriefDescription": "Retired instructions with at least 1 uncacheable load or lock.", + "Data_LA": "1", + "EventCode": "0xd4", + "EventName": "MEM_LOAD_MISC_RETIRED.UC", + "PEBS": "1", + "PublicDescription": "Retired instructions with at least one load to uncacheable memory-type, or at least one cache-line split locked access (Bus Lock).", + "SampleAfterValue": "100007", + "UMask": "0x4" + }, + { + "BriefDescription": "Number of completed demand load requests that missed the L1, but hit the FB(fill buffer), because a preceding miss to the same cacheline initiated the line to be brought into L1, but data is not yet ready in L1.", + "Data_LA": "1", + "EventCode": "0xd1", + "EventName": "MEM_LOAD_RETIRED.FB_HIT", + "PEBS": "1", + "PublicDescription": "Counts retired load instructions with at least one uop was load missed in L1 but hit FB (Fill Buffers) due to preceding miss to the same cache line with data not ready.", + "SampleAfterValue": "100007", + "UMask": "0x40" + }, + { + "BriefDescription": "Retired load instructions with L1 cache hits as data sources", + "Data_LA": "1", + "EventCode": "0xd1", + "EventName": "MEM_LOAD_RETIRED.L1_HIT", + "PEBS": "1", + "PublicDescription": "Counts retired load instructions with at least one uop that hit in the L1 data cache. This event includes all SW prefetches and lock instructions regardless of the data source.", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Retired load instructions missed L1 cache as data sources", + "Data_LA": "1", + "EventCode": "0xd1", + "EventName": "MEM_LOAD_RETIRED.L1_MISS", + "PEBS": "1", + "PublicDescription": "Counts retired load instructions with at least one uop that missed in the L1 cache.", + "SampleAfterValue": "200003", + "UMask": "0x8" + }, + { + "BriefDescription": "Retired load instructions with L2 cache hits as data sources", + "Data_LA": "1", + "EventCode": "0xd1", + "EventName": "MEM_LOAD_RETIRED.L2_HIT", + "PEBS": "1", + "PublicDescription": "Counts retired load instructions with L2 cache hits as data sources.", + "SampleAfterValue": "200003", + "UMask": "0x2" + }, + { + "BriefDescription": "Retired load instructions missed L2 cache as data sources", + "Data_LA": "1", + "EventCode": "0xd1", + "EventName": "MEM_LOAD_RETIRED.L2_MISS", + "PEBS": "1", + "PublicDescription": "Counts retired load instructions missed L2 cache as data sources.", + "SampleAfterValue": "100021", + "UMask": "0x10" + }, + { + "BriefDescription": "Retired load instructions with L3 cache hits as data sources", + "Data_LA": "1", + "EventCode": "0xd1", + "EventName": "MEM_LOAD_RETIRED.L3_HIT", + "PEBS": "1", + "PublicDescription": "Counts retired load instructions with at least one uop that hit in the L3 cache.", + "SampleAfterValue": "100021", + "UMask": "0x4" + }, + { + "BriefDescription": "Retired load instructions missed L3 cache as data sources", + "Data_LA": "1", + "EventCode": "0xd1", + "EventName": "MEM_LOAD_RETIRED.L3_MISS", + "PEBS": "1", + "PublicDescription": "Counts retired load instructions with at least one uop that missed in the L3 cache.", + "SampleAfterValue": "50021", + "UMask": "0x20" + }, + { + "BriefDescription": "MEM_STORE_RETIRED.L2_HIT", + "EventCode": "0x44", + "EventName": "MEM_STORE_RETIRED.L2_HIT", + "SampleAfterValue": "200003", + "UMask": "0x1" + }, + { + "BriefDescription": "Retired memory uops for any access", + "EventCode": "0xe5", + "EventName": "MEM_UOP_RETIRED.ANY", + "PublicDescription": "Number of retired micro-operations (uops) for load or store memory accesses", + "SampleAfterValue": "1000003", + "UMask": "0x3" + }, + { + "BriefDescription": "Counts demand instruction fetches and L1 instruction cache prefetches that hit in the L3 or were snooped from another core's caches on the same socket.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_CODE_RD.L3_HIT", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x3F803C0004", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts demand instruction fetches and L1 instruction cache prefetches that resulted in a snoop hit a modified line in another core's caches which forwarded the data.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_CODE_RD.L3_HIT.SNOOP_HITM", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x10003C0004", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts demand instruction fetches and L1 instruction cache prefetches that hit a modified line in a distant L3 Cache or were snooped from a distant core's L1/L2 caches on this socket when the system is in SNC (sub-NUMA cluster) mode.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_CODE_RD.SNC_CACHE.HITM", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x1008000004", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts demand instruction fetches and L1 instruction cache prefetches that either hit a non-modified line in a distant L3 Cache or were snooped from a distant core's L1/L2 caches on this socket when the system is in SNC (sub-NUMA cluster) mode.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_CODE_RD.SNC_CACHE.HIT_WITH_FWD", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x808000004", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts demand data reads that hit in the L3 or were snooped from another core's caches on the same socket.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_DATA_RD.L3_HIT", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x3F803C0001", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts demand data reads that resulted in a snoop hit a modified line in another core's caches which forwarded the data.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x10003C0001", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts demand data reads that resulted in a snoop that hit in another core, which did not forward the data.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HIT_NO_FWD", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x4003C0001", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts demand data reads that resulted in a snoop hit in another core's caches which forwarded the unmodified data to the requesting core.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HIT_WITH_FWD", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x8003C0001", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts demand data reads that were supplied by a cache on a remote socket where a snoop hit a modified line in another core's caches which forwarded the data.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_DATA_RD.REMOTE_CACHE.SNOOP_HITM", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x1030000001", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts demand data reads that were supplied by a cache on a remote socket where a snoop hit in another core's caches which forwarded the unmodified data to the requesting core.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_DATA_RD.REMOTE_CACHE.SNOOP_HIT_WITH_FWD", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x830000001", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts demand data reads that hit a modified line in a distant L3 Cache or were snooped from a distant core's L1/L2 caches on this socket when the system is in SNC (sub-NUMA cluster) mode.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_DATA_RD.SNC_CACHE.HITM", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x1008000001", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts demand data reads that either hit a non-modified line in a distant L3 Cache or were snooped from a distant core's L1/L2 caches on this socket when the system is in SNC (sub-NUMA cluster) mode.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_DATA_RD.SNC_CACHE.HIT_WITH_FWD", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x808000001", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts demand reads for ownership (RFO) requests and software prefetches for exclusive ownership (PREFETCHW) that hit in the L3 or were snooped from another core's caches on the same socket.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_RFO.L3_HIT", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x3F803C0002", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts demand reads for ownership (RFO) requests and software prefetches for exclusive ownership (PREFETCHW) that resulted in a snoop hit a modified line in another core's caches which forwarded the data.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_RFO.L3_HIT.SNOOP_HITM", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x10003C0002", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts demand reads for ownership (RFO) requests and software prefetches for exclusive ownership (PREFETCHW) that hit a modified line in a distant L3 Cache or were snooped from a distant core's L1/L2 caches on this socket when the system is in SNC (sub-NUMA cluster) mode.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_RFO.SNC_CACHE.HITM", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x1008000002", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts demand reads for ownership (RFO) requests and software prefetches for exclusive ownership (PREFETCHW) that either hit a non-modified line in a distant L3 Cache or were snooped from a distant core's L1/L2 caches on this socket when the system is in SNC (sub-NUMA cluster) mode.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_RFO.SNC_CACHE.HIT_WITH_FWD", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x808000002", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts hardware prefetches to the L3 only that hit in the L3 or were snooped from another core's caches on the same socket.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.HWPF_L3.L3_HIT", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x80082380", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts all (cacheable) data read, code read and RFO requests including demands and prefetches to the core caches (L1 or L2) that hit in the L3 or were snooped from another core's caches on the same socket.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.READS_TO_CORE.L3_HIT", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x3F003C4477", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts all (cacheable) data read, code read and RFO requests including demands and prefetches to the core caches (L1 or L2) that resulted in a snoop hit a modified line in another core's caches which forwarded the data.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.READS_TO_CORE.L3_HIT.SNOOP_HITM", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x10003C4477", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts all (cacheable) data read, code read and RFO requests including demands and prefetches to the core caches (L1 or L2) that resulted in a snoop that hit in another core, which did not forward the data.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.READS_TO_CORE.L3_HIT.SNOOP_HIT_NO_FWD", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x4003C4477", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts all (cacheable) data read, code read and RFO requests including demands and prefetches to the core caches (L1 or L2) that resulted in a snoop hit in another core's caches which forwarded the unmodified data to the requesting core.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.READS_TO_CORE.L3_HIT.SNOOP_HIT_WITH_FWD", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x8003C4477", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts all (cacheable) data read, code read and RFO requests including demands and prefetches to the core caches (L1 or L2) that were supplied by a cache on a remote socket where a snoop was sent and data was returned (Modified or Not Modified).", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.READS_TO_CORE.REMOTE_CACHE.SNOOP_FWD", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x1830004477", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts all (cacheable) data read, code read and RFO requests including demands and prefetches to the core caches (L1 or L2) that were supplied by a cache on a remote socket where a snoop hit a modified line in another core's caches which forwarded the data.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.READS_TO_CORE.REMOTE_CACHE.SNOOP_HITM", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x1030004477", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts all (cacheable) data read, code read and RFO requests including demands and prefetches to the core caches (L1 or L2) that were supplied by a cache on a remote socket where a snoop hit in another core's caches which forwarded the unmodified data to the requesting core.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.READS_TO_CORE.REMOTE_CACHE.SNOOP_HIT_WITH_FWD", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x830004477", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts all (cacheable) data read, code read and RFO requests including demands and prefetches to the core caches (L1 or L2) that hit a modified line in a distant L3 Cache or were snooped from a distant core's L1/L2 caches on this socket when the system is in SNC (sub-NUMA cluster) mode.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.READS_TO_CORE.SNC_CACHE.HITM", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x1008004477", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts all (cacheable) data read, code read and RFO requests including demands and prefetches to the core caches (L1 or L2) that either hit a non-modified line in a distant L3 Cache or were snooped from a distant core's L1/L2 caches on this socket when the system is in SNC (sub-NUMA cluster) mode.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.READS_TO_CORE.SNC_CACHE.HIT_WITH_FWD", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x808004477", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts demand reads for ownership (RFO), hardware prefetch RFOs (which bring data to L2), and software prefetches for exclusive ownership (PREFETCHW) that hit to a (M)odified cacheline in the L3 or snoop filter.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.RFO_TO_CORE.L3_HIT_M", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x1F80040022", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts streaming stores that hit in the L3 or were snooped from another core's caches on the same socket.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.STREAMING_WR.L3_HIT", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x80080800", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "OFFCORE_REQUESTS.ALL_REQUESTS", + "EventCode": "0x21", + "EventName": "OFFCORE_REQUESTS.ALL_REQUESTS", + "SampleAfterValue": "100003", + "UMask": "0x80" + }, + { + "BriefDescription": "Demand and prefetch data reads", + "EventCode": "0x21", + "EventName": "OFFCORE_REQUESTS.DATA_RD", + "PublicDescription": "Counts the demand and prefetch data reads. All Core Data Reads include cacheable 'Demands' and L2 prefetchers (not L3 prefetchers). Counting also covers reads due to page walks resulted from any request type.", + "SampleAfterValue": "100003", + "UMask": "0x8" + }, + { + "BriefDescription": "Demand Data Read requests sent to uncore", + "EventCode": "0x21", + "EventName": "OFFCORE_REQUESTS.DEMAND_DATA_RD", + "PublicDescription": "Counts the Demand Data Read requests sent to uncore. Use it in conjunction with OFFCORE_REQUESTS_OUTSTANDING to determine average latency in the uncore.", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "This event is deprecated. Refer to new event OFFCORE_REQUESTS_OUTSTANDING.DATA_RD", + "Deprecated": "1", + "EventCode": "0x20", + "EventName": "OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD", + "SampleAfterValue": "1000003", + "UMask": "0x8" + }, + { + "BriefDescription": "OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD", + "CounterMask": "1", + "EventCode": "0x20", + "EventName": "OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD", + "SampleAfterValue": "1000003", + "UMask": "0x8" + }, + { + "BriefDescription": "Cycles where at least 1 outstanding demand data read request is pending.", + "CounterMask": "1", + "EventCode": "0x20", + "EventName": "OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_DATA_RD", + "SampleAfterValue": "2000003", + "UMask": "0x1" + }, + { + "BriefDescription": "OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO", + "CounterMask": "1", + "EventCode": "0x20", + "EventName": "OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO", + "SampleAfterValue": "1000003", + "UMask": "0x4" + }, + { + "BriefDescription": "OFFCORE_REQUESTS_OUTSTANDING.DATA_RD", + "EventCode": "0x20", + "EventName": "OFFCORE_REQUESTS_OUTSTANDING.DATA_RD", + "SampleAfterValue": "1000003", + "UMask": "0x8" + }, + { + "BriefDescription": "For every cycle, increments by the number of outstanding demand data read requests pending.", + "EventCode": "0x20", + "EventName": "OFFCORE_REQUESTS_OUTSTANDING.DEMAND_DATA_RD", + "PublicDescription": "For every cycle, increments by the number of outstanding demand data read requests pending. Requests are considered outstanding from the time they miss the core's L2 cache until the transaction completion message is sent to the requestor.", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts bus locks, accounts for cache line split locks and UC locks.", + "EventCode": "0x2c", + "EventName": "SQ_MISC.BUS_LOCK", + "PublicDescription": "Counts the more expensive bus lock needed to enforce cache coherency for certain memory accesses that need to be done atomically. Can be created by issuing an atomic instruction (via the LOCK prefix) which causes a cache line split or accesses uncacheable memory.", + "SampleAfterValue": "100003", + "UMask": "0x10" + }, + { + "BriefDescription": "Number of PREFETCHNTA instructions executed.", + "EventCode": "0x40", + "EventName": "SW_PREFETCH_ACCESS.NTA", + "PublicDescription": "Counts the number of PREFETCHNTA instructions executed.", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Number of PREFETCHW instructions executed.", + "EventCode": "0x40", + "EventName": "SW_PREFETCH_ACCESS.PREFETCHW", + "PublicDescription": "Counts the number of PREFETCHW instructions executed.", + "SampleAfterValue": "100003", + "UMask": "0x8" + }, + { + "BriefDescription": "Number of PREFETCHT0 instructions executed.", + "EventCode": "0x40", + "EventName": "SW_PREFETCH_ACCESS.T0", + "PublicDescription": "Counts the number of PREFETCHT0 instructions executed.", + "SampleAfterValue": "100003", + "UMask": "0x2" + }, + { + "BriefDescription": "Number of PREFETCHT1 or PREFETCHT2 instructions executed.", + "EventCode": "0x40", + "EventName": "SW_PREFETCH_ACCESS.T1_T2", + "PublicDescription": "Counts the number of PREFETCHT1 or PREFETCHT2 instructions executed.", + "SampleAfterValue": "100003", + "UMask": "0x4" + } +] diff --git a/tools/perf/pmu-events/arch/x86/emeraldrapids/floating-point.json b/tools/perf/pmu-events/arch/x86/emeraldrapids/floating-point.json new file mode 100644 index 000000000000..4a9d211e9d4f --- /dev/null +++ b/tools/perf/pmu-events/arch/x86/emeraldrapids/floating-point.json @@ -0,0 +1,193 @@ +[ + { + "BriefDescription": "ARITH.FPDIV_ACTIVE", + "CounterMask": "1", + "EventCode": "0xb0", + "EventName": "ARITH.FPDIV_ACTIVE", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts all microcode FP assists.", + "EventCode": "0xc1", + "EventName": "ASSISTS.FP", + "PublicDescription": "Counts all microcode Floating Point assists.", + "SampleAfterValue": "100003", + "UMask": "0x2" + }, + { + "BriefDescription": "ASSISTS.SSE_AVX_MIX", + "EventCode": "0xc1", + "EventName": "ASSISTS.SSE_AVX_MIX", + "SampleAfterValue": "1000003", + "UMask": "0x10" + }, + { + "BriefDescription": "FP_ARITH_DISPATCHED.PORT_0", + "EventCode": "0xb3", + "EventName": "FP_ARITH_DISPATCHED.PORT_0", + "SampleAfterValue": "2000003", + "UMask": "0x1" + }, + { + "BriefDescription": "FP_ARITH_DISPATCHED.PORT_1", + "EventCode": "0xb3", + "EventName": "FP_ARITH_DISPATCHED.PORT_1", + "SampleAfterValue": "2000003", + "UMask": "0x2" + }, + { + "BriefDescription": "FP_ARITH_DISPATCHED.PORT_5", + "EventCode": "0xb3", + "EventName": "FP_ARITH_DISPATCHED.PORT_5", + "SampleAfterValue": "2000003", + "UMask": "0x4" + }, + { + "BriefDescription": "Counts number of SSE/AVX computational 128-bit packed double precision floating-point instructions retired; some instructions will count twice as noted below. Each count represents 2 computation operations, one for each element. Applies to SSE* and AVX* packed double precision floating-point instructions: ADD SUB HADD HSUB SUBADD MUL DIV MIN MAX SQRT DPP FM(N)ADD/SUB. DPP and FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element.", + "EventCode": "0xc7", + "EventName": "FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE", + "PublicDescription": "Number of SSE/AVX computational 128-bit packed double precision floating-point instructions retired; some instructions will count twice as noted below. Each count represents 2 computation operations, one for each element. Applies to SSE* and AVX* packed double precision floating-point instructions: ADD SUB HADD HSUB SUBADD MUL DIV MIN MAX SQRT DPP FM(N)ADD/SUB. DPP and FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element. The DAZ and FTZ flags in the MXCSR register need to be set when using these events.", + "SampleAfterValue": "100003", + "UMask": "0x4" + }, + { + "BriefDescription": "Number of SSE/AVX computational 128-bit packed single precision floating-point instructions retired; some instructions will count twice as noted below. Each count represents 4 computation operations, one for each element. Applies to SSE* and AVX* packed single precision floating-point instructions: ADD SUB MUL DIV MIN MAX RCP14 RSQRT14 SQRT DPP FM(N)ADD/SUB. DPP and FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element.", + "EventCode": "0xc7", + "EventName": "FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE", + "PublicDescription": "Number of SSE/AVX computational 128-bit packed single precision floating-point instructions retired; some instructions will count twice as noted below. Each count represents 4 computation operations, one for each element. Applies to SSE* and AVX* packed single precision floating-point instructions: ADD SUB HADD HSUB SUBADD MUL DIV MIN MAX SQRT RSQRT RCP DPP FM(N)ADD/SUB. DPP and FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element. The DAZ and FTZ flags in the MXCSR register need to be set when using these events.", + "SampleAfterValue": "100003", + "UMask": "0x8" + }, + { + "BriefDescription": "Counts number of SSE/AVX computational 256-bit packed double precision floating-point instructions retired; some instructions will count twice as noted below. Each count represents 4 computation operations, one for each element. Applies to SSE* and AVX* packed double precision floating-point instructions: ADD SUB HADD HSUB SUBADD MUL DIV MIN MAX SQRT FM(N)ADD/SUB. FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element.", + "EventCode": "0xc7", + "EventName": "FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE", + "PublicDescription": "Number of SSE/AVX computational 256-bit packed double precision floating-point instructions retired; some instructions will count twice as noted below. Each count represents 4 computation operations, one for each element. Applies to SSE* and AVX* packed double precision floating-point instructions: ADD SUB HADD HSUB SUBADD MUL DIV MIN MAX SQRT FM(N)ADD/SUB. FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element. The DAZ and FTZ flags in the MXCSR register need to be set when using these events.", + "SampleAfterValue": "100003", + "UMask": "0x10" + }, + { + "BriefDescription": "Counts number of SSE/AVX computational 256-bit packed single precision floating-point instructions retired; some instructions will count twice as noted below. Each count represents 8 computation operations, one for each element. Applies to SSE* and AVX* packed single precision floating-point instructions: ADD SUB HADD HSUB SUBADD MUL DIV MIN MAX SQRT RSQRT RCP DPP FM(N)ADD/SUB. DPP and FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element.", + "EventCode": "0xc7", + "EventName": "FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE", + "PublicDescription": "Number of SSE/AVX computational 256-bit packed single precision floating-point instructions retired; some instructions will count twice as noted below. Each count represents 8 computation operations, one for each element. Applies to SSE* and AVX* packed single precision floating-point instructions: ADD SUB HADD HSUB SUBADD MUL DIV MIN MAX SQRT RSQRT RCP DPP FM(N)ADD/SUB. DPP and FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element. The DAZ and FTZ flags in the MXCSR register need to be set when using these events.", + "SampleAfterValue": "100003", + "UMask": "0x20" + }, + { + "BriefDescription": "Number of SSE/AVX computational 128-bit packed single and 256-bit packed double precision FP instructions retired; some instructions will count twice as noted below. Each count represents 2 or/and 4 computation operations, 1 for each element. Applies to SSE* and AVX* packed single precision and packed double precision FP instructions: ADD SUB HADD HSUB SUBADD MUL DIV MIN MAX RCP14 RSQRT14 SQRT DPP FM(N)ADD/SUB. DPP and FM(N)ADD/SUB count twice as they perform 2 calculations per element.", + "EventCode": "0xc7", + "EventName": "FP_ARITH_INST_RETIRED.4_FLOPS", + "PublicDescription": "Number of SSE/AVX computational 128-bit packed single precision and 256-bit packed double precision floating-point instructions retired; some instructions will count twice as noted below. Each count represents 2 or/and 4 computation operations, one for each element. Applies to SSE* and AVX* packed single precision floating-point and packed double precision floating-point instructions: ADD SUB HADD HSUB SUBADD MUL DIV MIN MAX RCP14 RSQRT14 SQRT DPP FM(N)ADD/SUB. DPP and FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element. The DAZ and FTZ flags in the MXCSR register need to be set when using these events.", + "SampleAfterValue": "100003", + "UMask": "0x18" + }, + { + "BriefDescription": "Counts number of SSE/AVX computational 512-bit packed double precision floating-point instructions retired; some instructions will count twice as noted below. Each count represents 8 computation operations, one for each element. Applies to SSE* and AVX* packed double precision floating-point instructions: ADD SUB MUL DIV MIN MAX SQRT RSQRT14 RCP14 FM(N)ADD/SUB. FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element.", + "EventCode": "0xc7", + "EventName": "FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE", + "PublicDescription": "Number of SSE/AVX computational 512-bit packed double precision floating-point instructions retired; some instructions will count twice as noted below. Each count represents 8 computation operations, one for each element. Applies to SSE* and AVX* packed double precision floating-point instructions: ADD SUB MUL DIV MIN MAX SQRT RSQRT14 RCP14 FM(N)ADD/SUB. FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element. The DAZ and FTZ flags in the MXCSR register need to be set when using these events.", + "SampleAfterValue": "100003", + "UMask": "0x40" + }, + { + "BriefDescription": "Counts number of SSE/AVX computational 512-bit packed single precision floating-point instructions retired; some instructions will count twice as noted below. Each count represents 16 computation operations, one for each element. Applies to SSE* and AVX* packed single precision floating-point instructions: ADD SUB MUL DIV MIN MAX SQRT RSQRT14 RCP14 FM(N)ADD/SUB. FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element.", + "EventCode": "0xc7", + "EventName": "FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE", + "PublicDescription": "Number of SSE/AVX computational 512-bit packed single precision floating-point instructions retired; some instructions will count twice as noted below. Each count represents 16 computation operations, one for each element. Applies to SSE* and AVX* packed single precision floating-point instructions: ADD SUB MUL DIV MIN MAX SQRT RSQRT14 RCP14 FM(N)ADD/SUB. FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element. The DAZ and FTZ flags in the MXCSR register need to be set when using these events.", + "SampleAfterValue": "100003", + "UMask": "0x80" + }, + { + "BriefDescription": "Number of SSE/AVX computational 256-bit packed single precision and 512-bit packed double precision FP instructions retired; some instructions will count twice as noted below. Each count represents 8 computation operations, 1 for each element. Applies to SSE* and AVX* packed single precision and double precision FP instructions: ADD SUB HADD HSUB SUBADD MUL DIV MIN MAX SQRT RSQRT RSQRT14 RCP RCP14 DPP FM(N)ADD/SUB. DPP and FM(N)ADD/SUB count twice as they perform 2 calculations per element.", + "EventCode": "0xc7", + "EventName": "FP_ARITH_INST_RETIRED.8_FLOPS", + "PublicDescription": "Number of SSE/AVX computational 256-bit packed single precision and 512-bit packed double precision floating-point instructions retired; some instructions will count twice as noted below. Each count represents 8 computation operations, one for each element. Applies to SSE* and AVX* packed single precision and double precision floating-point instructions: ADD SUB HADD HSUB SUBADD MUL DIV MIN MAX SQRT RSQRT RSQRT14 RCP RCP14 DPP FM(N)ADD/SUB. DPP and FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element. The DAZ and FTZ flags in the MXCSR register need to be set when using these events.", + "SampleAfterValue": "100003", + "UMask": "0x60" + }, + { + "BriefDescription": "Number of SSE/AVX computational scalar floating-point instructions retired; some instructions will count twice as noted below. Applies to SSE* and AVX* scalar, double and single precision floating-point: ADD SUB MUL DIV MIN MAX RCP14 RSQRT14 RANGE SQRT DPP FM(N)ADD/SUB. DPP and FM(N)ADD/SUB instructions count twice as they perform multiple calculations per element.", + "EventCode": "0xc7", + "EventName": "FP_ARITH_INST_RETIRED.SCALAR", + "PublicDescription": "Number of SSE/AVX computational scalar single precision and double precision floating-point instructions retired; some instructions will count twice as noted below. Each count represents 1 computational operation. Applies to SSE* and AVX* scalar single precision floating-point instructions: ADD SUB MUL DIV MIN MAX SQRT RSQRT RCP FM(N)ADD/SUB. FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element. The DAZ and FTZ flags in the MXCSR register need to be set when using these events.", + "SampleAfterValue": "1000003", + "UMask": "0x3" + }, + { + "BriefDescription": "Counts number of SSE/AVX computational scalar double precision floating-point instructions retired; some instructions will count twice as noted below. Each count represents 1 computational operation. Applies to SSE* and AVX* scalar double precision floating-point instructions: ADD SUB MUL DIV MIN MAX SQRT FM(N)ADD/SUB. FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element.", + "EventCode": "0xc7", + "EventName": "FP_ARITH_INST_RETIRED.SCALAR_DOUBLE", + "PublicDescription": "Number of SSE/AVX computational scalar double precision floating-point instructions retired; some instructions will count twice as noted below. Each count represents 1 computational operation. Applies to SSE* and AVX* scalar double precision floating-point instructions: ADD SUB MUL DIV MIN MAX SQRT FM(N)ADD/SUB. FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element. The DAZ and FTZ flags in the MXCSR register need to be set when using these events.", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts number of SSE/AVX computational scalar single precision floating-point instructions retired; some instructions will count twice as noted below. Each count represents 1 computational operation. Applies to SSE* and AVX* scalar single precision floating-point instructions: ADD SUB MUL DIV MIN MAX SQRT RSQRT RCP FM(N)ADD/SUB. FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element.", + "EventCode": "0xc7", + "EventName": "FP_ARITH_INST_RETIRED.SCALAR_SINGLE", + "PublicDescription": "Number of SSE/AVX computational scalar single precision floating-point instructions retired; some instructions will count twice as noted below. Each count represents 1 computational operation. Applies to SSE* and AVX* scalar single precision floating-point instructions: ADD SUB MUL DIV MIN MAX SQRT RSQRT RCP FM(N)ADD/SUB. FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element. The DAZ and FTZ flags in the MXCSR register need to be set when using these events.", + "SampleAfterValue": "100003", + "UMask": "0x2" + }, + { + "BriefDescription": "Number of any Vector retired FP arithmetic instructions", + "EventCode": "0xc7", + "EventName": "FP_ARITH_INST_RETIRED.VECTOR", + "PublicDescription": "Number of any Vector retired FP arithmetic instructions. The DAZ and FTZ flags in the MXCSR register need to be set when using these events.", + "SampleAfterValue": "1000003", + "UMask": "0xfc" + }, + { + "BriefDescription": "FP_ARITH_INST_RETIRED2.128B_PACKED_HALF", + "EventCode": "0xcf", + "EventName": "FP_ARITH_INST_RETIRED2.128B_PACKED_HALF", + "SampleAfterValue": "100003", + "UMask": "0x4" + }, + { + "BriefDescription": "FP_ARITH_INST_RETIRED2.256B_PACKED_HALF", + "EventCode": "0xcf", + "EventName": "FP_ARITH_INST_RETIRED2.256B_PACKED_HALF", + "SampleAfterValue": "100003", + "UMask": "0x8" + }, + { + "BriefDescription": "FP_ARITH_INST_RETIRED2.512B_PACKED_HALF", + "EventCode": "0xcf", + "EventName": "FP_ARITH_INST_RETIRED2.512B_PACKED_HALF", + "SampleAfterValue": "100003", + "UMask": "0x10" + }, + { + "BriefDescription": "FP_ARITH_INST_RETIRED2.COMPLEX_SCALAR_HALF", + "EventCode": "0xcf", + "EventName": "FP_ARITH_INST_RETIRED2.COMPLEX_SCALAR_HALF", + "SampleAfterValue": "100003", + "UMask": "0x2" + }, + { + "BriefDescription": "Number of all Scalar Half-Precision FP arithmetic instructions(1) retired - regular and complex.", + "EventCode": "0xcf", + "EventName": "FP_ARITH_INST_RETIRED2.SCALAR", + "PublicDescription": "FP_ARITH_INST_RETIRED2.SCALAR", + "SampleAfterValue": "100003", + "UMask": "0x3" + }, + { + "BriefDescription": "FP_ARITH_INST_RETIRED2.SCALAR_HALF", + "EventCode": "0xcf", + "EventName": "FP_ARITH_INST_RETIRED2.SCALAR_HALF", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Number of all Vector (also called packed) Half-Precision FP arithmetic instructions(1) retired.", + "EventCode": "0xcf", + "EventName": "FP_ARITH_INST_RETIRED2.VECTOR", + "PublicDescription": "FP_ARITH_INST_RETIRED2.VECTOR", + "SampleAfterValue": "100003", + "UMask": "0x1c" + } +] diff --git a/tools/perf/pmu-events/arch/x86/emeraldrapids/frontend.json b/tools/perf/pmu-events/arch/x86/emeraldrapids/frontend.json new file mode 100644 index 000000000000..9e53da55d0c1 --- /dev/null +++ b/tools/perf/pmu-events/arch/x86/emeraldrapids/frontend.json @@ -0,0 +1,389 @@ +[ + { + "BriefDescription": "Clears due to Unknown Branches.", + "EventCode": "0x60", + "EventName": "BACLEARS.ANY", + "PublicDescription": "Number of times the front-end is resteered when it finds a branch instruction in a fetch line. This is called Unknown Branch which occurs for the first time a branch instruction is fetched or when the branch is not tracked by the BPU (Branch Prediction Unit) anymore.", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Stalls caused by changing prefix length of the instruction.", + "EventCode": "0x87", + "EventName": "DECODE.LCP", + "PublicDescription": "Counts cycles that the Instruction Length decoder (ILD) stalls occurred due to dynamically changing prefix length of the decoded instruction (by operand size prefix instruction 0x66, address size prefix instruction 0x67 or REX.W for Intel64). Count is proportional to the number of prefixes in a 16B-line. This may result in a three-cycle penalty for each LCP (Length changing prefix) in a 16-byte chunk.", + "SampleAfterValue": "500009", + "UMask": "0x1" + }, + { + "BriefDescription": "Cycles the Microcode Sequencer is busy.", + "EventCode": "0x87", + "EventName": "DECODE.MS_BUSY", + "SampleAfterValue": "500009", + "UMask": "0x2" + }, + { + "BriefDescription": "DSB-to-MITE switch true penalty cycles.", + "EventCode": "0x61", + "EventName": "DSB2MITE_SWITCHES.PENALTY_CYCLES", + "PublicDescription": "Decode Stream Buffer (DSB) is a Uop-cache that holds translations of previously fetched instructions that were decoded by the legacy x86 decode pipeline (MITE). This event counts fetch penalty cycles when a transition occurs from DSB to MITE.", + "SampleAfterValue": "100003", + "UMask": "0x2" + }, + { + "BriefDescription": "Retired Instructions who experienced DSB miss.", + "EventCode": "0xc6", + "EventName": "FRONTEND_RETIRED.ANY_DSB_MISS", + "MSRIndex": "0x3F7", + "MSRValue": "0x1", + "PEBS": "1", + "PublicDescription": "Counts retired Instructions that experienced DSB (Decode stream buffer i.e. the decoded instruction-cache) miss.", + "SampleAfterValue": "100007", + "UMask": "0x1" + }, + { + "BriefDescription": "Retired Instructions who experienced a critical DSB miss.", + "EventCode": "0xc6", + "EventName": "FRONTEND_RETIRED.DSB_MISS", + "MSRIndex": "0x3F7", + "MSRValue": "0x11", + "PEBS": "1", + "PublicDescription": "Number of retired Instructions that experienced a critical DSB (Decode stream buffer i.e. the decoded instruction-cache) miss. Critical means stalls were exposed to the back-end as a result of the DSB miss.", + "SampleAfterValue": "100007", + "UMask": "0x1" + }, + { + "BriefDescription": "Retired Instructions who experienced iTLB true miss.", + "EventCode": "0xc6", + "EventName": "FRONTEND_RETIRED.ITLB_MISS", + "MSRIndex": "0x3F7", + "MSRValue": "0x14", + "PEBS": "1", + "PublicDescription": "Counts retired Instructions that experienced iTLB (Instruction TLB) true miss.", + "SampleAfterValue": "100007", + "UMask": "0x1" + }, + { + "BriefDescription": "Retired Instructions who experienced Instruction L1 Cache true miss.", + "EventCode": "0xc6", + "EventName": "FRONTEND_RETIRED.L1I_MISS", + "MSRIndex": "0x3F7", + "MSRValue": "0x12", + "PEBS": "1", + "PublicDescription": "Counts retired Instructions who experienced Instruction L1 Cache true miss.", + "SampleAfterValue": "100007", + "UMask": "0x1" + }, + { + "BriefDescription": "Retired Instructions who experienced Instruction L2 Cache true miss.", + "EventCode": "0xc6", + "EventName": "FRONTEND_RETIRED.L2_MISS", + "MSRIndex": "0x3F7", + "MSRValue": "0x13", + "PEBS": "1", + "PublicDescription": "Counts retired Instructions who experienced Instruction L2 Cache true miss.", + "SampleAfterValue": "100007", + "UMask": "0x1" + }, + { + "BriefDescription": "Retired instructions after front-end starvation of at least 1 cycle", + "EventCode": "0xc6", + "EventName": "FRONTEND_RETIRED.LATENCY_GE_1", + "MSRIndex": "0x3F7", + "MSRValue": "0x600106", + "PEBS": "1", + "PublicDescription": "Retired instructions that are fetched after an interval where the front-end delivered no uops for a period of at least 1 cycle which was not interrupted by a back-end stall.", + "SampleAfterValue": "100007", + "UMask": "0x1" + }, + { + "BriefDescription": "Retired instructions that are fetched after an interval where the front-end delivered no uops for a period of 128 cycles which was not interrupted by a back-end stall.", + "EventCode": "0xc6", + "EventName": "FRONTEND_RETIRED.LATENCY_GE_128", + "MSRIndex": "0x3F7", + "MSRValue": "0x608006", + "PEBS": "1", + "PublicDescription": "Counts retired instructions that are fetched after an interval where the front-end delivered no uops for a period of 128 cycles which was not interrupted by a back-end stall.", + "SampleAfterValue": "100007", + "UMask": "0x1" + }, + { + "BriefDescription": "Retired instructions that are fetched after an interval where the front-end delivered no uops for a period of 16 cycles which was not interrupted by a back-end stall.", + "EventCode": "0xc6", + "EventName": "FRONTEND_RETIRED.LATENCY_GE_16", + "MSRIndex": "0x3F7", + "MSRValue": "0x601006", + "PEBS": "1", + "PublicDescription": "Counts retired instructions that are delivered to the back-end after a front-end stall of at least 16 cycles. During this period the front-end delivered no uops.", + "SampleAfterValue": "100007", + "UMask": "0x1" + }, + { + "BriefDescription": "Retired instructions after front-end starvation of at least 2 cycles", + "EventCode": "0xc6", + "EventName": "FRONTEND_RETIRED.LATENCY_GE_2", + "MSRIndex": "0x3F7", + "MSRValue": "0x600206", + "PEBS": "1", + "PublicDescription": "Retired instructions that are fetched after an interval where the front-end delivered no uops for a period of at least 2 cycles which was not interrupted by a back-end stall.", + "SampleAfterValue": "100007", + "UMask": "0x1" + }, + { + "BriefDescription": "Retired instructions that are fetched after an interval where the front-end delivered no uops for a period of 256 cycles which was not interrupted by a back-end stall.", + "EventCode": "0xc6", + "EventName": "FRONTEND_RETIRED.LATENCY_GE_256", + "MSRIndex": "0x3F7", + "MSRValue": "0x610006", + "PEBS": "1", + "PublicDescription": "Counts retired instructions that are fetched after an interval where the front-end delivered no uops for a period of 256 cycles which was not interrupted by a back-end stall.", + "SampleAfterValue": "100007", + "UMask": "0x1" + }, + { + "BriefDescription": "Retired instructions that are fetched after an interval where the front-end had at least 1 bubble-slot for a period of 2 cycles which was not interrupted by a back-end stall.", + "EventCode": "0xc6", + "EventName": "FRONTEND_RETIRED.LATENCY_GE_2_BUBBLES_GE_1", + "MSRIndex": "0x3F7", + "MSRValue": "0x100206", + "PEBS": "1", + "PublicDescription": "Counts retired instructions that are delivered to the back-end after the front-end had at least 1 bubble-slot for a period of 2 cycles. A bubble-slot is an empty issue-pipeline slot while there was no RAT stall.", + "SampleAfterValue": "100007", + "UMask": "0x1" + }, + { + "BriefDescription": "Retired instructions that are fetched after an interval where the front-end delivered no uops for a period of 32 cycles which was not interrupted by a back-end stall.", + "EventCode": "0xc6", + "EventName": "FRONTEND_RETIRED.LATENCY_GE_32", + "MSRIndex": "0x3F7", + "MSRValue": "0x602006", + "PEBS": "1", + "PublicDescription": "Counts retired instructions that are delivered to the back-end after a front-end stall of at least 32 cycles. During this period the front-end delivered no uops.", + "SampleAfterValue": "100007", + "UMask": "0x1" + }, + { + "BriefDescription": "Retired instructions that are fetched after an interval where the front-end delivered no uops for a period of 4 cycles which was not interrupted by a back-end stall.", + "EventCode": "0xc6", + "EventName": "FRONTEND_RETIRED.LATENCY_GE_4", + "MSRIndex": "0x3F7", + "MSRValue": "0x600406", + "PEBS": "1", + "PublicDescription": "Counts retired instructions that are fetched after an interval where the front-end delivered no uops for a period of 4 cycles which was not interrupted by a back-end stall.", + "SampleAfterValue": "100007", + "UMask": "0x1" + }, + { + "BriefDescription": "Retired instructions that are fetched after an interval where the front-end delivered no uops for a period of 512 cycles which was not interrupted by a back-end stall.", + "EventCode": "0xc6", + "EventName": "FRONTEND_RETIRED.LATENCY_GE_512", + "MSRIndex": "0x3F7", + "MSRValue": "0x620006", + "PEBS": "1", + "PublicDescription": "Counts retired instructions that are fetched after an interval where the front-end delivered no uops for a period of 512 cycles which was not interrupted by a back-end stall.", + "SampleAfterValue": "100007", + "UMask": "0x1" + }, + { + "BriefDescription": "Retired instructions that are fetched after an interval where the front-end delivered no uops for a period of 64 cycles which was not interrupted by a back-end stall.", + "EventCode": "0xc6", + "EventName": "FRONTEND_RETIRED.LATENCY_GE_64", + "MSRIndex": "0x3F7", + "MSRValue": "0x604006", + "PEBS": "1", + "PublicDescription": "Counts retired instructions that are fetched after an interval where the front-end delivered no uops for a period of 64 cycles which was not interrupted by a back-end stall.", + "SampleAfterValue": "100007", + "UMask": "0x1" + }, + { + "BriefDescription": "Retired instructions that are fetched after an interval where the front-end delivered no uops for a period of 8 cycles which was not interrupted by a back-end stall.", + "EventCode": "0xc6", + "EventName": "FRONTEND_RETIRED.LATENCY_GE_8", + "MSRIndex": "0x3F7", + "MSRValue": "0x600806", + "PEBS": "1", + "PublicDescription": "Counts retired instructions that are delivered to the back-end after a front-end stall of at least 8 cycles. During this period the front-end delivered no uops.", + "SampleAfterValue": "100007", + "UMask": "0x1" + }, + { + "BriefDescription": "FRONTEND_RETIRED.MS_FLOWS", + "EventCode": "0xc6", + "EventName": "FRONTEND_RETIRED.MS_FLOWS", + "MSRIndex": "0x3F7", + "MSRValue": "0x8", + "PEBS": "1", + "SampleAfterValue": "100007", + "UMask": "0x1" + }, + { + "BriefDescription": "Retired Instructions who experienced STLB (2nd level TLB) true miss.", + "EventCode": "0xc6", + "EventName": "FRONTEND_RETIRED.STLB_MISS", + "MSRIndex": "0x3F7", + "MSRValue": "0x15", + "PEBS": "1", + "PublicDescription": "Counts retired Instructions that experienced STLB (2nd level TLB) true miss.", + "SampleAfterValue": "100007", + "UMask": "0x1" + }, + { + "BriefDescription": "FRONTEND_RETIRED.UNKNOWN_BRANCH", + "EventCode": "0xc6", + "EventName": "FRONTEND_RETIRED.UNKNOWN_BRANCH", + "MSRIndex": "0x3F7", + "MSRValue": "0x17", + "PEBS": "1", + "SampleAfterValue": "100007", + "UMask": "0x1" + }, + { + "BriefDescription": "Cycles where a code fetch is stalled due to L1 instruction cache miss.", + "EventCode": "0x80", + "EventName": "ICACHE_DATA.STALLS", + "PublicDescription": "Counts cycles where a code line fetch is stalled due to an L1 instruction cache miss. The decode pipeline works at a 32 Byte granularity.", + "SampleAfterValue": "500009", + "UMask": "0x4" + }, + { + "BriefDescription": "Cycles where a code fetch is stalled due to L1 instruction cache tag miss.", + "EventCode": "0x83", + "EventName": "ICACHE_TAG.STALLS", + "PublicDescription": "Counts cycles where a code fetch is stalled due to L1 instruction cache tag miss.", + "SampleAfterValue": "200003", + "UMask": "0x4" + }, + { + "BriefDescription": "Cycles Decode Stream Buffer (DSB) is delivering any Uop", + "CounterMask": "1", + "EventCode": "0x79", + "EventName": "IDQ.DSB_CYCLES_ANY", + "PublicDescription": "Counts the number of cycles uops were delivered to Instruction Decode Queue (IDQ) from the Decode Stream Buffer (DSB) path.", + "SampleAfterValue": "2000003", + "UMask": "0x8" + }, + { + "BriefDescription": "Cycles DSB is delivering optimal number of Uops", + "CounterMask": "6", + "EventCode": "0x79", + "EventName": "IDQ.DSB_CYCLES_OK", + "PublicDescription": "Counts the number of cycles where optimal number of uops was delivered to the Instruction Decode Queue (IDQ) from the MITE (legacy decode pipeline) path. During these cycles uops are not being delivered from the Decode Stream Buffer (DSB).", + "SampleAfterValue": "2000003", + "UMask": "0x8" + }, + { + "BriefDescription": "Uops delivered to Instruction Decode Queue (IDQ) from the Decode Stream Buffer (DSB) path", + "EventCode": "0x79", + "EventName": "IDQ.DSB_UOPS", + "PublicDescription": "Counts the number of uops delivered to Instruction Decode Queue (IDQ) from the Decode Stream Buffer (DSB) path.", + "SampleAfterValue": "2000003", + "UMask": "0x8" + }, + { + "BriefDescription": "Cycles MITE is delivering any Uop", + "CounterMask": "1", + "EventCode": "0x79", + "EventName": "IDQ.MITE_CYCLES_ANY", + "PublicDescription": "Counts the number of cycles uops were delivered to the Instruction Decode Queue (IDQ) from the MITE (legacy decode pipeline) path. During these cycles uops are not being delivered from the Decode Stream Buffer (DSB).", + "SampleAfterValue": "2000003", + "UMask": "0x4" + }, + { + "BriefDescription": "Cycles MITE is delivering optimal number of Uops", + "CounterMask": "6", + "EventCode": "0x79", + "EventName": "IDQ.MITE_CYCLES_OK", + "PublicDescription": "Counts the number of cycles where optimal number of uops was delivered to the Instruction Decode Queue (IDQ) from the MITE (legacy decode pipeline) path. During these cycles uops are not being delivered from the Decode Stream Buffer (DSB).", + "SampleAfterValue": "2000003", + "UMask": "0x4" + }, + { + "BriefDescription": "Uops delivered to Instruction Decode Queue (IDQ) from MITE path", + "EventCode": "0x79", + "EventName": "IDQ.MITE_UOPS", + "PublicDescription": "Counts the number of uops delivered to Instruction Decode Queue (IDQ) from the MITE path. This also means that uops are not being delivered from the Decode Stream Buffer (DSB).", + "SampleAfterValue": "2000003", + "UMask": "0x4" + }, + { + "BriefDescription": "Cycles when uops are being delivered to IDQ while MS is busy", + "CounterMask": "1", + "EventCode": "0x79", + "EventName": "IDQ.MS_CYCLES_ANY", + "PublicDescription": "Counts cycles during which uops are being delivered to Instruction Decode Queue (IDQ) while the Microcode Sequencer (MS) is busy. Uops maybe initiated by Decode Stream Buffer (DSB) or MITE.", + "SampleAfterValue": "2000003", + "UMask": "0x20" + }, + { + "BriefDescription": "Number of switches from DSB or MITE to the MS", + "CounterMask": "1", + "EdgeDetect": "1", + "EventCode": "0x79", + "EventName": "IDQ.MS_SWITCHES", + "PublicDescription": "Number of switches from DSB (Decode Stream Buffer) or MITE (legacy decode pipeline) to the Microcode Sequencer.", + "SampleAfterValue": "100003", + "UMask": "0x20" + }, + { + "BriefDescription": "Uops delivered to IDQ while MS is busy", + "EventCode": "0x79", + "EventName": "IDQ.MS_UOPS", + "PublicDescription": "Counts the total number of uops delivered by the Microcode Sequencer (MS).", + "SampleAfterValue": "1000003", + "UMask": "0x20" + }, + { + "BriefDescription": "Uops not delivered by IDQ when backend of the machine is not stalled [This event is alias to IDQ_UOPS_NOT_DELIVERED.CORE]", + "EventCode": "0x9c", + "EventName": "IDQ_BUBBLES.CORE", + "PublicDescription": "Counts the number of uops not delivered to by the Instruction Decode Queue (IDQ) to the back-end of the pipeline when there was no back-end stalls. This event counts for one SMT thread in a given cycle. [This event is alias to IDQ_UOPS_NOT_DELIVERED.CORE]", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Cycles when no uops are not delivered by the IDQ when backend of the machine is not stalled [This event is alias to IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE]", + "CounterMask": "6", + "EventCode": "0x9c", + "EventName": "IDQ_BUBBLES.CYCLES_0_UOPS_DELIV.CORE", + "PublicDescription": "Counts the number of cycles when no uops were delivered by the Instruction Decode Queue (IDQ) to the back-end of the pipeline when there was no back-end stalls. This event counts for one SMT thread in a given cycle. [This event is alias to IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE]", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Cycles when optimal number of uops was delivered to the back-end when the back-end is not stalled [This event is alias to IDQ_UOPS_NOT_DELIVERED.CYCLES_FE_WAS_OK]", + "CounterMask": "1", + "EventCode": "0x9c", + "EventName": "IDQ_BUBBLES.CYCLES_FE_WAS_OK", + "Invert": "1", + "PublicDescription": "Counts the number of cycles when the optimal number of uops were delivered by the Instruction Decode Queue (IDQ) to the back-end of the pipeline when there was no back-end stalls. This event counts for one SMT thread in a given cycle. [This event is alias to IDQ_UOPS_NOT_DELIVERED.CYCLES_FE_WAS_OK]", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Uops not delivered by IDQ when backend of the machine is not stalled [This event is alias to IDQ_BUBBLES.CORE]", + "EventCode": "0x9c", + "EventName": "IDQ_UOPS_NOT_DELIVERED.CORE", + "PublicDescription": "Counts the number of uops not delivered to by the Instruction Decode Queue (IDQ) to the back-end of the pipeline when there was no back-end stalls. This event counts for one SMT thread in a given cycle. [This event is alias to IDQ_BUBBLES.CORE]", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Cycles when no uops are not delivered by the IDQ when backend of the machine is not stalled [This event is alias to IDQ_BUBBLES.CYCLES_0_UOPS_DELIV.CORE]", + "CounterMask": "6", + "EventCode": "0x9c", + "EventName": "IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE", + "PublicDescription": "Counts the number of cycles when no uops were delivered by the Instruction Decode Queue (IDQ) to the back-end of the pipeline when there was no back-end stalls. This event counts for one SMT thread in a given cycle. [This event is alias to IDQ_BUBBLES.CYCLES_0_UOPS_DELIV.CORE]", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Cycles when optimal number of uops was delivered to the back-end when the back-end is not stalled [This event is alias to IDQ_BUBBLES.CYCLES_FE_WAS_OK]", + "CounterMask": "1", + "EventCode": "0x9c", + "EventName": "IDQ_UOPS_NOT_DELIVERED.CYCLES_FE_WAS_OK", + "Invert": "1", + "PublicDescription": "Counts the number of cycles when the optimal number of uops were delivered by the Instruction Decode Queue (IDQ) to the back-end of the pipeline when there was no back-end stalls. This event counts for one SMT thread in a given cycle. [This event is alias to IDQ_BUBBLES.CYCLES_FE_WAS_OK]", + "SampleAfterValue": "1000003", + "UMask": "0x1" + } +] diff --git a/tools/perf/pmu-events/arch/x86/emeraldrapids/memory.json b/tools/perf/pmu-events/arch/x86/emeraldrapids/memory.json new file mode 100644 index 000000000000..e8bf7c9c44e1 --- /dev/null +++ b/tools/perf/pmu-events/arch/x86/emeraldrapids/memory.json @@ -0,0 +1,343 @@ +[ + { + "BriefDescription": "Execution stalls while L3 cache miss demand load is outstanding.", + "CounterMask": "6", + "EventCode": "0xa3", + "EventName": "CYCLE_ACTIVITY.STALLS_L3_MISS", + "SampleAfterValue": "1000003", + "UMask": "0x6" + }, + { + "BriefDescription": "Number of machine clears due to memory ordering conflicts.", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.MEMORY_ORDERING", + "PublicDescription": "Counts the number of Machine Clears detected dye to memory ordering. Memory Ordering Machine Clears may apply when a memory read may not conform to the memory ordering rules of the x86 architecture", + "SampleAfterValue": "100003", + "UMask": "0x2" + }, + { + "BriefDescription": "Cycles while L1 cache miss demand load is outstanding.", + "CounterMask": "2", + "EventCode": "0x47", + "EventName": "MEMORY_ACTIVITY.CYCLES_L1D_MISS", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Execution stalls while L1 cache miss demand load is outstanding.", + "CounterMask": "3", + "EventCode": "0x47", + "EventName": "MEMORY_ACTIVITY.STALLS_L1D_MISS", + "SampleAfterValue": "1000003", + "UMask": "0x3" + }, + { + "BriefDescription": "Execution stalls while L2 cache miss demand cacheable load request is outstanding.", + "CounterMask": "5", + "EventCode": "0x47", + "EventName": "MEMORY_ACTIVITY.STALLS_L2_MISS", + "PublicDescription": "Execution stalls while L2 cache miss demand cacheable load request is outstanding (will not count for uncacheable demand requests e.g. bus lock).", + "SampleAfterValue": "1000003", + "UMask": "0x5" + }, + { + "BriefDescription": "Execution stalls while L3 cache miss demand cacheable load request is outstanding.", + "CounterMask": "9", + "EventCode": "0x47", + "EventName": "MEMORY_ACTIVITY.STALLS_L3_MISS", + "PublicDescription": "Execution stalls while L3 cache miss demand cacheable load request is outstanding (will not count for uncacheable demand requests e.g. bus lock).", + "SampleAfterValue": "1000003", + "UMask": "0x9" + }, + { + "BriefDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 128 cycles.", + "Data_LA": "1", + "EventCode": "0xcd", + "EventName": "MEM_TRANS_RETIRED.LOAD_LATENCY_GT_128", + "MSRIndex": "0x3F6", + "MSRValue": "0x80", + "PEBS": "2", + "PublicDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 128 cycles. Reported latency may be longer than just the memory latency.", + "SampleAfterValue": "1009", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 16 cycles.", + "Data_LA": "1", + "EventCode": "0xcd", + "EventName": "MEM_TRANS_RETIRED.LOAD_LATENCY_GT_16", + "MSRIndex": "0x3F6", + "MSRValue": "0x10", + "PEBS": "2", + "PublicDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 16 cycles. Reported latency may be longer than just the memory latency.", + "SampleAfterValue": "20011", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 256 cycles.", + "Data_LA": "1", + "EventCode": "0xcd", + "EventName": "MEM_TRANS_RETIRED.LOAD_LATENCY_GT_256", + "MSRIndex": "0x3F6", + "MSRValue": "0x100", + "PEBS": "2", + "PublicDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 256 cycles. Reported latency may be longer than just the memory latency.", + "SampleAfterValue": "503", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 32 cycles.", + "Data_LA": "1", + "EventCode": "0xcd", + "EventName": "MEM_TRANS_RETIRED.LOAD_LATENCY_GT_32", + "MSRIndex": "0x3F6", + "MSRValue": "0x20", + "PEBS": "2", + "PublicDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 32 cycles. Reported latency may be longer than just the memory latency.", + "SampleAfterValue": "100007", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 4 cycles.", + "Data_LA": "1", + "EventCode": "0xcd", + "EventName": "MEM_TRANS_RETIRED.LOAD_LATENCY_GT_4", + "MSRIndex": "0x3F6", + "MSRValue": "0x4", + "PEBS": "2", + "PublicDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 4 cycles. Reported latency may be longer than just the memory latency.", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 512 cycles.", + "Data_LA": "1", + "EventCode": "0xcd", + "EventName": "MEM_TRANS_RETIRED.LOAD_LATENCY_GT_512", + "MSRIndex": "0x3F6", + "MSRValue": "0x200", + "PEBS": "2", + "PublicDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 512 cycles. Reported latency may be longer than just the memory latency.", + "SampleAfterValue": "101", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 64 cycles.", + "Data_LA": "1", + "EventCode": "0xcd", + "EventName": "MEM_TRANS_RETIRED.LOAD_LATENCY_GT_64", + "MSRIndex": "0x3F6", + "MSRValue": "0x40", + "PEBS": "2", + "PublicDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 64 cycles. Reported latency may be longer than just the memory latency.", + "SampleAfterValue": "2003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 8 cycles.", + "Data_LA": "1", + "EventCode": "0xcd", + "EventName": "MEM_TRANS_RETIRED.LOAD_LATENCY_GT_8", + "MSRIndex": "0x3F6", + "MSRValue": "0x8", + "PEBS": "2", + "PublicDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 8 cycles. Reported latency may be longer than just the memory latency.", + "SampleAfterValue": "50021", + "UMask": "0x1" + }, + { + "BriefDescription": "Retired memory store access operations. A PDist event for PEBS Store Latency Facility.", + "Data_LA": "1", + "EventCode": "0xcd", + "EventName": "MEM_TRANS_RETIRED.STORE_SAMPLE", + "PEBS": "2", + "PublicDescription": "Counts Retired memory accesses with at least 1 store operation. This PEBS event is the precisely-distributed (PDist) trigger covering all stores uops for sampling by the PEBS Store Latency Facility. The facility is described in Intel SDM Volume 3 section 19.9.8", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Counts demand instruction fetches and L1 instruction cache prefetches that were not supplied by the local socket's L1, L2, or L3 caches.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_CODE_RD.L3_MISS", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x3FBFC00004", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts demand data reads that were not supplied by the local socket's L1, L2, or L3 caches.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_DATA_RD.L3_MISS", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x3FBFC00001", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts demand reads for ownership (RFO) requests and software prefetches for exclusive ownership (PREFETCHW) that were not supplied by the local socket's L1, L2, or L3 caches.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_RFO.L3_MISS", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x3F3FC00002", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts hardware prefetches to the L3 only that missed the local socket's L1, L2, and L3 caches.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.HWPF_L3.L3_MISS", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x94002380", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts hardware prefetches to the L3 only that were not supplied by the local socket's L1, L2, or L3 caches and the cacheline is homed locally.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.HWPF_L3.L3_MISS_LOCAL", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x84002380", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts all (cacheable) data read, code read and RFO requests including demands and prefetches to the core caches (L1 or L2) that were not supplied by the local socket's L1, L2, or L3 caches.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.READS_TO_CORE.L3_MISS", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x3F3FC04477", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts all (cacheable) data read, code read and RFO requests including demands and prefetches to the core caches (L1 or L2) that were not supplied by the local socket's L1, L2, or L3 caches and the cacheline is homed locally.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.READS_TO_CORE.L3_MISS_LOCAL", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x3F04C04477", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts all (cacheable) data read, code read and RFO requests including demands and prefetches to the core caches (L1 or L2) that missed the L3 Cache and were supplied by the local socket (DRAM or PMM), whether or not in Sub NUMA Cluster(SNC) Mode. In SNC Mode counts PMM or DRAM accesses that are controlled by the close or distant SNC Cluster. It does not count misses to the L3 which go to Local CXL Type 2 Memory or Local Non DRAM.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.READS_TO_CORE.L3_MISS_LOCAL_SOCKET", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x70CC04477", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts streaming stores that missed the local socket's L1, L2, and L3 caches.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.STREAMING_WR.L3_MISS", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x94000800", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts streaming stores that were not supplied by the local socket's L1, L2, or L3 caches and the cacheline is homed locally.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.STREAMING_WR.L3_MISS_LOCAL", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x84000800", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts demand data read requests that miss the L3 cache.", + "EventCode": "0x21", + "EventName": "OFFCORE_REQUESTS.L3_MISS_DEMAND_DATA_RD", + "SampleAfterValue": "100003", + "UMask": "0x10" + }, + { + "BriefDescription": "For every cycle, increments by the number of demand data read requests pending that are known to have missed the L3 cache.", + "EventCode": "0x20", + "EventName": "OFFCORE_REQUESTS_OUTSTANDING.L3_MISS_DEMAND_DATA_RD", + "PublicDescription": "For every cycle, increments by the number of demand data read requests pending that are known to have missed the L3 cache. Note that this does not capture all elapsed cycles while requests are outstanding - only cycles from when the requests were known by the requesting core to have missed the L3 cache.", + "SampleAfterValue": "2000003", + "UMask": "0x10" + }, + { + "BriefDescription": "Number of times an RTM execution aborted.", + "EventCode": "0xc9", + "EventName": "RTM_RETIRED.ABORTED", + "PublicDescription": "Counts the number of times RTM abort was triggered.", + "SampleAfterValue": "100003", + "UMask": "0x4" + }, + { + "BriefDescription": "Number of times an RTM execution aborted due to none of the previous 4 categories (e.g. interrupt)", + "EventCode": "0xc9", + "EventName": "RTM_RETIRED.ABORTED_EVENTS", + "PublicDescription": "Counts the number of times an RTM execution aborted due to none of the previous 4 categories (e.g. interrupt).", + "SampleAfterValue": "100003", + "UMask": "0x80" + }, + { + "BriefDescription": "Number of times an RTM execution aborted due to various memory events (e.g. read/write capacity and conflicts)", + "EventCode": "0xc9", + "EventName": "RTM_RETIRED.ABORTED_MEM", + "PublicDescription": "Counts the number of times an RTM execution aborted due to various memory events (e.g. read/write capacity and conflicts).", + "SampleAfterValue": "100003", + "UMask": "0x8" + }, + { + "BriefDescription": "Number of times an RTM execution aborted due to incompatible memory type", + "EventCode": "0xc9", + "EventName": "RTM_RETIRED.ABORTED_MEMTYPE", + "PublicDescription": "Counts the number of times an RTM execution aborted due to incompatible memory type.", + "SampleAfterValue": "100003", + "UMask": "0x40" + }, + { + "BriefDescription": "Number of times an RTM execution aborted due to HLE-unfriendly instructions", + "EventCode": "0xc9", + "EventName": "RTM_RETIRED.ABORTED_UNFRIENDLY", + "PublicDescription": "Counts the number of times an RTM execution aborted due to HLE-unfriendly instructions.", + "SampleAfterValue": "100003", + "UMask": "0x20" + }, + { + "BriefDescription": "Number of times an RTM execution successfully committed", + "EventCode": "0xc9", + "EventName": "RTM_RETIRED.COMMIT", + "PublicDescription": "Counts the number of times RTM commit succeeded.", + "SampleAfterValue": "100003", + "UMask": "0x2" + }, + { + "BriefDescription": "Number of times an RTM execution started.", + "EventCode": "0xc9", + "EventName": "RTM_RETIRED.START", + "PublicDescription": "Counts the number of times we entered an RTM region. Does not count nested transactions.", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Speculatively counts the number of TSX aborts due to a data capacity limitation for transactional reads", + "EventCode": "0x54", + "EventName": "TX_MEM.ABORT_CAPACITY_READ", + "PublicDescription": "Speculatively counts the number of Transactional Synchronization Extensions (TSX) aborts due to a data capacity limitation for transactional reads", + "SampleAfterValue": "100003", + "UMask": "0x80" + }, + { + "BriefDescription": "Speculatively counts the number of TSX aborts due to a data capacity limitation for transactional writes.", + "EventCode": "0x54", + "EventName": "TX_MEM.ABORT_CAPACITY_WRITE", + "PublicDescription": "Speculatively counts the number of Transactional Synchronization Extensions (TSX) aborts due to a data capacity limitation for transactional writes.", + "SampleAfterValue": "100003", + "UMask": "0x2" + }, + { + "BriefDescription": "Number of times a transactional abort was signaled due to a data conflict on a transactionally accessed address", + "EventCode": "0x54", + "EventName": "TX_MEM.ABORT_CONFLICT", + "PublicDescription": "Counts the number of times a TSX line had a cache conflict.", + "SampleAfterValue": "100003", + "UMask": "0x1" + } +] diff --git a/tools/perf/pmu-events/arch/x86/emeraldrapids/other.json b/tools/perf/pmu-events/arch/x86/emeraldrapids/other.json new file mode 100644 index 000000000000..2f375a6badcd --- /dev/null +++ b/tools/perf/pmu-events/arch/x86/emeraldrapids/other.json @@ -0,0 +1,315 @@ +[ + { + "BriefDescription": "ASSISTS.PAGE_FAULT", + "EventCode": "0xc1", + "EventName": "ASSISTS.PAGE_FAULT", + "SampleAfterValue": "1000003", + "UMask": "0x8" + }, + { + "BriefDescription": "Counts the cycles where the AMX (Advance Matrix Extension) unit is busy performing an operation.", + "EventCode": "0xb7", + "EventName": "EXE.AMX_BUSY", + "SampleAfterValue": "2000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Counts demand instruction fetches and L1 instruction cache prefetches that have any type of response.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_CODE_RD.ANY_RESPONSE", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x10004", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts demand instruction fetches and L1 instruction cache prefetches that were supplied by DRAM.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_CODE_RD.DRAM", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x73C000004", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts demand instruction fetches and L1 instruction cache prefetches that were supplied by DRAM attached to this socket, unless in Sub NUMA Cluster(SNC) Mode. In SNC Mode counts only those DRAM accesses that are controlled by the close SNC Cluster.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_CODE_RD.LOCAL_DRAM", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x104000004", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts demand instruction fetches and L1 instruction cache prefetches that were supplied by DRAM on a distant memory controller of this socket when the system is in SNC (sub-NUMA cluster) mode.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_CODE_RD.SNC_DRAM", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x708000004", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts demand data reads that have any type of response.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_DATA_RD.ANY_RESPONSE", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x10001", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts demand data reads that were supplied by DRAM.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_DATA_RD.DRAM", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x73C000001", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts demand data reads that were supplied by DRAM attached to this socket, unless in Sub NUMA Cluster(SNC) Mode. In SNC Mode counts only those DRAM accesses that are controlled by the close SNC Cluster.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_DATA_RD.LOCAL_DRAM", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x104000001", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts demand data reads that were supplied by DRAM attached to another socket.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_DATA_RD.REMOTE_DRAM", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x730000001", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts demand data reads that were supplied by DRAM on a distant memory controller of this socket when the system is in SNC (sub-NUMA cluster) mode.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_DATA_RD.SNC_DRAM", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x708000001", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts demand reads for ownership (RFO) requests and software prefetches for exclusive ownership (PREFETCHW) that have any type of response.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_RFO.ANY_RESPONSE", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x3F3FFC0002", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts demand reads for ownership (RFO) requests and software prefetches for exclusive ownership (PREFETCHW) that were supplied by DRAM.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_RFO.DRAM", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x73C000002", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts demand reads for ownership (RFO) requests and software prefetches for exclusive ownership (PREFETCHW) that were supplied by DRAM attached to this socket, unless in Sub NUMA Cluster(SNC) Mode. In SNC Mode counts only those DRAM accesses that are controlled by the close SNC Cluster.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_RFO.LOCAL_DRAM", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x104000002", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts demand reads for ownership (RFO) requests and software prefetches for exclusive ownership (PREFETCHW) that were supplied by DRAM on a distant memory controller of this socket when the system is in SNC (sub-NUMA cluster) mode.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_RFO.SNC_DRAM", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x708000002", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts data load hardware prefetch requests to the L1 data cache that have any type of response.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.HWPF_L1D.ANY_RESPONSE", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x10400", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts hardware prefetches (which bring data to L2) that have any type of response.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.HWPF_L2.ANY_RESPONSE", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x10070", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts hardware prefetches to the L3 only that have any type of response.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.HWPF_L3.ANY_RESPONSE", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x12380", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts hardware prefetches to the L3 only that were not supplied by the local socket's L1, L2, or L3 caches and the cacheline was homed in a remote socket.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.HWPF_L3.REMOTE", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x90002380", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts writebacks of modified cachelines and streaming stores that have any type of response.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.MODIFIED_WRITE.ANY_RESPONSE", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x10808", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts all (cacheable) data read, code read and RFO requests including demands and prefetches to the core caches (L1 or L2) that have any type of response.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.READS_TO_CORE.ANY_RESPONSE", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x3F3FFC4477", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts all (cacheable) data read, code read and RFO requests including demands and prefetches to the core caches (L1 or L2) that were supplied by DRAM.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.READS_TO_CORE.DRAM", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x73C004477", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts all (cacheable) data read, code read and RFO requests including demands and prefetches to the core caches (L1 or L2) that were supplied by DRAM attached to this socket, unless in Sub NUMA Cluster(SNC) Mode. In SNC Mode counts only those DRAM accesses that are controlled by the close SNC Cluster.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.READS_TO_CORE.LOCAL_DRAM", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x104004477", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts all (cacheable) data read, code read and RFO requests including demands and prefetches to the core caches (L1 or L2) that were supplied by DRAM attached to this socket, whether or not in Sub NUMA Cluster(SNC) Mode. In SNC Mode counts DRAM accesses that are controlled by the close or distant SNC Cluster.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.READS_TO_CORE.LOCAL_SOCKET_DRAM", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x70C004477", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts all (cacheable) data read, code read and RFO requests including demands and prefetches to the core caches (L1 or L2) that were not supplied by the local socket's L1, L2, or L3 caches and were supplied by a remote socket.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.READS_TO_CORE.REMOTE", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x3F33004477", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts all (cacheable) data read, code read and RFO requests including demands and prefetches to the core caches (L1 or L2) that were supplied by DRAM attached to another socket.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.READS_TO_CORE.REMOTE_DRAM", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x730004477", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts all (cacheable) data read, code read and RFO requests including demands and prefetches to the core caches (L1 or L2) that were supplied by DRAM or PMM attached to another socket.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.READS_TO_CORE.REMOTE_MEMORY", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x733004477", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts all (cacheable) data read, code read and RFO requests including demands and prefetches to the core caches (L1 or L2) that were supplied by DRAM on a distant memory controller of this socket when the system is in SNC (sub-NUMA cluster) mode.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.READS_TO_CORE.SNC_DRAM", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x708004477", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts streaming stores that have any type of response.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.STREAMING_WR.ANY_RESPONSE", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x10800", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts Demand RFOs, ItoM's, PREFECTHW's, Hardware RFO Prefetches to the L1/L2 and Streaming stores that likely resulted in a store to Memory (DRAM or PMM)", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.WRITE_ESTIMATE.MEMORY", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0xFBFF80822", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Cycles when Reservation Station (RS) is empty for the thread.", + "EventCode": "0xa5", + "EventName": "RS.EMPTY", + "PublicDescription": "Counts cycles during which the reservation station (RS) is empty for this logical processor. This is usually caused when the front-end pipeline runs into starvation periods (e.g. branch mispredictions or i-cache misses)", + "SampleAfterValue": "1000003", + "UMask": "0x7" + }, + { + "BriefDescription": "Counts end of periods where the Reservation Station (RS) was empty.", + "CounterMask": "1", + "EdgeDetect": "1", + "EventCode": "0xa5", + "EventName": "RS.EMPTY_COUNT", + "Invert": "1", + "PublicDescription": "Counts end of periods where the Reservation Station (RS) was empty. Could be useful to closely sample on front-end latency issues (see the FRONTEND_RETIRED event of designated precise events)", + "SampleAfterValue": "100003", + "UMask": "0x7" + }, + { + "BriefDescription": "This event is deprecated. Refer to new event RS.EMPTY_COUNT", + "CounterMask": "1", + "Deprecated": "1", + "EdgeDetect": "1", + "EventCode": "0xa5", + "EventName": "RS_EMPTY.COUNT", + "Invert": "1", + "SampleAfterValue": "100003", + "UMask": "0x7" + }, + { + "BriefDescription": "This event is deprecated. Refer to new event RS.EMPTY", + "Deprecated": "1", + "EventCode": "0xa5", + "EventName": "RS_EMPTY.CYCLES", + "SampleAfterValue": "1000003", + "UMask": "0x7" + }, + { + "BriefDescription": "Cycles the uncore cannot take further requests", + "CounterMask": "1", + "EventCode": "0x2d", + "EventName": "XQ.FULL_CYCLES", + "PublicDescription": "number of cycles when the thread is active and the uncore cannot take any further requests (for example prefetches, loads or stores initiated by the Core that miss the L2 cache).", + "SampleAfterValue": "1000003", + "UMask": "0x1" + } +] diff --git a/tools/perf/pmu-events/arch/x86/emeraldrapids/pipeline.json b/tools/perf/pmu-events/arch/x86/emeraldrapids/pipeline.json new file mode 100644 index 000000000000..6dcf3b763af4 --- /dev/null +++ b/tools/perf/pmu-events/arch/x86/emeraldrapids/pipeline.json @@ -0,0 +1,975 @@ +[ + { + "BriefDescription": "AMX retired arithmetic BF16 operations.", + "EventCode": "0xce", + "EventName": "AMX_OPS_RETIRED.BF16", + "PublicDescription": "Number of AMX-based retired arithmetic bfloat16 (BF16) floating-point operations. Counts TDPBF16PS FP instructions. SW to use operation multiplier of 4", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "AMX retired arithmetic integer 8-bit operations.", + "EventCode": "0xce", + "EventName": "AMX_OPS_RETIRED.INT8", + "PublicDescription": "Number of AMX-based retired arithmetic integer operations of 8-bit width source operands. Counts TDPB[SS,UU,US,SU]D instructions. SW should use operation multiplier of 8.", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "This event is deprecated. Refer to new event ARITH.DIV_ACTIVE", + "CounterMask": "1", + "Deprecated": "1", + "EventCode": "0xb0", + "EventName": "ARITH.DIVIDER_ACTIVE", + "SampleAfterValue": "1000003", + "UMask": "0x9" + }, + { + "BriefDescription": "Cycles when divide unit is busy executing divide or square root operations.", + "CounterMask": "1", + "EventCode": "0xb0", + "EventName": "ARITH.DIV_ACTIVE", + "PublicDescription": "Counts cycles when divide unit is busy executing divide or square root operations. Accounts for integer and floating-point operations.", + "SampleAfterValue": "1000003", + "UMask": "0x9" + }, + { + "BriefDescription": "This event is deprecated. Refer to new event ARITH.FPDIV_ACTIVE", + "CounterMask": "1", + "Deprecated": "1", + "EventCode": "0xb0", + "EventName": "ARITH.FP_DIVIDER_ACTIVE", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "This event counts the cycles the integer divider is busy.", + "CounterMask": "1", + "EventCode": "0xb0", + "EventName": "ARITH.IDIV_ACTIVE", + "SampleAfterValue": "1000003", + "UMask": "0x8" + }, + { + "BriefDescription": "This event is deprecated. Refer to new event ARITH.IDIV_ACTIVE", + "CounterMask": "1", + "Deprecated": "1", + "EventCode": "0xb0", + "EventName": "ARITH.INT_DIVIDER_ACTIVE", + "SampleAfterValue": "1000003", + "UMask": "0x8" + }, + { + "BriefDescription": "Number of occurrences where a microcode assist is invoked by hardware.", + "EventCode": "0xc1", + "EventName": "ASSISTS.ANY", + "PublicDescription": "Counts the number of occurrences where a microcode assist is invoked by hardware. Examples include AD (page Access Dirty), FP and AVX related assists.", + "SampleAfterValue": "100003", + "UMask": "0x1b" + }, + { + "BriefDescription": "All branch instructions retired.", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.ALL_BRANCHES", + "PEBS": "1", + "PublicDescription": "Counts all branch instructions retired.", + "SampleAfterValue": "400009" + }, + { + "BriefDescription": "Conditional branch instructions retired.", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.COND", + "PEBS": "1", + "PublicDescription": "Counts conditional branch instructions retired.", + "SampleAfterValue": "400009", + "UMask": "0x11" + }, + { + "BriefDescription": "Not taken branch instructions retired.", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.COND_NTAKEN", + "PEBS": "1", + "PublicDescription": "Counts not taken branch instructions retired.", + "SampleAfterValue": "400009", + "UMask": "0x10" + }, + { + "BriefDescription": "Taken conditional branch instructions retired.", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.COND_TAKEN", + "PEBS": "1", + "PublicDescription": "Counts taken conditional branch instructions retired.", + "SampleAfterValue": "400009", + "UMask": "0x1" + }, + { + "BriefDescription": "Far branch instructions retired.", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.FAR_BRANCH", + "PEBS": "1", + "PublicDescription": "Counts far branch instructions retired.", + "SampleAfterValue": "100007", + "UMask": "0x40" + }, + { + "BriefDescription": "Indirect near branch instructions retired (excluding returns)", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.INDIRECT", + "PEBS": "1", + "PublicDescription": "Counts near indirect branch instructions retired excluding returns. TSX abort is an indirect branch.", + "SampleAfterValue": "100003", + "UMask": "0x80" + }, + { + "BriefDescription": "Direct and indirect near call instructions retired.", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.NEAR_CALL", + "PEBS": "1", + "PublicDescription": "Counts both direct and indirect near call instructions retired.", + "SampleAfterValue": "100007", + "UMask": "0x2" + }, + { + "BriefDescription": "Return instructions retired.", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.NEAR_RETURN", + "PEBS": "1", + "PublicDescription": "Counts return instructions retired.", + "SampleAfterValue": "100007", + "UMask": "0x8" + }, + { + "BriefDescription": "Taken branch instructions retired.", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.NEAR_TAKEN", + "PEBS": "1", + "PublicDescription": "Counts taken branch instructions retired.", + "SampleAfterValue": "400009", + "UMask": "0x20" + }, + { + "BriefDescription": "All mispredicted branch instructions retired.", + "EventCode": "0xc5", + "EventName": "BR_MISP_RETIRED.ALL_BRANCHES", + "PEBS": "1", + "PublicDescription": "Counts all the retired branch instructions that were mispredicted by the processor. A branch misprediction occurs when the processor incorrectly predicts the destination of the branch. When the misprediction is discovered at execution, all the instructions executed in the wrong (speculative) path must be discarded, and the processor must start fetching from the correct path.", + "SampleAfterValue": "400009" + }, + { + "BriefDescription": "Mispredicted conditional branch instructions retired.", + "EventCode": "0xc5", + "EventName": "BR_MISP_RETIRED.COND", + "PEBS": "1", + "PublicDescription": "Counts mispredicted conditional branch instructions retired.", + "SampleAfterValue": "400009", + "UMask": "0x11" + }, + { + "BriefDescription": "Mispredicted non-taken conditional branch instructions retired.", + "EventCode": "0xc5", + "EventName": "BR_MISP_RETIRED.COND_NTAKEN", + "PEBS": "1", + "PublicDescription": "Counts the number of conditional branch instructions retired that were mispredicted and the branch direction was not taken.", + "SampleAfterValue": "400009", + "UMask": "0x10" + }, + { + "BriefDescription": "number of branch instructions retired that were mispredicted and taken.", + "EventCode": "0xc5", + "EventName": "BR_MISP_RETIRED.COND_TAKEN", + "PEBS": "1", + "PublicDescription": "Counts taken conditional mispredicted branch instructions retired.", + "SampleAfterValue": "400009", + "UMask": "0x1" + }, + { + "BriefDescription": "Miss-predicted near indirect branch instructions retired (excluding returns)", + "EventCode": "0xc5", + "EventName": "BR_MISP_RETIRED.INDIRECT", + "PEBS": "1", + "PublicDescription": "Counts miss-predicted near indirect branch instructions retired excluding returns. TSX abort is an indirect branch.", + "SampleAfterValue": "100003", + "UMask": "0x80" + }, + { + "BriefDescription": "Mispredicted indirect CALL retired.", + "EventCode": "0xc5", + "EventName": "BR_MISP_RETIRED.INDIRECT_CALL", + "PEBS": "1", + "PublicDescription": "Counts retired mispredicted indirect (near taken) CALL instructions, including both register and memory indirect.", + "SampleAfterValue": "400009", + "UMask": "0x2" + }, + { + "BriefDescription": "Number of near branch instructions retired that were mispredicted and taken.", + "EventCode": "0xc5", + "EventName": "BR_MISP_RETIRED.NEAR_TAKEN", + "PEBS": "1", + "PublicDescription": "Counts number of near branch instructions retired that were mispredicted and taken.", + "SampleAfterValue": "400009", + "UMask": "0x20" + }, + { + "BriefDescription": "This event counts the number of mispredicted ret instructions retired. Non PEBS", + "EventCode": "0xc5", + "EventName": "BR_MISP_RETIRED.RET", + "PEBS": "1", + "PublicDescription": "This is a non-precise version (that is, does not use PEBS) of the event that counts mispredicted return instructions retired.", + "SampleAfterValue": "100007", + "UMask": "0x8" + }, + { + "BriefDescription": "Core clocks when the thread is in the C0.1 light-weight slower wakeup time but more power saving optimized state.", + "EventCode": "0xec", + "EventName": "CPU_CLK_UNHALTED.C01", + "PublicDescription": "Counts core clocks when the thread is in the C0.1 light-weight slower wakeup time but more power saving optimized state. This state can be entered via the TPAUSE or UMWAIT instructions.", + "SampleAfterValue": "2000003", + "UMask": "0x10" + }, + { + "BriefDescription": "Core clocks when the thread is in the C0.2 light-weight faster wakeup time but less power saving optimized state.", + "EventCode": "0xec", + "EventName": "CPU_CLK_UNHALTED.C02", + "PublicDescription": "Counts core clocks when the thread is in the C0.2 light-weight faster wakeup time but less power saving optimized state. This state can be entered via the TPAUSE or UMWAIT instructions.", + "SampleAfterValue": "2000003", + "UMask": "0x20" + }, + { + "BriefDescription": "Core clocks when the thread is in the C0.1 or C0.2 or running a PAUSE in C0 ACPI state.", + "EventCode": "0xec", + "EventName": "CPU_CLK_UNHALTED.C0_WAIT", + "PublicDescription": "Counts core clocks when the thread is in the C0.1 or C0.2 power saving optimized states (TPAUSE or UMWAIT instructions) or running the PAUSE instruction.", + "SampleAfterValue": "2000003", + "UMask": "0x70" + }, + { + "BriefDescription": "Cycle counts are evenly distributed between active threads in the Core.", + "EventCode": "0xec", + "EventName": "CPU_CLK_UNHALTED.DISTRIBUTED", + "PublicDescription": "This event distributes cycle counts between active hyperthreads, i.e., those in C0. A hyperthread becomes inactive when it executes the HLT or MWAIT instructions. If all other hyperthreads are inactive (or disabled or do not exist), all counts are attributed to this hyperthread. To obtain the full count when the Core is active, sum the counts from each hyperthread.", + "SampleAfterValue": "2000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Core crystal clock cycles when this thread is unhalted and the other thread is halted.", + "EventCode": "0x3c", + "EventName": "CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE", + "PublicDescription": "Counts Core crystal clock cycles when current thread is unhalted and the other thread is halted.", + "SampleAfterValue": "25003", + "UMask": "0x2" + }, + { + "BriefDescription": "CPU_CLK_UNHALTED.PAUSE", + "EventCode": "0xec", + "EventName": "CPU_CLK_UNHALTED.PAUSE", + "SampleAfterValue": "2000003", + "UMask": "0x40" + }, + { + "BriefDescription": "CPU_CLK_UNHALTED.PAUSE_INST", + "CounterMask": "1", + "EdgeDetect": "1", + "EventCode": "0xec", + "EventName": "CPU_CLK_UNHALTED.PAUSE_INST", + "SampleAfterValue": "2000003", + "UMask": "0x40" + }, + { + "BriefDescription": "Core crystal clock cycles. Cycle counts are evenly distributed between active threads in the Core.", + "EventCode": "0x3c", + "EventName": "CPU_CLK_UNHALTED.REF_DISTRIBUTED", + "PublicDescription": "This event distributes Core crystal clock cycle counts between active hyperthreads, i.e., those in C0 sleep-state. A hyperthread becomes inactive when it executes the HLT or MWAIT instructions. If one thread is active in a core, all counts are attributed to this hyperthread. To obtain the full count when the Core is active, sum the counts from each hyperthread.", + "SampleAfterValue": "2000003", + "UMask": "0x8" + }, + { + "BriefDescription": "Reference cycles when the core is not in halt state.", + "EventName": "CPU_CLK_UNHALTED.REF_TSC", + "PublicDescription": "Counts the number of reference cycles when the core is not in a halt state. The core enters the halt state when it is running the HLT instruction or the MWAIT instruction. This event is not affected by core frequency changes (for example, P states, TM2 transitions) but has the same incrementing frequency as the time stamp counter. This event can approximate elapsed time while the core was not in a halt state. It is counted on a dedicated fixed counter, leaving the eight programmable counters available for other events. Note: On all current platforms this event stops counting during 'throttling (TM)' states duty off periods the processor is 'halted'. The counter update is done at a lower clock rate then the core clock the overflow status bit for this counter may appear 'sticky'. After the counter has overflowed and software clears the overflow status bit and resets the counter to less than MAX. The reset value to the counter is not clocked immediately so the overflow status bit will flip 'high (1)' and generate another PMI (if enabled) after which the reset value gets clocked into the counter. Therefore, software will get the interrupt, read the overflow status bit '1 for bit 34 while the counter value is less than MAX. Software should ignore this case.", + "SampleAfterValue": "2000003", + "UMask": "0x3" + }, + { + "BriefDescription": "Reference cycles when the core is not in halt state.", + "EventCode": "0x3c", + "EventName": "CPU_CLK_UNHALTED.REF_TSC_P", + "PublicDescription": "Counts the number of reference cycles when the core is not in a halt state. The core enters the halt state when it is running the HLT instruction or the MWAIT instruction. This event is not affected by core frequency changes (for example, P states, TM2 transitions) but has the same incrementing frequency as the time stamp counter. This event can approximate elapsed time while the core was not in a halt state. It is counted on a dedicated fixed counter, leaving the four (eight when Hyperthreading is disabled) programmable counters available for other events. Note: On all current platforms this event stops counting during 'throttling (TM)' states duty off periods the processor is 'halted'. The counter update is done at a lower clock rate then the core clock the overflow status bit for this counter may appear 'sticky'. After the counter has overflowed and software clears the overflow status bit and resets the counter to less than MAX. The reset value to the counter is not clocked immediately so the overflow status bit will flip 'high (1)' and generate another PMI (if enabled) after which the reset value gets clocked into the counter. Therefore, software will get the interrupt, read the overflow status bit '1 for bit 34 while the counter value is less than MAX. Software should ignore this case.", + "SampleAfterValue": "2000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Core cycles when the thread is not in halt state", + "EventName": "CPU_CLK_UNHALTED.THREAD", + "PublicDescription": "Counts the number of core cycles while the thread is not in a halt state. The thread enters the halt state when it is running the HLT instruction. This event is a component in many key event ratios. The core frequency may change from time to time due to transitions associated with Enhanced Intel SpeedStep Technology or TM2. For this reason this event may have a changing ratio with regards to time. When the core frequency is constant, this event can approximate elapsed time while the core was not in the halt state. It is counted on a dedicated fixed counter, leaving the eight programmable counters available for other events.", + "SampleAfterValue": "2000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Thread cycles when thread is not in halt state", + "EventCode": "0x3c", + "EventName": "CPU_CLK_UNHALTED.THREAD_P", + "PublicDescription": "This is an architectural event that counts the number of thread cycles while the thread is not in a halt state. The thread enters the halt state when it is running the HLT instruction. The core frequency may change from time to time due to power or thermal throttling. For this reason, this event may have a changing ratio with regards to wall clock time.", + "SampleAfterValue": "2000003" + }, + { + "BriefDescription": "Cycles while L1 cache miss demand load is outstanding.", + "CounterMask": "8", + "EventCode": "0xa3", + "EventName": "CYCLE_ACTIVITY.CYCLES_L1D_MISS", + "SampleAfterValue": "1000003", + "UMask": "0x8" + }, + { + "BriefDescription": "Cycles while L2 cache miss demand load is outstanding.", + "CounterMask": "1", + "EventCode": "0xa3", + "EventName": "CYCLE_ACTIVITY.CYCLES_L2_MISS", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Cycles while memory subsystem has an outstanding load.", + "CounterMask": "16", + "EventCode": "0xa3", + "EventName": "CYCLE_ACTIVITY.CYCLES_MEM_ANY", + "SampleAfterValue": "1000003", + "UMask": "0x10" + }, + { + "BriefDescription": "Execution stalls while L1 cache miss demand load is outstanding.", + "CounterMask": "12", + "EventCode": "0xa3", + "EventName": "CYCLE_ACTIVITY.STALLS_L1D_MISS", + "SampleAfterValue": "1000003", + "UMask": "0xc" + }, + { + "BriefDescription": "Execution stalls while L2 cache miss demand load is outstanding.", + "CounterMask": "5", + "EventCode": "0xa3", + "EventName": "CYCLE_ACTIVITY.STALLS_L2_MISS", + "SampleAfterValue": "1000003", + "UMask": "0x5" + }, + { + "BriefDescription": "Total execution stalls.", + "CounterMask": "4", + "EventCode": "0xa3", + "EventName": "CYCLE_ACTIVITY.STALLS_TOTAL", + "SampleAfterValue": "1000003", + "UMask": "0x4" + }, + { + "BriefDescription": "Cycles total of 1 uop is executed on all ports and Reservation Station was not empty.", + "EventCode": "0xa6", + "EventName": "EXE_ACTIVITY.1_PORTS_UTIL", + "PublicDescription": "Counts cycles during which a total of 1 uop was executed on all ports and Reservation Station (RS) was not empty.", + "SampleAfterValue": "2000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Cycles total of 2 uops are executed on all ports and Reservation Station was not empty.", + "EventCode": "0xa6", + "EventName": "EXE_ACTIVITY.2_PORTS_UTIL", + "PublicDescription": "Counts cycles during which a total of 2 uops were executed on all ports and Reservation Station (RS) was not empty.", + "SampleAfterValue": "2000003", + "UMask": "0x4" + }, + { + "BriefDescription": "Cycles total of 3 uops are executed on all ports and Reservation Station was not empty.", + "EventCode": "0xa6", + "EventName": "EXE_ACTIVITY.3_PORTS_UTIL", + "PublicDescription": "Cycles total of 3 uops are executed on all ports and Reservation Station (RS) was not empty.", + "SampleAfterValue": "2000003", + "UMask": "0x8" + }, + { + "BriefDescription": "Cycles total of 4 uops are executed on all ports and Reservation Station was not empty.", + "EventCode": "0xa6", + "EventName": "EXE_ACTIVITY.4_PORTS_UTIL", + "PublicDescription": "Cycles total of 4 uops are executed on all ports and Reservation Station (RS) was not empty.", + "SampleAfterValue": "2000003", + "UMask": "0x10" + }, + { + "BriefDescription": "Execution stalls while memory subsystem has an outstanding load.", + "CounterMask": "5", + "EventCode": "0xa6", + "EventName": "EXE_ACTIVITY.BOUND_ON_LOADS", + "SampleAfterValue": "2000003", + "UMask": "0x21" + }, + { + "BriefDescription": "Cycles where the Store Buffer was full and no loads caused an execution stall.", + "CounterMask": "2", + "EventCode": "0xa6", + "EventName": "EXE_ACTIVITY.BOUND_ON_STORES", + "PublicDescription": "Counts cycles where the Store Buffer was full and no loads caused an execution stall.", + "SampleAfterValue": "1000003", + "UMask": "0x40" + }, + { + "BriefDescription": "Cycles no uop executed while RS was not empty, the SB was not full and there was no outstanding load.", + "EventCode": "0xa6", + "EventName": "EXE_ACTIVITY.EXE_BOUND_0_PORTS", + "PublicDescription": "Number of cycles total of 0 uops executed on all ports, Reservation Station (RS) was not empty, the Store Buffer (SB) was not full and there was no outstanding load.", + "SampleAfterValue": "1000003", + "UMask": "0x80" + }, + { + "BriefDescription": "Instruction decoders utilized in a cycle", + "EventCode": "0x75", + "EventName": "INST_DECODED.DECODERS", + "PublicDescription": "Number of decoders utilized in a cycle when the MITE (legacy decode pipeline) fetches instructions.", + "SampleAfterValue": "2000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Number of instructions retired. Fixed Counter - architectural event", + "EventName": "INST_RETIRED.ANY", + "PEBS": "1", + "PublicDescription": "Counts the number of X86 instructions retired - an Architectural PerfMon event. Counting continues during hardware interrupts, traps, and inside interrupt handlers. Notes: INST_RETIRED.ANY is counted by a designated fixed counter freeing up programmable counters to count other events. INST_RETIRED.ANY_P is counted by a programmable counter.", + "SampleAfterValue": "2000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Number of instructions retired. General Counter - architectural event", + "EventCode": "0xc0", + "EventName": "INST_RETIRED.ANY_P", + "PEBS": "1", + "PublicDescription": "Counts the number of X86 instructions retired - an Architectural PerfMon event. Counting continues during hardware interrupts, traps, and inside interrupt handlers. Notes: INST_RETIRED.ANY is counted by a designated fixed counter freeing up programmable counters to count other events. INST_RETIRED.ANY_P is counted by a programmable counter.", + "SampleAfterValue": "2000003" + }, + { + "BriefDescription": "INST_RETIRED.MACRO_FUSED", + "EventCode": "0xc0", + "EventName": "INST_RETIRED.MACRO_FUSED", + "SampleAfterValue": "2000003", + "UMask": "0x10" + }, + { + "BriefDescription": "Retired NOP instructions.", + "EventCode": "0xc0", + "EventName": "INST_RETIRED.NOP", + "PublicDescription": "Counts all retired NOP or ENDBR32/64 instructions", + "SampleAfterValue": "2000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Precise instruction retired with PEBS precise-distribution", + "EventName": "INST_RETIRED.PREC_DIST", + "PEBS": "1", + "PublicDescription": "A version of INST_RETIRED that allows for a precise distribution of samples across instructions retired. It utilizes the Precise Distribution of Instructions Retired (PDIR++) feature to fix bias in how retired instructions get sampled. Use on Fixed Counter 0.", + "SampleAfterValue": "2000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Iterations of Repeat string retired instructions.", + "EventCode": "0xc0", + "EventName": "INST_RETIRED.REP_ITERATION", + "PublicDescription": "Number of iterations of Repeat (REP) string retired instructions such as MOVS, CMPS, and SCAS. Each has a byte, word, and doubleword version and string instructions can be repeated using a repetition prefix, REP, that allows their architectural execution to be repeated a number of times as specified by the RCX register. Note the number of iterations is implementation-dependent.", + "SampleAfterValue": "2000003", + "UMask": "0x8" + }, + { + "BriefDescription": "Clears speculative count", + "CounterMask": "1", + "EdgeDetect": "1", + "EventCode": "0xad", + "EventName": "INT_MISC.CLEARS_COUNT", + "PublicDescription": "Counts the number of speculative clears due to any type of branch misprediction or machine clears", + "SampleAfterValue": "500009", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts cycles after recovery from a branch misprediction or machine clear till the first uop is issued from the resteered path.", + "EventCode": "0xad", + "EventName": "INT_MISC.CLEAR_RESTEER_CYCLES", + "PublicDescription": "Cycles after recovery from a branch misprediction or machine clear till the first uop is issued from the resteered path.", + "SampleAfterValue": "500009", + "UMask": "0x80" + }, + { + "BriefDescription": "INT_MISC.MBA_STALLS", + "EventCode": "0xad", + "EventName": "INT_MISC.MBA_STALLS", + "SampleAfterValue": "1000003", + "UMask": "0x20" + }, + { + "BriefDescription": "Core cycles the allocator was stalled due to recovery from earlier clear event for this thread", + "EventCode": "0xad", + "EventName": "INT_MISC.RECOVERY_CYCLES", + "PublicDescription": "Counts core cycles when the Resource allocator was stalled due to recovery from an earlier branch misprediction or machine clear event.", + "SampleAfterValue": "500009", + "UMask": "0x1" + }, + { + "BriefDescription": "INT_MISC.UNKNOWN_BRANCH_CYCLES", + "EventCode": "0xad", + "EventName": "INT_MISC.UNKNOWN_BRANCH_CYCLES", + "MSRIndex": "0x3F7", + "MSRValue": "0x7", + "SampleAfterValue": "1000003", + "UMask": "0x40" + }, + { + "BriefDescription": "TMA slots where uops got dropped", + "EventCode": "0xad", + "EventName": "INT_MISC.UOP_DROPPING", + "PublicDescription": "Estimated number of Top-down Microarchitecture Analysis slots that got dropped due to non front-end reasons", + "SampleAfterValue": "1000003", + "UMask": "0x10" + }, + { + "BriefDescription": "INT_VEC_RETIRED.128BIT", + "EventCode": "0xe7", + "EventName": "INT_VEC_RETIRED.128BIT", + "SampleAfterValue": "1000003", + "UMask": "0x13" + }, + { + "BriefDescription": "INT_VEC_RETIRED.256BIT", + "EventCode": "0xe7", + "EventName": "INT_VEC_RETIRED.256BIT", + "SampleAfterValue": "1000003", + "UMask": "0xac" + }, + { + "BriefDescription": "integer ADD, SUB, SAD 128-bit vector instructions.", + "EventCode": "0xe7", + "EventName": "INT_VEC_RETIRED.ADD_128", + "PublicDescription": "Number of retired integer ADD/SUB (regular or horizontal), SAD 128-bit vector instructions.", + "SampleAfterValue": "1000003", + "UMask": "0x3" + }, + { + "BriefDescription": "integer ADD, SUB, SAD 256-bit vector instructions.", + "EventCode": "0xe7", + "EventName": "INT_VEC_RETIRED.ADD_256", + "PublicDescription": "Number of retired integer ADD/SUB (regular or horizontal), SAD 256-bit vector instructions.", + "SampleAfterValue": "1000003", + "UMask": "0xc" + }, + { + "BriefDescription": "INT_VEC_RETIRED.MUL_256", + "EventCode": "0xe7", + "EventName": "INT_VEC_RETIRED.MUL_256", + "SampleAfterValue": "1000003", + "UMask": "0x80" + }, + { + "BriefDescription": "INT_VEC_RETIRED.SHUFFLES", + "EventCode": "0xe7", + "EventName": "INT_VEC_RETIRED.SHUFFLES", + "SampleAfterValue": "1000003", + "UMask": "0x40" + }, + { + "BriefDescription": "INT_VEC_RETIRED.VNNI_128", + "EventCode": "0xe7", + "EventName": "INT_VEC_RETIRED.VNNI_128", + "SampleAfterValue": "1000003", + "UMask": "0x10" + }, + { + "BriefDescription": "INT_VEC_RETIRED.VNNI_256", + "EventCode": "0xe7", + "EventName": "INT_VEC_RETIRED.VNNI_256", + "SampleAfterValue": "1000003", + "UMask": "0x20" + }, + { + "BriefDescription": "False dependencies in MOB due to partial compare on address.", + "EventCode": "0x03", + "EventName": "LD_BLOCKS.ADDRESS_ALIAS", + "PublicDescription": "Counts the number of times a load got blocked due to false dependencies in MOB due to partial compare on address.", + "SampleAfterValue": "100003", + "UMask": "0x4" + }, + { + "BriefDescription": "The number of times that split load operations are temporarily blocked because all resources for handling the split accesses are in use.", + "EventCode": "0x03", + "EventName": "LD_BLOCKS.NO_SR", + "PublicDescription": "Counts the number of times that split load operations are temporarily blocked because all resources for handling the split accesses are in use.", + "SampleAfterValue": "100003", + "UMask": "0x88" + }, + { + "BriefDescription": "Loads blocked due to overlapping with a preceding store that cannot be forwarded.", + "EventCode": "0x03", + "EventName": "LD_BLOCKS.STORE_FORWARD", + "PublicDescription": "Counts the number of times where store forwarding was prevented for a load operation. The most common case is a load blocked due to the address of memory access (partially) overlapping with a preceding uncompleted store. Note: See the table of not supported store forwards in the Optimization Guide.", + "SampleAfterValue": "100003", + "UMask": "0x82" + }, + { + "BriefDescription": "Counts the number of demand load dispatches that hit L1D fill buffer (FB) allocated for software prefetch.", + "EventCode": "0x4c", + "EventName": "LOAD_HIT_PREFETCH.SWPF", + "PublicDescription": "Counts all not software-prefetch load dispatches that hit the fill buffer (FB) allocated for the software prefetch. It can also be incremented by some lock instructions. So it should only be used with profiling so that the locks can be excluded by ASM (Assembly File) inspection of the nearby instructions.", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Cycles Uops delivered by the LSD, but didn't come from the decoder.", + "CounterMask": "1", + "EventCode": "0xa8", + "EventName": "LSD.CYCLES_ACTIVE", + "PublicDescription": "Counts the cycles when at least one uop is delivered by the LSD (Loop-stream detector).", + "SampleAfterValue": "2000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Cycles optimal number of Uops delivered by the LSD, but did not come from the decoder.", + "CounterMask": "6", + "EventCode": "0xa8", + "EventName": "LSD.CYCLES_OK", + "PublicDescription": "Counts the cycles when optimal number of uops is delivered by the LSD (Loop-stream detector).", + "SampleAfterValue": "2000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Number of Uops delivered by the LSD.", + "EventCode": "0xa8", + "EventName": "LSD.UOPS", + "PublicDescription": "Counts the number of uops delivered to the back-end by the LSD(Loop Stream Detector).", + "SampleAfterValue": "2000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Number of machine clears (nukes) of any type.", + "CounterMask": "1", + "EdgeDetect": "1", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.COUNT", + "PublicDescription": "Counts the number of machine clears (nukes) of any type.", + "SampleAfterValue": "100003", + "UMask": "0x1" + }, + { + "BriefDescription": "Self-modifying code (SMC) detected.", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.SMC", + "PublicDescription": "Counts self-modifying code (SMC) detected, which causes a machine clear.", + "SampleAfterValue": "100003", + "UMask": "0x4" + }, + { + "BriefDescription": "LFENCE instructions retired", + "EventCode": "0xe0", + "EventName": "MISC2_RETIRED.LFENCE", + "PublicDescription": "number of LFENCE retired instructions", + "SampleAfterValue": "400009", + "UMask": "0x20" + }, + { + "BriefDescription": "Increments whenever there is an update to the LBR array.", + "EventCode": "0xcc", + "EventName": "MISC_RETIRED.LBR_INSERTS", + "PublicDescription": "Increments when an entry is added to the Last Branch Record (LBR) array (or removed from the array in case of RETURNs in call stack mode). The event requires LBR enable via IA32_DEBUGCTL MSR and branch type selection via MSR_LBR_SELECT.", + "SampleAfterValue": "100003", + "UMask": "0x20" + }, + { + "BriefDescription": "Cycles stalled due to no store buffers available. (not including draining form sync).", + "EventCode": "0xa2", + "EventName": "RESOURCE_STALLS.SB", + "PublicDescription": "Counts allocation stall cycles caused by the store buffer (SB) being full. This counts cycles that the pipeline back-end blocked uop delivery from the front-end.", + "SampleAfterValue": "100003", + "UMask": "0x8" + }, + { + "BriefDescription": "Counts cycles where the pipeline is stalled due to serializing operations.", + "EventCode": "0xa2", + "EventName": "RESOURCE_STALLS.SCOREBOARD", + "SampleAfterValue": "100003", + "UMask": "0x2" + }, + { + "BriefDescription": "TMA slots where no uops were being issued due to lack of back-end resources.", + "EventCode": "0xa4", + "EventName": "TOPDOWN.BACKEND_BOUND_SLOTS", + "PublicDescription": "Number of slots in TMA method where no micro-operations were being issued from front-end to back-end of the machine due to lack of back-end resources.", + "SampleAfterValue": "10000003", + "UMask": "0x2" + }, + { + "BriefDescription": "TMA slots wasted due to incorrect speculations.", + "EventCode": "0xa4", + "EventName": "TOPDOWN.BAD_SPEC_SLOTS", + "PublicDescription": "Number of slots of TMA method that were wasted due to incorrect speculation. It covers all types of control-flow or data-related mis-speculations.", + "SampleAfterValue": "10000003", + "UMask": "0x4" + }, + { + "BriefDescription": "TMA slots wasted due to incorrect speculation by branch mispredictions", + "EventCode": "0xa4", + "EventName": "TOPDOWN.BR_MISPREDICT_SLOTS", + "PublicDescription": "Number of TMA slots that were wasted due to incorrect speculation by (any type of) branch mispredictions. This event estimates number of speculative operations that were issued but not retired as well as the out-of-order engine recovery past a branch misprediction.", + "SampleAfterValue": "10000003", + "UMask": "0x8" + }, + { + "BriefDescription": "TOPDOWN.MEMORY_BOUND_SLOTS", + "EventCode": "0xa4", + "EventName": "TOPDOWN.MEMORY_BOUND_SLOTS", + "SampleAfterValue": "10000003", + "UMask": "0x10" + }, + { + "BriefDescription": "TMA slots available for an unhalted logical processor. Fixed counter - architectural event", + "EventName": "TOPDOWN.SLOTS", + "PublicDescription": "Number of available slots for an unhalted logical processor. The event increments by machine-width of the narrowest pipeline as employed by the Top-down Microarchitecture Analysis method (TMA). The count is distributed among unhalted logical processors (hyper-threads) who share the same physical core. Software can use this event as the denominator for the top-level metrics of the TMA method. This architectural event is counted on a designated fixed counter (Fixed Counter 3).", + "SampleAfterValue": "10000003", + "UMask": "0x4" + }, + { + "BriefDescription": "TMA slots available for an unhalted logical processor. General counter - architectural event", + "EventCode": "0xa4", + "EventName": "TOPDOWN.SLOTS_P", + "PublicDescription": "Counts the number of available slots for an unhalted logical processor. The event increments by machine-width of the narrowest pipeline as employed by the Top-down Microarchitecture Analysis method. The count is distributed among unhalted logical processors (hyper-threads) who share the same physical core.", + "SampleAfterValue": "10000003", + "UMask": "0x1" + }, + { + "BriefDescription": "UOPS_DECODED.DEC0_UOPS", + "EventCode": "0x76", + "EventName": "UOPS_DECODED.DEC0_UOPS", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Uops executed on port 0", + "EventCode": "0xb2", + "EventName": "UOPS_DISPATCHED.PORT_0", + "PublicDescription": "Number of uops dispatch to execution port 0.", + "SampleAfterValue": "2000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Uops executed on port 1", + "EventCode": "0xb2", + "EventName": "UOPS_DISPATCHED.PORT_1", + "PublicDescription": "Number of uops dispatch to execution port 1.", + "SampleAfterValue": "2000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Uops executed on ports 2, 3 and 10", + "EventCode": "0xb2", + "EventName": "UOPS_DISPATCHED.PORT_2_3_10", + "PublicDescription": "Number of uops dispatch to execution ports 2, 3 and 10", + "SampleAfterValue": "2000003", + "UMask": "0x4" + }, + { + "BriefDescription": "Uops executed on ports 4 and 9", + "EventCode": "0xb2", + "EventName": "UOPS_DISPATCHED.PORT_4_9", + "PublicDescription": "Number of uops dispatch to execution ports 4 and 9", + "SampleAfterValue": "2000003", + "UMask": "0x10" + }, + { + "BriefDescription": "Uops executed on ports 5 and 11", + "EventCode": "0xb2", + "EventName": "UOPS_DISPATCHED.PORT_5_11", + "PublicDescription": "Number of uops dispatch to execution ports 5 and 11", + "SampleAfterValue": "2000003", + "UMask": "0x20" + }, + { + "BriefDescription": "Uops executed on port 6", + "EventCode": "0xb2", + "EventName": "UOPS_DISPATCHED.PORT_6", + "PublicDescription": "Number of uops dispatch to execution port 6.", + "SampleAfterValue": "2000003", + "UMask": "0x40" + }, + { + "BriefDescription": "Uops executed on ports 7 and 8", + "EventCode": "0xb2", + "EventName": "UOPS_DISPATCHED.PORT_7_8", + "PublicDescription": "Number of uops dispatch to execution ports 7 and 8.", + "SampleAfterValue": "2000003", + "UMask": "0x80" + }, + { + "BriefDescription": "Number of uops executed on the core.", + "EventCode": "0xb1", + "EventName": "UOPS_EXECUTED.CORE", + "PublicDescription": "Counts the number of uops executed from any thread.", + "SampleAfterValue": "2000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Cycles at least 1 micro-op is executed from any thread on physical core.", + "CounterMask": "1", + "EventCode": "0xb1", + "EventName": "UOPS_EXECUTED.CORE_CYCLES_GE_1", + "PublicDescription": "Counts cycles when at least 1 micro-op is executed from any thread on physical core.", + "SampleAfterValue": "2000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Cycles at least 2 micro-op is executed from any thread on physical core.", + "CounterMask": "2", + "EventCode": "0xb1", + "EventName": "UOPS_EXECUTED.CORE_CYCLES_GE_2", + "PublicDescription": "Counts cycles when at least 2 micro-ops are executed from any thread on physical core.", + "SampleAfterValue": "2000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Cycles at least 3 micro-op is executed from any thread on physical core.", + "CounterMask": "3", + "EventCode": "0xb1", + "EventName": "UOPS_EXECUTED.CORE_CYCLES_GE_3", + "PublicDescription": "Counts cycles when at least 3 micro-ops are executed from any thread on physical core.", + "SampleAfterValue": "2000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Cycles at least 4 micro-op is executed from any thread on physical core.", + "CounterMask": "4", + "EventCode": "0xb1", + "EventName": "UOPS_EXECUTED.CORE_CYCLES_GE_4", + "PublicDescription": "Counts cycles when at least 4 micro-ops are executed from any thread on physical core.", + "SampleAfterValue": "2000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Cycles where at least 1 uop was executed per-thread", + "CounterMask": "1", + "EventCode": "0xb1", + "EventName": "UOPS_EXECUTED.CYCLES_GE_1", + "PublicDescription": "Cycles where at least 1 uop was executed per-thread.", + "SampleAfterValue": "2000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Cycles where at least 2 uops were executed per-thread", + "CounterMask": "2", + "EventCode": "0xb1", + "EventName": "UOPS_EXECUTED.CYCLES_GE_2", + "PublicDescription": "Cycles where at least 2 uops were executed per-thread.", + "SampleAfterValue": "2000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Cycles where at least 3 uops were executed per-thread", + "CounterMask": "3", + "EventCode": "0xb1", + "EventName": "UOPS_EXECUTED.CYCLES_GE_3", + "PublicDescription": "Cycles where at least 3 uops were executed per-thread.", + "SampleAfterValue": "2000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Cycles where at least 4 uops were executed per-thread", + "CounterMask": "4", + "EventCode": "0xb1", + "EventName": "UOPS_EXECUTED.CYCLES_GE_4", + "PublicDescription": "Cycles where at least 4 uops were executed per-thread.", + "SampleAfterValue": "2000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts number of cycles no uops were dispatched to be executed on this thread.", + "CounterMask": "1", + "EventCode": "0xb1", + "EventName": "UOPS_EXECUTED.STALLS", + "Invert": "1", + "PublicDescription": "Counts cycles during which no uops were dispatched from the Reservation Station (RS) per thread.", + "SampleAfterValue": "2000003", + "UMask": "0x1" + }, + { + "BriefDescription": "This event is deprecated. Refer to new event UOPS_EXECUTED.STALLS", + "CounterMask": "1", + "Deprecated": "1", + "EventCode": "0xb1", + "EventName": "UOPS_EXECUTED.STALL_CYCLES", + "Invert": "1", + "SampleAfterValue": "2000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts the number of uops to be executed per-thread each cycle.", + "EventCode": "0xb1", + "EventName": "UOPS_EXECUTED.THREAD", + "SampleAfterValue": "2000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Counts the number of x87 uops dispatched.", + "EventCode": "0xb1", + "EventName": "UOPS_EXECUTED.X87", + "PublicDescription": "Counts the number of x87 uops executed.", + "SampleAfterValue": "2000003", + "UMask": "0x10" + }, + { + "BriefDescription": "Uops that RAT issues to RS", + "EventCode": "0xae", + "EventName": "UOPS_ISSUED.ANY", + "PublicDescription": "Counts the number of uops that the Resource Allocation Table (RAT) issues to the Reservation Station (RS).", + "SampleAfterValue": "2000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Cycles with retired uop(s).", + "CounterMask": "1", + "EventCode": "0xc2", + "EventName": "UOPS_RETIRED.CYCLES", + "PublicDescription": "Counts cycles where at least one uop has retired.", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Retired uops except the last uop of each instruction.", + "EventCode": "0xc2", + "EventName": "UOPS_RETIRED.HEAVY", + "PublicDescription": "Counts the number of retired micro-operations (uops) except the last uop of each instruction. An instruction that is decoded into less than two uops does not contribute to the count.", + "SampleAfterValue": "2000003", + "UMask": "0x1" + }, + { + "BriefDescription": "UOPS_RETIRED.MS", + "EventCode": "0xc2", + "EventName": "UOPS_RETIRED.MS", + "MSRIndex": "0x3F7", + "MSRValue": "0x8", + "SampleAfterValue": "2000003", + "UMask": "0x4" + }, + { + "BriefDescription": "Retirement slots used.", + "EventCode": "0xc2", + "EventName": "UOPS_RETIRED.SLOTS", + "PublicDescription": "Counts the retirement slots used each cycle.", + "SampleAfterValue": "2000003", + "UMask": "0x2" + }, + { + "BriefDescription": "Cycles without actually retired uops.", + "CounterMask": "1", + "EventCode": "0xc2", + "EventName": "UOPS_RETIRED.STALLS", + "Invert": "1", + "PublicDescription": "This event counts cycles without actually retired uops.", + "SampleAfterValue": "1000003", + "UMask": "0x2" + }, + { + "BriefDescription": "This event is deprecated. Refer to new event UOPS_RETIRED.STALLS", + "CounterMask": "1", + "Deprecated": "1", + "EventCode": "0xc2", + "EventName": "UOPS_RETIRED.STALL_CYCLES", + "Invert": "1", + "SampleAfterValue": "1000003", + "UMask": "0x2" + } +] diff --git a/tools/perf/pmu-events/arch/x86/emeraldrapids/uncore-cache.json b/tools/perf/pmu-events/arch/x86/emeraldrapids/uncore-cache.json new file mode 100644 index 000000000000..cf6fa70f37c1 --- /dev/null +++ b/tools/perf/pmu-events/arch/x86/emeraldrapids/uncore-cache.json @@ -0,0 +1,5952 @@ +[ + { + "BriefDescription": "CHA to iMC Bypass : Intermediate bypass Taken", + "EventCode": "0x57", + "EventName": "UNC_CHA_BYPASS_CHA_IMC.INTERMEDIATE", + "PerPkg": "1", + "PublicDescription": "CHA to iMC Bypass : Intermediate bypass Taken : Counts the number of times when the CHA was able to bypass HA pipe on the way to iMC. This is a latency optimization for situations when there is light loadings on the memory subsystem. This can be filtered by when the bypass was taken and when it was not. : Filter for transactions that succeeded in taking the intermediate bypass.", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "CHA to iMC Bypass : Not Taken", + "EventCode": "0x57", + "EventName": "UNC_CHA_BYPASS_CHA_IMC.NOT_TAKEN", + "PerPkg": "1", + "PublicDescription": "CHA to iMC Bypass : Not Taken : Counts the number of times when the CHA was able to bypass HA pipe on the way to iMC. This is a latency optimization for situations when there is light loadings on the memory subsystem. This can be filtered by when the bypass was taken and when it was not. : Filter for transactions that could not take the bypass, and issues a read to memory. Note that transactions that did not take the bypass but did not issue read to memory will not be counted.", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "CHA to iMC Bypass : Taken", + "EventCode": "0x57", + "EventName": "UNC_CHA_BYPASS_CHA_IMC.TAKEN", + "PerPkg": "1", + "PublicDescription": "CHA to iMC Bypass : Taken : Counts the number of times when the CHA was able to bypass HA pipe on the way to iMC. This is a latency optimization for situations when there is light loadings on the memory subsystem. This can be filtered by when the bypass was taken and when it was not. : Filter for transactions that succeeded in taking the full bypass.", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "CHA Clockticks", + "EventCode": "0x01", + "EventName": "UNC_CHA_CLOCKTICKS", + "PerPkg": "1", + "PublicDescription": "Number of CHA clock cycles while the event is enabled", + "Unit": "CHA" + }, + { + "BriefDescription": "CMS Clockticks", + "EventCode": "0xc0", + "EventName": "UNC_CHA_CMS_CLOCKTICKS", + "PerPkg": "1", + "Unit": "CHA" + }, + { + "BriefDescription": "Core Cross Snoops Issued : Any Cycle with Multiple Snoops", + "EventCode": "0x33", + "EventName": "UNC_CHA_CORE_SNP.ANY_GTONE", + "PerPkg": "1", + "PublicDescription": "Core Cross Snoops Issued : Any Cycle with Multiple Snoops : Counts the number of transactions that trigger a configurable number of cross snoops. Cores are snooped if the transaction looks up the cache and determines that it is necessary based on the operation type and what CoreValid bits are set. For example, if 2 CV bits are set on a data read, the cores must have the data in S state so it is not necessary to snoop them. However, if only 1 CV bit is set the core my have modified the data. If the transaction was an RFO, it would need to invalidate the lines. This event can be filtered based on who triggered the initial snoop(s).", + "UMask": "0xf2", + "Unit": "CHA" + }, + { + "BriefDescription": "Core Cross Snoops Issued : Any Single Snoop", + "EventCode": "0x33", + "EventName": "UNC_CHA_CORE_SNP.ANY_ONE", + "PerPkg": "1", + "PublicDescription": "Core Cross Snoops Issued : Any Single Snoop : Counts the number of transactions that trigger a configurable number of cross snoops. Cores are snooped if the transaction looks up the cache and determines that it is necessary based on the operation type and what CoreValid bits are set. For example, if 2 CV bits are set on a data read, the cores must have the data in S state so it is not necessary to snoop them. However, if only 1 CV bit is set the core my have modified the data. If the transaction was an RFO, it would need to invalidate the lines. This event can be filtered based on who triggered the initial snoop(s).", + "UMask": "0xf1", + "Unit": "CHA" + }, + { + "BriefDescription": "Core Cross Snoops Issued : Multiple Core Requests", + "EventCode": "0x33", + "EventName": "UNC_CHA_CORE_SNP.CORE_GTONE", + "PerPkg": "1", + "PublicDescription": "Core Cross Snoops Issued : Multiple Core Requests : Counts the number of transactions that trigger a configurable number of cross snoops. Cores are snooped if the transaction looks up the cache and determines that it is necessary based on the operation type and what CoreValid bits are set. For example, if 2 CV bits are set on a data read, the cores must have the data in S state so it is not necessary to snoop them. However, if only 1 CV bit is set the core my have modified the data. If the transaction was an RFO, it would need to invalidate the lines. This event can be filtered based on who triggered the initial snoop(s).", + "UMask": "0x42", + "Unit": "CHA" + }, + { + "BriefDescription": "Core Cross Snoops Issued : Single Core Requests", + "EventCode": "0x33", + "EventName": "UNC_CHA_CORE_SNP.CORE_ONE", + "PerPkg": "1", + "PublicDescription": "Core Cross Snoops Issued : Single Core Requests : Counts the number of transactions that trigger a configurable number of cross snoops. Cores are snooped if the transaction looks up the cache and determines that it is necessary based on the operation type and what CoreValid bits are set. For example, if 2 CV bits are set on a data read, the cores must have the data in S state so it is not necessary to snoop them. However, if only 1 CV bit is set the core my have modified the data. If the transaction was an RFO, it would need to invalidate the lines. This event can be filtered based on who triggered the initial snoop(s).", + "UMask": "0x41", + "Unit": "CHA" + }, + { + "BriefDescription": "Core Cross Snoops Issued : Multiple Eviction", + "EventCode": "0x33", + "EventName": "UNC_CHA_CORE_SNP.EVICT_GTONE", + "PerPkg": "1", + "PublicDescription": "Core Cross Snoops Issued : Multiple Eviction : Counts the number of transactions that trigger a configurable number of cross snoops. Cores are snooped if the transaction looks up the cache and determines that it is necessary based on the operation type and what CoreValid bits are set. For example, if 2 CV bits are set on a data read, the cores must have the data in S state so it is not necessary to snoop them. However, if only 1 CV bit is set the core my have modified the data. If the transaction was an RFO, it would need to invalidate the lines. This event can be filtered based on who triggered the initial snoop(s).", + "UMask": "0x82", + "Unit": "CHA" + }, + { + "BriefDescription": "Core Cross Snoops Issued : Single Eviction", + "EventCode": "0x33", + "EventName": "UNC_CHA_CORE_SNP.EVICT_ONE", + "PerPkg": "1", + "PublicDescription": "Core Cross Snoops Issued : Single Eviction : Counts the number of transactions that trigger a configurable number of cross snoops. Cores are snooped if the transaction looks up the cache and determines that it is necessary based on the operation type and what CoreValid bits are set. For example, if 2 CV bits are set on a data read, the cores must have the data in S state so it is not necessary to snoop them. However, if only 1 CV bit is set the core my have modified the data. If the transaction was an RFO, it would need to invalidate the lines. This event can be filtered based on who triggered the initial snoop(s).", + "UMask": "0x81", + "Unit": "CHA" + }, + { + "BriefDescription": "Core Cross Snoops Issued : Multiple External Snoops", + "EventCode": "0x33", + "EventName": "UNC_CHA_CORE_SNP.EXT_GTONE", + "PerPkg": "1", + "PublicDescription": "Core Cross Snoops Issued : Multiple External Snoops : Counts the number of transactions that trigger a configurable number of cross snoops. Cores are snooped if the transaction looks up the cache and determines that it is necessary based on the operation type and what CoreValid bits are set. For example, if 2 CV bits are set on a data read, the cores must have the data in S state so it is not necessary to snoop them. However, if only 1 CV bit is set the core my have modified the data. If the transaction was an RFO, it would need to invalidate the lines. This event can be filtered based on who triggered the initial snoop(s).", + "UMask": "0x22", + "Unit": "CHA" + }, + { + "BriefDescription": "Core Cross Snoops Issued : Single External Snoops", + "EventCode": "0x33", + "EventName": "UNC_CHA_CORE_SNP.EXT_ONE", + "PerPkg": "1", + "PublicDescription": "Core Cross Snoops Issued : Single External Snoops : Counts the number of transactions that trigger a configurable number of cross snoops. Cores are snooped if the transaction looks up the cache and determines that it is necessary based on the operation type and what CoreValid bits are set. For example, if 2 CV bits are set on a data read, the cores must have the data in S state so it is not necessary to snoop them. However, if only 1 CV bit is set the core my have modified the data. If the transaction was an RFO, it would need to invalidate the lines. This event can be filtered based on who triggered the initial snoop(s).", + "UMask": "0x21", + "Unit": "CHA" + }, + { + "BriefDescription": "Core Cross Snoops Issued : Multiple Snoop Targets from Remote", + "EventCode": "0x33", + "EventName": "UNC_CHA_CORE_SNP.REMOTE_GTONE", + "PerPkg": "1", + "PublicDescription": "Core Cross Snoops Issued : Multiple Snoop Targets from Remote : Counts the number of transactions that trigger a configurable number of cross snoops. Cores are snooped if the transaction looks up the cache and determines that it is necessary based on the operation type and what CoreValid bits are set. For example, if 2 CV bits are set on a data read, the cores must have the data in S state so it is not necessary to snoop them. However, if only 1 CV bit is set the core my have modified the data. If the transaction was an RFO, it would need to invalidate the lines. This event can be filtered based on who triggered the initial snoop(s).", + "UMask": "0x12", + "Unit": "CHA" + }, + { + "BriefDescription": "Core Cross Snoops Issued : Single Snoop Target from Remote", + "EventCode": "0x33", + "EventName": "UNC_CHA_CORE_SNP.REMOTE_ONE", + "PerPkg": "1", + "PublicDescription": "Core Cross Snoops Issued : Single Snoop Target from Remote : Counts the number of transactions that trigger a configurable number of cross snoops. Cores are snooped if the transaction looks up the cache and determines that it is necessary based on the operation type and what CoreValid bits are set. For example, if 2 CV bits are set on a data read, the cores must have the data in S state so it is not necessary to snoop them. However, if only 1 CV bit is set the core my have modified the data. If the transaction was an RFO, it would need to invalidate the lines. This event can be filtered based on who triggered the initial snoop(s).", + "UMask": "0x11", + "Unit": "CHA" + }, + { + "BriefDescription": "Direct GO", + "EventCode": "0x6e", + "EventName": "UNC_CHA_DIRECT_GO.HA_SUPPRESS_DRD", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "Direct GO", + "EventCode": "0x6e", + "EventName": "UNC_CHA_DIRECT_GO.HA_SUPPRESS_NO_D2C", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "Direct GO", + "EventCode": "0x6e", + "EventName": "UNC_CHA_DIRECT_GO.HA_TOR_DEALLOC", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "Direct GO", + "EventCode": "0x6d", + "EventName": "UNC_CHA_DIRECT_GO_OPC.EXTCMP", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "Direct GO", + "EventCode": "0x6d", + "EventName": "UNC_CHA_DIRECT_GO_OPC.FAST_GO", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "CHA" + }, + { + "BriefDescription": "Direct GO", + "EventCode": "0x6d", + "EventName": "UNC_CHA_DIRECT_GO_OPC.FAST_GO_PULL", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "CHA" + }, + { + "BriefDescription": "Direct GO", + "EventCode": "0x6d", + "EventName": "UNC_CHA_DIRECT_GO_OPC.GO", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "Direct GO", + "EventCode": "0x6d", + "EventName": "UNC_CHA_DIRECT_GO_OPC.GO_PULL", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "CHA" + }, + { + "BriefDescription": "Direct GO", + "EventCode": "0x6d", + "EventName": "UNC_CHA_DIRECT_GO_OPC.IDLE_DUE_SUPPRESS", + "PerPkg": "1", + "UMask": "0x80", + "Unit": "CHA" + }, + { + "BriefDescription": "Direct GO", + "EventCode": "0x6d", + "EventName": "UNC_CHA_DIRECT_GO_OPC.NOP", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "CHA" + }, + { + "BriefDescription": "Direct GO", + "EventCode": "0x6d", + "EventName": "UNC_CHA_DIRECT_GO_OPC.PULL", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "Multi-socket cacheline Directory state lookups; Snoop Not Needed", + "EventCode": "0x53", + "EventName": "UNC_CHA_DIR_LOOKUP.NO_SNP", + "PerPkg": "1", + "PublicDescription": "Counts transactions that looked into the multi-socket cacheline Directory state, and therefore did not send a snoop because the Directory indicated it was not needed.", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "Multi-socket cacheline Directory state lookups; Snoop Needed", + "EventCode": "0x53", + "EventName": "UNC_CHA_DIR_LOOKUP.SNP", + "PerPkg": "1", + "PublicDescription": "Counts transactions that looked into the multi-socket cacheline Directory state, and sent one or more snoops, because the Directory indicated it was needed.", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "Multi-socket cacheline Directory state updates; Directory Updated memory write from the HA pipe", + "EventCode": "0x54", + "EventName": "UNC_CHA_DIR_UPDATE.HA", + "PerPkg": "1", + "PublicDescription": "Counts only multi-socket cacheline Directory state updates memory writes issued from the HA pipe. This does not include memory write requests which are for I (Invalid) or E (Exclusive) cachelines.", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "Multi-socket cacheline Directory state updates; Directory Updated memory write from TOR pipe", + "EventCode": "0x54", + "EventName": "UNC_CHA_DIR_UPDATE.TOR", + "PerPkg": "1", + "PublicDescription": "Counts only multi-socket cacheline Directory state updates due to memory writes issued from the TOR pipe which are the result of remote transaction hitting the SF/LLC and returning data Core2Core. This does not include memory write requests which are for I (Invalid) or E (Exclusive) cachelines.", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "Egress Blocking due to Ordering requirements : Down", + "EventCode": "0xba", + "EventName": "UNC_CHA_EGRESS_ORDERING.IV_SNOOPGO_DN", + "PerPkg": "1", + "PublicDescription": "Egress Blocking due to Ordering requirements : Down : Counts number of cycles IV was blocked in the TGR Egress due to SNP/GO Ordering requirements", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "Egress Blocking due to Ordering requirements : Up", + "EventCode": "0xba", + "EventName": "UNC_CHA_EGRESS_ORDERING.IV_SNOOPGO_UP", + "PerPkg": "1", + "PublicDescription": "Egress Blocking due to Ordering requirements : Up : Counts number of cycles IV was blocked in the TGR Egress due to SNP/GO Ordering requirements", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "Read request from a remote socket which hit in the HitMe Cache to a line In the E state", + "EventCode": "0x5f", + "EventName": "UNC_CHA_HITME_HIT.EX_RDS", + "PerPkg": "1", + "PublicDescription": "Counts read requests from a remote socket which hit in the HitME cache (used to cache the multi-socket Directory state) to a line in the E(Exclusive) state. This includes the following read opcodes (RdCode, RdData, RdDataMigratory, RdCur, RdInv*, Inv*).", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "Counts Number of Hits in HitMe Cache : Shared hit and op is RdInvOwn, RdInv, Inv*", + "EventCode": "0x5f", + "EventName": "UNC_CHA_HITME_HIT.SHARED_OWNREQ", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "Counts Number of Hits in HitMe Cache : op is WbMtoE", + "EventCode": "0x5f", + "EventName": "UNC_CHA_HITME_HIT.WBMTOE", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "CHA" + }, + { + "BriefDescription": "Counts Number of Hits in HitMe Cache : op is WbMtoI, WbPushMtoI, WbFlush, or WbMtoS", + "EventCode": "0x5f", + "EventName": "UNC_CHA_HITME_HIT.WBMTOI_OR_S", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "CHA" + }, + { + "BriefDescription": "Counts Number of times HitMe Cache is accessed : op is RdCode, RdData, RdDataMigratory, RdCur, RdInvOwn, RdInv, Inv*", + "EventCode": "0x5e", + "EventName": "UNC_CHA_HITME_LOOKUP.READ", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "Counts Number of times HitMe Cache is accessed : op is WbMtoE, WbMtoI, WbPushMtoI, WbFlush, or WbMtoS", + "EventCode": "0x5e", + "EventName": "UNC_CHA_HITME_LOOKUP.WRITE", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "Counts Number of Misses in HitMe Cache : No SF/LLC HitS/F and op is RdInvOwn", + "EventCode": "0x60", + "EventName": "UNC_CHA_HITME_MISS.NOTSHARED_RDINVOWN", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "CHA" + }, + { + "BriefDescription": "Counts Number of Misses in HitMe Cache : op is RdCode, RdData, RdDataMigratory, RdCur, RdInv, Inv*", + "EventCode": "0x60", + "EventName": "UNC_CHA_HITME_MISS.READ_OR_INV", + "PerPkg": "1", + "UMask": "0x80", + "Unit": "CHA" + }, + { + "BriefDescription": "Counts Number of Misses in HitMe Cache : SF/LLC HitS/F and op is RdInvOwn", + "EventCode": "0x60", + "EventName": "UNC_CHA_HITME_MISS.SHARED_RDINVOWN", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "CHA" + }, + { + "BriefDescription": "Counts the number of Allocate/Update to HitMe Cache : Deallocate HitME$ on Reads without RspFwdI*", + "EventCode": "0x61", + "EventName": "UNC_CHA_HITME_UPDATE.DEALLOCATE", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "CHA" + }, + { + "BriefDescription": "Counts the number of Allocate/Update to HitMe Cache : op is RspIFwd or RspIFwdWb for a local request", + "EventCode": "0x61", + "EventName": "UNC_CHA_HITME_UPDATE.DEALLOCATE_RSPFWDI_LOC", + "PerPkg": "1", + "PublicDescription": "Counts the number of Allocate/Update to HitMe Cache : op is RspIFwd or RspIFwdWb for a local request : Received RspFwdI* for a local request, but converted HitME$ to SF entry", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "Counts the number of Allocate/Update to HitMe Cache : Update HitMe Cache on RdInvOwn even if not RspFwdI*", + "EventCode": "0x61", + "EventName": "UNC_CHA_HITME_UPDATE.RDINVOWN", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "CHA" + }, + { + "BriefDescription": "Counts the number of Allocate/Update to HitMe Cache : op is RspIFwd or RspIFwdWb for a remote request", + "EventCode": "0x61", + "EventName": "UNC_CHA_HITME_UPDATE.RSPFWDI_REM", + "PerPkg": "1", + "PublicDescription": "Counts the number of Allocate/Update to HitMe Cache : op is RspIFwd or RspIFwdWb for a remote request : Updated HitME$ on RspFwdI* or local HitM/E received for a remote request", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "Counts the number of Allocate/Update to HitMe Cache : Update HitMe Cache to SHARed", + "EventCode": "0x61", + "EventName": "UNC_CHA_HITME_UPDATE.SHARED", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "Normal priority reads issued to the memory controller from the CHA", + "EventCode": "0x59", + "EventName": "UNC_CHA_IMC_READS_COUNT.NORMAL", + "PerPkg": "1", + "PublicDescription": "Counts when a normal (Non-Isochronous) read is issued to any of the memory controller channels from the CHA.", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "HA to iMC Reads Issued : ISOCH", + "EventCode": "0x59", + "EventName": "UNC_CHA_IMC_READS_COUNT.PRIORITY", + "PerPkg": "1", + "PublicDescription": "HA to iMC Reads Issued : ISOCH : Count of the number of reads issued to any of the memory controller channels. This can be filtered by the priority of the reads.", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "CHA to iMC Full Line Writes Issued; Full Line Non-ISOCH", + "EventCode": "0x5b", + "EventName": "UNC_CHA_IMC_WRITES_COUNT.FULL", + "PerPkg": "1", + "PublicDescription": "Counts when a normal (Non-Isochronous) full line write is issued from the CHA to the any of the memory controller channels.", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "CHA to iMC Full Line Writes Issued : ISOCH Full Line", + "EventCode": "0x5b", + "EventName": "UNC_CHA_IMC_WRITES_COUNT.FULL_PRIORITY", + "PerPkg": "1", + "PublicDescription": "CHA to iMC Full Line Writes Issued : ISOCH Full Line : Counts the total number of full line writes issued from the HA into the memory controller.", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "CHA to iMC Full Line Writes Issued : Partial Non-ISOCH", + "EventCode": "0x5b", + "EventName": "UNC_CHA_IMC_WRITES_COUNT.PARTIAL", + "PerPkg": "1", + "PublicDescription": "CHA to iMC Full Line Writes Issued : Partial Non-ISOCH : Counts the total number of full line writes issued from the HA into the memory controller.", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "CHA to iMC Full Line Writes Issued : ISOCH Partial", + "EventCode": "0x5b", + "EventName": "UNC_CHA_IMC_WRITES_COUNT.PARTIAL_PRIORITY", + "PerPkg": "1", + "PublicDescription": "CHA to iMC Full Line Writes Issued : ISOCH Partial : Counts the total number of full line writes issued from the HA into the memory controller.", + "UMask": "0x8", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache and Snoop Filter Lookups; Any Request", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.ALL", + "PerPkg": "1", + "PublicDescription": "Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. CHAFilter0[24:21,17] bits correspond to [FMESI] state.; Filters for any transaction originating from the IPQ or IRQ. This does not include lookups originating from the ISMQ.", + "UMask": "0x1fffff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : All transactions from Remote Agents", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.ALL_REMOTE", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : All transactions from Remote Agents : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS select a state or states (in the umask field) to match. Otherwise, the event will count nothing.", + "UMask": "0x17e0ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : All Requests", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.ANY_F", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : All Requests : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. : Any local or remote transaction to the LLC, including prefetch.", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : CRd Requests", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.CODE", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : CRd Requests : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. : Local or remote CRd transactions to the LLC. This includes CRd prefetch.", + "UMask": "0x1bd0ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : CRd Requests", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.CODE_READ_F", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : CRd Requests : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. : Local or remote CRd transactions to the LLC. This includes CRd prefetch.", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : Local non-prefetch requests", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.COREPREF_OR_DMND_LOCAL_F", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : Local non-prefetch requests : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. : Any local transaction to the LLC, not including prefetch", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache and Snoop Filter Lookups; Data Read Request", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.DATA_RD", + "PerPkg": "1", + "PublicDescription": "Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. CHAFilter0[24:21,17] bits correspond to [FMESI] state. Read transactions", + "UMask": "0x1bc1ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : Data Reads", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.DATA_READ_ALL", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : Data Reads : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS select a state or states (in the umask field) to match. Otherwise, the event will count nothing.", + "UMask": "0x1fc1ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : Data Read Request", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.DATA_READ_F", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : Data Read Request : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. : Read transactions.", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : Demand Data Reads, Core and LLC prefetches", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.DATA_READ_LOCAL", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : Demand Data Reads, Core and LLC prefetches : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS select a state or states (in the umask field) to match. Otherwise, the event will count nothing.", + "UMask": "0x841ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : Data Read Misses", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.DATA_READ_MISS", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : Data Read Misses : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS select a state or states (in the umask field) to match. Otherwise, the event will count nothing.", + "UMask": "0x1fc101", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : E State", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.E", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : E State : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. : Hit Exclusive State", + "UMask": "0x20", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : F State", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.F", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : F State : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. : Hit Forward State", + "UMask": "0x80", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : Flush or Invalidate Requests", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.FLUSH_INV", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : Flush : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing.", + "UMask": "0x1a44ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : Flush", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.FLUSH_OR_INV_F", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : Flush : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing.", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : I State", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.I", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : I State : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. : Miss", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : Local LLC prefetch requests (from LLC)", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.LLCPREF_LOCAL_F", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : Local LLC prefetch requests (from LLC) : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. : Any local LLC prefetch to the LLC", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : Transactions homed locally", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.LOCALLY_HOMED_ADDRESS", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : Transactions homed locally : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. : Transaction whose address resides in the local MC.", + "UMask": "0xbdfff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : CRd Requests that come from the local socket (usually the core)", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.LOCAL_CODE", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : CRd Requests : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. : Local or remote CRd transactions to the LLC. This includes CRd prefetch.", + "UMask": "0x19d0ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache and Snoop Filter Lookups; Data Read Request that come from the local socket (usually the core)", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.LOCAL_DATA_RD", + "PerPkg": "1", + "PublicDescription": "Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. CHAFilter0[24:21,17] bits correspond to [FMESI] state. Read transactions", + "UMask": "0x19c1ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : Demand CRd Requests that come from the local socket (usually the core)", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.LOCAL_DMND_CODE", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : CRd Requests : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. : Local or remote CRd transactions to the LLC. This includes CRd prefetch.", + "UMask": "0x1850ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache and Snoop Filter Lookups; Demand Data Reads that come from the local socket (usually the core)", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.LOCAL_DMND_DATA_RD", + "PerPkg": "1", + "PublicDescription": "Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. CHAFilter0[24:21,17] bits correspond to [FMESI] state. Read transactions", + "UMask": "0x1841ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : Demand RFO Requests that come from the local socket (usually the core)", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.LOCAL_DMND_RFO", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : RFO Requests : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. : Local or remote RFO transactions to the LLC. This includes RFO prefetch.", + "UMask": "0x1848ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : Transactions homed locally", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.LOCAL_F", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : Transactions homed locally : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. : Transaction whose address resides in the local MC.", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : Flush or Invalidate Requests that come from the local socket (usually the core)", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.LOCAL_FLUSH_INV", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : Flush : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing.", + "UMask": "0x1844ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache and Snoop Filter Lookups; Prefetch requests to the LLC that come from the local socket (usually the core)", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.LOCAL_LLC_PF", + "PerPkg": "1", + "PublicDescription": "Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. CHAFilter0[24:21,17] bits correspond to [FMESI] state. Read transactions", + "UMask": "0x189dff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache and Snoop Filter Lookups; Data Read Prefetches that come from the local socket (usually the core)", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.LOCAL_PF", + "PerPkg": "1", + "PublicDescription": "Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. CHAFilter0[24:21,17] bits correspond to [FMESI] state. Read transactions", + "UMask": "0x199dff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : CRd Prefetches that come from the local socket (usually the core)", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.LOCAL_PF_CODE", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : CRd Requests : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. : Local or remote CRd transactions to the LLC. This includes CRd prefetch.", + "UMask": "0x1910ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache and Snoop Filter Lookups; Data Read Prefetches that come from the local socket (usually the core)", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.LOCAL_PF_DATA_RD", + "PerPkg": "1", + "PublicDescription": "Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. CHAFilter0[24:21,17] bits correspond to [FMESI] state. Read transactions", + "UMask": "0x1981ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : RFO Prefetches that come from the local socket (usually the core)", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.LOCAL_PF_RFO", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : RFO Requests : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. : Local or remote RFO transactions to the LLC. This includes RFO prefetch.", + "UMask": "0x1908ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : RFO Requests that come from the local socket (usually the core)", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.LOCAL_RFO", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : RFO Requests : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. : Local or remote RFO transactions to the LLC. This includes RFO prefetch.", + "UMask": "0x19c8ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : M State", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.M", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : M State : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. : Hit Modified State", + "UMask": "0x40", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : All Misses", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.MISS_ALL", + "PerPkg": "1", + "PublicDescription": "Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS select a state or states (in the umask field) to match. Otherwise, the event will count nothing.", + "UMask": "0x1fe001", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : Write Requests", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.OTHER_REQ_F", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : Write Requests : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. : Writeback transactions from L2 to the LLC This includes all write transactions -- both Cacheable and UC.", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : Remote non-snoop requests", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.PREF_OR_DMND_REMOTE_F", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : Remote non-snoop requests : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. : Remote non-snoop transactions to the LLC.", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : Transactions homed remotely", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.REMOTELY_HOMED_ADDRESS", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : Transactions homed remotely : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. : Transaction whose address resides in a remote MC", + "UMask": "0x15dfff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : CRd Requests that come from a Remote socket.", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.REMOTE_CODE", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : CRd Requests : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. : Local or remote CRd transactions to the LLC. This includes CRd prefetch.", + "UMask": "0x1a10ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache and Snoop Filter Lookups; Data Read Requests that come from a Remote socket", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.REMOTE_DATA_RD", + "PerPkg": "1", + "PublicDescription": "Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. CHAFilter0[24:21,17] bits correspond to [FMESI] state. Read transactions", + "UMask": "0x1a01ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : Transactions homed remotely", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.REMOTE_F", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : Transactions homed remotely : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. : Transaction whose address resides in a remote MC", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : Flush or Invalidate requests that come from a Remote socket.", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.REMOTE_FLUSH_INV", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : Flush : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing.", + "UMask": "0x1a04ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : Filters Requests for those that write info into the cache that come from a remote socket", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.REMOTE_OTHER", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : Write Requests : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. : Writeback transactions from L2 to the LLC This includes all write transactions -- both Cacheable and UC.", + "UMask": "0x1a02ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : RFO Requests that come from a Remote socket.", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.REMOTE_RFO", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : RFO Requests : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. : Local or remote RFO transactions to the LLC. This includes RFO prefetch.", + "UMask": "0x1a08ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : Remote snoop requests", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.REMOTE_SNOOP_F", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : Remote snoop requests : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. : Remote snoop transactions to the LLC.", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache and Snoop Filter Lookups; Snoop Requests from a Remote Socket", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.REMOTE_SNP", + "PerPkg": "1", + "PublicDescription": "Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. CHAFilter0[24:21,17] bits correspond to [FMESI] state.; Filters for any transaction originating from the IPQ or IRQ. This does not include lookups originating from the ISMQ.", + "UMask": "0x1c19ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : RFO Requests", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.RFO", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : RFO Requests : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. : Local or remote RFO transactions to the LLC. This includes RFO prefetch.", + "UMask": "0x1bc8ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : RFO Request Filter", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.RFO_F", + "PerPkg": "1", + "PublicDescription": "Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS select a state or states (in the umask field) to match. Otherwise, the event will count nothing. : Local or remote RFO transactions to the LLC. This includes RFO prefetch.", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : Locally HOMed RFOs - Demand and Prefetches", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.RFO_LOCAL", + "PerPkg": "1", + "PublicDescription": "Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS select a state or states (in the umask field) to match. Otherwise, the event will count nothing.", + "UMask": "0x9c8ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : S State", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.S", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : S State : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. : Hit Shared State", + "UMask": "0x10", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : SnoopFilter - E State", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.SF_E", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : SnoopFilter - E State : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. : SF Hit Exclusive State", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : SnoopFilter - H State", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.SF_H", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : SnoopFilter - H State : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. : SF Hit HitMe State", + "UMask": "0x8", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : SnoopFilter - S State", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.SF_S", + "PerPkg": "1", + "PublicDescription": "Cache Lookups : SnoopFilter - S State : Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set umask bit 0 and select a state or states to match. Otherwise, the event will count nothing. : SF Hit Shared State", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : Writes", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.WRITE_LOCAL", + "PerPkg": "1", + "PublicDescription": "Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS select a state or states (in the umask field) to match. Otherwise, the event will count nothing. : Requests that install or change a line in the LLC. Examples: Writebacks from Core L2's and UPI. Prefetches into the LLC.", + "UMask": "0x842ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Cache Lookups : Remote Writes", + "EventCode": "0x34", + "EventName": "UNC_CHA_LLC_LOOKUP.WRITE_REMOTE", + "PerPkg": "1", + "PublicDescription": "Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS select a state or states (in the umask field) to match. Otherwise, the event will count nothing.", + "UMask": "0x17c2ff", + "Unit": "CHA" + }, + { + "BriefDescription": "Lines Victimized : Lines in E state", + "EventCode": "0x37", + "EventName": "UNC_CHA_LLC_VICTIMS.E_STATE", + "PerPkg": "1", + "PublicDescription": "Lines Victimized : Lines in E state : Counts the number of lines that were victimized on a fill. This can be filtered by the state that the line was in.", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "Lines Victimized : IA traffic", + "EventCode": "0x37", + "EventName": "UNC_CHA_LLC_VICTIMS.IA", + "PerPkg": "1", + "PublicDescription": "Lines Victimized : IA traffic : Counts the number of lines that were victimized on a fill. This can be filtered by the state that the line was in.", + "UMask": "0x20", + "Unit": "CHA" + }, + { + "BriefDescription": "Lines Victimized : IO traffic", + "EventCode": "0x37", + "EventName": "UNC_CHA_LLC_VICTIMS.IO", + "PerPkg": "1", + "PublicDescription": "Lines Victimized : IO traffic : Counts the number of lines that were victimized on a fill. This can be filtered by the state that the line was in.", + "UMask": "0x10", + "Unit": "CHA" + }, + { + "BriefDescription": "All LLC lines in E state that are victimized on a fill from an IO device", + "EventCode": "0x37", + "EventName": "UNC_CHA_LLC_VICTIMS.IO_E", + "PerPkg": "1", + "PublicDescription": "Counts the number of lines that were victimized on a fill. This can be filtered by the state that the line was in.", + "UMask": "0x12", + "Unit": "CHA" + }, + { + "BriefDescription": "All LLC lines in F or S state that are victimized on a fill from an IO device", + "EventCode": "0x37", + "EventName": "UNC_CHA_LLC_VICTIMS.IO_FS", + "PerPkg": "1", + "PublicDescription": "Counts the number of lines that were victimized on a fill. This can be filtered by the state that the line was in.", + "UMask": "0x1c", + "Unit": "CHA" + }, + { + "BriefDescription": "All LLC lines in M state that are victimized on a fill from an IO device", + "EventCode": "0x37", + "EventName": "UNC_CHA_LLC_VICTIMS.IO_M", + "PerPkg": "1", + "PublicDescription": "Counts the number of lines that were victimized on a fill. This can be filtered by the state that the line was in.", + "UMask": "0x11", + "Unit": "CHA" + }, + { + "BriefDescription": "All LLC lines in any state that are victimized on a fill from an IO device", + "EventCode": "0x37", + "EventName": "UNC_CHA_LLC_VICTIMS.IO_MESF", + "PerPkg": "1", + "PublicDescription": "Counts the number of lines that were victimized on a fill. This can be filtered by the state that the line was in.", + "UMask": "0x1f", + "Unit": "CHA" + }, + { + "BriefDescription": "Lines Victimized; Local - All Lines", + "EventCode": "0x37", + "EventName": "UNC_CHA_LLC_VICTIMS.LOCAL_ALL", + "PerPkg": "1", + "PublicDescription": "Counts the number of lines that were victimized on a fill. This can be filtered by the state that the line was in.", + "UMask": "0x200f", + "Unit": "CHA" + }, + { + "BriefDescription": "Lines Victimized", + "EventCode": "0x37", + "EventName": "UNC_CHA_LLC_VICTIMS.LOCAL_E", + "PerPkg": "1", + "PublicDescription": "Lines Victimized : Counts the number of lines that were victimized on a fill. This can be filtered by the state that the line was in.", + "UMask": "0x2002", + "Unit": "CHA" + }, + { + "BriefDescription": "Lines Victimized", + "EventCode": "0x37", + "EventName": "UNC_CHA_LLC_VICTIMS.LOCAL_M", + "PerPkg": "1", + "PublicDescription": "Lines Victimized : Counts the number of lines that were victimized on a fill. This can be filtered by the state that the line was in.", + "UMask": "0x2001", + "Unit": "CHA" + }, + { + "BriefDescription": "Lines Victimized : Local Only", + "EventCode": "0x37", + "EventName": "UNC_CHA_LLC_VICTIMS.LOCAL_ONLY", + "PerPkg": "1", + "PublicDescription": "Lines Victimized : Local Only : Counts the number of lines that were victimized on a fill. This can be filtered by the state that the line was in.", + "Unit": "CHA" + }, + { + "BriefDescription": "Lines Victimized", + "EventCode": "0x37", + "EventName": "UNC_CHA_LLC_VICTIMS.LOCAL_S", + "PerPkg": "1", + "PublicDescription": "Lines Victimized : Counts the number of lines that were victimized on a fill. This can be filtered by the state that the line was in.", + "UMask": "0x2004", + "Unit": "CHA" + }, + { + "BriefDescription": "Lines Victimized : Lines in M state", + "EventCode": "0x37", + "EventName": "UNC_CHA_LLC_VICTIMS.M_STATE", + "PerPkg": "1", + "PublicDescription": "Lines Victimized : Lines in M state : Counts the number of lines that were victimized on a fill. This can be filtered by the state that the line was in.", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "Lines Victimized; Remote - All Lines", + "EventCode": "0x37", + "EventName": "UNC_CHA_LLC_VICTIMS.REMOTE_ALL", + "PerPkg": "1", + "PublicDescription": "Counts the number of lines that were victimized on a fill. This can be filtered by the state that the line was in.", + "UMask": "0x800f", + "Unit": "CHA" + }, + { + "BriefDescription": "Lines Victimized", + "EventCode": "0x37", + "EventName": "UNC_CHA_LLC_VICTIMS.REMOTE_E", + "PerPkg": "1", + "PublicDescription": "Lines Victimized : Counts the number of lines that were victimized on a fill. This can be filtered by the state that the line was in.", + "UMask": "0x8002", + "Unit": "CHA" + }, + { + "BriefDescription": "Lines Victimized", + "EventCode": "0x37", + "EventName": "UNC_CHA_LLC_VICTIMS.REMOTE_M", + "PerPkg": "1", + "PublicDescription": "Lines Victimized : Counts the number of lines that were victimized on a fill. This can be filtered by the state that the line was in.", + "UMask": "0x8001", + "Unit": "CHA" + }, + { + "BriefDescription": "Lines Victimized : Remote Only", + "EventCode": "0x37", + "EventName": "UNC_CHA_LLC_VICTIMS.REMOTE_ONLY", + "PerPkg": "1", + "PublicDescription": "Lines Victimized : Remote Only : Counts the number of lines that were victimized on a fill. This can be filtered by the state that the line was in.", + "Unit": "CHA" + }, + { + "BriefDescription": "Lines Victimized", + "EventCode": "0x37", + "EventName": "UNC_CHA_LLC_VICTIMS.REMOTE_S", + "PerPkg": "1", + "PublicDescription": "Lines Victimized : Counts the number of lines that were victimized on a fill. This can be filtered by the state that the line was in.", + "UMask": "0x8004", + "Unit": "CHA" + }, + { + "BriefDescription": "Lines Victimized : Lines in S State", + "EventCode": "0x37", + "EventName": "UNC_CHA_LLC_VICTIMS.S_STATE", + "PerPkg": "1", + "PublicDescription": "Lines Victimized : Lines in S State : Counts the number of lines that were victimized on a fill. This can be filtered by the state that the line was in.", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "All LLC lines in E state that are victimized on a fill", + "EventCode": "0x37", + "EventName": "UNC_CHA_LLC_VICTIMS.TOTAL_E", + "PerPkg": "1", + "PublicDescription": "Counts the number of lines that were victimized on a fill. This can be filtered by the state that the line was in.", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "All LLC lines in M state that are victimized on a fill", + "EventCode": "0x37", + "EventName": "UNC_CHA_LLC_VICTIMS.TOTAL_M", + "PerPkg": "1", + "PublicDescription": "Counts the number of lines that were victimized on a fill. This can be filtered by the state that the line was in.", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "All LLC lines in S state that are victimized on a fill", + "EventCode": "0x37", + "EventName": "UNC_CHA_LLC_VICTIMS.TOTAL_S", + "PerPkg": "1", + "PublicDescription": "Counts the number of lines that were victimized on a fill. This can be filtered by the state that the line was in.", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "Cbo Misc : CV0 Prefetch Miss", + "EventCode": "0x39", + "EventName": "UNC_CHA_MISC.CV0_PREF_MISS", + "PerPkg": "1", + "PublicDescription": "Cbo Misc : CV0 Prefetch Miss : Miscellaneous events in the Cbo.", + "UMask": "0x20", + "Unit": "CHA" + }, + { + "BriefDescription": "Cbo Misc : CV0 Prefetch Victim", + "EventCode": "0x39", + "EventName": "UNC_CHA_MISC.CV0_PREF_VIC", + "PerPkg": "1", + "PublicDescription": "Cbo Misc : CV0 Prefetch Victim : Miscellaneous events in the Cbo.", + "UMask": "0x10", + "Unit": "CHA" + }, + { + "BriefDescription": "Number of times that an RFO hit in S state.", + "EventCode": "0x39", + "EventName": "UNC_CHA_MISC.RFO_HIT_S", + "PerPkg": "1", + "PublicDescription": "Counts when a RFO (the Read for Ownership issued before a write) request hit a cacheline in the S (Shared) state.", + "UMask": "0x8", + "Unit": "CHA" + }, + { + "BriefDescription": "Cbo Misc : Silent Snoop Eviction", + "EventCode": "0x39", + "EventName": "UNC_CHA_MISC.RSPI_WAS_FSE", + "PerPkg": "1", + "PublicDescription": "Cbo Misc : Silent Snoop Eviction : Miscellaneous events in the Cbo. : Counts the number of times when a Snoop hit in FSE states and triggered a silent eviction. This is useful because this information is lost in the PRE encodings.", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "Cbo Misc : Write Combining Aliasing", + "EventCode": "0x39", + "EventName": "UNC_CHA_MISC.WC_ALIASING", + "PerPkg": "1", + "PublicDescription": "Cbo Misc : Write Combining Aliasing : Miscellaneous events in the Cbo. : Counts the number of times that a USWC write (WCIL(F)) transaction hit in the LLC in M state, triggering a WBMtoI followed by the USWC write. This occurs when there is WC aliasing.", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "OSB Snoop Broadcast : Local InvItoE", + "EventCode": "0x55", + "EventName": "UNC_CHA_OSB.LOCAL_INVITOE", + "PerPkg": "1", + "PublicDescription": "OSB Snoop Broadcast : Local InvItoE : Count of OSB snoop broadcasts. Counts by 1 per request causing OSB snoops to be broadcast. Does not count all the snoops generated by OSB.", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "OSB Snoop Broadcast : Local Rd", + "EventCode": "0x55", + "EventName": "UNC_CHA_OSB.LOCAL_READ", + "PerPkg": "1", + "PublicDescription": "OSB Snoop Broadcast : Local Rd : Count of OSB snoop broadcasts. Counts by 1 per request causing OSB snoops to be broadcast. Does not count all the snoops generated by OSB.", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "OSB Snoop Broadcast : Off", + "EventCode": "0x55", + "EventName": "UNC_CHA_OSB.OFF_PWRHEURISTIC", + "PerPkg": "1", + "PublicDescription": "OSB Snoop Broadcast : Off : Count of OSB snoop broadcasts. Counts by 1 per request causing OSB snoops to be broadcast. Does not count all the snoops generated by OSB.", + "UMask": "0x20", + "Unit": "CHA" + }, + { + "BriefDescription": "OSB Snoop Broadcast : Remote Rd", + "EventCode": "0x55", + "EventName": "UNC_CHA_OSB.REMOTE_READ", + "PerPkg": "1", + "PublicDescription": "OSB Snoop Broadcast : Remote Rd : Count of OSB snoop broadcasts. Counts by 1 per request causing OSB snoops to be broadcast. Does not count all the snoops generated by OSB.", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "OSB Snoop Broadcast : Remote Rd InvItoE", + "EventCode": "0x55", + "EventName": "UNC_CHA_OSB.REMOTE_READINVITOE", + "PerPkg": "1", + "PublicDescription": "OSB Snoop Broadcast : Remote Rd InvItoE : Count of OSB snoop broadcasts. Counts by 1 per request causing OSB snoops to be broadcast. Does not count all the snoops generated by OSB.", + "UMask": "0x8", + "Unit": "CHA" + }, + { + "BriefDescription": "OSB Snoop Broadcast : RFO HitS Snoop Broadcast", + "EventCode": "0x55", + "EventName": "UNC_CHA_OSB.RFO_HITS_SNP_BCAST", + "PerPkg": "1", + "PublicDescription": "OSB Snoop Broadcast : RFO HitS Snoop Broadcast : Count of OSB snoop broadcasts. Counts by 1 per request causing OSB snoops to be broadcast. Does not count all the snoops generated by OSB.", + "UMask": "0x10", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_PMM_MEMMODE_NM_INVITOX.LOCAL", + "EventCode": "0x65", + "EventName": "UNC_CHA_PMM_MEMMODE_NM_INVITOX.LOCAL", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_PMM_MEMMODE_NM_INVITOX.REMOTE", + "EventCode": "0x65", + "EventName": "UNC_CHA_PMM_MEMMODE_NM_INVITOX.REMOTE", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_PMM_MEMMODE_NM_INVITOX.SETCONFLICT", + "EventCode": "0x65", + "EventName": "UNC_CHA_PMM_MEMMODE_NM_INVITOX.SETCONFLICT", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "Memory Mode related events; Counts the number of times CHA saw a Near Memory set conflict in SF/LLC", + "EventCode": "0x64", + "EventName": "UNC_CHA_PMM_MEMMODE_NM_SETCONFLICTS.LLC", + "PerPkg": "1", + "PublicDescription": "Near Memory evictions due to another read to the same Near Memory set in the LLC.", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "Memory Mode related events; Counts the number of times CHA saw a Near memory set conflict in SF/LLC", + "EventCode": "0x64", + "EventName": "UNC_CHA_PMM_MEMMODE_NM_SETCONFLICTS.SF", + "PerPkg": "1", + "PublicDescription": "Near Memory evictions due to another read to the same Near Memory set in the SF", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "Memory Mode related events; Counts the number of times CHA saw a Near Memory set conflict in TOR", + "EventCode": "0x64", + "EventName": "UNC_CHA_PMM_MEMMODE_NM_SETCONFLICTS.TOR", + "PerPkg": "1", + "PublicDescription": "No Reject in the CHA due to a pending read to the same Near Memory set in the TOR.", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_PMM_MEMMODE_NM_SETCONFLICTS2.IODC", + "EventCode": "0x70", + "EventName": "UNC_CHA_PMM_MEMMODE_NM_SETCONFLICTS2.IODC", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_PMM_MEMMODE_NM_SETCONFLICTS2.MEMWR", + "EventCode": "0x70", + "EventName": "UNC_CHA_PMM_MEMMODE_NM_SETCONFLICTS2.MEMWR", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_PMM_MEMMODE_NM_SETCONFLICTS2.MEMWRNI", + "EventCode": "0x70", + "EventName": "UNC_CHA_PMM_MEMMODE_NM_SETCONFLICTS2.MEMWRNI", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_PMM_QOS.DDR4_FAST_INSERT", + "EventCode": "0x66", + "EventName": "UNC_CHA_PMM_QOS.DDR4_FAST_INSERT", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_PMM_QOS.REJ_IRQ", + "EventCode": "0x66", + "EventName": "UNC_CHA_PMM_QOS.REJ_IRQ", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_PMM_QOS.SLOWTORQ_SKIP", + "EventCode": "0x66", + "EventName": "UNC_CHA_PMM_QOS.SLOWTORQ_SKIP", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_PMM_QOS.SLOW_INSERT", + "EventCode": "0x66", + "EventName": "UNC_CHA_PMM_QOS.SLOW_INSERT", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_PMM_QOS.THROTTLE", + "EventCode": "0x66", + "EventName": "UNC_CHA_PMM_QOS.THROTTLE", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_PMM_QOS.THROTTLE_IRQ", + "EventCode": "0x66", + "EventName": "UNC_CHA_PMM_QOS.THROTTLE_IRQ", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_PMM_QOS.THROTTLE_PRQ", + "EventCode": "0x66", + "EventName": "UNC_CHA_PMM_QOS.THROTTLE_PRQ", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_PMM_QOS_OCCUPANCY.DDR_FAST_FIFO", + "EventCode": "0x67", + "EventName": "UNC_CHA_PMM_QOS_OCCUPANCY.DDR_FAST_FIFO", + "PerPkg": "1", + "PublicDescription": ": count # of FAST TOR Request inserted to ha_tor_req_fifo", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "Number of SLOW TOR Request inserted to ha_pmm_tor_req_fifo", + "EventCode": "0x67", + "EventName": "UNC_CHA_PMM_QOS_OCCUPANCY.DDR_SLOW_FIFO", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "CHA iMC CHNx READ Credits Empty : MC0", + "EventCode": "0x58", + "EventName": "UNC_CHA_READ_NO_CREDITS.MC0", + "PerPkg": "1", + "PublicDescription": "CHA iMC CHNx READ Credits Empty : MC0 : Counts the number of times when there are no credits available for sending reads from the CHA into the iMC. In order to send reads into the memory controller, the HA must first acquire a credit for the iMC's AD Ingress queue. : Filter for memory controller 0 only.", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "CHA iMC CHNx READ Credits Empty : MC1", + "EventCode": "0x58", + "EventName": "UNC_CHA_READ_NO_CREDITS.MC1", + "PerPkg": "1", + "PublicDescription": "CHA iMC CHNx READ Credits Empty : MC1 : Counts the number of times when there are no credits available for sending reads from the CHA into the iMC. In order to send reads into the memory controller, the HA must first acquire a credit for the iMC's AD Ingress queue. : Filter for memory controller 1 only.", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "CHA iMC CHNx READ Credits Empty : MC2", + "EventCode": "0x58", + "EventName": "UNC_CHA_READ_NO_CREDITS.MC2", + "PerPkg": "1", + "PublicDescription": "CHA iMC CHNx READ Credits Empty : MC2 : Counts the number of times when there are no credits available for sending reads from the CHA into the iMC. In order to send reads into the memory controller, the HA must first acquire a credit for the iMC's AD Ingress queue. : Filter for memory controller 2 only.", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "CHA iMC CHNx READ Credits Empty : MC3", + "EventCode": "0x58", + "EventName": "UNC_CHA_READ_NO_CREDITS.MC3", + "PerPkg": "1", + "PublicDescription": "CHA iMC CHNx READ Credits Empty : MC3 : Counts the number of times when there are no credits available for sending reads from the CHA into the iMC. In order to send reads into the memory controller, the HA must first acquire a credit for the iMC's AD Ingress queue. : Filter for memory controller 3 only.", + "UMask": "0x8", + "Unit": "CHA" + }, + { + "BriefDescription": "CHA iMC CHNx READ Credits Empty : MC4", + "EventCode": "0x58", + "EventName": "UNC_CHA_READ_NO_CREDITS.MC4", + "PerPkg": "1", + "PublicDescription": "CHA iMC CHNx READ Credits Empty : MC4 : Counts the number of times when there are no credits available for sending reads from the CHA into the iMC. In order to send reads into the memory controller, the HA must first acquire a credit for the iMC's AD Ingress queue. : Filter for memory controller 4 only.", + "UMask": "0x10", + "Unit": "CHA" + }, + { + "BriefDescription": "CHA iMC CHNx READ Credits Empty : MC5", + "EventCode": "0x58", + "EventName": "UNC_CHA_READ_NO_CREDITS.MC5", + "PerPkg": "1", + "PublicDescription": "CHA iMC CHNx READ Credits Empty : MC5 : Counts the number of times when there are no credits available for sending reads from the CHA into the iMC. In order to send reads into the memory controller, the HA must first acquire a credit for the iMC's AD Ingress queue. : Filter for memory controller 5 only.", + "UMask": "0x20", + "Unit": "CHA" + }, + { + "BriefDescription": "Requests for exclusive ownership of a cache line without receiving data", + "EventCode": "0x50", + "EventName": "UNC_CHA_REQUESTS.INVITOE", + "PerPkg": "1", + "PublicDescription": "Counts the total number of requests coming from a unit on this socket for exclusive ownership of a cache line without receiving data (INVITOE) to the CHA.", + "UMask": "0x30", + "Unit": "CHA" + }, + { + "BriefDescription": "Local requests for exclusive ownership of a cache line without receiving data", + "EventCode": "0x50", + "EventName": "UNC_CHA_REQUESTS.INVITOE_LOCAL", + "PerPkg": "1", + "PublicDescription": "Counts the total number of requests coming from a unit on this socket for exclusive ownership of a cache line without receiving data (INVITOE) to the CHA.", + "UMask": "0x10", + "Unit": "CHA" + }, + { + "BriefDescription": "Remote requests for exclusive ownership of a cache line without receiving data", + "EventCode": "0x50", + "EventName": "UNC_CHA_REQUESTS.INVITOE_REMOTE", + "PerPkg": "1", + "PublicDescription": "Counts the total number of requests coming from a remote socket for exclusive ownership of a cache line without receiving data (INVITOE) to the CHA.", + "UMask": "0x20", + "Unit": "CHA" + }, + { + "BriefDescription": "Read requests made into the CHA", + "EventCode": "0x50", + "EventName": "UNC_CHA_REQUESTS.READS", + "PerPkg": "1", + "PublicDescription": "Counts read requests made into this CHA. Reads include all read opcodes (including RFO: the Read for Ownership issued before a write) .", + "UMask": "0x3", + "Unit": "CHA" + }, + { + "BriefDescription": "Read requests from a unit on this socket", + "EventCode": "0x50", + "EventName": "UNC_CHA_REQUESTS.READS_LOCAL", + "PerPkg": "1", + "PublicDescription": "Counts read requests coming from a unit on this socket made into this CHA. Reads include all read opcodes (including RFO: the Read for Ownership issued before a write).", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "Read requests from a remote socket", + "EventCode": "0x50", + "EventName": "UNC_CHA_REQUESTS.READS_REMOTE", + "PerPkg": "1", + "PublicDescription": "Counts read requests coming from a remote socket made into the CHA. Reads include all read opcodes (including RFO: the Read for Ownership issued before a write).", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "Write requests made into the CHA", + "EventCode": "0x50", + "EventName": "UNC_CHA_REQUESTS.WRITES", + "PerPkg": "1", + "PublicDescription": "Counts write requests made into the CHA, including streaming, evictions, HitM (Reads from another core to a Modified cacheline), etc.", + "UMask": "0xc", + "Unit": "CHA" + }, + { + "BriefDescription": "Write Requests from a unit on this socket", + "EventCode": "0x50", + "EventName": "UNC_CHA_REQUESTS.WRITES_LOCAL", + "PerPkg": "1", + "PublicDescription": "Counts write requests coming from a unit on this socket made into this CHA, including streaming, evictions, HitM (Reads from another core to a Modified cacheline), etc.", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "Read and Write Requests; Writes Remote", + "EventCode": "0x50", + "EventName": "UNC_CHA_REQUESTS.WRITES_REMOTE", + "PerPkg": "1", + "PublicDescription": "Counts the total number of read requests made into the Home Agent. Reads include all read opcodes (including RFO). Writes include all writes (streaming, evictions, HitM, etc).", + "UMask": "0x8", + "Unit": "CHA" + }, + { + "BriefDescription": "Ingress (from CMS) Allocations : IPQ", + "EventCode": "0x13", + "EventName": "UNC_CHA_RxC_INSERTS.IPQ", + "PerPkg": "1", + "PublicDescription": "Ingress (from CMS) Allocations : IPQ : Counts number of allocations per cycle into the specified Ingress queue.", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "Ingress (from CMS) Allocations : IRQ", + "EventCode": "0x13", + "EventName": "UNC_CHA_RxC_INSERTS.IRQ", + "PerPkg": "1", + "PublicDescription": "Ingress (from CMS) Allocations : IRQ : Counts number of allocations per cycle into the specified Ingress queue.", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "Ingress (from CMS) Allocations : IRQ Rejected", + "EventCode": "0x13", + "EventName": "UNC_CHA_RxC_INSERTS.IRQ_REJ", + "PerPkg": "1", + "PublicDescription": "Ingress (from CMS) Allocations : IRQ Rejected : Counts number of allocations per cycle into the specified Ingress queue.", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "Ingress (from CMS) Allocations : PRQ", + "EventCode": "0x13", + "EventName": "UNC_CHA_RxC_INSERTS.PRQ", + "PerPkg": "1", + "PublicDescription": "Ingress (from CMS) Allocations : PRQ : Counts number of allocations per cycle into the specified Ingress queue.", + "UMask": "0x10", + "Unit": "CHA" + }, + { + "BriefDescription": "Ingress (from CMS) Allocations : PRQ", + "EventCode": "0x13", + "EventName": "UNC_CHA_RxC_INSERTS.PRQ_REJ", + "PerPkg": "1", + "PublicDescription": "Ingress (from CMS) Allocations : PRQ : Counts number of allocations per cycle into the specified Ingress queue.", + "UMask": "0x20", + "Unit": "CHA" + }, + { + "BriefDescription": "Ingress (from CMS) Allocations : RRQ", + "EventCode": "0x13", + "EventName": "UNC_CHA_RxC_INSERTS.RRQ", + "PerPkg": "1", + "PublicDescription": "Ingress (from CMS) Allocations : RRQ : Counts number of allocations per cycle into the specified Ingress queue.", + "UMask": "0x40", + "Unit": "CHA" + }, + { + "BriefDescription": "Ingress (from CMS) Allocations : WBQ", + "EventCode": "0x13", + "EventName": "UNC_CHA_RxC_INSERTS.WBQ", + "PerPkg": "1", + "PublicDescription": "Ingress (from CMS) Allocations : WBQ : Counts number of allocations per cycle into the specified Ingress queue.", + "UMask": "0x80", + "Unit": "CHA" + }, + { + "BriefDescription": "IPQ Requests (from CMS) Rejected - Set 0 : AD REQ on VN0", + "EventCode": "0x22", + "EventName": "UNC_CHA_RxC_IPQ0_REJECT.AD_REQ_VN0", + "PerPkg": "1", + "PublicDescription": "IPQ Requests (from CMS) Rejected - Set 0 : AD REQ on VN0 : No AD VN0 credit for generating a request", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "IPQ Requests (from CMS) Rejected - Set 0 : AD RSP on VN0", + "EventCode": "0x22", + "EventName": "UNC_CHA_RxC_IPQ0_REJECT.AD_RSP_VN0", + "PerPkg": "1", + "PublicDescription": "IPQ Requests (from CMS) Rejected - Set 0 : AD RSP on VN0 : No AD VN0 credit for generating a response", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "IPQ Requests (from CMS) Rejected - Set 0 : Non UPI AK Request", + "EventCode": "0x22", + "EventName": "UNC_CHA_RxC_IPQ0_REJECT.AK_NON_UPI", + "PerPkg": "1", + "PublicDescription": "IPQ Requests (from CMS) Rejected - Set 0 : Non UPI AK Request : Can't inject AK ring message", + "UMask": "0x40", + "Unit": "CHA" + }, + { + "BriefDescription": "IPQ Requests (from CMS) Rejected - Set 0 : BL NCB on VN0", + "EventCode": "0x22", + "EventName": "UNC_CHA_RxC_IPQ0_REJECT.BL_NCB_VN0", + "PerPkg": "1", + "PublicDescription": "IPQ Requests (from CMS) Rejected - Set 0 : BL NCB on VN0 : No BL VN0 credit for NCB", + "UMask": "0x10", + "Unit": "CHA" + }, + { + "BriefDescription": "IPQ Requests (from CMS) Rejected - Set 0 : BL NCS on VN0", + "EventCode": "0x22", + "EventName": "UNC_CHA_RxC_IPQ0_REJECT.BL_NCS_VN0", + "PerPkg": "1", + "PublicDescription": "IPQ Requests (from CMS) Rejected - Set 0 : BL NCS on VN0 : No BL VN0 credit for NCS", + "UMask": "0x20", + "Unit": "CHA" + }, + { + "BriefDescription": "IPQ Requests (from CMS) Rejected - Set 0 : BL RSP on VN0", + "EventCode": "0x22", + "EventName": "UNC_CHA_RxC_IPQ0_REJECT.BL_RSP_VN0", + "PerPkg": "1", + "PublicDescription": "IPQ Requests (from CMS) Rejected - Set 0 : BL RSP on VN0 : No BL VN0 credit for generating a response", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "IPQ Requests (from CMS) Rejected - Set 0 : BL WB on VN0", + "EventCode": "0x22", + "EventName": "UNC_CHA_RxC_IPQ0_REJECT.BL_WB_VN0", + "PerPkg": "1", + "PublicDescription": "IPQ Requests (from CMS) Rejected - Set 0 : BL WB on VN0 : No BL VN0 credit for generating a writeback", + "UMask": "0x8", + "Unit": "CHA" + }, + { + "BriefDescription": "IPQ Requests (from CMS) Rejected - Set 0 : Non UPI IV Request", + "EventCode": "0x22", + "EventName": "UNC_CHA_RxC_IPQ0_REJECT.IV_NON_UPI", + "PerPkg": "1", + "PublicDescription": "IPQ Requests (from CMS) Rejected - Set 0 : Non UPI IV Request : Can't inject IV ring message", + "UMask": "0x80", + "Unit": "CHA" + }, + { + "BriefDescription": "IPQ Requests (from CMS) Rejected - Set 1 : Allow Snoop", + "EventCode": "0x23", + "EventName": "UNC_CHA_RxC_IPQ1_REJECT.ALLOW_SNP", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "CHA" + }, + { + "BriefDescription": "IPQ Requests (from CMS) Rejected - Set 1 : ANY0", + "EventCode": "0x23", + "EventName": "UNC_CHA_RxC_IPQ1_REJECT.ANY0", + "PerPkg": "1", + "PublicDescription": "IPQ Requests (from CMS) Rejected - Set 1 : ANY0 : Any condition listed in the IPQ0 Reject counter was true", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "IPQ Requests (from CMS) Rejected - Set 1 : HA", + "EventCode": "0x23", + "EventName": "UNC_CHA_RxC_IPQ1_REJECT.HA", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "IPQ Requests (from CMS) Rejected - Set 1 : LLC OR SF Way", + "EventCode": "0x23", + "EventName": "UNC_CHA_RxC_IPQ1_REJECT.LLC_OR_SF_WAY", + "PerPkg": "1", + "PublicDescription": "IPQ Requests (from CMS) Rejected - Set 1 : LLC OR SF Way : Way conflict with another request that caused the reject", + "UMask": "0x20", + "Unit": "CHA" + }, + { + "BriefDescription": "IPQ Requests (from CMS) Rejected - Set 1 : LLC Victim", + "EventCode": "0x23", + "EventName": "UNC_CHA_RxC_IPQ1_REJECT.LLC_VICTIM", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "IPQ Requests (from CMS) Rejected - Set 1 : PhyAddr Match", + "EventCode": "0x23", + "EventName": "UNC_CHA_RxC_IPQ1_REJECT.PA_MATCH", + "PerPkg": "1", + "PublicDescription": "IPQ Requests (from CMS) Rejected - Set 1 : PhyAddr Match : Address match with an outstanding request that was rejected.", + "UMask": "0x80", + "Unit": "CHA" + }, + { + "BriefDescription": "IPQ Requests (from CMS) Rejected - Set 1 : SF Victim", + "EventCode": "0x23", + "EventName": "UNC_CHA_RxC_IPQ1_REJECT.SF_VICTIM", + "PerPkg": "1", + "PublicDescription": "IPQ Requests (from CMS) Rejected - Set 1 : SF Victim : Requests did not generate Snoop filter victim", + "UMask": "0x8", + "Unit": "CHA" + }, + { + "BriefDescription": "IPQ Requests (from CMS) Rejected - Set 1 : Victim", + "EventCode": "0x23", + "EventName": "UNC_CHA_RxC_IPQ1_REJECT.VICTIM", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "CHA" + }, + { + "BriefDescription": "IRQ Requests (from CMS) Rejected - Set 0 : AD REQ on VN0", + "EventCode": "0x18", + "EventName": "UNC_CHA_RxC_IRQ0_REJECT.AD_REQ_VN0", + "PerPkg": "1", + "PublicDescription": "IRQ Requests (from CMS) Rejected - Set 0 : AD REQ on VN0 : No AD VN0 credit for generating a request", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "IRQ Requests (from CMS) Rejected - Set 0 : AD RSP on VN0", + "EventCode": "0x18", + "EventName": "UNC_CHA_RxC_IRQ0_REJECT.AD_RSP_VN0", + "PerPkg": "1", + "PublicDescription": "IRQ Requests (from CMS) Rejected - Set 0 : AD RSP on VN0 : No AD VN0 credit for generating a response", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "IRQ Requests (from CMS) Rejected - Set 0 : Non UPI AK Request", + "EventCode": "0x18", + "EventName": "UNC_CHA_RxC_IRQ0_REJECT.AK_NON_UPI", + "PerPkg": "1", + "PublicDescription": "IRQ Requests (from CMS) Rejected - Set 0 : Non UPI AK Request : Can't inject AK ring message", + "UMask": "0x40", + "Unit": "CHA" + }, + { + "BriefDescription": "IRQ Requests (from CMS) Rejected - Set 0 : BL NCB on VN0", + "EventCode": "0x18", + "EventName": "UNC_CHA_RxC_IRQ0_REJECT.BL_NCB_VN0", + "PerPkg": "1", + "PublicDescription": "IRQ Requests (from CMS) Rejected - Set 0 : BL NCB on VN0 : No BL VN0 credit for NCB", + "UMask": "0x10", + "Unit": "CHA" + }, + { + "BriefDescription": "IRQ Requests (from CMS) Rejected - Set 0 : BL NCS on VN0", + "EventCode": "0x18", + "EventName": "UNC_CHA_RxC_IRQ0_REJECT.BL_NCS_VN0", + "PerPkg": "1", + "PublicDescription": "IRQ Requests (from CMS) Rejected - Set 0 : BL NCS on VN0 : No BL VN0 credit for NCS", + "UMask": "0x20", + "Unit": "CHA" + }, + { + "BriefDescription": "IRQ Requests (from CMS) Rejected - Set 0 : BL RSP on VN0", + "EventCode": "0x18", + "EventName": "UNC_CHA_RxC_IRQ0_REJECT.BL_RSP_VN0", + "PerPkg": "1", + "PublicDescription": "IRQ Requests (from CMS) Rejected - Set 0 : BL RSP on VN0 : No BL VN0 credit for generating a response", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "IRQ Requests (from CMS) Rejected - Set 0 : BL WB on VN0", + "EventCode": "0x18", + "EventName": "UNC_CHA_RxC_IRQ0_REJECT.BL_WB_VN0", + "PerPkg": "1", + "PublicDescription": "IRQ Requests (from CMS) Rejected - Set 0 : BL WB on VN0 : No BL VN0 credit for generating a writeback", + "UMask": "0x8", + "Unit": "CHA" + }, + { + "BriefDescription": "IRQ Requests (from CMS) Rejected - Set 0 : Non UPI IV Request", + "EventCode": "0x18", + "EventName": "UNC_CHA_RxC_IRQ0_REJECT.IV_NON_UPI", + "PerPkg": "1", + "PublicDescription": "IRQ Requests (from CMS) Rejected - Set 0 : Non UPI IV Request : Can't inject IV ring message", + "UMask": "0x80", + "Unit": "CHA" + }, + { + "BriefDescription": "IRQ Requests (from CMS) Rejected - Set 1 : Allow Snoop", + "EventCode": "0x19", + "EventName": "UNC_CHA_RxC_IRQ1_REJECT.ALLOW_SNP", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "CHA" + }, + { + "BriefDescription": "IRQ Requests (from CMS) Rejected - Set 1 : ANY0", + "EventCode": "0x19", + "EventName": "UNC_CHA_RxC_IRQ1_REJECT.ANY0", + "PerPkg": "1", + "PublicDescription": "IRQ Requests (from CMS) Rejected - Set 1 : ANY0 : Any condition listed in the IRQ0 Reject counter was true", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "IRQ Requests (from CMS) Rejected - Set 1 : HA", + "EventCode": "0x19", + "EventName": "UNC_CHA_RxC_IRQ1_REJECT.HA", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "IRQ Requests (from CMS) Rejected - Set 1 : LLC or SF Way", + "EventCode": "0x19", + "EventName": "UNC_CHA_RxC_IRQ1_REJECT.LLC_OR_SF_WAY", + "PerPkg": "1", + "PublicDescription": "IRQ Requests (from CMS) Rejected - Set 1 : LLC or SF Way : Way conflict with another request that caused the reject", + "UMask": "0x20", + "Unit": "CHA" + }, + { + "BriefDescription": "IRQ Requests (from CMS) Rejected - Set 1 : LLC Victim", + "EventCode": "0x19", + "EventName": "UNC_CHA_RxC_IRQ1_REJECT.LLC_VICTIM", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "Ingress (from CMS) Request Queue Rejects; PhyAddr Match", + "EventCode": "0x19", + "EventName": "UNC_CHA_RxC_IRQ1_REJECT.PA_MATCH", + "PerPkg": "1", + "UMask": "0x80", + "Unit": "CHA" + }, + { + "BriefDescription": "IRQ Requests (from CMS) Rejected - Set 1 : SF Victim", + "EventCode": "0x19", + "EventName": "UNC_CHA_RxC_IRQ1_REJECT.SF_VICTIM", + "PerPkg": "1", + "PublicDescription": "IRQ Requests (from CMS) Rejected - Set 1 : SF Victim : Requests did not generate Snoop filter victim", + "UMask": "0x8", + "Unit": "CHA" + }, + { + "BriefDescription": "IRQ Requests (from CMS) Rejected - Set 1 : Victim", + "EventCode": "0x19", + "EventName": "UNC_CHA_RxC_IRQ1_REJECT.VICTIM", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "CHA" + }, + { + "BriefDescription": "ISMQ Rejects - Set 0 : AD REQ on VN0", + "EventCode": "0x24", + "EventName": "UNC_CHA_RxC_ISMQ0_REJECT.AD_REQ_VN0", + "PerPkg": "1", + "PublicDescription": "ISMQ Rejects - Set 0 : AD REQ on VN0 : Number of times a transaction flowing through the ISMQ had to retry. Transaction pass through the ISMQ as responses for requests that already exist in the Cbo. Some examples include: when data is returned or when snoop responses come back from the cores. : No AD VN0 credit for generating a request", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "ISMQ Rejects - Set 0 : AD RSP on VN0", + "EventCode": "0x24", + "EventName": "UNC_CHA_RxC_ISMQ0_REJECT.AD_RSP_VN0", + "PerPkg": "1", + "PublicDescription": "ISMQ Rejects - Set 0 : AD RSP on VN0 : Number of times a transaction flowing through the ISMQ had to retry. Transaction pass through the ISMQ as responses for requests that already exist in the Cbo. Some examples include: when data is returned or when snoop responses come back from the cores. : No AD VN0 credit for generating a response", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "ISMQ Rejects - Set 0 : Non UPI AK Request", + "EventCode": "0x24", + "EventName": "UNC_CHA_RxC_ISMQ0_REJECT.AK_NON_UPI", + "PerPkg": "1", + "PublicDescription": "ISMQ Rejects - Set 0 : Non UPI AK Request : Number of times a transaction flowing through the ISMQ had to retry. Transaction pass through the ISMQ as responses for requests that already exist in the Cbo. Some examples include: when data is returned or when snoop responses come back from the cores. : Can't inject AK ring message", + "UMask": "0x40", + "Unit": "CHA" + }, + { + "BriefDescription": "ISMQ Rejects - Set 0 : BL NCB on VN0", + "EventCode": "0x24", + "EventName": "UNC_CHA_RxC_ISMQ0_REJECT.BL_NCB_VN0", + "PerPkg": "1", + "PublicDescription": "ISMQ Rejects - Set 0 : BL NCB on VN0 : Number of times a transaction flowing through the ISMQ had to retry. Transaction pass through the ISMQ as responses for requests that already exist in the Cbo. Some examples include: when data is returned or when snoop responses come back from the cores. : No BL VN0 credit for NCB", + "UMask": "0x10", + "Unit": "CHA" + }, + { + "BriefDescription": "ISMQ Rejects - Set 0 : BL NCS on VN0", + "EventCode": "0x24", + "EventName": "UNC_CHA_RxC_ISMQ0_REJECT.BL_NCS_VN0", + "PerPkg": "1", + "PublicDescription": "ISMQ Rejects - Set 0 : BL NCS on VN0 : Number of times a transaction flowing through the ISMQ had to retry. Transaction pass through the ISMQ as responses for requests that already exist in the Cbo. Some examples include: when data is returned or when snoop responses come back from the cores. : No BL VN0 credit for NCS", + "UMask": "0x20", + "Unit": "CHA" + }, + { + "BriefDescription": "ISMQ Rejects - Set 0 : BL RSP on VN0", + "EventCode": "0x24", + "EventName": "UNC_CHA_RxC_ISMQ0_REJECT.BL_RSP_VN0", + "PerPkg": "1", + "PublicDescription": "ISMQ Rejects - Set 0 : BL RSP on VN0 : Number of times a transaction flowing through the ISMQ had to retry. Transaction pass through the ISMQ as responses for requests that already exist in the Cbo. Some examples include: when data is returned or when snoop responses come back from the cores. : No BL VN0 credit for generating a response", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "ISMQ Rejects - Set 0 : BL WB on VN0", + "EventCode": "0x24", + "EventName": "UNC_CHA_RxC_ISMQ0_REJECT.BL_WB_VN0", + "PerPkg": "1", + "PublicDescription": "ISMQ Rejects - Set 0 : BL WB on VN0 : Number of times a transaction flowing through the ISMQ had to retry. Transaction pass through the ISMQ as responses for requests that already exist in the Cbo. Some examples include: when data is returned or when snoop responses come back from the cores. : No BL VN0 credit for generating a writeback", + "UMask": "0x8", + "Unit": "CHA" + }, + { + "BriefDescription": "ISMQ Rejects - Set 0 : Non UPI IV Request", + "EventCode": "0x24", + "EventName": "UNC_CHA_RxC_ISMQ0_REJECT.IV_NON_UPI", + "PerPkg": "1", + "PublicDescription": "ISMQ Rejects - Set 0 : Non UPI IV Request : Number of times a transaction flowing through the ISMQ had to retry. Transaction pass through the ISMQ as responses for requests that already exist in the Cbo. Some examples include: when data is returned or when snoop responses come back from the cores. : Can't inject IV ring message", + "UMask": "0x80", + "Unit": "CHA" + }, + { + "BriefDescription": "ISMQ Retries - Set 0 : AD REQ on VN0", + "EventCode": "0x2c", + "EventName": "UNC_CHA_RxC_ISMQ0_RETRY.AD_REQ_VN0", + "PerPkg": "1", + "PublicDescription": "ISMQ Retries - Set 0 : AD REQ on VN0 : Number of times a transaction flowing through the ISMQ had to retry. Transaction pass through the ISMQ as responses for requests that already exist in the Cbo. Some examples include: when data is returned or when snoop responses come back from the cores. : No AD VN0 credit for generating a request", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "ISMQ Retries - Set 0 : AD RSP on VN0", + "EventCode": "0x2c", + "EventName": "UNC_CHA_RxC_ISMQ0_RETRY.AD_RSP_VN0", + "PerPkg": "1", + "PublicDescription": "ISMQ Retries - Set 0 : AD RSP on VN0 : Number of times a transaction flowing through the ISMQ had to retry. Transaction pass through the ISMQ as responses for requests that already exist in the Cbo. Some examples include: when data is returned or when snoop responses come back from the cores. : No AD VN0 credit for generating a response", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "ISMQ Retries - Set 0 : Non UPI AK Request", + "EventCode": "0x2c", + "EventName": "UNC_CHA_RxC_ISMQ0_RETRY.AK_NON_UPI", + "PerPkg": "1", + "PublicDescription": "ISMQ Retries - Set 0 : Non UPI AK Request : Number of times a transaction flowing through the ISMQ had to retry. Transaction pass through the ISMQ as responses for requests that already exist in the Cbo. Some examples include: when data is returned or when snoop responses come back from the cores. : Can't inject AK ring message", + "UMask": "0x40", + "Unit": "CHA" + }, + { + "BriefDescription": "ISMQ Retries - Set 0 : BL NCB on VN0", + "EventCode": "0x2c", + "EventName": "UNC_CHA_RxC_ISMQ0_RETRY.BL_NCB_VN0", + "PerPkg": "1", + "PublicDescription": "ISMQ Retries - Set 0 : BL NCB on VN0 : Number of times a transaction flowing through the ISMQ had to retry. Transaction pass through the ISMQ as responses for requests that already exist in the Cbo. Some examples include: when data is returned or when snoop responses come back from the cores. : No BL VN0 credit for NCB", + "UMask": "0x10", + "Unit": "CHA" + }, + { + "BriefDescription": "ISMQ Retries - Set 0 : BL NCS on VN0", + "EventCode": "0x2c", + "EventName": "UNC_CHA_RxC_ISMQ0_RETRY.BL_NCS_VN0", + "PerPkg": "1", + "PublicDescription": "ISMQ Retries - Set 0 : BL NCS on VN0 : Number of times a transaction flowing through the ISMQ had to retry. Transaction pass through the ISMQ as responses for requests that already exist in the Cbo. Some examples include: when data is returned or when snoop responses come back from the cores. : No BL VN0 credit for NCS", + "UMask": "0x20", + "Unit": "CHA" + }, + { + "BriefDescription": "ISMQ Retries - Set 0 : BL RSP on VN0", + "EventCode": "0x2c", + "EventName": "UNC_CHA_RxC_ISMQ0_RETRY.BL_RSP_VN0", + "PerPkg": "1", + "PublicDescription": "ISMQ Retries - Set 0 : BL RSP on VN0 : Number of times a transaction flowing through the ISMQ had to retry. Transaction pass through the ISMQ as responses for requests that already exist in the Cbo. Some examples include: when data is returned or when snoop responses come back from the cores. : No BL VN0 credit for generating a response", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "ISMQ Retries - Set 0 : BL WB on VN0", + "EventCode": "0x2c", + "EventName": "UNC_CHA_RxC_ISMQ0_RETRY.BL_WB_VN0", + "PerPkg": "1", + "PublicDescription": "ISMQ Retries - Set 0 : BL WB on VN0 : Number of times a transaction flowing through the ISMQ had to retry. Transaction pass through the ISMQ as responses for requests that already exist in the Cbo. Some examples include: when data is returned or when snoop responses come back from the cores. : No BL VN0 credit for generating a writeback", + "UMask": "0x8", + "Unit": "CHA" + }, + { + "BriefDescription": "ISMQ Retries - Set 0 : Non UPI IV Request", + "EventCode": "0x2c", + "EventName": "UNC_CHA_RxC_ISMQ0_RETRY.IV_NON_UPI", + "PerPkg": "1", + "PublicDescription": "ISMQ Retries - Set 0 : Non UPI IV Request : Number of times a transaction flowing through the ISMQ had to retry. Transaction pass through the ISMQ as responses for requests that already exist in the Cbo. Some examples include: when data is returned or when snoop responses come back from the cores. : Can't inject IV ring message", + "UMask": "0x80", + "Unit": "CHA" + }, + { + "BriefDescription": "ISMQ Rejects - Set 1 : ANY0", + "EventCode": "0x25", + "EventName": "UNC_CHA_RxC_ISMQ1_REJECT.ANY0", + "PerPkg": "1", + "PublicDescription": "ISMQ Rejects - Set 1 : ANY0 : Number of times a transaction flowing through the ISMQ had to retry. Transaction pass through the ISMQ as responses for requests that already exist in the Cbo. Some examples include: when data is returned or when snoop responses come back from the cores. : Any condition listed in the ISMQ0 Reject counter was true", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "ISMQ Rejects - Set 1 : HA", + "EventCode": "0x25", + "EventName": "UNC_CHA_RxC_ISMQ1_REJECT.HA", + "PerPkg": "1", + "PublicDescription": "ISMQ Rejects - Set 1 : HA : Number of times a transaction flowing through the ISMQ had to retry. Transaction pass through the ISMQ as responses for requests that already exist in the Cbo. Some examples include: when data is returned or when snoop responses come back from the cores.", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "ISMQ Retries - Set 1 : ANY0", + "EventCode": "0x2d", + "EventName": "UNC_CHA_RxC_ISMQ1_RETRY.ANY0", + "PerPkg": "1", + "PublicDescription": "ISMQ Retries - Set 1 : ANY0 : Number of times a transaction flowing through the ISMQ had to retry. Transaction pass through the ISMQ as responses for requests that already exist in the Cbo. Some examples include: when data is returned or when snoop responses come back from the cores. : Any condition listed in the ISMQ0 Reject counter was true", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "ISMQ Retries - Set 1 : HA", + "EventCode": "0x2d", + "EventName": "UNC_CHA_RxC_ISMQ1_RETRY.HA", + "PerPkg": "1", + "PublicDescription": "ISMQ Retries - Set 1 : HA : Number of times a transaction flowing through the ISMQ had to retry. Transaction pass through the ISMQ as responses for requests that already exist in the Cbo. Some examples include: when data is returned or when snoop responses come back from the cores.", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "Ingress (from CMS) Occupancy : IPQ", + "EventCode": "0x11", + "EventName": "UNC_CHA_RxC_OCCUPANCY.IPQ", + "PerPkg": "1", + "PublicDescription": "Ingress (from CMS) Occupancy : IPQ : Counts number of entries in the specified Ingress queue in each cycle.", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "Ingress (from CMS) Occupancy : RRQ", + "EventCode": "0x11", + "EventName": "UNC_CHA_RxC_OCCUPANCY.RRQ", + "PerPkg": "1", + "PublicDescription": "Ingress (from CMS) Occupancy : RRQ : Counts number of entries in the specified Ingress queue in each cycle.", + "UMask": "0x40", + "Unit": "CHA" + }, + { + "BriefDescription": "Ingress (from CMS) Occupancy : WBQ", + "EventCode": "0x11", + "EventName": "UNC_CHA_RxC_OCCUPANCY.WBQ", + "PerPkg": "1", + "PublicDescription": "Ingress (from CMS) Occupancy : WBQ : Counts number of entries in the specified Ingress queue in each cycle.", + "UMask": "0x80", + "Unit": "CHA" + }, + { + "BriefDescription": "Other Retries - Set 0 : AD REQ on VN0", + "EventCode": "0x2e", + "EventName": "UNC_CHA_RxC_OTHER0_RETRY.AD_REQ_VN0", + "PerPkg": "1", + "PublicDescription": "Other Retries - Set 0 : AD REQ on VN0 : Retry Queue Inserts of Transactions that were already in another Retry Q (sub-events encode the reason for the next reject) : No AD VN0 credit for generating a request", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "Other Retries - Set 0 : AD RSP on VN0", + "EventCode": "0x2e", + "EventName": "UNC_CHA_RxC_OTHER0_RETRY.AD_RSP_VN0", + "PerPkg": "1", + "PublicDescription": "Other Retries - Set 0 : AD RSP on VN0 : Retry Queue Inserts of Transactions that were already in another Retry Q (sub-events encode the reason for the next reject) : No AD VN0 credit for generating a response", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "Other Retries - Set 0 : Non UPI AK Request", + "EventCode": "0x2e", + "EventName": "UNC_CHA_RxC_OTHER0_RETRY.AK_NON_UPI", + "PerPkg": "1", + "PublicDescription": "Other Retries - Set 0 : Non UPI AK Request : Retry Queue Inserts of Transactions that were already in another Retry Q (sub-events encode the reason for the next reject) : Can't inject AK ring message", + "UMask": "0x40", + "Unit": "CHA" + }, + { + "BriefDescription": "Other Retries - Set 0 : BL NCB on VN0", + "EventCode": "0x2e", + "EventName": "UNC_CHA_RxC_OTHER0_RETRY.BL_NCB_VN0", + "PerPkg": "1", + "PublicDescription": "Other Retries - Set 0 : BL NCB on VN0 : Retry Queue Inserts of Transactions that were already in another Retry Q (sub-events encode the reason for the next reject) : No BL VN0 credit for NCB", + "UMask": "0x10", + "Unit": "CHA" + }, + { + "BriefDescription": "Other Retries - Set 0 : BL NCS on VN0", + "EventCode": "0x2e", + "EventName": "UNC_CHA_RxC_OTHER0_RETRY.BL_NCS_VN0", + "PerPkg": "1", + "PublicDescription": "Other Retries - Set 0 : BL NCS on VN0 : Retry Queue Inserts of Transactions that were already in another Retry Q (sub-events encode the reason for the next reject) : No BL VN0 credit for NCS", + "UMask": "0x20", + "Unit": "CHA" + }, + { + "BriefDescription": "Other Retries - Set 0 : BL RSP on VN0", + "EventCode": "0x2e", + "EventName": "UNC_CHA_RxC_OTHER0_RETRY.BL_RSP_VN0", + "PerPkg": "1", + "PublicDescription": "Other Retries - Set 0 : BL RSP on VN0 : Retry Queue Inserts of Transactions that were already in another Retry Q (sub-events encode the reason for the next reject) : No BL VN0 credit for generating a response", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "Other Retries - Set 0 : BL WB on VN0", + "EventCode": "0x2e", + "EventName": "UNC_CHA_RxC_OTHER0_RETRY.BL_WB_VN0", + "PerPkg": "1", + "PublicDescription": "Other Retries - Set 0 : BL WB on VN0 : Retry Queue Inserts of Transactions that were already in another Retry Q (sub-events encode the reason for the next reject) : No BL VN0 credit for generating a writeback", + "UMask": "0x8", + "Unit": "CHA" + }, + { + "BriefDescription": "Other Retries - Set 0 : Non UPI IV Request", + "EventCode": "0x2e", + "EventName": "UNC_CHA_RxC_OTHER0_RETRY.IV_NON_UPI", + "PerPkg": "1", + "PublicDescription": "Other Retries - Set 0 : Non UPI IV Request : Retry Queue Inserts of Transactions that were already in another Retry Q (sub-events encode the reason for the next reject) : Can't inject IV ring message", + "UMask": "0x80", + "Unit": "CHA" + }, + { + "BriefDescription": "Other Retries - Set 1 : Allow Snoop", + "EventCode": "0x2f", + "EventName": "UNC_CHA_RxC_OTHER1_RETRY.ALLOW_SNP", + "PerPkg": "1", + "PublicDescription": "Other Retries - Set 1 : Allow Snoop : Retry Queue Inserts of Transactions that were already in another Retry Q (sub-events encode the reason for the next reject)", + "UMask": "0x40", + "Unit": "CHA" + }, + { + "BriefDescription": "Other Retries - Set 1 : ANY0", + "EventCode": "0x2f", + "EventName": "UNC_CHA_RxC_OTHER1_RETRY.ANY0", + "PerPkg": "1", + "PublicDescription": "Other Retries - Set 1 : ANY0 : Retry Queue Inserts of Transactions that were already in another Retry Q (sub-events encode the reason for the next reject) : Any condition listed in the Other0 Reject counter was true", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "Other Retries - Set 1 : HA", + "EventCode": "0x2f", + "EventName": "UNC_CHA_RxC_OTHER1_RETRY.HA", + "PerPkg": "1", + "PublicDescription": "Other Retries - Set 1 : HA : Retry Queue Inserts of Transactions that were already in another Retry Q (sub-events encode the reason for the next reject)", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "Other Retries - Set 1 : LLC OR SF Way", + "EventCode": "0x2f", + "EventName": "UNC_CHA_RxC_OTHER1_RETRY.LLC_OR_SF_WAY", + "PerPkg": "1", + "PublicDescription": "Other Retries - Set 1 : LLC OR SF Way : Retry Queue Inserts of Transactions that were already in another Retry Q (sub-events encode the reason for the next reject) : Way conflict with another request that caused the reject", + "UMask": "0x20", + "Unit": "CHA" + }, + { + "BriefDescription": "Other Retries - Set 1 : LLC Victim", + "EventCode": "0x2f", + "EventName": "UNC_CHA_RxC_OTHER1_RETRY.LLC_VICTIM", + "PerPkg": "1", + "PublicDescription": "Other Retries - Set 1 : LLC Victim : Retry Queue Inserts of Transactions that were already in another Retry Q (sub-events encode the reason for the next reject)", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "Other Retries - Set 1 : PhyAddr Match", + "EventCode": "0x2f", + "EventName": "UNC_CHA_RxC_OTHER1_RETRY.PA_MATCH", + "PerPkg": "1", + "PublicDescription": "Other Retries - Set 1 : PhyAddr Match : Retry Queue Inserts of Transactions that were already in another Retry Q (sub-events encode the reason for the next reject) : Address match with an outstanding request that was rejected.", + "UMask": "0x80", + "Unit": "CHA" + }, + { + "BriefDescription": "Other Retries - Set 1 : SF Victim", + "EventCode": "0x2f", + "EventName": "UNC_CHA_RxC_OTHER1_RETRY.SF_VICTIM", + "PerPkg": "1", + "PublicDescription": "Other Retries - Set 1 : SF Victim : Retry Queue Inserts of Transactions that were already in another Retry Q (sub-events encode the reason for the next reject) : Requests did not generate Snoop filter victim", + "UMask": "0x8", + "Unit": "CHA" + }, + { + "BriefDescription": "Other Retries - Set 1 : Victim", + "EventCode": "0x2f", + "EventName": "UNC_CHA_RxC_OTHER1_RETRY.VICTIM", + "PerPkg": "1", + "PublicDescription": "Other Retries - Set 1 : Victim : Retry Queue Inserts of Transactions that were already in another Retry Q (sub-events encode the reason for the next reject)", + "UMask": "0x10", + "Unit": "CHA" + }, + { + "BriefDescription": "PRQ Requests (from CMS) Rejected - Set 0 : AD REQ on VN0", + "EventCode": "0x20", + "EventName": "UNC_CHA_RxC_PRQ0_REJECT.AD_REQ_VN0", + "PerPkg": "1", + "PublicDescription": "PRQ Requests (from CMS) Rejected - Set 0 : AD REQ on VN0 : No AD VN0 credit for generating a request", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "PRQ Requests (from CMS) Rejected - Set 0 : AD RSP on VN0", + "EventCode": "0x20", + "EventName": "UNC_CHA_RxC_PRQ0_REJECT.AD_RSP_VN0", + "PerPkg": "1", + "PublicDescription": "PRQ Requests (from CMS) Rejected - Set 0 : AD RSP on VN0 : No AD VN0 credit for generating a response", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "PRQ Requests (from CMS) Rejected - Set 0 : Non UPI AK Request", + "EventCode": "0x20", + "EventName": "UNC_CHA_RxC_PRQ0_REJECT.AK_NON_UPI", + "PerPkg": "1", + "PublicDescription": "PRQ Requests (from CMS) Rejected - Set 0 : Non UPI AK Request : Can't inject AK ring message", + "UMask": "0x40", + "Unit": "CHA" + }, + { + "BriefDescription": "PRQ Requests (from CMS) Rejected - Set 0 : BL NCB on VN0", + "EventCode": "0x20", + "EventName": "UNC_CHA_RxC_PRQ0_REJECT.BL_NCB_VN0", + "PerPkg": "1", + "PublicDescription": "PRQ Requests (from CMS) Rejected - Set 0 : BL NCB on VN0 : No BL VN0 credit for NCB", + "UMask": "0x10", + "Unit": "CHA" + }, + { + "BriefDescription": "PRQ Requests (from CMS) Rejected - Set 0 : BL NCS on VN0", + "EventCode": "0x20", + "EventName": "UNC_CHA_RxC_PRQ0_REJECT.BL_NCS_VN0", + "PerPkg": "1", + "PublicDescription": "PRQ Requests (from CMS) Rejected - Set 0 : BL NCS on VN0 : No BL VN0 credit for NCS", + "UMask": "0x20", + "Unit": "CHA" + }, + { + "BriefDescription": "PRQ Requests (from CMS) Rejected - Set 0 : BL RSP on VN0", + "EventCode": "0x20", + "EventName": "UNC_CHA_RxC_PRQ0_REJECT.BL_RSP_VN0", + "PerPkg": "1", + "PublicDescription": "PRQ Requests (from CMS) Rejected - Set 0 : BL RSP on VN0 : No BL VN0 credit for generating a response", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "PRQ Requests (from CMS) Rejected - Set 0 : BL WB on VN0", + "EventCode": "0x20", + "EventName": "UNC_CHA_RxC_PRQ0_REJECT.BL_WB_VN0", + "PerPkg": "1", + "PublicDescription": "PRQ Requests (from CMS) Rejected - Set 0 : BL WB on VN0 : No BL VN0 credit for generating a writeback", + "UMask": "0x8", + "Unit": "CHA" + }, + { + "BriefDescription": "PRQ Requests (from CMS) Rejected - Set 0 : Non UPI IV Request", + "EventCode": "0x20", + "EventName": "UNC_CHA_RxC_PRQ0_REJECT.IV_NON_UPI", + "PerPkg": "1", + "PublicDescription": "PRQ Requests (from CMS) Rejected - Set 0 : Non UPI IV Request : Can't inject IV ring message", + "UMask": "0x80", + "Unit": "CHA" + }, + { + "BriefDescription": "PRQ Requests (from CMS) Rejected - Set 1 : Allow Snoop", + "EventCode": "0x21", + "EventName": "UNC_CHA_RxC_PRQ1_REJECT.ALLOW_SNP", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "CHA" + }, + { + "BriefDescription": "PRQ Requests (from CMS) Rejected - Set 1 : ANY0", + "EventCode": "0x21", + "EventName": "UNC_CHA_RxC_PRQ1_REJECT.ANY0", + "PerPkg": "1", + "PublicDescription": "PRQ Requests (from CMS) Rejected - Set 1 : ANY0 : Any condition listed in the PRQ0 Reject counter was true", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "PRQ Requests (from CMS) Rejected - Set 1 : HA", + "EventCode": "0x21", + "EventName": "UNC_CHA_RxC_PRQ1_REJECT.HA", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "PRQ Requests (from CMS) Rejected - Set 1 : LLC OR SF Way", + "EventCode": "0x21", + "EventName": "UNC_CHA_RxC_PRQ1_REJECT.LLC_OR_SF_WAY", + "PerPkg": "1", + "PublicDescription": "PRQ Requests (from CMS) Rejected - Set 1 : LLC OR SF Way : Way conflict with another request that caused the reject", + "UMask": "0x20", + "Unit": "CHA" + }, + { + "BriefDescription": "PRQ Requests (from CMS) Rejected - Set 1 : LLC Victim", + "EventCode": "0x21", + "EventName": "UNC_CHA_RxC_PRQ1_REJECT.LLC_VICTIM", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "PRQ Requests (from CMS) Rejected - Set 1 : PhyAddr Match", + "EventCode": "0x21", + "EventName": "UNC_CHA_RxC_PRQ1_REJECT.PA_MATCH", + "PerPkg": "1", + "PublicDescription": "PRQ Requests (from CMS) Rejected - Set 1 : PhyAddr Match : Address match with an outstanding request that was rejected.", + "UMask": "0x80", + "Unit": "CHA" + }, + { + "BriefDescription": "PRQ Requests (from CMS) Rejected - Set 1 : SF Victim", + "EventCode": "0x21", + "EventName": "UNC_CHA_RxC_PRQ1_REJECT.SF_VICTIM", + "PerPkg": "1", + "PublicDescription": "PRQ Requests (from CMS) Rejected - Set 1 : SF Victim : Requests did not generate Snoop filter victim", + "UMask": "0x8", + "Unit": "CHA" + }, + { + "BriefDescription": "PRQ Requests (from CMS) Rejected - Set 1 : Victim", + "EventCode": "0x21", + "EventName": "UNC_CHA_RxC_PRQ1_REJECT.VICTIM", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "CHA" + }, + { + "BriefDescription": "Request Queue Retries - Set 0 : AD REQ on VN0", + "EventCode": "0x2a", + "EventName": "UNC_CHA_RxC_REQ_Q0_RETRY.AD_REQ_VN0", + "PerPkg": "1", + "PublicDescription": "Request Queue Retries - Set 0 : AD REQ on VN0 : REQUESTQ includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ) : No AD VN0 credit for generating a request", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "Request Queue Retries - Set 0 : AD RSP on VN0", + "EventCode": "0x2a", + "EventName": "UNC_CHA_RxC_REQ_Q0_RETRY.AD_RSP_VN0", + "PerPkg": "1", + "PublicDescription": "Request Queue Retries - Set 0 : AD RSP on VN0 : REQUESTQ includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ) : No AD VN0 credit for generating a response", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "Request Queue Retries - Set 0 : Non UPI AK Request", + "EventCode": "0x2a", + "EventName": "UNC_CHA_RxC_REQ_Q0_RETRY.AK_NON_UPI", + "PerPkg": "1", + "PublicDescription": "Request Queue Retries - Set 0 : Non UPI AK Request : REQUESTQ includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ) : Can't inject AK ring message", + "UMask": "0x40", + "Unit": "CHA" + }, + { + "BriefDescription": "Request Queue Retries - Set 0 : BL NCB on VN0", + "EventCode": "0x2a", + "EventName": "UNC_CHA_RxC_REQ_Q0_RETRY.BL_NCB_VN0", + "PerPkg": "1", + "PublicDescription": "Request Queue Retries - Set 0 : BL NCB on VN0 : REQUESTQ includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ) : No BL VN0 credit for NCB", + "UMask": "0x10", + "Unit": "CHA" + }, + { + "BriefDescription": "Request Queue Retries - Set 0 : BL NCS on VN0", + "EventCode": "0x2a", + "EventName": "UNC_CHA_RxC_REQ_Q0_RETRY.BL_NCS_VN0", + "PerPkg": "1", + "PublicDescription": "Request Queue Retries - Set 0 : BL NCS on VN0 : REQUESTQ includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ) : No BL VN0 credit for NCS", + "UMask": "0x20", + "Unit": "CHA" + }, + { + "BriefDescription": "Request Queue Retries - Set 0 : BL RSP on VN0", + "EventCode": "0x2a", + "EventName": "UNC_CHA_RxC_REQ_Q0_RETRY.BL_RSP_VN0", + "PerPkg": "1", + "PublicDescription": "Request Queue Retries - Set 0 : BL RSP on VN0 : REQUESTQ includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ) : No BL VN0 credit for generating a response", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "Request Queue Retries - Set 0 : BL WB on VN0", + "EventCode": "0x2a", + "EventName": "UNC_CHA_RxC_REQ_Q0_RETRY.BL_WB_VN0", + "PerPkg": "1", + "PublicDescription": "Request Queue Retries - Set 0 : BL WB on VN0 : REQUESTQ includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ) : No BL VN0 credit for generating a writeback", + "UMask": "0x8", + "Unit": "CHA" + }, + { + "BriefDescription": "Request Queue Retries - Set 0 : Non UPI IV Request", + "EventCode": "0x2a", + "EventName": "UNC_CHA_RxC_REQ_Q0_RETRY.IV_NON_UPI", + "PerPkg": "1", + "PublicDescription": "Request Queue Retries - Set 0 : Non UPI IV Request : REQUESTQ includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ) : Can't inject IV ring message", + "UMask": "0x80", + "Unit": "CHA" + }, + { + "BriefDescription": "Request Queue Retries - Set 1 : Allow Snoop", + "EventCode": "0x2b", + "EventName": "UNC_CHA_RxC_REQ_Q1_RETRY.ALLOW_SNP", + "PerPkg": "1", + "PublicDescription": "Request Queue Retries - Set 1 : Allow Snoop : REQUESTQ includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ)", + "UMask": "0x40", + "Unit": "CHA" + }, + { + "BriefDescription": "Request Queue Retries - Set 1 : ANY0", + "EventCode": "0x2b", + "EventName": "UNC_CHA_RxC_REQ_Q1_RETRY.ANY0", + "PerPkg": "1", + "PublicDescription": "Request Queue Retries - Set 1 : ANY0 : REQUESTQ includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ) : Any condition listed in the WBQ0 Reject counter was true", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "Request Queue Retries - Set 1 : HA", + "EventCode": "0x2b", + "EventName": "UNC_CHA_RxC_REQ_Q1_RETRY.HA", + "PerPkg": "1", + "PublicDescription": "Request Queue Retries - Set 1 : HA : REQUESTQ includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ)", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "Request Queue Retries - Set 1 : LLC OR SF Way", + "EventCode": "0x2b", + "EventName": "UNC_CHA_RxC_REQ_Q1_RETRY.LLC_OR_SF_WAY", + "PerPkg": "1", + "PublicDescription": "Request Queue Retries - Set 1 : LLC OR SF Way : REQUESTQ includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ) : Way conflict with another request that caused the reject", + "UMask": "0x20", + "Unit": "CHA" + }, + { + "BriefDescription": "Request Queue Retries - Set 1 : LLC Victim", + "EventCode": "0x2b", + "EventName": "UNC_CHA_RxC_REQ_Q1_RETRY.LLC_VICTIM", + "PerPkg": "1", + "PublicDescription": "Request Queue Retries - Set 1 : LLC Victim : REQUESTQ includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ)", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "Request Queue Retries - Set 1 : PhyAddr Match", + "EventCode": "0x2b", + "EventName": "UNC_CHA_RxC_REQ_Q1_RETRY.PA_MATCH", + "PerPkg": "1", + "PublicDescription": "Request Queue Retries - Set 1 : PhyAddr Match : REQUESTQ includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ) : Address match with an outstanding request that was rejected.", + "UMask": "0x80", + "Unit": "CHA" + }, + { + "BriefDescription": "Request Queue Retries - Set 1 : SF Victim", + "EventCode": "0x2b", + "EventName": "UNC_CHA_RxC_REQ_Q1_RETRY.SF_VICTIM", + "PerPkg": "1", + "PublicDescription": "Request Queue Retries - Set 1 : SF Victim : REQUESTQ includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ) : Requests did not generate Snoop filter victim", + "UMask": "0x8", + "Unit": "CHA" + }, + { + "BriefDescription": "Request Queue Retries - Set 1 : Victim", + "EventCode": "0x2b", + "EventName": "UNC_CHA_RxC_REQ_Q1_RETRY.VICTIM", + "PerPkg": "1", + "PublicDescription": "Request Queue Retries - Set 1 : Victim : REQUESTQ includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ)", + "UMask": "0x10", + "Unit": "CHA" + }, + { + "BriefDescription": "RRQ Rejects - Set 0 : AD REQ on VN0", + "EventCode": "0x26", + "EventName": "UNC_CHA_RxC_RRQ0_REJECT.AD_REQ_VN0", + "PerPkg": "1", + "PublicDescription": "RRQ Rejects - Set 0 : AD REQ on VN0 : Number of times a transaction flowing through the RRQ (Remote Response Queue) had to retry. : No AD VN0 credit for generating a request", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "RRQ Rejects - Set 0 : AD RSP on VN0", + "EventCode": "0x26", + "EventName": "UNC_CHA_RxC_RRQ0_REJECT.AD_RSP_VN0", + "PerPkg": "1", + "PublicDescription": "RRQ Rejects - Set 0 : AD RSP on VN0 : Number of times a transaction flowing through the RRQ (Remote Response Queue) had to retry. : No AD VN0 credit for generating a response", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "RRQ Rejects - Set 0 : Non UPI AK Request", + "EventCode": "0x26", + "EventName": "UNC_CHA_RxC_RRQ0_REJECT.AK_NON_UPI", + "PerPkg": "1", + "PublicDescription": "RRQ Rejects - Set 0 : Non UPI AK Request : Number of times a transaction flowing through the RRQ (Remote Response Queue) had to retry. : Can't inject AK ring message", + "UMask": "0x40", + "Unit": "CHA" + }, + { + "BriefDescription": "RRQ Rejects - Set 0 : BL NCB on VN0", + "EventCode": "0x26", + "EventName": "UNC_CHA_RxC_RRQ0_REJECT.BL_NCB_VN0", + "PerPkg": "1", + "PublicDescription": "RRQ Rejects - Set 0 : BL NCB on VN0 : Number of times a transaction flowing through the RRQ (Remote Response Queue) had to retry. : No BL VN0 credit for NCB", + "UMask": "0x10", + "Unit": "CHA" + }, + { + "BriefDescription": "RRQ Rejects - Set 0 : BL NCS on VN0", + "EventCode": "0x26", + "EventName": "UNC_CHA_RxC_RRQ0_REJECT.BL_NCS_VN0", + "PerPkg": "1", + "PublicDescription": "RRQ Rejects - Set 0 : BL NCS on VN0 : Number of times a transaction flowing through the RRQ (Remote Response Queue) had to retry. : No BL VN0 credit for NCS", + "UMask": "0x20", + "Unit": "CHA" + }, + { + "BriefDescription": "RRQ Rejects - Set 0 : BL RSP on VN0", + "EventCode": "0x26", + "EventName": "UNC_CHA_RxC_RRQ0_REJECT.BL_RSP_VN0", + "PerPkg": "1", + "PublicDescription": "RRQ Rejects - Set 0 : BL RSP on VN0 : Number of times a transaction flowing through the RRQ (Remote Response Queue) had to retry. : No BL VN0 credit for generating a response", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "RRQ Rejects - Set 0 : BL WB on VN0", + "EventCode": "0x26", + "EventName": "UNC_CHA_RxC_RRQ0_REJECT.BL_WB_VN0", + "PerPkg": "1", + "PublicDescription": "RRQ Rejects - Set 0 : BL WB on VN0 : Number of times a transaction flowing through the RRQ (Remote Response Queue) had to retry. : No BL VN0 credit for generating a writeback", + "UMask": "0x8", + "Unit": "CHA" + }, + { + "BriefDescription": "RRQ Rejects - Set 0 : Non UPI IV Request", + "EventCode": "0x26", + "EventName": "UNC_CHA_RxC_RRQ0_REJECT.IV_NON_UPI", + "PerPkg": "1", + "PublicDescription": "RRQ Rejects - Set 0 : Non UPI IV Request : Number of times a transaction flowing through the RRQ (Remote Response Queue) had to retry. : Can't inject IV ring message", + "UMask": "0x80", + "Unit": "CHA" + }, + { + "BriefDescription": "RRQ Rejects - Set 1 : Allow Snoop", + "EventCode": "0x27", + "EventName": "UNC_CHA_RxC_RRQ1_REJECT.ALLOW_SNP", + "PerPkg": "1", + "PublicDescription": "RRQ Rejects - Set 1 : Allow Snoop : Number of times a transaction flowing through the RRQ (Remote Response Queue) had to retry.", + "UMask": "0x40", + "Unit": "CHA" + }, + { + "BriefDescription": "RRQ Rejects - Set 1 : ANY0", + "EventCode": "0x27", + "EventName": "UNC_CHA_RxC_RRQ1_REJECT.ANY0", + "PerPkg": "1", + "PublicDescription": "RRQ Rejects - Set 1 : ANY0 : Number of times a transaction flowing through the RRQ (Remote Response Queue) had to retry. : Any condition listed in the RRQ0 Reject counter was true", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "RRQ Rejects - Set 1 : HA", + "EventCode": "0x27", + "EventName": "UNC_CHA_RxC_RRQ1_REJECT.HA", + "PerPkg": "1", + "PublicDescription": "RRQ Rejects - Set 1 : HA : Number of times a transaction flowing through the RRQ (Remote Response Queue) had to retry.", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "RRQ Rejects - Set 1 : LLC OR SF Way", + "EventCode": "0x27", + "EventName": "UNC_CHA_RxC_RRQ1_REJECT.LLC_OR_SF_WAY", + "PerPkg": "1", + "PublicDescription": "RRQ Rejects - Set 1 : LLC OR SF Way : Number of times a transaction flowing through the RRQ (Remote Response Queue) had to retry. : Way conflict with another request that caused the reject", + "UMask": "0x20", + "Unit": "CHA" + }, + { + "BriefDescription": "RRQ Rejects - Set 1 : LLC Victim", + "EventCode": "0x27", + "EventName": "UNC_CHA_RxC_RRQ1_REJECT.LLC_VICTIM", + "PerPkg": "1", + "PublicDescription": "RRQ Rejects - Set 1 : LLC Victim : Number of times a transaction flowing through the RRQ (Remote Response Queue) had to retry.", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "RRQ Rejects - Set 1 : PhyAddr Match", + "EventCode": "0x27", + "EventName": "UNC_CHA_RxC_RRQ1_REJECT.PA_MATCH", + "PerPkg": "1", + "PublicDescription": "RRQ Rejects - Set 1 : PhyAddr Match : Number of times a transaction flowing through the RRQ (Remote Response Queue) had to retry. : Address match with an outstanding request that was rejected.", + "UMask": "0x80", + "Unit": "CHA" + }, + { + "BriefDescription": "RRQ Rejects - Set 1 : SF Victim", + "EventCode": "0x27", + "EventName": "UNC_CHA_RxC_RRQ1_REJECT.SF_VICTIM", + "PerPkg": "1", + "PublicDescription": "RRQ Rejects - Set 1 : SF Victim : Number of times a transaction flowing through the RRQ (Remote Response Queue) had to retry. : Requests did not generate Snoop filter victim", + "UMask": "0x8", + "Unit": "CHA" + }, + { + "BriefDescription": "RRQ Rejects - Set 1 : Victim", + "EventCode": "0x27", + "EventName": "UNC_CHA_RxC_RRQ1_REJECT.VICTIM", + "PerPkg": "1", + "PublicDescription": "RRQ Rejects - Set 1 : Victim : Number of times a transaction flowing through the RRQ (Remote Response Queue) had to retry.", + "UMask": "0x10", + "Unit": "CHA" + }, + { + "BriefDescription": "WBQ Rejects - Set 0 : AD REQ on VN0", + "EventCode": "0x28", + "EventName": "UNC_CHA_RxC_WBQ0_REJECT.AD_REQ_VN0", + "PerPkg": "1", + "PublicDescription": "WBQ Rejects - Set 0 : AD REQ on VN0 : Number of times a transaction flowing through the WBQ (Writeback Queue) had to retry. : No AD VN0 credit for generating a request", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "WBQ Rejects - Set 0 : AD RSP on VN0", + "EventCode": "0x28", + "EventName": "UNC_CHA_RxC_WBQ0_REJECT.AD_RSP_VN0", + "PerPkg": "1", + "PublicDescription": "WBQ Rejects - Set 0 : AD RSP on VN0 : Number of times a transaction flowing through the WBQ (Writeback Queue) had to retry. : No AD VN0 credit for generating a response", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "WBQ Rejects - Set 0 : Non UPI AK Request", + "EventCode": "0x28", + "EventName": "UNC_CHA_RxC_WBQ0_REJECT.AK_NON_UPI", + "PerPkg": "1", + "PublicDescription": "WBQ Rejects - Set 0 : Non UPI AK Request : Number of times a transaction flowing through the WBQ (Writeback Queue) had to retry. : Can't inject AK ring message", + "UMask": "0x40", + "Unit": "CHA" + }, + { + "BriefDescription": "WBQ Rejects - Set 0 : BL NCB on VN0", + "EventCode": "0x28", + "EventName": "UNC_CHA_RxC_WBQ0_REJECT.BL_NCB_VN0", + "PerPkg": "1", + "PublicDescription": "WBQ Rejects - Set 0 : BL NCB on VN0 : Number of times a transaction flowing through the WBQ (Writeback Queue) had to retry. : No BL VN0 credit for NCB", + "UMask": "0x10", + "Unit": "CHA" + }, + { + "BriefDescription": "WBQ Rejects - Set 0 : BL NCS on VN0", + "EventCode": "0x28", + "EventName": "UNC_CHA_RxC_WBQ0_REJECT.BL_NCS_VN0", + "PerPkg": "1", + "PublicDescription": "WBQ Rejects - Set 0 : BL NCS on VN0 : Number of times a transaction flowing through the WBQ (Writeback Queue) had to retry. : No BL VN0 credit for NCS", + "UMask": "0x20", + "Unit": "CHA" + }, + { + "BriefDescription": "WBQ Rejects - Set 0 : BL RSP on VN0", + "EventCode": "0x28", + "EventName": "UNC_CHA_RxC_WBQ0_REJECT.BL_RSP_VN0", + "PerPkg": "1", + "PublicDescription": "WBQ Rejects - Set 0 : BL RSP on VN0 : Number of times a transaction flowing through the WBQ (Writeback Queue) had to retry. : No BL VN0 credit for generating a response", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "WBQ Rejects - Set 0 : BL WB on VN0", + "EventCode": "0x28", + "EventName": "UNC_CHA_RxC_WBQ0_REJECT.BL_WB_VN0", + "PerPkg": "1", + "PublicDescription": "WBQ Rejects - Set 0 : BL WB on VN0 : Number of times a transaction flowing through the WBQ (Writeback Queue) had to retry. : No BL VN0 credit for generating a writeback", + "UMask": "0x8", + "Unit": "CHA" + }, + { + "BriefDescription": "WBQ Rejects - Set 0 : Non UPI IV Request", + "EventCode": "0x28", + "EventName": "UNC_CHA_RxC_WBQ0_REJECT.IV_NON_UPI", + "PerPkg": "1", + "PublicDescription": "WBQ Rejects - Set 0 : Non UPI IV Request : Number of times a transaction flowing through the WBQ (Writeback Queue) had to retry. : Can't inject IV ring message", + "UMask": "0x80", + "Unit": "CHA" + }, + { + "BriefDescription": "WBQ Rejects - Set 1 : Allow Snoop", + "EventCode": "0x29", + "EventName": "UNC_CHA_RxC_WBQ1_REJECT.ALLOW_SNP", + "PerPkg": "1", + "PublicDescription": "WBQ Rejects - Set 1 : Allow Snoop : Number of times a transaction flowing through the WBQ (Writeback Queue) had to retry.", + "UMask": "0x40", + "Unit": "CHA" + }, + { + "BriefDescription": "WBQ Rejects - Set 1 : ANY0", + "EventCode": "0x29", + "EventName": "UNC_CHA_RxC_WBQ1_REJECT.ANY0", + "PerPkg": "1", + "PublicDescription": "WBQ Rejects - Set 1 : ANY0 : Number of times a transaction flowing through the WBQ (Writeback Queue) had to retry. : Any condition listed in the WBQ0 Reject counter was true", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "WBQ Rejects - Set 1 : HA", + "EventCode": "0x29", + "EventName": "UNC_CHA_RxC_WBQ1_REJECT.HA", + "PerPkg": "1", + "PublicDescription": "WBQ Rejects - Set 1 : HA : Number of times a transaction flowing through the WBQ (Writeback Queue) had to retry.", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "WBQ Rejects - Set 1 : LLC OR SF Way", + "EventCode": "0x29", + "EventName": "UNC_CHA_RxC_WBQ1_REJECT.LLC_OR_SF_WAY", + "PerPkg": "1", + "PublicDescription": "WBQ Rejects - Set 1 : LLC OR SF Way : Number of times a transaction flowing through the WBQ (Writeback Queue) had to retry. : Way conflict with another request that caused the reject", + "UMask": "0x20", + "Unit": "CHA" + }, + { + "BriefDescription": "WBQ Rejects - Set 1 : LLC Victim", + "EventCode": "0x29", + "EventName": "UNC_CHA_RxC_WBQ1_REJECT.LLC_VICTIM", + "PerPkg": "1", + "PublicDescription": "WBQ Rejects - Set 1 : LLC Victim : Number of times a transaction flowing through the WBQ (Writeback Queue) had to retry.", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "WBQ Rejects - Set 1 : PhyAddr Match", + "EventCode": "0x29", + "EventName": "UNC_CHA_RxC_WBQ1_REJECT.PA_MATCH", + "PerPkg": "1", + "PublicDescription": "WBQ Rejects - Set 1 : PhyAddr Match : Number of times a transaction flowing through the WBQ (Writeback Queue) had to retry. : Address match with an outstanding request that was rejected.", + "UMask": "0x80", + "Unit": "CHA" + }, + { + "BriefDescription": "WBQ Rejects - Set 1 : SF Victim", + "EventCode": "0x29", + "EventName": "UNC_CHA_RxC_WBQ1_REJECT.SF_VICTIM", + "PerPkg": "1", + "PublicDescription": "WBQ Rejects - Set 1 : SF Victim : Number of times a transaction flowing through the WBQ (Writeback Queue) had to retry. : Requests did not generate Snoop filter victim", + "UMask": "0x8", + "Unit": "CHA" + }, + { + "BriefDescription": "WBQ Rejects - Set 1 : Victim", + "EventCode": "0x29", + "EventName": "UNC_CHA_RxC_WBQ1_REJECT.VICTIM", + "PerPkg": "1", + "PublicDescription": "WBQ Rejects - Set 1 : Victim : Number of times a transaction flowing through the WBQ (Writeback Queue) had to retry.", + "UMask": "0x10", + "Unit": "CHA" + }, + { + "BriefDescription": "Snoops Sent : All", + "EventCode": "0x51", + "EventName": "UNC_CHA_SNOOPS_SENT.ALL", + "PerPkg": "1", + "PublicDescription": "Snoops Sent : All : Counts the number of snoops issued by the HA.", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "Snoops Sent : Broadcast snoop for Local Requests", + "EventCode": "0x51", + "EventName": "UNC_CHA_SNOOPS_SENT.BCST_LOCAL", + "PerPkg": "1", + "PublicDescription": "Snoops Sent : Broadcast snoop for Local Requests : Counts the number of snoops issued by the HA. : Counts the number of broadcast snoops issued by the HA. This filter includes only requests coming from local sockets.", + "UMask": "0x10", + "Unit": "CHA" + }, + { + "BriefDescription": "Snoops Sent : Broadcast snoops for Remote Requests", + "EventCode": "0x51", + "EventName": "UNC_CHA_SNOOPS_SENT.BCST_REMOTE", + "PerPkg": "1", + "PublicDescription": "Snoops Sent : Broadcast snoops for Remote Requests : Counts the number of snoops issued by the HA. : Counts the number of broadcast snoops issued by the HA.This filter includes only requests coming from remote sockets.", + "UMask": "0x20", + "Unit": "CHA" + }, + { + "BriefDescription": "Snoops Sent : Directed snoops for Local Requests", + "EventCode": "0x51", + "EventName": "UNC_CHA_SNOOPS_SENT.DIRECT_LOCAL", + "PerPkg": "1", + "PublicDescription": "Snoops Sent : Directed snoops for Local Requests : Counts the number of snoops issued by the HA. : Counts the number of directed snoops issued by the HA. This filter includes only requests coming from local sockets.", + "UMask": "0x40", + "Unit": "CHA" + }, + { + "BriefDescription": "Snoops Sent : Directed snoops for Remote Requests", + "EventCode": "0x51", + "EventName": "UNC_CHA_SNOOPS_SENT.DIRECT_REMOTE", + "PerPkg": "1", + "PublicDescription": "Snoops Sent : Directed snoops for Remote Requests : Counts the number of snoops issued by the HA. : Counts the number of directed snoops issued by the HA. This filter includes only requests coming from remote sockets.", + "UMask": "0x80", + "Unit": "CHA" + }, + { + "BriefDescription": "Snoops Sent : Broadcast or directed Snoops sent for Local Requests", + "EventCode": "0x51", + "EventName": "UNC_CHA_SNOOPS_SENT.LOCAL", + "PerPkg": "1", + "PublicDescription": "Snoops Sent : Broadcast or directed Snoops sent for Local Requests : Counts the number of snoops issued by the HA. : Counts the number of broadcast or directed snoops issued by the HA per request. This filter includes only requests coming from the local socket.", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "Snoops Sent : Broadcast or directed Snoops sent for Remote Requests", + "EventCode": "0x51", + "EventName": "UNC_CHA_SNOOPS_SENT.REMOTE", + "PerPkg": "1", + "PublicDescription": "Snoops Sent : Broadcast or directed Snoops sent for Remote Requests : Counts the number of snoops issued by the HA. : Counts the number of broadcast or directed snoops issued by the HA per request. This filter includes only requests coming from the remote socket.", + "UMask": "0x8", + "Unit": "CHA" + }, + { + "BriefDescription": "Snoop Responses Received : RSPCNFLCT*", + "EventCode": "0x5c", + "EventName": "UNC_CHA_SNOOP_RESP.RSPCNFLCT", + "PerPkg": "1", + "PublicDescription": "Snoop Responses Received : RSPCNFLCT* : Counts the total number of RspI snoop responses received. Whenever a snoops are issued, one or more snoop responses will be returned depending on the topology of the system. In systems larger than 2s, when multiple snoops are returned this will count all the snoops that are received. For example, if 3 snoops were issued and returned RspI, RspS, and RspSFwd; then each of these sub-events would increment by 1. : Filters for snoops responses of RspConflict. This is returned when a snoop finds an existing outstanding transaction in a remote caching agent when it CAMs that caching agent. This triggers conflict resolution hardware. This covers both RspCnflct and RspCnflctWbI.", + "UMask": "0x40", + "Unit": "CHA" + }, + { + "BriefDescription": "Snoop Responses Received : RspFwd", + "EventCode": "0x5c", + "EventName": "UNC_CHA_SNOOP_RESP.RSPFWD", + "PerPkg": "1", + "PublicDescription": "Snoop Responses Received : RspFwd : Counts the total number of RspI snoop responses received. Whenever a snoops are issued, one or more snoop responses will be returned depending on the topology of the system. In systems larger than 2s, when multiple snoops are returned this will count all the snoops that are received. For example, if 3 snoops were issued and returned RspI, RspS, and RspSFwd; then each of these sub-events would increment by 1. : Filters for a snoop response of RspFwd to a CA request. This snoop response is only possible for RdCur when a snoop HITM/E in a remote caching agent and it directly forwards data to a requestor without changing the requestor's cache line state.", + "UMask": "0x80", + "Unit": "CHA" + }, + { + "BriefDescription": "Snoop Responses Received : Rsp*Fwd*WB", + "EventCode": "0x5c", + "EventName": "UNC_CHA_SNOOP_RESP.RSPFWDWB", + "PerPkg": "1", + "PublicDescription": "Snoop Responses Received : Rsp*Fwd*WB : Counts the total number of RspI snoop responses received. Whenever a snoops are issued, one or more snoop responses will be returned depending on the topology of the system. In systems larger than 2s, when multiple snoops are returned this will count all the snoops that are received. For example, if 3 snoops were issued and returned RspI, RspS, and RspSFwd; then each of these sub-events would increment by 1. : Filters for a snoop response of Rsp*Fwd*WB. This snoop response is only used in 4s systems. It is used when a snoop HITM's in a remote caching agent and it directly forwards data to a requestor, and simultaneously returns data to the home to be written back to memory.", + "UMask": "0x20", + "Unit": "CHA" + }, + { + "BriefDescription": "RspI Snoop Responses Received", + "EventCode": "0x5c", + "EventName": "UNC_CHA_SNOOP_RESP.RSPI", + "PerPkg": "1", + "PublicDescription": "Counts when a transaction with the opcode type RspI Snoop Response was received which indicates the remote cache does not have the data, or when the remote cache silently evicts data (such as when an RFO: the Read for Ownership issued before a write hits non-modified data).", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "RspIFwd Snoop Responses Received", + "EventCode": "0x5c", + "EventName": "UNC_CHA_SNOOP_RESP.RSPIFWD", + "PerPkg": "1", + "PublicDescription": "Counts when a a transaction with the opcode type RspIFwd Snoop Response was received which indicates a remote caching agent forwarded the data and the requesting agent is able to acquire the data in E (Exclusive) or M (modified) states. This is commonly returned with RFO (the Read for Ownership issued before a write) transactions. The snoop could have either been to a cacheline in the M,E,F (Modified, Exclusive or Forward) states.", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "RspS Snoop Responses Received", + "EventCode": "0x5c", + "EventName": "UNC_CHA_SNOOP_RESP.RSPS", + "PerPkg": "1", + "PublicDescription": "Counts when a transaction with the opcode type RspS Snoop Response was received which indicates when a remote cache has data but is not forwarding it. It is a way to let the requesting socket know that it cannot allocate the data in E state. No data is sent with S RspS.", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "RspSFwd Snoop Responses Received", + "EventCode": "0x5c", + "EventName": "UNC_CHA_SNOOP_RESP.RSPSFWD", + "PerPkg": "1", + "PublicDescription": "Counts when a a transaction with the opcode type RspSFwd Snoop Response was received which indicates a remote caching agent forwarded the data but held on to its current copy. This is common for data and code reads that hit in a remote socket in E (Exclusive) or F (Forward) state.", + "UMask": "0x8", + "Unit": "CHA" + }, + { + "BriefDescription": "Snoop Responses Received : Rsp*WB", + "EventCode": "0x5c", + "EventName": "UNC_CHA_SNOOP_RESP.RSPWB", + "PerPkg": "1", + "PublicDescription": "Snoop Responses Received : Rsp*WB : Counts the total number of RspI snoop responses received. Whenever a snoops are issued, one or more snoop responses will be returned depending on the topology of the system. In systems larger than 2s, when multiple snoops are returned this will count all the snoops that are received. For example, if 3 snoops were issued and returned RspI, RspS, and RspSFwd; then each of these sub-events would increment by 1. : Filters for a snoop response of RspIWB or RspSWB. This is returned when a non-RFO request hits in M state. Data and Code Reads can return either RspIWB or RspSWB depending on how the system has been configured. InvItoE transactions will also return RspIWB because they must acquire ownership.", + "UMask": "0x10", + "Unit": "CHA" + }, + { + "BriefDescription": "Snoop Responses Received Local : RspCnflct", + "EventCode": "0x5d", + "EventName": "UNC_CHA_SNOOP_RESP_LOCAL.RSPCNFLCT", + "PerPkg": "1", + "PublicDescription": "Snoop Responses Received Local : RspCnflct : Number of snoop responses received for a Local request : Filters for snoops responses of RspConflict to local CA requests. This is returned when a snoop finds an existing outstanding transaction in a remote caching agent when it CAMs that caching agent. This triggers conflict resolution hardware. This covers both RspCnflct and RspCnflctWbI.", + "UMask": "0x40", + "Unit": "CHA" + }, + { + "BriefDescription": "Snoop Responses Received Local : RspFwd", + "EventCode": "0x5d", + "EventName": "UNC_CHA_SNOOP_RESP_LOCAL.RSPFWD", + "PerPkg": "1", + "PublicDescription": "Snoop Responses Received Local : RspFwd : Number of snoop responses received for a Local request : Filters for a snoop response of RspFwd to local CA requests. This snoop response is only possible for RdCur when a snoop HITM/E in a remote caching agent and it directly forwards data to a requestor without changing the requestor's cache line state.", + "UMask": "0x80", + "Unit": "CHA" + }, + { + "BriefDescription": "Snoop Responses Received Local : Rsp*FWD*WB", + "EventCode": "0x5d", + "EventName": "UNC_CHA_SNOOP_RESP_LOCAL.RSPFWDWB", + "PerPkg": "1", + "PublicDescription": "Snoop Responses Received Local : Rsp*FWD*WB : Number of snoop responses received for a Local request : Filters for a snoop response of Rsp*Fwd*WB to local CA requests. This snoop response is only used in 4s systems. It is used when a snoop HITM's in a remote caching agent and it directly forwards data to a requestor, and simultaneously returns data to the home to be written back to memory.", + "UMask": "0x20", + "Unit": "CHA" + }, + { + "BriefDescription": "Snoop Responses Received Local : RspI", + "EventCode": "0x5d", + "EventName": "UNC_CHA_SNOOP_RESP_LOCAL.RSPI", + "PerPkg": "1", + "PublicDescription": "Snoop Responses Received Local : RspI : Number of snoop responses received for a Local request : Filters for snoops responses of RspI to local CA requests. RspI is returned when the remote cache does not have the data, or when the remote cache silently evicts data (such as when an RFO hits non-modified data).", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "Snoop Responses Received Local : RspIFwd", + "EventCode": "0x5d", + "EventName": "UNC_CHA_SNOOP_RESP_LOCAL.RSPIFWD", + "PerPkg": "1", + "PublicDescription": "Snoop Responses Received Local : RspIFwd : Number of snoop responses received for a Local request : Filters for snoop responses of RspIFwd to local CA requests. This is returned when a remote caching agent forwards data and the requesting agent is able to acquire the data in E or M states. This is commonly returned with RFO transactions. It can be either a HitM or a HitFE.", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "Snoop Responses Received Local : RspS", + "EventCode": "0x5d", + "EventName": "UNC_CHA_SNOOP_RESP_LOCAL.RSPS", + "PerPkg": "1", + "PublicDescription": "Snoop Responses Received Local : RspS : Number of snoop responses received for a Local request : Filters for snoop responses of RspS to local CA requests. RspS is returned when a remote cache has data but is not forwarding it. It is a way to let the requesting socket know that it cannot allocate the data in E state. No data is sent with S RspS.", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "Snoop Responses Received Local : RspSFwd", + "EventCode": "0x5d", + "EventName": "UNC_CHA_SNOOP_RESP_LOCAL.RSPSFWD", + "PerPkg": "1", + "PublicDescription": "Snoop Responses Received Local : RspSFwd : Number of snoop responses received for a Local request : Filters for a snoop response of RspSFwd to local CA requests. This is returned when a remote caching agent forwards data but holds on to its current copy. This is common for data and code reads that hit in a remote socket in E or F state.", + "UMask": "0x8", + "Unit": "CHA" + }, + { + "BriefDescription": "Snoop Responses Received Local : Rsp*WB", + "EventCode": "0x5d", + "EventName": "UNC_CHA_SNOOP_RESP_LOCAL.RSPWB", + "PerPkg": "1", + "PublicDescription": "Snoop Responses Received Local : Rsp*WB : Number of snoop responses received for a Local request : Filters for a snoop response of RspIWB or RspSWB to local CA requests. This is returned when a non-RFO request hits in M state. Data and Code Reads can return either RspIWB or RspSWB depending on how the system has been configured. InvItoE transactions will also return RspIWB because they must acquire ownership.", + "UMask": "0x10", + "Unit": "CHA" + }, + { + "BriefDescription": "Misc Snoop Responses Received : MtoI RspIDataM", + "EventCode": "0x6b", + "EventName": "UNC_CHA_SNOOP_RSP_MISC.MTOI_RSPDATAM", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "Misc Snoop Responses Received : MtoI RspIFwdM", + "EventCode": "0x6b", + "EventName": "UNC_CHA_SNOOP_RSP_MISC.MTOI_RSPIFWDM", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "Misc Snoop Responses Received : Pull Data Partial - Hit LLC", + "EventCode": "0x6b", + "EventName": "UNC_CHA_SNOOP_RSP_MISC.PULLDATAPTL_HITLLC", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "CHA" + }, + { + "BriefDescription": "Misc Snoop Responses Received : Pull Data Partial - Hit SF", + "EventCode": "0x6b", + "EventName": "UNC_CHA_SNOOP_RSP_MISC.PULLDATAPTL_HITSF", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "CHA" + }, + { + "BriefDescription": "Misc Snoop Responses Received : RspIFwdPtl Hit LLC", + "EventCode": "0x6b", + "EventName": "UNC_CHA_SNOOP_RSP_MISC.RSPIFWDMPTL_HITLLC", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "CHA" + }, + { + "BriefDescription": "Misc Snoop Responses Received : RspIFwdPtl Hit SF", + "EventCode": "0x6b", + "EventName": "UNC_CHA_SNOOP_RSP_MISC.RSPIFWDMPTL_HITSF", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : All", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.ALL", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : All : Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent.", + "UMask": "0xc001ffff", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : DDR Access", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.DDR", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : DDR Access : Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent.", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : SF/LLC Evictions", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.EVICT", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : SF/LLC Evictions : Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. : TOR allocation occurred as a result of SF/LLC evictions (came from the ISMQ)", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : Just Hits", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.HIT", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : Just Hits : Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent.", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; All from Local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent.; All locally initiated requests from IA Cores", + "UMask": "0xc001ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts;CLFlush from Local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_CLFLUSH", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent.; CLFlush events that are initiated from the Core", + "UMask": "0xc8c7ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts;CLFlushOpt from Local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_CLFLUSHOPT", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent.; CLFlushOpt events that are initiated from the Core", + "UMask": "0xc8d7ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; CRd from local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_CRD", + "PerPkg": "1", + "PublicDescription": "TOR Inserts; Code read from local IA that misses in the snoop filter", + "UMask": "0xc80fff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; CRd Pref from local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_CRD_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Inserts; Code read prefetch from local IA that misses in the snoop filter", + "UMask": "0xc88fff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; DRd from local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_DRD", + "PerPkg": "1", + "PublicDescription": "TOR Inserts; Data read from local IA that misses in the snoop filter", + "UMask": "0xc817ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : DRd PTEs issued by iA Cores", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_DRDPTE", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : DRd PTEs issued by iA Cores due to a page walk : Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc837ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; DRd Opt from local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_DRD_OPT", + "PerPkg": "1", + "PublicDescription": "TOR Inserts; Data read opt from local IA that misses in the snoop filter", + "UMask": "0xc827ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; DRd Opt Pref from local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_DRD_OPT_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Inserts; Data read opt prefetch from local IA that misses in the snoop filter", + "UMask": "0xc8a7ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; DRd Pref from local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_DRD_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Inserts; Data read prefetch from local IA that misses in the snoop filter", + "UMask": "0xc897ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; Hits from Local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_HIT", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent.", + "UMask": "0xc001fd01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; CRd hits from local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_HIT_CRD", + "PerPkg": "1", + "PublicDescription": "TOR Inserts; Code read from local IA that hits in the snoop filter", + "UMask": "0xc80ffd01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; CRd Pref hits from local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_HIT_CRD_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Inserts; Code read prefetch from local IA that hits in the snoop filter", + "UMask": "0xc88ffd01", + "Unit": "CHA" + }, + { + "BriefDescription": "All requests issued from IA cores to CXL accelerator memory regions that hit the LLC.", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_HIT_CXL_ACC", + "PerPkg": "1", + "UMask": "0x10c0018101", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_TOR_INSERTS.IA_HIT_CXL_ACC_LOCAL", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_HIT_CXL_ACC_LOCAL", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x10c0008101", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; DRd hits from local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_HIT_DRD", + "PerPkg": "1", + "PublicDescription": "TOR Inserts; Data read from local IA that hits in the snoop filter", + "UMask": "0xc817fd01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : DRd PTEs issued by iA Cores that Hit the LLC", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_HIT_DRDPTE", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : DRd PTEs issued by iA Cores due to page walks that hit the LLC : Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc837fd01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; DRd Opt hits from local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_HIT_DRD_OPT", + "PerPkg": "1", + "PublicDescription": "TOR Inserts; Data read opt from local IA that hits in the snoop filter", + "UMask": "0xc827fd01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; DRd Opt Pref hits from local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_HIT_DRD_OPT_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Inserts; Data read opt prefetch from local IA that hits in the snoop filter", + "UMask": "0xc8a7fd01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; DRd Pref hits from local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_HIT_DRD_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Inserts; Data read prefetch from local IA that hits in the snoop filter", + "UMask": "0xc897fd01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : ItoMs issued by iA Cores that Hit LLC", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_HIT_ITOM", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xcc47fd01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; LLCPrefCode hits from local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_HIT_LLCPREFCODE", + "PerPkg": "1", + "PublicDescription": "TOR Inserts; Last level cache prefetch code read from local IA that hits in the snoop filter", + "UMask": "0xcccffd01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; LLCPrefData hits from local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_HIT_LLCPREFDATA", + "PerPkg": "1", + "PublicDescription": "TOR Inserts; Last level cache prefetch data read from local IA that hits in the snoop filter", + "UMask": "0xccd7fd01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; LLCPrefRFO hits from local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_HIT_LLCPREFRFO", + "PerPkg": "1", + "PublicDescription": "TOR Inserts; Last level cache prefetch read for ownership from local IA that hits in the snoop filter", + "UMask": "0xccc7fd01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; RFO hits from local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_HIT_RFO", + "PerPkg": "1", + "PublicDescription": "TOR Inserts; Read for ownership from local IA that hits in the snoop filter", + "UMask": "0xc807fd01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; RFO Pref hits from local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_HIT_RFO_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Inserts; Read for ownership prefetch from local IA that hits in the snoop filter", + "UMask": "0xc887fd01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts;ItoM from Local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_ITOM", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent.; ItoM events that are initiated from the Core", + "UMask": "0xcc47ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : ItoMCacheNears issued by iA Cores", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_ITOMCACHENEAR", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xcd47ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; LLCPrefCode from local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_LLCPREFCODE", + "PerPkg": "1", + "PublicDescription": "TOR Inserts; Last level cache prefetch code read from local IA.", + "UMask": "0xcccfff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; LLCPrefData from local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_LLCPREFDATA", + "PerPkg": "1", + "PublicDescription": "TOR Inserts; Last level cache prefetch data read from local IA.", + "UMask": "0xccd7ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; LLCPrefRFO from local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_LLCPREFRFO", + "PerPkg": "1", + "PublicDescription": "TOR Inserts; Last level cache prefetch read for ownership from local IA that misses in the snoop filter", + "UMask": "0xccc7ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; misses from Local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : All requests from iA Cores that Missed the LLC : Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc001fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts for CRd misses from local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_CRD", + "PerPkg": "1", + "PublicDescription": "Inserts into the TOR from local IA cores which miss the LLC and snoop filter with the opcode CRd", + "UMask": "0xc80ffe01", + "Unit": "CHA" + }, + { + "BriefDescription": "CRds and equivalent opcodes issued from an IA core which miss the L3 and target memory in a CXL type 2 accelerator.", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_CRDMORPH_CXL_ACC", + "PerPkg": "1", + "UMask": "0x10c80b8201", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : CRd issued by iA Cores that Missed the LLC - HOMed locally", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_CRD_LOCAL", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc80efe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; CRd Pref misses from local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_CRD_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Inserts; Code read prefetch from local IA that misses in the snoop filter", + "UMask": "0xc88ffe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : CRd_Prefs issued by iA Cores that Missed the LLC - HOMed locally", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_CRD_PREF_LOCAL", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc88efe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : CRd_Prefs issued by iA Cores that Missed the LLC - HOMed remotely", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_CRD_PREF_REMOTE", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc88f7e01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : CRd issued by iA Cores that Missed the LLC - HOMed remotely", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_CRD_REMOTE", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc80f7e01", + "Unit": "CHA" + }, + { + "BriefDescription": "All requests issued from IA cores to CXL accelerator memory regions that miss the LLC.", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_CXL_ACC", + "PerPkg": "1", + "UMask": "0x10c0018201", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_TOR_INSERTS.IA_MISS_CXL_ACC_LOCAL", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_CXL_ACC_LOCAL", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x10c0008201", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts for DRd misses from local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD", + "PerPkg": "1", + "PublicDescription": "Inserts into the TOR from local IA cores which miss the LLC and snoop filter with the opcode DRd", + "UMask": "0xc817fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "DRds and equivalent opcodes issued from an IA core which miss the L3 and target memory in a CXL type 2 accelerator.", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_DRDMORPH_CXL_ACC", + "PerPkg": "1", + "UMask": "0x10c8138201", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : DRd PTEs issued by iA Cores that Missed the LLC", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_DRDPTE", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : DRd PTEs issued by iA Cores due to a page walk that missed the LLC : Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc837fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "DRds issued from an IA core which miss the L3 and target memory in a CXL type 2 memory expander card.", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD_CXL_ACC", + "PerPkg": "1", + "UMask": "0x10c8178201", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD_CXL_ACC_LOCAL", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD_CXL_ACC_LOCAL", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x10c8168201", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts for DRds issued by IA Cores targeting DDR Mem that Missed the LLC", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD_DDR", + "PerPkg": "1", + "PublicDescription": "Inserts into the TOR from local IA cores which miss the LLC and snoop filter with the opcode DRd, and which target DDR memory", + "UMask": "0xc8178601", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts for DRd misses from local IA targeting local memory", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD_LOCAL", + "PerPkg": "1", + "PublicDescription": "Inserts into the TOR from local IA cores which miss the LLC and snoop filter with the opcode DRd, and which target local memory", + "UMask": "0xc816fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : DRds issued by iA Cores targeting DDR Mem that Missed the LLC - HOMed locally", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD_LOCAL_DDR", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8168601", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : DRds issued by iA Cores targeting PMM Mem that Missed the LLC - HOMed locally", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD_LOCAL_PMM", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8168a01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; DRd Opt misses from local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD_OPT", + "PerPkg": "1", + "PublicDescription": "TOR Inserts; Data read opt from local IA that misses in the snoop filter", + "UMask": "0xc827fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD_OPT_CXL_ACC_LOCAL", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD_OPT_CXL_ACC_LOCAL", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x10c8268201", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; DRd Opt Pref misses from local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD_OPT_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Inserts; Data read opt prefetch from local IA that misses in the snoop filter", + "UMask": "0xc8a7fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD_OPT_PREF_CXL_ACC_LOCAL", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD_OPT_PREF_CXL_ACC_LOCAL", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x10c8a68201", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts for DRds issued by iA Cores targeting PMM Mem that Missed the LLC", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PMM", + "PerPkg": "1", + "PublicDescription": "Inserts into the TOR from local IA cores which miss the LLC and snoop filter with the opcode DRd, and which target PMM memory", + "UMask": "0xc8178a01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts for DRd Pref misses from local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF", + "PerPkg": "1", + "PublicDescription": "Inserts into the TOR from local IA cores which miss the LLC and snoop filter with the opcode DRD_PREF", + "UMask": "0xc897fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "L2 data prefetches issued from an IA core which miss the L3 and target memory in a CXL type 2 accelerator.", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF_CXL_ACC", + "PerPkg": "1", + "UMask": "0x10c8978201", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF_CXL_ACC_LOCAL", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF_CXL_ACC_LOCAL", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x10c8968201", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : DRd_Prefs issued by iA Cores targeting DDR Mem that Missed the LLC", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF_DDR", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8978601", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts for DRd Pref misses from local IA targeting local memory", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF_LOCAL", + "PerPkg": "1", + "PublicDescription": "Inserts into the TOR from local IA cores which miss the LLC and snoop filter with the opcode DRD_PREF, and target local memory", + "UMask": "0xc896fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : DRd_Prefs issued by iA Cores targeting DDR Mem that Missed the LLC - HOMed locally", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF_LOCAL_DDR", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8968601", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : DRd_Prefs issued by iA Cores targeting PMM Mem that Missed the LLC - HOMed locally", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF_LOCAL_PMM", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8968a01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : DRd_Prefs issued by iA Cores targeting PMM Mem that Missed the LLC", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF_PMM", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8978a01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts for DRd Pref misses from local IA targeting remote memory", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF_REMOTE", + "PerPkg": "1", + "PublicDescription": "Inserts into the TOR from local IA cores which miss the LLC and snoop filter with the opcode DRD_PREF, and target remote memory", + "UMask": "0xc8977e01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : DRd_Prefs issued by iA Cores targeting DDR Mem that Missed the LLC - HOMed remotely", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF_REMOTE_DDR", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8970601", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : DRd_Prefs issued by iA Cores targeting PMM Mem that Missed the LLC - HOMed remotely", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF_REMOTE_PMM", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8970a01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts for DRd misses from local IA targeting remote memory", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD_REMOTE", + "PerPkg": "1", + "PublicDescription": "Inserts into the TOR from local IA cores which miss the LLC and snoop filter with the opcode DRd, and target remote memory", + "UMask": "0xc8177e01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : DRds issued by iA Cores targeting DDR Mem that Missed the LLC - HOMed remotely", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD_REMOTE_DDR", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8170601", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : DRds issued by iA Cores targeting PMM Mem that Missed the LLC - HOMed remotely", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD_REMOTE_PMM", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8170a01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : ItoMs issued by iA Cores that Missed LLC", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_ITOM", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xcc47fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; LLCPrefCode misses from local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_LLCPREFCODE", + "PerPkg": "1", + "PublicDescription": "TOR Inserts; Last level cache prefetch code read from local IA that misses in the snoop filter", + "UMask": "0xcccffe01", + "Unit": "CHA" + }, + { + "BriefDescription": "LLC Prefetch Code transactions issued from an IA core which miss the L3 and target memory in a CXL type 2 accelerator.", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_LLCPREFCODE_CXL_ACC", + "PerPkg": "1", + "UMask": "0x10cccf8201", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; LLCPrefData misses from local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_LLCPREFDATA", + "PerPkg": "1", + "PublicDescription": "TOR Inserts; Last level cache prefetch data read from local IA that misses in the snoop filter", + "UMask": "0xccd7fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "LLC data prefetches issued from an IA core which miss the L3 and target memory in a CXL type 2 accelerator.", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_LLCPREFDATA_CXL_ACC", + "PerPkg": "1", + "UMask": "0x10ccd78201", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_TOR_INSERTS.IA_MISS_LLCPREFDATA_CXL_ACC_LOCAL", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_LLCPREFDATA_CXL_ACC_LOCAL", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x10ccd68201", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; LLCPrefRFO misses from local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_LLCPREFRFO", + "PerPkg": "1", + "PublicDescription": "TOR Inserts; Last level cache prefetch read for ownership from local IA that misses in the snoop filter", + "UMask": "0xccc7fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "L2 RFO prefetches issued from an IA core which miss the L3 and target memory in a CXL type 2 accelerator.", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_LLCPREFRFO_CXL_ACC", + "PerPkg": "1", + "UMask": "0x10c8878201", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_TOR_INSERTS.IA_MISS_LLCPREFRFO_CXL_ACC_LOCAL", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_LLCPREFRFO_CXL_ACC_LOCAL", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x10c8868201", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : WCiLFs issued by iA Cores targeting DDR that missed the LLC - HOMed locally", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_LOCAL_WCILF_DDR", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8668601", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : WCiLFs issued by iA Cores targeting PMM that missed the LLC - HOMed locally", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_LOCAL_WCILF_PMM", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8668a01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : WCiLs issued by iA Cores targeting DDR that missed the LLC - HOMed locally", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_LOCAL_WCIL_DDR", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc86e8601", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : WCiLs issued by iA Cores targeting PMM that missed the LLC - HOMed locally", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_LOCAL_WCIL_PMM", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc86e8a01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : WCiLFs issued by iA Cores targeting DDR that missed the LLC - HOMed remotely", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_REMOTE_WCILF_DDR", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8670601", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : WCiLFs issued by iA Cores targeting PMM that missed the LLC - HOMed remotely", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_REMOTE_WCILF_PMM", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8670a01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : WCiLs issued by iA Cores targeting DDR that missed the LLC - HOMed remotely", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_REMOTE_WCIL_DDR", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc86f0601", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : WCiLs issued by iA Cores targeting PMM that missed the LLC - HOMed remotely", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_REMOTE_WCIL_PMM", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc86f0a01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; RFO misses from local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_RFO", + "PerPkg": "1", + "PublicDescription": "TOR Inserts; Read for ownership from local IA that misses in the snoop filter", + "UMask": "0xc807fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "RFO and L2 RFO prefetches issued from an IA core which miss the L3 and target memory in a CXL type 2 accelerator.", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_RFOMORPH_CXL_ACC", + "PerPkg": "1", + "UMask": "0x10c8038201", + "Unit": "CHA" + }, + { + "BriefDescription": "RFOs issued from an IA core which miss the L3 and target memory in a CXL type 2 accelerator.", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_RFO_CXL_ACC", + "PerPkg": "1", + "UMask": "0x10c8078201", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_TOR_INSERTS.IA_MISS_RFO_CXL_ACC_LOCAL", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_RFO_CXL_ACC_LOCAL", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x10c8068201", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts RFO misses from local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_RFO_LOCAL", + "PerPkg": "1", + "PublicDescription": "TOR Inserts; Read for ownership from local IA that misses in the snoop filter", + "UMask": "0xc806fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; RFO pref misses from local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_RFO_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Inserts; Read for ownership prefetch from local IA that misses in the snoop filter", + "UMask": "0xc887fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "LLC RFO prefetches issued from an IA core which miss the L3 and target memory in a CXL type 2 accelerator.", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_RFO_PREF_CXL_ACC", + "PerPkg": "1", + "UMask": "0x10ccc78201", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_TOR_INSERTS.IA_MISS_RFO_PREF_CXL_ACC_LOCAL", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_RFO_PREF_CXL_ACC_LOCAL", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x10ccc68201", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; RFO prefetch misses from local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_RFO_PREF_LOCAL", + "PerPkg": "1", + "PublicDescription": "TOR Inserts; Read for ownership prefetch from local IA that misses in the snoop filter", + "UMask": "0xc886fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; RFO prefetch misses from local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_RFO_PREF_REMOTE", + "PerPkg": "1", + "PublicDescription": "TOR Inserts; Read for ownership prefetch from local IA that misses in the snoop filter", + "UMask": "0xc8877e01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; RFO misses from local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_RFO_REMOTE", + "PerPkg": "1", + "PublicDescription": "TOR Inserts Read for ownership from local IA that misses in the snoop filter", + "UMask": "0xc8077e01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : UCRdFs issued by iA Cores that Missed LLC", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_UCRDF", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc877de01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : WCiLs issued by iA Cores that Missed the LLC", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_WCIL", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc86ffe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : WCiLF issued by iA Cores that Missed the LLC", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_WCILF", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc867fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : WCiLFs issued by iA Cores targeting DDR that missed the LLC", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_WCILF_DDR", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8678601", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : WCiLFs issued by iA Cores targeting PMM that missed the LLC", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_WCILF_PMM", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8678a01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : WCiLs issued by iA Cores targeting DDR that missed the LLC", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_WCIL_DDR", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc86f8601", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : WCiLs issued by iA Cores targeting PMM that missed the LLC", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_WCIL_PMM", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc86f8a01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : WiLs issued by iA Cores that Missed LLC", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_WIL", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc87fde01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; RFO from local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_RFO", + "PerPkg": "1", + "PublicDescription": "TOR Inserts; Read for ownership from local IA that misses in the snoop filter", + "UMask": "0xc807ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; RFO pref from local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_RFO_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Inserts; Read for ownership prefetch from local IA that misses in the snoop filter", + "UMask": "0xc887ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts;SpecItoM from Local IA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_SPECITOM", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent.; SpecItoM events that are initiated from the Core", + "UMask": "0xcc57ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : WBEFtoEs issued by an IA Core. Non Modified Write Backs", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_WBEFTOE", + "PerPkg": "1", + "PublicDescription": "WbEFtoEs issued by iA Cores . (Non Modified Write Backs) :Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xcc3fff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : WBEFtoEs issued by an IA Core. Non Modified Write Backs", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_WBEFTOI", + "PerPkg": "1", + "PublicDescription": "WbEFtoEs issued by iA Cores . (Non Modified Write Backs) :Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xcc37ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : WBEFtoEs issued by an IA Core. Non Modified Write Backs", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_WBMTOE", + "PerPkg": "1", + "PublicDescription": "WbEFtoEs issued by iA Cores . (Non Modified Write Backs) :Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xcc2fff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : WbMtoIs issued by an iA Cores. Modified Write Backs", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_WBMTOI", + "PerPkg": "1", + "PublicDescription": "WbMtoIs issued by iA Cores . (Modified Write Backs) :Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xcc27ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : WBEFtoEs issued by an IA Core. Non Modified Write Backs", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_WBSTOI", + "PerPkg": "1", + "PublicDescription": "WbEFtoEs issued by iA Cores . (Non Modified Write Backs) :Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xcc67ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : WCiLs issued by iA Cores", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_WCIL", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc86fff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : WCiLF issued by iA Cores", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_WCILF", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc867ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; All from local IO", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc001ff04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : CLFlushes issued by IO Devices", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO_CLFLUSH", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8c3ff04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; Hits from local IO", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO_HIT", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc001fd04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; ItoM hits from local IO", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO_HIT_ITOM", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xcc43fd04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : ItoMCacheNears, indicating a partial write request, from IO Devices that hit the LLC", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO_HIT_ITOMCACHENEAR", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : ItoMCacheNears, indicating a partial write request, from IO Devices that hit the LLC : Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xcd43fd04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; RdCur and FsRdCur hits from local IO", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO_HIT_PCIRDCUR", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : PCIRdCurs issued by IO Devices that hit the LLC : Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8f3fd04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; RFO hits from local IO", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO_HIT_RFO", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc803fd04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts for ItoM from local IO", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO_ITOM", + "PerPkg": "1", + "PublicDescription": "Inserts into the TOR from local IO with the opcode ItoM", + "UMask": "0xcc43ff04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts for ItoMCacheNears from IO devices.", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO_ITOMCACHENEAR", + "PerPkg": "1", + "PublicDescription": "Inserts into the TOR from local IO devices with the opcode ItoMCacheNears. This event indicates a partial write request.", + "UMask": "0xcd43ff04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; Misses from local IO", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO_MISS", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc001fe04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; ItoM misses from local IO", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO_MISS_ITOM", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xcc43fe04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : ItoMCacheNears, indicating a partial write request, from IO Devices that missed the LLC", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO_MISS_ITOMCACHENEAR", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : ItoMCacheNears, indicating a partial write request, from IO Devices that missed the LLC : Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xcd43fe04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; RdCur and FsRdCur misses from local IO", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO_MISS_PCIRDCUR", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : PCIRdCurs issued by IO Devices that missed the LLC : Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8f3fe04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; RFO misses from local IO", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO_MISS_RFO", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : RFOs issued by IO Devices that missed the LLC : Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc803fe04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts for RdCur from local IO", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO_PCIRDCUR", + "PerPkg": "1", + "PublicDescription": "Inserts into the TOR from local IO with the opcode RdCur", + "UMask": "0xc8f3ff04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; RFO from local IO", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO_RFO", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : RFOs issued by IO Devices : Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc803ff04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : WbMtoIs issued by IO Devices", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IO_WBMTOI", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xcc23ff04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : IPQ", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IPQ", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : IPQ : Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent.", + "UMask": "0x8", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : IRQ - iA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IRQ_IA", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : IRQ - iA : Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. : From an iA Core", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : IRQ - Non iA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IRQ_NON_IA", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : IRQ - Non iA : Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent.", + "UMask": "0x10", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : Just ISOC", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.ISOC", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : Just ISOC : Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent.", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : Just Local Targets", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.LOCAL_TGT", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : Just Local Targets : Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent.", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : All from Local iA and IO", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.LOC_ALL", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : All from Local iA and IO : Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. : All locally initiated requests", + "UMask": "0xc000ff05", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : All from Local iA", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.LOC_IA", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : All from Local iA : Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. : All locally initiated requests from iA Cores", + "UMask": "0xc000ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : All from Local IO", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.LOC_IO", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : All from Local IO : Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. : All locally generated IO traffic", + "UMask": "0xc000ff04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : Match the Opcode in b[29:19] of the extended umask field", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.MATCH_OPC", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : Match the Opcode in b[29:19] of the extended umask field : Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent.", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : Just Misses", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.MISS", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : Just Misses : Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent.", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : MMCFG Access", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.MMCFG", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : MMCFG Access : Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent.", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : MMIO Access", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.MMIO", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : MMIO Access : Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent.", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : Just NearMem", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.NEARMEM", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : Just NearMem : Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent.", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : Just NonCoherent", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.NONCOH", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : Just NonCoherent : Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent.", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : Just NotNearMem", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.NOT_NEARMEM", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : Just NotNearMem : Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent.", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : PMM Access", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.PMM", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : PM Access : Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent.", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : Match the PreMorphed Opcode in b[29:19] of the extended umask field", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.PREMORPH_OPC", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : Match the PreMorphed Opcode in b[29:19] of the extended umask field : Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent.", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : PRQ - IOSF", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.PRQ_IOSF", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : PRQ - IOSF : Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. : From a PCIe Device", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : PRQ - Non IOSF", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.PRQ_NON_IOSF", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : PRQ - Non IOSF : Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent.", + "UMask": "0x20", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : Just Remote Targets", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.REMOTE_TGT", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : Just Remote Targets : Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent.", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : All from Remote", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.REM_ALL", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : All from Remote : Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. : All remote requests (e.g. snoops, writebacks) that came from remote sockets", + "UMask": "0xc001ffc8", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : All Snoops from Remote", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.REM_SNPS", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : All Snoops from Remote : Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. : All snoops to this LLC that came from remote sockets", + "UMask": "0xc001ff08", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : RRQ", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.RRQ", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : RRQ : Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent.", + "UMask": "0x40", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts; All Snoops from Remote", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.SNPS_FROM_REM", + "PerPkg": "1", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. All snoops to this LLC that came from remote sockets.", + "UMask": "0xc001ff08", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Inserts : WBQ", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.WBQ", + "PerPkg": "1", + "PublicDescription": "TOR Inserts : WBQ : Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent.", + "UMask": "0x80", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : All", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.ALL", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : All : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. T", + "UMask": "0xc001ffff", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : DDR Access", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.DDR", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : DDR Access : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent.", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : SF/LLC Evictions", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.EVICT", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : SF/LLC Evictions : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. T : TOR allocation occurred as a result of SF/LLC evictions (came from the ISMQ)", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : Just Hits", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.HIT", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : Just Hits : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. T", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; All from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : All requests from iA Cores : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc001ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : CLFlushes issued by iA Cores", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_CLFLUSH", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : CLFlushes issued by iA Cores : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8c7ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : CLFlushOpts issued by iA Cores", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_CLFLUSHOPT", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : CLFlushOpts issued by iA Cores : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8d7ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; CRd from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_CRD", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy; Code read from local IA that misses in the snoop filter", + "UMask": "0xc80fff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; CRd Pref from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_CRD_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy; Code read prefetch from local IA that misses in the snoop filter", + "UMask": "0xc88fff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; DRd from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_DRD", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy; Data read from local IA that misses in the snoop filter", + "UMask": "0xc817ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : DRdPte issued by iA Cores due to a page walk", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_DRDPTE", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "PublicDescription": "TOR Occupancy : DRdPte issued by iA Cores due to a page walk : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc837ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; DRd Opt from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_DRD_OPT", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy; Data read opt from local IA that misses in the snoop filter", + "UMask": "0xc827ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; DRd Opt Pref from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_DRD_OPT_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy; Data read opt prefetch from local IA that misses in the snoop filter", + "UMask": "0xc8a7ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; DRd Pref from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_DRD_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy; Data read prefetch from local IA that misses in the snoop filter", + "UMask": "0xc897ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; Hits from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_HIT", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : All requests from iA Cores that Hit the LLC : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc001fd01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; CRd hits from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_HIT_CRD", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy; Code read from local IA that hits in the snoop filter", + "UMask": "0xc80ffd01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; CRd Pref hits from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_HIT_CRD_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy; Code read prefetch from local IA that hits in the snoop filter", + "UMask": "0xc88ffd01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for All requests issued from IA cores to CXL accelerator memory regions that hit the LLC.", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_HIT_CXL_ACC", + "PerPkg": "1", + "UMask": "0x10c0018101", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_TOR_OCCUPANCY.IA_HIT_CXL_ACC_LOCAL", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_HIT_CXL_ACC_LOCAL", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x10c0008101", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; DRd hits from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_HIT_DRD", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy; Data read from local IA that hits in the snoop filter", + "UMask": "0xc817fd01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : DRdPte issued by iA Cores due to a page walk that hit the LLC", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_HIT_DRDPTE", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "PublicDescription": "TOR Occupancy : DRdPte issued by iA Cores due to a page walk that hit the LLC : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc837fd01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; DRd Opt hits from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_HIT_DRD_OPT", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy; Data read opt from local IA that hits in the snoop filter", + "UMask": "0xc827fd01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; DRd Opt Pref hits from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_HIT_DRD_OPT_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy; Data read opt prefetch from local IA that hits in the snoop filter", + "UMask": "0xc8a7fd01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; DRd Pref hits from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_HIT_DRD_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy; Data read prefetch from local IA that hits in the snoop filter", + "UMask": "0xc897fd01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : ItoMs issued by iA Cores that Hit LLC", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_HIT_ITOM", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : ItoMs issued by iA Cores that Hit LLC : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xcc47fd01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; LLCPrefCode hits from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_HIT_LLCPREFCODE", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy; Last level cache prefetch code read from local IA that hits in the snoop filter", + "UMask": "0xcccffd01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; LLCPrefData hits from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_HIT_LLCPREFDATA", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy; Last level cache prefetch data read from local IA that hits in the snoop filter", + "UMask": "0xccd7fd01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; LLCPrefRFO hits from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_HIT_LLCPREFRFO", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy; Last level cache prefetch read for ownership from local IA that hits in the snoop filter", + "UMask": "0xccc7fd01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; RFO hits from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_HIT_RFO", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy; Read for ownership from local IA that hits in the snoop filter", + "UMask": "0xc807fd01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; RFO Pref hits from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_HIT_RFO_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy; Read for ownership prefetch from local IA that hits in the snoop filter", + "UMask": "0xc887fd01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : ItoMs issued by iA Cores", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_ITOM", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : ItoMs issued by iA Cores : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xcc47ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : ItoMCacheNears issued by iA Cores", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_ITOMCACHENEAR", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : ItoMCacheNears issued by iA Cores : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xcd47ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; LLCPrefCode from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_LLCPREFCODE", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy; Last level cache prefetch data read from local IA.", + "UMask": "0xcccfff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; LLCPrefData from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_LLCPREFDATA", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy; Last level cache prefetch data read from local IA that misses in the snoop filter", + "UMask": "0xccd7ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; LLCPrefRFO from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_LLCPREFRFO", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy; Last level cache prefetch read for ownership from local IA that misses in the snoop filter", + "UMask": "0xccc7ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; Misses from Local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : All requests from iA Cores that Missed the LLC : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc001fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; CRd misses from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_CRD", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy; Code read from local IA that misses in the snoop filter", + "UMask": "0xc80ffe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for CRds and equivalent opcodes issued from an IA core which miss the L3 and target memory in a CXL type 2 accelerator.", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_CRDMORPH_CXL_ACC", + "PerPkg": "1", + "UMask": "0x10c80b8201", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : CRd issued by iA Cores that Missed the LLC - HOMed locally", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_CRD_LOCAL", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : CRd issued by iA Cores that Missed the LLC - HOMed locally : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc80efe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; CRd Pref misses from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_CRD_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy; Code read prefetch from local IA that misses in the snoop filter", + "UMask": "0xc88ffe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : CRd_Prefs issued by iA Cores that Missed the LLC - HOMed locally", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_CRD_PREF_LOCAL", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : CRd_Prefs issued by iA Cores that Missed the LLC - HOMed locally : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc88efe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : CRd_Prefs issued by iA Cores that Missed the LLC - HOMed remotely", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_CRD_PREF_REMOTE", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : CRd_Prefs issued by iA Cores that Missed the LLC - HOMed remotely : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc88f7e01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : CRd issued by iA Cores that Missed the LLC - HOMed remotely", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_CRD_REMOTE", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : CRd issued by iA Cores that Missed the LLC - HOMed remotely : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc80f7e01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for All requests issued from IA cores to CXL accelerator memory regions that miss the LLC.", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_CXL_ACC", + "PerPkg": "1", + "UMask": "0x10c0018201", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_CXL_ACC_LOCAL", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_CXL_ACC_LOCAL", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x10c0008201", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for DRd misses from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD", + "PerPkg": "1", + "PublicDescription": "Number of cycles for elements in the TOR from local IA cores which miss the LLC and snoop filter with the opcode DRd", + "UMask": "0xc817fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for DRds and equivalent opcodes issued from an IA core which miss the L3 and target memory in a CXL type 2 accelerator.", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRDMORPH_CXL_ACC", + "PerPkg": "1", + "UMask": "0x10c8138201", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : DRdPte issued by iA Cores due to a page walk that missed the LLC", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRDPTE", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "PublicDescription": "TOR Occupancy : DRdPte issued by iA Cores due to a page walk that missed the LLC : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc837fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for DRds and equivalent opcodes issued from an IA core which miss the L3 and target memory in a CXL type 2 memory expander card.", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_CXL_ACC", + "PerPkg": "1", + "UMask": "0x10c8178201", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_CXL_ACC_LOCAL", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_CXL_ACC_LOCAL", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x10c8168201", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for DRds issued by iA Cores targeting DDR Mem that Missed the LLC", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_DDR", + "PerPkg": "1", + "PublicDescription": "Number of cycles for elements in the TOR from local IA cores which miss the LLC and snoop filter with the opcode DRd, and which target DDR memory", + "UMask": "0xc8178601", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for DRd misses from local IA targeting local memory", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_LOCAL", + "PerPkg": "1", + "PublicDescription": "Number of cycles for elements in the TOR from local IA cores which miss the LLC and snoop filter with the opcode DRd, and which target local memory", + "UMask": "0xc816fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : DRds issued by iA Cores targeting DDR Mem that Missed the LLC - HOMed locally", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_LOCAL_DDR", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : DRds issued by iA Cores targeting DDR Mem that Missed the LLC - HOMed locally : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8168601", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : DRds issued by iA Cores targeting PMM Mem that Missed the LLC - HOMed locally", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_LOCAL_PMM", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : DRds issued by iA Cores targeting PMM Mem that Missed the LLC - HOMed locally : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8168a01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; DRd Opt misses from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_OPT", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy; Data read opt from local IA that misses in the snoop filter", + "UMask": "0xc827fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_OPT_CXL_ACC_LOCAL", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_OPT_CXL_ACC_LOCAL", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x10c8268201", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; DRd Opt Pref misses from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_OPT_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy; Data read opt prefetch from local IA that misses in the snoop filter", + "UMask": "0xc8a7fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_OPT_PREF_CXL_ACC_LOCAL", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_OPT_PREF_CXL_ACC_LOCAL", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x10c8a68201", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for DRds issued by iA Cores targeting PMM Mem that Missed the LLC", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PMM", + "PerPkg": "1", + "PublicDescription": "Number of cycles for elements in the TOR from local IA cores which miss the LLC and snoop filter with the opcode DRd, and which target PMM memory", + "UMask": "0xc8178a01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; DRd Pref misses from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy; Data read prefetch from local IA that misses in the snoop filter", + "UMask": "0xc897fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for L2 data prefetches issued from an IA core which miss the L3 and target memory in a CXL type 2 accelerator.", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PREF_CXL_ACC", + "PerPkg": "1", + "UMask": "0x10c8978201", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PREF_CXL_ACC_LOCAL", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PREF_CXL_ACC_LOCAL", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x10c8968201", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : DRd_Prefs issued by iA Cores targeting DDR Mem that Missed the LLC", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PREF_DDR", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : DRd_Prefs issued by iA Cores targeting DDR Mem that Missed the LLC : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8978601", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; DRd Pref misses from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PREF_LOCAL", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy; Data read prefetch from local IA that misses in the snoop filter", + "UMask": "0xc896fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : DRd_Prefs issued by iA Cores targeting DDR Mem that Missed the LLC - HOMed locally", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PREF_LOCAL_DDR", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : DRd_Prefs issued by iA Cores targeting DDR Mem that Missed the LLC - HOMed locally : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8968601", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : DRd_Prefs issued by iA Cores targeting PMM Mem that Missed the LLC - HOMed locally", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PREF_LOCAL_PMM", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : DRd_Prefs issued by iA Cores targeting PMM Mem that Missed the LLC - HOMed locally : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8968a01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : DRd_Prefs issued by iA Cores targeting PMM Mem that Missed the LLC", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PREF_PMM", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : DRd_Prefs issued by iA Cores targeting PMM Mem that Missed the LLC : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8978a01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; DRd Pref misses from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PREF_REMOTE", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy; Data read prefetch from local IA that misses in the snoop filter", + "UMask": "0xc8977e01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : DRd_Prefs issued by iA Cores targeting DDR Mem that Missed the LLC - HOMed remotely", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PREF_REMOTE_DDR", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : DRd_Prefs issued by iA Cores targeting DDR Mem that Missed the LLC - HOMed remotely : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8970601", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : DRd_Prefs issued by iA Cores targeting PMM Mem that Missed the LLC - HOMed remotely", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PREF_REMOTE_PMM", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : DRd_Prefs issued by iA Cores targeting PMM Mem that Missed the LLC - HOMed remotely : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8970a01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for DRd misses from local IA targeting remote memory", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_REMOTE", + "PerPkg": "1", + "PublicDescription": "Number of cycles for elements in the TOR from local IA cores which miss the LLC and snoop filter with the opcode DRd, and which target remote memory", + "UMask": "0xc8177e01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : DRds issued by iA Cores targeting DDR Mem that Missed the LLC - HOMed remotely", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_REMOTE_DDR", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : DRds issued by iA Cores targeting DDR Mem that Missed the LLC - HOMed remotely : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8170601", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : DRds issued by iA Cores targeting PMM Mem that Missed the LLC - HOMed remotely", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_REMOTE_PMM", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : DRds issued by iA Cores targeting PMM Mem that Missed the LLC - HOMed remotely : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8170a01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : ItoMs issued by iA Cores that Missed LLC", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_ITOM", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : ItoMs issued by iA Cores that Missed LLC : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xcc47fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; LLCPrefCode misses from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_LLCPREFCODE", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy; Last level cache prefetch code read from local IA that misses in the snoop filter", + "UMask": "0xcccffe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for LLC Prefetch Code transactions issued from an IA core which miss the L3 and target memory in a CXL type 2 accelerator.", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_LLCPREFCODE_CXL_ACC", + "PerPkg": "1", + "UMask": "0x10cccf8201", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; LLCPrefData misses from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_LLCPREFDATA", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy; Last level cache prefetch data read from local IA that misses in the snoop filter", + "UMask": "0xccd7fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for LLC data prefetches issued from an IA core which miss the L3 and target memory in a CXL type 2 accelerator.", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_LLCPREFDATA_CXL_ACC", + "PerPkg": "1", + "UMask": "0x10ccd78201", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_LLCPREFDATA_CXL_ACC_LOCAL", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_LLCPREFDATA_CXL_ACC_LOCAL", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x10ccd68201", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; LLCPrefRFO misses from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_LLCPREFRFO", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy; Last level cache prefetch read for ownership from local IA that misses in the snoop filter", + "UMask": "0xccc7fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for L2 RFO prefetches issued from an IA core which miss the L3 and target memory in a CXL type 2 accelerator.", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_LLCPREFRFO_CXL_ACC", + "PerPkg": "1", + "UMask": "0x10c8878201", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_LLCPREFRFO_CXL_ACC_LOCAL", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_LLCPREFRFO_CXL_ACC_LOCAL", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x10c8868201", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : WCiLFs issued by iA Cores targeting DDR that missed the LLC - HOMed locally", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_LOCAL_WCILF_DDR", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : WCiLFs issued by iA Cores targeting DDR that missed the LLC - HOMed locally : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8668601", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : WCiLFs issued by iA Cores targeting PMM that missed the LLC - HOMed locally", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_LOCAL_WCILF_PMM", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : WCiLFs issued by iA Cores targeting PMM that missed the LLC - HOMed locally : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8668a01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : WCiLs issued by iA Cores targeting DDR that missed the LLC - HOMed locally", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_LOCAL_WCIL_DDR", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : WCiLs issued by iA Cores targeting DDR that missed the LLC - HOMed locally : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc86e8601", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : WCiLs issued by iA Cores targeting PMM that missed the LLC - HOMed locally", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_LOCAL_WCIL_PMM", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : WCiLs issued by iA Cores targeting PMM that missed the LLC - HOMed locally : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc86e8a01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : WCiLFs issued by iA Cores targeting DDR that missed the LLC - HOMed remotely", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_REMOTE_WCILF_DDR", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : WCiLFs issued by iA Cores targeting DDR that missed the LLC - HOMed remotely : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8670601", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : WCiLFs issued by iA Cores targeting PMM that missed the LLC - HOMed remotely", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_REMOTE_WCILF_PMM", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : WCiLFs issued by iA Cores targeting PMM that missed the LLC - HOMed remotely : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8670a01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : WCiLs issued by iA Cores targeting DDR that missed the LLC - HOMed remotely", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_REMOTE_WCIL_DDR", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : WCiLs issued by iA Cores targeting DDR that missed the LLC - HOMed remotely : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc86f0601", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : WCiLs issued by iA Cores targeting PMM that missed the LLC - HOMed remotely", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_REMOTE_WCIL_PMM", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : WCiLs issued by iA Cores targeting PMM that missed the LLC - HOMed remotely : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc86f0a01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; RFO misses from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_RFO", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy; Read for ownership from local IA that misses in the snoop filter", + "UMask": "0xc807fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for RFO and L2 RFO prefetches issued from an IA core which miss the L3 and target memory in a CXL type 2 accelerator.", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_RFOMORPH_CXL_ACC", + "PerPkg": "1", + "UMask": "0x10c8038201", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for RFOs issued from an IA core which miss the L3 and target memory in a CXL type 2 accelerator.", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_RFO_CXL_ACC", + "PerPkg": "1", + "UMask": "0x10c8078201", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_RFO_CXL_ACC_LOCAL", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_RFO_CXL_ACC_LOCAL", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x10c8068201", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; RFO misses from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_RFO_LOCAL", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy; Read for ownership from local IA that misses in the snoop filter", + "UMask": "0xc806fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; RFO prefetch misses from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_RFO_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy; Read for ownership prefetch from local IA that misses in the snoop filter", + "UMask": "0xc887fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy for LLC RFO prefetches issued from an IA core which miss the L3 and target memory in a CXL type 2 accelerator.", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_RFO_PREF_CXL_ACC", + "PerPkg": "1", + "UMask": "0x10ccc78201", + "Unit": "CHA" + }, + { + "BriefDescription": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_RFO_PREF_CXL_ACC_LOCAL", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_RFO_PREF_CXL_ACC_LOCAL", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x10ccc68201", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; RFO prefetch misses from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_RFO_PREF_LOCAL", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy; Read for ownership prefetch from local IA that misses in the snoop filter", + "UMask": "0xc886fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; RFO prefetch misses from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_RFO_PREF_REMOTE", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy; Read for ownership prefetch from local IA that misses in the snoop filter", + "UMask": "0xc8877e01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; RFO misses from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_RFO_REMOTE", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy; Read for ownership from local IA that misses in the snoop filter", + "UMask": "0xc8077e01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : UCRdFs issued by iA Cores that Missed LLC", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_UCRDF", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : UCRdFs issued by iA Cores that Missed LLC : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc877de01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : WCiLs issued by iA Cores that Missed the LLC", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_WCIL", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : WCiLs issued by iA Cores that Missed the LLC : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc86ffe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : WCiLF issued by iA Cores that Missed the LLC", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_WCILF", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : WCiLF issued by iA Cores that Missed the LLC : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc867fe01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : WCiLFs issued by iA Cores targeting DDR that missed the LLC", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_WCILF_DDR", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : WCiLFs issued by iA Cores targeting DDR that missed the LLC : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8678601", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : WCiLFs issued by iA Cores targeting PMM that missed the LLC", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_WCILF_PMM", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : WCiLFs issued by iA Cores targeting PMM that missed the LLC : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8678a01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : WCiLs issued by iA Cores targeting DDR that missed the LLC", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_WCIL_DDR", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : WCiLs issued by iA Cores targeting DDR that missed the LLC : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc86f8601", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : WCiLs issued by iA Cores targeting PMM that missed the LLC", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_WCIL_PMM", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : WCiLs issued by iA Cores targeting PMM that missed the LLC : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc86f8a01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : WiLs issued by iA Cores that Missed LLC", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_WIL", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : WiLs issued by iA Cores that Missed LLC : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc87fde01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; RFO from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_RFO", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy; Read for ownership from local IA that misses in the snoop filter", + "UMask": "0xc807ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; RFO prefetch from local IA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_RFO_PREF", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy; Read for ownership prefetch from local IA that misses in the snoop filter", + "UMask": "0xc887ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : SpecItoMs issued by iA Cores", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_SPECITOM", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : SpecItoMs issued by iA Cores : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xcc57ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : WbMtoIs issued by iA Cores", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_WBMTOI", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : WbMtoIs issued by iA Cores : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xcc27ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : WCiLs issued by iA Cores", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_WCIL", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : WCiLs issued by iA Cores : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc86fff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : WCiLF issued by iA Cores", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_WCILF", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : WCiLF issued by iA Cores : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc867ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; All from local IO", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : All requests from IO Devices : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc001ff04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : CLFlushes issued by IO Devices", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO_CLFLUSH", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : CLFlushes issued by IO Devices : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8c3ff04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; Hits from local IO", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO_HIT", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : All requests from IO Devices that hit the LLC : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc001fd04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; ITOM hits from local IO", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO_HIT_ITOM", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : ItoMs issued by IO Devices that Hit the LLC : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xcc43fd04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : ItoMCacheNears, indicating a partial write request, from IO Devices that hit the LLC", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO_HIT_ITOMCACHENEAR", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : ItoMCacheNears, indicating a partial write request, from IO Devices that hit the LLC : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xcd43fd04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; RdCur and FsRdCur hits from local IO", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO_HIT_PCIRDCUR", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : PCIRdCurs issued by IO Devices that hit the LLC : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8f3fd04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; RFO hits from local IO", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO_HIT_RFO", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : RFOs issued by IO Devices that hit the LLC : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc803fd04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; ITOM from local IO", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO_ITOM", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : ItoMs issued by IO Devices : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xcc43ff04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : ItoMCacheNears, indicating a partial write request, from IO Devices", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO_ITOMCACHENEAR", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "PublicDescription": "TOR Occupancy : ItoMCacheNears, indicating a partial write request, from IO Devices : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xcd43ff04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; Misses from local IO", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO_MISS", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : All requests from IO Devices that missed the LLC : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc001fe04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; ITOM misses from local IO", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO_MISS_ITOM", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : ItoMs issued by IO Devices that missed the LLC : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xcc43fe04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : ItoMCacheNears, indicating a partial write request, from IO Devices that missed the LLC", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO_MISS_ITOMCACHENEAR", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : ItoMCacheNears, indicating a partial write request, from IO Devices that missed the LLC : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xcd43fe04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; RdCur and FsRdCur misses from local IO", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO_MISS_PCIRDCUR", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : PCIRdCurs issued by IO Devices that missed the LLC : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8f3fe04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; RFO misses from local IO", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO_MISS_RFO", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : RFOs issued by IO Devices that missed the LLC : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc803fe04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; RdCur and FsRdCur from local IO", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO_PCIRDCUR", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : PCIRdCurs issued by IO Devices : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc8f3ff04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; ItoM from local IO", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO_RFO", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : RFOs issued by IO Devices : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xc803ff04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : WbMtoIs issued by IO Devices", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IO_WBMTOI", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : WbMtoIs issued by IO Devices : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. Does not include addressless requests such as locks and interrupts.", + "UMask": "0xcc23ff04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : IPQ", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IPQ", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : IPQ : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. T", + "UMask": "0x8", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : IRQ - iA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IRQ_IA", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : IRQ - iA : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. T : From an iA Core", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : IRQ - Non iA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IRQ_NON_IA", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : IRQ - Non iA : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. T", + "UMask": "0x10", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : Just ISOC", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.ISOC", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : Just ISOC : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. T", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : Just Local Targets", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.LOCAL_TGT", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : Just Local Targets : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. T", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : All from Local iA and IO", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.LOC_ALL", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : All from Local iA and IO : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. T : All locally initiated requests", + "UMask": "0xc000ff05", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : All from Local iA", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.LOC_IA", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : All from Local iA : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. T : All locally initiated requests from iA Cores", + "UMask": "0xc000ff01", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : All from Local IO", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.LOC_IO", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : All from Local IO : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. T : All locally generated IO traffic", + "UMask": "0xc000ff04", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : Match the Opcode in b[29:19] of the extended umask field", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.MATCH_OPC", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : Match the Opcode in b[29:19] of the extended umask field : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. T", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : Just Misses", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.MISS", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : Just Misses : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. T", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : MMCFG Access", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.MMCFG", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : MMCFG Access : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. T", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : MMIO Access", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.MMIO", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : MMIO Access : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. T", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : Just NearMem", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.NEARMEM", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : Just NearMem : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. T", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : Just NonCoherent", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.NONCOH", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : Just NonCoherent : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. T", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : Just NotNearMem", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.NOT_NEARMEM", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : Just NotNearMem : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. T", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : PMM Access", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.PMM", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : PMM Access : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent.", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : Match the PreMorphed Opcode in b[29:19] of the extended umask field", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.PREMORPH_OPC", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : Match the PreMorphed Opcode in b[29:19] of the extended umask field : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. T", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : PRQ - IOSF", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.PRQ", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : PRQ - IOSF : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. T : From a PCIe Device", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : PRQ - Non IOSF", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.PRQ_NON_IOSF", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : PRQ - Non IOSF : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. T", + "UMask": "0x20", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : Just Remote Targets", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.REMOTE_TGT", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : Just Remote Targets : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. T", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : All from Remote", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.REM_ALL", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : All from Remote : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. T : All remote requests (e.g. snoops, writebacks) that came from remote sockets", + "UMask": "0xc001ffc8", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : All Snoops from Remote", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.REM_SNPS", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : All Snoops from Remote : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. T : All snoops to this LLC that came from remote sockets", + "UMask": "0xc001ff08", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : RRQ", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.RRQ", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : RRQ : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. T", + "UMask": "0x40", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy; All Snoops from Remote", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.SNPS_FROM_REM", + "PerPkg": "1", + "PublicDescription": "For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. All snoops to this LLC that came from remote sockets.", + "UMask": "0xc001ff08", + "Unit": "CHA" + }, + { + "BriefDescription": "TOR Occupancy : WBQ", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.WBQ", + "PerPkg": "1", + "PublicDescription": "TOR Occupancy : WBQ : For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. T", + "UMask": "0x80", + "Unit": "CHA" + }, + { + "BriefDescription": "WbPushMtoI : Pushed to LLC", + "EventCode": "0x56", + "EventName": "UNC_CHA_WB_PUSH_MTOI.LLC", + "PerPkg": "1", + "PublicDescription": "WbPushMtoI : Pushed to LLC : Counts the number of times when the CHA was received WbPushMtoI : Counts the number of times when the CHA was able to push WbPushMToI to LLC", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "WbPushMtoI : Pushed to Memory", + "EventCode": "0x56", + "EventName": "UNC_CHA_WB_PUSH_MTOI.MEM", + "PerPkg": "1", + "PublicDescription": "WbPushMtoI : Pushed to Memory : Counts the number of times when the CHA was received WbPushMtoI : Counts the number of times when the CHA was unable to push WbPushMToI to LLC (hence pushed it to MEM)", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "CHA iMC CHNx WRITE Credits Empty : MC0", + "EventCode": "0x5a", + "EventName": "UNC_CHA_WRITE_NO_CREDITS.MC0", + "PerPkg": "1", + "PublicDescription": "CHA iMC CHNx WRITE Credits Empty : MC0 : Counts the number of times when there are no credits available for sending WRITEs from the CHA into the iMC. In order to send WRITEs into the memory controller, the HA must first acquire a credit for the iMC's BL Ingress queue. : Filter for memory controller 0 only.", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "CHA iMC CHNx WRITE Credits Empty : MC1", + "EventCode": "0x5a", + "EventName": "UNC_CHA_WRITE_NO_CREDITS.MC1", + "PerPkg": "1", + "PublicDescription": "CHA iMC CHNx WRITE Credits Empty : MC1 : Counts the number of times when there are no credits available for sending WRITEs from the CHA into the iMC. In order to send WRITEs into the memory controller, the HA must first acquire a credit for the iMC's BL Ingress queue. : Filter for memory controller 1 only.", + "UMask": "0x2", + "Unit": "CHA" + }, + { + "BriefDescription": "CHA iMC CHNx WRITE Credits Empty : MC2", + "EventCode": "0x5a", + "EventName": "UNC_CHA_WRITE_NO_CREDITS.MC2", + "PerPkg": "1", + "PublicDescription": "CHA iMC CHNx WRITE Credits Empty : MC2 : Counts the number of times when there are no credits available for sending WRITEs from the CHA into the iMC. In order to send WRITEs into the memory controller, the HA must first acquire a credit for the iMC's BL Ingress queue. : Filter for memory controller 2 only.", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "CHA iMC CHNx WRITE Credits Empty : MC3", + "EventCode": "0x5a", + "EventName": "UNC_CHA_WRITE_NO_CREDITS.MC3", + "PerPkg": "1", + "PublicDescription": "CHA iMC CHNx WRITE Credits Empty : MC3 : Counts the number of times when there are no credits available for sending WRITEs from the CHA into the iMC. In order to send WRITEs into the memory controller, the HA must first acquire a credit for the iMC's BL Ingress queue. : Filter for memory controller 3 only.", + "UMask": "0x8", + "Unit": "CHA" + }, + { + "BriefDescription": "CHA iMC CHNx WRITE Credits Empty : MC4", + "EventCode": "0x5a", + "EventName": "UNC_CHA_WRITE_NO_CREDITS.MC4", + "PerPkg": "1", + "PublicDescription": "CHA iMC CHNx WRITE Credits Empty : MC4 : Counts the number of times when there are no credits available for sending WRITEs from the CHA into the iMC. In order to send WRITEs into the memory controller, the HA must first acquire a credit for the iMC's BL Ingress queue. : Filter for memory controller 4 only.", + "UMask": "0x10", + "Unit": "CHA" + }, + { + "BriefDescription": "CHA iMC CHNx WRITE Credits Empty : MC5", + "EventCode": "0x5a", + "EventName": "UNC_CHA_WRITE_NO_CREDITS.MC5", + "PerPkg": "1", + "PublicDescription": "CHA iMC CHNx WRITE Credits Empty : MC5 : Counts the number of times when there are no credits available for sending WRITEs from the CHA into the iMC. In order to send WRITEs into the memory controller, the HA must first acquire a credit for the iMC's BL Ingress queue. : Filter for memory controller 5 only.", + "UMask": "0x20", + "Unit": "CHA" + }, + { + "BriefDescription": "XPT Prefetches : Dropped (on 0?) - Conflict", + "EventCode": "0x6f", + "EventName": "UNC_CHA_XPT_PREF.DROP0_CONFLICT", + "PerPkg": "1", + "PublicDescription": "XPT Prefetches : Dropped (on 0?) - Conflict : Number of XPT prefetches dropped due to AD CMS write port contention", + "UMask": "0x8", + "Unit": "CHA" + }, + { + "BriefDescription": "XPT Prefetches : Dropped (on 0?) - No Credits", + "EventCode": "0x6f", + "EventName": "UNC_CHA_XPT_PREF.DROP0_NOCRD", + "PerPkg": "1", + "PublicDescription": "XPT Prefetches : Dropped (on 0?) - No Credits : Number of XPT prefetches dropped due to lack of XPT AD egress credits", + "UMask": "0x4", + "Unit": "CHA" + }, + { + "BriefDescription": "XPT Prefetches : Dropped (on 1?) - Conflict", + "EventCode": "0x6f", + "EventName": "UNC_CHA_XPT_PREF.DROP1_CONFLICT", + "PerPkg": "1", + "PublicDescription": "XPT Prefetches : Dropped (on 1?) - Conflict : Number of XPT prefetches dropped due to AD CMS write port contention", + "UMask": "0x80", + "Unit": "CHA" + }, + { + "BriefDescription": "XPT Prefetches : Dropped (on 1?) - No Credits", + "EventCode": "0x6f", + "EventName": "UNC_CHA_XPT_PREF.DROP1_NOCRD", + "PerPkg": "1", + "PublicDescription": "XPT Prefetches : Dropped (on 1?) - No Credits : Number of XPT prefetches dropped due to lack of XPT AD egress credits", + "UMask": "0x40", + "Unit": "CHA" + }, + { + "BriefDescription": "XPT Prefetches : Sent (on 0?)", + "EventCode": "0x6f", + "EventName": "UNC_CHA_XPT_PREF.SENT0", + "PerPkg": "1", + "PublicDescription": "XPT Prefetches : Sent (on 0?) : Number of XPT prefetches sent", + "UMask": "0x1", + "Unit": "CHA" + }, + { + "BriefDescription": "XPT Prefetches : Sent (on 1?)", + "EventCode": "0x6f", + "EventName": "UNC_CHA_XPT_PREF.SENT1", + "PerPkg": "1", + "PublicDescription": "XPT Prefetches : Sent (on 1?) : Number of XPT prefetches sent", + "UMask": "0x10", + "Unit": "CHA" + } +] diff --git a/tools/perf/pmu-events/arch/x86/emeraldrapids/uncore-cxl.json b/tools/perf/pmu-events/arch/x86/emeraldrapids/uncore-cxl.json new file mode 100644 index 000000000000..f3e84fd88de3 --- /dev/null +++ b/tools/perf/pmu-events/arch/x86/emeraldrapids/uncore-cxl.json @@ -0,0 +1,450 @@ +[ + { + "BriefDescription": "Counts the number of lfclk ticks", + "EventCode": "0x01", + "EventName": "UNC_CXLCM_CLOCKTICKS", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Number of Allocation to Mem Rxx AGF 0", + "EventCode": "0x43", + "EventName": "UNC_CXLCM_RxC_AGF_INSERTS.CACHE_DATA", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Number of Allocation to Cache Req AGF0", + "EventCode": "0x43", + "EventName": "UNC_CXLCM_RxC_AGF_INSERTS.CACHE_REQ0", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Number of Allocation to Cache Rsp AGF", + "EventCode": "0x43", + "EventName": "UNC_CXLCM_RxC_AGF_INSERTS.CACHE_REQ1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Number of Allocation to Cache Data AGF", + "EventCode": "0x43", + "EventName": "UNC_CXLCM_RxC_AGF_INSERTS.CACHE_RSP0", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Number of Allocation to Cache Rsp AGF", + "EventCode": "0x43", + "EventName": "UNC_CXLCM_RxC_AGF_INSERTS.CACHE_RSP1", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Number of Allocation to Cache Req AGF 1", + "EventCode": "0x43", + "EventName": "UNC_CXLCM_RxC_AGF_INSERTS.MEM_DATA", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Number of Allocation to Mem Data AGF", + "EventCode": "0x43", + "EventName": "UNC_CXLCM_RxC_AGF_INSERTS.MEM_REQ", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Count the number of Flits with AK set", + "EventCode": "0x4b", + "EventName": "UNC_CXLCM_RxC_FLITS.AK_HDR", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Count the number of Flits with BE set", + "EventCode": "0x4b", + "EventName": "UNC_CXLCM_RxC_FLITS.BE_HDR", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Count the number of control flits received", + "EventCode": "0x4b", + "EventName": "UNC_CXLCM_RxC_FLITS.CTRL", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Count the number of Headerless flits received", + "EventCode": "0x4b", + "EventName": "UNC_CXLCM_RxC_FLITS.NO_HDR", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Count the number of protocol flits received", + "EventCode": "0x4b", + "EventName": "UNC_CXLCM_RxC_FLITS.PROT", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Count the number of Flits with SZ set", + "EventCode": "0x4b", + "EventName": "UNC_CXLCM_RxC_FLITS.SZ_HDR", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Count the number of flits received", + "EventCode": "0x4b", + "EventName": "UNC_CXLCM_RxC_FLITS.VALID", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Count the number of valid messages in the flit", + "EventCode": "0x4b", + "EventName": "UNC_CXLCM_RxC_FLITS.VALID_MSG", + "PerPkg": "1", + "UMask": "0x80", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Count the number of CRC errors detected", + "EventCode": "0x40", + "EventName": "UNC_CXLCM_RxC_MISC.CRC_ERRORS", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Count the number of Init flits sent", + "EventCode": "0x40", + "EventName": "UNC_CXLCM_RxC_MISC.INIT", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Count the number of LLCRD flits sent", + "EventCode": "0x40", + "EventName": "UNC_CXLCM_RxC_MISC.LLCRD", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Count the number of Retry flits sent", + "EventCode": "0x40", + "EventName": "UNC_CXLCM_RxC_MISC.RETRY", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Number of cycles the Packing Buffer is Full", + "EventCode": "0x52", + "EventName": "UNC_CXLCM_RxC_PACK_BUF_FULL.CACHE_DATA", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Number of cycles the Packing Buffer is Full", + "EventCode": "0x52", + "EventName": "UNC_CXLCM_RxC_PACK_BUF_FULL.CACHE_REQ", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Number of cycles the Packing Buffer is Full", + "EventCode": "0x52", + "EventName": "UNC_CXLCM_RxC_PACK_BUF_FULL.CACHE_RSP", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Number of cycles the Packing Buffer is Full", + "EventCode": "0x52", + "EventName": "UNC_CXLCM_RxC_PACK_BUF_FULL.MEM_DATA", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Number of cycles the Packing Buffer is Full", + "EventCode": "0x52", + "EventName": "UNC_CXLCM_RxC_PACK_BUF_FULL.MEM_REQ", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Number of Allocation to Cache Data Packing buffer", + "EventCode": "0x41", + "EventName": "UNC_CXLCM_RxC_PACK_BUF_INSERTS.CACHE_DATA", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Number of Allocation to Cache Req Packing buffer", + "EventCode": "0x41", + "EventName": "UNC_CXLCM_RxC_PACK_BUF_INSERTS.CACHE_REQ", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Number of Allocation to Cache Rsp Packing buffer", + "EventCode": "0x41", + "EventName": "UNC_CXLCM_RxC_PACK_BUF_INSERTS.CACHE_RSP", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Number of Allocation to Mem Data Packing buffer", + "EventCode": "0x41", + "EventName": "UNC_CXLCM_RxC_PACK_BUF_INSERTS.MEM_DATA", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Number of Allocation to Mem Rxx Packing buffer", + "EventCode": "0x41", + "EventName": "UNC_CXLCM_RxC_PACK_BUF_INSERTS.MEM_REQ", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Number of cycles of Not Empty for Cache Data Packing buffer", + "EventCode": "0x42", + "EventName": "UNC_CXLCM_RxC_PACK_BUF_NE.CACHE_DATA", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Number of cycles of Not Empty for Cache Req Packing buffer", + "EventCode": "0x42", + "EventName": "UNC_CXLCM_RxC_PACK_BUF_NE.CACHE_REQ", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Number of cycles of Not Empty for Cache Rsp Packing buffer", + "EventCode": "0x42", + "EventName": "UNC_CXLCM_RxC_PACK_BUF_NE.CACHE_RSP", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Number of cycles of Not Empty for Mem Data Packing buffer", + "EventCode": "0x42", + "EventName": "UNC_CXLCM_RxC_PACK_BUF_NE.MEM_DATA", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Number of cycles of Not Empty for Mem Rxx Packing buffer", + "EventCode": "0x42", + "EventName": "UNC_CXLCM_RxC_PACK_BUF_NE.MEM_REQ", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Count the number of Flits with AK set", + "EventCode": "0x05", + "EventName": "UNC_CXLCM_TxC_FLITS.AK_HDR", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Count the number of Flits with BE set", + "EventCode": "0x05", + "EventName": "UNC_CXLCM_TxC_FLITS.BE_HDR", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Count the number of control flits packed", + "EventCode": "0x05", + "EventName": "UNC_CXLCM_TxC_FLITS.CTRL", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Count the number of Headerless flits packed", + "EventCode": "0x05", + "EventName": "UNC_CXLCM_TxC_FLITS.NO_HDR", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Count the number of protocol flits packed", + "EventCode": "0x05", + "EventName": "UNC_CXLCM_TxC_FLITS.PROT", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Count the number of Flits with SZ set", + "EventCode": "0x05", + "EventName": "UNC_CXLCM_TxC_FLITS.SZ_HDR", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Count the number of flits packed", + "EventCode": "0x05", + "EventName": "UNC_CXLCM_TxC_FLITS.VALID", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Number of Allocation to Cache Data Packing buffer", + "EventCode": "0x02", + "EventName": "UNC_CXLCM_TxC_PACK_BUF_INSERTS.CACHE_DATA", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Number of Allocation to Cache Req Packing buffer", + "EventCode": "0x02", + "EventName": "UNC_CXLCM_TxC_PACK_BUF_INSERTS.CACHE_REQ0", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Number of Allocation to Cache Rsp1 Packing buffer", + "EventCode": "0x02", + "EventName": "UNC_CXLCM_TxC_PACK_BUF_INSERTS.CACHE_REQ1", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Number of Allocation to Cache Rsp0 Packing buffer", + "EventCode": "0x02", + "EventName": "UNC_CXLCM_TxC_PACK_BUF_INSERTS.CACHE_RSP0", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Number of Allocation to Cache Req Packing buffer", + "EventCode": "0x02", + "EventName": "UNC_CXLCM_TxC_PACK_BUF_INSERTS.CACHE_RSP1", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Number of Allocation to Mem Data Packing buffer", + "EventCode": "0x02", + "EventName": "UNC_CXLCM_TxC_PACK_BUF_INSERTS.MEM_DATA", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Number of Allocation to Mem Rxx Packing buffer", + "EventCode": "0x02", + "EventName": "UNC_CXLCM_TxC_PACK_BUF_INSERTS.MEM_REQ", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "CXLCM" + }, + { + "BriefDescription": "Counts the number of uclk ticks", + "EventCode": "0x01", + "EventName": "UNC_CXLDP_CLOCKTICKS", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "CXLDP" + }, + { + "BriefDescription": "Number of Allocation to M2S Data AGF", + "EventCode": "0x02", + "EventName": "UNC_CXLDP_TxC_AGF_INSERTS.M2S_DATA", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "CXLDP" + }, + { + "BriefDescription": "Number of Allocation to M2S Req AGF", + "EventCode": "0x02", + "EventName": "UNC_CXLDP_TxC_AGF_INSERTS.M2S_REQ", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "CXLDP" + }, + { + "BriefDescription": "Number of Allocation to U2C Data AGF", + "EventCode": "0x02", + "EventName": "UNC_CXLDP_TxC_AGF_INSERTS.U2C_DATA", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "CXLDP" + }, + { + "BriefDescription": "Number of Allocation to U2C Req AGF", + "EventCode": "0x02", + "EventName": "UNC_CXLDP_TxC_AGF_INSERTS.U2C_REQ", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "CXLDP" + }, + { + "BriefDescription": "Number of Allocation to U2C Rsp AGF 0", + "EventCode": "0x02", + "EventName": "UNC_CXLDP_TxC_AGF_INSERTS.U2C_RSP0", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "CXLDP" + }, + { + "BriefDescription": "Number of Allocation to U2C Rsp AGF 1", + "EventCode": "0x02", + "EventName": "UNC_CXLDP_TxC_AGF_INSERTS.U2C_RSP1", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "CXLDP" + } +] diff --git a/tools/perf/pmu-events/arch/x86/emeraldrapids/uncore-interconnect.json b/tools/perf/pmu-events/arch/x86/emeraldrapids/uncore-interconnect.json new file mode 100644 index 000000000000..09d840c7da4c --- /dev/null +++ b/tools/perf/pmu-events/arch/x86/emeraldrapids/uncore-interconnect.json @@ -0,0 +1,6199 @@ +[ + { + "BriefDescription": "Total IRP occupancy of inbound read and write requests to coherent memory.", + "EventCode": "0x0f", + "EventName": "UNC_I_CACHE_TOTAL_OCCUPANCY.MEM", + "PerPkg": "1", + "PublicDescription": "Total IRP occupancy of inbound read and write requests to coherent memory. This is effectively the sum of read occupancy and write occupancy.", + "UMask": "0x4", + "Unit": "IRP" + }, + { + "BriefDescription": "IRP Clockticks", + "EventCode": "0x01", + "EventName": "UNC_I_CLOCKTICKS", + "PerPkg": "1", + "PublicDescription": "Number of IRP clock cycles while the event is enabled", + "Unit": "IRP" + }, + { + "BriefDescription": "FAF RF full", + "EventCode": "0x17", + "EventName": "UNC_I_FAF_FULL", + "PerPkg": "1", + "Unit": "IRP" + }, + { + "BriefDescription": "FAF - request insert from TC.", + "EventCode": "0x18", + "EventName": "UNC_I_FAF_INSERTS", + "PerPkg": "1", + "Unit": "IRP" + }, + { + "BriefDescription": "FAF occupancy", + "EventCode": "0x19", + "EventName": "UNC_I_FAF_OCCUPANCY", + "PerPkg": "1", + "Unit": "IRP" + }, + { + "BriefDescription": "FAF allocation -- sent to ADQ", + "EventCode": "0x16", + "EventName": "UNC_I_FAF_TRANSACTIONS", + "PerPkg": "1", + "Unit": "IRP" + }, + { + "BriefDescription": ": All Inserts Outbound (BL, AK, Snoops)", + "EventCode": "0x20", + "EventName": "UNC_I_IRP_ALL.EVICTS", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "IRP" + }, + { + "BriefDescription": ": All Inserts Inbound (p2p + faf + cset)", + "EventCode": "0x20", + "EventName": "UNC_I_IRP_ALL.INBOUND_INSERTS", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "IRP" + }, + { + "BriefDescription": ": All Inserts Outbound (BL, AK, Snoops)", + "EventCode": "0x20", + "EventName": "UNC_I_IRP_ALL.OUTBOUND_INSERTS", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "IRP" + }, + { + "BriefDescription": "Counts Timeouts - Set 0 : Cache Inserts of Atomic Transactions as Secondary", + "EventCode": "0x1e", + "EventName": "UNC_I_MISC0.2ND_ATOMIC_INSERT", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "IRP" + }, + { + "BriefDescription": "Counts Timeouts - Set 0 : Cache Inserts of Read Transactions as Secondary", + "EventCode": "0x1e", + "EventName": "UNC_I_MISC0.2ND_RD_INSERT", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "IRP" + }, + { + "BriefDescription": "Counts Timeouts - Set 0 : Cache Inserts of Write Transactions as Secondary", + "EventCode": "0x1e", + "EventName": "UNC_I_MISC0.2ND_WR_INSERT", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "IRP" + }, + { + "BriefDescription": "Counts Timeouts - Set 0 : Fastpath Rejects", + "EventCode": "0x1e", + "EventName": "UNC_I_MISC0.FAST_REJ", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "IRP" + }, + { + "BriefDescription": "Counts Timeouts - Set 0 : Fastpath Requests", + "EventCode": "0x1e", + "EventName": "UNC_I_MISC0.FAST_REQ", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "IRP" + }, + { + "BriefDescription": "Counts Timeouts - Set 0 : Fastpath Transfers From Primary to Secondary", + "EventCode": "0x1e", + "EventName": "UNC_I_MISC0.FAST_XFER", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "IRP" + }, + { + "BriefDescription": "Counts Timeouts - Set 0 : Prefetch Ack Hints From Primary to Secondary", + "EventCode": "0x1e", + "EventName": "UNC_I_MISC0.PF_ACK_HINT", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "IRP" + }, + { + "BriefDescription": "Counts Timeouts - Set 0 : Slow path fwpf didn't find prefetch", + "EventCode": "0x1e", + "EventName": "UNC_I_MISC0.SLOWPATH_FWPF_NO_PRF", + "PerPkg": "1", + "UMask": "0x80", + "Unit": "IRP" + }, + { + "BriefDescription": "Misc Events - Set 1 : Lost Forward", + "EventCode": "0x1f", + "EventName": "UNC_I_MISC1.LOST_FWD", + "PerPkg": "1", + "PublicDescription": "Misc Events - Set 1 : Lost Forward : Snoop pulled away ownership before a write was committed", + "UMask": "0x10", + "Unit": "IRP" + }, + { + "BriefDescription": "Misc Events - Set 1 : Received Invalid", + "EventCode": "0x1f", + "EventName": "UNC_I_MISC1.SEC_RCVD_INVLD", + "PerPkg": "1", + "PublicDescription": "Misc Events - Set 1 : Received Invalid : Secondary received a transfer that did not have sufficient MESI state", + "UMask": "0x20", + "Unit": "IRP" + }, + { + "BriefDescription": "Misc Events - Set 1 : Received Valid", + "EventCode": "0x1f", + "EventName": "UNC_I_MISC1.SEC_RCVD_VLD", + "PerPkg": "1", + "PublicDescription": "Misc Events - Set 1 : Received Valid : Secondary received a transfer that did have sufficient MESI state", + "UMask": "0x40", + "Unit": "IRP" + }, + { + "BriefDescription": "Misc Events - Set 1 : Slow Transfer of E Line", + "EventCode": "0x1f", + "EventName": "UNC_I_MISC1.SLOW_E", + "PerPkg": "1", + "PublicDescription": "Misc Events - Set 1 : Slow Transfer of E Line : Secondary received a transfer that did have sufficient MESI state", + "UMask": "0x4", + "Unit": "IRP" + }, + { + "BriefDescription": "Misc Events - Set 1 : Slow Transfer of I Line", + "EventCode": "0x1f", + "EventName": "UNC_I_MISC1.SLOW_I", + "PerPkg": "1", + "PublicDescription": "Misc Events - Set 1 : Slow Transfer of I Line : Snoop took cacheline ownership before write from data was committed.", + "UMask": "0x1", + "Unit": "IRP" + }, + { + "BriefDescription": "Misc Events - Set 1 : Slow Transfer of M Line", + "EventCode": "0x1f", + "EventName": "UNC_I_MISC1.SLOW_M", + "PerPkg": "1", + "PublicDescription": "Misc Events - Set 1 : Slow Transfer of M Line : Snoop took cacheline ownership before write from data was committed.", + "UMask": "0x8", + "Unit": "IRP" + }, + { + "BriefDescription": "Misc Events - Set 1 : Slow Transfer of S Line", + "EventCode": "0x1f", + "EventName": "UNC_I_MISC1.SLOW_S", + "PerPkg": "1", + "PublicDescription": "Misc Events - Set 1 : Slow Transfer of S Line : Secondary received a transfer that did not have sufficient MESI state", + "UMask": "0x2", + "Unit": "IRP" + }, + { + "BriefDescription": "Responses to snoops of any type that hit M, E, S or I line in the IIO", + "EventCode": "0x12", + "EventName": "UNC_I_SNOOP_RESP.ALL_HIT", + "PerPkg": "1", + "PublicDescription": "Responses to snoops of any type (code, data, invalidate) that hit M, E, S or I line in the IIO", + "UMask": "0x7e", + "Unit": "IRP" + }, + { + "BriefDescription": "Responses to snoops of any type that hit E or S line in the IIO cache", + "EventCode": "0x12", + "EventName": "UNC_I_SNOOP_RESP.ALL_HIT_ES", + "PerPkg": "1", + "PublicDescription": "Responses to snoops of any type (code, data, invalidate) that hit E or S line in the IIO cache", + "UMask": "0x74", + "Unit": "IRP" + }, + { + "BriefDescription": "Responses to snoops of any type that hit I line in the IIO cache", + "EventCode": "0x12", + "EventName": "UNC_I_SNOOP_RESP.ALL_HIT_I", + "PerPkg": "1", + "PublicDescription": "Responses to snoops of any type (code, data, invalidate) that hit I line in the IIO cache", + "UMask": "0x72", + "Unit": "IRP" + }, + { + "BriefDescription": "Responses to snoops of any type that hit M line in the IIO cache", + "EventCode": "0x12", + "EventName": "UNC_I_SNOOP_RESP.ALL_HIT_M", + "PerPkg": "1", + "PublicDescription": "Responses to snoops of any type (code, data, invalidate) that hit M line in the IIO cache", + "UMask": "0x78", + "Unit": "IRP" + }, + { + "BriefDescription": "Responses to snoops of any type that miss the IIO cache", + "EventCode": "0x12", + "EventName": "UNC_I_SNOOP_RESP.ALL_MISS", + "PerPkg": "1", + "PublicDescription": "Responses to snoops of any type (code, data, invalidate) that miss the IIO cache", + "UMask": "0x71", + "Unit": "IRP" + }, + { + "BriefDescription": "Snoop Responses : Hit E or S", + "EventCode": "0x12", + "EventName": "UNC_I_SNOOP_RESP.HIT_ES", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "IRP" + }, + { + "BriefDescription": "Snoop Responses : Hit I", + "EventCode": "0x12", + "EventName": "UNC_I_SNOOP_RESP.HIT_I", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "IRP" + }, + { + "BriefDescription": "Snoop Responses : Hit M", + "EventCode": "0x12", + "EventName": "UNC_I_SNOOP_RESP.HIT_M", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "IRP" + }, + { + "BriefDescription": "Snoop Responses : Miss", + "EventCode": "0x12", + "EventName": "UNC_I_SNOOP_RESP.MISS", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "IRP" + }, + { + "BriefDescription": "Snoop Responses : SnpCode", + "EventCode": "0x12", + "EventName": "UNC_I_SNOOP_RESP.SNPCODE", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "IRP" + }, + { + "BriefDescription": "Snoop Responses : SnpData", + "EventCode": "0x12", + "EventName": "UNC_I_SNOOP_RESP.SNPDATA", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "IRP" + }, + { + "BriefDescription": "Snoop Responses : SnpInv", + "EventCode": "0x12", + "EventName": "UNC_I_SNOOP_RESP.SNPINV", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "IRP" + }, + { + "BriefDescription": "Inbound write (fast path) requests received by the IRP.", + "EventCode": "0x11", + "EventName": "UNC_I_TRANSACTIONS.WR_PREF", + "PerPkg": "1", + "PublicDescription": "Inbound write (fast path) requests to coherent memory, received by the IRP resulting in write ownership requests issued by IRP to the mesh.", + "UMask": "0x8", + "Unit": "IRP" + }, + { + "BriefDescription": "AK Egress Allocations", + "EventCode": "0x0b", + "EventName": "UNC_I_TxC_AK_INSERTS", + "PerPkg": "1", + "Unit": "IRP" + }, + { + "BriefDescription": "BL DRS Egress Cycles Full", + "EventCode": "0x05", + "EventName": "UNC_I_TxC_BL_DRS_CYCLES_FULL", + "PerPkg": "1", + "Unit": "IRP" + }, + { + "BriefDescription": "BL DRS Egress Inserts", + "EventCode": "0x02", + "EventName": "UNC_I_TxC_BL_DRS_INSERTS", + "PerPkg": "1", + "Unit": "IRP" + }, + { + "BriefDescription": "BL DRS Egress Occupancy", + "EventCode": "0x08", + "EventName": "UNC_I_TxC_BL_DRS_OCCUPANCY", + "PerPkg": "1", + "Unit": "IRP" + }, + { + "BriefDescription": "BL NCB Egress Cycles Full", + "EventCode": "0x06", + "EventName": "UNC_I_TxC_BL_NCB_CYCLES_FULL", + "PerPkg": "1", + "Unit": "IRP" + }, + { + "BriefDescription": "BL NCB Egress Inserts", + "EventCode": "0x03", + "EventName": "UNC_I_TxC_BL_NCB_INSERTS", + "PerPkg": "1", + "Unit": "IRP" + }, + { + "BriefDescription": "BL NCB Egress Occupancy", + "EventCode": "0x09", + "EventName": "UNC_I_TxC_BL_NCB_OCCUPANCY", + "PerPkg": "1", + "Unit": "IRP" + }, + { + "BriefDescription": "BL NCS Egress Cycles Full", + "EventCode": "0x07", + "EventName": "UNC_I_TxC_BL_NCS_CYCLES_FULL", + "PerPkg": "1", + "Unit": "IRP" + }, + { + "BriefDescription": "BL NCS Egress Inserts", + "EventCode": "0x04", + "EventName": "UNC_I_TxC_BL_NCS_INSERTS", + "PerPkg": "1", + "Unit": "IRP" + }, + { + "BriefDescription": "BL NCS Egress Occupancy", + "EventCode": "0x0a", + "EventName": "UNC_I_TxC_BL_NCS_OCCUPANCY", + "PerPkg": "1", + "Unit": "IRP" + }, + { + "BriefDescription": "UNC_I_TxR2_AD01_STALL_CREDIT_CYCLES", + "EventCode": "0x1c", + "EventName": "UNC_I_TxR2_AD01_STALL_CREDIT_CYCLES", + "PerPkg": "1", + "PublicDescription": ": Counts the number times when it is not possible to issue a request to the M2PCIe because there are no Egress Credits available on AD0, A1 or AD0AD1 both. Stalls on both AD0 and AD1 will count as 2", + "Unit": "IRP" + }, + { + "BriefDescription": "No AD0 Egress Credits Stalls", + "EventCode": "0x1a", + "EventName": "UNC_I_TxR2_AD0_STALL_CREDIT_CYCLES", + "PerPkg": "1", + "PublicDescription": "No AD0 Egress Credits Stalls : Counts the number times when it is not possible to issue a request to the M2PCIe because there are no AD0 Egress Credits available.", + "Unit": "IRP" + }, + { + "BriefDescription": "No AD1 Egress Credits Stalls", + "EventCode": "0x1b", + "EventName": "UNC_I_TxR2_AD1_STALL_CREDIT_CYCLES", + "PerPkg": "1", + "PublicDescription": "No AD1 Egress Credits Stalls : Counts the number times when it is not possible to issue a request to the M2PCIe because there are no AD1 Egress Credits available.", + "Unit": "IRP" + }, + { + "BriefDescription": "No BL Egress Credit Stalls", + "EventCode": "0x1d", + "EventName": "UNC_I_TxR2_BL_STALL_CREDIT_CYCLES", + "PerPkg": "1", + "PublicDescription": "No BL Egress Credit Stalls : Counts the number times when it is not possible to issue data to the R2PCIe because there are no BL Egress Credits available.", + "Unit": "IRP" + }, + { + "BriefDescription": "Outbound Read Requests", + "EventCode": "0x0d", + "EventName": "UNC_I_TxS_DATA_INSERTS_NCB", + "PerPkg": "1", + "PublicDescription": "Outbound Read Requests : Counts the number of requests issued to the switch (towards the devices).", + "Unit": "IRP" + }, + { + "BriefDescription": "Outbound Read Requests", + "EventCode": "0x0e", + "EventName": "UNC_I_TxS_DATA_INSERTS_NCS", + "PerPkg": "1", + "PublicDescription": "Outbound Read Requests : Counts the number of requests issued to the switch (towards the devices).", + "Unit": "IRP" + }, + { + "BriefDescription": "Outbound Request Queue Occupancy", + "EventCode": "0x0c", + "EventName": "UNC_I_TxS_REQUEST_OCCUPANCY", + "PerPkg": "1", + "PublicDescription": "Outbound Request Queue Occupancy : Accumulates the number of outstanding outbound requests from the IRP to the switch (towards the devices). This can be used in conjunction with the allocations event in order to calculate average latency of outbound requests.", + "Unit": "IRP" + }, + { + "BriefDescription": "M2M Clockticks", + "EventCode": "0x01", + "EventName": "UNC_M2M_CLOCKTICKS", + "PerPkg": "1", + "PublicDescription": "Clockticks of the mesh to memory (M2M)", + "Unit": "M2M" + }, + { + "BriefDescription": "CMS Clockticks", + "EventCode": "0xc0", + "EventName": "UNC_M2M_CMS_CLOCKTICKS", + "PerPkg": "1", + "Unit": "M2M" + }, + { + "BriefDescription": "Cycles when direct to core mode (which bypasses the CHA) was disabled", + "EventCode": "0x17", + "EventName": "UNC_M2M_DIRECT2CORE_NOT_TAKEN_DIRSTATE", + "PerPkg": "1", + "UMask": "0x7", + "Unit": "M2M" + }, + { + "BriefDescription": "Cycles when direct to core mode, which bypasses the CHA, was disabled : Non Cisgress", + "EventCode": "0x17", + "EventName": "UNC_M2M_DIRECT2CORE_NOT_TAKEN_DIRSTATE.NON_CISGRESS", + "PerPkg": "1", + "PublicDescription": "Cycles when direct to core mode, which bypasses the CHA, was disabled : Non Cisgress : Counts the number of time non cisgress D2C was not honoured by egress due to directory state constraints", + "UMask": "0x2", + "Unit": "M2M" + }, + { + "BriefDescription": "Counts the time when FM didn't do d2c for fill reads (cross tile case)", + "EventCode": "0x4a", + "EventName": "UNC_M2M_DIRECT2CORE_NOT_TAKEN_NOTFORKED", + "PerPkg": "1", + "Unit": "M2M" + }, + { + "BriefDescription": "Number of reads in which direct to core transaction were overridden", + "EventCode": "0x18", + "EventName": "UNC_M2M_DIRECT2CORE_TXN_OVERRIDE", + "PerPkg": "1", + "UMask": "0x3", + "Unit": "M2M" + }, + { + "BriefDescription": "Number of reads in which direct to core transaction was overridden : Cisgress", + "EventCode": "0x18", + "EventName": "UNC_M2M_DIRECT2CORE_TXN_OVERRIDE.CISGRESS", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2M" + }, + { + "BriefDescription": "Number of reads in which direct to core transaction was overridden : 2LM Hit?", + "EventCode": "0x18", + "EventName": "UNC_M2M_DIRECT2CORE_TXN_OVERRIDE.PMM_HIT", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2M" + }, + { + "BriefDescription": "Number of times a direct to UPI transaction was overridden.", + "EventCode": "0x1C", + "EventName": "UNC_M2M_DIRECT2UPITXN_OVERRIDE.PMM_HIT", + "PerPkg": "1", + "PublicDescription": "Number of times a direct to UPI transaction was overridden. : Counts the number of times D2K wasn't honored even though the incoming request had d2k set", + "UMask": "0x1", + "Unit": "M2M" + }, + { + "BriefDescription": "Number of reads in which direct to Intel UPI transactions were overridden", + "EventCode": "0x1b", + "EventName": "UNC_M2M_DIRECT2UPI_NOT_TAKEN_CREDITS", + "PerPkg": "1", + "UMask": "0x7", + "Unit": "M2M" + }, + { + "BriefDescription": "Cycles when direct to Intel UPI was disabled", + "EventCode": "0x1a", + "EventName": "UNC_M2M_DIRECT2UPI_NOT_TAKEN_DIRSTATE", + "PerPkg": "1", + "UMask": "0x7", + "Unit": "M2M" + }, + { + "BriefDescription": "Cycles when Direct2UPI was Disabled : Cisgress D2U Ignored", + "EventCode": "0x1A", + "EventName": "UNC_M2M_DIRECT2UPI_NOT_TAKEN_DIRSTATE.CISGRESS", + "PerPkg": "1", + "PublicDescription": "Cycles when Direct2UPI was Disabled : Cisgress D2U Ignored : Counts cisgress d2K that was not honored due to directory constraints", + "UMask": "0x4", + "Unit": "M2M" + }, + { + "BriefDescription": "Cycles when Direct2UPI was Disabled : Egress Ignored D2U", + "EventCode": "0x1A", + "EventName": "UNC_M2M_DIRECT2UPI_NOT_TAKEN_DIRSTATE.EGRESS", + "PerPkg": "1", + "PublicDescription": "Cycles when Direct2UPI was Disabled : Egress Ignored D2U : Counts the number of time D2K was not honoured by egress due to directory state constraints", + "UMask": "0x1", + "Unit": "M2M" + }, + { + "BriefDescription": "Cycles when Direct2UPI was Disabled : Non Cisgress D2U Ignored", + "EventCode": "0x1A", + "EventName": "UNC_M2M_DIRECT2UPI_NOT_TAKEN_DIRSTATE.NON_CISGRESS", + "PerPkg": "1", + "PublicDescription": "Cycles when Direct2UPI was Disabled : Non Cisgress D2U Ignored : Counts non cisgress d2K that was not honored due to directory constraints", + "UMask": "0x2", + "Unit": "M2M" + }, + { + "BriefDescription": "Messages sent direct to the Intel UPI", + "EventCode": "0x19", + "EventName": "UNC_M2M_DIRECT2UPI_TAKEN", + "PerPkg": "1", + "PublicDescription": "Counts the number of times egress did D2K (Direct to KTI)", + "UMask": "0x7", + "Unit": "M2M" + }, + { + "BriefDescription": "Number of reads that a message sent direct2 Intel UPI was overridden", + "EventCode": "0x1c", + "EventName": "UNC_M2M_DIRECT2UPI_TXN_OVERRIDE", + "PerPkg": "1", + "UMask": "0x3", + "Unit": "M2M" + }, + { + "BriefDescription": "Number of times a direct to UPI transaction was overridden.", + "EventCode": "0x1C", + "EventName": "UNC_M2M_DIRECT2UPI_TXN_OVERRIDE.CISGRESS", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2M" + }, + { + "BriefDescription": "Directory Hit : On NonDirty Line in A State", + "EventCode": "0x1d", + "EventName": "UNC_M2M_DIRECTORY_HIT.CLEAN_A", + "PerPkg": "1", + "UMask": "0x80", + "Unit": "M2M" + }, + { + "BriefDescription": "Directory Hit : On NonDirty Line in I State", + "EventCode": "0x1d", + "EventName": "UNC_M2M_DIRECTORY_HIT.CLEAN_I", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "M2M" + }, + { + "BriefDescription": "Directory Hit : On NonDirty Line in L State", + "EventCode": "0x1d", + "EventName": "UNC_M2M_DIRECTORY_HIT.CLEAN_P", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "M2M" + }, + { + "BriefDescription": "Directory Hit : On NonDirty Line in S State", + "EventCode": "0x1d", + "EventName": "UNC_M2M_DIRECTORY_HIT.CLEAN_S", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "M2M" + }, + { + "BriefDescription": "Directory Hit : On Dirty Line in A State", + "EventCode": "0x1d", + "EventName": "UNC_M2M_DIRECTORY_HIT.DIRTY_A", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "M2M" + }, + { + "BriefDescription": "Directory Hit : On Dirty Line in I State", + "EventCode": "0x1d", + "EventName": "UNC_M2M_DIRECTORY_HIT.DIRTY_I", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2M" + }, + { + "BriefDescription": "Directory Hit : On Dirty Line in L State", + "EventCode": "0x1d", + "EventName": "UNC_M2M_DIRECTORY_HIT.DIRTY_P", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "M2M" + }, + { + "BriefDescription": "Directory Hit : On Dirty Line in S State", + "EventCode": "0x1d", + "EventName": "UNC_M2M_DIRECTORY_HIT.DIRTY_S", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2M" + }, + { + "BriefDescription": "Multi-socket cacheline Directory lookups (any state found)", + "EventCode": "0x20", + "EventName": "UNC_M2M_DIRECTORY_LOOKUP.ANY", + "PerPkg": "1", + "PublicDescription": "Counts the number of hit data returns to egress with any directory to non persistent memory", + "UMask": "0x1", + "Unit": "M2M" + }, + { + "BriefDescription": "Multi-socket cacheline Directory lookups (cacheline found in A state)", + "EventCode": "0x20", + "EventName": "UNC_M2M_DIRECTORY_LOOKUP.STATE_A", + "PerPkg": "1", + "PublicDescription": "Counts the number of hit data returns to egress with directory A to non persistent memory", + "UMask": "0x8", + "Unit": "M2M" + }, + { + "BriefDescription": "Multi-socket cacheline Directory lookup (cacheline found in I state)", + "EventCode": "0x20", + "EventName": "UNC_M2M_DIRECTORY_LOOKUP.STATE_I", + "PerPkg": "1", + "PublicDescription": "Counts the number of hit data returns to egress with directory I to non persistent memory", + "UMask": "0x2", + "Unit": "M2M" + }, + { + "BriefDescription": "Multi-socket cacheline Directory lookup (cacheline found in S state)", + "EventCode": "0x20", + "EventName": "UNC_M2M_DIRECTORY_LOOKUP.STATE_S", + "PerPkg": "1", + "PublicDescription": "Counts the number of hit data returns to egress with directory S to non persistent memory", + "UMask": "0x4", + "Unit": "M2M" + }, + { + "BriefDescription": "Directory Miss : On NonDirty Line in A State", + "EventCode": "0x1e", + "EventName": "UNC_M2M_DIRECTORY_MISS.CLEAN_A", + "PerPkg": "1", + "UMask": "0x80", + "Unit": "M2M" + }, + { + "BriefDescription": "Directory Miss : On NonDirty Line in I State", + "EventCode": "0x1e", + "EventName": "UNC_M2M_DIRECTORY_MISS.CLEAN_I", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "M2M" + }, + { + "BriefDescription": "Directory Miss : On NonDirty Line in L State", + "EventCode": "0x1e", + "EventName": "UNC_M2M_DIRECTORY_MISS.CLEAN_P", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "M2M" + }, + { + "BriefDescription": "Directory Miss : On NonDirty Line in S State", + "EventCode": "0x1e", + "EventName": "UNC_M2M_DIRECTORY_MISS.CLEAN_S", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "M2M" + }, + { + "BriefDescription": "Directory Miss : On Dirty Line in A State", + "EventCode": "0x1e", + "EventName": "UNC_M2M_DIRECTORY_MISS.DIRTY_A", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "M2M" + }, + { + "BriefDescription": "Directory Miss : On Dirty Line in I State", + "EventCode": "0x1e", + "EventName": "UNC_M2M_DIRECTORY_MISS.DIRTY_I", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2M" + }, + { + "BriefDescription": "Directory Miss : On Dirty Line in L State", + "EventCode": "0x1e", + "EventName": "UNC_M2M_DIRECTORY_MISS.DIRTY_P", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "M2M" + }, + { + "BriefDescription": "Directory Miss : On Dirty Line in S State", + "EventCode": "0x1e", + "EventName": "UNC_M2M_DIRECTORY_MISS.DIRTY_S", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2M" + }, + { + "BriefDescription": "Multi-socket cacheline Directory update from A to I", + "EventCode": "0x21", + "EventName": "UNC_M2M_DIRECTORY_UPDATE.A2I", + "PerPkg": "1", + "UMask": "0x320", + "Unit": "M2M" + }, + { + "BriefDescription": "Multi-socket cacheline Directory update from A to S", + "EventCode": "0x21", + "EventName": "UNC_M2M_DIRECTORY_UPDATE.A2S", + "PerPkg": "1", + "UMask": "0x340", + "Unit": "M2M" + }, + { + "BriefDescription": "Multi-socket cacheline Directory update from/to Any state", + "EventCode": "0x21", + "EventName": "UNC_M2M_DIRECTORY_UPDATE.ANY", + "PerPkg": "1", + "UMask": "0x301", + "Unit": "M2M" + }, + { + "BriefDescription": "Multi-socket cacheline Directory Updates", + "EventCode": "0x21", + "EventName": "UNC_M2M_DIRECTORY_UPDATE.A_TO_I_HIT_NON_PMM", + "PerPkg": "1", + "PublicDescription": "Counts 1lm or 2lm hit data returns that would result in directory update from A to I to non persistent memory (DRAM or HBM)", + "UMask": "0x120", + "Unit": "M2M" + }, + { + "BriefDescription": "Multi-socket cacheline Directory Updates", + "EventCode": "0x21", + "EventName": "UNC_M2M_DIRECTORY_UPDATE.A_TO_I_MISS_NON_PMM", + "PerPkg": "1", + "PublicDescription": "Counts 2lm miss data returns that would result in directory update from A to I to non persistent memory (DRAM or HBM)", + "UMask": "0x220", + "Unit": "M2M" + }, + { + "BriefDescription": "Multi-socket cacheline Directory Updates", + "EventCode": "0x21", + "EventName": "UNC_M2M_DIRECTORY_UPDATE.A_TO_S_HIT_NON_PMM", + "PerPkg": "1", + "PublicDescription": "Counts 1lm or 2lm hit data returns that would result in directory update from A to S to non persistent memory (DRAM or HBM)", + "UMask": "0x140", + "Unit": "M2M" + }, + { + "BriefDescription": "Multi-socket cacheline Directory Updates", + "EventCode": "0x21", + "EventName": "UNC_M2M_DIRECTORY_UPDATE.A_TO_S_MISS_NON_PMM", + "PerPkg": "1", + "PublicDescription": "Counts 2lm miss data returns that would result in directory update from A to S to non persistent memory (DRAM or HBM)", + "UMask": "0x240", + "Unit": "M2M" + }, + { + "BriefDescription": "Multi-socket cacheline Directory Updates", + "EventCode": "0x21", + "EventName": "UNC_M2M_DIRECTORY_UPDATE.HIT_NON_PMM", + "PerPkg": "1", + "PublicDescription": "Counts any 1lm or 2lm hit data return that would result in directory update to non persistent memory (DRAM or HBM)", + "UMask": "0x101", + "Unit": "M2M" + }, + { + "BriefDescription": "Multi-socket cacheline Directory update from I to A", + "EventCode": "0x21", + "EventName": "UNC_M2M_DIRECTORY_UPDATE.I2A", + "PerPkg": "1", + "UMask": "0x304", + "Unit": "M2M" + }, + { + "BriefDescription": "Multi-socket cacheline Directory update from I to S", + "EventCode": "0x21", + "EventName": "UNC_M2M_DIRECTORY_UPDATE.I2S", + "PerPkg": "1", + "UMask": "0x302", + "Unit": "M2M" + }, + { + "BriefDescription": "Multi-socket cacheline Directory Updates", + "EventCode": "0x21", + "EventName": "UNC_M2M_DIRECTORY_UPDATE.I_TO_A_HIT_NON_PMM", + "PerPkg": "1", + "PublicDescription": "Counts 1lm or 2lm hit data returns that would result in directory update from I to A to non persistent memory (DRAM or HBM)", + "UMask": "0x104", + "Unit": "M2M" + }, + { + "BriefDescription": "Multi-socket cacheline Directory Updates", + "EventCode": "0x21", + "EventName": "UNC_M2M_DIRECTORY_UPDATE.I_TO_A_MISS_NON_PMM", + "PerPkg": "1", + "PublicDescription": "Counts 2lm miss data returns that would result in directory update from I to A to non persistent memory (DRAM or HBM)", + "UMask": "0x204", + "Unit": "M2M" + }, + { + "BriefDescription": "Multi-socket cacheline Directory Updates", + "EventCode": "0x21", + "EventName": "UNC_M2M_DIRECTORY_UPDATE.I_TO_S_HIT_NON_PMM", + "PerPkg": "1", + "PublicDescription": "Counts 1lm or 2lm hit data returns that would result in directory update from I to S to non persistent memory (DRAM or HBM)", + "UMask": "0x102", + "Unit": "M2M" + }, + { + "BriefDescription": "Multi-socket cacheline Directory Updates", + "EventCode": "0x21", + "EventName": "UNC_M2M_DIRECTORY_UPDATE.I_TO_S_MISS_NON_PMM", + "PerPkg": "1", + "PublicDescription": "Counts 2lm miss data returns that would result in directory update from I to S to non persistent memory (DRAM or HBM)", + "UMask": "0x202", + "Unit": "M2M" + }, + { + "BriefDescription": "Multi-socket cacheline Directory Updates", + "EventCode": "0x21", + "EventName": "UNC_M2M_DIRECTORY_UPDATE.MISS_NON_PMM", + "PerPkg": "1", + "PublicDescription": "Counts any 2lm miss data return that would result in directory update to non persistent memory (DRAM or HBM)", + "UMask": "0x201", + "Unit": "M2M" + }, + { + "BriefDescription": "Multi-socket cacheline Directory update from S to A", + "EventCode": "0x21", + "EventName": "UNC_M2M_DIRECTORY_UPDATE.S2A", + "PerPkg": "1", + "UMask": "0x310", + "Unit": "M2M" + }, + { + "BriefDescription": "Multi-socket cacheline Directory update from S to I", + "EventCode": "0x21", + "EventName": "UNC_M2M_DIRECTORY_UPDATE.S2I", + "PerPkg": "1", + "UMask": "0x308", + "Unit": "M2M" + }, + { + "BriefDescription": "Multi-socket cacheline Directory Updates", + "EventCode": "0x21", + "EventName": "UNC_M2M_DIRECTORY_UPDATE.S_TO_A_HIT_NON_PMM", + "PerPkg": "1", + "PublicDescription": "Counts 1lm or 2lm hit data returns that would result in directory update from S to A to non persistent memory (DRAM or HBM)", + "UMask": "0x110", + "Unit": "M2M" + }, + { + "BriefDescription": "Multi-socket cacheline Directory Updates", + "EventCode": "0x21", + "EventName": "UNC_M2M_DIRECTORY_UPDATE.S_TO_A_MISS_NON_PMM", + "PerPkg": "1", + "PublicDescription": "Counts 2lm miss data returns that would result in directory update from S to A to non persistent memory (DRAM or HBM)", + "UMask": "0x210", + "Unit": "M2M" + }, + { + "BriefDescription": "Multi-socket cacheline Directory Updates", + "EventCode": "0x21", + "EventName": "UNC_M2M_DIRECTORY_UPDATE.S_TO_I_HIT_NON_PMM", + "PerPkg": "1", + "PublicDescription": "Counts 1lm or 2lm hit data returns that would result in directory update from S to I to non persistent memory (DRAM or HBM)", + "UMask": "0x108", + "Unit": "M2M" + }, + { + "BriefDescription": "Multi-socket cacheline Directory Updates", + "EventCode": "0x21", + "EventName": "UNC_M2M_DIRECTORY_UPDATE.S_TO_I_MISS_NON_PMM", + "PerPkg": "1", + "PublicDescription": "Counts 2lm miss data returns that would result in directory update from S to I to non persistent memory (DRAM or HBM)", + "UMask": "0x208", + "Unit": "M2M" + }, + { + "BriefDescription": "Egress Blocking due to Ordering requirements : Down", + "EventCode": "0xba", + "EventName": "UNC_M2M_EGRESS_ORDERING.IV_SNOOPGO_DN", + "PerPkg": "1", + "PublicDescription": "Egress Blocking due to Ordering requirements : Down : Counts number of cycles IV was blocked in the TGR Egress due to SNP/GO Ordering requirements", + "UMask": "0x80000004", + "Unit": "M2M" + }, + { + "BriefDescription": "Egress Blocking due to Ordering requirements : Up", + "EventCode": "0xba", + "EventName": "UNC_M2M_EGRESS_ORDERING.IV_SNOOPGO_UP", + "PerPkg": "1", + "PublicDescription": "Egress Blocking due to Ordering requirements : Up : Counts number of cycles IV was blocked in the TGR Egress due to SNP/GO Ordering requirements", + "UMask": "0x80000001", + "Unit": "M2M" + }, + { + "BriefDescription": "Count when Starve Glocab counter is at 7", + "EventCode": "0x44", + "EventName": "UNC_M2M_IGR_STARVE_WINNER.MASK7", + "PerPkg": "1", + "UMask": "0x80", + "Unit": "M2M" + }, + { + "BriefDescription": "Reads to iMC issued", + "EventCode": "0x24", + "EventName": "UNC_M2M_IMC_READS.ALL", + "PerPkg": "1", + "UMask": "0x304", + "Unit": "M2M" + }, + { + "BriefDescription": "UNC_M2M_IMC_READS.CH0.TO_NM1LM", + "EventCode": "0x24", + "EventName": "UNC_M2M_IMC_READS.CH0.TO_NM1LM", + "PerPkg": "1", + "UMask": "0x108", + "Unit": "M2M" + }, + { + "BriefDescription": "UNC_M2M_IMC_READS.CH0.TO_NMCache", + "EventCode": "0x24", + "EventName": "UNC_M2M_IMC_READS.CH0.TO_NMCache", + "PerPkg": "1", + "UMask": "0x110", + "Unit": "M2M" + }, + { + "BriefDescription": "UNC_M2M_IMC_READS.CH0_ALL", + "EventCode": "0x24", + "EventName": "UNC_M2M_IMC_READS.CH0_ALL", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "UMask": "0x104", + "Unit": "M2M" + }, + { + "BriefDescription": "UNC_M2M_IMC_READS.CH0_FROM_TGR", + "EventCode": "0x24", + "EventName": "UNC_M2M_IMC_READS.CH0_FROM_TGR", + "PerPkg": "1", + "UMask": "0x140", + "Unit": "M2M" + }, + { + "BriefDescription": "UNC_M2M_IMC_READS.CH0_ISOCH", + "EventCode": "0x24", + "EventName": "UNC_M2M_IMC_READS.CH0_ISOCH", + "PerPkg": "1", + "UMask": "0x102", + "Unit": "M2M" + }, + { + "BriefDescription": "UNC_M2M_IMC_READS.CH0_NORMAL", + "EventCode": "0x24", + "EventName": "UNC_M2M_IMC_READS.CH0_NORMAL", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "UMask": "0x101", + "Unit": "M2M" + }, + { + "BriefDescription": "UNC_M2M_IMC_READS.CH0_TO_DDR_AS_CACHE", + "EventCode": "0x24", + "EventName": "UNC_M2M_IMC_READS.CH0_TO_DDR_AS_CACHE", + "PerPkg": "1", + "UMask": "0x110", + "Unit": "M2M" + }, + { + "BriefDescription": "UNC_M2M_IMC_READS.CH0_TO_DDR_AS_MEM", + "EventCode": "0x24", + "EventName": "UNC_M2M_IMC_READS.CH0_TO_DDR_AS_MEM", + "PerPkg": "1", + "UMask": "0x108", + "Unit": "M2M" + }, + { + "BriefDescription": "UNC_M2M_IMC_READS.CH0_TO_PMM", + "EventCode": "0x24", + "EventName": "UNC_M2M_IMC_READS.CH0_TO_PMM", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "UMask": "0x120", + "Unit": "M2M" + }, + { + "BriefDescription": "UNC_M2M_IMC_READS.CH1.TO_NM1LM", + "EventCode": "0x24", + "EventName": "UNC_M2M_IMC_READS.CH1.TO_NM1LM", + "PerPkg": "1", + "UMask": "0x208", + "Unit": "M2M" + }, + { + "BriefDescription": "UNC_M2M_IMC_READS.CH1.TO_NMCache", + "EventCode": "0x24", + "EventName": "UNC_M2M_IMC_READS.CH1.TO_NMCache", + "PerPkg": "1", + "UMask": "0x210", + "Unit": "M2M" + }, + { + "BriefDescription": "UNC_M2M_IMC_READS.CH1_ALL", + "EventCode": "0x24", + "EventName": "UNC_M2M_IMC_READS.CH1_ALL", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "UMask": "0x204", + "Unit": "M2M" + }, + { + "BriefDescription": "UNC_M2M_IMC_READS.CH1_FROM_TGR", + "EventCode": "0x24", + "EventName": "UNC_M2M_IMC_READS.CH1_FROM_TGR", + "PerPkg": "1", + "UMask": "0x240", + "Unit": "M2M" + }, + { + "BriefDescription": "UNC_M2M_IMC_READS.CH1_ISOCH", + "EventCode": "0x24", + "EventName": "UNC_M2M_IMC_READS.CH1_ISOCH", + "PerPkg": "1", + "UMask": "0x202", + "Unit": "M2M" + }, + { + "BriefDescription": "UNC_M2M_IMC_READS.CH1_NORMAL", + "EventCode": "0x24", + "EventName": "UNC_M2M_IMC_READS.CH1_NORMAL", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "UMask": "0x201", + "Unit": "M2M" + }, + { + "BriefDescription": "UNC_M2M_IMC_READS.CH1_TO_DDR_AS_CACHE", + "EventCode": "0x24", + "EventName": "UNC_M2M_IMC_READS.CH1_TO_DDR_AS_CACHE", + "PerPkg": "1", + "UMask": "0x210", + "Unit": "M2M" + }, + { + "BriefDescription": "UNC_M2M_IMC_READS.CH1_TO_DDR_AS_MEM", + "EventCode": "0x24", + "EventName": "UNC_M2M_IMC_READS.CH1_TO_DDR_AS_MEM", + "PerPkg": "1", + "UMask": "0x208", + "Unit": "M2M" + }, + { + "BriefDescription": "UNC_M2M_IMC_READS.CH1_TO_PMM", + "EventCode": "0x24", + "EventName": "UNC_M2M_IMC_READS.CH1_TO_PMM", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "UMask": "0x220", + "Unit": "M2M" + }, + { + "BriefDescription": "UNC_M2M_IMC_READS.FROM_TGR", + "EventCode": "0x24", + "EventName": "UNC_M2M_IMC_READS.FROM_TGR", + "PerPkg": "1", + "UMask": "0x340", + "Unit": "M2M" + }, + { + "BriefDescription": "UNC_M2M_IMC_READS.ISOCH", + "EventCode": "0x24", + "EventName": "UNC_M2M_IMC_READS.ISOCH", + "PerPkg": "1", + "UMask": "0x302", + "Unit": "M2M" + }, + { + "BriefDescription": "UNC_M2M_IMC_READS.NORMAL", + "EventCode": "0x24", + "EventName": "UNC_M2M_IMC_READS.NORMAL", + "PerPkg": "1", + "UMask": "0x301", + "Unit": "M2M" + }, + { + "BriefDescription": "UNC_M2M_IMC_READS.TO_DDR_AS_CACHE", + "EventCode": "0x24", + "EventName": "UNC_M2M_IMC_READS.TO_DDR_AS_CACHE", + "PerPkg": "1", + "UMask": "0x310", + "Unit": "M2M" + }, + { + "BriefDescription": "UNC_M2M_IMC_READS.TO_DDR_AS_MEM", + "EventCode": "0x24", + "EventName": "UNC_M2M_IMC_READS.TO_DDR_AS_MEM", + "PerPkg": "1", + "UMask": "0x308", + "Unit": "M2M" + }, + { + "BriefDescription": "UNC_M2M_IMC_READS.TO_NM1LM", + "EventCode": "0x24", + "EventName": "UNC_M2M_IMC_READS.TO_NM1LM", + "PerPkg": "1", + "UMask": "0x308", + "Unit": "M2M" + }, + { + "BriefDescription": "UNC_M2M_IMC_READS.TO_NMCACHE", + "EventCode": "0x24", + "EventName": "UNC_M2M_IMC_READS.TO_NMCACHE", + "PerPkg": "1", + "UMask": "0x310", + "Unit": "M2M" + }, + { + "BriefDescription": "UNC_M2M_IMC_READS.TO_PMM", + "EventCode": "0x24", + "EventName": "UNC_M2M_IMC_READS.TO_PMM", + "PerPkg": "1", + "UMask": "0x320", + "Unit": "M2M" + }, + { + "BriefDescription": "All Writes - All Channels", + "EventCode": "0x25", + "EventName": "UNC_M2M_IMC_WRITES.ALL", + "PerPkg": "1", + "UMask": "0x1810", + "Unit": "M2M" + }, + { + "BriefDescription": "Non-Inclusive - Ch0", + "EventCode": "0x25", + "EventName": "UNC_M2M_IMC_WRITES.CH0.NI", + "PerPkg": "1", + "Unit": "M2M" + }, + { + "BriefDescription": "UNC_M2M_IMC_WRITES.CH0_ALL", + "EventCode": "0x25", + "EventName": "UNC_M2M_IMC_WRITES.CH0_ALL", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "UMask": "0x810", + "Unit": "M2M" + }, + { + "BriefDescription": "From TGR - Ch0", + "EventCode": "0x25", + "EventName": "UNC_M2M_IMC_WRITES.CH0_FROM_TGR", + "PerPkg": "1", + "Unit": "M2M" + }, + { + "BriefDescription": "UNC_M2M_IMC_WRITES.CH0_FULL", + "EventCode": "0x25", + "EventName": "UNC_M2M_IMC_WRITES.CH0_FULL", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "UMask": "0x801", + "Unit": "M2M" + }, + { + "BriefDescription": "UNC_M2M_IMC_WRITES.CH0_FULL_ISOCH", + "EventCode": "0x25", + "EventName": "UNC_M2M_IMC_WRITES.CH0_FULL_ISOCH", + "PerPkg": "1", + "UMask": "0x804", + "Unit": "M2M" + }, + { + "BriefDescription": "Non-Inclusive - Ch0", + "EventCode": "0x25", + "EventName": "UNC_M2M_IMC_WRITES.CH0_NI", + "PerPkg": "1", + "Unit": "M2M" + }, + { + "BriefDescription": "Non-Inclusive Miss - Ch0", + "EventCode": "0x25", + "EventName": "UNC_M2M_IMC_WRITES.CH0_NI_MISS", + "PerPkg": "1", + "Unit": "M2M" + }, + { + "BriefDescription": "UNC_M2M_IMC_WRITES.CH0_PARTIAL", + "EventCode": "0x25", + "EventName": "UNC_M2M_IMC_WRITES.CH0_PARTIAL", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "UMask": "0x802", + "Unit": "M2M" + }, + { + "BriefDescription": "UNC_M2M_IMC_WRITES.CH0_PARTIAL_ISOCH", + "EventCode": "0x25", + "EventName": "UNC_M2M_IMC_WRITES.CH0_PARTIAL_ISOCH", + "PerPkg": "1", + "UMask": "0x808", + "Unit": "M2M" + }, + { + "BriefDescription": "DDR, acting as Cache - Ch0", + "EventCode": "0x25", + "EventName": "UNC_M2M_IMC_WRITES.CH0_TO_DDR_AS_CACHE", + "PerPkg": "1", + "UMask": "0x840", + "Unit": "M2M" + }, + { + "BriefDescription": "UNC_M2M_IMC_WRITES.CH0_TO_DDR_AS_MEM", + "EventCode": "0x25", + "EventName": "UNC_M2M_IMC_WRITES.CH0_TO_DDR_AS_MEM", + "PerPkg": "1", + "UMask": "0x820", + "Unit": "M2M" + }, + { + "BriefDescription": "PMM - Ch0", + "EventCode": "0x25", + "EventName": "UNC_M2M_IMC_WRITES.CH0_TO_PMM", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "PublicDescription": "PMM - Ch0 : Counts all PMM dimm writes requests(full line and partial) sent from M2M to iMC", + "UMask": "0x880", + "Unit": "M2M" + }, + { + "BriefDescription": "Non-Inclusive - Ch1", + "EventCode": "0x25", + "EventName": "UNC_M2M_IMC_WRITES.CH1.NI", + "PerPkg": "1", + "Unit": "M2M" + }, + { + "BriefDescription": "All Writes - Ch1", + "EventCode": "0x25", + "EventName": "UNC_M2M_IMC_WRITES.CH1_ALL", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "UMask": "0x1010", + "Unit": "M2M" + }, + { + "BriefDescription": "From TGR - Ch1", + "EventCode": "0x25", + "EventName": "UNC_M2M_IMC_WRITES.CH1_FROM_TGR", + "PerPkg": "1", + "Unit": "M2M" + }, + { + "BriefDescription": "Full Line Non-ISOCH - Ch1", + "EventCode": "0x25", + "EventName": "UNC_M2M_IMC_WRITES.CH1_FULL", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "UMask": "0x1001", + "Unit": "M2M" + }, + { + "BriefDescription": "ISOCH Full Line - Ch1", + "EventCode": "0x25", + "EventName": "UNC_M2M_IMC_WRITES.CH1_FULL_ISOCH", + "PerPkg": "1", + "UMask": "0x1004", + "Unit": "M2M" + }, + { + "BriefDescription": "Non-Inclusive - Ch1", + "EventCode": "0x25", + "EventName": "UNC_M2M_IMC_WRITES.CH1_NI", + "PerPkg": "1", + "Unit": "M2M" + }, + { + "BriefDescription": "Non-Inclusive Miss - Ch1", + "EventCode": "0x25", + "EventName": "UNC_M2M_IMC_WRITES.CH1_NI_MISS", + "PerPkg": "1", + "Unit": "M2M" + }, + { + "BriefDescription": "Partial Non-ISOCH - Ch1", + "EventCode": "0x25", + "EventName": "UNC_M2M_IMC_WRITES.CH1_PARTIAL", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "UMask": "0x1002", + "Unit": "M2M" + }, + { + "BriefDescription": "ISOCH Partial - Ch1", + "EventCode": "0x25", + "EventName": "UNC_M2M_IMC_WRITES.CH1_PARTIAL_ISOCH", + "PerPkg": "1", + "UMask": "0x1008", + "Unit": "M2M" + }, + { + "BriefDescription": "DDR, acting as Cache - Ch1", + "EventCode": "0x25", + "EventName": "UNC_M2M_IMC_WRITES.CH1_TO_DDR_AS_CACHE", + "PerPkg": "1", + "UMask": "0x1040", + "Unit": "M2M" + }, + { + "BriefDescription": "DDR - Ch1", + "EventCode": "0x25", + "EventName": "UNC_M2M_IMC_WRITES.CH1_TO_DDR_AS_MEM", + "PerPkg": "1", + "UMask": "0x1020", + "Unit": "M2M" + }, + { + "BriefDescription": "PMM - Ch1", + "EventCode": "0x25", + "EventName": "UNC_M2M_IMC_WRITES.CH1_TO_PMM", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "PublicDescription": "PMM - Ch1 : Counts all PMM dimm writes requests(full line and partial) sent from M2M to iMC", + "UMask": "0x1080", + "Unit": "M2M" + }, + { + "BriefDescription": "From TGR - All Channels", + "EventCode": "0x25", + "EventName": "UNC_M2M_IMC_WRITES.FROM_TGR", + "PerPkg": "1", + "Unit": "M2M" + }, + { + "BriefDescription": "Full Non-ISOCH - All Channels", + "EventCode": "0x25", + "EventName": "UNC_M2M_IMC_WRITES.FULL", + "PerPkg": "1", + "UMask": "0x1801", + "Unit": "M2M" + }, + { + "BriefDescription": "ISOCH Full Line - All Channels", + "EventCode": "0x25", + "EventName": "UNC_M2M_IMC_WRITES.FULL_ISOCH", + "PerPkg": "1", + "UMask": "0x1804", + "Unit": "M2M" + }, + { + "BriefDescription": "Non-Inclusive - All Channels", + "EventCode": "0x25", + "EventName": "UNC_M2M_IMC_WRITES.NI", + "PerPkg": "1", + "Unit": "M2M" + }, + { + "BriefDescription": "Non-Inclusive Miss - All Channels", + "EventCode": "0x25", + "EventName": "UNC_M2M_IMC_WRITES.NI_MISS", + "PerPkg": "1", + "Unit": "M2M" + }, + { + "BriefDescription": "Partial Non-ISOCH - All Channels", + "EventCode": "0x25", + "EventName": "UNC_M2M_IMC_WRITES.PARTIAL", + "PerPkg": "1", + "UMask": "0x1802", + "Unit": "M2M" + }, + { + "BriefDescription": "ISOCH Partial - All Channels", + "EventCode": "0x25", + "EventName": "UNC_M2M_IMC_WRITES.PARTIAL_ISOCH", + "PerPkg": "1", + "UMask": "0x1808", + "Unit": "M2M" + }, + { + "BriefDescription": "DDR, acting as Cache - All Channels", + "EventCode": "0x25", + "EventName": "UNC_M2M_IMC_WRITES.TO_DDR_AS_CACHE", + "PerPkg": "1", + "UMask": "0x1840", + "Unit": "M2M" + }, + { + "BriefDescription": "DDR - All Channels", + "EventCode": "0x25", + "EventName": "UNC_M2M_IMC_WRITES.TO_DDR_AS_MEM", + "PerPkg": "1", + "UMask": "0x1820", + "Unit": "M2M" + }, + { + "BriefDescription": "PMM - All Channels", + "EventCode": "0x25", + "EventName": "UNC_M2M_IMC_WRITES.TO_PMM", + "PerPkg": "1", + "UMask": "0x1880", + "Unit": "M2M" + }, + { + "BriefDescription": "UNC_M2M_PREFCAM_CIS_DROPS", + "EventCode": "0x5c", + "EventName": "UNC_M2M_PREFCAM_CIS_DROPS", + "PerPkg": "1", + "Unit": "M2M" + }, + { + "BriefDescription": "Data Prefetches Dropped", + "EventCode": "0x58", + "EventName": "UNC_M2M_PREFCAM_DEMAND_DROPS.CH0_UPI", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2M" + }, + { + "BriefDescription": "Data Prefetches Dropped", + "EventCode": "0x58", + "EventName": "UNC_M2M_PREFCAM_DEMAND_DROPS.CH0_XPT", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2M" + }, + { + "BriefDescription": "Data Prefetches Dropped", + "EventCode": "0x58", + "EventName": "UNC_M2M_PREFCAM_DEMAND_DROPS.CH1_UPI", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "M2M" + }, + { + "BriefDescription": "Data Prefetches Dropped", + "EventCode": "0x58", + "EventName": "UNC_M2M_PREFCAM_DEMAND_DROPS.CH1_XPT", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "M2M" + }, + { + "BriefDescription": "Data Prefetches Dropped : UPI - All Channels", + "EventCode": "0x58", + "EventName": "UNC_M2M_PREFCAM_DEMAND_DROPS.UPI_ALLCH", + "PerPkg": "1", + "UMask": "0xa", + "Unit": "M2M" + }, + { + "BriefDescription": "Data Prefetches Dropped", + "EventCode": "0x58", + "EventName": "UNC_M2M_PREFCAM_DEMAND_DROPS.XPT_ALLCH", + "PerPkg": "1", + "UMask": "0x5", + "Unit": "M2M" + }, + { + "BriefDescription": ": UPI - All Channels", + "EventCode": "0x5d", + "EventName": "UNC_M2M_PREFCAM_DEMAND_MERGE.UPI_ALLCH", + "PerPkg": "1", + "UMask": "0xa", + "Unit": "M2M" + }, + { + "BriefDescription": ": XPT - All Channels", + "EventCode": "0x5d", + "EventName": "UNC_M2M_PREFCAM_DEMAND_MERGE.XPT_ALLCH", + "PerPkg": "1", + "UMask": "0x5", + "Unit": "M2M" + }, + { + "BriefDescription": "Demands Not Merged with CAMed Prefetches", + "EventCode": "0x5E", + "EventName": "UNC_M2M_PREFCAM_DEMAND_NO_MERGE.RD_MERGED", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "M2M" + }, + { + "BriefDescription": "Demands Not Merged with CAMed Prefetches", + "EventCode": "0x5E", + "EventName": "UNC_M2M_PREFCAM_DEMAND_NO_MERGE.WR_MERGED", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "M2M" + }, + { + "BriefDescription": "Demands Not Merged with CAMed Prefetches", + "EventCode": "0x5E", + "EventName": "UNC_M2M_PREFCAM_DEMAND_NO_MERGE.WR_SQUASHED", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "M2M" + }, + { + "BriefDescription": "Prefetch CAM Inserts : UPI - Ch 0", + "EventCode": "0x56", + "EventName": "UNC_M2M_PREFCAM_INSERTS.CH0_UPI", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2M" + }, + { + "BriefDescription": "Prefetch CAM Inserts : XPT - Ch 0", + "EventCode": "0x56", + "EventName": "UNC_M2M_PREFCAM_INSERTS.CH0_XPT", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2M" + }, + { + "BriefDescription": "Prefetch CAM Inserts : UPI - Ch 1", + "EventCode": "0x56", + "EventName": "UNC_M2M_PREFCAM_INSERTS.CH1_UPI", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "M2M" + }, + { + "BriefDescription": "Prefetch CAM Inserts : XPT - Ch 1", + "EventCode": "0x56", + "EventName": "UNC_M2M_PREFCAM_INSERTS.CH1_XPT", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "M2M" + }, + { + "BriefDescription": "Prefetch CAM Inserts : UPI - All Channels", + "EventCode": "0x56", + "EventName": "UNC_M2M_PREFCAM_INSERTS.UPI_ALLCH", + "PerPkg": "1", + "UMask": "0xa", + "Unit": "M2M" + }, + { + "BriefDescription": "Prefetch CAM Inserts : XPT - All Channels", + "EventCode": "0x56", + "EventName": "UNC_M2M_PREFCAM_INSERTS.XPT_ALLCH", + "PerPkg": "1", + "PublicDescription": "Prefetch CAM Inserts : XPT -All Channels", + "UMask": "0x5", + "Unit": "M2M" + }, + { + "BriefDescription": "Prefetch CAM Occupancy : All Channels", + "EventCode": "0x54", + "EventName": "UNC_M2M_PREFCAM_OCCUPANCY.ALLCH", + "PerPkg": "1", + "UMask": "0x3", + "Unit": "M2M" + }, + { + "BriefDescription": "Prefetch CAM Occupancy : Channel 0", + "EventCode": "0x54", + "EventName": "UNC_M2M_PREFCAM_OCCUPANCY.CH0", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2M" + }, + { + "BriefDescription": "Prefetch CAM Occupancy : Channel 1", + "EventCode": "0x54", + "EventName": "UNC_M2M_PREFCAM_OCCUPANCY.CH1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2M" + }, + { + "BriefDescription": "All Channels", + "EventCode": "0x5F", + "EventName": "UNC_M2M_PREFCAM_RESP_MISS.ALLCH", + "PerPkg": "1", + "UMask": "0x3", + "Unit": "M2M" + }, + { + "BriefDescription": ": Channel 0", + "EventCode": "0x5f", + "EventName": "UNC_M2M_PREFCAM_RESP_MISS.CH0", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2M" + }, + { + "BriefDescription": ": Channel 1", + "EventCode": "0x5f", + "EventName": "UNC_M2M_PREFCAM_RESP_MISS.CH1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2M" + }, + { + "BriefDescription": "UNC_M2M_PREFCAM_RxC_DEALLOCS.1LM_POSTED", + "EventCode": "0x62", + "EventName": "UNC_M2M_PREFCAM_RxC_DEALLOCS.1LM_POSTED", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2M" + }, + { + "BriefDescription": "UNC_M2M_PREFCAM_RxC_DEALLOCS.CIS", + "EventCode": "0x62", + "EventName": "UNC_M2M_PREFCAM_RxC_DEALLOCS.CIS", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "M2M" + }, + { + "BriefDescription": "UNC_M2M_PREFCAM_RxC_DEALLOCS.PMM_MEMMODE_ACCEPT", + "EventCode": "0x62", + "EventName": "UNC_M2M_PREFCAM_RxC_DEALLOCS.PMM_MEMMODE_ACCEPT", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "M2M" + }, + { + "BriefDescription": "UNC_M2M_PREFCAM_RxC_DEALLOCS.SQUASHED", + "EventCode": "0x62", + "EventName": "UNC_M2M_PREFCAM_RxC_DEALLOCS.SQUASHED", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2M" + }, + { + "BriefDescription": "AD Ingress (from CMS) Occupancy - Prefetches", + "EventCode": "0x60", + "EventName": "UNC_M2M_PREFCAM_RxC_OCCUPANCY", + "PerPkg": "1", + "Unit": "M2M" + }, + { + "BriefDescription": "AD Ingress (from CMS) : AD Ingress (from CMS) Allocations", + "EventCode": "0x02", + "EventName": "UNC_M2M_RxC_AD_INSERTS", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2M" + }, + { + "BriefDescription": "AD Ingress (from CMS) Occupancy", + "EventCode": "0x03", + "EventName": "UNC_M2M_RxC_AD_OCCUPANCY", + "PerPkg": "1", + "Unit": "M2M" + }, + { + "BriefDescription": "Clean NearMem Read Hit", + "EventCode": "0x1F", + "EventName": "UNC_M2M_TAG_HIT.NM_RD_HIT_CLEAN", + "PerPkg": "1", + "PublicDescription": "Counts clean full line read hits (reads and RFOs).", + "UMask": "0x1", + "Unit": "M2M" + }, + { + "BriefDescription": "Dirty NearMem Read Hit", + "EventCode": "0x1F", + "EventName": "UNC_M2M_TAG_HIT.NM_RD_HIT_DIRTY", + "PerPkg": "1", + "PublicDescription": "Counts dirty full line read hits (reads and RFOs).", + "UMask": "0x2", + "Unit": "M2M" + }, + { + "BriefDescription": "Tag Hit : Clean NearMem Underfill Hit", + "EventCode": "0x1F", + "EventName": "UNC_M2M_TAG_HIT.NM_UFILL_HIT_CLEAN", + "PerPkg": "1", + "PublicDescription": "Tag Hit indicates when a request sent to the iMC hit in Near Memory. : Counts clean underfill hits due to a partial write", + "UMask": "0x4", + "Unit": "M2M" + }, + { + "BriefDescription": "Tag Hit : Dirty NearMem Underfill Hit", + "EventCode": "0x1F", + "EventName": "UNC_M2M_TAG_HIT.NM_UFILL_HIT_DIRTY", + "PerPkg": "1", + "PublicDescription": "Tag Hit indicates when a request sent to the iMC hit in Near Memory. : Counts dirty underfill read hits due to a partial write", + "UMask": "0x8", + "Unit": "M2M" + }, + { + "BriefDescription": "UNC_M2M_TAG_MISS", + "EventCode": "0x4b", + "EventName": "UNC_M2M_TAG_MISS", + "PerPkg": "1", + "UMask": "0x3", + "Unit": "M2M" + }, + { + "BriefDescription": "Number AD Ingress Credits", + "EventCode": "0x2e", + "EventName": "UNC_M2M_TGR_AD_CREDITS", + "PerPkg": "1", + "Unit": "M2M" + }, + { + "BriefDescription": "Number BL Ingress Credits", + "EventCode": "0x2f", + "EventName": "UNC_M2M_TGR_BL_CREDITS", + "PerPkg": "1", + "Unit": "M2M" + }, + { + "BriefDescription": "Tracker Inserts : Channel 0", + "EventCode": "0x32", + "EventName": "UNC_M2M_TRACKER_INSERTS.CH0", + "PerPkg": "1", + "UMask": "0x104", + "Unit": "M2M" + }, + { + "BriefDescription": "Tracker Inserts : Channel 1", + "EventCode": "0x32", + "EventName": "UNC_M2M_TRACKER_INSERTS.CH1", + "PerPkg": "1", + "UMask": "0x204", + "Unit": "M2M" + }, + { + "BriefDescription": "Tracker Occupancy : Channel 0", + "EventCode": "0x33", + "EventName": "UNC_M2M_TRACKER_OCCUPANCY.CH0", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2M" + }, + { + "BriefDescription": "Tracker Occupancy : Channel 1", + "EventCode": "0x33", + "EventName": "UNC_M2M_TRACKER_OCCUPANCY.CH1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2M" + }, + { + "BriefDescription": "WPQ Flush : Channel 0", + "EventCode": "0x42", + "EventName": "UNC_M2M_WPQ_FLUSH.CH0", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2M" + }, + { + "BriefDescription": "WPQ Flush : Channel 1", + "EventCode": "0x42", + "EventName": "UNC_M2M_WPQ_FLUSH.CH1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2M" + }, + { + "BriefDescription": "M2M->iMC WPQ Cycles w/Credits - Regular : Channel 0", + "EventCode": "0x37", + "EventName": "UNC_M2M_WPQ_NO_REG_CRD.CHN0", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2M" + }, + { + "BriefDescription": "M2M->iMC WPQ Cycles w/Credits - Regular : Channel 1", + "EventCode": "0x37", + "EventName": "UNC_M2M_WPQ_NO_REG_CRD.CHN1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2M" + }, + { + "BriefDescription": "M2M->iMC WPQ Cycles w/Credits - Special : Channel 0", + "EventCode": "0x38", + "EventName": "UNC_M2M_WPQ_NO_SPEC_CRD.CHN0", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2M" + }, + { + "BriefDescription": "M2M->iMC WPQ Cycles w/Credits - Special : Channel 1", + "EventCode": "0x38", + "EventName": "UNC_M2M_WPQ_NO_SPEC_CRD.CHN1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2M" + }, + { + "BriefDescription": "Write Tracker Inserts : Channel 0", + "EventCode": "0x40", + "EventName": "UNC_M2M_WR_TRACKER_INSERTS.CH0", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2M" + }, + { + "BriefDescription": "Write Tracker Inserts : Channel 1", + "EventCode": "0x40", + "EventName": "UNC_M2M_WR_TRACKER_INSERTS.CH1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2M" + }, + { + "BriefDescription": "Write Tracker Cycles Not Empty : Channel 0", + "EventCode": "0x35", + "EventName": "UNC_M2M_WR_TRACKER_NE.CH0", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2M" + }, + { + "BriefDescription": "Write Tracker Cycles Not Empty : Channel 1", + "EventCode": "0x35", + "EventName": "UNC_M2M_WR_TRACKER_NE.CH1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2M" + }, + { + "BriefDescription": "Write Tracker Cycles Not Empty : Mirror", + "EventCode": "0x35", + "EventName": "UNC_M2M_WR_TRACKER_NE.MIRR", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "M2M" + }, + { + "BriefDescription": "Write Tracker Cycles Not Empty", + "EventCode": "0x35", + "EventName": "UNC_M2M_WR_TRACKER_NE.MIRR_NONTGR", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "M2M" + }, + { + "BriefDescription": "Write Tracker Cycles Not Empty", + "EventCode": "0x35", + "EventName": "UNC_M2M_WR_TRACKER_NE.MIRR_PWR", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "M2M" + }, + { + "BriefDescription": "Write Tracker Non-Posted Inserts : Channel 0", + "EventCode": "0x4d", + "EventName": "UNC_M2M_WR_TRACKER_NONPOSTED_INSERTS.CH0", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2M" + }, + { + "BriefDescription": "Write Tracker Non-Posted Inserts : Channel 1", + "EventCode": "0x4d", + "EventName": "UNC_M2M_WR_TRACKER_NONPOSTED_INSERTS.CH1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2M" + }, + { + "BriefDescription": "Write Tracker Non-Posted Occupancy : Channel 0", + "EventCode": "0x4c", + "EventName": "UNC_M2M_WR_TRACKER_NONPOSTED_OCCUPANCY.CH0", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2M" + }, + { + "BriefDescription": "Write Tracker Non-Posted Occupancy : Channel 1", + "EventCode": "0x4c", + "EventName": "UNC_M2M_WR_TRACKER_NONPOSTED_OCCUPANCY.CH1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2M" + }, + { + "BriefDescription": "Write Tracker Posted Inserts : Channel 0", + "EventCode": "0x48", + "EventName": "UNC_M2M_WR_TRACKER_POSTED_INSERTS.CH0", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2M" + }, + { + "BriefDescription": "Write Tracker Posted Inserts : Channel 1", + "EventCode": "0x48", + "EventName": "UNC_M2M_WR_TRACKER_POSTED_INSERTS.CH1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2M" + }, + { + "BriefDescription": "Write Tracker Posted Occupancy : Channel 0", + "EventCode": "0x47", + "EventName": "UNC_M2M_WR_TRACKER_POSTED_OCCUPANCY.CH0", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2M" + }, + { + "BriefDescription": "Write Tracker Posted Occupancy : Channel 1", + "EventCode": "0x47", + "EventName": "UNC_M2M_WR_TRACKER_POSTED_OCCUPANCY.CH1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2M" + }, + { + "BriefDescription": "CBox AD Credits Empty : Requests", + "EventCode": "0x22", + "EventName": "UNC_M3UPI_CHA_AD_CREDITS_EMPTY.REQ", + "PerPkg": "1", + "PublicDescription": "CBox AD Credits Empty : Requests : No credits available to send to Cbox on the AD Ring (covers higher CBoxes)", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "CBox AD Credits Empty : Snoops", + "EventCode": "0x22", + "EventName": "UNC_M3UPI_CHA_AD_CREDITS_EMPTY.SNP", + "PerPkg": "1", + "PublicDescription": "CBox AD Credits Empty : Snoops : No credits available to send to Cbox on the AD Ring (covers higher CBoxes)", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "CBox AD Credits Empty : VNA Messages", + "EventCode": "0x22", + "EventName": "UNC_M3UPI_CHA_AD_CREDITS_EMPTY.VNA", + "PerPkg": "1", + "PublicDescription": "CBox AD Credits Empty : VNA Messages : No credits available to send to Cbox on the AD Ring (covers higher CBoxes)", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "CBox AD Credits Empty : Writebacks", + "EventCode": "0x22", + "EventName": "UNC_M3UPI_CHA_AD_CREDITS_EMPTY.WB", + "PerPkg": "1", + "PublicDescription": "CBox AD Credits Empty : Writebacks : No credits available to send to Cbox on the AD Ring (covers higher CBoxes)", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "M3UPI Clockticks", + "EventCode": "0x01", + "EventName": "UNC_M3UPI_CLOCKTICKS", + "PerPkg": "1", + "PublicDescription": "Number of M2UPI clock cycles while the event is enabled", + "Unit": "M3UPI" + }, + { + "BriefDescription": "M3UPI CMS Clockticks", + "EventCode": "0xc0", + "EventName": "UNC_M3UPI_CMS_CLOCKTICKS", + "PerPkg": "1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "D2C Sent", + "EventCode": "0x2b", + "EventName": "UNC_M3UPI_D2C_SENT", + "PerPkg": "1", + "PublicDescription": "D2C Sent : Count cases BL sends direct to core", + "Unit": "M3UPI" + }, + { + "BriefDescription": "D2U Sent", + "EventCode": "0x2a", + "EventName": "UNC_M3UPI_D2U_SENT", + "PerPkg": "1", + "PublicDescription": "D2U Sent : Cases where SMI3 sends D2U command", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Egress Blocking due to Ordering requirements : Down", + "EventCode": "0xba", + "EventName": "UNC_M3UPI_EGRESS_ORDERING.IV_SNOOPGO_DN", + "PerPkg": "1", + "PublicDescription": "Egress Blocking due to Ordering requirements : Down : Counts number of cycles IV was blocked in the TGR Egress due to SNP/GO Ordering requirements", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Egress Blocking due to Ordering requirements : Up", + "EventCode": "0xba", + "EventName": "UNC_M3UPI_EGRESS_ORDERING.IV_SNOOPGO_UP", + "PerPkg": "1", + "PublicDescription": "Egress Blocking due to Ordering requirements : Up : Counts number of cycles IV was blocked in the TGR Egress due to SNP/GO Ordering requirements", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "M2 BL Credits Empty : IIO0 and IIO1 share the same ring destination. (1 VN0 credit only)", + "EventCode": "0x23", + "EventName": "UNC_M3UPI_M2_BL_CREDITS_EMPTY.IIO1_NCB", + "PerPkg": "1", + "PublicDescription": "M2 BL Credits Empty : IIO0 and IIO1 share the same ring destination. (1 VN0 credit only) : No vn0 and vna credits available to send to M2", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "M2 BL Credits Empty : IIO2", + "EventCode": "0x23", + "EventName": "UNC_M3UPI_M2_BL_CREDITS_EMPTY.IIO2_NCB", + "PerPkg": "1", + "PublicDescription": "M2 BL Credits Empty : IIO2 : No vn0 and vna credits available to send to M2", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "M2 BL Credits Empty : IIO3", + "EventCode": "0x23", + "EventName": "UNC_M3UPI_M2_BL_CREDITS_EMPTY.IIO3_NCB", + "PerPkg": "1", + "PublicDescription": "M2 BL Credits Empty : IIO3 : No vn0 and vna credits available to send to M2", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "M2 BL Credits Empty : IIO4", + "EventCode": "0x23", + "EventName": "UNC_M3UPI_M2_BL_CREDITS_EMPTY.IIO4_NCB", + "PerPkg": "1", + "PublicDescription": "M2 BL Credits Empty : IIO4 : No vn0 and vna credits available to send to M2", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "M2 BL Credits Empty : IIO5", + "EventCode": "0x23", + "EventName": "UNC_M3UPI_M2_BL_CREDITS_EMPTY.IIO5_NCB", + "PerPkg": "1", + "PublicDescription": "M2 BL Credits Empty : IIO5 : No vn0 and vna credits available to send to M2", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "M2 BL Credits Empty : All IIO targets for NCS are in single mask. ORs them together", + "EventCode": "0x23", + "EventName": "UNC_M3UPI_M2_BL_CREDITS_EMPTY.NCS", + "PerPkg": "1", + "PublicDescription": "M2 BL Credits Empty : All IIO targets for NCS are in single mask. ORs them together : No vn0 and vna credits available to send to M2", + "UMask": "0x40", + "Unit": "M3UPI" + }, + { + "BriefDescription": "M2 BL Credits Empty : Selected M2p BL NCS credits", + "EventCode": "0x23", + "EventName": "UNC_M3UPI_M2_BL_CREDITS_EMPTY.NCS_SEL", + "PerPkg": "1", + "PublicDescription": "M2 BL Credits Empty : Selected M2p BL NCS credits : No vn0 and vna credits available to send to M2", + "UMask": "0x80", + "Unit": "M3UPI" + }, + { + "BriefDescription": "M2 BL Credits Empty : IIO5", + "EventCode": "0x23", + "EventName": "UNC_M3UPI_M2_BL_CREDITS_EMPTY.UBOX_NCB", + "PerPkg": "1", + "PublicDescription": "M2 BL Credits Empty : IIO5 : No vn0 and vna credits available to send to M2", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Multi Slot Flit Received : AD - Slot 0", + "EventCode": "0x3e", + "EventName": "UNC_M3UPI_MULTI_SLOT_RCVD.AD_SLOT0", + "PerPkg": "1", + "PublicDescription": "Multi Slot Flit Received : AD - Slot 0 : Multi slot flit received - S0, S1 and/or S2 populated (can use AK S0/S1 masks for AK allocations)", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Multi Slot Flit Received : AD - Slot 1", + "EventCode": "0x3e", + "EventName": "UNC_M3UPI_MULTI_SLOT_RCVD.AD_SLOT1", + "PerPkg": "1", + "PublicDescription": "Multi Slot Flit Received : AD - Slot 1 : Multi slot flit received - S0, S1 and/or S2 populated (can use AK S0/S1 masks for AK allocations)", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Multi Slot Flit Received : AD - Slot 2", + "EventCode": "0x3e", + "EventName": "UNC_M3UPI_MULTI_SLOT_RCVD.AD_SLOT2", + "PerPkg": "1", + "PublicDescription": "Multi Slot Flit Received : AD - Slot 2 : Multi slot flit received - S0, S1 and/or S2 populated (can use AK S0/S1 masks for AK allocations)", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Multi Slot Flit Received : AK - Slot 0", + "EventCode": "0x3e", + "EventName": "UNC_M3UPI_MULTI_SLOT_RCVD.AK_SLOT0", + "PerPkg": "1", + "PublicDescription": "Multi Slot Flit Received : AK - Slot 0 : Multi slot flit received - S0, S1 and/or S2 populated (can use AK S0/S1 masks for AK allocations)", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Multi Slot Flit Received : AK - Slot 2", + "EventCode": "0x3e", + "EventName": "UNC_M3UPI_MULTI_SLOT_RCVD.AK_SLOT2", + "PerPkg": "1", + "PublicDescription": "Multi Slot Flit Received : AK - Slot 2 : Multi slot flit received - S0, S1 and/or S2 populated (can use AK S0/S1 masks for AK allocations)", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Multi Slot Flit Received : BL - Slot 0", + "EventCode": "0x3e", + "EventName": "UNC_M3UPI_MULTI_SLOT_RCVD.BL_SLOT0", + "PerPkg": "1", + "PublicDescription": "Multi Slot Flit Received : BL - Slot 0 : Multi slot flit received - S0, S1 and/or S2 populated (can use AK S0/S1 masks for AK allocations)", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Lost Arb for VN0 : REQ on AD", + "EventCode": "0x4b", + "EventName": "UNC_M3UPI_RxC_ARB_LOST_VN0.AD_REQ", + "PerPkg": "1", + "PublicDescription": "Lost Arb for VN0 : REQ on AD : VN0 message requested but lost arbitration : Home (REQ) messages on AD. REQ is generally used to send requests, request responses, and snoop responses.", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Lost Arb for VN0 : RSP on AD", + "EventCode": "0x4b", + "EventName": "UNC_M3UPI_RxC_ARB_LOST_VN0.AD_RSP", + "PerPkg": "1", + "PublicDescription": "Lost Arb for VN0 : RSP on AD : VN0 message requested but lost arbitration : Response (RSP) messages on AD. RSP packets are used to transmit a variety of protocol flits including grants and completions (CMP).", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Lost Arb for VN0 : SNP on AD", + "EventCode": "0x4b", + "EventName": "UNC_M3UPI_RxC_ARB_LOST_VN0.AD_SNP", + "PerPkg": "1", + "PublicDescription": "Lost Arb for VN0 : SNP on AD : VN0 message requested but lost arbitration : Snoops (SNP) messages on AD. SNP is used for outgoing snoops.", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Lost Arb for VN0 : NCB on BL", + "EventCode": "0x4b", + "EventName": "UNC_M3UPI_RxC_ARB_LOST_VN0.BL_NCB", + "PerPkg": "1", + "PublicDescription": "Lost Arb for VN0 : NCB on BL : VN0 message requested but lost arbitration : Non-Coherent Broadcast (NCB) messages on BL. NCB is generally used to transmit data without coherency. For example, non-coherent read data returns.", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Lost Arb for VN0 : NCS on BL", + "EventCode": "0x4b", + "EventName": "UNC_M3UPI_RxC_ARB_LOST_VN0.BL_NCS", + "PerPkg": "1", + "PublicDescription": "Lost Arb for VN0 : NCS on BL : VN0 message requested but lost arbitration : Non-Coherent Standard (NCS) messages on BL.", + "UMask": "0x40", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Lost Arb for VN0 : RSP on BL", + "EventCode": "0x4b", + "EventName": "UNC_M3UPI_RxC_ARB_LOST_VN0.BL_RSP", + "PerPkg": "1", + "PublicDescription": "Lost Arb for VN0 : RSP on BL : VN0 message requested but lost arbitration : Response (RSP) messages on BL. RSP packets are used to transmit a variety of protocol flits including grants and completions (CMP).", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Lost Arb for VN0 : WB on BL", + "EventCode": "0x4b", + "EventName": "UNC_M3UPI_RxC_ARB_LOST_VN0.BL_WB", + "PerPkg": "1", + "PublicDescription": "Lost Arb for VN0 : WB on BL : VN0 message requested but lost arbitration : Data Response (WB) messages on BL. WB is generally used to transmit data with coherency. For example, remote reads and writes, or cache to cache transfers will transmit their data using WB.", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Lost Arb for VN1 : REQ on AD", + "EventCode": "0x4c", + "EventName": "UNC_M3UPI_RxC_ARB_LOST_VN1.AD_REQ", + "PerPkg": "1", + "PublicDescription": "Lost Arb for VN1 : REQ on AD : VN1 message requested but lost arbitration : Home (REQ) messages on AD. REQ is generally used to send requests, request responses, and snoop responses.", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Lost Arb for VN1 : RSP on AD", + "EventCode": "0x4c", + "EventName": "UNC_M3UPI_RxC_ARB_LOST_VN1.AD_RSP", + "PerPkg": "1", + "PublicDescription": "Lost Arb for VN1 : RSP on AD : VN1 message requested but lost arbitration : Response (RSP) messages on AD. RSP packets are used to transmit a variety of protocol flits including grants and completions (CMP).", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Lost Arb for VN1 : SNP on AD", + "EventCode": "0x4c", + "EventName": "UNC_M3UPI_RxC_ARB_LOST_VN1.AD_SNP", + "PerPkg": "1", + "PublicDescription": "Lost Arb for VN1 : SNP on AD : VN1 message requested but lost arbitration : Snoops (SNP) messages on AD. SNP is used for outgoing snoops.", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Lost Arb for VN1 : NCB on BL", + "EventCode": "0x4c", + "EventName": "UNC_M3UPI_RxC_ARB_LOST_VN1.BL_NCB", + "PerPkg": "1", + "PublicDescription": "Lost Arb for VN1 : NCB on BL : VN1 message requested but lost arbitration : Non-Coherent Broadcast (NCB) messages on BL. NCB is generally used to transmit data without coherency. For example, non-coherent read data returns.", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Lost Arb for VN1 : NCS on BL", + "EventCode": "0x4c", + "EventName": "UNC_M3UPI_RxC_ARB_LOST_VN1.BL_NCS", + "PerPkg": "1", + "PublicDescription": "Lost Arb for VN1 : NCS on BL : VN1 message requested but lost arbitration : Non-Coherent Standard (NCS) messages on BL.", + "UMask": "0x40", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Lost Arb for VN1 : RSP on BL", + "EventCode": "0x4c", + "EventName": "UNC_M3UPI_RxC_ARB_LOST_VN1.BL_RSP", + "PerPkg": "1", + "PublicDescription": "Lost Arb for VN1 : RSP on BL : VN1 message requested but lost arbitration : Response (RSP) messages on BL. RSP packets are used to transmit a variety of protocol flits including grants and completions (CMP).", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Lost Arb for VN1 : WB on BL", + "EventCode": "0x4c", + "EventName": "UNC_M3UPI_RxC_ARB_LOST_VN1.BL_WB", + "PerPkg": "1", + "PublicDescription": "Lost Arb for VN1 : WB on BL : VN1 message requested but lost arbitration : Data Response (WB) messages on BL. WB is generally used to transmit data with coherency. For example, remote reads and writes, or cache to cache transfers will transmit their data using WB.", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Arb Miscellaneous : AD, BL Parallel Win VN0", + "EventCode": "0x4d", + "EventName": "UNC_M3UPI_RxC_ARB_MISC.ADBL_PARALLEL_WIN_VN0", + "PerPkg": "1", + "PublicDescription": "Arb Miscellaneous : AD, BL Parallel Win VN0 : AD and BL messages won arbitration concurrently / in parallel", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Arb Miscellaneous : AD, BL Parallel Win VN1", + "EventCode": "0x4d", + "EventName": "UNC_M3UPI_RxC_ARB_MISC.ADBL_PARALLEL_WIN_VN1", + "PerPkg": "1", + "PublicDescription": "Arb Miscellaneous : AD, BL Parallel Win VN1 : AD and BL messages won arbitration concurrently / in parallel", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Arb Miscellaneous : Max Parallel Win", + "EventCode": "0x4d", + "EventName": "UNC_M3UPI_RxC_ARB_MISC.ALL_PARALLEL_WIN", + "PerPkg": "1", + "PublicDescription": "Arb Miscellaneous : Max Parallel Win : VN0 and VN1 arbitration sub-pipelines both produced AD and BL winners (maximum possible parallel winners)", + "UMask": "0x80", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Arb Miscellaneous : No Progress on Pending AD VN0", + "EventCode": "0x4d", + "EventName": "UNC_M3UPI_RxC_ARB_MISC.NO_PROG_AD_VN0", + "PerPkg": "1", + "PublicDescription": "Arb Miscellaneous : No Progress on Pending AD VN0 : Arbitration stage made no progress on pending ad vn0 messages because slotting stage cannot accept new message", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Arb Miscellaneous : No Progress on Pending AD VN1", + "EventCode": "0x4d", + "EventName": "UNC_M3UPI_RxC_ARB_MISC.NO_PROG_AD_VN1", + "PerPkg": "1", + "PublicDescription": "Arb Miscellaneous : No Progress on Pending AD VN1 : Arbitration stage made no progress on pending ad vn1 messages because slotting stage cannot accept new message", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Arb Miscellaneous : No Progress on Pending BL VN0", + "EventCode": "0x4d", + "EventName": "UNC_M3UPI_RxC_ARB_MISC.NO_PROG_BL_VN0", + "PerPkg": "1", + "PublicDescription": "Arb Miscellaneous : No Progress on Pending BL VN0 : Arbitration stage made no progress on pending bl vn0 messages because slotting stage cannot accept new message", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Arb Miscellaneous : No Progress on Pending BL VN1", + "EventCode": "0x4d", + "EventName": "UNC_M3UPI_RxC_ARB_MISC.NO_PROG_BL_VN1", + "PerPkg": "1", + "PublicDescription": "Arb Miscellaneous : No Progress on Pending BL VN1 : Arbitration stage made no progress on pending bl vn1 messages because slotting stage cannot accept new message", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Arb Miscellaneous : VN0, VN1 Parallel Win", + "EventCode": "0x4d", + "EventName": "UNC_M3UPI_RxC_ARB_MISC.VN01_PARALLEL_WIN", + "PerPkg": "1", + "PublicDescription": "Arb Miscellaneous : VN0, VN1 Parallel Win : VN0 and VN1 arbitration sub-pipelines had parallel winners (at least one AD or BL on each side)", + "UMask": "0x40", + "Unit": "M3UPI" + }, + { + "BriefDescription": "No Credits to Arb for VN0 : REQ on AD", + "EventCode": "0x47", + "EventName": "UNC_M3UPI_RxC_ARB_NOCRD_VN0.AD_REQ", + "PerPkg": "1", + "PublicDescription": "No Credits to Arb for VN0 : REQ on AD : VN0 message is blocked from requesting arbitration due to lack of remote UPI credits : Home (REQ) messages on AD. REQ is generally used to send requests, request responses, and snoop responses.", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "No Credits to Arb for VN0 : RSP on AD", + "EventCode": "0x47", + "EventName": "UNC_M3UPI_RxC_ARB_NOCRD_VN0.AD_RSP", + "PerPkg": "1", + "PublicDescription": "No Credits to Arb for VN0 : RSP on AD : VN0 message is blocked from requesting arbitration due to lack of remote UPI credits : Response (RSP) messages on AD. RSP packets are used to transmit a variety of protocol flits including grants and completions (CMP).", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "No Credits to Arb for VN0 : SNP on AD", + "EventCode": "0x47", + "EventName": "UNC_M3UPI_RxC_ARB_NOCRD_VN0.AD_SNP", + "PerPkg": "1", + "PublicDescription": "No Credits to Arb for VN0 : SNP on AD : VN0 message is blocked from requesting arbitration due to lack of remote UPI credits : Snoops (SNP) messages on AD. SNP is used for outgoing snoops.", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "No Credits to Arb for VN0 : NCB on BL", + "EventCode": "0x47", + "EventName": "UNC_M3UPI_RxC_ARB_NOCRD_VN0.BL_NCB", + "PerPkg": "1", + "PublicDescription": "No Credits to Arb for VN0 : NCB on BL : VN0 message is blocked from requesting arbitration due to lack of remote UPI credits : Non-Coherent Broadcast (NCB) messages on BL. NCB is generally used to transmit data without coherency. For example, non-coherent read data returns.", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "No Credits to Arb for VN0 : NCS on BL", + "EventCode": "0x47", + "EventName": "UNC_M3UPI_RxC_ARB_NOCRD_VN0.BL_NCS", + "PerPkg": "1", + "PublicDescription": "No Credits to Arb for VN0 : NCS on BL : VN0 message is blocked from requesting arbitration due to lack of remote UPI credits : Non-Coherent Standard (NCS) messages on BL.", + "UMask": "0x40", + "Unit": "M3UPI" + }, + { + "BriefDescription": "No Credits to Arb for VN0 : RSP on BL", + "EventCode": "0x47", + "EventName": "UNC_M3UPI_RxC_ARB_NOCRD_VN0.BL_RSP", + "PerPkg": "1", + "PublicDescription": "No Credits to Arb for VN0 : RSP on BL : VN0 message is blocked from requesting arbitration due to lack of remote UPI credits : Response (RSP) messages on BL. RSP packets are used to transmit a variety of protocol flits including grants and completions (CMP).", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "No Credits to Arb for VN0 : WB on BL", + "EventCode": "0x47", + "EventName": "UNC_M3UPI_RxC_ARB_NOCRD_VN0.BL_WB", + "PerPkg": "1", + "PublicDescription": "No Credits to Arb for VN0 : WB on BL : VN0 message is blocked from requesting arbitration due to lack of remote UPI credits : Data Response (WB) messages on BL. WB is generally used to transmit data with coherency. For example, remote reads and writes, or cache to cache transfers will transmit their data using WB.", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "No Credits to Arb for VN1 : REQ on AD", + "EventCode": "0x48", + "EventName": "UNC_M3UPI_RxC_ARB_NOCRD_VN1.AD_REQ", + "PerPkg": "1", + "PublicDescription": "No Credits to Arb for VN1 : REQ on AD : VN1 message is blocked from requesting arbitration due to lack of remote UPI credits : Home (REQ) messages on AD. REQ is generally used to send requests, request responses, and snoop responses.", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "No Credits to Arb for VN1 : RSP on AD", + "EventCode": "0x48", + "EventName": "UNC_M3UPI_RxC_ARB_NOCRD_VN1.AD_RSP", + "PerPkg": "1", + "PublicDescription": "No Credits to Arb for VN1 : RSP on AD : VN1 message is blocked from requesting arbitration due to lack of remote UPI credits : Response (RSP) messages on AD. RSP packets are used to transmit a variety of protocol flits including grants and completions (CMP).", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "No Credits to Arb for VN1 : SNP on AD", + "EventCode": "0x48", + "EventName": "UNC_M3UPI_RxC_ARB_NOCRD_VN1.AD_SNP", + "PerPkg": "1", + "PublicDescription": "No Credits to Arb for VN1 : SNP on AD : VN1 message is blocked from requesting arbitration due to lack of remote UPI credits : Snoops (SNP) messages on AD. SNP is used for outgoing snoops.", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "No Credits to Arb for VN1 : NCB on BL", + "EventCode": "0x48", + "EventName": "UNC_M3UPI_RxC_ARB_NOCRD_VN1.BL_NCB", + "PerPkg": "1", + "PublicDescription": "No Credits to Arb for VN1 : NCB on BL : VN1 message is blocked from requesting arbitration due to lack of remote UPI credits : Non-Coherent Broadcast (NCB) messages on BL. NCB is generally used to transmit data without coherency. For example, non-coherent read data returns.", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "No Credits to Arb for VN1 : NCS on BL", + "EventCode": "0x48", + "EventName": "UNC_M3UPI_RxC_ARB_NOCRD_VN1.BL_NCS", + "PerPkg": "1", + "PublicDescription": "No Credits to Arb for VN1 : NCS on BL : VN1 message is blocked from requesting arbitration due to lack of remote UPI credits : Non-Coherent Standard (NCS) messages on BL.", + "UMask": "0x40", + "Unit": "M3UPI" + }, + { + "BriefDescription": "No Credits to Arb for VN1 : RSP on BL", + "EventCode": "0x48", + "EventName": "UNC_M3UPI_RxC_ARB_NOCRD_VN1.BL_RSP", + "PerPkg": "1", + "PublicDescription": "No Credits to Arb for VN1 : RSP on BL : VN1 message is blocked from requesting arbitration due to lack of remote UPI credits : Response (RSP) messages on BL. RSP packets are used to transmit a variety of protocol flits including grants and completions (CMP).", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "No Credits to Arb for VN1 : WB on BL", + "EventCode": "0x48", + "EventName": "UNC_M3UPI_RxC_ARB_NOCRD_VN1.BL_WB", + "PerPkg": "1", + "PublicDescription": "No Credits to Arb for VN1 : WB on BL : VN1 message is blocked from requesting arbitration due to lack of remote UPI credits : Data Response (WB) messages on BL. WB is generally used to transmit data with coherency. For example, remote reads and writes, or cache to cache transfers will transmit their data using WB.", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Can't Arb for VN0 : REQ on AD", + "EventCode": "0x49", + "EventName": "UNC_M3UPI_RxC_ARB_NOREQ_VN0.AD_REQ", + "PerPkg": "1", + "PublicDescription": "Can't Arb for VN0 : REQ on AD : VN0 message was not able to request arbitration while some other message won arbitration : Home (REQ) messages on AD. REQ is generally used to send requests, request responses, and snoop responses.", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Can't Arb for VN0 : RSP on AD", + "EventCode": "0x49", + "EventName": "UNC_M3UPI_RxC_ARB_NOREQ_VN0.AD_RSP", + "PerPkg": "1", + "PublicDescription": "Can't Arb for VN0 : RSP on AD : VN0 message was not able to request arbitration while some other message won arbitration : Response (RSP) messages on AD. RSP packets are used to transmit a variety of protocol flits including grants and completions (CMP).", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Can't Arb for VN0 : SNP on AD", + "EventCode": "0x49", + "EventName": "UNC_M3UPI_RxC_ARB_NOREQ_VN0.AD_SNP", + "PerPkg": "1", + "PublicDescription": "Can't Arb for VN0 : SNP on AD : VN0 message was not able to request arbitration while some other message won arbitration : Snoops (SNP) messages on AD. SNP is used for outgoing snoops.", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Can't Arb for VN0 : NCB on BL", + "EventCode": "0x49", + "EventName": "UNC_M3UPI_RxC_ARB_NOREQ_VN0.BL_NCB", + "PerPkg": "1", + "PublicDescription": "Can't Arb for VN0 : NCB on BL : VN0 message was not able to request arbitration while some other message won arbitration : Non-Coherent Broadcast (NCB) messages on BL. NCB is generally used to transmit data without coherency. For example, non-coherent read data returns.", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Can't Arb for VN0 : NCS on BL", + "EventCode": "0x49", + "EventName": "UNC_M3UPI_RxC_ARB_NOREQ_VN0.BL_NCS", + "PerPkg": "1", + "PublicDescription": "Can't Arb for VN0 : NCS on BL : VN0 message was not able to request arbitration while some other message won arbitration : Non-Coherent Standard (NCS) messages on BL.", + "UMask": "0x40", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Can't Arb for VN0 : RSP on BL", + "EventCode": "0x49", + "EventName": "UNC_M3UPI_RxC_ARB_NOREQ_VN0.BL_RSP", + "PerPkg": "1", + "PublicDescription": "Can't Arb for VN0 : RSP on BL : VN0 message was not able to request arbitration while some other message won arbitration : Response (RSP) messages on BL. RSP packets are used to transmit a variety of protocol flits including grants and completions (CMP).", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Can't Arb for VN0 : WB on BL", + "EventCode": "0x49", + "EventName": "UNC_M3UPI_RxC_ARB_NOREQ_VN0.BL_WB", + "PerPkg": "1", + "PublicDescription": "Can't Arb for VN0 : WB on BL : VN0 message was not able to request arbitration while some other message won arbitration : Data Response (WB) messages on BL. WB is generally used to transmit data with coherency. For example, remote reads and writes, or cache to cache transfers will transmit their data using WB.", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Can't Arb for VN1 : REQ on AD", + "EventCode": "0x4a", + "EventName": "UNC_M3UPI_RxC_ARB_NOREQ_VN1.AD_REQ", + "PerPkg": "1", + "PublicDescription": "Can't Arb for VN1 : REQ on AD : VN1 message was not able to request arbitration while some other message won arbitration : Home (REQ) messages on AD. REQ is generally used to send requests, request responses, and snoop responses.", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Can't Arb for VN1 : RSP on AD", + "EventCode": "0x4a", + "EventName": "UNC_M3UPI_RxC_ARB_NOREQ_VN1.AD_RSP", + "PerPkg": "1", + "PublicDescription": "Can't Arb for VN1 : RSP on AD : VN1 message was not able to request arbitration while some other message won arbitration : Response (RSP) messages on AD. RSP packets are used to transmit a variety of protocol flits including grants and completions (CMP).", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Can't Arb for VN1 : SNP on AD", + "EventCode": "0x4a", + "EventName": "UNC_M3UPI_RxC_ARB_NOREQ_VN1.AD_SNP", + "PerPkg": "1", + "PublicDescription": "Can't Arb for VN1 : SNP on AD : VN1 message was not able to request arbitration while some other message won arbitration : Snoops (SNP) messages on AD. SNP is used for outgoing snoops.", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Can't Arb for VN1 : NCB on BL", + "EventCode": "0x4a", + "EventName": "UNC_M3UPI_RxC_ARB_NOREQ_VN1.BL_NCB", + "PerPkg": "1", + "PublicDescription": "Can't Arb for VN1 : NCB on BL : VN1 message was not able to request arbitration while some other message won arbitration : Non-Coherent Broadcast (NCB) messages on BL. NCB is generally used to transmit data without coherency. For example, non-coherent read data returns.", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Can't Arb for VN1 : NCS on BL", + "EventCode": "0x4a", + "EventName": "UNC_M3UPI_RxC_ARB_NOREQ_VN1.BL_NCS", + "PerPkg": "1", + "PublicDescription": "Can't Arb for VN1 : NCS on BL : VN1 message was not able to request arbitration while some other message won arbitration : Non-Coherent Standard (NCS) messages on BL.", + "UMask": "0x40", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Can't Arb for VN1 : RSP on BL", + "EventCode": "0x4a", + "EventName": "UNC_M3UPI_RxC_ARB_NOREQ_VN1.BL_RSP", + "PerPkg": "1", + "PublicDescription": "Can't Arb for VN1 : RSP on BL : VN1 message was not able to request arbitration while some other message won arbitration : Response (RSP) messages on BL. RSP packets are used to transmit a variety of protocol flits including grants and completions (CMP).", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Can't Arb for VN1 : WB on BL", + "EventCode": "0x4a", + "EventName": "UNC_M3UPI_RxC_ARB_NOREQ_VN1.BL_WB", + "PerPkg": "1", + "PublicDescription": "Can't Arb for VN1 : WB on BL : VN1 message was not able to request arbitration while some other message won arbitration : Data Response (WB) messages on BL. WB is generally used to transmit data with coherency. For example, remote reads and writes, or cache to cache transfers will transmit their data using WB.", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Ingress Queue Bypasses : AD to Slot 0 on BL Arb", + "EventCode": "0x40", + "EventName": "UNC_M3UPI_RxC_BYPASSED.AD_S0_BL_ARB", + "PerPkg": "1", + "PublicDescription": "Ingress Queue Bypasses : AD to Slot 0 on BL Arb : Number of times message is bypassed around the Ingress Queue : AD is taking bypass to slot 0 of independent flit while bl message is in arbitration", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Ingress Queue Bypasses : AD to Slot 0 on Idle", + "EventCode": "0x40", + "EventName": "UNC_M3UPI_RxC_BYPASSED.AD_S0_IDLE", + "PerPkg": "1", + "PublicDescription": "Ingress Queue Bypasses : AD to Slot 0 on Idle : Number of times message is bypassed around the Ingress Queue : AD is taking bypass to slot 0 of independent flit while pipeline is idle", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Ingress Queue Bypasses : AD + BL to Slot 1", + "EventCode": "0x40", + "EventName": "UNC_M3UPI_RxC_BYPASSED.AD_S1_BL_SLOT", + "PerPkg": "1", + "PublicDescription": "Ingress Queue Bypasses : AD + BL to Slot 1 : Number of times message is bypassed around the Ingress Queue : AD is taking bypass to flit slot 1 while merging with bl message in same flit", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Ingress Queue Bypasses : AD + BL to Slot 2", + "EventCode": "0x40", + "EventName": "UNC_M3UPI_RxC_BYPASSED.AD_S2_BL_SLOT", + "PerPkg": "1", + "PublicDescription": "Ingress Queue Bypasses : AD + BL to Slot 2 : Number of times message is bypassed around the Ingress Queue : AD is taking bypass to flit slot 2 while merging with bl message in same flit", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Miscellaneous Credit Events : Any In BGF FIFO", + "EventCode": "0x5f", + "EventName": "UNC_M3UPI_RxC_CRD_MISC.ANY_BGF_FIFO", + "PerPkg": "1", + "PublicDescription": "Miscellaneous Credit Events : Any In BGF FIFO : Indication that at least one packet (flit) is in the bgf (fifo only)", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Miscellaneous Credit Events : Any in BGF Path", + "EventCode": "0x5f", + "EventName": "UNC_M3UPI_RxC_CRD_MISC.ANY_BGF_PATH", + "PerPkg": "1", + "PublicDescription": "Miscellaneous Credit Events : Any in BGF Path : Indication that at least one packet (flit) is in the bgf path (i.e. pipe to fifo)", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Miscellaneous Credit Events", + "EventCode": "0x5f", + "EventName": "UNC_M3UPI_RxC_CRD_MISC.LT1_FOR_D2K", + "PerPkg": "1", + "PublicDescription": "Miscellaneous Credit Events : d2k credit count is less than 1", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Miscellaneous Credit Events", + "EventCode": "0x5f", + "EventName": "UNC_M3UPI_RxC_CRD_MISC.LT2_FOR_D2K", + "PerPkg": "1", + "PublicDescription": "Miscellaneous Credit Events : d2k credit count is less than 2", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Miscellaneous Credit Events : No D2K For Arb", + "EventCode": "0x5f", + "EventName": "UNC_M3UPI_RxC_CRD_MISC.VN0_NO_D2K_FOR_ARB", + "PerPkg": "1", + "PublicDescription": "Miscellaneous Credit Events : No D2K For Arb : VN0 BL RSP message was blocked from arbitration request due to lack of D2K CMP credit", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Miscellaneous Credit Events", + "EventCode": "0x5f", + "EventName": "UNC_M3UPI_RxC_CRD_MISC.VN1_NO_D2K_FOR_ARB", + "PerPkg": "1", + "PublicDescription": "Miscellaneous Credit Events : VN1 BL RSP message was blocked from arbitration request due to lack of D2K CMP credits", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Credit Occupancy : Credits Consumed", + "EventCode": "0x60", + "EventName": "UNC_M3UPI_RxC_CRD_OCC.CONSUMED", + "PerPkg": "1", + "PublicDescription": "Credit Occupancy : Credits Consumed : number of remote vna credits consumed per cycle", + "UMask": "0x80", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Credit Occupancy : D2K Credits", + "EventCode": "0x60", + "EventName": "UNC_M3UPI_RxC_CRD_OCC.D2K_CRD", + "PerPkg": "1", + "PublicDescription": "Credit Occupancy : D2K Credits : D2K completion fifo credit occupancy (credits in use), accumulated across all cycles", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Credit Occupancy : Packets in BGF FIFO", + "EventCode": "0x60", + "EventName": "UNC_M3UPI_RxC_CRD_OCC.FLITS_IN_FIFO", + "PerPkg": "1", + "PublicDescription": "Credit Occupancy : Packets in BGF FIFO : Occupancy of m3upi ingress -> upi link layer bgf; packets (flits) in fifo", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Credit Occupancy : Packets in BGF Path", + "EventCode": "0x60", + "EventName": "UNC_M3UPI_RxC_CRD_OCC.FLITS_IN_PATH", + "PerPkg": "1", + "PublicDescription": "Credit Occupancy : Packets in BGF Path : Occupancy of m3upi ingress -> upi link layer bgf; packets (flits) in path (i.e. pipe to fifo or fifo)", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Credit Occupancy", + "EventCode": "0x60", + "EventName": "UNC_M3UPI_RxC_CRD_OCC.P1P_FIFO", + "PerPkg": "1", + "PublicDescription": "Credit Occupancy : count of bl messages in pump-1-pending state, in completion fifo only", + "UMask": "0x40", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Credit Occupancy", + "EventCode": "0x60", + "EventName": "UNC_M3UPI_RxC_CRD_OCC.P1P_TOTAL", + "PerPkg": "1", + "PublicDescription": "Credit Occupancy : count of bl messages in pump-1-pending state, in marker table and in fifo", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Credit Occupancy : Transmit Credits", + "EventCode": "0x60", + "EventName": "UNC_M3UPI_RxC_CRD_OCC.TxQ_CRD", + "PerPkg": "1", + "PublicDescription": "Credit Occupancy : Transmit Credits : Link layer transmit queue credit occupancy (credits in use), accumulated across all cycles", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Credit Occupancy : VNA In Use", + "EventCode": "0x60", + "EventName": "UNC_M3UPI_RxC_CRD_OCC.VNA_IN_USE", + "PerPkg": "1", + "PublicDescription": "Credit Occupancy : VNA In Use : Remote UPI VNA credit occupancy (number of credits in use), accumulated across all cycles", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN0 Ingress (from CMS) Queue - Cycles Not Empty : REQ on AD", + "EventCode": "0x43", + "EventName": "UNC_M3UPI_RxC_CYCLES_NE_VN0.AD_REQ", + "PerPkg": "1", + "PublicDescription": "VN0 Ingress (from CMS) Queue - Cycles Not Empty : REQ on AD : Counts the number of cycles when the UPI Ingress is not empty. This tracks one of the three rings that are used by the UPI agent. This can be used in conjunction with the UPI Ingress Occupancy Accumulator event in order to calculate average queue occupancy. Multiple ingress buffers can be tracked at a given time using multiple counters. : Home (REQ) messages on AD. REQ is generally used to send requests, request responses, and snoop responses.", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN0 Ingress (from CMS) Queue - Cycles Not Empty : RSP on AD", + "EventCode": "0x43", + "EventName": "UNC_M3UPI_RxC_CYCLES_NE_VN0.AD_RSP", + "PerPkg": "1", + "PublicDescription": "VN0 Ingress (from CMS) Queue - Cycles Not Empty : RSP on AD : Counts the number of cycles when the UPI Ingress is not empty. This tracks one of the three rings that are used by the UPI agent. This can be used in conjunction with the UPI Ingress Occupancy Accumulator event in order to calculate average queue occupancy. Multiple ingress buffers can be tracked at a given time using multiple counters. : Response (RSP) messages on AD. RSP packets are used to transmit a variety of protocol flits including grants and completions (CMP).", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN0 Ingress (from CMS) Queue - Cycles Not Empty : SNP on AD", + "EventCode": "0x43", + "EventName": "UNC_M3UPI_RxC_CYCLES_NE_VN0.AD_SNP", + "PerPkg": "1", + "PublicDescription": "VN0 Ingress (from CMS) Queue - Cycles Not Empty : SNP on AD : Counts the number of cycles when the UPI Ingress is not empty. This tracks one of the three rings that are used by the UPI agent. This can be used in conjunction with the UPI Ingress Occupancy Accumulator event in order to calculate average queue occupancy. Multiple ingress buffers can be tracked at a given time using multiple counters. : Snoops (SNP) messages on AD. SNP is used for outgoing snoops.", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN0 Ingress (from CMS) Queue - Cycles Not Empty : NCB on BL", + "EventCode": "0x43", + "EventName": "UNC_M3UPI_RxC_CYCLES_NE_VN0.BL_NCB", + "PerPkg": "1", + "PublicDescription": "VN0 Ingress (from CMS) Queue - Cycles Not Empty : NCB on BL : Counts the number of cycles when the UPI Ingress is not empty. This tracks one of the three rings that are used by the UPI agent. This can be used in conjunction with the UPI Ingress Occupancy Accumulator event in order to calculate average queue occupancy. Multiple ingress buffers can be tracked at a given time using multiple counters. : Non-Coherent Broadcast (NCB) messages on BL. NCB is generally used to transmit data without coherency. For example, non-coherent read data returns.", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN0 Ingress (from CMS) Queue - Cycles Not Empty : NCS on BL", + "EventCode": "0x43", + "EventName": "UNC_M3UPI_RxC_CYCLES_NE_VN0.BL_NCS", + "PerPkg": "1", + "PublicDescription": "VN0 Ingress (from CMS) Queue - Cycles Not Empty : NCS on BL : Counts the number of cycles when the UPI Ingress is not empty. This tracks one of the three rings that are used by the UPI agent. This can be used in conjunction with the UPI Ingress Occupancy Accumulator event in order to calculate average queue occupancy. Multiple ingress buffers can be tracked at a given time using multiple counters. : Non-Coherent Standard (NCS) messages on BL.", + "UMask": "0x40", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN0 Ingress (from CMS) Queue - Cycles Not Empty : RSP on BL", + "EventCode": "0x43", + "EventName": "UNC_M3UPI_RxC_CYCLES_NE_VN0.BL_RSP", + "PerPkg": "1", + "PublicDescription": "VN0 Ingress (from CMS) Queue - Cycles Not Empty : RSP on BL : Counts the number of cycles when the UPI Ingress is not empty. This tracks one of the three rings that are used by the UPI agent. This can be used in conjunction with the UPI Ingress Occupancy Accumulator event in order to calculate average queue occupancy. Multiple ingress buffers can be tracked at a given time using multiple counters. : Response (RSP) messages on BL. RSP packets are used to transmit a variety of protocol flits including grants and completions (CMP).", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN0 Ingress (from CMS) Queue - Cycles Not Empty : WB on BL", + "EventCode": "0x43", + "EventName": "UNC_M3UPI_RxC_CYCLES_NE_VN0.BL_WB", + "PerPkg": "1", + "PublicDescription": "VN0 Ingress (from CMS) Queue - Cycles Not Empty : WB on BL : Counts the number of cycles when the UPI Ingress is not empty. This tracks one of the three rings that are used by the UPI agent. This can be used in conjunction with the UPI Ingress Occupancy Accumulator event in order to calculate average queue occupancy. Multiple ingress buffers can be tracked at a given time using multiple counters. : Data Response (WB) messages on BL. WB is generally used to transmit data with coherency. For example, remote reads and writes, or cache to cache transfers will transmit their data using WB.", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Data Flit Not Sent : All", + "EventCode": "0x55", + "EventName": "UNC_M3UPI_RxC_DATA_FLITS_NOT_SENT.ALL", + "PerPkg": "1", + "PublicDescription": "Data Flit Not Sent : All : Data flit is ready for transmission but could not be sent : data flit is ready for transmission but could not be sent for any reason, e.g. low credits, low tsv, stall injection", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Data Flit Not Sent : No BGF Credits", + "EventCode": "0x55", + "EventName": "UNC_M3UPI_RxC_DATA_FLITS_NOT_SENT.NO_BGF", + "PerPkg": "1", + "PublicDescription": "Data Flit Not Sent : No BGF Credits : Data flit is ready for transmission but could not be sent", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Data Flit Not Sent : No TxQ Credits", + "EventCode": "0x55", + "EventName": "UNC_M3UPI_RxC_DATA_FLITS_NOT_SENT.NO_TXQ", + "PerPkg": "1", + "PublicDescription": "Data Flit Not Sent : No TxQ Credits : Data flit is ready for transmission but could not be sent", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Data Flit Not Sent : TSV High", + "EventCode": "0x55", + "EventName": "UNC_M3UPI_RxC_DATA_FLITS_NOT_SENT.TSV_HI", + "PerPkg": "1", + "PublicDescription": "Data Flit Not Sent : TSV High : Data flit is ready for transmission but could not be sent : data flit is ready for transmission but was not sent while tsv high", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Data Flit Not Sent : Cycle valid for Flit", + "EventCode": "0x55", + "EventName": "UNC_M3UPI_RxC_DATA_FLITS_NOT_SENT.VALID_FOR_FLIT", + "PerPkg": "1", + "PublicDescription": "Data Flit Not Sent : Cycle valid for Flit : Data flit is ready for transmission but could not be sent : data flit is ready for transmission but was not sent while cycle is valid for flit transmission", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Generating BL Data Flit Sequence : Wait on Pump 0", + "EventCode": "0x57", + "EventName": "UNC_M3UPI_RxC_FLITS_GEN_BL.P0_WAIT", + "PerPkg": "1", + "PublicDescription": "Generating BL Data Flit Sequence : Wait on Pump 0 : generating bl data flit sequence; waiting for data pump 0", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Generating BL Data Flit Sequence", + "EventCode": "0x57", + "EventName": "UNC_M3UPI_RxC_FLITS_GEN_BL.P1P_AT_LIMIT", + "PerPkg": "1", + "PublicDescription": "Generating BL Data Flit Sequence : pump-1-pending logic is at capacity (pending table plus completion fifo at limit)", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Generating BL Data Flit Sequence", + "EventCode": "0x57", + "EventName": "UNC_M3UPI_RxC_FLITS_GEN_BL.P1P_BUSY", + "PerPkg": "1", + "PublicDescription": "Generating BL Data Flit Sequence : pump-1-pending logic is tracking at least one message", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Generating BL Data Flit Sequence", + "EventCode": "0x57", + "EventName": "UNC_M3UPI_RxC_FLITS_GEN_BL.P1P_FIFO_FULL", + "PerPkg": "1", + "PublicDescription": "Generating BL Data Flit Sequence : pump-1-pending completion fifo is full", + "UMask": "0x40", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Generating BL Data Flit Sequence", + "EventCode": "0x57", + "EventName": "UNC_M3UPI_RxC_FLITS_GEN_BL.P1P_HOLD_P0", + "PerPkg": "1", + "PublicDescription": "Generating BL Data Flit Sequence : pump-1-pending logic is at or near capacity, such that pump-0-only bl messages are getting stalled in slotting stage", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Generating BL Data Flit Sequence", + "EventCode": "0x57", + "EventName": "UNC_M3UPI_RxC_FLITS_GEN_BL.P1P_TO_LIMBO", + "PerPkg": "1", + "PublicDescription": "Generating BL Data Flit Sequence : a bl message finished but is in limbo and moved to pump-1-pending logic", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Generating BL Data Flit Sequence : Wait on Pump 1", + "EventCode": "0x57", + "EventName": "UNC_M3UPI_RxC_FLITS_GEN_BL.P1_WAIT", + "PerPkg": "1", + "PublicDescription": "Generating BL Data Flit Sequence : Wait on Pump 1 : generating bl data flit sequence; waiting for data pump 1", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UNC_M3UPI_RxC_FLITS_MISC.S2REQ_IN_HOLDOFF", + "EventCode": "0x58", + "EventName": "UNC_M3UPI_RxC_FLITS_MISC.S2REQ_IN_HOLDOFF", + "PerPkg": "1", + "PublicDescription": ": slot 2 request naturally serviced during hold-off period", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UNC_M3UPI_RxC_FLITS_MISC.S2REQ_IN_SERVICE", + "EventCode": "0x58", + "EventName": "UNC_M3UPI_RxC_FLITS_MISC.S2REQ_IN_SERVICE", + "PerPkg": "1", + "PublicDescription": ": slot 2 request forcibly serviced during service window", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UNC_M3UPI_RxC_FLITS_MISC.S2REQ_RECEIVED", + "EventCode": "0x58", + "EventName": "UNC_M3UPI_RxC_FLITS_MISC.S2REQ_RECEIVED", + "PerPkg": "1", + "PublicDescription": ": slot 2 request received from link layer while idle (with no slot 2 request active immediately prior)", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UNC_M3UPI_RxC_FLITS_MISC.S2REQ_WITHDRAWN", + "EventCode": "0x58", + "EventName": "UNC_M3UPI_RxC_FLITS_MISC.S2REQ_WITHDRAWN", + "PerPkg": "1", + "PublicDescription": ": slot 2 request withdrawn during hold-off period or service window", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Slotting BL Message Into Header Flit : All", + "EventCode": "0x56", + "EventName": "UNC_M3UPI_RxC_FLITS_SLOT_BL.ALL", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Slotting BL Message Into Header Flit : Needs Data Flit", + "EventCode": "0x56", + "EventName": "UNC_M3UPI_RxC_FLITS_SLOT_BL.NEED_DATA", + "PerPkg": "1", + "PublicDescription": "Slotting BL Message Into Header Flit : Needs Data Flit : BL message requires data flit sequence", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Slotting BL Message Into Header Flit : Wait on Pump 0", + "EventCode": "0x56", + "EventName": "UNC_M3UPI_RxC_FLITS_SLOT_BL.P0_WAIT", + "PerPkg": "1", + "PublicDescription": "Slotting BL Message Into Header Flit : Wait on Pump 0 : Waiting for header pump 0", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Slotting BL Message Into Header Flit : Don't Need Pump 1", + "EventCode": "0x56", + "EventName": "UNC_M3UPI_RxC_FLITS_SLOT_BL.P1_NOT_REQ", + "PerPkg": "1", + "PublicDescription": "Slotting BL Message Into Header Flit : Don't Need Pump 1 : Header pump 1 is not required for flit", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Slotting BL Message Into Header Flit : Don't Need Pump 1 - Bubble", + "EventCode": "0x56", + "EventName": "UNC_M3UPI_RxC_FLITS_SLOT_BL.P1_NOT_REQ_BUT_BUBBLE", + "PerPkg": "1", + "PublicDescription": "Slotting BL Message Into Header Flit : Don't Need Pump 1 - Bubble : Header pump 1 is not required for flit but flit transmission delayed", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Slotting BL Message Into Header Flit : Don't Need Pump 1 - Not Avail", + "EventCode": "0x56", + "EventName": "UNC_M3UPI_RxC_FLITS_SLOT_BL.P1_NOT_REQ_NOT_AVAIL", + "PerPkg": "1", + "PublicDescription": "Slotting BL Message Into Header Flit : Don't Need Pump 1 - Not Avail : Header pump 1 is not required for flit and not available", + "UMask": "0x40", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Slotting BL Message Into Header Flit : Wait on Pump 1", + "EventCode": "0x56", + "EventName": "UNC_M3UPI_RxC_FLITS_SLOT_BL.P1_WAIT", + "PerPkg": "1", + "PublicDescription": "Slotting BL Message Into Header Flit : Wait on Pump 1 : Waiting for header pump 1", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Flit Gen - Header 1 : Accumulate", + "EventCode": "0x51", + "EventName": "UNC_M3UPI_RxC_FLIT_GEN_HDR1.ACCUM", + "PerPkg": "1", + "PublicDescription": "Flit Gen - Header 1 : Accumulate : Events related to Header Flit Generation - Set 1 : Header flit slotting control state machine is in any accumulate state; multi-message flit may be assembled over multiple cycles", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Flit Gen - Header 1 : Accumulate Ready", + "EventCode": "0x51", + "EventName": "UNC_M3UPI_RxC_FLIT_GEN_HDR1.ACCUM_READ", + "PerPkg": "1", + "PublicDescription": "Flit Gen - Header 1 : Accumulate Ready : Events related to Header Flit Generation - Set 1 : header flit slotting control state machine is in accum_ready state; flit is ready to send but transmission is blocked; more messages may be slotted into flit", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Flit Gen - Header 1 : Accumulate Wasted", + "EventCode": "0x51", + "EventName": "UNC_M3UPI_RxC_FLIT_GEN_HDR1.ACCUM_WASTED", + "PerPkg": "1", + "PublicDescription": "Flit Gen - Header 1 : Accumulate Wasted : Events related to Header Flit Generation - Set 1 : Flit is being assembled over multiple cycles, but no additional message is being slotted into flit in current cycle; accumulate cycle is wasted", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Flit Gen - Header 1 : Run-Ahead - Blocked", + "EventCode": "0x51", + "EventName": "UNC_M3UPI_RxC_FLIT_GEN_HDR1.AHEAD_BLOCKED", + "PerPkg": "1", + "PublicDescription": "Flit Gen - Header 1 : Run-Ahead - Blocked : Events related to Header Flit Generation - Set 1 : Header flit slotting entered run-ahead state; new header flit is started while transmission of prior, fully assembled flit is blocked", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Flit Gen - Header 1", + "EventCode": "0x51", + "EventName": "UNC_M3UPI_RxC_FLIT_GEN_HDR1.AHEAD_MSG1_AFTER", + "PerPkg": "1", + "PublicDescription": "Flit Gen - Header 1 : Events related to Header Flit Generation - Set 1 : run-ahead mode: message was slotted only after run-ahead was over; run-ahead mode definitely wasted", + "UMask": "0x80", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Flit Gen - Header 1 : Run-Ahead - Message", + "EventCode": "0x51", + "EventName": "UNC_M3UPI_RxC_FLIT_GEN_HDR1.AHEAD_MSG1_DURING", + "PerPkg": "1", + "PublicDescription": "Flit Gen - Header 1 : Run-Ahead - Message : Events related to Header Flit Generation - Set 1 : run-ahead mode: one message slotted during run-ahead", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Flit Gen - Header 1", + "EventCode": "0x51", + "EventName": "UNC_M3UPI_RxC_FLIT_GEN_HDR1.AHEAD_MSG2_AFTER", + "PerPkg": "1", + "PublicDescription": "Flit Gen - Header 1 : Events related to Header Flit Generation - Set 1 : run-ahead mode: second message slotted immediately after run-ahead; potential run-ahead success", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Flit Gen - Header 1", + "EventCode": "0x51", + "EventName": "UNC_M3UPI_RxC_FLIT_GEN_HDR1.AHEAD_MSG2_SENT", + "PerPkg": "1", + "PublicDescription": "Flit Gen - Header 1 : Events related to Header Flit Generation - Set 1 : run-ahead mode: two (or three) message flit sent immediately after run-ahead; complete run-ahead success", + "UMask": "0x40", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Flit Gen - Header 2 : Parallel Ok", + "EventCode": "0x52", + "EventName": "UNC_M3UPI_RxC_FLIT_GEN_HDR2.PAR", + "PerPkg": "1", + "PublicDescription": "Flit Gen - Header 2 : Parallel Ok : Events related to Header Flit Generation - Set 2 : new header flit construction may proceed in parallel with data flit sequence", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Flit Gen - Header 2 : Parallel Flit Finished", + "EventCode": "0x52", + "EventName": "UNC_M3UPI_RxC_FLIT_GEN_HDR2.PAR_FLIT", + "PerPkg": "1", + "PublicDescription": "Flit Gen - Header 2 : Parallel Flit Finished : Events related to Header Flit Generation - Set 2 : header flit finished assembly in parallel with data flit sequence", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Flit Gen - Header 2 : Parallel Message", + "EventCode": "0x52", + "EventName": "UNC_M3UPI_RxC_FLIT_GEN_HDR2.PAR_MSG", + "PerPkg": "1", + "PublicDescription": "Flit Gen - Header 2 : Parallel Message : Events related to Header Flit Generation - Set 2 : message is slotted into header flit in parallel with data flit sequence", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Flit Gen - Header 2 : Rate-matching Stall", + "EventCode": "0x52", + "EventName": "UNC_M3UPI_RxC_FLIT_GEN_HDR2.RMSTALL", + "PerPkg": "1", + "PublicDescription": "Flit Gen - Header 2 : Rate-matching Stall : Events related to Header Flit Generation - Set 2 : Rate-matching stall injected", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Flit Gen - Header 2 : Rate-matching Stall - No Message", + "EventCode": "0x52", + "EventName": "UNC_M3UPI_RxC_FLIT_GEN_HDR2.RMSTALL_NOMSG", + "PerPkg": "1", + "PublicDescription": "Flit Gen - Header 2 : Rate-matching Stall - No Message : Events related to Header Flit Generation - Set 2 : Rate matching stall injected, but no additional message slotted during stall cycle", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Sent Header Flit : One Message", + "EventCode": "0x54", + "EventName": "UNC_M3UPI_RxC_HDR_FLITS_SENT.1_MSG", + "PerPkg": "1", + "PublicDescription": "Sent Header Flit : One Message : One message in flit; VNA or non-VNA flit", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Sent Header Flit : One Message in non-VNA", + "EventCode": "0x54", + "EventName": "UNC_M3UPI_RxC_HDR_FLITS_SENT.1_MSG_VNX", + "PerPkg": "1", + "PublicDescription": "Sent Header Flit : One Message in non-VNA : One message in flit; non-VNA flit", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Sent Header Flit : Two Messages", + "EventCode": "0x54", + "EventName": "UNC_M3UPI_RxC_HDR_FLITS_SENT.2_MSGS", + "PerPkg": "1", + "PublicDescription": "Sent Header Flit : Two Messages : Two messages in flit; VNA flit", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Sent Header Flit : Three Messages", + "EventCode": "0x54", + "EventName": "UNC_M3UPI_RxC_HDR_FLITS_SENT.3_MSGS", + "PerPkg": "1", + "PublicDescription": "Sent Header Flit : Three Messages : Three messages in flit; VNA flit", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Sent Header Flit : One Slot Taken", + "EventCode": "0x54", + "EventName": "UNC_M3UPI_RxC_HDR_FLITS_SENT.SLOTS_1", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Sent Header Flit : Two Slots Taken", + "EventCode": "0x54", + "EventName": "UNC_M3UPI_RxC_HDR_FLITS_SENT.SLOTS_2", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Sent Header Flit : All Slots Taken", + "EventCode": "0x54", + "EventName": "UNC_M3UPI_RxC_HDR_FLITS_SENT.SLOTS_3", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Header Not Sent : All", + "EventCode": "0x53", + "EventName": "UNC_M3UPI_RxC_HDR_FLIT_NOT_SENT.ALL", + "PerPkg": "1", + "PublicDescription": "Header Not Sent : All : header flit is ready for transmission but could not be sent : header flit is ready for transmission but could not be sent for any reason, e.g. no credits, low tsv, stall injection", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Header Not Sent : No BGF Credits", + "EventCode": "0x53", + "EventName": "UNC_M3UPI_RxC_HDR_FLIT_NOT_SENT.NO_BGF_CRD", + "PerPkg": "1", + "PublicDescription": "Header Not Sent : No BGF Credits : header flit is ready for transmission but could not be sent : No BGF credits available", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Header Not Sent : No BGF Credits + No Extra Message Slotted", + "EventCode": "0x53", + "EventName": "UNC_M3UPI_RxC_HDR_FLIT_NOT_SENT.NO_BGF_NO_MSG", + "PerPkg": "1", + "PublicDescription": "Header Not Sent : No BGF Credits + No Extra Message Slotted : header flit is ready for transmission but could not be sent : No BGF credits available; no additional message slotted into flit", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Header Not Sent : No TxQ Credits", + "EventCode": "0x53", + "EventName": "UNC_M3UPI_RxC_HDR_FLIT_NOT_SENT.NO_TXQ_CRD", + "PerPkg": "1", + "PublicDescription": "Header Not Sent : No TxQ Credits : header flit is ready for transmission but could not be sent : No TxQ credits available", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Header Not Sent : No TxQ Credits + No Extra Message Slotted", + "EventCode": "0x53", + "EventName": "UNC_M3UPI_RxC_HDR_FLIT_NOT_SENT.NO_TXQ_NO_MSG", + "PerPkg": "1", + "PublicDescription": "Header Not Sent : No TxQ Credits + No Extra Message Slotted : header flit is ready for transmission but could not be sent : No TxQ credits available; no additional message slotted into flit", + "UMask": "0x40", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Header Not Sent : TSV High", + "EventCode": "0x53", + "EventName": "UNC_M3UPI_RxC_HDR_FLIT_NOT_SENT.TSV_HI", + "PerPkg": "1", + "PublicDescription": "Header Not Sent : TSV High : header flit is ready for transmission but could not be sent : header flit is ready for transmission but was not sent while tsv high", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Header Not Sent : Cycle valid for Flit", + "EventCode": "0x53", + "EventName": "UNC_M3UPI_RxC_HDR_FLIT_NOT_SENT.VALID_FOR_FLIT", + "PerPkg": "1", + "PublicDescription": "Header Not Sent : Cycle valid for Flit : header flit is ready for transmission but could not be sent : header flit is ready for transmission but was not sent while cycle is valid for flit transmission", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Message Held : Can't Slot AD", + "EventCode": "0x50", + "EventName": "UNC_M3UPI_RxC_HELD.CANT_SLOT_AD", + "PerPkg": "1", + "PublicDescription": "Message Held : Can't Slot AD : some AD message could not be slotted (logical OR of all AD events under INGR_SLOT_CANT_MC_VN{0,1})", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Message Held : Can't Slot BL", + "EventCode": "0x50", + "EventName": "UNC_M3UPI_RxC_HELD.CANT_SLOT_BL", + "PerPkg": "1", + "PublicDescription": "Message Held : Can't Slot BL : some BL message could not be slotted (logical OR of all BL events under INGR_SLOT_CANT_MC_VN{0,1})", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Message Held : Parallel Attempt", + "EventCode": "0x50", + "EventName": "UNC_M3UPI_RxC_HELD.PARALLEL_ATTEMPT", + "PerPkg": "1", + "PublicDescription": "Message Held : Parallel Attempt : ad and bl messages attempted to slot into the same flit in parallel", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Message Held : Parallel Success", + "EventCode": "0x50", + "EventName": "UNC_M3UPI_RxC_HELD.PARALLEL_SUCCESS", + "PerPkg": "1", + "PublicDescription": "Message Held : Parallel Success : ad and bl messages were actually slotted into the same flit in parallel", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Message Held : VN0", + "EventCode": "0x50", + "EventName": "UNC_M3UPI_RxC_HELD.VN0", + "PerPkg": "1", + "PublicDescription": "Message Held : VN0 : vn0 message(s) that couldn't be slotted into last vn0 flit are held in slotting stage while processing vn1 flit", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Message Held : VN1", + "EventCode": "0x50", + "EventName": "UNC_M3UPI_RxC_HELD.VN1", + "PerPkg": "1", + "PublicDescription": "Message Held : VN1 : vn1 message(s) that couldn't be slotted into last vn1 flit are held in slotting stage while processing vn0 flit", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN0 message can't slot into flit : REQ on AD", + "EventCode": "0x4e", + "EventName": "UNC_M3UPI_RxC_PACKING_MISS_VN0.AD_REQ", + "PerPkg": "1", + "PublicDescription": "VN0 message can't slot into flit : REQ on AD : Count cases where Ingress has packets to send but did not have time to pack into flit before sending to Agent so slot was left NULL which could have been used. : Home (REQ) messages on AD. REQ is generally used to send requests, request responses, and snoop responses.", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN0 message can't slot into flit : RSP on AD", + "EventCode": "0x4e", + "EventName": "UNC_M3UPI_RxC_PACKING_MISS_VN0.AD_RSP", + "PerPkg": "1", + "PublicDescription": "VN0 message can't slot into flit : RSP on AD : Count cases where Ingress has packets to send but did not have time to pack into flit before sending to Agent so slot was left NULL which could have been used. : Response (RSP) messages on AD. RSP packets are used to transmit a variety of protocol flits including grants and completions (CMP).", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN0 message can't slot into flit : SNP on AD", + "EventCode": "0x4e", + "EventName": "UNC_M3UPI_RxC_PACKING_MISS_VN0.AD_SNP", + "PerPkg": "1", + "PublicDescription": "VN0 message can't slot into flit : SNP on AD : Count cases where Ingress has packets to send but did not have time to pack into flit before sending to Agent so slot was left NULL which could have been used. : Snoops (SNP) messages on AD. SNP is used for outgoing snoops.", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN0 message can't slot into flit : NCB on BL", + "EventCode": "0x4e", + "EventName": "UNC_M3UPI_RxC_PACKING_MISS_VN0.BL_NCB", + "PerPkg": "1", + "PublicDescription": "VN0 message can't slot into flit : NCB on BL : Count cases where Ingress has packets to send but did not have time to pack into flit before sending to Agent so slot was left NULL which could have been used. : Non-Coherent Broadcast (NCB) messages on BL. NCB is generally used to transmit data without coherency. For example, non-coherent read data returns.", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN0 message can't slot into flit : NCS on BL", + "EventCode": "0x4e", + "EventName": "UNC_M3UPI_RxC_PACKING_MISS_VN0.BL_NCS", + "PerPkg": "1", + "PublicDescription": "VN0 message can't slot into flit : NCS on BL : Count cases where Ingress has packets to send but did not have time to pack into flit before sending to Agent so slot was left NULL which could have been used. : Non-Coherent Standard (NCS) messages on BL.", + "UMask": "0x40", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN0 message can't slot into flit : RSP on BL", + "EventCode": "0x4e", + "EventName": "UNC_M3UPI_RxC_PACKING_MISS_VN0.BL_RSP", + "PerPkg": "1", + "PublicDescription": "VN0 message can't slot into flit : RSP on BL : Count cases where Ingress has packets to send but did not have time to pack into flit before sending to Agent so slot was left NULL which could have been used. : Response (RSP) messages on BL. RSP packets are used to transmit a variety of protocol flits including grants and completions (CMP).", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN0 message can't slot into flit : WB on BL", + "EventCode": "0x4e", + "EventName": "UNC_M3UPI_RxC_PACKING_MISS_VN0.BL_WB", + "PerPkg": "1", + "PublicDescription": "VN0 message can't slot into flit : WB on BL : Count cases where Ingress has packets to send but did not have time to pack into flit before sending to Agent so slot was left NULL which could have been used. : Data Response (WB) messages on BL. WB is generally used to transmit data with coherency. For example, remote reads and writes, or cache to cache transfers will transmit their data using WB.", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN1 message can't slot into flit : REQ on AD", + "EventCode": "0x4f", + "EventName": "UNC_M3UPI_RxC_PACKING_MISS_VN1.AD_REQ", + "PerPkg": "1", + "PublicDescription": "VN1 message can't slot into flit : REQ on AD : Count cases where Ingress has packets to send but did not have time to pack into flit before sending to Agent so slot was left NULL which could have been used. : Home (REQ) messages on AD. REQ is generally used to send requests, request responses, and snoop responses.", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN1 message can't slot into flit : RSP on AD", + "EventCode": "0x4f", + "EventName": "UNC_M3UPI_RxC_PACKING_MISS_VN1.AD_RSP", + "PerPkg": "1", + "PublicDescription": "VN1 message can't slot into flit : RSP on AD : Count cases where Ingress has packets to send but did not have time to pack into flit before sending to Agent so slot was left NULL which could have been used. : Response (RSP) messages on AD. RSP packets are used to transmit a variety of protocol flits including grants and completions (CMP).", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN1 message can't slot into flit : SNP on AD", + "EventCode": "0x4f", + "EventName": "UNC_M3UPI_RxC_PACKING_MISS_VN1.AD_SNP", + "PerPkg": "1", + "PublicDescription": "VN1 message can't slot into flit : SNP on AD : Count cases where Ingress has packets to send but did not have time to pack into flit before sending to Agent so slot was left NULL which could have been used. : Snoops (SNP) messages on AD. SNP is used for outgoing snoops.", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN1 message can't slot into flit : NCB on BL", + "EventCode": "0x4f", + "EventName": "UNC_M3UPI_RxC_PACKING_MISS_VN1.BL_NCB", + "PerPkg": "1", + "PublicDescription": "VN1 message can't slot into flit : NCB on BL : Count cases where Ingress has packets to send but did not have time to pack into flit before sending to Agent so slot was left NULL which could have been used. : Non-Coherent Broadcast (NCB) messages on BL. NCB is generally used to transmit data without coherency. For example, non-coherent read data returns.", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN1 message can't slot into flit : NCS on BL", + "EventCode": "0x4f", + "EventName": "UNC_M3UPI_RxC_PACKING_MISS_VN1.BL_NCS", + "PerPkg": "1", + "PublicDescription": "VN1 message can't slot into flit : NCS on BL : Count cases where Ingress has packets to send but did not have time to pack into flit before sending to Agent so slot was left NULL which could have been used. : Non-Coherent Standard (NCS) messages on BL.", + "UMask": "0x40", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN1 message can't slot into flit : RSP on BL", + "EventCode": "0x4f", + "EventName": "UNC_M3UPI_RxC_PACKING_MISS_VN1.BL_RSP", + "PerPkg": "1", + "PublicDescription": "VN1 message can't slot into flit : RSP on BL : Count cases where Ingress has packets to send but did not have time to pack into flit before sending to Agent so slot was left NULL which could have been used. : Response (RSP) messages on BL. RSP packets are used to transmit a variety of protocol flits including grants and completions (CMP).", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN1 message can't slot into flit : WB on BL", + "EventCode": "0x4f", + "EventName": "UNC_M3UPI_RxC_PACKING_MISS_VN1.BL_WB", + "PerPkg": "1", + "PublicDescription": "VN1 message can't slot into flit : WB on BL : Count cases where Ingress has packets to send but did not have time to pack into flit before sending to Agent so slot was left NULL which could have been used. : Data Response (WB) messages on BL. WB is generally used to transmit data with coherency. For example, remote reads and writes, or cache to cache transfers will transmit their data using WB.", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Remote VNA Credits : Any In Use", + "EventCode": "0x5a", + "EventName": "UNC_M3UPI_RxC_VNA_CRD.ANY_IN_USE", + "PerPkg": "1", + "PublicDescription": "Remote VNA Credits : Any In Use : At least one remote vna credit is in use", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Remote VNA Credits : Corrected", + "EventCode": "0x5a", + "EventName": "UNC_M3UPI_RxC_VNA_CRD.CORRECTED", + "PerPkg": "1", + "PublicDescription": "Remote VNA Credits : Corrected : Number of remote vna credits corrected (local return) per cycle", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Remote VNA Credits : Level < 1", + "EventCode": "0x5a", + "EventName": "UNC_M3UPI_RxC_VNA_CRD.LT1", + "PerPkg": "1", + "PublicDescription": "Remote VNA Credits : Level < 1 : Remote vna credit level is less than 1 (i.e. no vna credits available)", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Remote VNA Credits : Level < 10", + "EventCode": "0x5a", + "EventName": "UNC_M3UPI_RxC_VNA_CRD.LT10", + "PerPkg": "1", + "PublicDescription": "Remote VNA Credits : Level < 10 : remote vna credit level is less than 10; parallel vn0/vn1 arb not possible", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Remote VNA Credits : Level < 4", + "EventCode": "0x5a", + "EventName": "UNC_M3UPI_RxC_VNA_CRD.LT4", + "PerPkg": "1", + "PublicDescription": "Remote VNA Credits : Level < 4 : Remote vna credit level is less than 4; bl (or ad requiring 4 vna) cannot arb on vna", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Remote VNA Credits : Level < 5", + "EventCode": "0x5a", + "EventName": "UNC_M3UPI_RxC_VNA_CRD.LT5", + "PerPkg": "1", + "PublicDescription": "Remote VNA Credits : Level < 5 : Remote vna credit level is less than 5; parallel ad/bl arb on vna not possible", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UNC_M3UPI_RxC_VNA_CRD_MISC.REQ_ADBL_ALLOC_L5", + "EventCode": "0x59", + "EventName": "UNC_M3UPI_RxC_VNA_CRD_MISC.REQ_ADBL_ALLOC_L5", + "PerPkg": "1", + "PublicDescription": ": remote vna credit count was less than 5 and allocation to ad or bl messages was required", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UNC_M3UPI_RxC_VNA_CRD_MISC.REQ_VN01_ALLOC_LT10", + "EventCode": "0x59", + "EventName": "UNC_M3UPI_RxC_VNA_CRD_MISC.REQ_VN01_ALLOC_LT10", + "PerPkg": "1", + "PublicDescription": ": remote vna credit count was less than 10 and allocation to vn0 or vn1 was required", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UNC_M3UPI_RxC_VNA_CRD_MISC.VN0_JUST_AD", + "EventCode": "0x59", + "EventName": "UNC_M3UPI_RxC_VNA_CRD_MISC.VN0_JUST_AD", + "PerPkg": "1", + "PublicDescription": ": on vn0, remote vna credits were allocated only to ad messages, not to bl", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UNC_M3UPI_RxC_VNA_CRD_MISC.VN0_JUST_BL", + "EventCode": "0x59", + "EventName": "UNC_M3UPI_RxC_VNA_CRD_MISC.VN0_JUST_BL", + "PerPkg": "1", + "PublicDescription": ": on vn0, remote vna credits were allocated only to bl messages, not to ad", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UNC_M3UPI_RxC_VNA_CRD_MISC.VN0_ONLY", + "EventCode": "0x59", + "EventName": "UNC_M3UPI_RxC_VNA_CRD_MISC.VN0_ONLY", + "PerPkg": "1", + "PublicDescription": ": remote vna credits were allocated only to vn0, not to vn1", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UNC_M3UPI_RxC_VNA_CRD_MISC.VN1_JUST_AD", + "EventCode": "0x59", + "EventName": "UNC_M3UPI_RxC_VNA_CRD_MISC.VN1_JUST_AD", + "PerPkg": "1", + "PublicDescription": ": on vn1, remote vna credits were allocated only to ad messages, not to bl", + "UMask": "0x40", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UNC_M3UPI_RxC_VNA_CRD_MISC.VN1_JUST_BL", + "EventCode": "0x59", + "EventName": "UNC_M3UPI_RxC_VNA_CRD_MISC.VN1_JUST_BL", + "PerPkg": "1", + "PublicDescription": ": on vn1, remote vna credits were allocated only to bl messages, not to ad", + "UMask": "0x80", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UNC_M3UPI_RxC_VNA_CRD_MISC.VN1_ONLY", + "EventCode": "0x59", + "EventName": "UNC_M3UPI_RxC_VNA_CRD_MISC.VN1_ONLY", + "PerPkg": "1", + "PublicDescription": ": remote vna credits were allocated only to vn1, not to vn0", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Failed ARB for AD : VN0 REQ Messages", + "EventCode": "0x30", + "EventName": "UNC_M3UPI_TxC_AD_ARB_FAIL.VN0_REQ", + "PerPkg": "1", + "PublicDescription": "Failed ARB for AD : VN0 REQ Messages : AD arb but no win; arb request asserted but not won", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Failed ARB for AD : VN0 RSP Messages", + "EventCode": "0x30", + "EventName": "UNC_M3UPI_TxC_AD_ARB_FAIL.VN0_RSP", + "PerPkg": "1", + "PublicDescription": "Failed ARB for AD : VN0 RSP Messages : AD arb but no win; arb request asserted but not won", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Failed ARB for AD : VN0 SNP Messages", + "EventCode": "0x30", + "EventName": "UNC_M3UPI_TxC_AD_ARB_FAIL.VN0_SNP", + "PerPkg": "1", + "PublicDescription": "Failed ARB for AD : VN0 SNP Messages : AD arb but no win; arb request asserted but not won", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Failed ARB for AD : VN0 WB Messages", + "EventCode": "0x30", + "EventName": "UNC_M3UPI_TxC_AD_ARB_FAIL.VN0_WB", + "PerPkg": "1", + "PublicDescription": "Failed ARB for AD : VN0 WB Messages : AD arb but no win; arb request asserted but not won", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Failed ARB for AD : VN1 REQ Messages", + "EventCode": "0x30", + "EventName": "UNC_M3UPI_TxC_AD_ARB_FAIL.VN1_REQ", + "PerPkg": "1", + "PublicDescription": "Failed ARB for AD : VN1 REQ Messages : AD arb but no win; arb request asserted but not won", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Failed ARB for AD : VN1 RSP Messages", + "EventCode": "0x30", + "EventName": "UNC_M3UPI_TxC_AD_ARB_FAIL.VN1_RSP", + "PerPkg": "1", + "PublicDescription": "Failed ARB for AD : VN1 RSP Messages : AD arb but no win; arb request asserted but not won", + "UMask": "0x40", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Failed ARB for AD : VN1 SNP Messages", + "EventCode": "0x30", + "EventName": "UNC_M3UPI_TxC_AD_ARB_FAIL.VN1_SNP", + "PerPkg": "1", + "PublicDescription": "Failed ARB for AD : VN1 SNP Messages : AD arb but no win; arb request asserted but not won", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Failed ARB for AD : VN1 WB Messages", + "EventCode": "0x30", + "EventName": "UNC_M3UPI_TxC_AD_ARB_FAIL.VN1_WB", + "PerPkg": "1", + "PublicDescription": "Failed ARB for AD : VN1 WB Messages : AD arb but no win; arb request asserted but not won", + "UMask": "0x80", + "Unit": "M3UPI" + }, + { + "BriefDescription": "AD FlowQ Bypass", + "EventCode": "0x2C", + "EventName": "UNC_M3UPI_TxC_AD_FLQ_BYPASS", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "PublicDescription": "Counts cases when the AD flowQ is bypassed (S0, S1 and S2 indicate which slot was bypassed with S0 having the highest priority and S2 the least)", + "Unit": "M3UPI" + }, + { + "BriefDescription": "AD FlowQ Bypass", + "EventCode": "0x2c", + "EventName": "UNC_M3UPI_TxC_AD_FLQ_BYPASS.AD_SLOT0", + "PerPkg": "1", + "PublicDescription": "AD FlowQ Bypass : Counts cases when the AD flowQ is bypassed (S0, S1 and S2 indicate which slot was bypassed with S0 having the highest priority and S2 the least)", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "AD FlowQ Bypass", + "EventCode": "0x2c", + "EventName": "UNC_M3UPI_TxC_AD_FLQ_BYPASS.AD_SLOT1", + "PerPkg": "1", + "PublicDescription": "AD FlowQ Bypass : Counts cases when the AD flowQ is bypassed (S0, S1 and S2 indicate which slot was bypassed with S0 having the highest priority and S2 the least)", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "AD FlowQ Bypass", + "EventCode": "0x2c", + "EventName": "UNC_M3UPI_TxC_AD_FLQ_BYPASS.AD_SLOT2", + "PerPkg": "1", + "PublicDescription": "AD FlowQ Bypass : Counts cases when the AD flowQ is bypassed (S0, S1 and S2 indicate which slot was bypassed with S0 having the highest priority and S2 the least)", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "AD FlowQ Bypass", + "EventCode": "0x2c", + "EventName": "UNC_M3UPI_TxC_AD_FLQ_BYPASS.BL_EARLY_RSP", + "PerPkg": "1", + "PublicDescription": "AD FlowQ Bypass : Counts cases when the AD flowQ is bypassed (S0, S1 and S2 indicate which slot was bypassed with S0 having the highest priority and S2 the least)", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "AD Flow Q Not Empty : VN0 REQ Messages", + "EventCode": "0x27", + "EventName": "UNC_M3UPI_TxC_AD_FLQ_CYCLES_NE.VN0_REQ", + "PerPkg": "1", + "PublicDescription": "AD Flow Q Not Empty : VN0 REQ Messages : Number of cycles the AD Egress queue is Not Empty", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "AD Flow Q Not Empty : VN0 RSP Messages", + "EventCode": "0x27", + "EventName": "UNC_M3UPI_TxC_AD_FLQ_CYCLES_NE.VN0_RSP", + "PerPkg": "1", + "PublicDescription": "AD Flow Q Not Empty : VN0 RSP Messages : Number of cycles the AD Egress queue is Not Empty", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "AD Flow Q Not Empty : VN0 SNP Messages", + "EventCode": "0x27", + "EventName": "UNC_M3UPI_TxC_AD_FLQ_CYCLES_NE.VN0_SNP", + "PerPkg": "1", + "PublicDescription": "AD Flow Q Not Empty : VN0 SNP Messages : Number of cycles the AD Egress queue is Not Empty", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "AD Flow Q Not Empty : VN0 WB Messages", + "EventCode": "0x27", + "EventName": "UNC_M3UPI_TxC_AD_FLQ_CYCLES_NE.VN0_WB", + "PerPkg": "1", + "PublicDescription": "AD Flow Q Not Empty : VN0 WB Messages : Number of cycles the AD Egress queue is Not Empty", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "AD Flow Q Not Empty : VN1 REQ Messages", + "EventCode": "0x27", + "EventName": "UNC_M3UPI_TxC_AD_FLQ_CYCLES_NE.VN1_REQ", + "PerPkg": "1", + "PublicDescription": "AD Flow Q Not Empty : VN1 REQ Messages : Number of cycles the AD Egress queue is Not Empty", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "AD Flow Q Not Empty : VN1 RSP Messages", + "EventCode": "0x27", + "EventName": "UNC_M3UPI_TxC_AD_FLQ_CYCLES_NE.VN1_RSP", + "PerPkg": "1", + "PublicDescription": "AD Flow Q Not Empty : VN1 RSP Messages : Number of cycles the AD Egress queue is Not Empty", + "UMask": "0x40", + "Unit": "M3UPI" + }, + { + "BriefDescription": "AD Flow Q Not Empty : VN1 SNP Messages", + "EventCode": "0x27", + "EventName": "UNC_M3UPI_TxC_AD_FLQ_CYCLES_NE.VN1_SNP", + "PerPkg": "1", + "PublicDescription": "AD Flow Q Not Empty : VN1 SNP Messages : Number of cycles the AD Egress queue is Not Empty", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "AD Flow Q Not Empty : VN1 WB Messages", + "EventCode": "0x27", + "EventName": "UNC_M3UPI_TxC_AD_FLQ_CYCLES_NE.VN1_WB", + "PerPkg": "1", + "PublicDescription": "AD Flow Q Not Empty : VN1 WB Messages : Number of cycles the AD Egress queue is Not Empty", + "UMask": "0x80", + "Unit": "M3UPI" + }, + { + "BriefDescription": "AD Flow Q Inserts : VN0 REQ Messages", + "EventCode": "0x2d", + "EventName": "UNC_M3UPI_TxC_AD_FLQ_INSERTS.VN0_REQ", + "PerPkg": "1", + "PublicDescription": "AD Flow Q Inserts : VN0 REQ Messages : Counts the number of allocations into the QPI FlowQ. This can be used in conjunction with the QPI FlowQ Occupancy Accumulator event in order to calculate average queue latency. Only a single FlowQ queue can be tracked at any given time. It is not possible to filter based on direction or polarity.", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "AD Flow Q Inserts : VN0 RSP Messages", + "EventCode": "0x2d", + "EventName": "UNC_M3UPI_TxC_AD_FLQ_INSERTS.VN0_RSP", + "PerPkg": "1", + "PublicDescription": "AD Flow Q Inserts : VN0 RSP Messages : Counts the number of allocations into the QPI FlowQ. This can be used in conjunction with the QPI FlowQ Occupancy Accumulator event in order to calculate average queue latency. Only a single FlowQ queue can be tracked at any given time. It is not possible to filter based on direction or polarity.", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "AD Flow Q Inserts : VN0 SNP Messages", + "EventCode": "0x2d", + "EventName": "UNC_M3UPI_TxC_AD_FLQ_INSERTS.VN0_SNP", + "PerPkg": "1", + "PublicDescription": "AD Flow Q Inserts : VN0 SNP Messages : Counts the number of allocations into the QPI FlowQ. This can be used in conjunction with the QPI FlowQ Occupancy Accumulator event in order to calculate average queue latency. Only a single FlowQ queue can be tracked at any given time. It is not possible to filter based on direction or polarity.", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "AD Flow Q Inserts : VN0 WB Messages", + "EventCode": "0x2d", + "EventName": "UNC_M3UPI_TxC_AD_FLQ_INSERTS.VN0_WB", + "PerPkg": "1", + "PublicDescription": "AD Flow Q Inserts : VN0 WB Messages : Counts the number of allocations into the QPI FlowQ. This can be used in conjunction with the QPI FlowQ Occupancy Accumulator event in order to calculate average queue latency. Only a single FlowQ queue can be tracked at any given time. It is not possible to filter based on direction or polarity.", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "AD Flow Q Inserts : VN1 REQ Messages", + "EventCode": "0x2d", + "EventName": "UNC_M3UPI_TxC_AD_FLQ_INSERTS.VN1_REQ", + "PerPkg": "1", + "PublicDescription": "AD Flow Q Inserts : VN1 REQ Messages : Counts the number of allocations into the QPI FlowQ. This can be used in conjunction with the QPI FlowQ Occupancy Accumulator event in order to calculate average queue latency. Only a single FlowQ queue can be tracked at any given time. It is not possible to filter based on direction or polarity.", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "AD Flow Q Inserts : VN1 RSP Messages", + "EventCode": "0x2d", + "EventName": "UNC_M3UPI_TxC_AD_FLQ_INSERTS.VN1_RSP", + "PerPkg": "1", + "PublicDescription": "AD Flow Q Inserts : VN1 RSP Messages : Counts the number of allocations into the QPI FlowQ. This can be used in conjunction with the QPI FlowQ Occupancy Accumulator event in order to calculate average queue latency. Only a single FlowQ queue can be tracked at any given time. It is not possible to filter based on direction or polarity.", + "UMask": "0x40", + "Unit": "M3UPI" + }, + { + "BriefDescription": "AD Flow Q Inserts : VN1 SNP Messages", + "EventCode": "0x2d", + "EventName": "UNC_M3UPI_TxC_AD_FLQ_INSERTS.VN1_SNP", + "PerPkg": "1", + "PublicDescription": "AD Flow Q Inserts : VN1 SNP Messages : Counts the number of allocations into the QPI FlowQ. This can be used in conjunction with the QPI FlowQ Occupancy Accumulator event in order to calculate average queue latency. Only a single FlowQ queue can be tracked at any given time. It is not possible to filter based on direction or polarity.", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "AD Flow Q Occupancy : VN0 REQ Messages", + "EventCode": "0x1c", + "EventName": "UNC_M3UPI_TxC_AD_FLQ_OCCUPANCY.VN0_REQ", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "AD Flow Q Occupancy : VN0 RSP Messages", + "EventCode": "0x1c", + "EventName": "UNC_M3UPI_TxC_AD_FLQ_OCCUPANCY.VN0_RSP", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "AD Flow Q Occupancy : VN0 SNP Messages", + "EventCode": "0x1c", + "EventName": "UNC_M3UPI_TxC_AD_FLQ_OCCUPANCY.VN0_SNP", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "AD Flow Q Occupancy : VN0 WB Messages", + "EventCode": "0x1c", + "EventName": "UNC_M3UPI_TxC_AD_FLQ_OCCUPANCY.VN0_WB", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "AD Flow Q Occupancy : VN1 REQ Messages", + "EventCode": "0x1c", + "EventName": "UNC_M3UPI_TxC_AD_FLQ_OCCUPANCY.VN1_REQ", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "AD Flow Q Occupancy : VN1 RSP Messages", + "EventCode": "0x1c", + "EventName": "UNC_M3UPI_TxC_AD_FLQ_OCCUPANCY.VN1_RSP", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "M3UPI" + }, + { + "BriefDescription": "AD Flow Q Occupancy : VN1 SNP Messages", + "EventCode": "0x1c", + "EventName": "UNC_M3UPI_TxC_AD_FLQ_OCCUPANCY.VN1_SNP", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "AK Flow Q Inserts", + "EventCode": "0x2f", + "EventName": "UNC_M3UPI_TxC_AK_FLQ_INSERTS", + "PerPkg": "1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "AK Flow Q Occupancy", + "EventCode": "0x1e", + "EventName": "UNC_M3UPI_TxC_AK_FLQ_OCCUPANCY", + "PerPkg": "1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Failed ARB for BL : VN0 NCB Messages", + "EventCode": "0x35", + "EventName": "UNC_M3UPI_TxC_BL_ARB_FAIL.VN0_NCB", + "PerPkg": "1", + "PublicDescription": "Failed ARB for BL : VN0 NCB Messages : BL arb but no win; arb request asserted but not won", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Failed ARB for BL : VN0 NCS Messages", + "EventCode": "0x35", + "EventName": "UNC_M3UPI_TxC_BL_ARB_FAIL.VN0_NCS", + "PerPkg": "1", + "PublicDescription": "Failed ARB for BL : VN0 NCS Messages : BL arb but no win; arb request asserted but not won", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Failed ARB for BL : VN0 RSP Messages", + "EventCode": "0x35", + "EventName": "UNC_M3UPI_TxC_BL_ARB_FAIL.VN0_RSP", + "PerPkg": "1", + "PublicDescription": "Failed ARB for BL : VN0 RSP Messages : BL arb but no win; arb request asserted but not won", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Failed ARB for BL : VN0 WB Messages", + "EventCode": "0x35", + "EventName": "UNC_M3UPI_TxC_BL_ARB_FAIL.VN0_WB", + "PerPkg": "1", + "PublicDescription": "Failed ARB for BL : VN0 WB Messages : BL arb but no win; arb request asserted but not won", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Failed ARB for BL : VN1 NCS Messages", + "EventCode": "0x35", + "EventName": "UNC_M3UPI_TxC_BL_ARB_FAIL.VN1_NCB", + "PerPkg": "1", + "PublicDescription": "Failed ARB for BL : VN1 NCS Messages : BL arb but no win; arb request asserted but not won", + "UMask": "0x40", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Failed ARB for BL : VN1 NCB Messages", + "EventCode": "0x35", + "EventName": "UNC_M3UPI_TxC_BL_ARB_FAIL.VN1_NCS", + "PerPkg": "1", + "PublicDescription": "Failed ARB for BL : VN1 NCB Messages : BL arb but no win; arb request asserted but not won", + "UMask": "0x80", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Failed ARB for BL : VN1 RSP Messages", + "EventCode": "0x35", + "EventName": "UNC_M3UPI_TxC_BL_ARB_FAIL.VN1_RSP", + "PerPkg": "1", + "PublicDescription": "Failed ARB for BL : VN1 RSP Messages : BL arb but no win; arb request asserted but not won", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Failed ARB for BL : VN1 WB Messages", + "EventCode": "0x35", + "EventName": "UNC_M3UPI_TxC_BL_ARB_FAIL.VN1_WB", + "PerPkg": "1", + "PublicDescription": "Failed ARB for BL : VN1 WB Messages : BL arb but no win; arb request asserted but not won", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "BL Flow Q Not Empty : VN0 REQ Messages", + "EventCode": "0x28", + "EventName": "UNC_M3UPI_TxC_BL_FLQ_CYCLES_NE.VN0_REQ", + "PerPkg": "1", + "PublicDescription": "BL Flow Q Not Empty : VN0 REQ Messages : Number of cycles the BL Egress queue is Not Empty", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "BL Flow Q Not Empty : VN0 RSP Messages", + "EventCode": "0x28", + "EventName": "UNC_M3UPI_TxC_BL_FLQ_CYCLES_NE.VN0_RSP", + "PerPkg": "1", + "PublicDescription": "BL Flow Q Not Empty : VN0 RSP Messages : Number of cycles the BL Egress queue is Not Empty", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "BL Flow Q Not Empty : VN0 SNP Messages", + "EventCode": "0x28", + "EventName": "UNC_M3UPI_TxC_BL_FLQ_CYCLES_NE.VN0_SNP", + "PerPkg": "1", + "PublicDescription": "BL Flow Q Not Empty : VN0 SNP Messages : Number of cycles the BL Egress queue is Not Empty", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "BL Flow Q Not Empty : VN0 WB Messages", + "EventCode": "0x28", + "EventName": "UNC_M3UPI_TxC_BL_FLQ_CYCLES_NE.VN0_WB", + "PerPkg": "1", + "PublicDescription": "BL Flow Q Not Empty : VN0 WB Messages : Number of cycles the BL Egress queue is Not Empty", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "BL Flow Q Not Empty : VN1 REQ Messages", + "EventCode": "0x28", + "EventName": "UNC_M3UPI_TxC_BL_FLQ_CYCLES_NE.VN1_REQ", + "PerPkg": "1", + "PublicDescription": "BL Flow Q Not Empty : VN1 REQ Messages : Number of cycles the BL Egress queue is Not Empty", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "BL Flow Q Not Empty : VN1 RSP Messages", + "EventCode": "0x28", + "EventName": "UNC_M3UPI_TxC_BL_FLQ_CYCLES_NE.VN1_RSP", + "PerPkg": "1", + "PublicDescription": "BL Flow Q Not Empty : VN1 RSP Messages : Number of cycles the BL Egress queue is Not Empty", + "UMask": "0x40", + "Unit": "M3UPI" + }, + { + "BriefDescription": "BL Flow Q Not Empty : VN1 SNP Messages", + "EventCode": "0x28", + "EventName": "UNC_M3UPI_TxC_BL_FLQ_CYCLES_NE.VN1_SNP", + "PerPkg": "1", + "PublicDescription": "BL Flow Q Not Empty : VN1 SNP Messages : Number of cycles the BL Egress queue is Not Empty", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "BL Flow Q Not Empty : VN1 WB Messages", + "EventCode": "0x28", + "EventName": "UNC_M3UPI_TxC_BL_FLQ_CYCLES_NE.VN1_WB", + "PerPkg": "1", + "PublicDescription": "BL Flow Q Not Empty : VN1 WB Messages : Number of cycles the BL Egress queue is Not Empty", + "UMask": "0x80", + "Unit": "M3UPI" + }, + { + "BriefDescription": "BL Flow Q Inserts : VN0 RSP Messages", + "EventCode": "0x2e", + "EventName": "UNC_M3UPI_TxC_BL_FLQ_INSERTS.VN0_NCB", + "PerPkg": "1", + "PublicDescription": "BL Flow Q Inserts : VN0 RSP Messages : Counts the number of allocations into the QPI FlowQ. This can be used in conjunction with the QPI FlowQ Occupancy Accumulator event in order to calculate average queue latency. Only a single FlowQ queue can be tracked at any given time. It is not possible to filter based on direction or polarity.", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "BL Flow Q Inserts : VN0 WB Messages", + "EventCode": "0x2e", + "EventName": "UNC_M3UPI_TxC_BL_FLQ_INSERTS.VN0_NCS", + "PerPkg": "1", + "PublicDescription": "BL Flow Q Inserts : VN0 WB Messages : Counts the number of allocations into the QPI FlowQ. This can be used in conjunction with the QPI FlowQ Occupancy Accumulator event in order to calculate average queue latency. Only a single FlowQ queue can be tracked at any given time. It is not possible to filter based on direction or polarity.", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "BL Flow Q Inserts : VN0 NCS Messages", + "EventCode": "0x2e", + "EventName": "UNC_M3UPI_TxC_BL_FLQ_INSERTS.VN0_RSP", + "PerPkg": "1", + "PublicDescription": "BL Flow Q Inserts : VN0 NCS Messages : Counts the number of allocations into the QPI FlowQ. This can be used in conjunction with the QPI FlowQ Occupancy Accumulator event in order to calculate average queue latency. Only a single FlowQ queue can be tracked at any given time. It is not possible to filter based on direction or polarity.", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "BL Flow Q Inserts : VN0 NCB Messages", + "EventCode": "0x2e", + "EventName": "UNC_M3UPI_TxC_BL_FLQ_INSERTS.VN0_WB", + "PerPkg": "1", + "PublicDescription": "BL Flow Q Inserts : VN0 NCB Messages : Counts the number of allocations into the QPI FlowQ. This can be used in conjunction with the QPI FlowQ Occupancy Accumulator event in order to calculate average queue latency. Only a single FlowQ queue can be tracked at any given time. It is not possible to filter based on direction or polarity.", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "BL Flow Q Inserts : VN1 RSP Messages", + "EventCode": "0x2e", + "EventName": "UNC_M3UPI_TxC_BL_FLQ_INSERTS.VN1_NCB", + "PerPkg": "1", + "PublicDescription": "BL Flow Q Inserts : VN1 RSP Messages : Counts the number of allocations into the QPI FlowQ. This can be used in conjunction with the QPI FlowQ Occupancy Accumulator event in order to calculate average queue latency. Only a single FlowQ queue can be tracked at any given time. It is not possible to filter based on direction or polarity.", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "BL Flow Q Inserts : VN1 WB Messages", + "EventCode": "0x2e", + "EventName": "UNC_M3UPI_TxC_BL_FLQ_INSERTS.VN1_NCS", + "PerPkg": "1", + "PublicDescription": "BL Flow Q Inserts : VN1 WB Messages : Counts the number of allocations into the QPI FlowQ. This can be used in conjunction with the QPI FlowQ Occupancy Accumulator event in order to calculate average queue latency. Only a single FlowQ queue can be tracked at any given time. It is not possible to filter based on direction or polarity.", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "BL Flow Q Inserts : VN1_NCB Messages", + "EventCode": "0x2e", + "EventName": "UNC_M3UPI_TxC_BL_FLQ_INSERTS.VN1_RSP", + "PerPkg": "1", + "PublicDescription": "BL Flow Q Inserts : VN1_NCB Messages : Counts the number of allocations into the QPI FlowQ. This can be used in conjunction with the QPI FlowQ Occupancy Accumulator event in order to calculate average queue latency. Only a single FlowQ queue can be tracked at any given time. It is not possible to filter based on direction or polarity.", + "UMask": "0x80", + "Unit": "M3UPI" + }, + { + "BriefDescription": "BL Flow Q Inserts : VN1_NCS Messages", + "EventCode": "0x2e", + "EventName": "UNC_M3UPI_TxC_BL_FLQ_INSERTS.VN1_WB", + "PerPkg": "1", + "PublicDescription": "BL Flow Q Inserts : VN1_NCS Messages : Counts the number of allocations into the QPI FlowQ. This can be used in conjunction with the QPI FlowQ Occupancy Accumulator event in order to calculate average queue latency. Only a single FlowQ queue can be tracked at any given time. It is not possible to filter based on direction or polarity.", + "UMask": "0x40", + "Unit": "M3UPI" + }, + { + "BriefDescription": "BL Flow Q Occupancy : VN0 NCB Messages", + "EventCode": "0x1d", + "EventName": "UNC_M3UPI_TxC_BL_FLQ_OCCUPANCY.VN0_NCB", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "BL Flow Q Occupancy : VN0 NCS Messages", + "EventCode": "0x1d", + "EventName": "UNC_M3UPI_TxC_BL_FLQ_OCCUPANCY.VN0_NCS", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "BL Flow Q Occupancy : VN0 RSP Messages", + "EventCode": "0x1d", + "EventName": "UNC_M3UPI_TxC_BL_FLQ_OCCUPANCY.VN0_RSP", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "BL Flow Q Occupancy : VN0 WB Messages", + "EventCode": "0x1d", + "EventName": "UNC_M3UPI_TxC_BL_FLQ_OCCUPANCY.VN0_WB", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "BL Flow Q Occupancy : VN1_NCS Messages", + "EventCode": "0x1d", + "EventName": "UNC_M3UPI_TxC_BL_FLQ_OCCUPANCY.VN1_NCB", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "M3UPI" + }, + { + "BriefDescription": "BL Flow Q Occupancy : VN1_NCB Messages", + "EventCode": "0x1d", + "EventName": "UNC_M3UPI_TxC_BL_FLQ_OCCUPANCY.VN1_NCS", + "PerPkg": "1", + "UMask": "0x80", + "Unit": "M3UPI" + }, + { + "BriefDescription": "BL Flow Q Occupancy : VN1 RSP Messages", + "EventCode": "0x1d", + "EventName": "UNC_M3UPI_TxC_BL_FLQ_OCCUPANCY.VN1_RSP", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "BL Flow Q Occupancy : VN1 WB Messages", + "EventCode": "0x1d", + "EventName": "UNC_M3UPI_TxC_BL_FLQ_OCCUPANCY.VN1_WB", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "BL Flow Q Occupancy : VN0 RSP Messages", + "EventCode": "0x1f", + "EventName": "UNC_M3UPI_TxC_BL_WB_FLQ_OCCUPANCY.VN0_LOCAL", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "BL Flow Q Occupancy : VN0 WB Messages", + "EventCode": "0x1f", + "EventName": "UNC_M3UPI_TxC_BL_WB_FLQ_OCCUPANCY.VN0_THROUGH", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "BL Flow Q Occupancy : VN0 NCB Messages", + "EventCode": "0x1f", + "EventName": "UNC_M3UPI_TxC_BL_WB_FLQ_OCCUPANCY.VN0_WRPULL", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "BL Flow Q Occupancy : VN1 RSP Messages", + "EventCode": "0x1f", + "EventName": "UNC_M3UPI_TxC_BL_WB_FLQ_OCCUPANCY.VN1_LOCAL", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "BL Flow Q Occupancy : VN1 WB Messages", + "EventCode": "0x1f", + "EventName": "UNC_M3UPI_TxC_BL_WB_FLQ_OCCUPANCY.VN1_THROUGH", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "BL Flow Q Occupancy : VN1_NCS Messages", + "EventCode": "0x1f", + "EventName": "UNC_M3UPI_TxC_BL_WB_FLQ_OCCUPANCY.VN1_WRPULL", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UPI0 AD Credits Empty : VN0 REQ Messages", + "EventCode": "0x20", + "EventName": "UNC_M3UPI_UPI_PEER_AD_CREDITS_EMPTY.VN0_REQ", + "PerPkg": "1", + "PublicDescription": "UPI0 AD Credits Empty : VN0 REQ Messages : No credits available to send to UPIs on the AD Ring", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UPI0 AD Credits Empty : VN0 RSP Messages", + "EventCode": "0x20", + "EventName": "UNC_M3UPI_UPI_PEER_AD_CREDITS_EMPTY.VN0_RSP", + "PerPkg": "1", + "PublicDescription": "UPI0 AD Credits Empty : VN0 RSP Messages : No credits available to send to UPIs on the AD Ring", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UPI0 AD Credits Empty : VN0 SNP Messages", + "EventCode": "0x20", + "EventName": "UNC_M3UPI_UPI_PEER_AD_CREDITS_EMPTY.VN0_SNP", + "PerPkg": "1", + "PublicDescription": "UPI0 AD Credits Empty : VN0 SNP Messages : No credits available to send to UPIs on the AD Ring", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UPI0 AD Credits Empty : VN1 REQ Messages", + "EventCode": "0x20", + "EventName": "UNC_M3UPI_UPI_PEER_AD_CREDITS_EMPTY.VN1_REQ", + "PerPkg": "1", + "PublicDescription": "UPI0 AD Credits Empty : VN1 REQ Messages : No credits available to send to UPIs on the AD Ring", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UPI0 AD Credits Empty : VN1 RSP Messages", + "EventCode": "0x20", + "EventName": "UNC_M3UPI_UPI_PEER_AD_CREDITS_EMPTY.VN1_RSP", + "PerPkg": "1", + "PublicDescription": "UPI0 AD Credits Empty : VN1 RSP Messages : No credits available to send to UPIs on the AD Ring", + "UMask": "0x40", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UPI0 AD Credits Empty : VN1 SNP Messages", + "EventCode": "0x20", + "EventName": "UNC_M3UPI_UPI_PEER_AD_CREDITS_EMPTY.VN1_SNP", + "PerPkg": "1", + "PublicDescription": "UPI0 AD Credits Empty : VN1 SNP Messages : No credits available to send to UPIs on the AD Ring", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UPI0 AD Credits Empty : VNA", + "EventCode": "0x20", + "EventName": "UNC_M3UPI_UPI_PEER_AD_CREDITS_EMPTY.VNA", + "PerPkg": "1", + "PublicDescription": "UPI0 AD Credits Empty : VNA : No credits available to send to UPIs on the AD Ring", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UPI0 BL Credits Empty : VN0 RSP Messages", + "EventCode": "0x21", + "EventName": "UNC_M3UPI_UPI_PEER_BL_CREDITS_EMPTY.VN0_NCS_NCB", + "PerPkg": "1", + "PublicDescription": "UPI0 BL Credits Empty : VN0 RSP Messages : No credits available to send to UPI on the BL Ring (diff between non-SMI and SMI mode)", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UPI0 BL Credits Empty : VN0 REQ Messages", + "EventCode": "0x21", + "EventName": "UNC_M3UPI_UPI_PEER_BL_CREDITS_EMPTY.VN0_RSP", + "PerPkg": "1", + "PublicDescription": "UPI0 BL Credits Empty : VN0 REQ Messages : No credits available to send to UPI on the BL Ring (diff between non-SMI and SMI mode)", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UPI0 BL Credits Empty : VN0 SNP Messages", + "EventCode": "0x21", + "EventName": "UNC_M3UPI_UPI_PEER_BL_CREDITS_EMPTY.VN0_WB", + "PerPkg": "1", + "PublicDescription": "UPI0 BL Credits Empty : VN0 SNP Messages : No credits available to send to UPI on the BL Ring (diff between non-SMI and SMI mode)", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UPI0 BL Credits Empty : VN1 RSP Messages", + "EventCode": "0x21", + "EventName": "UNC_M3UPI_UPI_PEER_BL_CREDITS_EMPTY.VN1_NCS_NCB", + "PerPkg": "1", + "PublicDescription": "UPI0 BL Credits Empty : VN1 RSP Messages : No credits available to send to UPI on the BL Ring (diff between non-SMI and SMI mode)", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UPI0 BL Credits Empty : VN1 REQ Messages", + "EventCode": "0x21", + "EventName": "UNC_M3UPI_UPI_PEER_BL_CREDITS_EMPTY.VN1_RSP", + "PerPkg": "1", + "PublicDescription": "UPI0 BL Credits Empty : VN1 REQ Messages : No credits available to send to UPI on the BL Ring (diff between non-SMI and SMI mode)", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UPI0 BL Credits Empty : VN1 SNP Messages", + "EventCode": "0x21", + "EventName": "UNC_M3UPI_UPI_PEER_BL_CREDITS_EMPTY.VN1_WB", + "PerPkg": "1", + "PublicDescription": "UPI0 BL Credits Empty : VN1 SNP Messages : No credits available to send to UPI on the BL Ring (diff between non-SMI and SMI mode)", + "UMask": "0x40", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UPI0 BL Credits Empty : VNA", + "EventCode": "0x21", + "EventName": "UNC_M3UPI_UPI_PEER_BL_CREDITS_EMPTY.VNA", + "PerPkg": "1", + "PublicDescription": "UPI0 BL Credits Empty : VNA : No credits available to send to UPI on the BL Ring (diff between non-SMI and SMI mode)", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "FlowQ Generated Prefetch", + "EventCode": "0x29", + "EventName": "UNC_M3UPI_UPI_PREFETCH_SPAWN", + "PerPkg": "1", + "PublicDescription": "FlowQ Generated Prefetch : Count cases where FlowQ causes spawn of Prefetch to iMC/SMI3 target", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN0 Credit Used : WB on BL", + "EventCode": "0x5b", + "EventName": "UNC_M3UPI_VN0_CREDITS_USED.NCB", + "PerPkg": "1", + "PublicDescription": "VN0 Credit Used : WB on BL : Number of times a VN0 credit was used on the DRS message channel. In order for a request to be transferred across UPI, it must be guaranteed to have a flit buffer on the remote socket to sink into. There are two credit pools, VNA and VN0. VNA is a shared pool used to achieve high performance. The VN0 pool has reserved entries for each message class and is used to prevent deadlock. Requests first attempt to acquire a VNA credit, and then fall back to VN0 if they fail. This counts the number of times a VN0 credit was used. Note that a single VN0 credit holds access to potentially multiple flit buffers. For example, a transfer that uses VNA could use 9 flit buffers and in that case uses 9 credits. A transfer on VN0 will only count a single credit even though it may use multiple buffers. : Data Response (WB) messages on BL. WB is generally used to transmit data with coherency. For example, remote reads and writes, or cache to cache transfers will transmit their data using WB.", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN0 Credit Used : NCB on BL", + "EventCode": "0x5b", + "EventName": "UNC_M3UPI_VN0_CREDITS_USED.NCS", + "PerPkg": "1", + "PublicDescription": "VN0 Credit Used : NCB on BL : Number of times a VN0 credit was used on the DRS message channel. In order for a request to be transferred across UPI, it must be guaranteed to have a flit buffer on the remote socket to sink into. There are two credit pools, VNA and VN0. VNA is a shared pool used to achieve high performance. The VN0 pool has reserved entries for each message class and is used to prevent deadlock. Requests first attempt to acquire a VNA credit, and then fall back to VN0 if they fail. This counts the number of times a VN0 credit was used. Note that a single VN0 credit holds access to potentially multiple flit buffers. For example, a transfer that uses VNA could use 9 flit buffers and in that case uses 9 credits. A transfer on VN0 will only count a single credit even though it may use multiple buffers. : Non-Coherent Broadcast (NCB) messages on BL. NCB is generally used to transmit data without coherency. For example, non-coherent read data returns.", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN0 Credit Used : REQ on AD", + "EventCode": "0x5b", + "EventName": "UNC_M3UPI_VN0_CREDITS_USED.REQ", + "PerPkg": "1", + "PublicDescription": "VN0 Credit Used : REQ on AD : Number of times a VN0 credit was used on the DRS message channel. In order for a request to be transferred across UPI, it must be guaranteed to have a flit buffer on the remote socket to sink into. There are two credit pools, VNA and VN0. VNA is a shared pool used to achieve high performance. The VN0 pool has reserved entries for each message class and is used to prevent deadlock. Requests first attempt to acquire a VNA credit, and then fall back to VN0 if they fail. This counts the number of times a VN0 credit was used. Note that a single VN0 credit holds access to potentially multiple flit buffers. For example, a transfer that uses VNA could use 9 flit buffers and in that case uses 9 credits. A transfer on VN0 will only count a single credit even though it may use multiple buffers. : Home (REQ) messages on AD. REQ is generally used to send requests, request responses, and snoop responses.", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN0 Credit Used : RSP on AD", + "EventCode": "0x5b", + "EventName": "UNC_M3UPI_VN0_CREDITS_USED.RSP", + "PerPkg": "1", + "PublicDescription": "VN0 Credit Used : RSP on AD : Number of times a VN0 credit was used on the DRS message channel. In order for a request to be transferred across UPI, it must be guaranteed to have a flit buffer on the remote socket to sink into. There are two credit pools, VNA and VN0. VNA is a shared pool used to achieve high performance. The VN0 pool has reserved entries for each message class and is used to prevent deadlock. Requests first attempt to acquire a VNA credit, and then fall back to VN0 if they fail. This counts the number of times a VN0 credit was used. Note that a single VN0 credit holds access to potentially multiple flit buffers. For example, a transfer that uses VNA could use 9 flit buffers and in that case uses 9 credits. A transfer on VN0 will only count a single credit even though it may use multiple buffers. : Response (RSP) messages on AD. RSP packets are used to transmit a variety of protocol flits including grants and completions (CMP).", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN0 Credit Used : SNP on AD", + "EventCode": "0x5b", + "EventName": "UNC_M3UPI_VN0_CREDITS_USED.SNP", + "PerPkg": "1", + "PublicDescription": "VN0 Credit Used : SNP on AD : Number of times a VN0 credit was used on the DRS message channel. In order for a request to be transferred across UPI, it must be guaranteed to have a flit buffer on the remote socket to sink into. There are two credit pools, VNA and VN0. VNA is a shared pool used to achieve high performance. The VN0 pool has reserved entries for each message class and is used to prevent deadlock. Requests first attempt to acquire a VNA credit, and then fall back to VN0 if they fail. This counts the number of times a VN0 credit was used. Note that a single VN0 credit holds access to potentially multiple flit buffers. For example, a transfer that uses VNA could use 9 flit buffers and in that case uses 9 credits. A transfer on VN0 will only count a single credit even though it may use multiple buffers. : Snoops (SNP) messages on AD. SNP is used for outgoing snoops.", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN0 Credit Used : RSP on BL", + "EventCode": "0x5b", + "EventName": "UNC_M3UPI_VN0_CREDITS_USED.WB", + "PerPkg": "1", + "PublicDescription": "VN0 Credit Used : RSP on BL : Number of times a VN0 credit was used on the DRS message channel. In order for a request to be transferred across UPI, it must be guaranteed to have a flit buffer on the remote socket to sink into. There are two credit pools, VNA and VN0. VNA is a shared pool used to achieve high performance. The VN0 pool has reserved entries for each message class and is used to prevent deadlock. Requests first attempt to acquire a VNA credit, and then fall back to VN0 if they fail. This counts the number of times a VN0 credit was used. Note that a single VN0 credit holds access to potentially multiple flit buffers. For example, a transfer that uses VNA could use 9 flit buffers and in that case uses 9 credits. A transfer on VN0 will only count a single credit even though it may use multiple buffers. : Response (RSP) messages on BL. RSP packets are used to transmit a variety of protocol flits including grants and completions (CMP).", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN0 No Credits : WB on BL", + "EventCode": "0x5d", + "EventName": "UNC_M3UPI_VN0_NO_CREDITS.NCB", + "PerPkg": "1", + "PublicDescription": "VN0 No Credits : WB on BL : Number of Cycles there were no VN0 Credits : Data Response (WB) messages on BL. WB is generally used to transmit data with coherency. For example, remote reads and writes, or cache to cache transfers will transmit their data using WB.", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN0 No Credits : NCB on BL", + "EventCode": "0x5d", + "EventName": "UNC_M3UPI_VN0_NO_CREDITS.NCS", + "PerPkg": "1", + "PublicDescription": "VN0 No Credits : NCB on BL : Number of Cycles there were no VN0 Credits : Non-Coherent Broadcast (NCB) messages on BL. NCB is generally used to transmit data without coherency. For example, non-coherent read data returns.", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN0 No Credits : REQ on AD", + "EventCode": "0x5d", + "EventName": "UNC_M3UPI_VN0_NO_CREDITS.REQ", + "PerPkg": "1", + "PublicDescription": "VN0 No Credits : REQ on AD : Number of Cycles there were no VN0 Credits : Home (REQ) messages on AD. REQ is generally used to send requests, request responses, and snoop responses.", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN0 No Credits : RSP on AD", + "EventCode": "0x5d", + "EventName": "UNC_M3UPI_VN0_NO_CREDITS.RSP", + "PerPkg": "1", + "PublicDescription": "VN0 No Credits : RSP on AD : Number of Cycles there were no VN0 Credits : Response (RSP) messages on AD. RSP packets are used to transmit a variety of protocol flits including grants and completions (CMP).", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN0 No Credits : SNP on AD", + "EventCode": "0x5d", + "EventName": "UNC_M3UPI_VN0_NO_CREDITS.SNP", + "PerPkg": "1", + "PublicDescription": "VN0 No Credits : SNP on AD : Number of Cycles there were no VN0 Credits : Snoops (SNP) messages on AD. SNP is used for outgoing snoops.", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN0 No Credits : RSP on BL", + "EventCode": "0x5d", + "EventName": "UNC_M3UPI_VN0_NO_CREDITS.WB", + "PerPkg": "1", + "PublicDescription": "VN0 No Credits : RSP on BL : Number of Cycles there were no VN0 Credits : Response (RSP) messages on BL. RSP packets are used to transmit a variety of protocol flits including grants and completions (CMP).", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN1 Credit Used : WB on BL", + "EventCode": "0x5c", + "EventName": "UNC_M3UPI_VN1_CREDITS_USED.NCB", + "PerPkg": "1", + "PublicDescription": "VN1 Credit Used : WB on BL : Number of times a VN1 credit was used on the WB message channel. In order for a request to be transferred across QPI, it must be guaranteed to have a flit buffer on the remote socket to sink into. There are two credit pools, VNA and VN1. VNA is a shared pool used to achieve high performance. The VN1 pool has reserved entries for each message class and is used to prevent deadlock. Requests first attempt to acquire a VNA credit, and then fall back to VN1 if they fail. This counts the number of times a VN1 credit was used. Note that a single VN1 credit holds access to potentially multiple flit buffers. For example, a transfer that uses VNA could use 9 flit buffers and in that case uses 9 credits. A transfer on VN1 will only count a single credit even though it may use multiple buffers. : Data Response (WB) messages on BL. WB is generally used to transmit data with coherency. For example, remote reads and writes, or cache to cache transfers will transmit their data using WB.", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN1 Credit Used : NCB on BL", + "EventCode": "0x5c", + "EventName": "UNC_M3UPI_VN1_CREDITS_USED.NCS", + "PerPkg": "1", + "PublicDescription": "VN1 Credit Used : NCB on BL : Number of times a VN1 credit was used on the WB message channel. In order for a request to be transferred across QPI, it must be guaranteed to have a flit buffer on the remote socket to sink into. There are two credit pools, VNA and VN1. VNA is a shared pool used to achieve high performance. The VN1 pool has reserved entries for each message class and is used to prevent deadlock. Requests first attempt to acquire a VNA credit, and then fall back to VN1 if they fail. This counts the number of times a VN1 credit was used. Note that a single VN1 credit holds access to potentially multiple flit buffers. For example, a transfer that uses VNA could use 9 flit buffers and in that case uses 9 credits. A transfer on VN1 will only count a single credit even though it may use multiple buffers. : Non-Coherent Broadcast (NCB) messages on BL. NCB is generally used to transmit data without coherency. For example, non-coherent read data returns.", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN1 Credit Used : REQ on AD", + "EventCode": "0x5c", + "EventName": "UNC_M3UPI_VN1_CREDITS_USED.REQ", + "PerPkg": "1", + "PublicDescription": "VN1 Credit Used : REQ on AD : Number of times a VN1 credit was used on the WB message channel. In order for a request to be transferred across QPI, it must be guaranteed to have a flit buffer on the remote socket to sink into. There are two credit pools, VNA and VN1. VNA is a shared pool used to achieve high performance. The VN1 pool has reserved entries for each message class and is used to prevent deadlock. Requests first attempt to acquire a VNA credit, and then fall back to VN1 if they fail. This counts the number of times a VN1 credit was used. Note that a single VN1 credit holds access to potentially multiple flit buffers. For example, a transfer that uses VNA could use 9 flit buffers and in that case uses 9 credits. A transfer on VN1 will only count a single credit even though it may use multiple buffers. : Home (REQ) messages on AD. REQ is generally used to send requests, request responses, and snoop responses.", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN1 Credit Used : RSP on AD", + "EventCode": "0x5c", + "EventName": "UNC_M3UPI_VN1_CREDITS_USED.RSP", + "PerPkg": "1", + "PublicDescription": "VN1 Credit Used : RSP on AD : Number of times a VN1 credit was used on the WB message channel. In order for a request to be transferred across QPI, it must be guaranteed to have a flit buffer on the remote socket to sink into. There are two credit pools, VNA and VN1. VNA is a shared pool used to achieve high performance. The VN1 pool has reserved entries for each message class and is used to prevent deadlock. Requests first attempt to acquire a VNA credit, and then fall back to VN1 if they fail. This counts the number of times a VN1 credit was used. Note that a single VN1 credit holds access to potentially multiple flit buffers. For example, a transfer that uses VNA could use 9 flit buffers and in that case uses 9 credits. A transfer on VN1 will only count a single credit even though it may use multiple buffers. : Response (RSP) messages on AD. RSP packets are used to transmit a variety of protocol flits including grants and completions (CMP).", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN1 Credit Used : SNP on AD", + "EventCode": "0x5c", + "EventName": "UNC_M3UPI_VN1_CREDITS_USED.SNP", + "PerPkg": "1", + "PublicDescription": "VN1 Credit Used : SNP on AD : Number of times a VN1 credit was used on the WB message channel. In order for a request to be transferred across QPI, it must be guaranteed to have a flit buffer on the remote socket to sink into. There are two credit pools, VNA and VN1. VNA is a shared pool used to achieve high performance. The VN1 pool has reserved entries for each message class and is used to prevent deadlock. Requests first attempt to acquire a VNA credit, and then fall back to VN1 if they fail. This counts the number of times a VN1 credit was used. Note that a single VN1 credit holds access to potentially multiple flit buffers. For example, a transfer that uses VNA could use 9 flit buffers and in that case uses 9 credits. A transfer on VN1 will only count a single credit even though it may use multiple buffers. : Snoops (SNP) messages on AD. SNP is used for outgoing snoops.", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN1 Credit Used : RSP on BL", + "EventCode": "0x5c", + "EventName": "UNC_M3UPI_VN1_CREDITS_USED.WB", + "PerPkg": "1", + "PublicDescription": "VN1 Credit Used : RSP on BL : Number of times a VN1 credit was used on the WB message channel. In order for a request to be transferred across QPI, it must be guaranteed to have a flit buffer on the remote socket to sink into. There are two credit pools, VNA and VN1. VNA is a shared pool used to achieve high performance. The VN1 pool has reserved entries for each message class and is used to prevent deadlock. Requests first attempt to acquire a VNA credit, and then fall back to VN1 if they fail. This counts the number of times a VN1 credit was used. Note that a single VN1 credit holds access to potentially multiple flit buffers. For example, a transfer that uses VNA could use 9 flit buffers and in that case uses 9 credits. A transfer on VN1 will only count a single credit even though it may use multiple buffers. : Response (RSP) messages on BL. RSP packets are used to transmit a variety of protocol flits including grants and completions (CMP).", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN1 No Credits : WB on BL", + "EventCode": "0x5e", + "EventName": "UNC_M3UPI_VN1_NO_CREDITS.NCB", + "PerPkg": "1", + "PublicDescription": "VN1 No Credits : WB on BL : Number of Cycles there were no VN1 Credits : Data Response (WB) messages on BL. WB is generally used to transmit data with coherency. For example, remote reads and writes, or cache to cache transfers will transmit their data using WB.", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN1 No Credits : NCB on BL", + "EventCode": "0x5e", + "EventName": "UNC_M3UPI_VN1_NO_CREDITS.NCS", + "PerPkg": "1", + "PublicDescription": "VN1 No Credits : NCB on BL : Number of Cycles there were no VN1 Credits : Non-Coherent Broadcast (NCB) messages on BL. NCB is generally used to transmit data without coherency. For example, non-coherent read data returns.", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN1 No Credits : REQ on AD", + "EventCode": "0x5e", + "EventName": "UNC_M3UPI_VN1_NO_CREDITS.REQ", + "PerPkg": "1", + "PublicDescription": "VN1 No Credits : REQ on AD : Number of Cycles there were no VN1 Credits : Home (REQ) messages on AD. REQ is generally used to send requests, request responses, and snoop responses.", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN1 No Credits : RSP on AD", + "EventCode": "0x5e", + "EventName": "UNC_M3UPI_VN1_NO_CREDITS.RSP", + "PerPkg": "1", + "PublicDescription": "VN1 No Credits : RSP on AD : Number of Cycles there were no VN1 Credits : Response (RSP) messages on AD. RSP packets are used to transmit a variety of protocol flits including grants and completions (CMP).", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN1 No Credits : SNP on AD", + "EventCode": "0x5e", + "EventName": "UNC_M3UPI_VN1_NO_CREDITS.SNP", + "PerPkg": "1", + "PublicDescription": "VN1 No Credits : SNP on AD : Number of Cycles there were no VN1 Credits : Snoops (SNP) messages on AD. SNP is used for outgoing snoops.", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "VN1 No Credits : RSP on BL", + "EventCode": "0x5e", + "EventName": "UNC_M3UPI_VN1_NO_CREDITS.WB", + "PerPkg": "1", + "PublicDescription": "VN1 No Credits : RSP on BL : Number of Cycles there were no VN1 Credits : Response (RSP) messages on BL. RSP packets are used to transmit a variety of protocol flits including grants and completions (CMP).", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UNC_M3UPI_WB_OCC_COMPARE.BOTHNONZERO_RT_EQ_LOCALDEST_VN0", + "EventCode": "0x7e", + "EventName": "UNC_M3UPI_WB_OCC_COMPARE.BOTHNONZERO_RT_EQ_LOCALDEST_VN0", + "PerPkg": "1", + "UMask": "0x82", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UNC_M3UPI_WB_OCC_COMPARE.BOTHNONZERO_RT_EQ_LOCALDEST_VN1", + "EventCode": "0x7e", + "EventName": "UNC_M3UPI_WB_OCC_COMPARE.BOTHNONZERO_RT_EQ_LOCALDEST_VN1", + "PerPkg": "1", + "UMask": "0xa0", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UNC_M3UPI_WB_OCC_COMPARE.BOTHNONZERO_RT_GT_LOCALDEST_VN0", + "EventCode": "0x7e", + "EventName": "UNC_M3UPI_WB_OCC_COMPARE.BOTHNONZERO_RT_GT_LOCALDEST_VN0", + "PerPkg": "1", + "UMask": "0x81", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UNC_M3UPI_WB_OCC_COMPARE.BOTHNONZERO_RT_GT_LOCALDEST_VN1", + "EventCode": "0x7e", + "EventName": "UNC_M3UPI_WB_OCC_COMPARE.BOTHNONZERO_RT_GT_LOCALDEST_VN1", + "PerPkg": "1", + "UMask": "0x90", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UNC_M3UPI_WB_OCC_COMPARE.BOTHNONZERO_RT_LT_LOCALDEST_VN0", + "EventCode": "0x7e", + "EventName": "UNC_M3UPI_WB_OCC_COMPARE.BOTHNONZERO_RT_LT_LOCALDEST_VN0", + "PerPkg": "1", + "UMask": "0x84", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UNC_M3UPI_WB_OCC_COMPARE.BOTHNONZERO_RT_LT_LOCALDEST_VN1", + "EventCode": "0x7e", + "EventName": "UNC_M3UPI_WB_OCC_COMPARE.BOTHNONZERO_RT_LT_LOCALDEST_VN1", + "PerPkg": "1", + "UMask": "0xc0", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UNC_M3UPI_WB_OCC_COMPARE.RT_EQ_LOCALDEST_VN0", + "EventCode": "0x7e", + "EventName": "UNC_M3UPI_WB_OCC_COMPARE.RT_EQ_LOCALDEST_VN0", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UNC_M3UPI_WB_OCC_COMPARE.RT_EQ_LOCALDEST_VN1", + "EventCode": "0x7e", + "EventName": "UNC_M3UPI_WB_OCC_COMPARE.RT_EQ_LOCALDEST_VN1", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UNC_M3UPI_WB_OCC_COMPARE.RT_GT_LOCALDEST_VN0", + "EventCode": "0x7e", + "EventName": "UNC_M3UPI_WB_OCC_COMPARE.RT_GT_LOCALDEST_VN0", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UNC_M3UPI_WB_OCC_COMPARE.RT_GT_LOCALDEST_VN1", + "EventCode": "0x7e", + "EventName": "UNC_M3UPI_WB_OCC_COMPARE.RT_GT_LOCALDEST_VN1", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UNC_M3UPI_WB_OCC_COMPARE.RT_LT_LOCALDEST_VN0", + "EventCode": "0x7e", + "EventName": "UNC_M3UPI_WB_OCC_COMPARE.RT_LT_LOCALDEST_VN0", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UNC_M3UPI_WB_OCC_COMPARE.RT_LT_LOCALDEST_VN1", + "EventCode": "0x7e", + "EventName": "UNC_M3UPI_WB_OCC_COMPARE.RT_LT_LOCALDEST_VN1", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UNC_M3UPI_WB_PENDING.LOCALDEST_VN0", + "EventCode": "0x7d", + "EventName": "UNC_M3UPI_WB_PENDING.LOCALDEST_VN0", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UNC_M3UPI_WB_PENDING.LOCALDEST_VN1", + "EventCode": "0x7d", + "EventName": "UNC_M3UPI_WB_PENDING.LOCALDEST_VN1", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UNC_M3UPI_WB_PENDING.LOCAL_AND_RT_VN0", + "EventCode": "0x7d", + "EventName": "UNC_M3UPI_WB_PENDING.LOCAL_AND_RT_VN0", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UNC_M3UPI_WB_PENDING.LOCAL_AND_RT_VN1", + "EventCode": "0x7d", + "EventName": "UNC_M3UPI_WB_PENDING.LOCAL_AND_RT_VN1", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UNC_M3UPI_WB_PENDING.ROUTETHRU_VN0", + "EventCode": "0x7d", + "EventName": "UNC_M3UPI_WB_PENDING.ROUTETHRU_VN0", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UNC_M3UPI_WB_PENDING.ROUTETHRU_VN1", + "EventCode": "0x7d", + "EventName": "UNC_M3UPI_WB_PENDING.ROUTETHRU_VN1", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UNC_M3UPI_WB_PENDING.WAITING4PULL_VN0", + "EventCode": "0x7d", + "EventName": "UNC_M3UPI_WB_PENDING.WAITING4PULL_VN0", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UNC_M3UPI_WB_PENDING.WAITING4PULL_VN1", + "EventCode": "0x7d", + "EventName": "UNC_M3UPI_WB_PENDING.WAITING4PULL_VN1", + "PerPkg": "1", + "UMask": "0x80", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UNC_M3UPI_XPT_PFTCH.ARB", + "EventCode": "0x61", + "EventName": "UNC_M3UPI_XPT_PFTCH.ARB", + "PerPkg": "1", + "PublicDescription": ": xpt prefetch message is making arbitration request", + "UMask": "0x4", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UNC_M3UPI_XPT_PFTCH.ARRIVED", + "EventCode": "0x61", + "EventName": "UNC_M3UPI_XPT_PFTCH.ARRIVED", + "PerPkg": "1", + "PublicDescription": ": xpt prefetch message arrived in ingress pipeline", + "UMask": "0x1", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UNC_M3UPI_XPT_PFTCH.BYPASS", + "EventCode": "0x61", + "EventName": "UNC_M3UPI_XPT_PFTCH.BYPASS", + "PerPkg": "1", + "PublicDescription": ": xpt prefetch message took bypass path", + "UMask": "0x2", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UNC_M3UPI_XPT_PFTCH.FLITTED", + "EventCode": "0x61", + "EventName": "UNC_M3UPI_XPT_PFTCH.FLITTED", + "PerPkg": "1", + "PublicDescription": ": xpt prefetch message was slotted into flit (non bypass)", + "UMask": "0x10", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UNC_M3UPI_XPT_PFTCH.LOST_ARB", + "EventCode": "0x61", + "EventName": "UNC_M3UPI_XPT_PFTCH.LOST_ARB", + "PerPkg": "1", + "PublicDescription": ": xpt prefetch message lost arbitration", + "UMask": "0x8", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UNC_M3UPI_XPT_PFTCH.LOST_OLD", + "EventCode": "0x61", + "EventName": "UNC_M3UPI_XPT_PFTCH.LOST_OLD", + "PerPkg": "1", + "PublicDescription": ": xpt prefetch message was dropped because it became too old", + "UMask": "0x20", + "Unit": "M3UPI" + }, + { + "BriefDescription": "UNC_M3UPI_XPT_PFTCH.LOST_QFULL", + "EventCode": "0x61", + "EventName": "UNC_M3UPI_XPT_PFTCH.LOST_QFULL", + "PerPkg": "1", + "PublicDescription": ": xpt prefetch message was dropped because it was overwritten by new message while prefetch queue was full", + "UMask": "0x40", + "Unit": "M3UPI" + }, + { + "BriefDescription": "Number of allocations into the CRS Egress used to queue up requests destined to the mesh (AD Bouncable)", + "EventCode": "0x47", + "EventName": "UNC_MDF_CRS_TxR_INSERTS.AD_BNC", + "PerPkg": "1", + "PublicDescription": "AD Bouncable : Number of allocations into the CRS Egress", + "UMask": "0x1", + "Unit": "MDF" + }, + { + "BriefDescription": "Number of allocations into the CRS Egress used to queue up requests destined to the mesh (AD credited)", + "EventCode": "0x47", + "EventName": "UNC_MDF_CRS_TxR_INSERTS.AD_CRD", + "PerPkg": "1", + "PublicDescription": "AD credited : Number of allocations into the CRS Egress", + "UMask": "0x2", + "Unit": "MDF" + }, + { + "BriefDescription": "Number of allocations into the CRS Egress used to queue up requests destined to the mesh (AK)", + "EventCode": "0x47", + "EventName": "UNC_MDF_CRS_TxR_INSERTS.AK", + "PerPkg": "1", + "PublicDescription": "AK : Number of allocations into the CRS Egress", + "UMask": "0x10", + "Unit": "MDF" + }, + { + "BriefDescription": "Number of allocations into the CRS Egress used to queue up requests destined to the mesh (AKC)", + "EventCode": "0x47", + "EventName": "UNC_MDF_CRS_TxR_INSERTS.AKC", + "PerPkg": "1", + "PublicDescription": "AKC : Number of allocations into the CRS Egress", + "UMask": "0x40", + "Unit": "MDF" + }, + { + "BriefDescription": "Number of allocations into the CRS Egress used to queue up requests destined to the mesh (BL Bouncable)", + "EventCode": "0x47", + "EventName": "UNC_MDF_CRS_TxR_INSERTS.BL_BNC", + "PerPkg": "1", + "PublicDescription": "BL Bouncable : Number of allocations into the CRS Egress", + "UMask": "0x4", + "Unit": "MDF" + }, + { + "BriefDescription": "Number of allocations into the CRS Egress used to queue up requests destined to the mesh (BL credited)", + "EventCode": "0x47", + "EventName": "UNC_MDF_CRS_TxR_INSERTS.BL_CRD", + "PerPkg": "1", + "PublicDescription": "BL credited : Number of allocations into the CRS Egress", + "UMask": "0x8", + "Unit": "MDF" + }, + { + "BriefDescription": "Number of allocations into the CRS Egress used to queue up requests destined to the mesh (IV)", + "EventCode": "0x47", + "EventName": "UNC_MDF_CRS_TxR_INSERTS.IV", + "PerPkg": "1", + "PublicDescription": "IV : Number of allocations into the CRS Egress", + "UMask": "0x20", + "Unit": "MDF" + }, + { + "BriefDescription": "Number of cycles incoming messages from the vertical ring that are bounced at the SBO\r\nIngress (V-EMIB) (AD)", + "EventCode": "0x4B", + "EventName": "UNC_MDF_CRS_TxR_V_BOUNCES.AD", + "PerPkg": "1", + "PublicDescription": "AD : Number of cycles incoming messages from the vertical ring that are bounced at the SBO", + "UMask": "0x1", + "Unit": "MDF" + }, + { + "BriefDescription": "Number of cycles incoming messages from the vertical ring that are bounced at the SBO\r\nIngress (V-EMIB) (AK)", + "EventCode": "0x4B", + "EventName": "UNC_MDF_CRS_TxR_V_BOUNCES.AK", + "PerPkg": "1", + "PublicDescription": "AK : Number of cycles incoming messages from the vertical ring that are bounced at the SBO", + "UMask": "0x4", + "Unit": "MDF" + }, + { + "BriefDescription": "Number of cycles incoming messages from the vertical ring that are bounced at the SBO\r\nIngress (V-EMIB) (AKC)", + "EventCode": "0x4B", + "EventName": "UNC_MDF_CRS_TxR_V_BOUNCES.AKC", + "PerPkg": "1", + "PublicDescription": "AKC : Number of cycles incoming messages from the vertical ring that are bounced at the SBO", + "UMask": "0x10", + "Unit": "MDF" + }, + { + "BriefDescription": "Number of cycles incoming messages from the vertical ring that are bounced at the SBO\r\nIngress (V-EMIB) (BL)", + "EventCode": "0x4B", + "EventName": "UNC_MDF_CRS_TxR_V_BOUNCES.BL", + "PerPkg": "1", + "PublicDescription": "BL : Number of cycles incoming messages from the vertical ring that are bounced at the SBO", + "UMask": "0x2", + "Unit": "MDF" + }, + { + "BriefDescription": "Number of cycles incoming messages from the vertical ring that are bounced at the SBO\r\nIngress (V-EMIB) (IV)", + "EventCode": "0x4B", + "EventName": "UNC_MDF_CRS_TxR_V_BOUNCES.IV", + "PerPkg": "1", + "PublicDescription": "IV : Number of cycles incoming messages from the vertical ring that are bounced at the SBO", + "UMask": "0x8", + "Unit": "MDF" + }, + { + "BriefDescription": "Counts the number of cycles when the distress signals are asserted based on SBO Ingress threshold", + "EventCode": "0x15", + "EventName": "UNC_MDF_FAST_ASSERTED.AD_BNC", + "PerPkg": "1", + "PublicDescription": "AD bnc : Counts the number of cycles when the distress signals are asserted based on SBO Ingress threshold", + "UMask": "0x1", + "Unit": "MDF" + }, + { + "BriefDescription": "Counts the number of cycles when the distress signals are asserted based on SBO Ingress threshold", + "EventCode": "0x15", + "EventName": "UNC_MDF_FAST_ASSERTED.BL_CRD", + "PerPkg": "1", + "PublicDescription": "BL bnc : Counts the number of cycles when the distress signals are asserted based on SBO Ingress threshold", + "UMask": "0x2", + "Unit": "MDF" + }, + { + "BriefDescription": "UPI Clockticks", + "EventCode": "0x01", + "EventName": "UNC_UPI_CLOCKTICKS", + "PerPkg": "1", + "PublicDescription": "Number of UPI LL clock cycles while the event is enabled", + "Unit": "UPI" + }, + { + "BriefDescription": "Direct packet attempts : D2C", + "EventCode": "0x12", + "EventName": "UNC_UPI_DIRECT_ATTEMPTS.D2C", + "PerPkg": "1", + "PublicDescription": "Direct packet attempts : D2C : Counts the number of DRS packets that we attempted to do direct2core/direct2UPI on. There are 4 mutually exclusive filters. Filter [0] can be used to get successful spawns, while [1:3] provide the different failure cases. Note that this does not count packets that are not candidates for Direct2Core. The only candidates for Direct2Core are DRS packets destined for Cbos.", + "UMask": "0x1", + "Unit": "UPI" + }, + { + "BriefDescription": "Direct packet attempts : D2K", + "EventCode": "0x12", + "EventName": "UNC_UPI_DIRECT_ATTEMPTS.D2K", + "PerPkg": "1", + "PublicDescription": "Direct packet attempts : D2K : Counts the number of DRS packets that we attempted to do direct2core/direct2UPI on. There are 4 mutually exclusive filters. Filter [0] can be used to get successful spawns, while [1:3] provide the different failure cases. Note that this does not count packets that are not candidates for Direct2Core. The only candidates for Direct2Core are DRS packets destined for Cbos.", + "UMask": "0x2", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_FLOWQ_NO_VNA_CRD.AD_VNA_EQ0", + "EventCode": "0x18", + "EventName": "UNC_UPI_FLOWQ_NO_VNA_CRD.AD_VNA_EQ0", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_FLOWQ_NO_VNA_CRD.AD_VNA_EQ1", + "EventCode": "0x18", + "EventName": "UNC_UPI_FLOWQ_NO_VNA_CRD.AD_VNA_EQ1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_FLOWQ_NO_VNA_CRD.AD_VNA_EQ2", + "EventCode": "0x18", + "EventName": "UNC_UPI_FLOWQ_NO_VNA_CRD.AD_VNA_EQ2", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_FLOWQ_NO_VNA_CRD.AK_VNA_EQ0", + "EventCode": "0x18", + "EventName": "UNC_UPI_FLOWQ_NO_VNA_CRD.AK_VNA_EQ0", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_FLOWQ_NO_VNA_CRD.AK_VNA_EQ1", + "EventCode": "0x18", + "EventName": "UNC_UPI_FLOWQ_NO_VNA_CRD.AK_VNA_EQ1", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_FLOWQ_NO_VNA_CRD.AK_VNA_EQ2", + "EventCode": "0x18", + "EventName": "UNC_UPI_FLOWQ_NO_VNA_CRD.AK_VNA_EQ2", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_FLOWQ_NO_VNA_CRD.AK_VNA_EQ3", + "EventCode": "0x18", + "EventName": "UNC_UPI_FLOWQ_NO_VNA_CRD.AK_VNA_EQ3", + "PerPkg": "1", + "UMask": "0x80", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_FLOWQ_NO_VNA_CRD.BL_VNA_EQ0", + "EventCode": "0x18", + "EventName": "UNC_UPI_FLOWQ_NO_VNA_CRD.BL_VNA_EQ0", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "UPI" + }, + { + "BriefDescription": "Cycles in L1", + "EventCode": "0x21", + "EventName": "UNC_UPI_L1_POWER_CYCLES", + "PerPkg": "1", + "PublicDescription": "Cycles in L1 : Number of UPI qfclk cycles spent in L1 power mode. L1 is a mode that totally shuts down a UPI link. Use edge detect to count the number of instances when the UPI link entered L1. Link power states are per link and per direction, so for example the Tx direction could be in one state while Rx was in another. Because L1 totally shuts down the link, it takes a good amount of time to exit this mode.", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_M3_BYP_BLOCKED.BGF_CRD", + "EventCode": "0x14", + "EventName": "UNC_UPI_M3_BYP_BLOCKED.BGF_CRD", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_M3_BYP_BLOCKED.FLOWQ_AD_VNA_LE2", + "EventCode": "0x14", + "EventName": "UNC_UPI_M3_BYP_BLOCKED.FLOWQ_AD_VNA_LE2", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_M3_BYP_BLOCKED.FLOWQ_AK_VNA_LE3", + "EventCode": "0x14", + "EventName": "UNC_UPI_M3_BYP_BLOCKED.FLOWQ_AK_VNA_LE3", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_M3_BYP_BLOCKED.FLOWQ_BL_VNA_EQ0", + "EventCode": "0x14", + "EventName": "UNC_UPI_M3_BYP_BLOCKED.FLOWQ_BL_VNA_EQ0", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_M3_BYP_BLOCKED.GV_BLOCK", + "EventCode": "0x14", + "EventName": "UNC_UPI_M3_BYP_BLOCKED.GV_BLOCK", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_M3_CRD_RETURN_BLOCKED", + "EventCode": "0x16", + "EventName": "UNC_UPI_M3_CRD_RETURN_BLOCKED", + "PerPkg": "1", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_M3_RXQ_BLOCKED.BGF_CRD", + "EventCode": "0x15", + "EventName": "UNC_UPI_M3_RXQ_BLOCKED.BGF_CRD", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_M3_RXQ_BLOCKED.FLOWQ_AD_VNA_BTW_2_THRESH", + "EventCode": "0x15", + "EventName": "UNC_UPI_M3_RXQ_BLOCKED.FLOWQ_AD_VNA_BTW_2_THRESH", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_M3_RXQ_BLOCKED.FLOWQ_AD_VNA_LE2", + "EventCode": "0x15", + "EventName": "UNC_UPI_M3_RXQ_BLOCKED.FLOWQ_AD_VNA_LE2", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_M3_RXQ_BLOCKED.FLOWQ_AK_VNA_LE3", + "EventCode": "0x15", + "EventName": "UNC_UPI_M3_RXQ_BLOCKED.FLOWQ_AK_VNA_LE3", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_M3_RXQ_BLOCKED.FLOWQ_BL_VNA_BTW_0_THRESH", + "EventCode": "0x15", + "EventName": "UNC_UPI_M3_RXQ_BLOCKED.FLOWQ_BL_VNA_BTW_0_THRESH", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_M3_RXQ_BLOCKED.FLOWQ_BL_VNA_EQ0", + "EventCode": "0x15", + "EventName": "UNC_UPI_M3_RXQ_BLOCKED.FLOWQ_BL_VNA_EQ0", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_M3_RXQ_BLOCKED.GV_BLOCK", + "EventCode": "0x15", + "EventName": "UNC_UPI_M3_RXQ_BLOCKED.GV_BLOCK", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "UPI" + }, + { + "BriefDescription": "Cycles where phy is not in L0, L0c, L0p, L1", + "EventCode": "0x20", + "EventName": "UNC_UPI_PHY_INIT_CYCLES", + "PerPkg": "1", + "Unit": "UPI" + }, + { + "BriefDescription": "L1 Req Nack", + "EventCode": "0x23", + "EventName": "UNC_UPI_POWER_L1_NACK", + "PerPkg": "1", + "PublicDescription": "L1 Req Nack : Counts the number of times a link sends/receives a LinkReqNAck. When the UPI links would like to change power state, the Tx side initiates a request to the Rx side requesting to change states. This requests can either be accepted or denied. If the Rx side replies with an Ack, the power mode will change. If it replies with NAck, no change will take place. This can be filtered based on Rx and Tx. An Rx LinkReqNAck refers to receiving an NAck (meaning this agent's Tx originally requested the power change). A Tx LinkReqNAck refers to sending this command (meaning the peer agent's Tx originally requested the power change and this agent accepted it).", + "Unit": "UPI" + }, + { + "BriefDescription": "L1 Req (same as L1 Ack).", + "EventCode": "0x22", + "EventName": "UNC_UPI_POWER_L1_REQ", + "PerPkg": "1", + "PublicDescription": "L1 Req (same as L1 Ack). : Counts the number of times a link sends/receives a LinkReqAck. When the UPI links would like to change power state, the Tx side initiates a request to the Rx side requesting to change states. This requests can either be accepted or denied. If the Rx side replies with an Ack, the power mode will change. If it replies with NAck, no change will take place. This can be filtered based on Rx and Tx. An Rx LinkReqAck refers to receiving an Ack (meaning this agent's Tx originally requested the power change). A Tx LinkReqAck refers to sending this command (meaning the peer agent's Tx originally requested the power change and this agent accepted it).", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_REQ_SLOT2_FROM_M3.ACK", + "EventCode": "0x46", + "EventName": "UNC_UPI_REQ_SLOT2_FROM_M3.ACK", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_REQ_SLOT2_FROM_M3.VN0", + "EventCode": "0x46", + "EventName": "UNC_UPI_REQ_SLOT2_FROM_M3.VN0", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_REQ_SLOT2_FROM_M3.VN1", + "EventCode": "0x46", + "EventName": "UNC_UPI_REQ_SLOT2_FROM_M3.VN1", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_REQ_SLOT2_FROM_M3.VNA", + "EventCode": "0x46", + "EventName": "UNC_UPI_REQ_SLOT2_FROM_M3.VNA", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "UPI" + }, + { + "BriefDescription": "Cycles in L0p", + "EventCode": "0x25", + "EventName": "UNC_UPI_RxL0P_POWER_CYCLES", + "PerPkg": "1", + "PublicDescription": "Cycles in L0p : Number of UPI qfclk cycles spent in L0p power mode. L0p is a mode where we disable 1/2 of the UPI lanes, decreasing our bandwidth in order to save power. It increases snoop and data transfer latencies and decreases overall bandwidth. This mode can be very useful in NUMA optimized workloads that largely only utilize UPI for snoops and their responses. Use edge detect to count the number of instances when the UPI link entered L0p. Link power states are per link and per direction, so for example the Tx direction could be in one state while Rx was in another.", + "Unit": "UPI" + }, + { + "BriefDescription": "Cycles in L0", + "EventCode": "0x24", + "EventName": "UNC_UPI_RxL0_POWER_CYCLES", + "PerPkg": "1", + "PublicDescription": "Cycles in L0 : Number of UPI qfclk cycles spent in L0 power mode in the Link Layer. L0 is the default mode which provides the highest performance with the most power. Use edge detect to count the number of instances that the link entered L0. Link power states are per link and per direction, so for example the Tx direction could be in one state while Rx was in another. The phy layer sometimes leaves L0 for training, which will not be captured by this event.", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_RxL_ANY_FLITS.DATA", + "EventCode": "0x4B", + "EventName": "UNC_UPI_RxL_ANY_FLITS.DATA", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_RxL_ANY_FLITS.LLCRD", + "EventCode": "0x4B", + "EventName": "UNC_UPI_RxL_ANY_FLITS.LLCRD", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_RxL_ANY_FLITS.LLCTRL", + "EventCode": "0x4B", + "EventName": "UNC_UPI_RxL_ANY_FLITS.LLCTRL", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_RxL_ANY_FLITS.NULL", + "EventCode": "0x4B", + "EventName": "UNC_UPI_RxL_ANY_FLITS.NULL", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_RxL_ANY_FLITS.PROTHDR", + "EventCode": "0x4B", + "EventName": "UNC_UPI_RxL_ANY_FLITS.PROTHDR", + "PerPkg": "1", + "UMask": "0x80", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_RxL_ANY_FLITS.SLOT0", + "EventCode": "0x4B", + "EventName": "UNC_UPI_RxL_ANY_FLITS.SLOT0", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_RxL_ANY_FLITS.SLOT1", + "EventCode": "0x4B", + "EventName": "UNC_UPI_RxL_ANY_FLITS.SLOT1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_RxL_ANY_FLITS.SLOT2", + "EventCode": "0x4B", + "EventName": "UNC_UPI_RxL_ANY_FLITS.SLOT2", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Receive path of a UPI Port : Non-Coherent Bypass", + "EventCode": "0x05", + "EventName": "UNC_UPI_RxL_BASIC_HDR_MATCH.NCB", + "PerPkg": "1", + "PublicDescription": "Matches on Receive path of a UPI Port : Non-Coherent Bypass : Matches on Receive path of a UPI port.\r\nMatch based on UMask specific bits:\r\nZ: Message Class (3-bit)\r\nY: Message Class Enable\r\nW: Opcode (4-bit)\r\nV: Opcode Enable\r\nU: Local Enable\r\nT: Remote Enable\r\nS: Data Hdr Enable\r\nR: Non-Data Hdr Enable\r\nQ: Dual Slot Hdr Enable\r\nP: Single Slot Hdr Enable\r\nLink Layer control types are excluded (LL CTRL, slot NULL, LLCRD) even under specific opcode match_en cases.\r\nNote: If Message Class is disabled, we expect opcode to also be disabled.", + "UMask": "0xe", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Receive path of a UPI Port : Non-Coherent Bypass, Match Opcode", + "EventCode": "0x05", + "EventName": "UNC_UPI_RxL_BASIC_HDR_MATCH.NCB_OPC", + "PerPkg": "1", + "PublicDescription": "Matches on Receive path of a UPI Port : Non-Coherent Bypass, Match Opcode : Matches on Receive path of a UPI port.\r\nMatch based on UMask specific bits:\r\nZ: Message Class (3-bit)\r\nY: Message Class Enable\r\nW: Opcode (4-bit)\r\nV: Opcode Enable\r\nU: Local Enable\r\nT: Remote Enable\r\nS: Data Hdr Enable\r\nR: Non-Data Hdr Enable\r\nQ: Dual Slot Hdr Enable\r\nP: Single Slot Hdr Enable\r\nLink Layer control types are excluded (LL CTRL, slot NULL, LLCRD) even under specific opcode match_en cases.\r\nNote: If Message Class is disabled, we expect opcode to also be disabled.", + "UMask": "0x10e", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Receive path of a UPI Port : Non-Coherent Standard", + "EventCode": "0x05", + "EventName": "UNC_UPI_RxL_BASIC_HDR_MATCH.NCS", + "PerPkg": "1", + "PublicDescription": "Matches on Receive path of a UPI Port : Non-Coherent Standard : Matches on Receive path of a UPI port.\r\nMatch based on UMask specific bits:\r\nZ: Message Class (3-bit)\r\nY: Message Class Enable\r\nW: Opcode (4-bit)\r\nV: Opcode Enable\r\nU: Local Enable\r\nT: Remote Enable\r\nS: Data Hdr Enable\r\nR: Non-Data Hdr Enable\r\nQ: Dual Slot Hdr Enable\r\nP: Single Slot Hdr Enable\r\nLink Layer control types are excluded (LL CTRL, slot NULL, LLCRD) even under specific opcode match_en cases.\r\nNote: If Message Class is disabled, we expect opcode to also be disabled.", + "UMask": "0xf", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Receive path of a UPI Port : Non-Coherent Standard, Match Opcode", + "EventCode": "0x05", + "EventName": "UNC_UPI_RxL_BASIC_HDR_MATCH.NCS_OPC", + "PerPkg": "1", + "PublicDescription": "Matches on Receive path of a UPI Port : Non-Coherent Standard, Match Opcode : Matches on Receive path of a UPI port.\r\nMatch based on UMask specific bits:\r\nZ: Message Class (3-bit)\r\nY: Message Class Enable\r\nW: Opcode (4-bit)\r\nV: Opcode Enable\r\nU: Local Enable\r\nT: Remote Enable\r\nS: Data Hdr Enable\r\nR: Non-Data Hdr Enable\r\nQ: Dual Slot Hdr Enable\r\nP: Single Slot Hdr Enable\r\nLink Layer control types are excluded (LL CTRL, slot NULL, LLCRD) even under specific opcode match_en cases.\r\nNote: If Message Class is disabled, we expect opcode to also be disabled.", + "UMask": "0x10f", + "Unit": "UPI" + }, + { + "BriefDescription": "RxQ Flit Buffer Bypassed : Slot 0", + "EventCode": "0x31", + "EventName": "UNC_UPI_RxL_BYPASSED.SLOT0", + "PerPkg": "1", + "PublicDescription": "RxQ Flit Buffer Bypassed : Slot 0 : Counts the number of times that an incoming flit was able to bypass the flit buffer and pass directly across the BGF and into the Egress. This is a latency optimization, and should generally be the common case. If this value is less than the number of flits transferred, it implies that there was queueing getting onto the ring, and thus the transactions saw higher latency.", + "UMask": "0x1", + "Unit": "UPI" + }, + { + "BriefDescription": "RxQ Flit Buffer Bypassed : Slot 1", + "EventCode": "0x31", + "EventName": "UNC_UPI_RxL_BYPASSED.SLOT1", + "PerPkg": "1", + "PublicDescription": "RxQ Flit Buffer Bypassed : Slot 1 : Counts the number of times that an incoming flit was able to bypass the flit buffer and pass directly across the BGF and into the Egress. This is a latency optimization, and should generally be the common case. If this value is less than the number of flits transferred, it implies that there was queueing getting onto the ring, and thus the transactions saw higher latency.", + "UMask": "0x2", + "Unit": "UPI" + }, + { + "BriefDescription": "RxQ Flit Buffer Bypassed : Slot 2", + "EventCode": "0x31", + "EventName": "UNC_UPI_RxL_BYPASSED.SLOT2", + "PerPkg": "1", + "PublicDescription": "RxQ Flit Buffer Bypassed : Slot 2 : Counts the number of times that an incoming flit was able to bypass the flit buffer and pass directly across the BGF and into the Egress. This is a latency optimization, and should generally be the common case. If this value is less than the number of flits transferred, it implies that there was queueing getting onto the ring, and thus the transactions saw higher latency.", + "UMask": "0x4", + "Unit": "UPI" + }, + { + "BriefDescription": "CRC Errors Detected", + "EventCode": "0x0b", + "EventName": "UNC_UPI_RxL_CRC_ERRORS", + "PerPkg": "1", + "PublicDescription": "CRC Errors Detected : Number of CRC errors detected in the UPI Agent. Each UPI flit incorporates 8 bits of CRC for error detection. This counts the number of flits where the CRC was able to detect an error. After an error has been detected, the UPI agent will send a request to the transmitting socket to resend the flit (as well as any flits that came after it).", + "Unit": "UPI" + }, + { + "BriefDescription": "LLR Requests Sent", + "EventCode": "0x08", + "EventName": "UNC_UPI_RxL_CRC_LLR_REQ_TRANSMIT", + "PerPkg": "1", + "PublicDescription": "LLR Requests Sent : Number of LLR Requests were transmitted. This should generally be <= the number of CRC errors detected. If multiple errors are detected before the Rx side receives a LLC_REQ_ACK from the Tx side, there is no need to send more LLR_REQ_NACKs..", + "Unit": "UPI" + }, + { + "BriefDescription": "VN0 Credit Consumed", + "EventCode": "0x39", + "EventName": "UNC_UPI_RxL_CREDITS_CONSUMED_VN0", + "PerPkg": "1", + "PublicDescription": "VN0 Credit Consumed : Counts the number of times that an RxQ VN0 credit was consumed (i.e. message uses a VN0 credit for the Rx Buffer). This includes packets that went through the RxQ and those that were bypasssed.", + "Unit": "UPI" + }, + { + "BriefDescription": "VN1 Credit Consumed", + "EventCode": "0x3a", + "EventName": "UNC_UPI_RxL_CREDITS_CONSUMED_VN1", + "PerPkg": "1", + "PublicDescription": "VN1 Credit Consumed : Counts the number of times that an RxQ VN1 credit was consumed (i.e. message uses a VN1 credit for the Rx Buffer). This includes packets that went through the RxQ and those that were bypasssed.", + "Unit": "UPI" + }, + { + "BriefDescription": "VNA Credit Consumed", + "EventCode": "0x38", + "EventName": "UNC_UPI_RxL_CREDITS_CONSUMED_VNA", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "PublicDescription": "Counts the number of times that an RxQ VNA credit was consumed (i.e. message uses a VNA credit for the Rx Buffer). This includes packets that went through the RxQ and those that were bypasssed.", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Received : All Data", + "EventCode": "0x03", + "EventName": "UNC_UPI_RxL_FLITS.ALL_DATA", + "PerPkg": "1", + "PublicDescription": "Valid Flits Received : All Data : Shows legal flit time (hides impact of L0p and L0c).", + "UMask": "0xf", + "Unit": "UPI" + }, + { + "BriefDescription": "Null FLITs received from any slot", + "EventCode": "0x03", + "EventName": "UNC_UPI_RxL_FLITS.ALL_NULL", + "PerPkg": "1", + "UMask": "0x27", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Received : Data", + "EventCode": "0x03", + "EventName": "UNC_UPI_RxL_FLITS.DATA", + "PerPkg": "1", + "PublicDescription": "Valid Flits Received : Data : Shows legal flit time (hides impact of L0p and L0c). : Count Data Flits (which consume all slots), but how much to count is based on Slot0-2 mask, so count can be 0-3 depending on which slots are enabled for counting..", + "UMask": "0x8", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Received : Idle", + "EventCode": "0x03", + "EventName": "UNC_UPI_RxL_FLITS.IDLE", + "PerPkg": "1", + "PublicDescription": "Valid Flits Received : Idle : Shows legal flit time (hides impact of L0p and L0c).", + "UMask": "0x47", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Received : LLCRD Not Empty", + "EventCode": "0x03", + "EventName": "UNC_UPI_RxL_FLITS.LLCRD", + "PerPkg": "1", + "PublicDescription": "Valid Flits Received : LLCRD Not Empty : Shows legal flit time (hides impact of L0p and L0c). : Enables counting of LLCRD (with non-zero payload). This only applies to slot 2 since LLCRD is only allowed in slot 2", + "UMask": "0x10", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Received : LLCTRL", + "EventCode": "0x03", + "EventName": "UNC_UPI_RxL_FLITS.LLCTRL", + "PerPkg": "1", + "PublicDescription": "Valid Flits Received : LLCTRL : Shows legal flit time (hides impact of L0p and L0c). : Equivalent to an idle packet. Enables counting of slot 0 LLCTRL messages.", + "UMask": "0x40", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Received : All Non Data", + "EventCode": "0x03", + "EventName": "UNC_UPI_RxL_FLITS.NON_DATA", + "PerPkg": "1", + "PublicDescription": "Valid Flits Received : All Non Data : Shows legal flit time (hides impact of L0p and L0c).", + "UMask": "0x97", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Received : Slot NULL or LLCRD Empty", + "EventCode": "0x03", + "EventName": "UNC_UPI_RxL_FLITS.NULL", + "PerPkg": "1", + "PublicDescription": "Valid Flits Received : Slot NULL or LLCRD Empty : Shows legal flit time (hides impact of L0p and L0c). : LLCRD with all zeros is treated as NULL. Slot 1 is not treated as NULL if slot 0 is a dual slot. This can apply to slot 0,1, or 2.", + "UMask": "0x20", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Received : Protocol Header", + "EventCode": "0x03", + "EventName": "UNC_UPI_RxL_FLITS.PROTHDR", + "PerPkg": "1", + "PublicDescription": "Valid Flits Received : Protocol Header : Shows legal flit time (hides impact of L0p and L0c). : Enables count of protocol headers in slot 0,1,2 (depending on slot uMask bits)", + "UMask": "0x80", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Received : Slot 0", + "EventCode": "0x03", + "EventName": "UNC_UPI_RxL_FLITS.SLOT0", + "PerPkg": "1", + "PublicDescription": "Valid Flits Received : Slot 0 : Shows legal flit time (hides impact of L0p and L0c). : Count Slot 0 - Other mask bits determine types of headers to count.", + "UMask": "0x1", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Received : Slot 1", + "EventCode": "0x03", + "EventName": "UNC_UPI_RxL_FLITS.SLOT1", + "PerPkg": "1", + "PublicDescription": "Valid Flits Received : Slot 1 : Shows legal flit time (hides impact of L0p and L0c). : Count Slot 1 - Other mask bits determine types of headers to count.", + "UMask": "0x2", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Received : Slot 2", + "EventCode": "0x03", + "EventName": "UNC_UPI_RxL_FLITS.SLOT2", + "PerPkg": "1", + "PublicDescription": "Valid Flits Received : Slot 2 : Shows legal flit time (hides impact of L0p and L0c). : Count Slot 2 - Other mask bits determine types of headers to count.", + "UMask": "0x4", + "Unit": "UPI" + }, + { + "BriefDescription": "RxQ Flit Buffer Allocations : Slot 0", + "EventCode": "0x30", + "EventName": "UNC_UPI_RxL_INSERTS.SLOT0", + "PerPkg": "1", + "PublicDescription": "RxQ Flit Buffer Allocations : Slot 0 : Number of allocations into the UPI Rx Flit Buffer. Generally, when data is transmitted across UPI, it will bypass the RxQ and pass directly to the ring interface. If things back up getting transmitted onto the ring, however, it may need to allocate into this buffer, thus increasing the latency. This event can be used in conjunction with the Flit Buffer Occupancy event in order to calculate the average flit buffer lifetime.", + "UMask": "0x1", + "Unit": "UPI" + }, + { + "BriefDescription": "RxQ Flit Buffer Allocations : Slot 1", + "EventCode": "0x30", + "EventName": "UNC_UPI_RxL_INSERTS.SLOT1", + "PerPkg": "1", + "PublicDescription": "RxQ Flit Buffer Allocations : Slot 1 : Number of allocations into the UPI Rx Flit Buffer. Generally, when data is transmitted across UPI, it will bypass the RxQ and pass directly to the ring interface. If things back up getting transmitted onto the ring, however, it may need to allocate into this buffer, thus increasing the latency. This event can be used in conjunction with the Flit Buffer Occupancy event in order to calculate the average flit buffer lifetime.", + "UMask": "0x2", + "Unit": "UPI" + }, + { + "BriefDescription": "RxQ Flit Buffer Allocations : Slot 2", + "EventCode": "0x30", + "EventName": "UNC_UPI_RxL_INSERTS.SLOT2", + "PerPkg": "1", + "PublicDescription": "RxQ Flit Buffer Allocations : Slot 2 : Number of allocations into the UPI Rx Flit Buffer. Generally, when data is transmitted across UPI, it will bypass the RxQ and pass directly to the ring interface. If things back up getting transmitted onto the ring, however, it may need to allocate into this buffer, thus increasing the latency. This event can be used in conjunction with the Flit Buffer Occupancy event in order to calculate the average flit buffer lifetime.", + "UMask": "0x4", + "Unit": "UPI" + }, + { + "BriefDescription": "RxQ Occupancy - All Packets : Slot 0", + "EventCode": "0x32", + "EventName": "UNC_UPI_RxL_OCCUPANCY.SLOT0", + "PerPkg": "1", + "PublicDescription": "RxQ Occupancy - All Packets : Slot 0 : Accumulates the number of elements in the UPI RxQ in each cycle. Generally, when data is transmitted across UPI, it will bypass the RxQ and pass directly to the ring interface. If things back up getting transmitted onto the ring, however, it may need to allocate into this buffer, thus increasing the latency. This event can be used in conjunction with the Flit Buffer Not Empty event to calculate average occupancy, or with the Flit Buffer Allocations event to track average lifetime.", + "UMask": "0x1", + "Unit": "UPI" + }, + { + "BriefDescription": "RxQ Occupancy - All Packets : Slot 1", + "EventCode": "0x32", + "EventName": "UNC_UPI_RxL_OCCUPANCY.SLOT1", + "PerPkg": "1", + "PublicDescription": "RxQ Occupancy - All Packets : Slot 1 : Accumulates the number of elements in the UPI RxQ in each cycle. Generally, when data is transmitted across UPI, it will bypass the RxQ and pass directly to the ring interface. If things back up getting transmitted onto the ring, however, it may need to allocate into this buffer, thus increasing the latency. This event can be used in conjunction with the Flit Buffer Not Empty event to calculate average occupancy, or with the Flit Buffer Allocations event to track average lifetime.", + "UMask": "0x2", + "Unit": "UPI" + }, + { + "BriefDescription": "RxQ Occupancy - All Packets : Slot 2", + "EventCode": "0x32", + "EventName": "UNC_UPI_RxL_OCCUPANCY.SLOT2", + "PerPkg": "1", + "PublicDescription": "RxQ Occupancy - All Packets : Slot 2 : Accumulates the number of elements in the UPI RxQ in each cycle. Generally, when data is transmitted across UPI, it will bypass the RxQ and pass directly to the ring interface. If things back up getting transmitted onto the ring, however, it may need to allocate into this buffer, thus increasing the latency. This event can be used in conjunction with the Flit Buffer Not Empty event to calculate average occupancy, or with the Flit Buffer Allocations event to track average lifetime.", + "UMask": "0x4", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_RxL_SLOT_BYPASS.S0_RXQ1", + "EventCode": "0x33", + "EventName": "UNC_UPI_RxL_SLOT_BYPASS.S0_RXQ1", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_RxL_SLOT_BYPASS.S0_RXQ2", + "EventCode": "0x33", + "EventName": "UNC_UPI_RxL_SLOT_BYPASS.S0_RXQ2", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_RxL_SLOT_BYPASS.S1_RXQ0", + "EventCode": "0x33", + "EventName": "UNC_UPI_RxL_SLOT_BYPASS.S1_RXQ0", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_RxL_SLOT_BYPASS.S1_RXQ2", + "EventCode": "0x33", + "EventName": "UNC_UPI_RxL_SLOT_BYPASS.S1_RXQ2", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_RxL_SLOT_BYPASS.S2_RXQ0", + "EventCode": "0x33", + "EventName": "UNC_UPI_RxL_SLOT_BYPASS.S2_RXQ0", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_RxL_SLOT_BYPASS.S2_RXQ1", + "EventCode": "0x33", + "EventName": "UNC_UPI_RxL_SLOT_BYPASS.S2_RXQ1", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_TxL0P_CLK_ACTIVE.CFG_CTL", + "EventCode": "0x2a", + "EventName": "UNC_UPI_TxL0P_CLK_ACTIVE.CFG_CTL", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_TxL0P_CLK_ACTIVE.DFX", + "EventCode": "0x2a", + "EventName": "UNC_UPI_TxL0P_CLK_ACTIVE.DFX", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_TxL0P_CLK_ACTIVE.RETRY", + "EventCode": "0x2a", + "EventName": "UNC_UPI_TxL0P_CLK_ACTIVE.RETRY", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_TxL0P_CLK_ACTIVE.RXQ", + "EventCode": "0x2a", + "EventName": "UNC_UPI_TxL0P_CLK_ACTIVE.RXQ", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_TxL0P_CLK_ACTIVE.RXQ_BYPASS", + "EventCode": "0x2a", + "EventName": "UNC_UPI_TxL0P_CLK_ACTIVE.RXQ_BYPASS", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_TxL0P_CLK_ACTIVE.RXQ_CRED", + "EventCode": "0x2a", + "EventName": "UNC_UPI_TxL0P_CLK_ACTIVE.RXQ_CRED", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_TxL0P_CLK_ACTIVE.SPARE", + "EventCode": "0x2a", + "EventName": "UNC_UPI_TxL0P_CLK_ACTIVE.SPARE", + "PerPkg": "1", + "UMask": "0x80", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_TxL0P_CLK_ACTIVE.TXQ", + "EventCode": "0x2a", + "EventName": "UNC_UPI_TxL0P_CLK_ACTIVE.TXQ", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "UPI" + }, + { + "BriefDescription": "Cycles in L0p", + "EventCode": "0x27", + "EventName": "UNC_UPI_TxL0P_POWER_CYCLES", + "PerPkg": "1", + "PublicDescription": "Cycles in L0p : Number of UPI qfclk cycles spent in L0p power mode. L0p is a mode where we disable 1/2 of the UPI lanes, decreasing our bandwidth in order to save power. It increases snoop and data transfer latencies and decreases overall bandwidth. This mode can be very useful in NUMA optimized workloads that largely only utilize UPI for snoops and their responses. Use edge detect to count the number of instances when the UPI link entered L0p. Link power states are per link and per direction, so for example the Tx direction could be in one state while Rx was in another.", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_TxL0P_POWER_CYCLES_LL_ENTER", + "EventCode": "0x28", + "EventName": "UNC_UPI_TxL0P_POWER_CYCLES_LL_ENTER", + "PerPkg": "1", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_TxL0P_POWER_CYCLES_M3_EXIT", + "EventCode": "0x29", + "EventName": "UNC_UPI_TxL0P_POWER_CYCLES_M3_EXIT", + "PerPkg": "1", + "Unit": "UPI" + }, + { + "BriefDescription": "Cycles in L0", + "EventCode": "0x26", + "EventName": "UNC_UPI_TxL0_POWER_CYCLES", + "PerPkg": "1", + "PublicDescription": "Cycles in L0 : Number of UPI qfclk cycles spent in L0 power mode in the Link Layer. L0 is the default mode which provides the highest performance with the most power. Use edge detect to count the number of instances that the link entered L0. Link power states are per link and per direction, so for example the Tx direction could be in one state while Rx was in another. The phy layer sometimes leaves L0 for training, which will not be captured by this event.", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_TxL_ANY_FLITS.DATA", + "EventCode": "0x4A", + "EventName": "UNC_UPI_TxL_ANY_FLITS.DATA", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_TxL_ANY_FLITS.LLCRD", + "EventCode": "0x4A", + "EventName": "UNC_UPI_TxL_ANY_FLITS.LLCRD", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_TxL_ANY_FLITS.LLCTRL", + "EventCode": "0x4A", + "EventName": "UNC_UPI_TxL_ANY_FLITS.LLCTRL", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_TxL_ANY_FLITS.NULL", + "EventCode": "0x4A", + "EventName": "UNC_UPI_TxL_ANY_FLITS.NULL", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_TxL_ANY_FLITS.PROTHDR", + "EventCode": "0x4A", + "EventName": "UNC_UPI_TxL_ANY_FLITS.PROTHDR", + "PerPkg": "1", + "UMask": "0x80", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_TxL_ANY_FLITS.SLOT0", + "EventCode": "0x4A", + "EventName": "UNC_UPI_TxL_ANY_FLITS.SLOT0", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_TxL_ANY_FLITS.SLOT1", + "EventCode": "0x4A", + "EventName": "UNC_UPI_TxL_ANY_FLITS.SLOT1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_TxL_ANY_FLITS.SLOT2", + "EventCode": "0x4A", + "EventName": "UNC_UPI_TxL_ANY_FLITS.SLOT2", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Transmit path of a UPI Port : Non-Coherent Bypass", + "EventCode": "0x04", + "EventName": "UNC_UPI_TxL_BASIC_HDR_MATCH.NCB", + "PerPkg": "1", + "PublicDescription": "Matches on Transmit path of a UPI Port : Non-Coherent Bypass : Matches on Transmit path of a UPI port.\r\nMatch based on UMask specific bits:\r\nZ: Message Class (3-bit)\r\nY: Message Class Enable\r\nW: Opcode (4-bit)\r\nV: Opcode Enable\r\nU: Local Enable\r\nT: Remote Enable\r\nS: Data Hdr Enable\r\nR: Non-Data Hdr Enable\r\nQ: Dual Slot Hdr Enable\r\nP: Single Slot Hdr Enable\r\nLink Layer control types are excluded (LL CTRL, slot NULL, LLCRD) even under specific opcode match_en cases.\r\nNote: If Message Class is disabled, we expect opcode to also be disabled.", + "UMask": "0xe", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Transmit path of a UPI Port : Non-Coherent Bypass, Match Opcode", + "EventCode": "0x04", + "EventName": "UNC_UPI_TxL_BASIC_HDR_MATCH.NCB_OPC", + "PerPkg": "1", + "PublicDescription": "Matches on Transmit path of a UPI Port : Non-Coherent Bypass, Match Opcode : Matches on Transmit path of a UPI port.\r\nMatch based on UMask specific bits:\r\nZ: Message Class (3-bit)\r\nY: Message Class Enable\r\nW: Opcode (4-bit)\r\nV: Opcode Enable\r\nU: Local Enable\r\nT: Remote Enable\r\nS: Data Hdr Enable\r\nR: Non-Data Hdr Enable\r\nQ: Dual Slot Hdr Enable\r\nP: Single Slot Hdr Enable\r\nLink Layer control types are excluded (LL CTRL, slot NULL, LLCRD) even under specific opcode match_en cases.\r\nNote: If Message Class is disabled, we expect opcode to also be disabled.", + "UMask": "0x10e", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Transmit path of a UPI Port : Non-Coherent Standard", + "EventCode": "0x04", + "EventName": "UNC_UPI_TxL_BASIC_HDR_MATCH.NCS", + "PerPkg": "1", + "PublicDescription": "Matches on Transmit path of a UPI Port : Non-Coherent Standard : Matches on Transmit path of a UPI port.\r\nMatch based on UMask specific bits:\r\nZ: Message Class (3-bit)\r\nY: Message Class Enable\r\nW: Opcode (4-bit)\r\nV: Opcode Enable\r\nU: Local Enable\r\nT: Remote Enable\r\nS: Data Hdr Enable\r\nR: Non-Data Hdr Enable\r\nQ: Dual Slot Hdr Enable\r\nP: Single Slot Hdr Enable\r\nLink Layer control types are excluded (LL CTRL, slot NULL, LLCRD) even under specific opcode match_en cases.\r\nNote: If Message Class is disabled, we expect opcode to also be disabled.", + "UMask": "0xf", + "Unit": "UPI" + }, + { + "BriefDescription": "Matches on Transmit path of a UPI Port : Non-Coherent Standard, Match Opcode", + "EventCode": "0x04", + "EventName": "UNC_UPI_TxL_BASIC_HDR_MATCH.NCS_OPC", + "PerPkg": "1", + "PublicDescription": "Matches on Transmit path of a UPI Port : Non-Coherent Standard, Match Opcode : Matches on Transmit path of a UPI port.\r\nMatch based on UMask specific bits:\r\nZ: Message Class (3-bit)\r\nY: Message Class Enable\r\nW: Opcode (4-bit)\r\nV: Opcode Enable\r\nU: Local Enable\r\nT: Remote Enable\r\nS: Data Hdr Enable\r\nR: Non-Data Hdr Enable\r\nQ: Dual Slot Hdr Enable\r\nP: Single Slot Hdr Enable\r\nLink Layer control types are excluded (LL CTRL, slot NULL, LLCRD) even under specific opcode match_en cases.\r\nNote: If Message Class is disabled, we expect opcode to also be disabled.", + "UMask": "0x10f", + "Unit": "UPI" + }, + { + "BriefDescription": "Tx Flit Buffer Bypassed", + "EventCode": "0x41", + "EventName": "UNC_UPI_TxL_BYPASSED", + "PerPkg": "1", + "PublicDescription": "Tx Flit Buffer Bypassed : Counts the number of times that an incoming flit was able to bypass the Tx flit buffer and pass directly out the UPI Link. Generally, when data is transmitted across UPI, it will bypass the TxQ and pass directly to the link. However, the TxQ will be used with L0p and when LLR occurs, increasing latency to transfer out to the link.", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Sent : All Data", + "EventCode": "0x02", + "EventName": "UNC_UPI_TxL_FLITS.ALL_DATA", + "PerPkg": "1", + "PublicDescription": "Valid Flits Sent : All Data : Counts number of data flits across this UPI link.", + "UMask": "0xf", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Sent : All LLCRD Not Empty", + "EventCode": "0x02", + "EventName": "UNC_UPI_TxL_FLITS.ALL_LLCRD", + "PerPkg": "1", + "PublicDescription": "Valid Flits Sent : All Data : Shows legal flit time (hides impact of L0p and L0c).", + "UMask": "0x17", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Sent : All LLCTRL", + "EventCode": "0x02", + "EventName": "UNC_UPI_TxL_FLITS.ALL_LLCTRL", + "PerPkg": "1", + "PublicDescription": "Valid Flits Sent : All LLCTRL : Shows legal flit time (hides impact of L0p and L0c).", + "UMask": "0x47", + "Unit": "UPI" + }, + { + "BriefDescription": "All Null Flits", + "EventCode": "0x02", + "EventName": "UNC_UPI_TxL_FLITS.ALL_NULL", + "PerPkg": "1", + "UMask": "0x27", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Sent : All Protocol Header", + "EventCode": "0x02", + "EventName": "UNC_UPI_TxL_FLITS.ALL_PROTHDR", + "PerPkg": "1", + "PublicDescription": "Valid Flits Sent : All ProtDDR : Shows legal flit time (hides impact of L0p and L0c).", + "UMask": "0x87", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Sent : Data", + "EventCode": "0x02", + "EventName": "UNC_UPI_TxL_FLITS.DATA", + "PerPkg": "1", + "PublicDescription": "Valid Flits Sent : Data : Shows legal flit time (hides impact of L0p and L0c). : Count Data Flits (which consume all slots), but how much to count is based on Slot0-2 mask, so count can be 0-3 depending on which slots are enabled for counting..", + "UMask": "0x8", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Sent : Idle", + "EventCode": "0x02", + "EventName": "UNC_UPI_TxL_FLITS.IDLE", + "PerPkg": "1", + "PublicDescription": "Valid Flits Sent : Idle : Shows legal flit time (hides impact of L0p and L0c).", + "UMask": "0x47", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Sent : LLCRD Not Empty", + "EventCode": "0x02", + "EventName": "UNC_UPI_TxL_FLITS.LLCRD", + "PerPkg": "1", + "PublicDescription": "Valid Flits Sent : LLCRD Not Empty : Shows legal flit time (hides impact of L0p and L0c). : Enables counting of LLCRD (with non-zero payload). This only applies to slot 2 since LLCRD is only allowed in slot 2", + "UMask": "0x10", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Sent : LLCTRL", + "EventCode": "0x02", + "EventName": "UNC_UPI_TxL_FLITS.LLCTRL", + "PerPkg": "1", + "PublicDescription": "Valid Flits Sent : LLCTRL : Shows legal flit time (hides impact of L0p and L0c). : Equivalent to an idle packet. Enables counting of slot 0 LLCTRL messages.", + "UMask": "0x40", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Sent : All Non Data", + "EventCode": "0x02", + "EventName": "UNC_UPI_TxL_FLITS.NON_DATA", + "PerPkg": "1", + "PublicDescription": "Valid Flits Sent : All Non Data : Shows legal flit time (hides impact of L0p and L0c).", + "UMask": "0x97", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Sent : Slot NULL or LLCRD Empty", + "EventCode": "0x02", + "EventName": "UNC_UPI_TxL_FLITS.NULL", + "PerPkg": "1", + "PublicDescription": "Valid Flits Sent : Slot NULL or LLCRD Empty : Shows legal flit time (hides impact of L0p and L0c). : LLCRD with all zeros is treated as NULL. Slot 1 is not treated as NULL if slot 0 is a dual slot. This can apply to slot 0,1, or 2.", + "UMask": "0x20", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Sent : Protocol Header", + "EventCode": "0x02", + "EventName": "UNC_UPI_TxL_FLITS.PROTHDR", + "PerPkg": "1", + "PublicDescription": "Valid Flits Sent : Protocol Header : Shows legal flit time (hides impact of L0p and L0c). : Enables count of protocol headers in slot 0,1,2 (depending on slot uMask bits)", + "UMask": "0x80", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Sent : Slot 0", + "EventCode": "0x02", + "EventName": "UNC_UPI_TxL_FLITS.SLOT0", + "PerPkg": "1", + "PublicDescription": "Valid Flits Sent : Slot 0 : Shows legal flit time (hides impact of L0p and L0c). : Count Slot 0 - Other mask bits determine types of headers to count.", + "UMask": "0x1", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Sent : Slot 1", + "EventCode": "0x02", + "EventName": "UNC_UPI_TxL_FLITS.SLOT1", + "PerPkg": "1", + "PublicDescription": "Valid Flits Sent : Slot 1 : Shows legal flit time (hides impact of L0p and L0c). : Count Slot 1 - Other mask bits determine types of headers to count.", + "UMask": "0x2", + "Unit": "UPI" + }, + { + "BriefDescription": "Valid Flits Sent : Slot 2", + "EventCode": "0x02", + "EventName": "UNC_UPI_TxL_FLITS.SLOT2", + "PerPkg": "1", + "PublicDescription": "Valid Flits Sent : Slot 2 : Shows legal flit time (hides impact of L0p and L0c). : Count Slot 2 - Other mask bits determine types of headers to count.", + "UMask": "0x4", + "Unit": "UPI" + }, + { + "BriefDescription": "Tx Flit Buffer Allocations", + "EventCode": "0x40", + "EventName": "UNC_UPI_TxL_INSERTS", + "PerPkg": "1", + "PublicDescription": "Tx Flit Buffer Allocations : Number of allocations into the UPI Tx Flit Buffer. Generally, when data is transmitted across UPI, it will bypass the TxQ and pass directly to the link. However, the TxQ will be used with L0p and when LLR occurs, increasing latency to transfer out to the link. This event can be used in conjunction with the Flit Buffer Occupancy event in order to calculate the average flit buffer lifetime.", + "Unit": "UPI" + }, + { + "BriefDescription": "Tx Flit Buffer Occupancy", + "EventCode": "0x42", + "EventName": "UNC_UPI_TxL_OCCUPANCY", + "PerPkg": "1", + "PublicDescription": "Tx Flit Buffer Occupancy : Accumulates the number of flits in the TxQ. Generally, when data is transmitted across UPI, it will bypass the TxQ and pass directly to the link. However, the TxQ will be used with L0p and when LLR occurs, increasing latency to transfer out to the link. This can be used with the cycles not empty event to track average occupancy, or the allocations event to track average lifetime in the TxQ.", + "Unit": "UPI" + }, + { + "BriefDescription": "UNC_UPI_VNA_CREDIT_RETURN_BLOCKED_VN01", + "EventCode": "0x45", + "EventName": "UNC_UPI_VNA_CREDIT_RETURN_BLOCKED_VN01", + "PerPkg": "1", + "Unit": "UPI" + }, + { + "BriefDescription": "VNA Credits Pending Return - Occupancy", + "EventCode": "0x44", + "EventName": "UNC_UPI_VNA_CREDIT_RETURN_OCCUPANCY", + "PerPkg": "1", + "PublicDescription": "VNA Credits Pending Return - Occupancy : Number of VNA credits in the Rx side that are waitng to be returned back across the link.", + "Unit": "UPI" + }, + { + "BriefDescription": "Message Received : Doorbell", + "EventCode": "0x42", + "EventName": "UNC_U_EVENT_MSG.DOORBELL_RCVD", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "UBOX" + }, + { + "BriefDescription": "Message Received : Interrupt", + "EventCode": "0x42", + "EventName": "UNC_U_EVENT_MSG.INT_PRIO", + "PerPkg": "1", + "PublicDescription": "Message Received : Interrupt : Interrupts", + "UMask": "0x10", + "Unit": "UBOX" + }, + { + "BriefDescription": "Message Received : IPI", + "EventCode": "0x42", + "EventName": "UNC_U_EVENT_MSG.IPI_RCVD", + "PerPkg": "1", + "PublicDescription": "Message Received : IPI : Inter Processor Interrupts", + "UMask": "0x4", + "Unit": "UBOX" + }, + { + "BriefDescription": "Message Received : MSI", + "EventCode": "0x42", + "EventName": "UNC_U_EVENT_MSG.MSI_RCVD", + "PerPkg": "1", + "PublicDescription": "Message Received : MSI : Message Signaled Interrupts - interrupts sent by devices (including PCIe via IOxAPIC) (Socket Mode only)", + "UMask": "0x2", + "Unit": "UBOX" + }, + { + "BriefDescription": "Message Received : VLW", + "EventCode": "0x42", + "EventName": "UNC_U_EVENT_MSG.VLW_RCVD", + "PerPkg": "1", + "PublicDescription": "Message Received : VLW : Virtual Logical Wire (legacy) message were received from Uncore.", + "UMask": "0x1", + "Unit": "UBOX" + }, + { + "BriefDescription": "UNC_U_M2U_MISC1.RxC_CYCLES_NE_CBO_NCB", + "EventCode": "0x4d", + "EventName": "UNC_U_M2U_MISC1.RxC_CYCLES_NE_CBO_NCB", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "UBOX" + }, + { + "BriefDescription": "UNC_U_M2U_MISC1.RxC_CYCLES_NE_CBO_NCS", + "EventCode": "0x4d", + "EventName": "UNC_U_M2U_MISC1.RxC_CYCLES_NE_CBO_NCS", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "UBOX" + }, + { + "BriefDescription": "UNC_U_M2U_MISC1.RxC_CYCLES_NE_UPI_NCB", + "EventCode": "0x4d", + "EventName": "UNC_U_M2U_MISC1.RxC_CYCLES_NE_UPI_NCB", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "UBOX" + }, + { + "BriefDescription": "UNC_U_M2U_MISC1.RxC_CYCLES_NE_UPI_NCS", + "EventCode": "0x4d", + "EventName": "UNC_U_M2U_MISC1.RxC_CYCLES_NE_UPI_NCS", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "UBOX" + }, + { + "BriefDescription": "UNC_U_M2U_MISC1.TxC_CYCLES_CRD_OVF_CBO_NCB", + "EventCode": "0x4d", + "EventName": "UNC_U_M2U_MISC1.TxC_CYCLES_CRD_OVF_CBO_NCB", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "UBOX" + }, + { + "BriefDescription": "UNC_U_M2U_MISC1.TxC_CYCLES_CRD_OVF_CBO_NCS", + "EventCode": "0x4d", + "EventName": "UNC_U_M2U_MISC1.TxC_CYCLES_CRD_OVF_CBO_NCS", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "UBOX" + }, + { + "BriefDescription": "UNC_U_M2U_MISC1.TxC_CYCLES_CRD_OVF_UPI_NCB", + "EventCode": "0x4d", + "EventName": "UNC_U_M2U_MISC1.TxC_CYCLES_CRD_OVF_UPI_NCB", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "UBOX" + }, + { + "BriefDescription": "UNC_U_M2U_MISC1.TxC_CYCLES_CRD_OVF_UPI_NCS", + "EventCode": "0x4d", + "EventName": "UNC_U_M2U_MISC1.TxC_CYCLES_CRD_OVF_UPI_NCS", + "PerPkg": "1", + "UMask": "0x80", + "Unit": "UBOX" + }, + { + "BriefDescription": "UNC_U_M2U_MISC2.RxC_CYCLES_EMPTY_BL", + "EventCode": "0x4e", + "EventName": "UNC_U_M2U_MISC2.RxC_CYCLES_EMPTY_BL", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "UBOX" + }, + { + "BriefDescription": "UNC_U_M2U_MISC2.RxC_CYCLES_FULL_BL", + "EventCode": "0x4e", + "EventName": "UNC_U_M2U_MISC2.RxC_CYCLES_FULL_BL", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "UBOX" + }, + { + "BriefDescription": "UNC_U_M2U_MISC2.TxC_CYCLES_CRD_OVF_VN0_NCB", + "EventCode": "0x4e", + "EventName": "UNC_U_M2U_MISC2.TxC_CYCLES_CRD_OVF_VN0_NCB", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "UBOX" + }, + { + "BriefDescription": "UNC_U_M2U_MISC2.TxC_CYCLES_CRD_OVF_VN0_NCS", + "EventCode": "0x4e", + "EventName": "UNC_U_M2U_MISC2.TxC_CYCLES_CRD_OVF_VN0_NCS", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "UBOX" + }, + { + "BriefDescription": "UNC_U_M2U_MISC2.TxC_CYCLES_EMPTY_AK", + "EventCode": "0x4e", + "EventName": "UNC_U_M2U_MISC2.TxC_CYCLES_EMPTY_AK", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "UBOX" + }, + { + "BriefDescription": "UNC_U_M2U_MISC2.TxC_CYCLES_EMPTY_AKC", + "EventCode": "0x4e", + "EventName": "UNC_U_M2U_MISC2.TxC_CYCLES_EMPTY_AKC", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "UBOX" + }, + { + "BriefDescription": "UNC_U_M2U_MISC2.TxC_CYCLES_EMPTY_BL", + "EventCode": "0x4e", + "EventName": "UNC_U_M2U_MISC2.TxC_CYCLES_EMPTY_BL", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "UBOX" + }, + { + "BriefDescription": "UNC_U_M2U_MISC2.TxC_CYCLES_FULL_BL", + "EventCode": "0x4e", + "EventName": "UNC_U_M2U_MISC2.TxC_CYCLES_FULL_BL", + "PerPkg": "1", + "UMask": "0x80", + "Unit": "UBOX" + }, + { + "BriefDescription": "UNC_U_M2U_MISC3.TxC_CYCLES_FULL_AK", + "EventCode": "0x4f", + "EventName": "UNC_U_M2U_MISC3.TxC_CYCLES_FULL_AK", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "UBOX" + }, + { + "BriefDescription": "UNC_U_M2U_MISC3.TxC_CYCLES_FULL_AKC", + "EventCode": "0x4f", + "EventName": "UNC_U_M2U_MISC3.TxC_CYCLES_FULL_AKC", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "UBOX" + }, + { + "BriefDescription": "Cycles PHOLD Assert to Ack : Assert to ACK", + "EventCode": "0x45", + "EventName": "UNC_U_PHOLD_CYCLES.ASSERT_TO_ACK", + "PerPkg": "1", + "PublicDescription": "Cycles PHOLD Assert to Ack : Assert to ACK : PHOLD cycles.", + "UMask": "0x1", + "Unit": "UBOX" + }, + { + "BriefDescription": "UNC_U_RACU_DRNG.PFTCH_BUF_EMPTY", + "EventCode": "0x4c", + "EventName": "UNC_U_RACU_DRNG.PFTCH_BUF_EMPTY", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "UBOX" + }, + { + "BriefDescription": "UNC_U_RACU_DRNG.RDRAND", + "EventCode": "0x4c", + "EventName": "UNC_U_RACU_DRNG.RDRAND", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "UBOX" + }, + { + "BriefDescription": "UNC_U_RACU_DRNG.RDSEED", + "EventCode": "0x4c", + "EventName": "UNC_U_RACU_DRNG.RDSEED", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "UBOX" + }, + { + "BriefDescription": "RACU Request", + "EventCode": "0x46", + "EventName": "UNC_U_RACU_REQUESTS", + "PerPkg": "1", + "PublicDescription": "RACU Request : Number outstanding register requests within message channel tracker", + "Unit": "UBOX" + } +] diff --git a/tools/perf/pmu-events/arch/x86/emeraldrapids/uncore-io.json b/tools/perf/pmu-events/arch/x86/emeraldrapids/uncore-io.json new file mode 100644 index 000000000000..557080b74ee5 --- /dev/null +++ b/tools/perf/pmu-events/arch/x86/emeraldrapids/uncore-io.json @@ -0,0 +1,3587 @@ +[ + { + "BriefDescription": "Free running counter that increments for every 32 bytes of data sent from the IO agent to the SOC", + "EventCode": "0xff", + "EventName": "UNC_IIO_BANDWIDTH_IN.PART0_FREERUN", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "iio_free_running" + }, + { + "BriefDescription": "Free running counter that increments for every 32 bytes of data sent from the IO agent to the SOC", + "EventCode": "0xff", + "EventName": "UNC_IIO_BANDWIDTH_IN.PART1_FREERUN", + "PerPkg": "1", + "UMask": "0x21", + "Unit": "iio_free_running" + }, + { + "BriefDescription": "Free running counter that increments for every 32 bytes of data sent from the IO agent to the SOC", + "EventCode": "0xff", + "EventName": "UNC_IIO_BANDWIDTH_IN.PART2_FREERUN", + "PerPkg": "1", + "UMask": "0x22", + "Unit": "iio_free_running" + }, + { + "BriefDescription": "Free running counter that increments for every 32 bytes of data sent from the IO agent to the SOC", + "EventCode": "0xff", + "EventName": "UNC_IIO_BANDWIDTH_IN.PART3_FREERUN", + "PerPkg": "1", + "UMask": "0x23", + "Unit": "iio_free_running" + }, + { + "BriefDescription": "Free running counter that increments for every 32 bytes of data sent from the IO agent to the SOC", + "EventCode": "0xff", + "EventName": "UNC_IIO_BANDWIDTH_IN.PART4_FREERUN", + "PerPkg": "1", + "UMask": "0x24", + "Unit": "iio_free_running" + }, + { + "BriefDescription": "Free running counter that increments for every 32 bytes of data sent from the IO agent to the SOC", + "EventCode": "0xff", + "EventName": "UNC_IIO_BANDWIDTH_IN.PART5_FREERUN", + "PerPkg": "1", + "UMask": "0x25", + "Unit": "iio_free_running" + }, + { + "BriefDescription": "Free running counter that increments for every 32 bytes of data sent from the IO agent to the SOC", + "EventCode": "0xff", + "EventName": "UNC_IIO_BANDWIDTH_IN.PART6_FREERUN", + "PerPkg": "1", + "UMask": "0x26", + "Unit": "iio_free_running" + }, + { + "BriefDescription": "Free running counter that increments for every 32 bytes of data sent from the IO agent to the SOC", + "EventCode": "0xff", + "EventName": "UNC_IIO_BANDWIDTH_IN.PART7_FREERUN", + "PerPkg": "1", + "UMask": "0x27", + "Unit": "iio_free_running" + }, + { + "BriefDescription": "IIO Clockticks", + "EventCode": "0x01", + "EventName": "UNC_IIO_CLOCKTICKS", + "PerPkg": "1", + "PortMask": "0x0000", + "PublicDescription": "Number of IIO clock cycles while the event is enabled", + "Unit": "IIO" + }, + { + "BriefDescription": "Free running counter that increments for IIO clocktick", + "EventCode": "0xff", + "EventName": "UNC_IIO_CLOCKTICKS_FREERUN", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "iio_free_running" + }, + { + "BriefDescription": "PCIe Completion Buffer Inserts of completions with data: Part 0-7", + "EventCode": "0xc2", + "EventName": "UNC_IIO_COMP_BUF_INSERTS.CMPD.ALL_PARTS", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0xff", + "PublicDescription": "PCIe Completion Buffer Inserts of completions with data : Part 0-7", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "PCIe Completion Buffer Inserts of completions with data: Part 0", + "EventCode": "0xc2", + "EventName": "UNC_IIO_COMP_BUF_INSERTS.CMPD.PART0", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0001", + "PublicDescription": "PCIe Completion Buffer Inserts of completions with data : Part 0 : x16 card plugged in to Lane 0/1/2/3, Or x8 card plugged in to Lane 0/1, Or x4 card is plugged in to slot 0", + "UMask": "0x7001004", + "Unit": "IIO" + }, + { + "BriefDescription": "PCIe Completion Buffer Inserts of completions with data: Part 1", + "EventCode": "0xc2", + "EventName": "UNC_IIO_COMP_BUF_INSERTS.CMPD.PART1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0002", + "PublicDescription": "PCIe Completion Buffer Inserts of completions with data : Part 1 : x16 card plugged in to Lane 0/1/2/3, Or x8 card plugged in to Lane 0/1, Or x4 card is plugged in to slot 1", + "UMask": "0x7002004", + "Unit": "IIO" + }, + { + "BriefDescription": "PCIe Completion Buffer Inserts of completions with data: Part 2", + "EventCode": "0xc2", + "EventName": "UNC_IIO_COMP_BUF_INSERTS.CMPD.PART2", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0004", + "PublicDescription": "PCIe Completion Buffer Inserts of completions with data : Part 2 : x16 card plugged in to Lane 0/1/2/3, Or x8 card plugged in to Lane 0/1, Or x4 card is plugged in to slot 2", + "UMask": "0x7004004", + "Unit": "IIO" + }, + { + "BriefDescription": "PCIe Completion Buffer Inserts of completions with data: Part 3", + "EventCode": "0xc2", + "EventName": "UNC_IIO_COMP_BUF_INSERTS.CMPD.PART3", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0008", + "PublicDescription": "PCIe Completion Buffer Inserts of completions with data : Part 2 : x16 card plugged in to Lane 0/1/2/3, Or x8 card plugged in to Lane 0/1, Or x4 card is plugged in to slot 3", + "UMask": "0x7008004", + "Unit": "IIO" + }, + { + "BriefDescription": "PCIe Completion Buffer Inserts of completions with data: Part 4", + "EventCode": "0xc2", + "EventName": "UNC_IIO_COMP_BUF_INSERTS.CMPD.PART4", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0010", + "PublicDescription": "PCIe Completion Buffer Inserts of completions with data : Part 0 : x16 card plugged in to Lane 0/1/2/3, Or x8 card plugged in to Lane 0/1, Or x4 card is plugged in to slot 4", + "UMask": "0x7010004", + "Unit": "IIO" + }, + { + "BriefDescription": "PCIe Completion Buffer Inserts of completions with data: Part 5", + "EventCode": "0xc2", + "EventName": "UNC_IIO_COMP_BUF_INSERTS.CMPD.PART5", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0020", + "PublicDescription": "PCIe Completion Buffer Inserts of completions with data : Part 1 : x16 card plugged in to Lane 0/1/2/3, Or x8 card plugged in to Lane 0/1, Or x4 card is plugged in to slot 5", + "UMask": "0x7020004", + "Unit": "IIO" + }, + { + "BriefDescription": "PCIe Completion Buffer Inserts of completions with data: Part 6", + "EventCode": "0xc2", + "EventName": "UNC_IIO_COMP_BUF_INSERTS.CMPD.PART6", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0040", + "PublicDescription": "PCIe Completion Buffer Inserts of completions with data : Part 2 : x16 card plugged in to Lane 0/1/2/3, Or x8 card plugged in to Lane 0/1, Or x4 card is plugged in to slot 6", + "UMask": "0x7040004", + "Unit": "IIO" + }, + { + "BriefDescription": "PCIe Completion Buffer Inserts of completions with data: Part 7", + "EventCode": "0xc2", + "EventName": "UNC_IIO_COMP_BUF_INSERTS.CMPD.PART7", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0080", + "PublicDescription": "PCIe Completion Buffer Inserts of completions with data : Part 2 : x16 card plugged in to Lane 0/1/2/3, Or x8 card plugged in to Lane 0/1, Or x4 card is plugged in to slot 7", + "UMask": "0x7080004", + "Unit": "IIO" + }, + { + "BriefDescription": "UNC_IIO_COMP_BUF_OCCUPANCY.CMPD.ALL_PARTS", + "EventCode": "0xd5", + "EventName": "UNC_IIO_COMP_BUF_OCCUPANCY.CMPD.ALL_PARTS", + "FCMask": "0x07", + "PerPkg": "1", + "UMask": "0xff", + "Unit": "IIO" + }, + { + "BriefDescription": "PCIe Completion Buffer Occupancy : Part 0", + "EventCode": "0xd5", + "EventName": "UNC_IIO_COMP_BUF_OCCUPANCY.CMPD.PART0", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0000", + "PublicDescription": "x16 card plugged in to stack, Or x8 card plugged in to Lane 0/1, Or x4 card is plugged in to slot 0", + "UMask": "0x7000001", + "Unit": "IIO" + }, + { + "BriefDescription": "PCIe Completion Buffer Occupancy : Part 1", + "EventCode": "0xd5", + "EventName": "UNC_IIO_COMP_BUF_OCCUPANCY.CMPD.PART1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0000", + "PublicDescription": "x4 card is plugged in to slot 1", + "UMask": "0x7000002", + "Unit": "IIO" + }, + { + "BriefDescription": "PCIe Completion Buffer Occupancy : Part 2", + "EventCode": "0xd5", + "EventName": "UNC_IIO_COMP_BUF_OCCUPANCY.CMPD.PART2", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0000", + "PublicDescription": "x8 card plugged in to Lane 2/3, Or x4 card is plugged in to slot 1", + "UMask": "0x7000004", + "Unit": "IIO" + }, + { + "BriefDescription": "PCIe Completion Buffer Occupancy : Part 3", + "EventCode": "0xd5", + "EventName": "UNC_IIO_COMP_BUF_OCCUPANCY.CMPD.PART3", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0000", + "PublicDescription": "x4 card is plugged in to slot 3", + "UMask": "0x7000008", + "Unit": "IIO" + }, + { + "BriefDescription": "PCIe Completion Buffer Occupancy : Part 4", + "EventCode": "0xd5", + "EventName": "UNC_IIO_COMP_BUF_OCCUPANCY.CMPD.PART4", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0000", + "PublicDescription": "x16 card plugged in to stack, Or x8 card plugged in to Lane 0/1, Or x4 card is plugged in to slot 0", + "UMask": "0x7000010", + "Unit": "IIO" + }, + { + "BriefDescription": "PCIe Completion Buffer Occupancy : Part 5", + "EventCode": "0xd5", + "EventName": "UNC_IIO_COMP_BUF_OCCUPANCY.CMPD.PART5", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0000", + "PublicDescription": "x4 card is plugged in to slot 1", + "UMask": "0x7000020", + "Unit": "IIO" + }, + { + "BriefDescription": "PCIe Completion Buffer Occupancy : Part 6", + "EventCode": "0xd5", + "EventName": "UNC_IIO_COMP_BUF_OCCUPANCY.CMPD.PART6", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0000", + "PublicDescription": "x8 card plugged in to Lane 2/3, Or x4 card is plugged in to slot 1", + "UMask": "0x7000040", + "Unit": "IIO" + }, + { + "BriefDescription": "PCIe Completion Buffer Occupancy : Part 7", + "EventCode": "0xd5", + "EventName": "UNC_IIO_COMP_BUF_OCCUPANCY.CMPD.PART7", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0000", + "PublicDescription": "x4 card is plugged in to slot 3", + "UMask": "0x7000080", + "Unit": "IIO" + }, + { + "BriefDescription": "Read request for 4 bytes made by the CPU to IIO Part0-7", + "EventCode": "0xc0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.MEM_READ.ALL_PARTS", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x00ff", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Read request for 4 bytes made by the CPU to IIO Part0", + "EventCode": "0xc0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.MEM_READ.PART0", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0001", + "PublicDescription": "Data requested by the CPU : Core reading from Card's MMIO space : Number of DWs (4 bytes) requested by the main die. Includes all requests initiated by the main die, including reads and writes. : x16 card plugged in to stack, Or x8 card plugged in to Lane 0/1, Or x4 card is plugged in to slot 0", + "UMask": "0x7001004", + "Unit": "IIO" + }, + { + "BriefDescription": "Read request for 4 bytes made by the CPU to IIO Part1", + "EventCode": "0xc0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.MEM_READ.PART1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0002", + "PublicDescription": "Data requested by the CPU : Core reading from Card's MMIO space : Number of DWs (4 bytes) requested by the main die. Includes all requests initiated by the main die, including reads and writes. : x4 card is plugged in to slot 1", + "UMask": "0x7002004", + "Unit": "IIO" + }, + { + "BriefDescription": "Read request for 4 bytes made by the CPU to IIO Part2", + "EventCode": "0xc0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.MEM_READ.PART2", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0004", + "PublicDescription": "Data requested by the CPU : Core reading from Card's MMIO space : Number of DWs (4 bytes) requested by the main die. Includes all requests initiated by the main die, including reads and writes. : x8 card plugged in to Lane 2/3, Or x4 card is plugged in to slot 1", + "UMask": "0x7004004", + "Unit": "IIO" + }, + { + "BriefDescription": "Read request for 4 bytes made by the CPU to IIO Part3", + "EventCode": "0xc0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.MEM_READ.PART3", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0008", + "PublicDescription": "Data requested by the CPU : Core reading from Card's MMIO space : Number of DWs (4 bytes) requested by the main die. Includes all requests initiated by the main die, including reads and writes. : x4 card is plugged in to slot 3", + "UMask": "0x7008004", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested by the CPU : Core reading from Cards MMIO space", + "EventCode": "0xc0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.MEM_READ.PART4", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0010", + "PublicDescription": "Data requested by the CPU : Core reading from Cards MMIO space : Number of DWs (4 bytes) requested by the main die. Includes all requests initiated by the main die, including reads and writes. : x16 card plugged in to stack, Or x8 card plugged in to Lane 0/1, Or x4 card is plugged in to slot 0", + "UMask": "0x7010004", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested by the CPU : Core reading from Cards MMIO space", + "EventCode": "0xc0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.MEM_READ.PART5", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0020", + "PublicDescription": "Data requested by the CPU : Core reading from Cards MMIO space : Number of DWs (4 bytes) requested by the main die. Includes all requests initiated by the main die, including reads and writes. : x4 card is plugged in to slot 1", + "UMask": "0x7020004", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested by the CPU : Core reading from Cards MMIO space", + "EventCode": "0xc0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.MEM_READ.PART6", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0040", + "PublicDescription": "Data requested by the CPU : Core reading from Cards MMIO space : Number of DWs (4 bytes) requested by the main die. Includes all requests initiated by the main die, including reads and writes. : x8 card plugged in to Lane 2/3, Or x4 card is plugged in to slot 1", + "UMask": "0x7040004", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested by the CPU : Core reading from Cards MMIO space", + "EventCode": "0xc0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.MEM_READ.PART7", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0080", + "PublicDescription": "Data requested by the CPU : Core reading from Cards MMIO space : Number of DWs (4 bytes) requested by the main die. Includes all requests initiated by the main die, including reads and writes. : x4 card is plugged in to slot 3", + "UMask": "0x7080004", + "Unit": "IIO" + }, + { + "BriefDescription": "Write request of 4 bytes made to IIO Part0-7 by the CPU", + "EventCode": "0xc0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.MEM_WRITE.ALL_PARTS", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x00ff", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested by the CPU : Core writing to Cards MMIO space", + "EventCode": "0xc0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.MEM_WRITE.IOMMU0", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0100", + "PublicDescription": "Data requested by the CPU : Core writing to Cards MMIO space : Number of DWs (4 bytes) requested by the main die. Includes all requests initiated by the main die, including reads and writes. : IOMMU - Type 0", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested by the CPU : Core writing to Cards MMIO space", + "EventCode": "0xc0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.MEM_WRITE.IOMMU1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0200", + "PublicDescription": "Data requested by the CPU : Core writing to Cards MMIO space : Number of DWs (4 bytes) requested by the main die. Includes all requests initiated by the main die, including reads and writes. : IOMMU - Type 1", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Write request of 4 bytes made to IIO Part0 by the CPU", + "EventCode": "0xc0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.MEM_WRITE.PART0", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0001", + "PublicDescription": "Data requested by the CPU : Core writing to Card's MMIO space : Number of DWs (4 bytes) requested by the main die. Includes all requests initiated by the main die, including reads and writes. : x16 card plugged in to stack, Or x8 card plugged in to Lane 0/1, Or x4 card is plugged in to slot 0", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Write request of 4 bytes made to IIO Part1 by the CPU", + "EventCode": "0xc0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.MEM_WRITE.PART1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0002", + "PublicDescription": "Data requested by the CPU : Core writing to Card's MMIO space : Number of DWs (4 bytes) requested by the main die. Includes all requests initiated by the main die, including reads and writes. : x4 card is plugged in to slot 1", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Write request of 4 bytes made to IIO Part2 by the CPU", + "EventCode": "0xc0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.MEM_WRITE.PART2", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0004", + "PublicDescription": "Data requested by the CPU : Core writing to Card's MMIO space : Number of DWs (4 bytes) requested by the main die. Includes all requests initiated by the main die, including reads and writes. : x8 card plugged in to Lane 2/3, Or x4 card is plugged in to slot 1", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Write request of 4 bytes made to IIO Part3 by the CPU", + "EventCode": "0xc0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.MEM_WRITE.PART3", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0008", + "PublicDescription": "Data requested by the CPU : Core writing to Card's MMIO space : Number of DWs (4 bytes) requested by the main die. Includes all requests initiated by the main die, including reads and writes. : x4 card is plugged in to slot 3", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested by the CPU : Core writing to Cards MMIO space", + "EventCode": "0xc0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.MEM_WRITE.PART4", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0010", + "PublicDescription": "Data requested by the CPU : Core writing to Cards MMIO space : Number of DWs (4 bytes) requested by the main die. Includes all requests initiated by the main die, including reads and writes. : x16 card plugged in to stack, Or x8 card plugged in to Lane 0/1, Or x4 card is plugged in to slot 0", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested by the CPU : Core writing to Cards MMIO space", + "EventCode": "0xc0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.MEM_WRITE.PART5", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0020", + "PublicDescription": "Data requested by the CPU : Core writing to Cards MMIO space : Number of DWs (4 bytes) requested by the main die. Includes all requests initiated by the main die, including reads and writes. : x4 card is plugged in to slot 1", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested by the CPU : Core writing to Cards MMIO space", + "EventCode": "0xc0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.MEM_WRITE.PART6", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0040", + "PublicDescription": "Data requested by the CPU : Core writing to Cards MMIO space : Number of DWs (4 bytes) requested by the main die. Includes all requests initiated by the main die, including reads and writes. : x8 card plugged in to Lane 2/3, Or x4 card is plugged in to slot 1", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested by the CPU : Core writing to Cards MMIO space", + "EventCode": "0xc0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.MEM_WRITE.PART7", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0080", + "PublicDescription": "Data requested by the CPU : Core writing to Cards MMIO space : Number of DWs (4 bytes) requested by the main die. Includes all requests initiated by the main die, including reads and writes. : x4 card is plugged in to slot 3", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Peer to peer read request for 4 bytes made by a different IIO unit to IIO Part0", + "EventCode": "0xc0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.PEER_READ.PART0", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0001", + "PublicDescription": "Data requested by the CPU : Another card (different IIO stack) reading from this card. : Number of DWs (4 bytes) requested by the main die. Includes all requests initiated by the main die, including reads and writes. : x16 card plugged in to stack, Or x8 card plugged in to Lane 0/1, Or x4 card is plugged in to slot 0", + "UMask": "0x7001008", + "Unit": "IIO" + }, + { + "BriefDescription": "Peer to peer read request for 4 bytes made by a different IIO unit to IIO Part0", + "EventCode": "0xc0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.PEER_READ.PART1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0002", + "PublicDescription": "Data requested by the CPU : Another card (different IIO stack) reading from this card. : Number of DWs (4 bytes) requested by the main die. Includes all requests initiated by the main die, including reads and writes. : x4 card is plugged in to slot 1", + "UMask": "0x7002008", + "Unit": "IIO" + }, + { + "BriefDescription": "Peer to peer read request for 4 bytes made by a different IIO unit to IIO Part0", + "EventCode": "0xc0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.PEER_READ.PART2", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0004", + "PublicDescription": "Data requested by the CPU : Another card (different IIO stack) reading from this card. : Number of DWs (4 bytes) requested by the main die. Includes all requests initiated by the main die, including reads and writes. : x8 card plugged in to Lane 2/3, Or x4 card is plugged in to slot 1", + "UMask": "0x7004008", + "Unit": "IIO" + }, + { + "BriefDescription": "Peer to peer read request for 4 bytes made by a different IIO unit to IIO Part0", + "EventCode": "0xc0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.PEER_READ.PART3", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0008", + "PublicDescription": "Data requested by the CPU : Another card (different IIO stack) reading from this card. : Number of DWs (4 bytes) requested by the main die. Includes all requests initiated by the main die, including reads and writes. : x4 card is plugged in to slot 3", + "UMask": "0x7008008", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested by the CPU : Another card (different IIO stack) reading from this card.", + "EventCode": "0xc0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.PEER_READ.PART4", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0010", + "PublicDescription": "Data requested by the CPU : Another card (different IIO stack) reading from this card. : Number of DWs (4 bytes) requested by the main die. Includes all requests initiated by the main die, including reads and writes. : x16 card plugged in to stack, Or x8 card plugged in to Lane 0/1, Or x4 card is plugged in to slot 0", + "UMask": "0x7010008", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested by the CPU : Another card (different IIO stack) reading from this card.", + "EventCode": "0xc0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.PEER_READ.PART5", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0020", + "PublicDescription": "Data requested by the CPU : Another card (different IIO stack) reading from this card. : Number of DWs (4 bytes) requested by the main die. Includes all requests initiated by the main die, including reads and writes. : x4 card is plugged in to slot 1", + "UMask": "0x7020008", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested by the CPU : Another card (different IIO stack) reading from this card.", + "EventCode": "0xc0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.PEER_READ.PART6", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0040", + "PublicDescription": "Data requested by the CPU : Another card (different IIO stack) reading from this card. : Number of DWs (4 bytes) requested by the main die. Includes all requests initiated by the main die, including reads and writes. : x8 card plugged in to Lane 2/3, Or x4 card is plugged in to slot 1", + "UMask": "0x7040008", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested by the CPU : Another card (different IIO stack) reading from this card.", + "EventCode": "0xc0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.PEER_READ.PART7", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0080", + "PublicDescription": "Data requested by the CPU : Another card (different IIO stack) reading from this card. : Number of DWs (4 bytes) requested by the main die. Includes all requests initiated by the main die, including reads and writes. : x4 card is plugged in to slot 3", + "UMask": "0x7080008", + "Unit": "IIO" + }, + { + "BriefDescription": "Peer to peer write request of 4 bytes made to IIO Part0 by a different IIO unit", + "EventCode": "0xc0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.PEER_WRITE.PART0", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0001", + "PublicDescription": "Data requested by the CPU : Another card (different IIO stack) writing to this card. : Number of DWs (4 bytes) requested by the main die. Includes all requests initiated by the main die, including reads and writes. : x16 card plugged in to stack, Or x8 card plugged in to Lane 0/1, Or x4 card is plugged in to slot 0", + "UMask": "0x7001002", + "Unit": "IIO" + }, + { + "BriefDescription": "Peer to peer write request of 4 bytes made to IIO Part0 by a different IIO unit", + "EventCode": "0xc0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.PEER_WRITE.PART1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0002", + "PublicDescription": "Data requested by the CPU : Another card (different IIO stack) writing to this card. : Number of DWs (4 bytes) requested by the main die. Includes all requests initiated by the main die, including reads and writes. : x4 card is plugged in to slot 1", + "UMask": "0x7002002", + "Unit": "IIO" + }, + { + "BriefDescription": "Peer to peer write request of 4 bytes made to IIO Part0 by a different IIO unit", + "EventCode": "0xc0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.PEER_WRITE.PART2", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0004", + "PublicDescription": "Data requested by the CPU : Another card (different IIO stack) writing to this card. : Number of DWs (4 bytes) requested by the main die. Includes all requests initiated by the main die, including reads and writes. : x8 card plugged in to Lane 2/3, Or x4 card is plugged in to slot 1", + "UMask": "0x7004002", + "Unit": "IIO" + }, + { + "BriefDescription": "Peer to peer write request of 4 bytes made to IIO Part0 by a different IIO unit", + "EventCode": "0xc0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.PEER_WRITE.PART3", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0008", + "PublicDescription": "Data requested by the CPU : Another card (different IIO stack) writing to this card. : Number of DWs (4 bytes) requested by the main die. Includes all requests initiated by the main die, including reads and writes. : x4 card is plugged in to slot 3", + "UMask": "0x7008002", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested by the CPU : Another card (different IIO stack) writing to this card.", + "EventCode": "0xc0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.PEER_WRITE.PART4", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0010", + "PublicDescription": "Data requested by the CPU : Another card (different IIO stack) writing to this card. : Number of DWs (4 bytes) requested by the main die. Includes all requests initiated by the main die, including reads and writes. : x16 card plugged in to stack, Or x8 card plugged in to Lane 0/1, Or x4 card is plugged in to slot 0", + "UMask": "0x7010002", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested by the CPU : Another card (different IIO stack) writing to this card.", + "EventCode": "0xc0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.PEER_WRITE.PART5", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0020", + "PublicDescription": "Data requested by the CPU : Another card (different IIO stack) writing to this card. : Number of DWs (4 bytes) requested by the main die. Includes all requests initiated by the main die, including reads and writes. : x4 card is plugged in to slot 1", + "UMask": "0x7020002", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested by the CPU : Another card (different IIO stack) writing to this card.", + "EventCode": "0xc0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.PEER_WRITE.PART6", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0040", + "PublicDescription": "Data requested by the CPU : Another card (different IIO stack) writing to this card. : Number of DWs (4 bytes) requested by the main die. Includes all requests initiated by the main die, including reads and writes. : x8 card plugged in to Lane 2/3, Or x4 card is plugged in to slot 1", + "UMask": "0x7040002", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested by the CPU : Another card (different IIO stack) writing to this card.", + "EventCode": "0xc0", + "EventName": "UNC_IIO_DATA_REQ_BY_CPU.PEER_WRITE.PART7", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0080", + "PublicDescription": "Data requested by the CPU : Another card (different IIO stack) writing to this card. : Number of DWs (4 bytes) requested by the main die. Includes all requests initiated by the main die, including reads and writes. : x4 card is plugged in to slot 3", + "UMask": "0x7080002", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested of the CPU : CmpD - device sending completion to CPU request", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.CMPD.ALL_PARTS", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0xff", + "PublicDescription": "Data requested of the CPU : CmpD - device sending completion to CPU request : Number of DWs (4 bytes) the card requests of the main die. Includes all requests initiated by the Card, including reads and writes. : x16 card plugged in to Lane 0/1/2/3, Or x8 card plugged in to Lane 0/1, Or x4 card is plugged in to slot 0", + "UMask": "0x80", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested of the CPU : CmpD - device sending completion to CPU request", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.CMPD.PART0", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0001", + "PublicDescription": "Data requested of the CPU : CmpD - device sending completion to CPU request : Number of DWs (4 bytes) the card requests of the main die. Includes all requests initiated by the Card, including reads and writes. : x16 card plugged in to Lane 0/1/2/3, Or x8 card plugged in to Lane 0/1, Or x4 card is plugged in to slot 0", + "UMask": "0x80", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested of the CPU : CmpD - device sending completion to CPU request", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.CMPD.PART1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0002", + "PublicDescription": "Data requested of the CPU : CmpD - device sending completion to CPU request : Number of DWs (4 bytes) the card requests of the main die. Includes all requests initiated by the Card, including reads and writes. : x4 card is plugged in to slot 1", + "UMask": "0x80", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested of the CPU : CmpD - device sending completion to CPU request", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.CMPD.PART2", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0004", + "PublicDescription": "Data requested of the CPU : CmpD - device sending completion to CPU request : Number of DWs (4 bytes) the card requests of the main die. Includes all requests initiated by the Card, including reads and writes. : x8 card plugged in to Lane 2/3, Or x4 card is plugged in to slot 2", + "UMask": "0x80", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested of the CPU : CmpD - device sending completion to CPU request", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.CMPD.PART3", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0008", + "PublicDescription": "Data requested of the CPU : CmpD - device sending completion to CPU request : Number of DWs (4 bytes) the card requests of the main die. Includes all requests initiated by the Card, including reads and writes. : x4 card is plugged in to slot 3", + "UMask": "0x80", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested of the CPU : CmpD - device sending completion to CPU request", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.CMPD.PART4", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0010", + "PublicDescription": "Data requested of the CPU : CmpD - device sending completion to CPU request : Number of DWs (4 bytes) the card requests of the main die. Includes all requests initiated by the Card, including reads and writes. : x16 card plugged in to Lane 4/5/6/7, Or x8 card plugged in to Lane 4/5, Or x4 card is plugged in to slot 4", + "UMask": "0x80", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested of the CPU : CmpD - device sending completion to CPU request", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.CMPD.PART5", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0020", + "PublicDescription": "Data requested of the CPU : CmpD - device sending completion to CPU request : Number of DWs (4 bytes) the card requests of the main die. Includes all requests initiated by the Card, including reads and writes. : x4 card is plugged in to slot 5", + "UMask": "0x80", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested of the CPU : CmpD - device sending completion to CPU request", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.CMPD.PART6", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0040", + "PublicDescription": "Data requested of the CPU : CmpD - device sending completion to CPU request : Number of DWs (4 bytes) the card requests of the main die. Includes all requests initiated by the Card, including reads and writes. : x8 card plugged in to Lane 6/7, Or x4 card is plugged in to slot 6", + "UMask": "0x80", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested of the CPU : CmpD - device sending completion to CPU request", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.CMPD.PART7", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0080", + "PublicDescription": "Data requested of the CPU : CmpD - device sending completion to CPU request : Number of DWs (4 bytes) the card requests of the main die. Includes all requests initiated by the Card, including reads and writes. : x4 card is plugged in to slot 7", + "UMask": "0x80", + "Unit": "IIO" + }, + { + "BriefDescription": "Read request for 4 bytes made by IIO Part0-7 to Memory", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.ALL_PARTS", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x00ff", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Read request for 4 bytes made by IIO Part0 to Memory", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART0", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0001", + "PublicDescription": "Data requested of the CPU : Card reading from DRAM : Number of DWs (4 bytes) the card requests of the main die. Includes all requests initiated by the Card, including reads and writes. : x16 card plugged in to stack, Or x8 card plugged in to Lane 0/1, Or x4 card is plugged in to slot 0", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Read request for 4 bytes made by IIO Part1 to Memory", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0002", + "PublicDescription": "Data requested of the CPU : Card reading from DRAM : Number of DWs (4 bytes) the card requests of the main die. Includes all requests initiated by the Card, including reads and writes. : x4 card is plugged in to slot 1", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Read request for 4 bytes made by IIO Part2 to Memory", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART2", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0004", + "PublicDescription": "Data requested of the CPU : Card reading from DRAM : Number of DWs (4 bytes) the card requests of the main die. Includes all requests initiated by the Card, including reads and writes. : x8 card plugged in to Lane 2/3, Or x4 card is plugged in to slot 1", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Read request for 4 bytes made by IIO Part3 to Memory", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART3", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0008", + "PublicDescription": "Data requested of the CPU : Card reading from DRAM : Number of DWs (4 bytes) the card requests of the main die. Includes all requests initiated by the Card, including reads and writes. : x4 card is plugged in to slot 3", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested of the CPU : Card reading from DRAM", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART4", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0010", + "PublicDescription": "Data requested of the CPU : Card reading from DRAM : Number of DWs (4 bytes) the card requests of the main die. Includes all requests initiated by the Card, including reads and writes. : x16 card plugged in to stack, Or x8 card plugged in to Lane 0/1, Or x4 card is plugged in to slot 0", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested of the CPU : Card reading from DRAM", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART5", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0020", + "PublicDescription": "Data requested of the CPU : Card reading from DRAM : Number of DWs (4 bytes) the card requests of the main die. Includes all requests initiated by the Card, including reads and writes. : x4 card is plugged in to slot 1", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested of the CPU : Card reading from DRAM", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART6", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0040", + "PublicDescription": "Data requested of the CPU : Card reading from DRAM : Number of DWs (4 bytes) the card requests of the main die. Includes all requests initiated by the Card, including reads and writes. : x8 card plugged in to Lane 2/3, Or x4 card is plugged in to slot 1", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested of the CPU : Card reading from DRAM", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART7", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0080", + "PublicDescription": "Data requested of the CPU : Card reading from DRAM : Number of DWs (4 bytes) the card requests of the main die. Includes all requests initiated by the Card, including reads and writes. : x4 card is plugged in to slot 3", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Write request of 4 bytes made by IIO Part0-7 to Memory", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.ALL_PARTS", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x00ff", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Write request of 4 bytes made by IIO Part0 to Memory", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART0", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0001", + "PublicDescription": "Data requested of the CPU : Card writing to DRAM : Number of DWs (4 bytes) the card requests of the main die. Includes all requests initiated by the Card, including reads and writes. : x16 card plugged in to stack, Or x8 card plugged in to Lane 0/1, Or x4 card is plugged in to slot 0", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Write request of 4 bytes made by IIO Part1 to Memory", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0002", + "PublicDescription": "Data requested of the CPU : Card writing to DRAM : Number of DWs (4 bytes) the card requests of the main die. Includes all requests initiated by the Card, including reads and writes. : x4 card is plugged in to slot 1", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Write request of 4 bytes made by IIO Part2 to Memory", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART2", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0004", + "PublicDescription": "Data requested of the CPU : Card writing to DRAM : Number of DWs (4 bytes) the card requests of the main die. Includes all requests initiated by the Card, including reads and writes. : x8 card plugged in to Lane 2/3, Or x4 card is plugged in to slot 1", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Write request of 4 bytes made by IIO Part3 to Memory", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART3", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0008", + "PublicDescription": "Data requested of the CPU : Card writing to DRAM : Number of DWs (4 bytes) the card requests of the main die. Includes all requests initiated by the Card, including reads and writes. : x4 card is plugged in to slot 3", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested of the CPU : Card writing to DRAM", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART4", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0010", + "PublicDescription": "Data requested of the CPU : Card writing to DRAM : Number of DWs (4 bytes) the card requests of the main die. Includes all requests initiated by the Card, including reads and writes. : x16 card plugged in to stack, Or x8 card plugged in to Lane 0/1, Or x4 card is plugged in to slot 0", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested of the CPU : Card writing to DRAM", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART5", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0020", + "PublicDescription": "Data requested of the CPU : Card writing to DRAM : Number of DWs (4 bytes) the card requests of the main die. Includes all requests initiated by the Card, including reads and writes. : x4 card is plugged in to slot 1", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested of the CPU : Card writing to DRAM", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART6", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0040", + "PublicDescription": "Data requested of the CPU : Card writing to DRAM : Number of DWs (4 bytes) the card requests of the main die. Includes all requests initiated by the Card, including reads and writes. : x8 card plugged in to Lane 2/3, Or x4 card is plugged in to slot 1", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested of the CPU : Card writing to DRAM", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART7", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0080", + "PublicDescription": "Data requested of the CPU : Card writing to DRAM : Number of DWs (4 bytes) the card requests of the main die. Includes all requests initiated by the Card, including reads and writes. : x4 card is plugged in to slot 3", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Peer to peer write request of 4 bytes made by IIO Part0 to an IIO target", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.PEER_WRITE.PART0", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0001", + "PublicDescription": "Data requested of the CPU : Card writing to another Card (same or different stack) : Number of DWs (4 bytes) the card requests of the main die. Includes all requests initiated by the Card, including reads and writes. : x16 card plugged in to stack, Or x8 card plugged in to Lane 0/1, Or x4 card is plugged in to slot 0", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "Peer to peer write request of 4 bytes made by IIO Part0 to an IIO target", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.PEER_WRITE.PART1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0002", + "PublicDescription": "Data requested of the CPU : Card writing to another Card (same or different stack) : Number of DWs (4 bytes) the card requests of the main die. Includes all requests initiated by the Card, including reads and writes. : x4 card is plugged in to slot 1", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "Peer to peer write request of 4 bytes made by IIO Part0 to an IIO target", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.PEER_WRITE.PART2", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0004", + "PublicDescription": "Data requested of the CPU : Card writing to another Card (same or different stack) : Number of DWs (4 bytes) the card requests of the main die. Includes all requests initiated by the Card, including reads and writes. : x8 card plugged in to Lane 2/3, Or x4 card is plugged in to slot 1", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "Peer to peer write request of 4 bytes made by IIO Part0 to an IIO target", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.PEER_WRITE.PART3", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0008", + "PublicDescription": "Data requested of the CPU : Card writing to another Card (same or different stack) : Number of DWs (4 bytes) the card requests of the main die. Includes all requests initiated by the Card, including reads and writes. : x4 card is plugged in to slot 3", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested of the CPU : Card writing to another Card (same or different stack)", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.PEER_WRITE.PART4", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0010", + "PublicDescription": "Data requested of the CPU : Card writing to another Card (same or different stack) : Number of DWs (4 bytes) the card requests of the main die. Includes all requests initiated by the Card, including reads and writes. : x16 card plugged in to stack, Or x8 card plugged in to Lane 0/1, Or x4 card is plugged in to slot 0", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested of the CPU : Card writing to another Card (same or different stack)", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.PEER_WRITE.PART5", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0020", + "PublicDescription": "Data requested of the CPU : Card writing to another Card (same or different stack) : Number of DWs (4 bytes) the card requests of the main die. Includes all requests initiated by the Card, including reads and writes. : x4 card is plugged in to slot 1", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested of the CPU : Card writing to another Card (same or different stack)", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.PEER_WRITE.PART6", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0040", + "PublicDescription": "Data requested of the CPU : Card writing to another Card (same or different stack) : Number of DWs (4 bytes) the card requests of the main die. Includes all requests initiated by the Card, including reads and writes. : x8 card plugged in to Lane 2/3, Or x4 card is plugged in to slot 1", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "Data requested of the CPU : Card writing to another Card (same or different stack)", + "EventCode": "0x83", + "EventName": "UNC_IIO_DATA_REQ_OF_CPU.PEER_WRITE.PART7", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0080", + "PublicDescription": "Data requested of the CPU : Card writing to another Card (same or different stack) : Number of DWs (4 bytes) the card requests of the main die. Includes all requests initiated by the Card, including reads and writes. : x4 card is plugged in to slot 3", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "Incoming arbitration requests : Passing data to be written", + "EventCode": "0x86", + "EventName": "UNC_IIO_INBOUND_ARB_REQ.DATA", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x00FF", + "PublicDescription": "Incoming arbitration requests : Passing data to be written : How often different queues (e.g. channel / fc) ask to send request into pipeline : Only for posted requests", + "UMask": "0x70ff020", + "Unit": "IIO" + }, + { + "BriefDescription": "Incoming arbitration requests : Issuing final read or write of line", + "EventCode": "0x86", + "EventName": "UNC_IIO_INBOUND_ARB_REQ.FINAL_RD_WR", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x00FF", + "PublicDescription": "Incoming arbitration requests : Issuing final read or write of line : How often different queues (e.g. channel / fc) ask to send request into pipeline", + "UMask": "0x8", + "Unit": "IIO" + }, + { + "BriefDescription": "Incoming arbitration requests : Processing response from IOMMU", + "EventCode": "0x86", + "EventName": "UNC_IIO_INBOUND_ARB_REQ.IOMMU_HIT", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x00FF", + "PublicDescription": "Incoming arbitration requests : Processing response from IOMMU : How often different queues (e.g. channel / fc) ask to send request into pipeline", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "Incoming arbitration requests : Issuing to IOMMU", + "EventCode": "0x86", + "EventName": "UNC_IIO_INBOUND_ARB_REQ.IOMMU_REQ", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x00FF", + "PublicDescription": "Incoming arbitration requests : Issuing to IOMMU : How often different queues (e.g. channel / fc) ask to send request into pipeline", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Incoming arbitration requests : Request Ownership", + "EventCode": "0x86", + "EventName": "UNC_IIO_INBOUND_ARB_REQ.REQ_OWN", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x00FF", + "PublicDescription": "Incoming arbitration requests : Request Ownership : How often different queues (e.g. channel / fc) ask to send request into pipeline : Only for posted requests", + "UMask": "0x70ff004", + "Unit": "IIO" + }, + { + "BriefDescription": "Incoming arbitration requests : Writing line", + "EventCode": "0x86", + "EventName": "UNC_IIO_INBOUND_ARB_REQ.WR", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x00FF", + "PublicDescription": "Incoming arbitration requests : Writing line : How often different queues (e.g. channel / fc) ask to send request into pipeline : Only for posted requests", + "UMask": "0x70ff010", + "Unit": "IIO" + }, + { + "BriefDescription": "Incoming arbitration requests granted : Passing data to be written", + "EventCode": "0x87", + "EventName": "UNC_IIO_INBOUND_ARB_WON.DATA", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x00FF", + "PublicDescription": "Incoming arbitration requests granted : Passing data to be written : How often different queues (e.g. channel / fc) are allowed to send request into pipeline : Only for posted requests", + "UMask": "0x70ff020", + "Unit": "IIO" + }, + { + "BriefDescription": "Incoming arbitration requests granted : Issuing final read or write of line", + "EventCode": "0x87", + "EventName": "UNC_IIO_INBOUND_ARB_WON.FINAL_RD_WR", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x00FF", + "PublicDescription": "Incoming arbitration requests granted : Issuing final read or write of line : How often different queues (e.g. channel / fc) are allowed to send request into pipeline", + "UMask": "0x70ff008", + "Unit": "IIO" + }, + { + "BriefDescription": "Incoming arbitration requests granted : Processing response from IOMMU", + "EventCode": "0x87", + "EventName": "UNC_IIO_INBOUND_ARB_WON.IOMMU_HIT", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x00FF", + "PublicDescription": "Incoming arbitration requests granted : Processing response from IOMMU : How often different queues (e.g. channel / fc) are allowed to send request into pipeline", + "UMask": "0x70ff002", + "Unit": "IIO" + }, + { + "BriefDescription": "Incoming arbitration requests granted : Issuing to IOMMU", + "EventCode": "0x87", + "EventName": "UNC_IIO_INBOUND_ARB_WON.IOMMU_REQ", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x00FF", + "PublicDescription": "Incoming arbitration requests granted : Issuing to IOMMU : How often different queues (e.g. channel / fc) are allowed to send request into pipeline", + "UMask": "0x70ff001", + "Unit": "IIO" + }, + { + "BriefDescription": "Incoming arbitration requests granted : Request Ownership", + "EventCode": "0x87", + "EventName": "UNC_IIO_INBOUND_ARB_WON.REQ_OWN", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x00FF", + "PublicDescription": "Incoming arbitration requests granted : Request Ownership : How often different queues (e.g. channel / fc) are allowed to send request into pipeline : Only for posted requests", + "UMask": "0x70ff004", + "Unit": "IIO" + }, + { + "BriefDescription": "Incoming arbitration requests granted : Writing line", + "EventCode": "0x87", + "EventName": "UNC_IIO_INBOUND_ARB_WON.WR", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x00FF", + "PublicDescription": "Incoming arbitration requests granted : Writing line : How often different queues (e.g. channel / fc) are allowed to send request into pipeline : Only for posted requests", + "UMask": "0x70ff010", + "Unit": "IIO" + }, + { + "BriefDescription": ": Context cache hits", + "EventCode": "0x40", + "EventName": "UNC_IIO_IOMMU0.CTXT_CACHE_HITS", + "PerPkg": "1", + "PortMask": "0x0000", + "PublicDescription": ": Context cache hits : Counts each time a first look up of the transaction hits the RCC.", + "UMask": "0x80", + "Unit": "IIO" + }, + { + "BriefDescription": ": Context cache lookups", + "EventCode": "0x40", + "EventName": "UNC_IIO_IOMMU0.CTXT_CACHE_LOOKUPS", + "PerPkg": "1", + "PortMask": "0x0000", + "PublicDescription": ": Context cache lookups : Counts each time a transaction looks up root context cache.", + "UMask": "0x40", + "Unit": "IIO" + }, + { + "BriefDescription": ": IOTLB lookups first", + "EventCode": "0x40", + "EventName": "UNC_IIO_IOMMU0.FIRST_LOOKUPS", + "PerPkg": "1", + "PortMask": "0x0000", + "PublicDescription": ": IOTLB lookups first : Some transactions have to look up IOTLB multiple times. Counts the first time a request looks up IOTLB.", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "IOTLB Fills (same as IOTLB miss)", + "EventCode": "0x40", + "EventName": "UNC_IIO_IOMMU0.MISSES", + "PerPkg": "1", + "PortMask": "0x0000", + "PublicDescription": "IOTLB Fills (same as IOTLB miss) : When a transaction misses IOTLB, it does a page walk to look up memory and bring in the relevant page translation. Counts when this page translation is written to IOTLB.", + "UMask": "0x20", + "Unit": "IIO" + }, + { + "BriefDescription": ": IOMMU memory access", + "EventCode": "0x41", + "EventName": "UNC_IIO_IOMMU1.NUM_MEM_ACCESSES", + "PerPkg": "1", + "PublicDescription": ": IOMMU memory access : IOMMU sends out memory fetches when it misses the cache look up which is indicated by this signal. M2IOSF only uses low priority channel", + "UMask": "0xc0", + "Unit": "IIO" + }, + { + "BriefDescription": ": PWC Hit to a 2M page", + "EventCode": "0x41", + "EventName": "UNC_IIO_IOMMU1.PWC_1G_HITS", + "PerPkg": "1", + "PublicDescription": ": PWC Hit to a 2M page : Counts each time a transaction's first look up hits the SLPWC at the 2M level", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": ": PWT Hit to a 256T page", + "EventCode": "0x41", + "EventName": "UNC_IIO_IOMMU1.PWC_256T_HITS", + "PerPkg": "1", + "PublicDescription": ": PWT Hit to a 256T page : Counts each time a transaction's first look up hits the SLPWC at the 512G level", + "UMask": "0x10", + "Unit": "IIO" + }, + { + "BriefDescription": ": PWC Hit to a 4K page", + "EventCode": "0x41", + "EventName": "UNC_IIO_IOMMU1.PWC_2M_HITS", + "PerPkg": "1", + "PublicDescription": ": PWC Hit to a 4K page : Counts each time a transaction's first look up hits the SLPWC at the 4K level", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": ": PWC Hit to a 1G page", + "EventCode": "0x41", + "EventName": "UNC_IIO_IOMMU1.PWC_512G_HITS", + "PerPkg": "1", + "PublicDescription": ": PWC Hit to a 1G page : Counts each time a transaction's first look up hits the SLPWC at the 1G level", + "UMask": "0x8", + "Unit": "IIO" + }, + { + "BriefDescription": ": PageWalk cache fill", + "EventCode": "0x41", + "EventName": "UNC_IIO_IOMMU1.PWC_CACHE_FILLS", + "PerPkg": "1", + "PublicDescription": ": PageWalk cache fill : When a transaction misses SLPWC, it does a page walk to look up memory and bring in the relevant page translation. When this page translation is written to SLPWC, ObsPwcFillValid_nnnH is asserted.", + "UMask": "0x20", + "Unit": "IIO" + }, + { + "BriefDescription": ": PageWalk cache lookup", + "EventCode": "0x41", + "EventName": "UNC_IIO_IOMMU1.PWT_CACHE_LOOKUPS", + "PerPkg": "1", + "PublicDescription": ": PageWalk cache lookup : Counts each time a transaction looks up second level page walk cache.", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": ": PWC Hit to a 2M page", + "EventCode": "0x41", + "EventName": "UNC_IIO_IOMMU1.SLPWC_1G_HITS", + "PerPkg": "1", + "PublicDescription": ": PWC Hit to a 2M page : Counts each time a transaction's first look up hits the SLPWC at the 2M level", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": ": PWC Hit to a 2M page", + "EventCode": "0x41", + "EventName": "UNC_IIO_IOMMU1.SLPWC_256T_HITS", + "PerPkg": "1", + "PublicDescription": ": PWC Hit to a 2M page : Counts each time a transaction's first look up hits the SLPWC at the 2M level", + "UMask": "0x10", + "Unit": "IIO" + }, + { + "BriefDescription": ": PWC Hit to a 1G page", + "EventCode": "0x41", + "EventName": "UNC_IIO_IOMMU1.SLPWC_512G_HITS", + "PerPkg": "1", + "PublicDescription": ": PWC Hit to a 1G page : Counts each time a transaction's first look up hits the SLPWC at the 1G level", + "UMask": "0x8", + "Unit": "IIO" + }, + { + "BriefDescription": ": Global IOTLB invalidation cycles", + "EventCode": "0x43", + "EventName": "UNC_IIO_IOMMU3.PWT_OCCUPANCY_MSB", + "PerPkg": "1", + "PortMask": "0x0000", + "PublicDescription": ": Global IOTLB invalidation cycles : Indicates that IOMMU is doing global invalidation.", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "AND Mask/match for debug bus : Non-PCIE bus", + "EventCode": "0x02", + "EventName": "UNC_IIO_MASK_MATCH_AND.BUS0", + "PerPkg": "1", + "PortMask": "0x0000", + "PublicDescription": "AND Mask/match for debug bus : Non-PCIE bus : Asserted if all bits specified by mask match", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "AND Mask/match for debug bus : Non-PCIE bus and PCIE bus", + "EventCode": "0x02", + "EventName": "UNC_IIO_MASK_MATCH_AND.BUS0_BUS1", + "PerPkg": "1", + "PortMask": "0x0000", + "PublicDescription": "AND Mask/match for debug bus : Non-PCIE bus and PCIE bus : Asserted if all bits specified by mask match", + "UMask": "0x8", + "Unit": "IIO" + }, + { + "BriefDescription": "AND Mask/match for debug bus : Non-PCIE bus and !(PCIE bus)", + "EventCode": "0x02", + "EventName": "UNC_IIO_MASK_MATCH_AND.BUS0_NOT_BUS1", + "PerPkg": "1", + "PortMask": "0x0000", + "PublicDescription": "AND Mask/match for debug bus : Non-PCIE bus and !(PCIE bus) : Asserted if all bits specified by mask match", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "AND Mask/match for debug bus : PCIE bus", + "EventCode": "0x02", + "EventName": "UNC_IIO_MASK_MATCH_AND.BUS1", + "PerPkg": "1", + "PortMask": "0x0000", + "PublicDescription": "AND Mask/match for debug bus : PCIE bus : Asserted if all bits specified by mask match", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "AND Mask/match for debug bus : !(Non-PCIE bus) and PCIE bus", + "EventCode": "0x02", + "EventName": "UNC_IIO_MASK_MATCH_AND.NOT_BUS0_BUS1", + "PerPkg": "1", + "PortMask": "0x0000", + "PublicDescription": "AND Mask/match for debug bus : !(Non-PCIE bus) and PCIE bus : Asserted if all bits specified by mask match", + "UMask": "0x10", + "Unit": "IIO" + }, + { + "BriefDescription": "AND Mask/match for debug bus : !(Non-PCIE bus) and !(PCIE bus)", + "EventCode": "0x02", + "EventName": "UNC_IIO_MASK_MATCH_AND.NOT_BUS0_NOT_BUS1", + "PerPkg": "1", + "PortMask": "0x0000", + "PublicDescription": "AND Mask/match for debug bus : !(Non-PCIE bus) and !(PCIE bus) : Asserted if all bits specified by mask match", + "UMask": "0x20", + "Unit": "IIO" + }, + { + "BriefDescription": "OR Mask/match for debug bus : Non-PCIE bus", + "EventCode": "0x03", + "EventName": "UNC_IIO_MASK_MATCH_OR.BUS0", + "PerPkg": "1", + "PortMask": "0x0000", + "PublicDescription": "OR Mask/match for debug bus : Non-PCIE bus : Asserted if any bits specified by mask match", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "OR Mask/match for debug bus : Non-PCIE bus and PCIE bus", + "EventCode": "0x03", + "EventName": "UNC_IIO_MASK_MATCH_OR.BUS0_BUS1", + "PerPkg": "1", + "PortMask": "0x0000", + "PublicDescription": "OR Mask/match for debug bus : Non-PCIE bus and PCIE bus : Asserted if any bits specified by mask match", + "UMask": "0x8", + "Unit": "IIO" + }, + { + "BriefDescription": "OR Mask/match for debug bus : Non-PCIE bus and !(PCIE bus)", + "EventCode": "0x03", + "EventName": "UNC_IIO_MASK_MATCH_OR.BUS0_NOT_BUS1", + "PerPkg": "1", + "PortMask": "0x0000", + "PublicDescription": "OR Mask/match for debug bus : Non-PCIE bus and !(PCIE bus) : Asserted if any bits specified by mask match", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "OR Mask/match for debug bus : PCIE bus", + "EventCode": "0x03", + "EventName": "UNC_IIO_MASK_MATCH_OR.BUS1", + "PerPkg": "1", + "PortMask": "0x0000", + "PublicDescription": "OR Mask/match for debug bus : PCIE bus : Asserted if any bits specified by mask match", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "OR Mask/match for debug bus : !(Non-PCIE bus) and PCIE bus", + "EventCode": "0x03", + "EventName": "UNC_IIO_MASK_MATCH_OR.NOT_BUS0_BUS1", + "PerPkg": "1", + "PortMask": "0x0000", + "PublicDescription": "OR Mask/match for debug bus : !(Non-PCIE bus) and PCIE bus : Asserted if any bits specified by mask match", + "UMask": "0x10", + "Unit": "IIO" + }, + { + "BriefDescription": "OR Mask/match for debug bus : !(Non-PCIE bus) and !(PCIE bus)", + "EventCode": "0x03", + "EventName": "UNC_IIO_MASK_MATCH_OR.NOT_BUS0_NOT_BUS1", + "PerPkg": "1", + "PortMask": "0x0000", + "PublicDescription": "OR Mask/match for debug bus : !(Non-PCIE bus) and !(PCIE bus) : Asserted if any bits specified by mask match", + "UMask": "0x20", + "Unit": "IIO" + }, + { + "BriefDescription": "Number requests PCIe makes of the main die : All", + "EventCode": "0x85", + "EventName": "UNC_IIO_NUM_REQ_OF_CPU.COMMIT.ALL", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0FFF", + "PublicDescription": "Number requests PCIe makes of the main die : All : Counts full PCIe requests before they're broken into a series of cache-line size requests as measured by DATA_REQ_OF_CPU and TXN_REQ_OF_CPU.", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Num requests sent by PCIe - by target : Abort", + "EventCode": "0x8e", + "EventName": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.ABORT", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x00FF", + "UMask": "0x80", + "Unit": "IIO" + }, + { + "BriefDescription": "Num requests sent by PCIe - by target : Confined P2P", + "EventCode": "0x8e", + "EventName": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.CONFINED_P2P", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x00FF", + "UMask": "0x40", + "Unit": "IIO" + }, + { + "BriefDescription": "Num requests sent by PCIe - by target : Local P2P", + "EventCode": "0x8e", + "EventName": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.LOC_P2P", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x00FF", + "UMask": "0x20", + "Unit": "IIO" + }, + { + "BriefDescription": "Num requests sent by PCIe - by target : Multi-cast", + "EventCode": "0x8e", + "EventName": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.MCAST", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x00FF", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "Num requests sent by PCIe - by target : Memory", + "EventCode": "0x8e", + "EventName": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.MEM", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x00FF", + "UMask": "0x8", + "Unit": "IIO" + }, + { + "BriefDescription": "Num requests sent by PCIe - by target : MsgB", + "EventCode": "0x8e", + "EventName": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.MSGB", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x00FF", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Num requests sent by PCIe - by target : Remote P2P", + "EventCode": "0x8e", + "EventName": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.REM_P2P", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x00FF", + "UMask": "0x10", + "Unit": "IIO" + }, + { + "BriefDescription": "Num requests sent by PCIe - by target : Ubox", + "EventCode": "0x8e", + "EventName": "UNC_IIO_NUM_REQ_OF_CPU_BY_TGT.UBOX", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x00FF", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "ITC address map 1", + "EventCode": "0x8f", + "EventName": "UNC_IIO_NUM_TGT_MATCHED_REQ_OF_CPU", + "PerPkg": "1", + "PortMask": "0x0000", + "PublicDescription": "UNC_IIO_NUM_TGT_MATCHED_REQ_OF_CPU", + "Unit": "IIO" + }, + { + "BriefDescription": "Outbound cacheline requests issued : 64B requests issued to device", + "EventCode": "0xd0", + "EventName": "UNC_IIO_OUTBOUND_CL_REQS_ISSUED.TO_IO", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x00FF", + "PublicDescription": "Outbound cacheline requests issued : 64B requests issued to device : Each outbound cacheline granular request may need to make multiple passes through the pipeline. Each time a cacheline completes all its passes it advances line", + "UMask": "0x8", + "Unit": "IIO" + }, + { + "BriefDescription": "Outbound TLP (transaction layer packet) requests issued : To device", + "EventCode": "0xd1", + "EventName": "UNC_IIO_OUTBOUND_TLP_REQS_ISSUED.TO_IO", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x00FF", + "PublicDescription": "Outbound TLP (transaction layer packet) requests issued : To device : Each time an outbound completes all its passes it advances the pointer", + "UMask": "0x8", + "Unit": "IIO" + }, + { + "BriefDescription": "PWT occupancy. Does not include 9th bit of occupancy (will undercount if PWT is greater than 255 per cycle).", + "EventCode": "0x42", + "EventName": "UNC_IIO_PWT_OCCUPANCY", + "PerPkg": "1", + "PortMask": "0x0000", + "PublicDescription": "PWT occupancy : Indicates how many page walks are outstanding at any point in time.", + "UMask": "0xff", + "Unit": "IIO" + }, + { + "BriefDescription": "Request Ownership : PCIe Request complete", + "EventCode": "0x91", + "EventName": "UNC_IIO_REQ_FROM_PCIE_CL_CMPL.DATA", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x00FF", + "PublicDescription": "Request Ownership : PCIe Request complete : Only for posted requests : Each PCIe request is broken down into a series of cacheline granular requests and each cacheline size request may need to make multiple passes through the pipeline (e.g. for posted interrupts or multi-cast). Each time a single PCIe request completes all its cacheline granular requests, it advances pointer.", + "UMask": "0x70ff020", + "Unit": "IIO" + }, + { + "BriefDescription": "Request Ownership : Writing line", + "EventCode": "0x91", + "EventName": "UNC_IIO_REQ_FROM_PCIE_CL_CMPL.FINAL_RD_WR", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x00FF", + "PublicDescription": "Request Ownership : Writing line : Only for posted requests : Only for posted requests", + "UMask": "0x70ff008", + "Unit": "IIO" + }, + { + "BriefDescription": "Request Ownership : Issuing final read or write of line", + "EventCode": "0x91", + "EventName": "UNC_IIO_REQ_FROM_PCIE_CL_CMPL.REQ_OWN", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x00FF", + "PublicDescription": "Request Ownership : Issuing final read or write of line : Only for posted requests", + "UMask": "0x70ff004", + "Unit": "IIO" + }, + { + "BriefDescription": "Request Ownership : Passing data to be written", + "EventCode": "0x91", + "EventName": "UNC_IIO_REQ_FROM_PCIE_CL_CMPL.WR", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x00FF", + "PublicDescription": "Request Ownership : Passing data to be written : Only for posted requests : Only for posted requests", + "UMask": "0x70ff010", + "Unit": "IIO" + }, + { + "BriefDescription": "Processing response from IOMMU : Passing data to be written", + "EventCode": "0x92", + "EventName": "UNC_IIO_REQ_FROM_PCIE_CMPL.FINAL_RD_WR", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x00FF", + "PublicDescription": "Processing response from IOMMU : Passing data to be written : Only for posted requests", + "UMask": "0x70ff008", + "Unit": "IIO" + }, + { + "BriefDescription": "Processing response from IOMMU : Issuing final read or write of line", + "EventCode": "0x92", + "EventName": "UNC_IIO_REQ_FROM_PCIE_CMPL.IOMMU_HIT", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x00FF", + "UMask": "0x70ff002", + "Unit": "IIO" + }, + { + "BriefDescription": "Processing response from IOMMU : Request Ownership", + "EventCode": "0x92", + "EventName": "UNC_IIO_REQ_FROM_PCIE_CMPL.IOMMU_REQ", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x00FF", + "PublicDescription": "Processing response from IOMMU : Request Ownership : Only for posted requests", + "UMask": "0x70ff001", + "Unit": "IIO" + }, + { + "BriefDescription": "Processing response from IOMMU : Writing line", + "EventCode": "0x92", + "EventName": "UNC_IIO_REQ_FROM_PCIE_CMPL.REQ_OWN", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x00FF", + "PublicDescription": "Processing response from IOMMU : Writing line : Only for posted requests", + "UMask": "0x70ff004", + "Unit": "IIO" + }, + { + "BriefDescription": "PCIe Request - pass complete : Passing data to be written", + "EventCode": "0x90", + "EventName": "UNC_IIO_REQ_FROM_PCIE_PASS_CMPL.DATA", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x00FF", + "PublicDescription": "PCIe Request - pass complete : Passing data to be written : Each PCIe request is broken down into a series of cacheline granular requests and each cacheline size request may need to make multiple passes through the pipeline (e.g. for posted interrupts or multi-cast). Each time a cacheline completes a single pass (e.g. posts a write to single multi-cast target) it advances state : Only for posted requests", + "UMask": "0x70ff020", + "Unit": "IIO" + }, + { + "BriefDescription": "PCIe Request - pass complete : Issuing final read or write of line", + "EventCode": "0x90", + "EventName": "UNC_IIO_REQ_FROM_PCIE_PASS_CMPL.FINAL_RD_WR", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x00FF", + "PublicDescription": "PCIe Request - pass complete : Issuing final read or write of line : Each PCIe request is broken down into a series of cacheline granular requests and each cacheline size request may need to make multiple passes through the pipeline (e.g. for posted interrupts or multi-cast). Each time a cacheline completes a single pass (e.g. posts a write to single multi-cast target) it advances state", + "UMask": "0x8", + "Unit": "IIO" + }, + { + "BriefDescription": "PCIe Request - pass complete : Request Ownership", + "EventCode": "0x90", + "EventName": "UNC_IIO_REQ_FROM_PCIE_PASS_CMPL.REQ_OWN", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x00FF", + "PublicDescription": "PCIe Request - pass complete : Request Ownership : Each PCIe request is broken down into a series of cacheline granular requests and each cacheline size request may need to make multiple passes through the pipeline (e.g. for posted interrupts or multi-cast). Each time a cacheline completes a single pass (e.g. posts a write to single multi-cast target) it advances state : Only for posted requests", + "UMask": "0x70ff004", + "Unit": "IIO" + }, + { + "BriefDescription": "PCIe Request - pass complete : Writing line", + "EventCode": "0x90", + "EventName": "UNC_IIO_REQ_FROM_PCIE_PASS_CMPL.WR", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x00FF", + "PublicDescription": "PCIe Request - pass complete : Writing line : Each PCIe request is broken down into a series of cacheline granular requests and each cacheline size request may need to make multiple passes through the pipeline (e.g. for posted interrupts or multi-cast). Each time a cacheline completes a single pass (e.g. posts a write to single multi-cast target) it advances state : Only for posted requests", + "UMask": "0x70ff010", + "Unit": "IIO" + }, + { + "BriefDescription": "Read request for up to a 64 byte transaction is made by the CPU to IIO Part0", + "EventCode": "0xc1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.MEM_READ.PART0", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0001", + "PublicDescription": "Number Transactions requested by the CPU : Core reading from Card's MMIO space : Also known as Outbound. Number of requests initiated by the main die, including reads and writes. : x16 card plugged in to stack, Or x8 card plugged in to Lane 0/1, Or x4 card is plugged in to slot 0", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Read request for up to a 64 byte transaction is made by the CPU to IIO Part1", + "EventCode": "0xc1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.MEM_READ.PART1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0002", + "PublicDescription": "Number Transactions requested by the CPU : Core reading from Card's MMIO space : Also known as Outbound. Number of requests initiated by the main die, including reads and writes. : x4 card is plugged in to slot 1", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Read request for up to a 64 byte transaction is made by the CPU to IIO Part2", + "EventCode": "0xc1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.MEM_READ.PART2", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0004", + "PublicDescription": "Number Transactions requested by the CPU : Core reading from Card's MMIO space : Also known as Outbound. Number of requests initiated by the main die, including reads and writes. : x8 card plugged in to Lane 2/3, Or x4 card is plugged in to slot 1", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Read request for up to a 64 byte transaction is made by the CPU to IIO Part3", + "EventCode": "0xc1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.MEM_READ.PART3", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0008", + "PublicDescription": "Number Transactions requested by the CPU : Core reading from Card's MMIO space : Also known as Outbound. Number of requests initiated by the main die, including reads and writes. : x4 card is plugged in to slot 3", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested by the CPU : Core reading from Cards MMIO space", + "EventCode": "0xc1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.MEM_READ.PART4", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0010", + "PublicDescription": "Number Transactions requested by the CPU : Core reading from Cards MMIO space : Also known as Outbound. Number of requests initiated by the main die, including reads and writes. : x16 card plugged in to stack, Or x8 card plugged in to Lane 0/1, Or x4 card is plugged in to slot 0", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested by the CPU : Core reading from Cards MMIO space", + "EventCode": "0xc1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.MEM_READ.PART5", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0020", + "PublicDescription": "Number Transactions requested by the CPU : Core reading from Cards MMIO space : Also known as Outbound. Number of requests initiated by the main die, including reads and writes. : x4 card is plugged in to slot 1", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested by the CPU : Core reading from Cards MMIO space", + "EventCode": "0xc1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.MEM_READ.PART6", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0040", + "PublicDescription": "Number Transactions requested by the CPU : Core reading from Cards MMIO space : Also known as Outbound. Number of requests initiated by the main die, including reads and writes. : x8 card plugged in to Lane 2/3, Or x4 card is plugged in to slot 1", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested by the CPU : Core reading from Cards MMIO space", + "EventCode": "0xc1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.MEM_READ.PART7", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0080", + "PublicDescription": "Number Transactions requested by the CPU : Core reading from Cards MMIO space : Also known as Outbound. Number of requests initiated by the main die, including reads and writes. : x4 card is plugged in to slot 3", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Write request of up to a 64 byte transaction is made to IIO Part0 by the CPU", + "EventCode": "0xc1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.MEM_WRITE.PART0", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0001", + "PublicDescription": "Number Transactions requested by the CPU : Core writing to Card's MMIO space : Also known as Outbound. Number of requests initiated by the main die, including reads and writes. : x16 card plugged in to stack, Or x8 card plugged in to Lane 0/1, Or x4 card is plugged in to slot 0", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Write request of up to a 64 byte transaction is made to IIO Part1 by the CPU", + "EventCode": "0xc1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.MEM_WRITE.PART1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0002", + "PublicDescription": "Number Transactions requested by the CPU : Core writing to Card's MMIO space : Also known as Outbound. Number of requests initiated by the main die, including reads and writes. : x4 card is plugged in to slot 1", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Write request of up to a 64 byte transaction is made to IIO Part2 by the CPU", + "EventCode": "0xc1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.MEM_WRITE.PART2", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0004", + "PublicDescription": "Number Transactions requested by the CPU : Core writing to Card's MMIO space : Also known as Outbound. Number of requests initiated by the main die, including reads and writes. : x8 card plugged in to Lane 2/3, Or x4 card is plugged in to slot 1", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Write request of up to a 64 byte transaction is made to IIO Part3 by the CPU", + "EventCode": "0xc1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.MEM_WRITE.PART3", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0008", + "PublicDescription": "Number Transactions requested by the CPU : Core writing to Card's MMIO space : Also known as Outbound. Number of requests initiated by the main die, including reads and writes. : x4 card is plugged in to slot 3", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested by the CPU : Core writing to Cards MMIO space", + "EventCode": "0xc1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.MEM_WRITE.PART4", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0010", + "PublicDescription": "Number Transactions requested by the CPU : Core writing to Cards MMIO space : Also known as Outbound. Number of requests initiated by the main die, including reads and writes. : x16 card plugged in to stack, Or x8 card plugged in to Lane 0/1, Or x4 card is plugged in to slot 0", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested by the CPU : Core writing to Cards MMIO space", + "EventCode": "0xc1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.MEM_WRITE.PART5", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0020", + "PublicDescription": "Number Transactions requested by the CPU : Core writing to Cards MMIO space : Also known as Outbound. Number of requests initiated by the main die, including reads and writes. : x4 card is plugged in to slot 1", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested by the CPU : Core writing to Cards MMIO space", + "EventCode": "0xc1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.MEM_WRITE.PART6", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0040", + "PublicDescription": "Number Transactions requested by the CPU : Core writing to Cards MMIO space : Also known as Outbound. Number of requests initiated by the main die, including reads and writes. : x8 card plugged in to Lane 2/3, Or x4 card is plugged in to slot 1", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested by the CPU : Core writing to Cards MMIO space", + "EventCode": "0xc1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.MEM_WRITE.PART7", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0080", + "PublicDescription": "Number Transactions requested by the CPU : Core writing to Cards MMIO space : Also known as Outbound. Number of requests initiated by the main die, including reads and writes. : x4 card is plugged in to slot 3", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested by the CPU : Another card (different IIO stack) writing to this card.", + "EventCode": "0xc1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.PEER_WRITE.PART0", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0001", + "PublicDescription": "Number Transactions requested by the CPU : Another card (different IIO stack) writing to this card. : Also known as Outbound. Number of requests initiated by the main die, including reads and writes. : x16 card plugged in to Lane 0/1/2/3, Or x8 card plugged in to Lane 0/1, Or x4 card is plugged in to slot 0", + "UMask": "0x7001002", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested by the CPU : Another card (different IIO stack) writing to this card.", + "EventCode": "0xc1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.PEER_WRITE.PART1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0002", + "PublicDescription": "Number Transactions requested by the CPU : Another card (different IIO stack) writing to this card. : Also known as Outbound. Number of requests initiated by the main die, including reads and writes. : x4 card is plugged in to slot 1", + "UMask": "0x7002002", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested by the CPU : Another card (different IIO stack) writing to this card.", + "EventCode": "0xc1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.PEER_WRITE.PART2", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0004", + "PublicDescription": "Number Transactions requested by the CPU : Another card (different IIO stack) writing to this card. : Also known as Outbound. Number of requests initiated by the main die, including reads and writes. : x8 card plugged in to Lane 2/3, Or x4 card is plugged in to slot 2", + "UMask": "0x7004002", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested by the CPU : Another card (different IIO stack) writing to this card.", + "EventCode": "0xc1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.PEER_WRITE.PART3", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0008", + "PublicDescription": "Number Transactions requested by the CPU : Another card (different IIO stack) writing to this card. : Also known as Outbound. Number of requests initiated by the main die, including reads and writes. : x4 card is plugged in to slot 3", + "UMask": "0x7008002", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested by the CPU : Another card (different IIO stack) writing to this card.", + "EventCode": "0xc1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.PEER_WRITE.PART4", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0010", + "PublicDescription": "Number Transactions requested by the CPU : Another card (different IIO stack) writing to this card. : Also known as Outbound. Number of requests initiated by the main die, including reads and writes. : x16 card plugged in to Lane 4/5/6/7, Or x8 card plugged in to Lane 4/5, Or x4 card is plugged in to slot 4", + "UMask": "0x7010002", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested by the CPU : Another card (different IIO stack) writing to this card.", + "EventCode": "0xc1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.PEER_WRITE.PART5", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0020", + "PublicDescription": "Number Transactions requested by the CPU : Another card (different IIO stack) writing to this card. : Also known as Outbound. Number of requests initiated by the main die, including reads and writes. : x4 card is plugged in to slot 5", + "UMask": "0x7020002", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested by the CPU : Another card (different IIO stack) writing to this card.", + "EventCode": "0xc1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.PEER_WRITE.PART6", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0040", + "PublicDescription": "Number Transactions requested by the CPU : Another card (different IIO stack) writing to this card. : Also known as Outbound. Number of requests initiated by the main die, including reads and writes. : x8 card plugged in to Lane 6/7, Or x4 card is plugged in to slot 6", + "UMask": "0x7040002", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested by the CPU : Another card (different IIO stack) writing to this card.", + "EventCode": "0xc1", + "EventName": "UNC_IIO_TXN_REQ_BY_CPU.PEER_WRITE.PART7", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0080", + "PublicDescription": "Number Transactions requested by the CPU : Another card (different IIO stack) writing to this card. : Also known as Outbound. Number of requests initiated by the main die, including reads and writes. : x4 card is plugged in to slot 7", + "UMask": "0x7080002", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : CmpD - device sending completion to CPU request", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.CMPD.PART0", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0001", + "PublicDescription": "Number Transactions requested of the CPU : CmpD - device sending completion to CPU request : Also known as Inbound. Number of 64B cache line requests initiated by the Card, including reads and writes. : x16 card plugged in to Lane 0/1/2/3, Or x8 card plugged in to Lane 0/1, Or x4 card is plugged in to slot 0", + "UMask": "0x80", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : CmpD - device sending completion to CPU request", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.CMPD.PART1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0002", + "PublicDescription": "Number Transactions requested of the CPU : CmpD - device sending completion to CPU request : Also known as Inbound. Number of 64B cache line requests initiated by the Card, including reads and writes. : x4 card is plugged in to slot 1", + "UMask": "0x80", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : CmpD - device sending completion to CPU request", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.CMPD.PART2", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0004", + "PublicDescription": "Number Transactions requested of the CPU : CmpD - device sending completion to CPU request : Also known as Inbound. Number of 64B cache line requests initiated by the Card, including reads and writes. : x8 card plugged in to Lane 2/3, Or x4 card is plugged in to slot 2", + "UMask": "0x80", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : CmpD - device sending completion to CPU request", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.CMPD.PART3", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0008", + "PublicDescription": "Number Transactions requested of the CPU : CmpD - device sending completion to CPU request : Also known as Inbound. Number of 64B cache line requests initiated by the Card, including reads and writes. : x4 card is plugged in to slot 3", + "UMask": "0x80", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : CmpD - device sending completion to CPU request", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.CMPD.PART4", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0010", + "PublicDescription": "Number Transactions requested of the CPU : CmpD - device sending completion to CPU request : Also known as Inbound. Number of 64B cache line requests initiated by the Card, including reads and writes. : x16 card plugged in to Lane 4/5/6/7, Or x8 card plugged in to Lane 4/5, Or x4 card is plugged in to slot 4", + "UMask": "0x80", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : CmpD - device sending completion to CPU request", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.CMPD.PART5", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0020", + "PublicDescription": "Number Transactions requested of the CPU : CmpD - device sending completion to CPU request : Also known as Inbound. Number of 64B cache line requests initiated by the Card, including reads and writes. : x4 card is plugged in to slot 5", + "UMask": "0x80", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : CmpD - device sending completion to CPU request", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.CMPD.PART6", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0040", + "PublicDescription": "Number Transactions requested of the CPU : CmpD - device sending completion to CPU request : Also known as Inbound. Number of 64B cache line requests initiated by the Card, including reads and writes. : x8 card plugged in to Lane 6/7, Or x4 card is plugged in to slot 6", + "UMask": "0x80", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : CmpD - device sending completion to CPU request", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.CMPD.PART7", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0080", + "PublicDescription": "Number Transactions requested of the CPU : CmpD - device sending completion to CPU request : Also known as Inbound. Number of 64B cache line requests initiated by the Card, including reads and writes. : x4 card is plugged in to slot 7", + "UMask": "0x80", + "Unit": "IIO" + }, + { + "BriefDescription": "Read request for up to a 64 byte transaction is made by IIO Part0 to Memory", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.MEM_READ.PART0", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0001", + "PublicDescription": "Number Transactions requested of the CPU : Card reading from DRAM : Also known as Inbound. Number of 64B cache line requests initiated by the Card, including reads and writes. : x16 card plugged in to stack, Or x8 card plugged in to Lane 0/1, Or x4 card is plugged in to slot 0", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Read request for up to a 64 byte transaction is made by IIO Part1 to Memory", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.MEM_READ.PART1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0002", + "PublicDescription": "Number Transactions requested of the CPU : Card reading from DRAM : Also known as Inbound. Number of 64B cache line requests initiated by the Card, including reads and writes. : x4 card is plugged in to slot 1", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Read request for up to a 64 byte transaction is made by IIO Part2 to Memory", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.MEM_READ.PART2", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0004", + "PublicDescription": "Number Transactions requested of the CPU : Card reading from DRAM : Also known as Inbound. Number of 64B cache line requests initiated by the Card, including reads and writes. : x8 card plugged in to Lane 2/3, Or x4 card is plugged in to slot 1", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Read request for up to a 64 byte transaction is made by IIO Part3 to Memory", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.MEM_READ.PART3", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0008", + "PublicDescription": "Number Transactions requested of the CPU : Card reading from DRAM : Also known as Inbound. Number of 64B cache line requests initiated by the Card, including reads and writes. : x4 card is plugged in to slot 3", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card reading from DRAM", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.MEM_READ.PART4", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0010", + "PublicDescription": "Number Transactions requested of the CPU : Card reading from DRAM : Also known as Inbound. Number of 64B cache line requests initiated by the Card, including reads and writes. : x16 card plugged in to stack, Or x8 card plugged in to Lane 0/1, Or x4 card is plugged in to slot 0", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card reading from DRAM", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.MEM_READ.PART5", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0020", + "PublicDescription": "Number Transactions requested of the CPU : Card reading from DRAM : Also known as Inbound. Number of 64B cache line requests initiated by the Card, including reads and writes. : x4 card is plugged in to slot 1", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card reading from DRAM", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.MEM_READ.PART6", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0040", + "PublicDescription": "Number Transactions requested of the CPU : Card reading from DRAM : Also known as Inbound. Number of 64B cache line requests initiated by the Card, including reads and writes. : x8 card plugged in to Lane 2/3, Or x4 card is plugged in to slot 1", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card reading from DRAM", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.MEM_READ.PART7", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0080", + "PublicDescription": "Number Transactions requested of the CPU : Card reading from DRAM : Also known as Inbound. Number of 64B cache line requests initiated by the Card, including reads and writes. : x4 card is plugged in to slot 3", + "UMask": "0x4", + "Unit": "IIO" + }, + { + "BriefDescription": "Write request of up to a 64 byte transaction is made by IIO Part0 to Memory", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.MEM_WRITE.PART0", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0001", + "PublicDescription": "Number Transactions requested of the CPU : Card writing to DRAM : Also known as Inbound. Number of 64B cache line requests initiated by the Card, including reads and writes. : x16 card plugged in to stack, Or x8 card plugged in to Lane 0/1, Or x4 card is plugged in to slot 0", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Write request of up to a 64 byte transaction is made by IIO Part1 to Memory", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.MEM_WRITE.PART1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0002", + "PublicDescription": "Number Transactions requested of the CPU : Card writing to DRAM : Also known as Inbound. Number of 64B cache line requests initiated by the Card, including reads and writes. : x4 card is plugged in to slot 1", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Write request of up to a 64 byte transaction is made by IIO Part2 to Memory", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.MEM_WRITE.PART2", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0004", + "PublicDescription": "Number Transactions requested of the CPU : Card writing to DRAM : Also known as Inbound. Number of 64B cache line requests initiated by the Card, including reads and writes. : x8 card plugged in to Lane 2/3, Or x4 card is plugged in to slot 1", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Write request of up to a 64 byte transaction is made by IIO Part3 to Memory", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.MEM_WRITE.PART3", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0008", + "PublicDescription": "Number Transactions requested of the CPU : Card writing to DRAM : Also known as Inbound. Number of 64B cache line requests initiated by the Card, including reads and writes. : x4 card is plugged in to slot 3", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card writing to DRAM", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.MEM_WRITE.PART4", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0010", + "PublicDescription": "Number Transactions requested of the CPU : Card writing to DRAM : Also known as Inbound. Number of 64B cache line requests initiated by the Card, including reads and writes. : x16 card plugged in to stack, Or x8 card plugged in to Lane 0/1, Or x4 card is plugged in to slot 0", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card writing to DRAM", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.MEM_WRITE.PART5", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0020", + "PublicDescription": "Number Transactions requested of the CPU : Card writing to DRAM : Also known as Inbound. Number of 64B cache line requests initiated by the Card, including reads and writes. : x4 card is plugged in to slot 1", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card writing to DRAM", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.MEM_WRITE.PART6", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0040", + "PublicDescription": "Number Transactions requested of the CPU : Card writing to DRAM : Also known as Inbound. Number of 64B cache line requests initiated by the Card, including reads and writes. : x8 card plugged in to Lane 2/3, Or x4 card is plugged in to slot 1", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card writing to DRAM", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.MEM_WRITE.PART7", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0080", + "PublicDescription": "Number Transactions requested of the CPU : Card writing to DRAM : Also known as Inbound. Number of 64B cache line requests initiated by the Card, including reads and writes. : x4 card is plugged in to slot 3", + "UMask": "0x1", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card writing to another Card (same or different stack)", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.PEER_WRITE.PART0", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0001", + "PublicDescription": "Number Transactions requested of the CPU : Card writing to another Card (same or different stack) : Also known as Inbound. Number of 64B cache line requests initiated by the Card, including reads and writes. : x16 card plugged in to Lane 0/1/2/3, Or x8 card plugged in to Lane 0/1, Or x4 card is plugged in to slot 0", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card writing to another Card (same or different stack)", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.PEER_WRITE.PART1", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0002", + "PublicDescription": "Number Transactions requested of the CPU : Card writing to another Card (same or different stack) : Also known as Inbound. Number of 64B cache line requests initiated by the Card, including reads and writes. : x4 card is plugged in to slot 1", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card writing to another Card (same or different stack)", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.PEER_WRITE.PART2", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0004", + "PublicDescription": "Number Transactions requested of the CPU : Card writing to another Card (same or different stack) : Also known as Inbound. Number of 64B cache line requests initiated by the Card, including reads and writes. : x8 card plugged in to Lane 2/3, Or x4 card is plugged in to slot 2", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card writing to another Card (same or different stack)", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.PEER_WRITE.PART3", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0008", + "PublicDescription": "Number Transactions requested of the CPU : Card writing to another Card (same or different stack) : Also known as Inbound. Number of 64B cache line requests initiated by the Card, including reads and writes. : x4 card is plugged in to slot 3", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card writing to another Card (same or different stack)", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.PEER_WRITE.PART4", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0010", + "PublicDescription": "Number Transactions requested of the CPU : Card writing to another Card (same or different stack) : Also known as Inbound. Number of 64B cache line requests initiated by the Card, including reads and writes. : x16 card plugged in to Lane 4/5/6/7, Or x8 card plugged in to Lane 4/5, Or x4 card is plugged in to slot 4", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card writing to another Card (same or different stack)", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.PEER_WRITE.PART5", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0020", + "PublicDescription": "Number Transactions requested of the CPU : Card writing to another Card (same or different stack) : Also known as Inbound. Number of 64B cache line requests initiated by the Card, including reads and writes. : x4 card is plugged in to slot 5", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card writing to another Card (same or different stack)", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.PEER_WRITE.PART6", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0040", + "PublicDescription": "Number Transactions requested of the CPU : Card writing to another Card (same or different stack) : Also known as Inbound. Number of 64B cache line requests initiated by the Card, including reads and writes. : x8 card plugged in to Lane 6/7, Or x4 card is plugged in to slot 6", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "Number Transactions requested of the CPU : Card writing to another Card (same or different stack)", + "EventCode": "0x84", + "EventName": "UNC_IIO_TXN_REQ_OF_CPU.PEER_WRITE.PART7", + "FCMask": "0x07", + "PerPkg": "1", + "PortMask": "0x0080", + "PublicDescription": "Number Transactions requested of the CPU : Card writing to another Card (same or different stack) : Also known as Inbound. Number of 64B cache line requests initiated by the Card, including reads and writes. : x4 card is plugged in to slot 7", + "UMask": "0x2", + "Unit": "IIO" + }, + { + "BriefDescription": "M2P Clockticks", + "EventCode": "0x01", + "EventName": "UNC_M2P_CLOCKTICKS", + "PerPkg": "1", + "PublicDescription": "Number of M2P clock cycles while the event is enabled", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "CMS Clockticks", + "EventCode": "0xc0", + "EventName": "UNC_M2P_CMS_CLOCKTICKS", + "PerPkg": "1", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Egress Blocking due to Ordering requirements : Down", + "EventCode": "0xba", + "EventName": "UNC_M2P_EGRESS_ORDERING.IV_SNOOPGO_DN", + "PerPkg": "1", + "PublicDescription": "Egress Blocking due to Ordering requirements : Down : Counts number of cycles IV was blocked in the TGR Egress due to SNP/GO Ordering requirements", + "UMask": "0x4", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Egress Blocking due to Ordering requirements : Up", + "EventCode": "0xba", + "EventName": "UNC_M2P_EGRESS_ORDERING.IV_SNOOPGO_UP", + "PerPkg": "1", + "PublicDescription": "Egress Blocking due to Ordering requirements : Up : Counts number of cycles IV was blocked in the TGR Egress due to SNP/GO Ordering requirements", + "UMask": "0x1", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "M2PCIe IIO Credit Acquired : DRS", + "EventCode": "0x33", + "EventName": "UNC_M2P_IIO_CREDITS_ACQUIRED.DRS_0", + "PerPkg": "1", + "PublicDescription": "M2PCIe IIO Credit Acquired : DRS : Counts the number of credits that are acquired in the M2PCIe agent for sending transactions into the IIO on either NCB or NCS are in use. Transactions from the BL ring going into the IIO Agent must first acquire a credit. These credits are for either the NCB or NCS message classes. NCB, or non-coherent bypass messages are used to transmit data without coherency (and are common). NCS is used for reads to PCIe (and should be used sparingly). : Credits for transfer through CMS Port 0 to the IIO for the DRS message class.", + "UMask": "0x1", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "M2PCIe IIO Credit Acquired : DRS", + "EventCode": "0x33", + "EventName": "UNC_M2P_IIO_CREDITS_ACQUIRED.DRS_1", + "PerPkg": "1", + "PublicDescription": "M2PCIe IIO Credit Acquired : DRS : Counts the number of credits that are acquired in the M2PCIe agent for sending transactions into the IIO on either NCB or NCS are in use. Transactions from the BL ring going into the IIO Agent must first acquire a credit. These credits are for either the NCB or NCS message classes. NCB, or non-coherent bypass messages are used to transmit data without coherency (and are common). NCS is used for reads to PCIe (and should be used sparingly). : Credits for transfer through CMS Port 0 to the IIO for the DRS message class.", + "UMask": "0x2", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "M2PCIe IIO Credit Acquired : NCB", + "EventCode": "0x33", + "EventName": "UNC_M2P_IIO_CREDITS_ACQUIRED.NCB_0", + "PerPkg": "1", + "PublicDescription": "M2PCIe IIO Credit Acquired : NCB : Counts the number of credits that are acquired in the M2PCIe agent for sending transactions into the IIO on either NCB or NCS are in use. Transactions from the BL ring going into the IIO Agent must first acquire a credit. These credits are for either the NCB or NCS message classes. NCB, or non-coherent bypass messages are used to transmit data without coherency (and are common). NCS is used for reads to PCIe (and should be used sparingly). : Credits for transfer through CMS Port 0 to the IIO for the NCB message class.", + "UMask": "0x4", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "M2PCIe IIO Credit Acquired : NCB", + "EventCode": "0x33", + "EventName": "UNC_M2P_IIO_CREDITS_ACQUIRED.NCB_1", + "PerPkg": "1", + "PublicDescription": "M2PCIe IIO Credit Acquired : NCB : Counts the number of credits that are acquired in the M2PCIe agent for sending transactions into the IIO on either NCB or NCS are in use. Transactions from the BL ring going into the IIO Agent must first acquire a credit. These credits are for either the NCB or NCS message classes. NCB, or non-coherent bypass messages are used to transmit data without coherency (and are common). NCS is used for reads to PCIe (and should be used sparingly). : Credits for transfer through CMS Port 0 to the IIO for the NCB message class.", + "UMask": "0x8", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "M2PCIe IIO Credit Acquired : NCS", + "EventCode": "0x33", + "EventName": "UNC_M2P_IIO_CREDITS_ACQUIRED.NCS_0", + "PerPkg": "1", + "PublicDescription": "M2PCIe IIO Credit Acquired : NCS : Counts the number of credits that are acquired in the M2PCIe agent for sending transactions into the IIO on either NCB or NCS are in use. Transactions from the BL ring going into the IIO Agent must first acquire a credit. These credits are for either the NCB or NCS message classes. NCB, or non-coherent bypass messages are used to transmit data without coherency (and are common). NCS is used for reads to PCIe (and should be used sparingly). : Credits for transfer through CMS Port 0 to the IIO for the NCS message class.", + "UMask": "0x10", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "M2PCIe IIO Credit Acquired : NCS", + "EventCode": "0x33", + "EventName": "UNC_M2P_IIO_CREDITS_ACQUIRED.NCS_1", + "PerPkg": "1", + "PublicDescription": "M2PCIe IIO Credit Acquired : NCS : Counts the number of credits that are acquired in the M2PCIe agent for sending transactions into the IIO on either NCB or NCS are in use. Transactions from the BL ring going into the IIO Agent must first acquire a credit. These credits are for either the NCB or NCS message classes. NCB, or non-coherent bypass messages are used to transmit data without coherency (and are common). NCS is used for reads to PCIe (and should be used sparingly). : Credit for transfer through CMS Port 0s to the IIO for the NCS message class.", + "UMask": "0x20", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "M2PCIe IIO Failed to Acquire a Credit : DRS", + "EventCode": "0x34", + "EventName": "UNC_M2P_IIO_CREDITS_REJECT.DRS", + "PerPkg": "1", + "PublicDescription": "M2PCIe IIO Failed to Acquire a Credit : DRS : Counts the number of times that a request pending in the BL Ingress attempted to acquire either a NCB or NCS credit to transmit into the IIO, but was rejected because no credits were available. NCB, or non-coherent bypass messages are used to transmit data without coherency (and are common). NCS is used for reads to PCIe (and should be used sparingly). : Credits to the IIO for the DRS message class.", + "UMask": "0x8", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "M2PCIe IIO Failed to Acquire a Credit : NCB", + "EventCode": "0x34", + "EventName": "UNC_M2P_IIO_CREDITS_REJECT.NCB", + "PerPkg": "1", + "PublicDescription": "M2PCIe IIO Failed to Acquire a Credit : NCB : Counts the number of times that a request pending in the BL Ingress attempted to acquire either a NCB or NCS credit to transmit into the IIO, but was rejected because no credits were available. NCB, or non-coherent bypass messages are used to transmit data without coherency (and are common). NCS is used for reads to PCIe (and should be used sparingly). : Credits to the IIO for the NCB message class.", + "UMask": "0x10", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "M2PCIe IIO Failed to Acquire a Credit : NCS", + "EventCode": "0x34", + "EventName": "UNC_M2P_IIO_CREDITS_REJECT.NCS", + "PerPkg": "1", + "PublicDescription": "M2PCIe IIO Failed to Acquire a Credit : NCS : Counts the number of times that a request pending in the BL Ingress attempted to acquire either a NCB or NCS credit to transmit into the IIO, but was rejected because no credits were available. NCB, or non-coherent bypass messages are used to transmit data without coherency (and are common). NCS is used for reads to PCIe (and should be used sparingly). : Credits to the IIO for the NCS message class.", + "UMask": "0x20", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "M2PCIe IIO Credits in Use : DRS to CMS Port 0", + "EventCode": "0x32", + "EventName": "UNC_M2P_IIO_CREDITS_USED.DRS_0", + "PerPkg": "1", + "PublicDescription": "M2PCIe IIO Credits in Use : DRS to CMS Port 0 : Counts the number of cycles when one or more credits in the M2PCIe agent for sending transactions into the IIO on either NCB or NCS are in use. Transactions from the BL ring going into the IIO Agent must first acquire a credit. These credits are for either the NCB or NCS message classes. NCB, or non-coherent bypass messages are used to transmit data without coherency (and are common). NCS is used for reads to PCIe (and should be used sparingly). : Credits for transfer through CMS Port 0 to the IIO for the DRS message class.", + "UMask": "0x1", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "M2PCIe IIO Credits in Use : DRS to CMS Port 1", + "EventCode": "0x32", + "EventName": "UNC_M2P_IIO_CREDITS_USED.DRS_1", + "PerPkg": "1", + "PublicDescription": "M2PCIe IIO Credits in Use : DRS to CMS Port 1 : Counts the number of cycles when one or more credits in the M2PCIe agent for sending transactions into the IIO on either NCB or NCS are in use. Transactions from the BL ring going into the IIO Agent must first acquire a credit. These credits are for either the NCB or NCS message classes. NCB, or non-coherent bypass messages are used to transmit data without coherency (and are common). NCS is used for reads to PCIe (and should be used sparingly). : Credits for transfer through CMS Port 0 to the IIO for the DRS message class.", + "UMask": "0x2", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "M2PCIe IIO Credits in Use : NCB to CMS Port 0", + "EventCode": "0x32", + "EventName": "UNC_M2P_IIO_CREDITS_USED.NCB_0", + "PerPkg": "1", + "PublicDescription": "M2PCIe IIO Credits in Use : NCB to CMS Port 0 : Counts the number of cycles when one or more credits in the M2PCIe agent for sending transactions into the IIO on either NCB or NCS are in use. Transactions from the BL ring going into the IIO Agent must first acquire a credit. These credits are for either the NCB or NCS message classes. NCB, or non-coherent bypass messages are used to transmit data without coherency (and are common). NCS is used for reads to PCIe (and should be used sparingly). : Credits for transfer through CMS Port 0 to the IIO for the NCB message class.", + "UMask": "0x4", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "M2PCIe IIO Credits in Use : NCB to CMS Port 1", + "EventCode": "0x32", + "EventName": "UNC_M2P_IIO_CREDITS_USED.NCB_1", + "PerPkg": "1", + "PublicDescription": "M2PCIe IIO Credits in Use : NCB to CMS Port 1 : Counts the number of cycles when one or more credits in the M2PCIe agent for sending transactions into the IIO on either NCB or NCS are in use. Transactions from the BL ring going into the IIO Agent must first acquire a credit. These credits are for either the NCB or NCS message classes. NCB, or non-coherent bypass messages are used to transmit data without coherency (and are common). NCS is used for reads to PCIe (and should be used sparingly). : Credits for transfer through CMS Port 0 to the IIO for the NCB message class.", + "UMask": "0x8", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "M2PCIe IIO Credits in Use : NCS to CMS Port 0", + "EventCode": "0x32", + "EventName": "UNC_M2P_IIO_CREDITS_USED.NCS_0", + "PerPkg": "1", + "PublicDescription": "M2PCIe IIO Credits in Use : NCS to CMS Port 0 : Counts the number of cycles when one or more credits in the M2PCIe agent for sending transactions into the IIO on either NCB or NCS are in use. Transactions from the BL ring going into the IIO Agent must first acquire a credit. These credits are for either the NCB or NCS message classes. NCB, or non-coherent bypass messages are used to transmit data without coherency (and are common). NCS is used for reads to PCIe (and should be used sparingly). : Credits for transfer through CMS Port 0 to the IIO for the NCS message class.", + "UMask": "0x10", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "M2PCIe IIO Credits in Use : NCS to CMS Port 1", + "EventCode": "0x32", + "EventName": "UNC_M2P_IIO_CREDITS_USED.NCS_1", + "PerPkg": "1", + "PublicDescription": "M2PCIe IIO Credits in Use : NCS to CMS Port 1 : Counts the number of cycles when one or more credits in the M2PCIe agent for sending transactions into the IIO on either NCB or NCS are in use. Transactions from the BL ring going into the IIO Agent must first acquire a credit. These credits are for either the NCB or NCS message classes. NCB, or non-coherent bypass messages are used to transmit data without coherency (and are common). NCS is used for reads to PCIe (and should be used sparingly). : Credit for transfer through CMS Port 0s to the IIO for the NCS message class.", + "UMask": "0x20", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local Dedicated P2P Credit Taken - 0 : M2IOSF0 - NCB", + "EventCode": "0x46", + "EventName": "UNC_M2P_LOCAL_DED_P2P_CRD_TAKEN_0.M2IOSF0_NCB", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local Dedicated P2P Credit Taken - 0 : M2IOSF0 - NCS", + "EventCode": "0x46", + "EventName": "UNC_M2P_LOCAL_DED_P2P_CRD_TAKEN_0.M2IOSF0_NCS", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local Dedicated P2P Credit Taken - 0 : M2IOSF1 - NCB", + "EventCode": "0x46", + "EventName": "UNC_M2P_LOCAL_DED_P2P_CRD_TAKEN_0.M2IOSF1_NCB", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local Dedicated P2P Credit Taken - 0 : M2IOSF1 - NCS", + "EventCode": "0x46", + "EventName": "UNC_M2P_LOCAL_DED_P2P_CRD_TAKEN_0.M2IOSF1_NCS", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local Dedicated P2P Credit Taken - 0 : M2IOSF2 - NCB", + "EventCode": "0x46", + "EventName": "UNC_M2P_LOCAL_DED_P2P_CRD_TAKEN_0.M2IOSF2_NCB", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local Dedicated P2P Credit Taken - 0 : M2IOSF2 - NCS", + "EventCode": "0x46", + "EventName": "UNC_M2P_LOCAL_DED_P2P_CRD_TAKEN_0.M2IOSF2_NCS", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local Dedicated P2P Credit Taken - 0 : M2IOSF3 - NCB", + "EventCode": "0x46", + "EventName": "UNC_M2P_LOCAL_DED_P2P_CRD_TAKEN_0.M2IOSF3_NCB", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local Dedicated P2P Credit Taken - 0 : M2IOSF3 - NCS", + "EventCode": "0x46", + "EventName": "UNC_M2P_LOCAL_DED_P2P_CRD_TAKEN_0.M2IOSF3_NCS", + "PerPkg": "1", + "UMask": "0x80", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local Dedicated P2P Credit Taken - 1 : M2IOSF4 - NCB", + "EventCode": "0x47", + "EventName": "UNC_M2P_LOCAL_DED_P2P_CRD_TAKEN_1.M2IOSF4_NCB", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local Dedicated P2P Credit Taken - 1 : M2IOSF4 - NCS", + "EventCode": "0x47", + "EventName": "UNC_M2P_LOCAL_DED_P2P_CRD_TAKEN_1.M2IOSF4_NCS", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local Dedicated P2P Credit Taken - 1 : M2IOSF5 - NCB", + "EventCode": "0x47", + "EventName": "UNC_M2P_LOCAL_DED_P2P_CRD_TAKEN_1.M2IOSF5_NCB", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local Dedicated P2P Credit Taken - 1 : M2IOSF5 - NCS", + "EventCode": "0x47", + "EventName": "UNC_M2P_LOCAL_DED_P2P_CRD_TAKEN_1.M2IOSF5_NCS", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local P2P Dedicated Credits Returned - 0 : M2IOSF0 - NCB", + "EventCode": "0x19", + "EventName": "UNC_M2P_LOCAL_P2P_DED_RETURNED_0.MS2IOSF0_NCB", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local P2P Dedicated Credits Returned - 0 : M2IOSF0 - NCS", + "EventCode": "0x19", + "EventName": "UNC_M2P_LOCAL_P2P_DED_RETURNED_0.MS2IOSF0_NCS", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local P2P Dedicated Credits Returned - 0 : M2IOSF1 - NCB", + "EventCode": "0x19", + "EventName": "UNC_M2P_LOCAL_P2P_DED_RETURNED_0.MS2IOSF1_NCB", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local P2P Dedicated Credits Returned - 0 : M2IOSF1 - NCS", + "EventCode": "0x19", + "EventName": "UNC_M2P_LOCAL_P2P_DED_RETURNED_0.MS2IOSF1_NCS", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local P2P Dedicated Credits Returned - 0 : M2IOSF2 - NCB", + "EventCode": "0x19", + "EventName": "UNC_M2P_LOCAL_P2P_DED_RETURNED_0.MS2IOSF2_NCB", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local P2P Dedicated Credits Returned - 0 : M2IOSF2 - NCS", + "EventCode": "0x19", + "EventName": "UNC_M2P_LOCAL_P2P_DED_RETURNED_0.MS2IOSF2_NCS", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local P2P Dedicated Credits Returned - 0 : M2IOSF3 - NCB", + "EventCode": "0x19", + "EventName": "UNC_M2P_LOCAL_P2P_DED_RETURNED_0.MS2IOSF3_NCB", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local P2P Dedicated Credits Returned - 0 : M2IOSF3 - NCS", + "EventCode": "0x19", + "EventName": "UNC_M2P_LOCAL_P2P_DED_RETURNED_0.MS2IOSF3_NCS", + "PerPkg": "1", + "UMask": "0x80", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local P2P Dedicated Credits Returned - 1 : M2IOSF4 - NCB", + "EventCode": "0x1a", + "EventName": "UNC_M2P_LOCAL_P2P_DED_RETURNED_1.MS2IOSF4_NCB", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local P2P Dedicated Credits Returned - 1 : M2IOSF4 - NCS", + "EventCode": "0x1a", + "EventName": "UNC_M2P_LOCAL_P2P_DED_RETURNED_1.MS2IOSF4_NCS", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local P2P Dedicated Credits Returned - 1 : M2IOSF5 - NCB", + "EventCode": "0x1a", + "EventName": "UNC_M2P_LOCAL_P2P_DED_RETURNED_1.MS2IOSF5_NCB", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local P2P Dedicated Credits Returned - 1 : M2IOSF5 - NCS", + "EventCode": "0x1a", + "EventName": "UNC_M2P_LOCAL_P2P_DED_RETURNED_1.MS2IOSF5_NCS", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local P2P Shared Credits Returned : Agent0", + "EventCode": "0x17", + "EventName": "UNC_M2P_LOCAL_P2P_SHAR_RETURNED.AGENT_0", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local P2P Shared Credits Returned : Agent1", + "EventCode": "0x17", + "EventName": "UNC_M2P_LOCAL_P2P_SHAR_RETURNED.AGENT_1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local P2P Shared Credits Returned : Agent2", + "EventCode": "0x17", + "EventName": "UNC_M2P_LOCAL_P2P_SHAR_RETURNED.AGENT_2", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local Shared P2P Credit Returned to credit ring : Agent0", + "EventCode": "0x44", + "EventName": "UNC_M2P_LOCAL_SHAR_P2P_CRD_RETURNED.AGENT_0", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local Shared P2P Credit Returned to credit ring : Agent1", + "EventCode": "0x44", + "EventName": "UNC_M2P_LOCAL_SHAR_P2P_CRD_RETURNED.AGENT_1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local Shared P2P Credit Returned to credit ring : Agent2", + "EventCode": "0x44", + "EventName": "UNC_M2P_LOCAL_SHAR_P2P_CRD_RETURNED.AGENT_2", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local Shared P2P Credit Returned to credit ring : Agent3", + "EventCode": "0x44", + "EventName": "UNC_M2P_LOCAL_SHAR_P2P_CRD_RETURNED.AGENT_3", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local Shared P2P Credit Returned to credit ring : Agent4", + "EventCode": "0x44", + "EventName": "UNC_M2P_LOCAL_SHAR_P2P_CRD_RETURNED.AGENT_4", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local Shared P2P Credit Returned to credit ring : Agent5", + "EventCode": "0x44", + "EventName": "UNC_M2P_LOCAL_SHAR_P2P_CRD_RETURNED.AGENT_5", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local Shared P2P Credit Taken - 0 : M2IOSF0 - NCB", + "EventCode": "0x40", + "EventName": "UNC_M2P_LOCAL_SHAR_P2P_CRD_TAKEN_0.M2IOSF0_NCB", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local Shared P2P Credit Taken - 0 : M2IOSF0 - NCS", + "EventCode": "0x40", + "EventName": "UNC_M2P_LOCAL_SHAR_P2P_CRD_TAKEN_0.M2IOSF0_NCS", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local Shared P2P Credit Taken - 0 : M2IOSF1 - NCB", + "EventCode": "0x40", + "EventName": "UNC_M2P_LOCAL_SHAR_P2P_CRD_TAKEN_0.M2IOSF1_NCB", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local Shared P2P Credit Taken - 0 : M2IOSF1 - NCS", + "EventCode": "0x40", + "EventName": "UNC_M2P_LOCAL_SHAR_P2P_CRD_TAKEN_0.M2IOSF1_NCS", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local Shared P2P Credit Taken - 0 : M2IOSF2 - NCB", + "EventCode": "0x40", + "EventName": "UNC_M2P_LOCAL_SHAR_P2P_CRD_TAKEN_0.M2IOSF2_NCB", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local Shared P2P Credit Taken - 0 : M2IOSF2 - NCS", + "EventCode": "0x40", + "EventName": "UNC_M2P_LOCAL_SHAR_P2P_CRD_TAKEN_0.M2IOSF2_NCS", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local Shared P2P Credit Taken - 0 : M2IOSF3 - NCB", + "EventCode": "0x40", + "EventName": "UNC_M2P_LOCAL_SHAR_P2P_CRD_TAKEN_0.M2IOSF3_NCB", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local Shared P2P Credit Taken - 0 : M2IOSF3 - NCS", + "EventCode": "0x40", + "EventName": "UNC_M2P_LOCAL_SHAR_P2P_CRD_TAKEN_0.M2IOSF3_NCS", + "PerPkg": "1", + "UMask": "0x80", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local Shared P2P Credit Taken - 1 : M2IOSF4 - NCB", + "EventCode": "0x41", + "EventName": "UNC_M2P_LOCAL_SHAR_P2P_CRD_TAKEN_1.M2IOSF4_NCB", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local Shared P2P Credit Taken - 1 : M2IOSF4 - NCS", + "EventCode": "0x41", + "EventName": "UNC_M2P_LOCAL_SHAR_P2P_CRD_TAKEN_1.M2IOSF4_NCS", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local Shared P2P Credit Taken - 1 : M2IOSF5 - NCB", + "EventCode": "0x41", + "EventName": "UNC_M2P_LOCAL_SHAR_P2P_CRD_TAKEN_1.M2IOSF5_NCB", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Local Shared P2P Credit Taken - 1 : M2IOSF5 - NCS", + "EventCode": "0x41", + "EventName": "UNC_M2P_LOCAL_SHAR_P2P_CRD_TAKEN_1.M2IOSF5_NCS", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Waiting on Local Shared P2P Credit - 0 : M2IOSF0 - NCB", + "EventCode": "0x4a", + "EventName": "UNC_M2P_LOCAL_SHAR_P2P_CRD_WAIT_0.M2IOSF0_NCB", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Waiting on Local Shared P2P Credit - 0 : M2IOSF0 - NCS", + "EventCode": "0x4a", + "EventName": "UNC_M2P_LOCAL_SHAR_P2P_CRD_WAIT_0.M2IOSF0_NCS", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Waiting on Local Shared P2P Credit - 0 : M2IOSF1 - NCB", + "EventCode": "0x4a", + "EventName": "UNC_M2P_LOCAL_SHAR_P2P_CRD_WAIT_0.M2IOSF1_NCB", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Waiting on Local Shared P2P Credit - 0 : M2IOSF1 - NCS", + "EventCode": "0x4a", + "EventName": "UNC_M2P_LOCAL_SHAR_P2P_CRD_WAIT_0.M2IOSF1_NCS", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Waiting on Local Shared P2P Credit - 0 : M2IOSF2 - NCB", + "EventCode": "0x4a", + "EventName": "UNC_M2P_LOCAL_SHAR_P2P_CRD_WAIT_0.M2IOSF2_NCB", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Waiting on Local Shared P2P Credit - 0 : M2IOSF2 - NCS", + "EventCode": "0x4a", + "EventName": "UNC_M2P_LOCAL_SHAR_P2P_CRD_WAIT_0.M2IOSF2_NCS", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Waiting on Local Shared P2P Credit - 0 : M2IOSF3 - NCB", + "EventCode": "0x4a", + "EventName": "UNC_M2P_LOCAL_SHAR_P2P_CRD_WAIT_0.M2IOSF3_NCB", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Waiting on Local Shared P2P Credit - 0 : M2IOSF3 - NCS", + "EventCode": "0x4a", + "EventName": "UNC_M2P_LOCAL_SHAR_P2P_CRD_WAIT_0.M2IOSF3_NCS", + "PerPkg": "1", + "UMask": "0x80", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Waiting on Local Shared P2P Credit - 1 : M2IOSF4 - NCB", + "EventCode": "0x4b", + "EventName": "UNC_M2P_LOCAL_SHAR_P2P_CRD_WAIT_1.M2IOSF4_NCB", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Waiting on Local Shared P2P Credit - 1 : M2IOSF4 - NCS", + "EventCode": "0x4b", + "EventName": "UNC_M2P_LOCAL_SHAR_P2P_CRD_WAIT_1.M2IOSF4_NCS", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Waiting on Local Shared P2P Credit - 1 : M2IOSF5 - NCB", + "EventCode": "0x4b", + "EventName": "UNC_M2P_LOCAL_SHAR_P2P_CRD_WAIT_1.M2IOSF5_NCB", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Waiting on Local Shared P2P Credit - 1 : M2IOSF5 - NCS", + "EventCode": "0x4b", + "EventName": "UNC_M2P_LOCAL_SHAR_P2P_CRD_WAIT_1.M2IOSF5_NCS", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "P2P Credit Occupancy : All", + "EventCode": "0x14", + "EventName": "UNC_M2P_P2P_CRD_OCCUPANCY.ALL", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "P2P Credit Occupancy : Local NCB", + "EventCode": "0x14", + "EventName": "UNC_M2P_P2P_CRD_OCCUPANCY.LOCAL_NCB", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "P2P Credit Occupancy : Local NCS", + "EventCode": "0x14", + "EventName": "UNC_M2P_P2P_CRD_OCCUPANCY.LOCAL_NCS", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "P2P Credit Occupancy : Remote NCB", + "EventCode": "0x14", + "EventName": "UNC_M2P_P2P_CRD_OCCUPANCY.REMOTE_NCB", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "P2P Credit Occupancy : Remote NCS", + "EventCode": "0x14", + "EventName": "UNC_M2P_P2P_CRD_OCCUPANCY.REMOTE_NCS", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Dedicated Credits Received : All", + "EventCode": "0x16", + "EventName": "UNC_M2P_P2P_DED_RECEIVED.ALL", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Dedicated Credits Received : Local NCB", + "EventCode": "0x16", + "EventName": "UNC_M2P_P2P_DED_RECEIVED.LOCAL_NCB", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Dedicated Credits Received : Local NCS", + "EventCode": "0x16", + "EventName": "UNC_M2P_P2P_DED_RECEIVED.LOCAL_NCS", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Dedicated Credits Received : Remote NCB", + "EventCode": "0x16", + "EventName": "UNC_M2P_P2P_DED_RECEIVED.REMOTE_NCB", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Dedicated Credits Received : Remote NCS", + "EventCode": "0x16", + "EventName": "UNC_M2P_P2P_DED_RECEIVED.REMOTE_NCS", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Shared Credits Received : All", + "EventCode": "0x15", + "EventName": "UNC_M2P_P2P_SHAR_RECEIVED.ALL", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Shared Credits Received : Local NCB", + "EventCode": "0x15", + "EventName": "UNC_M2P_P2P_SHAR_RECEIVED.LOCAL_NCB", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Shared Credits Received : Local NCS", + "EventCode": "0x15", + "EventName": "UNC_M2P_P2P_SHAR_RECEIVED.LOCAL_NCS", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Shared Credits Received : Remote NCB", + "EventCode": "0x15", + "EventName": "UNC_M2P_P2P_SHAR_RECEIVED.REMOTE_NCB", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Shared Credits Received : Remote NCS", + "EventCode": "0x15", + "EventName": "UNC_M2P_P2P_SHAR_RECEIVED.REMOTE_NCS", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Remote Dedicated P2P Credit Taken - 0 : UPI0 - DRS", + "EventCode": "0x48", + "EventName": "UNC_M2P_REMOTE_DED_P2P_CRD_TAKEN_0.UPI0_DRS", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Remote Dedicated P2P Credit Taken - 0 : UPI0 - NCB", + "EventCode": "0x48", + "EventName": "UNC_M2P_REMOTE_DED_P2P_CRD_TAKEN_0.UPI0_NCB", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Remote Dedicated P2P Credit Taken - 0 : UPI0 - NCS", + "EventCode": "0x48", + "EventName": "UNC_M2P_REMOTE_DED_P2P_CRD_TAKEN_0.UPI0_NCS", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Remote Dedicated P2P Credit Taken - 0 : UPI1 - DRS", + "EventCode": "0x48", + "EventName": "UNC_M2P_REMOTE_DED_P2P_CRD_TAKEN_0.UPI1_DRS", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Remote Dedicated P2P Credit Taken - 0 : UPI1 - NCB", + "EventCode": "0x48", + "EventName": "UNC_M2P_REMOTE_DED_P2P_CRD_TAKEN_0.UPI1_NCB", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Remote Dedicated P2P Credit Taken - 0 : UPI1 - NCS", + "EventCode": "0x48", + "EventName": "UNC_M2P_REMOTE_DED_P2P_CRD_TAKEN_0.UPI1_NCS", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Remote Dedicated P2P Credit Taken - 1 : UPI2 - DRS", + "EventCode": "0x49", + "EventName": "UNC_M2P_REMOTE_DED_P2P_CRD_TAKEN_1.UPI2_DRS", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Remote Dedicated P2P Credit Taken - 1 : UPI2 - NCB", + "EventCode": "0x49", + "EventName": "UNC_M2P_REMOTE_DED_P2P_CRD_TAKEN_1.UPI2_NCB", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Remote Dedicated P2P Credit Taken - 1 : UPI2 - NCS", + "EventCode": "0x49", + "EventName": "UNC_M2P_REMOTE_DED_P2P_CRD_TAKEN_1.UPI2_NCS", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Remote P2P Dedicated Credits Returned : UPI0 - NCB", + "EventCode": "0x1b", + "EventName": "UNC_M2P_REMOTE_P2P_DED_RETURNED.UPI0_NCB", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Remote P2P Dedicated Credits Returned : UPI0 - NCS", + "EventCode": "0x1b", + "EventName": "UNC_M2P_REMOTE_P2P_DED_RETURNED.UPI0_NCS", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Remote P2P Dedicated Credits Returned : UPI1 - NCB", + "EventCode": "0x1b", + "EventName": "UNC_M2P_REMOTE_P2P_DED_RETURNED.UPI1_NCB", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Remote P2P Dedicated Credits Returned : UPI1 - NCS", + "EventCode": "0x1b", + "EventName": "UNC_M2P_REMOTE_P2P_DED_RETURNED.UPI1_NCS", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Remote P2P Dedicated Credits Returned : UPI2 - NCB", + "EventCode": "0x1b", + "EventName": "UNC_M2P_REMOTE_P2P_DED_RETURNED.UPI2_NCB", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Remote P2P Dedicated Credits Returned : UPI2 - NCS", + "EventCode": "0x1b", + "EventName": "UNC_M2P_REMOTE_P2P_DED_RETURNED.UPI2_NCS", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Remote P2P Shared Credits Returned : Agent0", + "EventCode": "0x18", + "EventName": "UNC_M2P_REMOTE_P2P_SHAR_RETURNED.AGENT_0", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Remote P2P Shared Credits Returned : Agent1", + "EventCode": "0x18", + "EventName": "UNC_M2P_REMOTE_P2P_SHAR_RETURNED.AGENT_1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Remote P2P Shared Credits Returned : Agent2", + "EventCode": "0x18", + "EventName": "UNC_M2P_REMOTE_P2P_SHAR_RETURNED.AGENT_2", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Remote Shared P2P Credit Returned to credit ring : Agent0", + "EventCode": "0x45", + "EventName": "UNC_M2P_REMOTE_SHAR_P2P_CRD_RETURNED.AGENT_0", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Remote Shared P2P Credit Returned to credit ring : Agent1", + "EventCode": "0x45", + "EventName": "UNC_M2P_REMOTE_SHAR_P2P_CRD_RETURNED.AGENT_1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Remote Shared P2P Credit Returned to credit ring : Agent2", + "EventCode": "0x45", + "EventName": "UNC_M2P_REMOTE_SHAR_P2P_CRD_RETURNED.AGENT_2", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Remote Shared P2P Credit Taken - 0 : UPI0 - DRS", + "EventCode": "0x42", + "EventName": "UNC_M2P_REMOTE_SHAR_P2P_CRD_TAKEN_0.UPI0_DRS", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Remote Shared P2P Credit Taken - 0 : UPI0 - NCB", + "EventCode": "0x42", + "EventName": "UNC_M2P_REMOTE_SHAR_P2P_CRD_TAKEN_0.UPI0_NCB", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Remote Shared P2P Credit Taken - 0 : UPI0 - NCS", + "EventCode": "0x42", + "EventName": "UNC_M2P_REMOTE_SHAR_P2P_CRD_TAKEN_0.UPI0_NCS", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Remote Shared P2P Credit Taken - 0 : UPI1 - DRS", + "EventCode": "0x42", + "EventName": "UNC_M2P_REMOTE_SHAR_P2P_CRD_TAKEN_0.UPI1_DRS", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Remote Shared P2P Credit Taken - 0 : UPI1 - NCB", + "EventCode": "0x42", + "EventName": "UNC_M2P_REMOTE_SHAR_P2P_CRD_TAKEN_0.UPI1_NCB", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Remote Shared P2P Credit Taken - 0 : UPI1 - NCS", + "EventCode": "0x42", + "EventName": "UNC_M2P_REMOTE_SHAR_P2P_CRD_TAKEN_0.UPI1_NCS", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Remote Shared P2P Credit Taken - 1 : UPI2 - DRS", + "EventCode": "0x43", + "EventName": "UNC_M2P_REMOTE_SHAR_P2P_CRD_TAKEN_1.UPI2_DRS", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Remote Shared P2P Credit Taken - 1 : UPI2 - NCB", + "EventCode": "0x43", + "EventName": "UNC_M2P_REMOTE_SHAR_P2P_CRD_TAKEN_1.UPI2_NCB", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Remote Shared P2P Credit Taken - 1 : UPI2 - NCS", + "EventCode": "0x43", + "EventName": "UNC_M2P_REMOTE_SHAR_P2P_CRD_TAKEN_1.UPI2_NCS", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Waiting on Remote Shared P2P Credit - 0 : UPI0 - DRS", + "EventCode": "0x4c", + "EventName": "UNC_M2P_REMOTE_SHAR_P2P_CRD_WAIT_0.UPI0_DRS", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Waiting on Remote Shared P2P Credit - 0 : UPI0 - NCB", + "EventCode": "0x4c", + "EventName": "UNC_M2P_REMOTE_SHAR_P2P_CRD_WAIT_0.UPI0_NCB", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Waiting on Remote Shared P2P Credit - 0 : UPI0 - NCS", + "EventCode": "0x4c", + "EventName": "UNC_M2P_REMOTE_SHAR_P2P_CRD_WAIT_0.UPI0_NCS", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Waiting on Remote Shared P2P Credit - 0 : UPI1 - DRS", + "EventCode": "0x4c", + "EventName": "UNC_M2P_REMOTE_SHAR_P2P_CRD_WAIT_0.UPI1_DRS", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Waiting on Remote Shared P2P Credit - 0 : UPI1 - NCB", + "EventCode": "0x4c", + "EventName": "UNC_M2P_REMOTE_SHAR_P2P_CRD_WAIT_0.UPI1_NCB", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Waiting on Remote Shared P2P Credit - 0 : UPI1 - NCS", + "EventCode": "0x4c", + "EventName": "UNC_M2P_REMOTE_SHAR_P2P_CRD_WAIT_0.UPI1_NCS", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Waiting on Remote Shared P2P Credit - 1 : UPI2 - DRS", + "EventCode": "0x4d", + "EventName": "UNC_M2P_REMOTE_SHAR_P2P_CRD_WAIT_1.UPI2_DRS", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Waiting on Remote Shared P2P Credit - 1 : UPI2 - NCB", + "EventCode": "0x4d", + "EventName": "UNC_M2P_REMOTE_SHAR_P2P_CRD_WAIT_1.UPI2_NCB", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Waiting on Remote Shared P2P Credit - 1 : UPI2 - NCS", + "EventCode": "0x4d", + "EventName": "UNC_M2P_REMOTE_SHAR_P2P_CRD_WAIT_1.UPI2_NCS", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Ingress (from CMS) Queue Cycles Not Empty", + "EventCode": "0x10", + "EventName": "UNC_M2P_RxC_CYCLES_NE.ALL", + "PerPkg": "1", + "PublicDescription": "Ingress (from CMS) Queue Cycles Not Empty : Counts the number of cycles when the M2PCIe Ingress is not empty.", + "UMask": "0x80", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Ingress (from CMS) Queue Cycles Not Empty", + "EventCode": "0x10", + "EventName": "UNC_M2P_RxC_CYCLES_NE.CHA_IDI", + "PerPkg": "1", + "PublicDescription": "Ingress (from CMS) Queue Cycles Not Empty : Counts the number of cycles when the M2PCIe Ingress is not empty.", + "UMask": "0x1", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Ingress (from CMS) Queue Cycles Not Empty", + "EventCode": "0x10", + "EventName": "UNC_M2P_RxC_CYCLES_NE.CHA_NCB", + "PerPkg": "1", + "PublicDescription": "Ingress (from CMS) Queue Cycles Not Empty : Counts the number of cycles when the M2PCIe Ingress is not empty.", + "UMask": "0x2", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Ingress (from CMS) Queue Cycles Not Empty", + "EventCode": "0x10", + "EventName": "UNC_M2P_RxC_CYCLES_NE.CHA_NCS", + "PerPkg": "1", + "PublicDescription": "Ingress (from CMS) Queue Cycles Not Empty : Counts the number of cycles when the M2PCIe Ingress is not empty.", + "UMask": "0x4", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Ingress (from CMS) Queue Cycles Not Empty", + "EventCode": "0x10", + "EventName": "UNC_M2P_RxC_CYCLES_NE.IIO_NCB", + "PerPkg": "1", + "PublicDescription": "Ingress (from CMS) Queue Cycles Not Empty : Counts the number of cycles when the M2PCIe Ingress is not empty.", + "UMask": "0x20", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Ingress (from CMS) Queue Cycles Not Empty", + "EventCode": "0x10", + "EventName": "UNC_M2P_RxC_CYCLES_NE.IIO_NCS", + "PerPkg": "1", + "PublicDescription": "Ingress (from CMS) Queue Cycles Not Empty : Counts the number of cycles when the M2PCIe Ingress is not empty.", + "UMask": "0x40", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Ingress (from CMS) Queue Cycles Not Empty", + "EventCode": "0x10", + "EventName": "UNC_M2P_RxC_CYCLES_NE.UPI_NCB", + "PerPkg": "1", + "PublicDescription": "Ingress (from CMS) Queue Cycles Not Empty : Counts the number of cycles when the M2PCIe Ingress is not empty.", + "UMask": "0x8", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Ingress (from CMS) Queue Cycles Not Empty", + "EventCode": "0x10", + "EventName": "UNC_M2P_RxC_CYCLES_NE.UPI_NCS", + "PerPkg": "1", + "PublicDescription": "Ingress (from CMS) Queue Cycles Not Empty : Counts the number of cycles when the M2PCIe Ingress is not empty.", + "UMask": "0x10", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Ingress (from CMS) Queue Inserts", + "EventCode": "0x11", + "EventName": "UNC_M2P_RxC_INSERTS.ALL", + "PerPkg": "1", + "PublicDescription": "Ingress (from CMS) Queue Inserts : Counts the number of entries inserted into the M2PCIe Ingress Queue. This can be used in conjunction with the M2PCIe Ingress Occupancy Accumulator event in order to calculate average queue latency.", + "UMask": "0x80", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Ingress (from CMS) Queue Inserts", + "EventCode": "0x11", + "EventName": "UNC_M2P_RxC_INSERTS.CHA_IDI", + "PerPkg": "1", + "PublicDescription": "Ingress (from CMS) Queue Inserts : Counts the number of entries inserted into the M2PCIe Ingress Queue. This can be used in conjunction with the M2PCIe Ingress Occupancy Accumulator event in order to calculate average queue latency.", + "UMask": "0x1", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Ingress (from CMS) Queue Inserts", + "EventCode": "0x11", + "EventName": "UNC_M2P_RxC_INSERTS.CHA_NCB", + "PerPkg": "1", + "PublicDescription": "Ingress (from CMS) Queue Inserts : Counts the number of entries inserted into the M2PCIe Ingress Queue. This can be used in conjunction with the M2PCIe Ingress Occupancy Accumulator event in order to calculate average queue latency.", + "UMask": "0x2", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Ingress (from CMS) Queue Inserts", + "EventCode": "0x11", + "EventName": "UNC_M2P_RxC_INSERTS.CHA_NCS", + "PerPkg": "1", + "PublicDescription": "Ingress (from CMS) Queue Inserts : Counts the number of entries inserted into the M2PCIe Ingress Queue. This can be used in conjunction with the M2PCIe Ingress Occupancy Accumulator event in order to calculate average queue latency.", + "UMask": "0x4", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Ingress (from CMS) Queue Inserts", + "EventCode": "0x11", + "EventName": "UNC_M2P_RxC_INSERTS.IIO_NCB", + "PerPkg": "1", + "PublicDescription": "Ingress (from CMS) Queue Inserts : Counts the number of entries inserted into the M2PCIe Ingress Queue. This can be used in conjunction with the M2PCIe Ingress Occupancy Accumulator event in order to calculate average queue latency.", + "UMask": "0x20", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Ingress (from CMS) Queue Inserts", + "EventCode": "0x11", + "EventName": "UNC_M2P_RxC_INSERTS.IIO_NCS", + "PerPkg": "1", + "PublicDescription": "Ingress (from CMS) Queue Inserts : Counts the number of entries inserted into the M2PCIe Ingress Queue. This can be used in conjunction with the M2PCIe Ingress Occupancy Accumulator event in order to calculate average queue latency.", + "UMask": "0x40", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Ingress (from CMS) Queue Inserts", + "EventCode": "0x11", + "EventName": "UNC_M2P_RxC_INSERTS.UPI_NCB", + "PerPkg": "1", + "PublicDescription": "Ingress (from CMS) Queue Inserts : Counts the number of entries inserted into the M2PCIe Ingress Queue. This can be used in conjunction with the M2PCIe Ingress Occupancy Accumulator event in order to calculate average queue latency.", + "UMask": "0x8", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Ingress (from CMS) Queue Inserts", + "EventCode": "0x11", + "EventName": "UNC_M2P_RxC_INSERTS.UPI_NCS", + "PerPkg": "1", + "PublicDescription": "Ingress (from CMS) Queue Inserts : Counts the number of entries inserted into the M2PCIe Ingress Queue. This can be used in conjunction with the M2PCIe Ingress Occupancy Accumulator event in order to calculate average queue latency.", + "UMask": "0x10", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "UNC_M2P_TxC_CREDITS.PMM", + "EventCode": "0x2d", + "EventName": "UNC_M2P_TxC_CREDITS.PMM", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "UNC_M2P_TxC_CREDITS.PRQ", + "EventCode": "0x2d", + "EventName": "UNC_M2P_TxC_CREDITS.PRQ", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Egress (to CMS) Cycles Full", + "EventCode": "0x25", + "EventName": "UNC_M2P_TxC_CYCLES_FULL.PMM_BLOCK_0", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "PublicDescription": "Egress (to CMS) Cycles Full : Counts the number of cycles when the M2PCIe Egress is full. This tracks messages for one of the two CMS ports that are used by the M2PCIe agent.", + "UMask": "0x80", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Egress (to CMS) Cycles Full", + "EventCode": "0x25", + "EventName": "UNC_M2P_TxC_CYCLES_FULL.PMM_BLOCK_1", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "PublicDescription": "Egress (to CMS) Cycles Full : Counts the number of cycles when the M2PCIe Egress is full. This tracks messages for one of the two CMS ports that are used by the M2PCIe agent.", + "UMask": "0x8", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Egress (to CMS) Cycles Not Empty", + "EventCode": "0x23", + "EventName": "UNC_M2P_TxC_CYCLES_NE.PMM_DISTRESS_0", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "PublicDescription": "Egress (to CMS) Cycles Not Empty : Counts the number of cycles when the M2PCIe Egress is not empty. This tracks messages for one of the two CMS ports that are used by the M2PCIe agent. This can be used in conjunction with the M2PCIe Ingress Occupancy Accumulator event in order to calculate average queue occupancy. Multiple egress buffers can be tracked at a given time using multiple counters.", + "UMask": "0x80", + "Unit": "M2PCIe" + }, + { + "BriefDescription": "Egress (to CMS) Cycles Not Empty", + "EventCode": "0x23", + "EventName": "UNC_M2P_TxC_CYCLES_NE.PMM_DISTRESS_1", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "PublicDescription": "Egress (to CMS) Cycles Not Empty : Counts the number of cycles when the M2PCIe Egress is not empty. This tracks messages for one of the two CMS ports that are used by the M2PCIe agent. This can be used in conjunction with the M2PCIe Ingress Occupancy Accumulator event in order to calculate average queue occupancy. Multiple egress buffers can be tracked at a given time using multiple counters.", + "UMask": "0x8", + "Unit": "M2PCIe" + } +] diff --git a/tools/perf/pmu-events/arch/x86/emeraldrapids/uncore-memory.json b/tools/perf/pmu-events/arch/x86/emeraldrapids/uncore-memory.json new file mode 100644 index 000000000000..3ff9e9b722c8 --- /dev/null +++ b/tools/perf/pmu-events/arch/x86/emeraldrapids/uncore-memory.json @@ -0,0 +1,3308 @@ +[ + { + "BriefDescription": "Cycles - at UCLK", + "EventCode": "0x01", + "EventName": "UNC_M2HBM_CLOCKTICKS", + "PerPkg": "1", + "Unit": "M2HBM" + }, + { + "BriefDescription": "CMS Clockticks", + "EventCode": "0xc0", + "EventName": "UNC_M2HBM_CMS_CLOCKTICKS", + "PerPkg": "1", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Cycles when direct to core mode (which bypasses the CHA) was disabled", + "EventCode": "0x17", + "EventName": "UNC_M2HBM_DIRECT2CORE_NOT_TAKEN_DIRSTATE", + "PerPkg": "1", + "UMask": "0x7", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Cycles when direct to core mode, which bypasses the CHA, was disabled : Non Cisgress", + "EventCode": "0x17", + "EventName": "UNC_M2HBM_DIRECT2CORE_NOT_TAKEN_DIRSTATE.NON_CISGRESS", + "PerPkg": "1", + "PublicDescription": "Counts the number of time non cisgress D2C was not honoured by egress due to directory state constraints", + "UMask": "0x2", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Counts the time when FM didn't do d2c for fill reads (cross tile case)", + "EventCode": "0x4a", + "EventName": "UNC_M2HBM_DIRECT2CORE_NOT_TAKEN_NOTFORKED", + "PerPkg": "1", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Number of reads in which direct to core transaction were overridden", + "EventCode": "0x18", + "EventName": "UNC_M2HBM_DIRECT2CORE_TXN_OVERRIDE", + "PerPkg": "1", + "UMask": "0x3", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Number of reads in which direct to core transaction was overridden : Cisgress", + "EventCode": "0x18", + "EventName": "UNC_M2HBM_DIRECT2CORE_TXN_OVERRIDE.CISGRESS", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Number of reads in which direct to Intel UPI transactions were overridden", + "EventCode": "0x1b", + "EventName": "UNC_M2HBM_DIRECT2UPI_NOT_TAKEN_CREDITS", + "PerPkg": "1", + "UMask": "0x7", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Cycles when direct to Intel UPI was disabled", + "EventCode": "0x1a", + "EventName": "UNC_M2HBM_DIRECT2UPI_NOT_TAKEN_DIRSTATE", + "PerPkg": "1", + "UMask": "0x7", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Cycles when Direct2UPI was Disabled : Cisgress D2U Ignored", + "EventCode": "0x1A", + "EventName": "UNC_M2HBM_DIRECT2UPI_NOT_TAKEN_DIRSTATE.CISGRESS", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "PublicDescription": "Counts cisgress d2K that was not honored due to directory constraints", + "UMask": "0x4", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Cycles when Direct2UPI was Disabled : Egress Ignored D2U", + "EventCode": "0x1A", + "EventName": "UNC_M2HBM_DIRECT2UPI_NOT_TAKEN_DIRSTATE.EGRESS", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "PublicDescription": "Counts the number of time D2K was not honoured by egress due to directory state constraints", + "UMask": "0x1", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Cycles when Direct2UPI was Disabled : Non Cisgress D2U Ignored", + "EventCode": "0x1A", + "EventName": "UNC_M2HBM_DIRECT2UPI_NOT_TAKEN_DIRSTATE.NON_CISGRESS", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "PublicDescription": "Counts non cisgress d2K that was not honored due to directory constraints", + "UMask": "0x2", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Number of reads that a message sent direct2 Intel UPI was overridden", + "EventCode": "0x1c", + "EventName": "UNC_M2HBM_DIRECT2UPI_TXN_OVERRIDE", + "PerPkg": "1", + "UMask": "0x3", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Number of times a direct to UPI transaction was overridden.", + "EventCode": "0x1c", + "EventName": "UNC_M2HBM_DIRECT2UPI_TXN_OVERRIDE.CISGRESS", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Directory Hit : On NonDirty Line in A State", + "EventCode": "0x1d", + "EventName": "UNC_M2HBM_DIRECTORY_HIT.CLEAN_A", + "PerPkg": "1", + "UMask": "0x80", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Directory Hit : On NonDirty Line in I State", + "EventCode": "0x1d", + "EventName": "UNC_M2HBM_DIRECTORY_HIT.CLEAN_I", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Directory Hit : On NonDirty Line in L State", + "EventCode": "0x1d", + "EventName": "UNC_M2HBM_DIRECTORY_HIT.CLEAN_P", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Directory Hit : On NonDirty Line in S State", + "EventCode": "0x1d", + "EventName": "UNC_M2HBM_DIRECTORY_HIT.CLEAN_S", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Directory Hit : On Dirty Line in A State", + "EventCode": "0x1d", + "EventName": "UNC_M2HBM_DIRECTORY_HIT.DIRTY_A", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Directory Hit : On Dirty Line in I State", + "EventCode": "0x1d", + "EventName": "UNC_M2HBM_DIRECTORY_HIT.DIRTY_I", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Directory Hit : On Dirty Line in L State", + "EventCode": "0x1d", + "EventName": "UNC_M2HBM_DIRECTORY_HIT.DIRTY_P", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Directory Hit : On Dirty Line in S State", + "EventCode": "0x1d", + "EventName": "UNC_M2HBM_DIRECTORY_HIT.DIRTY_S", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Multi-socket cacheline Directory lookups (any state found)", + "EventCode": "0x20", + "EventName": "UNC_M2HBM_DIRECTORY_LOOKUP.ANY", + "PerPkg": "1", + "PublicDescription": "Counts the number of hit data returns to egress with any directory to non persistent memory", + "UMask": "0x1", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Multi-socket cacheline Directory lookups (cacheline found in A state)", + "EventCode": "0x20", + "EventName": "UNC_M2HBM_DIRECTORY_LOOKUP.STATE_A", + "PerPkg": "1", + "PublicDescription": "Counts the number of hit data returns to egress with directory A to non persistent memory", + "UMask": "0x8", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Multi-socket cacheline Directory lookup (cacheline found in I state)", + "EventCode": "0x20", + "EventName": "UNC_M2HBM_DIRECTORY_LOOKUP.STATE_I", + "PerPkg": "1", + "PublicDescription": "Counts the number of hit data returns to egress with directory I to non persistent memory", + "UMask": "0x2", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Multi-socket cacheline Directory lookup (cacheline found in S state)", + "EventCode": "0x20", + "EventName": "UNC_M2HBM_DIRECTORY_LOOKUP.STATE_S", + "PerPkg": "1", + "PublicDescription": "Counts the number of hit data returns to egress with directory S to non persistent memory", + "UMask": "0x4", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Directory Miss : On NonDirty Line in A State", + "EventCode": "0x1e", + "EventName": "UNC_M2HBM_DIRECTORY_MISS.CLEAN_A", + "PerPkg": "1", + "UMask": "0x80", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Directory Miss : On NonDirty Line in I State", + "EventCode": "0x1e", + "EventName": "UNC_M2HBM_DIRECTORY_MISS.CLEAN_I", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Directory Miss : On NonDirty Line in L State", + "EventCode": "0x1e", + "EventName": "UNC_M2HBM_DIRECTORY_MISS.CLEAN_P", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Directory Miss : On NonDirty Line in S State", + "EventCode": "0x1e", + "EventName": "UNC_M2HBM_DIRECTORY_MISS.CLEAN_S", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Directory Miss : On Dirty Line in A State", + "EventCode": "0x1e", + "EventName": "UNC_M2HBM_DIRECTORY_MISS.DIRTY_A", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Directory Miss : On Dirty Line in I State", + "EventCode": "0x1e", + "EventName": "UNC_M2HBM_DIRECTORY_MISS.DIRTY_I", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Directory Miss : On Dirty Line in L State", + "EventCode": "0x1e", + "EventName": "UNC_M2HBM_DIRECTORY_MISS.DIRTY_P", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Directory Miss : On Dirty Line in S State", + "EventCode": "0x1e", + "EventName": "UNC_M2HBM_DIRECTORY_MISS.DIRTY_S", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Multi-socket cacheline Directory update from A to I", + "EventCode": "0x21", + "EventName": "UNC_M2HBM_DIRECTORY_UPDATE.A2I", + "PerPkg": "1", + "UMask": "0x320", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Multi-socket cacheline Directory update from A to S", + "EventCode": "0x21", + "EventName": "UNC_M2HBM_DIRECTORY_UPDATE.A2S", + "PerPkg": "1", + "UMask": "0x340", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Multi-socket cacheline Directory update from/to Any state", + "EventCode": "0x21", + "EventName": "UNC_M2HBM_DIRECTORY_UPDATE.ANY", + "PerPkg": "1", + "UMask": "0x301", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Multi-socket cacheline Directory Updates", + "EventCode": "0x21", + "EventName": "UNC_M2HBM_DIRECTORY_UPDATE.A_TO_I_HIT_NON_PMM", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "PublicDescription": "Counts 1lm or 2lm hit data returns that would result in directory update from A to I to non persistent memory", + "UMask": "0x120", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Multi-socket cacheline Directory Updates", + "EventCode": "0x21", + "EventName": "UNC_M2HBM_DIRECTORY_UPDATE.A_TO_I_MISS_NON_PMM", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "PublicDescription": "Counts 2lm miss data returns that would result in directory update from A to I to non persistent memory", + "UMask": "0x220", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Multi-socket cacheline Directory Updates", + "EventCode": "0x21", + "EventName": "UNC_M2HBM_DIRECTORY_UPDATE.A_TO_S_HIT_NON_PMM", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "PublicDescription": "Counts 1lm or 2lm hit data returns that would result in directory update from A to S to non persistent memory", + "UMask": "0x140", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Multi-socket cacheline Directory Updates", + "EventCode": "0x21", + "EventName": "UNC_M2HBM_DIRECTORY_UPDATE.A_TO_S_MISS_NON_PMM", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "PublicDescription": "Counts 2lm miss data returns that would result in directory update from A to S to non persistent memory", + "UMask": "0x240", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Multi-socket cacheline Directory Updates", + "EventCode": "0x21", + "EventName": "UNC_M2HBM_DIRECTORY_UPDATE.HIT_NON_PMM", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "PublicDescription": "Counts any 1lm or 2lm hit data return that would result in directory update to non persistent memory", + "UMask": "0x101", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Multi-socket cacheline Directory update from I to A", + "EventCode": "0x21", + "EventName": "UNC_M2HBM_DIRECTORY_UPDATE.I2A", + "PerPkg": "1", + "UMask": "0x304", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Multi-socket cacheline Directory update from I to S", + "EventCode": "0x21", + "EventName": "UNC_M2HBM_DIRECTORY_UPDATE.I2S", + "PerPkg": "1", + "UMask": "0x302", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Multi-socket cacheline Directory Updates", + "EventCode": "0x21", + "EventName": "UNC_M2HBM_DIRECTORY_UPDATE.I_TO_A_HIT_NON_PMM", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "PublicDescription": "Counts 1lm or 2lm hit data returns that would result in directory update from I to A to non persistent memory", + "UMask": "0x104", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Multi-socket cacheline Directory Updates", + "EventCode": "0x21", + "EventName": "UNC_M2HBM_DIRECTORY_UPDATE.I_TO_A_MISS_NON_PMM", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "PublicDescription": "Counts 2lm miss data returns that would result in directory update from I to A to non persistent memory", + "UMask": "0x204", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Multi-socket cacheline Directory Updates", + "EventCode": "0x21", + "EventName": "UNC_M2HBM_DIRECTORY_UPDATE.I_TO_S_HIT_NON_PMM", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "PublicDescription": "Counts 1lm or 2lm hit data returns that would result in directory update from I to S to non persistent memory", + "UMask": "0x102", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Multi-socket cacheline Directory Updates", + "EventCode": "0x21", + "EventName": "UNC_M2HBM_DIRECTORY_UPDATE.I_TO_S_MISS_NON_PMM", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "PublicDescription": "Counts 2lm miss data returns that would result in directory update from I to S to non persistent memory", + "UMask": "0x202", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Multi-socket cacheline Directory Updates", + "EventCode": "0x21", + "EventName": "UNC_M2HBM_DIRECTORY_UPDATE.MISS_NON_PMM", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "PublicDescription": "Counts any 2lm miss data return that would result in directory update to non persistent memory", + "UMask": "0x201", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Multi-socket cacheline Directory update from S to A", + "EventCode": "0x21", + "EventName": "UNC_M2HBM_DIRECTORY_UPDATE.S2A", + "PerPkg": "1", + "UMask": "0x310", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Multi-socket cacheline Directory update from S to I", + "EventCode": "0x21", + "EventName": "UNC_M2HBM_DIRECTORY_UPDATE.S2I", + "PerPkg": "1", + "UMask": "0x308", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Multi-socket cacheline Directory Updates", + "EventCode": "0x21", + "EventName": "UNC_M2HBM_DIRECTORY_UPDATE.S_TO_A_HIT_NON_PMM", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "PublicDescription": "Counts 1lm or 2lm hit data returns that would result in directory update from S to A to non persistent memory", + "UMask": "0x110", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Multi-socket cacheline Directory Updates", + "EventCode": "0x21", + "EventName": "UNC_M2HBM_DIRECTORY_UPDATE.S_TO_A_MISS_NON_PMM", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "PublicDescription": "Counts 2lm miss data returns that would result in directory update from S to A to non persistent memory", + "UMask": "0x210", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Multi-socket cacheline Directory Updates", + "EventCode": "0x21", + "EventName": "UNC_M2HBM_DIRECTORY_UPDATE.S_TO_I_HIT_NON_PMM", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "PublicDescription": "Counts 1lm or 2lm hit data returns that would result in directory update from S to I to non persistent memory", + "UMask": "0x108", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Multi-socket cacheline Directory Updates", + "EventCode": "0x21", + "EventName": "UNC_M2HBM_DIRECTORY_UPDATE.S_TO_I_MISS_NON_PMM", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "PublicDescription": "Counts 2lm miss data returns that would result in directory update from S to I to non persistent memory", + "UMask": "0x208", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Count distress signalled on AkAd cmp message", + "EventCode": "0x67", + "EventName": "UNC_M2HBM_DISTRESS.AD", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Count distress signalled on any packet type", + "EventCode": "0x67", + "EventName": "UNC_M2HBM_DISTRESS.ALL", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Count distress signalled on Bl Cmp message", + "EventCode": "0x67", + "EventName": "UNC_M2HBM_DISTRESS.BL_CMP", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Count distress signalled on NM fill write message", + "EventCode": "0x67", + "EventName": "UNC_M2HBM_DISTRESS.CROSSTILE_NMWR", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Count distress signalled on D2Cha message", + "EventCode": "0x67", + "EventName": "UNC_M2HBM_DISTRESS.D2CHA", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Count distress signalled on D2c message", + "EventCode": "0x67", + "EventName": "UNC_M2HBM_DISTRESS.D2CORE", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Count distress signalled on D2k message", + "EventCode": "0x67", + "EventName": "UNC_M2HBM_DISTRESS.D2UPI", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Egress Blocking due to Ordering requirements : Down", + "EventCode": "0xba", + "EventName": "UNC_M2HBM_EGRESS_ORDERING.IV_SNOOPGO_DN", + "PerPkg": "1", + "PublicDescription": "Egress Blocking due to Ordering requirements : Down : Counts number of cycles IV was blocked in the TGR Egress due to SNP/GO Ordering requirements", + "UMask": "0x80000004", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Egress Blocking due to Ordering requirements : Up", + "EventCode": "0xba", + "EventName": "UNC_M2HBM_EGRESS_ORDERING.IV_SNOOPGO_UP", + "PerPkg": "1", + "PublicDescription": "Egress Blocking due to Ordering requirements : Up : Counts number of cycles IV was blocked in the TGR Egress due to SNP/GO Ordering requirements", + "UMask": "0x80000001", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Count when Starve Glocab counter is at 7", + "EventCode": "0x44", + "EventName": "UNC_M2HBM_IGR_STARVE_WINNER.MASK7", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "UMask": "0x80", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Reads to iMC issued", + "EventCode": "0x24", + "EventName": "UNC_M2HBM_IMC_READS.ALL", + "PerPkg": "1", + "UMask": "0x304", + "Unit": "M2HBM" + }, + { + "BriefDescription": "UNC_M2HBM_IMC_READS.CH0.ALL", + "EventCode": "0x24", + "EventName": "UNC_M2HBM_IMC_READS.CH0.ALL", + "PerPkg": "1", + "UMask": "0x104", + "Unit": "M2HBM" + }, + { + "BriefDescription": "UNC_M2HBM_IMC_READS.CH0.NORMAL", + "EventCode": "0x24", + "EventName": "UNC_M2HBM_IMC_READS.CH0.NORMAL", + "PerPkg": "1", + "UMask": "0x101", + "Unit": "M2HBM" + }, + { + "BriefDescription": "UNC_M2HBM_IMC_READS.CH0_ALL", + "EventCode": "0x24", + "EventName": "UNC_M2HBM_IMC_READS.CH0_ALL", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "UMask": "0x104", + "Unit": "M2HBM" + }, + { + "BriefDescription": "UNC_M2HBM_IMC_READS.CH0_FROM_TGR", + "EventCode": "0x24", + "EventName": "UNC_M2HBM_IMC_READS.CH0_FROM_TGR", + "PerPkg": "1", + "UMask": "0x140", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Critical Priority - Ch0", + "EventCode": "0x24", + "EventName": "UNC_M2HBM_IMC_READS.CH0_ISOCH", + "PerPkg": "1", + "UMask": "0x102", + "Unit": "M2HBM" + }, + { + "BriefDescription": "UNC_M2HBM_IMC_READS.CH0_NORMAL", + "EventCode": "0x24", + "EventName": "UNC_M2HBM_IMC_READS.CH0_NORMAL", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "UMask": "0x101", + "Unit": "M2HBM" + }, + { + "BriefDescription": "UNC_M2HBM_IMC_READS.CH1.ALL", + "EventCode": "0x24", + "EventName": "UNC_M2HBM_IMC_READS.CH1.ALL", + "PerPkg": "1", + "UMask": "0x204", + "Unit": "M2HBM" + }, + { + "BriefDescription": "UNC_M2HBM_IMC_READS.CH1.NORMAL", + "EventCode": "0x24", + "EventName": "UNC_M2HBM_IMC_READS.CH1.NORMAL", + "PerPkg": "1", + "UMask": "0x201", + "Unit": "M2HBM" + }, + { + "BriefDescription": "UNC_M2HBM_IMC_READS.CH1_ALL", + "EventCode": "0x24", + "EventName": "UNC_M2HBM_IMC_READS.CH1_ALL", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "UMask": "0x204", + "Unit": "M2HBM" + }, + { + "BriefDescription": "From TGR - Ch1", + "EventCode": "0x24", + "EventName": "UNC_M2HBM_IMC_READS.CH1_FROM_TGR", + "PerPkg": "1", + "UMask": "0x240", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Critical Priority - Ch1", + "EventCode": "0x24", + "EventName": "UNC_M2HBM_IMC_READS.CH1_ISOCH", + "PerPkg": "1", + "UMask": "0x202", + "Unit": "M2HBM" + }, + { + "BriefDescription": "UNC_M2HBM_IMC_READS.CH1_NORMAL", + "EventCode": "0x24", + "EventName": "UNC_M2HBM_IMC_READS.CH1_NORMAL", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "UMask": "0x201", + "Unit": "M2HBM" + }, + { + "BriefDescription": "From TGR - All Channels", + "EventCode": "0x24", + "EventName": "UNC_M2HBM_IMC_READS.FROM_TGR", + "PerPkg": "1", + "UMask": "0x340", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Critical Priority - All Channels", + "EventCode": "0x24", + "EventName": "UNC_M2HBM_IMC_READS.ISOCH", + "PerPkg": "1", + "UMask": "0x302", + "Unit": "M2HBM" + }, + { + "BriefDescription": "UNC_M2HBM_IMC_READS.NORMAL", + "EventCode": "0x24", + "EventName": "UNC_M2HBM_IMC_READS.NORMAL", + "PerPkg": "1", + "UMask": "0x301", + "Unit": "M2HBM" + }, + { + "BriefDescription": "All Writes - All Channels", + "EventCode": "0x25", + "EventName": "UNC_M2HBM_IMC_WRITES.ALL", + "PerPkg": "1", + "UMask": "0x1810", + "Unit": "M2HBM" + }, + { + "BriefDescription": "UNC_M2HBM_IMC_WRITES.CH0.ALL", + "EventCode": "0x25", + "EventName": "UNC_M2HBM_IMC_WRITES.CH0.ALL", + "PerPkg": "1", + "UMask": "0x810", + "Unit": "M2HBM" + }, + { + "BriefDescription": "UNC_M2HBM_IMC_WRITES.CH0.FULL", + "EventCode": "0x25", + "EventName": "UNC_M2HBM_IMC_WRITES.CH0.FULL", + "PerPkg": "1", + "UMask": "0x801", + "Unit": "M2HBM" + }, + { + "BriefDescription": "UNC_M2HBM_IMC_WRITES.CH0.PARTIAL", + "EventCode": "0x25", + "EventName": "UNC_M2HBM_IMC_WRITES.CH0.PARTIAL", + "PerPkg": "1", + "UMask": "0x802", + "Unit": "M2HBM" + }, + { + "BriefDescription": "UNC_M2HBM_IMC_WRITES.CH0_ALL", + "EventCode": "0x25", + "EventName": "UNC_M2HBM_IMC_WRITES.CH0_ALL", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "UMask": "0x810", + "Unit": "M2HBM" + }, + { + "BriefDescription": "From TGR - Ch0", + "EventCode": "0x25", + "EventName": "UNC_M2HBM_IMC_WRITES.CH0_FROM_TGR", + "PerPkg": "1", + "Unit": "M2HBM" + }, + { + "BriefDescription": "UNC_M2HBM_IMC_WRITES.CH0_FULL", + "EventCode": "0x25", + "EventName": "UNC_M2HBM_IMC_WRITES.CH0_FULL", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "UMask": "0x801", + "Unit": "M2HBM" + }, + { + "BriefDescription": "ISOCH Full Line - Ch0", + "EventCode": "0x25", + "EventName": "UNC_M2HBM_IMC_WRITES.CH0_FULL_ISOCH", + "PerPkg": "1", + "UMask": "0x804", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Non-Inclusive - Ch0", + "EventCode": "0x25", + "EventName": "UNC_M2HBM_IMC_WRITES.CH0_NI", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Non-Inclusive Miss - Ch0", + "EventCode": "0x25", + "EventName": "UNC_M2HBM_IMC_WRITES.CH0_NI_MISS", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "Unit": "M2HBM" + }, + { + "BriefDescription": "UNC_M2HBM_IMC_WRITES.CH0_PARTIAL", + "EventCode": "0x25", + "EventName": "UNC_M2HBM_IMC_WRITES.CH0_PARTIAL", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "UMask": "0x802", + "Unit": "M2HBM" + }, + { + "BriefDescription": "ISOCH Partial - Ch0", + "EventCode": "0x25", + "EventName": "UNC_M2HBM_IMC_WRITES.CH0_PARTIAL_ISOCH", + "PerPkg": "1", + "UMask": "0x808", + "Unit": "M2HBM" + }, + { + "BriefDescription": "All Writes - Ch1", + "EventCode": "0x25", + "EventName": "UNC_M2HBM_IMC_WRITES.CH1.ALL", + "PerPkg": "1", + "UMask": "0x1010", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Full Line Non-ISOCH - Ch1", + "EventCode": "0x25", + "EventName": "UNC_M2HBM_IMC_WRITES.CH1.FULL", + "PerPkg": "1", + "UMask": "0x1001", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Partial Non-ISOCH - Ch1", + "EventCode": "0x25", + "EventName": "UNC_M2HBM_IMC_WRITES.CH1.PARTIAL", + "PerPkg": "1", + "UMask": "0x1002", + "Unit": "M2HBM" + }, + { + "BriefDescription": "All Writes - Ch1", + "EventCode": "0x25", + "EventName": "UNC_M2HBM_IMC_WRITES.CH1_ALL", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "UMask": "0x1010", + "Unit": "M2HBM" + }, + { + "BriefDescription": "From TGR - Ch1", + "EventCode": "0x25", + "EventName": "UNC_M2HBM_IMC_WRITES.CH1_FROM_TGR", + "PerPkg": "1", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Full Line Non-ISOCH - Ch1", + "EventCode": "0x25", + "EventName": "UNC_M2HBM_IMC_WRITES.CH1_FULL", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "UMask": "0x1001", + "Unit": "M2HBM" + }, + { + "BriefDescription": "ISOCH Full Line - Ch1", + "EventCode": "0x25", + "EventName": "UNC_M2HBM_IMC_WRITES.CH1_FULL_ISOCH", + "PerPkg": "1", + "UMask": "0x1004", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Non-Inclusive - Ch1", + "EventCode": "0x25", + "EventName": "UNC_M2HBM_IMC_WRITES.CH1_NI", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Non-Inclusive Miss - Ch1", + "EventCode": "0x25", + "EventName": "UNC_M2HBM_IMC_WRITES.CH1_NI_MISS", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Partial Non-ISOCH - Ch1", + "EventCode": "0x25", + "EventName": "UNC_M2HBM_IMC_WRITES.CH1_PARTIAL", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "UMask": "0x1002", + "Unit": "M2HBM" + }, + { + "BriefDescription": "ISOCH Partial - Ch1", + "EventCode": "0x25", + "EventName": "UNC_M2HBM_IMC_WRITES.CH1_PARTIAL_ISOCH", + "PerPkg": "1", + "UMask": "0x1008", + "Unit": "M2HBM" + }, + { + "BriefDescription": "From TGR - All Channels", + "EventCode": "0x25", + "EventName": "UNC_M2HBM_IMC_WRITES.FROM_TGR", + "PerPkg": "1", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Full Non-ISOCH - All Channels", + "EventCode": "0x25", + "EventName": "UNC_M2HBM_IMC_WRITES.FULL", + "PerPkg": "1", + "UMask": "0x1801", + "Unit": "M2HBM" + }, + { + "BriefDescription": "ISOCH Full Line - All Channels", + "EventCode": "0x25", + "EventName": "UNC_M2HBM_IMC_WRITES.FULL_ISOCH", + "PerPkg": "1", + "UMask": "0x1804", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Non-Inclusive - All Channels", + "EventCode": "0x25", + "EventName": "UNC_M2HBM_IMC_WRITES.NI", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Non-Inclusive Miss - All Channels", + "EventCode": "0x25", + "EventName": "UNC_M2HBM_IMC_WRITES.NI_MISS", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Partial Non-ISOCH - All Channels", + "EventCode": "0x25", + "EventName": "UNC_M2HBM_IMC_WRITES.PARTIAL", + "PerPkg": "1", + "UMask": "0x1802", + "Unit": "M2HBM" + }, + { + "BriefDescription": "ISOCH Partial - All Channels", + "EventCode": "0x25", + "EventName": "UNC_M2HBM_IMC_WRITES.PARTIAL_ISOCH", + "PerPkg": "1", + "UMask": "0x1808", + "Unit": "M2HBM" + }, + { + "BriefDescription": "UNC_M2HBM_PREFCAM_CIS_DROPS", + "EventCode": "0x5c", + "EventName": "UNC_M2HBM_PREFCAM_CIS_DROPS", + "PerPkg": "1", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Data Prefetches Dropped", + "EventCode": "0x58", + "EventName": "UNC_M2HBM_PREFCAM_DEMAND_DROPS.CH0_UPI", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Data Prefetches Dropped", + "EventCode": "0x58", + "EventName": "UNC_M2HBM_PREFCAM_DEMAND_DROPS.CH0_XPT", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Data Prefetches Dropped", + "EventCode": "0x58", + "EventName": "UNC_M2HBM_PREFCAM_DEMAND_DROPS.CH1_UPI", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Data Prefetches Dropped", + "EventCode": "0x58", + "EventName": "UNC_M2HBM_PREFCAM_DEMAND_DROPS.CH1_XPT", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Data Prefetches Dropped : UPI - All Channels", + "EventCode": "0x58", + "EventName": "UNC_M2HBM_PREFCAM_DEMAND_DROPS.UPI_ALLCH", + "PerPkg": "1", + "UMask": "0xa", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Data Prefetches Dropped", + "EventCode": "0x58", + "EventName": "UNC_M2HBM_PREFCAM_DEMAND_DROPS.XPT_ALLCH", + "PerPkg": "1", + "UMask": "0x5", + "Unit": "M2HBM" + }, + { + "BriefDescription": ": UPI - All Channels", + "EventCode": "0x5d", + "EventName": "UNC_M2HBM_PREFCAM_DEMAND_MERGE.UPI_ALLCH", + "PerPkg": "1", + "UMask": "0xa", + "Unit": "M2HBM" + }, + { + "BriefDescription": ": XPT - All Channels", + "EventCode": "0x5d", + "EventName": "UNC_M2HBM_PREFCAM_DEMAND_MERGE.XPT_ALLCH", + "PerPkg": "1", + "UMask": "0x5", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Demands Not Merged with CAMed Prefetches", + "EventCode": "0x5e", + "EventName": "UNC_M2HBM_PREFCAM_DEMAND_NO_MERGE.RD_MERGED", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Demands Not Merged with CAMed Prefetches", + "EventCode": "0x5e", + "EventName": "UNC_M2HBM_PREFCAM_DEMAND_NO_MERGE.WR_MERGED", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Demands Not Merged with CAMed Prefetches", + "EventCode": "0x5e", + "EventName": "UNC_M2HBM_PREFCAM_DEMAND_NO_MERGE.WR_SQUASHED", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Prefetch CAM Inserts : UPI - Ch 0", + "EventCode": "0x56", + "EventName": "UNC_M2HBM_PREFCAM_INSERTS.CH0_UPI", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Prefetch CAM Inserts : XPT - Ch 0", + "EventCode": "0x56", + "EventName": "UNC_M2HBM_PREFCAM_INSERTS.CH0_XPT", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Prefetch CAM Inserts : UPI - Ch 1", + "EventCode": "0x56", + "EventName": "UNC_M2HBM_PREFCAM_INSERTS.CH1_UPI", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Prefetch CAM Inserts : XPT - Ch 1", + "EventCode": "0x56", + "EventName": "UNC_M2HBM_PREFCAM_INSERTS.CH1_XPT", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Prefetch CAM Inserts : UPI - All Channels", + "EventCode": "0x56", + "EventName": "UNC_M2HBM_PREFCAM_INSERTS.UPI_ALLCH", + "PerPkg": "1", + "UMask": "0xa", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Prefetch CAM Inserts : XPT - All Channels", + "EventCode": "0x56", + "EventName": "UNC_M2HBM_PREFCAM_INSERTS.XPT_ALLCH", + "PerPkg": "1", + "PublicDescription": "Prefetch CAM Inserts : XPT -All Channels", + "UMask": "0x5", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Prefetch CAM Occupancy : All Channels", + "EventCode": "0x54", + "EventName": "UNC_M2HBM_PREFCAM_OCCUPANCY.ALLCH", + "PerPkg": "1", + "UMask": "0x3", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Prefetch CAM Occupancy : Channel 0", + "EventCode": "0x54", + "EventName": "UNC_M2HBM_PREFCAM_OCCUPANCY.CH0", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Prefetch CAM Occupancy : Channel 1", + "EventCode": "0x54", + "EventName": "UNC_M2HBM_PREFCAM_OCCUPANCY.CH1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2HBM" + }, + { + "BriefDescription": "All Channels", + "EventCode": "0x5f", + "EventName": "UNC_M2HBM_PREFCAM_RESP_MISS.ALLCH", + "PerPkg": "1", + "UMask": "0x3", + "Unit": "M2HBM" + }, + { + "BriefDescription": ": Channel 0", + "EventCode": "0x5f", + "EventName": "UNC_M2HBM_PREFCAM_RESP_MISS.CH0", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2HBM" + }, + { + "BriefDescription": ": Channel 1", + "EventCode": "0x5f", + "EventName": "UNC_M2HBM_PREFCAM_RESP_MISS.CH1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2HBM" + }, + { + "BriefDescription": "UNC_M2HBM_PREFCAM_RxC_DEALLOCS.1LM_POSTED", + "EventCode": "0x62", + "EventName": "UNC_M2HBM_PREFCAM_RxC_DEALLOCS.1LM_POSTED", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2HBM" + }, + { + "BriefDescription": "UNC_M2HBM_PREFCAM_RxC_DEALLOCS.CIS", + "EventCode": "0x62", + "EventName": "UNC_M2HBM_PREFCAM_RxC_DEALLOCS.CIS", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "M2HBM" + }, + { + "BriefDescription": "UNC_M2HBM_PREFCAM_RxC_DEALLOCS.SQUASHED", + "EventCode": "0x62", + "EventName": "UNC_M2HBM_PREFCAM_RxC_DEALLOCS.SQUASHED", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2HBM" + }, + { + "BriefDescription": "UNC_M2HBM_PREFCAM_RxC_OCCUPANCY", + "EventCode": "0x60", + "EventName": "UNC_M2HBM_PREFCAM_RxC_OCCUPANCY", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "Unit": "M2HBM" + }, + { + "BriefDescription": "AD Ingress (from CMS) : AD Ingress (from CMS) Allocations", + "EventCode": "0x02", + "EventName": "UNC_M2HBM_RxC_AD.INSERTS", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "UMask": "0x1", + "Unit": "M2HBM" + }, + { + "BriefDescription": "AD Ingress (from CMS) : AD Ingress (from CMS) Allocations", + "EventCode": "0x02", + "EventName": "UNC_M2HBM_RxC_AD_INSERTS", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2HBM" + }, + { + "BriefDescription": "AD Ingress (from CMS) Occupancy", + "EventCode": "0x03", + "EventName": "UNC_M2HBM_RxC_AD_OCCUPANCY", + "PerPkg": "1", + "Unit": "M2HBM" + }, + { + "BriefDescription": "BL Ingress (from CMS) : BL Ingress (from CMS) Allocations", + "EventCode": "0x04", + "EventName": "UNC_M2HBM_RxC_BL.INSERTS", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "PublicDescription": "Counts anytime a BL packet is added to Ingress", + "UMask": "0x1", + "Unit": "M2HBM" + }, + { + "BriefDescription": "BL Ingress (from CMS) : BL Ingress (from CMS) Allocations", + "EventCode": "0x04", + "EventName": "UNC_M2HBM_RxC_BL_INSERTS", + "PerPkg": "1", + "PublicDescription": "Counts anytime a BL packet is added to Ingress", + "UMask": "0x1", + "Unit": "M2HBM" + }, + { + "BriefDescription": "BL Ingress (from CMS) Occupancy", + "EventCode": "0x05", + "EventName": "UNC_M2HBM_RxC_BL_OCCUPANCY", + "PerPkg": "1", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Number AD Ingress Credits", + "EventCode": "0x2e", + "EventName": "UNC_M2HBM_TGR_AD_CREDITS", + "PerPkg": "1", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Number BL Ingress Credits", + "EventCode": "0x2f", + "EventName": "UNC_M2HBM_TGR_BL_CREDITS", + "PerPkg": "1", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Tracker Inserts : Channel 0", + "EventCode": "0x32", + "EventName": "UNC_M2HBM_TRACKER_INSERTS.CH0", + "PerPkg": "1", + "UMask": "0x104", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Tracker Inserts : Channel 1", + "EventCode": "0x32", + "EventName": "UNC_M2HBM_TRACKER_INSERTS.CH1", + "PerPkg": "1", + "UMask": "0x204", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Tracker Occupancy : Channel 0", + "EventCode": "0x33", + "EventName": "UNC_M2HBM_TRACKER_OCCUPANCY.CH0", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Tracker Occupancy : Channel 1", + "EventCode": "0x33", + "EventName": "UNC_M2HBM_TRACKER_OCCUPANCY.CH1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2HBM" + }, + { + "BriefDescription": "AD Egress (to CMS) : AD Egress (to CMS) Allocations", + "EventCode": "0x06", + "EventName": "UNC_M2HBM_TxC_AD.INSERTS", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "PublicDescription": "Counts anytime a AD packet is added to Egress", + "UMask": "0x1", + "Unit": "M2HBM" + }, + { + "BriefDescription": "AD Egress (to CMS) : AD Egress (to CMS) Allocations", + "EventCode": "0x06", + "EventName": "UNC_M2HBM_TxC_AD_INSERTS", + "PerPkg": "1", + "PublicDescription": "Counts anytime a AD packet is added to Egress", + "UMask": "0x1", + "Unit": "M2HBM" + }, + { + "BriefDescription": "AD Egress (to CMS) Occupancy", + "EventCode": "0x07", + "EventName": "UNC_M2HBM_TxC_AD_OCCUPANCY", + "PerPkg": "1", + "Unit": "M2HBM" + }, + { + "BriefDescription": "BL Egress (to CMS) : Inserts - CMS0 - Near Side", + "EventCode": "0x0E", + "EventName": "UNC_M2HBM_TxC_BL.INSERTS_CMS0", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "PublicDescription": "Counts the number of BL transactions to CMS add port 0", + "UMask": "0x101", + "Unit": "M2HBM" + }, + { + "BriefDescription": "BL Egress (to CMS) : Inserts - CMS1 - Far Side", + "EventCode": "0x0E", + "EventName": "UNC_M2HBM_TxC_BL.INSERTS_CMS1", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "PublicDescription": "Counts the number of BL transactions to CMS add port 1", + "UMask": "0x201", + "Unit": "M2HBM" + }, + { + "BriefDescription": "BL Egress (to CMS) Occupancy : All", + "EventCode": "0x0f", + "EventName": "UNC_M2HBM_TxC_BL_OCCUPANCY.ALL", + "PerPkg": "1", + "UMask": "0x3", + "Unit": "M2HBM" + }, + { + "BriefDescription": "BL Egress (to CMS) Occupancy : Common Mesh Stop - Near Side", + "EventCode": "0x0f", + "EventName": "UNC_M2HBM_TxC_BL_OCCUPANCY.CMS0", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2HBM" + }, + { + "BriefDescription": "BL Egress (to CMS) Occupancy : Common Mesh Stop - Far Side", + "EventCode": "0x0f", + "EventName": "UNC_M2HBM_TxC_BL_OCCUPANCY.CMS1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2HBM" + }, + { + "BriefDescription": "WPQ Flush : Channel 0", + "EventCode": "0x42", + "EventName": "UNC_M2HBM_WPQ_FLUSH.CH0", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2HBM" + }, + { + "BriefDescription": "WPQ Flush : Channel 1", + "EventCode": "0x42", + "EventName": "UNC_M2HBM_WPQ_FLUSH.CH1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2HBM" + }, + { + "BriefDescription": "M2M and iMC WPQ Cycles w/Credits - Regular : Channel 0", + "EventCode": "0x37", + "EventName": "UNC_M2HBM_WPQ_NO_REG_CRD.CHN0", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2HBM" + }, + { + "BriefDescription": "M2M and iMC WPQ Cycles w/Credits - Regular : Channel 1", + "EventCode": "0x37", + "EventName": "UNC_M2HBM_WPQ_NO_REG_CRD.CHN1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2HBM" + }, + { + "BriefDescription": "M2M and iMC WPQ Cycles w/Credits - Special : Channel 0", + "EventCode": "0x38", + "EventName": "UNC_M2HBM_WPQ_NO_SPEC_CRD.CHN0", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2HBM" + }, + { + "BriefDescription": "M2M and iMC WPQ Cycles w/Credits - Special : Channel 1", + "EventCode": "0x38", + "EventName": "UNC_M2HBM_WPQ_NO_SPEC_CRD.CHN1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Write Tracker Inserts : Channel 0", + "EventCode": "0x40", + "EventName": "UNC_M2HBM_WR_TRACKER_INSERTS.CH0", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Write Tracker Inserts : Channel 1", + "EventCode": "0x40", + "EventName": "UNC_M2HBM_WR_TRACKER_INSERTS.CH1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Write Tracker Non-Posted Inserts : Channel 0", + "EventCode": "0x4d", + "EventName": "UNC_M2HBM_WR_TRACKER_NONPOSTED_INSERTS.CH0", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Write Tracker Non-Posted Inserts : Channel 1", + "EventCode": "0x4d", + "EventName": "UNC_M2HBM_WR_TRACKER_NONPOSTED_INSERTS.CH1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Write Tracker Non-Posted Occupancy : Channel 0", + "EventCode": "0x4c", + "EventName": "UNC_M2HBM_WR_TRACKER_NONPOSTED_OCCUPANCY.CH0", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Write Tracker Non-Posted Occupancy : Channel 1", + "EventCode": "0x4c", + "EventName": "UNC_M2HBM_WR_TRACKER_NONPOSTED_OCCUPANCY.CH1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Write Tracker Posted Inserts : Channel 0", + "EventCode": "0x48", + "EventName": "UNC_M2HBM_WR_TRACKER_POSTED_INSERTS.CH0", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Write Tracker Posted Inserts : Channel 1", + "EventCode": "0x48", + "EventName": "UNC_M2HBM_WR_TRACKER_POSTED_INSERTS.CH1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Write Tracker Posted Occupancy : Channel 0", + "EventCode": "0x47", + "EventName": "UNC_M2HBM_WR_TRACKER_POSTED_OCCUPANCY.CH0", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Write Tracker Posted Occupancy : Channel 1", + "EventCode": "0x47", + "EventName": "UNC_M2HBM_WR_TRACKER_POSTED_OCCUPANCY.CH1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "M2HBM" + }, + { + "BriefDescription": "Activate due to read, write, underfill, or bypass", + "EventCode": "0x02", + "EventName": "UNC_MCHBM_ACT_COUNT.ALL", + "PerPkg": "1", + "PublicDescription": "Counts the number of HBM Activate commands sent on this channel. Activate commands are issued to open up a page on the HBM devices so that it can be read or written to with a CAS. One can calculate the number of Page Misses by subtracting the number of Page Miss precharges from the number of Activates.", + "UMask": "0xff", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Activate due to read", + "EventCode": "0x02", + "EventName": "UNC_MCHBM_ACT_COUNT.RD", + "PerPkg": "1", + "PublicDescription": "Counts the number of HBM Activate commands sent on this channel. Activate commands are issued to open up a page on the HBM devices so that it can be read or written to with a CAS. One can calculate the number of Page Misses by subtracting the number of Page Miss precharges from the number of Activates.", + "UMask": "0x11", + "Unit": "MCHBM" + }, + { + "BriefDescription": "HBM Activate Count : Activate due to Read in PCH0", + "EventCode": "0x02", + "EventName": "UNC_MCHBM_ACT_COUNT.RD_PCH0", + "PerPkg": "1", + "PublicDescription": "Counts the number of HBM Activate commands sent on this channel. Activate commands are issued to open up a page on the HBM devices so that it can be read or written to with a CAS. One can calculate the number of Page Misses by subtracting the number of Page Miss precharges from the number of Activates.", + "UMask": "0x1", + "Unit": "MCHBM" + }, + { + "BriefDescription": "HBM Activate Count : Activate due to Read in PCH1", + "EventCode": "0x02", + "EventName": "UNC_MCHBM_ACT_COUNT.RD_PCH1", + "PerPkg": "1", + "PublicDescription": "Counts the number of HBM Activate commands sent on this channel. Activate commands are issued to open up a page on the HBM devices so that it can be read or written to with a CAS. One can calculate the number of Page Misses by subtracting the number of Page Miss precharges from the number of Activates.", + "UMask": "0x10", + "Unit": "MCHBM" + }, + { + "BriefDescription": "HBM Activate Count : Underfill Read transaction on Page Empty or Page Miss", + "EventCode": "0x02", + "EventName": "UNC_MCHBM_ACT_COUNT.UFILL", + "PerPkg": "1", + "PublicDescription": "Counts the number of HBM Activate commands sent on this channel. Activate commands are issued to open up a page on the HBM devices so that it can be read or written to with a CAS. One can calculate the number of Page Misses by subtracting the number of Page Miss precharges from the number of Activates.", + "UMask": "0x44", + "Unit": "MCHBM" + }, + { + "BriefDescription": "HBM Activate Count", + "EventCode": "0x02", + "EventName": "UNC_MCHBM_ACT_COUNT.UFILL_PCH0", + "PerPkg": "1", + "PublicDescription": "Counts the number of HBM Activate commands sent on this channel. Activate commands are issued to open up a page on the HBM devices so that it can be read or written to with a CAS. One can calculate the number of Page Misses by subtracting the number of Page Miss precharges from the number of Activates.", + "UMask": "0x4", + "Unit": "MCHBM" + }, + { + "BriefDescription": "HBM Activate Count", + "EventCode": "0x02", + "EventName": "UNC_MCHBM_ACT_COUNT.UFILL_PCH1", + "PerPkg": "1", + "PublicDescription": "Counts the number of HBM Activate commands sent on this channel. Activate commands are issued to open up a page on the HBM devices so that it can be read or written to with a CAS. One can calculate the number of Page Misses by subtracting the number of Page Miss precharges from the number of Activates.", + "UMask": "0x40", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Activate due to write", + "EventCode": "0x02", + "EventName": "UNC_MCHBM_ACT_COUNT.WR", + "PerPkg": "1", + "PublicDescription": "Counts the number of HBM Activate commands sent on this channel. Activate commands are issued to open up a page on the HBM devices so that it can be read or written to with a CAS. One can calculate the number of Page Misses by subtracting the number of Page Miss precharges from the number of Activates.", + "UMask": "0x22", + "Unit": "MCHBM" + }, + { + "BriefDescription": "HBM Activate Count : Activate due to Write in PCH0", + "EventCode": "0x02", + "EventName": "UNC_MCHBM_ACT_COUNT.WR_PCH0", + "PerPkg": "1", + "PublicDescription": "Counts the number of HBM Activate commands sent on this channel. Activate commands are issued to open up a page on the HBM devices so that it can be read or written to with a CAS. One can calculate the number of Page Misses by subtracting the number of Page Miss precharges from the number of Activates.", + "UMask": "0x2", + "Unit": "MCHBM" + }, + { + "BriefDescription": "HBM Activate Count : Activate due to Write in PCH1", + "EventCode": "0x02", + "EventName": "UNC_MCHBM_ACT_COUNT.WR_PCH1", + "PerPkg": "1", + "PublicDescription": "Counts the number of HBM Activate commands sent on this channel. Activate commands are issued to open up a page on the HBM devices so that it can be read or written to with a CAS. One can calculate the number of Page Misses by subtracting the number of Page Miss precharges from the number of Activates.", + "UMask": "0x20", + "Unit": "MCHBM" + }, + { + "BriefDescription": "All CAS commands issued", + "EventCode": "0x05", + "EventName": "UNC_MCHBM_CAS_COUNT.ALL", + "PerPkg": "1", + "UMask": "0xff", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Pseudo Channel 0", + "EventCode": "0x05", + "EventName": "UNC_MCHBM_CAS_COUNT.PCH0", + "PerPkg": "1", + "PublicDescription": "HBM RD_CAS and WR_CAS Commands", + "UMask": "0x40", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Pseudo Channel 1", + "EventCode": "0x05", + "EventName": "UNC_MCHBM_CAS_COUNT.PCH1", + "PerPkg": "1", + "PublicDescription": "HBM RD_CAS and WR_CAS Commands", + "UMask": "0x80", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Read CAS commands issued (regular and underfill)", + "EventCode": "0x05", + "EventName": "UNC_MCHBM_CAS_COUNT.RD", + "PerPkg": "1", + "UMask": "0xcf", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Regular read CAS commands with precharge", + "EventCode": "0x05", + "EventName": "UNC_MCHBM_CAS_COUNT.RD_PRE_REG", + "PerPkg": "1", + "UMask": "0xc2", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Underfill read CAS commands with precharge", + "EventCode": "0x05", + "EventName": "UNC_MCHBM_CAS_COUNT.RD_PRE_UNDERFILL", + "PerPkg": "1", + "UMask": "0xc8", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Regular read CAS commands issued (does not include underfills)", + "EventCode": "0x05", + "EventName": "UNC_MCHBM_CAS_COUNT.RD_REG", + "PerPkg": "1", + "UMask": "0xc1", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Underfill read CAS commands issued", + "EventCode": "0x05", + "EventName": "UNC_MCHBM_CAS_COUNT.RD_UNDERFILL", + "PerPkg": "1", + "UMask": "0xc4", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Write CAS commands issued", + "EventCode": "0x05", + "EventName": "UNC_MCHBM_CAS_COUNT.WR", + "PerPkg": "1", + "UMask": "0xf0", + "Unit": "MCHBM" + }, + { + "BriefDescription": "HBM RD_CAS and WR_CAS Commands. : HBM WR_CAS commands w/o auto-pre", + "EventCode": "0x05", + "EventName": "UNC_MCHBM_CAS_COUNT.WR_NONPRE", + "PerPkg": "1", + "UMask": "0xd0", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Write CAS commands with precharge", + "EventCode": "0x05", + "EventName": "UNC_MCHBM_CAS_COUNT.WR_PRE", + "PerPkg": "1", + "UMask": "0xe0", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Pseudo Channel 0", + "EventCode": "0x06", + "EventName": "UNC_MCHBM_CAS_ISSUED_REQ_LEN.PCH0", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Pseudo Channel 1", + "EventCode": "0x06", + "EventName": "UNC_MCHBM_CAS_ISSUED_REQ_LEN.PCH1", + "PerPkg": "1", + "UMask": "0x80", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Read CAS Command in Interleaved Mode (32B)", + "EventCode": "0x06", + "EventName": "UNC_MCHBM_CAS_ISSUED_REQ_LEN.RD_32B", + "PerPkg": "1", + "UMask": "0xc8", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Read CAS Command in Regular Mode (64B) in Pseudochannel 0", + "EventCode": "0x06", + "EventName": "UNC_MCHBM_CAS_ISSUED_REQ_LEN.RD_64B", + "PerPkg": "1", + "UMask": "0xc1", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Underfill Read CAS Command in Interleaved Mode (32B)", + "EventCode": "0x06", + "EventName": "UNC_MCHBM_CAS_ISSUED_REQ_LEN.RD_UFILL_32B", + "PerPkg": "1", + "UMask": "0xd0", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Underfill Read CAS Command in Regular Mode (64B) in Pseudochannel 1", + "EventCode": "0x06", + "EventName": "UNC_MCHBM_CAS_ISSUED_REQ_LEN.RD_UFILL_64B", + "PerPkg": "1", + "UMask": "0xc2", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Write CAS Command in Interleaved Mode (32B)", + "EventCode": "0x06", + "EventName": "UNC_MCHBM_CAS_ISSUED_REQ_LEN.WR_32B", + "PerPkg": "1", + "UMask": "0xe0", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Write CAS Command in Regular Mode (64B) in Pseudochannel 0", + "EventCode": "0x06", + "EventName": "UNC_MCHBM_CAS_ISSUED_REQ_LEN.WR_64B", + "PerPkg": "1", + "UMask": "0xc4", + "Unit": "MCHBM" + }, + { + "BriefDescription": "IMC Clockticks at DCLK frequency", + "EventCode": "0x01", + "EventName": "UNC_MCHBM_CLOCKTICKS", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "MCHBM" + }, + { + "BriefDescription": "HBM Precharge All Commands", + "EventCode": "0x44", + "EventName": "UNC_MCHBM_HBM_PREALL.PCH0", + "PerPkg": "1", + "PublicDescription": "Counts the number of times that the precharge all command was sent.", + "UMask": "0x1", + "Unit": "MCHBM" + }, + { + "BriefDescription": "HBM Precharge All Commands", + "EventCode": "0x44", + "EventName": "UNC_MCHBM_HBM_PREALL.PCH1", + "PerPkg": "1", + "PublicDescription": "Counts the number of times that the precharge all command was sent.", + "UMask": "0x2", + "Unit": "MCHBM" + }, + { + "BriefDescription": "All Precharge Commands", + "EventCode": "0x44", + "EventName": "UNC_MCHBM_HBM_PRE_ALL", + "PerPkg": "1", + "PublicDescription": "Precharge All Commands: Counts the number of times that the precharge all command was sent.", + "UMask": "0x3", + "Unit": "MCHBM" + }, + { + "BriefDescription": "IMC Clockticks at HCLK frequency", + "EventCode": "0x01", + "EventName": "UNC_MCHBM_HCLOCKTICKS", + "PerPkg": "1", + "Unit": "MCHBM" + }, + { + "BriefDescription": "All precharge events", + "EventCode": "0x03", + "EventName": "UNC_MCHBM_PRE_COUNT.ALL", + "PerPkg": "1", + "PublicDescription": "Counts the number of HBM Precharge commands sent on this channel.", + "UMask": "0xff", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Precharge from MC page table", + "EventCode": "0x03", + "EventName": "UNC_MCHBM_PRE_COUNT.PGT", + "PerPkg": "1", + "PublicDescription": "Counts the number of HBM Precharge commands sent on this channel.", + "UMask": "0x88", + "Unit": "MCHBM" + }, + { + "BriefDescription": "HBM Precharge commands. : Precharges from Page Table", + "EventCode": "0x03", + "EventName": "UNC_MCHBM_PRE_COUNT.PGT_PCH0", + "PerPkg": "1", + "PublicDescription": "Counts the number of HBM Precharge commands sent on this channel. : Equivalent to PAGE_EMPTY", + "UMask": "0x8", + "Unit": "MCHBM" + }, + { + "BriefDescription": "HBM Precharge commands.", + "EventCode": "0x03", + "EventName": "UNC_MCHBM_PRE_COUNT.PGT_PCH1", + "PerPkg": "1", + "PublicDescription": "Counts the number of HBM Precharge commands sent on this channel.", + "UMask": "0x80", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Precharge due to read on page miss", + "EventCode": "0x03", + "EventName": "UNC_MCHBM_PRE_COUNT.RD", + "PerPkg": "1", + "PublicDescription": "Counts the number of HBM Precharge commands sent on this channel.", + "UMask": "0x11", + "Unit": "MCHBM" + }, + { + "BriefDescription": "HBM Precharge commands. : Precharge due to read", + "EventCode": "0x03", + "EventName": "UNC_MCHBM_PRE_COUNT.RD_PCH0", + "PerPkg": "1", + "PublicDescription": "Counts the number of HBM Precharge commands sent on this channel. : Precharge from read bank scheduler", + "UMask": "0x1", + "Unit": "MCHBM" + }, + { + "BriefDescription": "HBM Precharge commands.", + "EventCode": "0x03", + "EventName": "UNC_MCHBM_PRE_COUNT.RD_PCH1", + "PerPkg": "1", + "PublicDescription": "Counts the number of HBM Precharge commands sent on this channel.", + "UMask": "0x10", + "Unit": "MCHBM" + }, + { + "BriefDescription": "HBM Precharge commands.", + "EventCode": "0x03", + "EventName": "UNC_MCHBM_PRE_COUNT.UFILL", + "PerPkg": "1", + "PublicDescription": "Counts the number of HBM Precharge commands sent on this channel.", + "UMask": "0x44", + "Unit": "MCHBM" + }, + { + "BriefDescription": "HBM Precharge commands.", + "EventCode": "0x03", + "EventName": "UNC_MCHBM_PRE_COUNT.UFILL_PCH0", + "PerPkg": "1", + "PublicDescription": "Counts the number of HBM Precharge commands sent on this channel.", + "UMask": "0x4", + "Unit": "MCHBM" + }, + { + "BriefDescription": "HBM Precharge commands.", + "EventCode": "0x03", + "EventName": "UNC_MCHBM_PRE_COUNT.UFILL_PCH1", + "PerPkg": "1", + "PublicDescription": "Counts the number of HBM Precharge commands sent on this channel.", + "UMask": "0x40", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Precharge due to write on page miss", + "EventCode": "0x03", + "EventName": "UNC_MCHBM_PRE_COUNT.WR", + "PerPkg": "1", + "PublicDescription": "Counts the number of HBM Precharge commands sent on this channel.", + "UMask": "0x22", + "Unit": "MCHBM" + }, + { + "BriefDescription": "HBM Precharge commands. : Precharge due to write", + "EventCode": "0x03", + "EventName": "UNC_MCHBM_PRE_COUNT.WR_PCH0", + "PerPkg": "1", + "PublicDescription": "Counts the number of HBM Precharge commands sent on this channel. : Precharge from write bank scheduler", + "UMask": "0x2", + "Unit": "MCHBM" + }, + { + "BriefDescription": "HBM Precharge commands.", + "EventCode": "0x03", + "EventName": "UNC_MCHBM_PRE_COUNT.WR_PCH1", + "PerPkg": "1", + "PublicDescription": "Counts the number of HBM Precharge commands sent on this channel.", + "UMask": "0x20", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Counts the number of cycles where the read buffer has greater than UMASK elements. NOTE: Umask must be set to the maximum number of elements in the queue (24 entries for SPR).", + "EventCode": "0x19", + "EventName": "UNC_MCHBM_RDB_FULL", + "PerPkg": "1", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Counts the number of inserts into the read buffer.", + "EventCode": "0x17", + "EventName": "UNC_MCHBM_RDB_INSERTS", + "PerPkg": "1", + "UMask": "0x3", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Read Data Buffer Inserts", + "EventCode": "0x17", + "EventName": "UNC_MCHBM_RDB_INSERTS.PCH0", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Read Data Buffer Inserts", + "EventCode": "0x17", + "EventName": "UNC_MCHBM_RDB_INSERTS.PCH1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Counts the number of elements in the read buffer per cycle.", + "EventCode": "0x1a", + "EventName": "UNC_MCHBM_RDB_OCCUPANCY", + "PerPkg": "1", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Read Pending Queue Allocations", + "EventCode": "0x10", + "EventName": "UNC_MCHBM_RPQ_INSERTS.PCH0", + "PerPkg": "1", + "PublicDescription": "Read Pending Queue Allocations: Counts the number of allocations into the Read Pending Queue. This queue is used to schedule reads out to the memory controller and to track the requests. Requests allocate into the RPQ soon after they enter the memory controller, and need credits for an entry in this buffer before being sent from the HA to the iMC. They deallocate after the CAS command has been issued to memory. This includes both ISOCH and non-ISOCH requests.", + "UMask": "0x1", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Read Pending Queue Allocations", + "EventCode": "0x10", + "EventName": "UNC_MCHBM_RPQ_INSERTS.PCH1", + "PerPkg": "1", + "PublicDescription": "Read Pending Queue Allocations: Counts the number of allocations into the Read Pending Queue. This queue is used to schedule reads out to the memory controller and to track the requests. Requests allocate into the RPQ soon after they enter the memory controller, and need credits for an entry in this buffer before being sent from the HA to the iMC. They deallocate after the CAS command has been issued to memory. This includes both ISOCH and non-ISOCH requests.", + "UMask": "0x2", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Read Pending Queue Occupancy", + "EventCode": "0x80", + "EventName": "UNC_MCHBM_RPQ_OCCUPANCY_PCH0", + "PerPkg": "1", + "PublicDescription": "Read Pending Queue Occupancy: Accumulates the occupancies of the Read Pending Queue each cycle. This can then be used to calculate both the average occupancy (in conjunction with the number of cycles not empty) and the average latency (in conjunction with the number of allocations). The RPQ is used to schedule reads out to the memory controller and to track the requests. Requests allocate into the RPQ soon after they enter the memory controller, and need credits for an entry in this buffer before being sent from the HA to the iMC. They deallocate after the CAS command has been issued to memory.", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Read Pending Queue Occupancy", + "EventCode": "0x81", + "EventName": "UNC_MCHBM_RPQ_OCCUPANCY_PCH1", + "PerPkg": "1", + "PublicDescription": "Read Pending Queue Occupancy: Accumulates the occupancies of the Read Pending Queue each cycle. This can then be used to calculate both the average occupancy (in conjunction with the number of cycles not empty) and the average latency (in conjunction with the number of allocations). The RPQ is used to schedule reads out to the memory controller and to track the requests. Requests allocate into the RPQ soon after they enter the memory controller, and need credits for an entry in this buffer before being sent from the HA to the iMC. They deallocate after the CAS command has been issued to memory.", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Write Pending Queue Allocations", + "EventCode": "0x20", + "EventName": "UNC_MCHBM_WPQ_INSERTS.PCH0", + "PerPkg": "1", + "PublicDescription": "Write Pending Queue Allocations: Counts the number of allocations into the Write Pending Queue. This can then be used to calculate the average queuing latency (in conjunction with the WPQ occupancy count). The WPQ is used to schedule write out to the memory controller and to track the writes. Requests allocate into the WPQ soon after they enter the memory controller, and need credits for an entry in this buffer before being sent from the CHA to the iMC. They deallocate after being issued. Write requests themselves are able to complete (from the perspective of the rest of the system) as soon they have posted to the iMC.", + "UMask": "0x1", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Write Pending Queue Allocations", + "EventCode": "0x20", + "EventName": "UNC_MCHBM_WPQ_INSERTS.PCH1", + "PerPkg": "1", + "PublicDescription": "Write Pending Queue Allocations: Counts the number of allocations into the Write Pending Queue. This can then be used to calculate the average queuing latency (in conjunction with the WPQ occupancy count). The WPQ is used to schedule write out to the memory controller and to track the writes. Requests allocate into the WPQ soon after they enter the memory controller, and need credits for an entry in this buffer before being sent from the CHA to the iMC. They deallocate after being issued. Write requests themselves are able to complete (from the perspective of the rest of the system) as soon they have posted to the iMC.", + "UMask": "0x2", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Write Pending Queue Occupancy", + "EventCode": "0x82", + "EventName": "UNC_MCHBM_WPQ_OCCUPANCY_PCH0", + "PerPkg": "1", + "PublicDescription": "Write Pending Queue Occupancy: Accumulates the occupancies of the Write Pending Queue each cycle. This can then be used to calculate both the average queue occupancy (in conjunction with the number of cycles not empty) and the average latency (in conjunction with the number of allocations). The WPQ is used to schedule write out to the memory controller and to track the writes. Requests allocate into the WPQ soon after they enter the memory controller, and need credits for an entry in this buffer before being sent from the HA to the iMC. They deallocate after being issued to memory. Write requests themselves are able to complete (from the perspective of the rest of the system) as soon they have posted to the iMC. This is not to be confused with actually performing the write. Therefore, the average latency for this queue is actually not useful for deconstruction intermediate write latencies. So, we provide filtering based on if the request has posted or not. By using the not posted filter, we can track how long writes spent in the iMC before completions were sent to the HA. The posted filter, on the other hand, provides information about how much queueing is actually happening in the iMC for writes before they are actually issued to memory. High average occupancies will generally coincide with high write major mode counts.", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Write Pending Queue Occupancy", + "EventCode": "0x83", + "EventName": "UNC_MCHBM_WPQ_OCCUPANCY_PCH1", + "PerPkg": "1", + "PublicDescription": "Write Pending Queue Occupancy: Accumulates the occupancies of the Write Pending Queue each cycle. This can then be used to calculate both the average queue occupancy (in conjunction with the number of cycles not empty) and the average latency (in conjunction with the number of allocations). The WPQ is used to schedule write out to the memory controller and to track the writes. Requests allocate into the WPQ soon after they enter the memory controller, and need credits for an entry in this buffer before being sent from the HA to the iMC. They deallocate after being issued to memory. Write requests themselves are able to complete (from the perspective of the rest of the system) as soon they have posted to the iMC. This is not to be confused with actually performing the write. Therefore, the average latency for this queue is actually not useful for deconstruction intermediate write latencies. So, we provide filtering based on if the request has posted or not. By using the not posted filter, we can track how long writes spent in the iMC before completions were sent to the HA. The posted filter, on the other hand, provides information about how much queueing is actually happening in the iMC for writes before they are actually issued to memory. High average occupancies will generally coincide with high write major mode counts.", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Write Pending Queue CAM Match", + "EventCode": "0x23", + "EventName": "UNC_MCHBM_WPQ_READ_HIT", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "PublicDescription": "Counts the number of times a request hits in the WPQ (write-pending queue). The iMC allows writes and reads to pass up other writes to different addresses. Before a read or a write is issued, it will first CAM the WPQ to see if there is a write pending to that address. When reads hit, they are able to directly pull their data from the WPQ instead of going to memory. Writes that hit will overwrite the existing data. Partial writes that hit will not need to do underfill reads and will simply update their relevant sections.", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Write Pending Queue CAM Match", + "EventCode": "0x23", + "EventName": "UNC_MCHBM_WPQ_READ_HIT.PCH0", + "PerPkg": "1", + "PublicDescription": "Write Pending Queue CAM Match: Counts the number of times a request hits in the WPQ (write-pending queue). The iMC allows writes and reads to pass up other writes to different addresses. Before a read or a write is issued, it will first CAM the WPQ to see if there is a write pending to that address. When reads hit, they are able to directly pull their data from the WPQ instead of going to memory. Writes that hit will overwrite the existing data. Partial writes that hit will not need to do underfill reads and will simply update their relevant sections.", + "UMask": "0x1", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Write Pending Queue CAM Match", + "EventCode": "0x23", + "EventName": "UNC_MCHBM_WPQ_READ_HIT.PCH1", + "PerPkg": "1", + "PublicDescription": "Write Pending Queue CAM Match: Counts the number of times a request hits in the WPQ (write-pending queue). The iMC allows writes and reads to pass up other writes to different addresses. Before a read or a write is issued, it will first CAM the WPQ to see if there is a write pending to that address. When reads hit, they are able to directly pull their data from the WPQ instead of going to memory. Writes that hit will overwrite the existing data. Partial writes that hit will not need to do underfill reads and will simply update their relevant sections.", + "UMask": "0x2", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Write Pending Queue CAM Match", + "EventCode": "0x24", + "EventName": "UNC_MCHBM_WPQ_WRITE_HIT", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "PublicDescription": "Counts the number of times a request hits in the WPQ (write-pending queue). The iMC allows writes and reads to pass up other writes to different addresses. Before a read or a write is issued, it will first CAM the WPQ to see if there is a write pending to that address. When reads hit, they are able to directly pull their data from the WPQ instead of going to memory. Writes that hit will overwrite the existing data. Partial writes that hit will not need to do underfill reads and will simply update their relevant sections.", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Write Pending Queue CAM Match", + "EventCode": "0x24", + "EventName": "UNC_MCHBM_WPQ_WRITE_HIT.PCH0", + "PerPkg": "1", + "PublicDescription": "Write Pending Queue CAM Match: Counts the number of times a request hits in the WPQ (write-pending queue). The iMC allows writes and reads to pass up other writes to different addresses. Before a read or a write is issued, it will first CAM the WPQ to see if there is a write pending to that address. When reads hit, they are able to directly pull their data from the WPQ instead of going to memory. Writes that hit will overwrite the existing data. Partial writes that hit will not need to do underfill reads and will simply update their relevant sections.", + "UMask": "0x1", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Write Pending Queue CAM Match", + "EventCode": "0x24", + "EventName": "UNC_MCHBM_WPQ_WRITE_HIT.PCH1", + "PerPkg": "1", + "PublicDescription": "Write Pending Queue CAM Match: Counts the number of times a request hits in the WPQ (write-pending queue). The iMC allows writes and reads to pass up other writes to different addresses. Before a read or a write is issued, it will first CAM the WPQ to see if there is a write pending to that address. When reads hit, they are able to directly pull their data from the WPQ instead of going to memory. Writes that hit will overwrite the existing data. Partial writes that hit will not need to do underfill reads and will simply update their relevant sections.", + "UMask": "0x2", + "Unit": "MCHBM" + }, + { + "BriefDescription": "Activate due to read, write, underfill, or bypass", + "EventCode": "0x02", + "EventName": "UNC_M_ACT_COUNT.ALL", + "PerPkg": "1", + "PublicDescription": "DRAM Activate Count : Counts the number of DRAM Activate commands sent on this channel. Activate commands are issued to open up a page on the DRAM devices so that it can be read or written to with a CAS. One can calculate the number of Page Misses by subtracting the number of Page Miss precharges from the number of Activates.", + "UMask": "0xff", + "Unit": "iMC" + }, + { + "BriefDescription": "All DRAM CAS commands issued", + "EventCode": "0x05", + "EventName": "UNC_M_CAS_COUNT.ALL", + "PerPkg": "1", + "PublicDescription": "DRAM RD_CAS and WR_CAS Commands. : All DRAM Read and Write actions : DRAM RD_CAS and WR_CAS Commands : Counts the total number of DRAM CAS commands issued on this channel.", + "UMask": "0xff", + "Unit": "iMC" + }, + { + "BriefDescription": "DRAM RD_CAS and WR_CAS Commands. : Pseudo Channel 0", + "EventCode": "0x05", + "EventName": "UNC_M_CAS_COUNT.PCH0", + "PerPkg": "1", + "PublicDescription": "DRAM RD_CAS and WR_CAS Commands. : Pseudo Channel 0 : DRAM RD_CAS and WR_CAS Commands", + "UMask": "0x40", + "Unit": "iMC" + }, + { + "BriefDescription": "DRAM RD_CAS and WR_CAS Commands. : Pseudo Channel 1", + "EventCode": "0x05", + "EventName": "UNC_M_CAS_COUNT.PCH1", + "PerPkg": "1", + "PublicDescription": "DRAM RD_CAS and WR_CAS Commands. : Pseudo Channel 1 : DRAM RD_CAS and WR_CAS Commands", + "UMask": "0x80", + "Unit": "iMC" + }, + { + "BriefDescription": "All DRAM read CAS commands issued (including underfills)", + "EventCode": "0x05", + "EventName": "UNC_M_CAS_COUNT.RD", + "PerPkg": "1", + "PublicDescription": "DRAM RD_CAS and WR_CAS Commands : Counts the total number of DRAM Read CAS commands issued on this channel. This includes underfills.", + "UMask": "0xcf", + "Unit": "iMC" + }, + { + "BriefDescription": "DRAM RD_CAS and WR_CAS Commands.", + "EventCode": "0x05", + "EventName": "UNC_M_CAS_COUNT.RD_PRE_REG", + "PerPkg": "1", + "PublicDescription": "DRAM RD_CAS and WR_CAS Commands. : DRAM RD_CAS and WR_CAS Commands", + "UMask": "0xc2", + "Unit": "iMC" + }, + { + "BriefDescription": "DRAM RD_CAS and WR_CAS Commands.", + "EventCode": "0x05", + "EventName": "UNC_M_CAS_COUNT.RD_PRE_UNDERFILL", + "PerPkg": "1", + "PublicDescription": "DRAM RD_CAS and WR_CAS Commands. : DRAM RD_CAS and WR_CAS Commands", + "UMask": "0xc8", + "Unit": "iMC" + }, + { + "BriefDescription": "All DRAM read CAS commands issued (does not include underfills)", + "EventCode": "0x05", + "EventName": "UNC_M_CAS_COUNT.RD_REG", + "PerPkg": "1", + "PublicDescription": "DRAM RD_CAS and WR_CAS Commands. : DRAM RD_CAS commands w/out auto-pre : DRAM RD_CAS and WR_CAS Commands : Counts the total number or DRAM Read CAS commands issued on this channel. This includes both regular RD CAS commands as well as those with implicit Precharge. We do not filter based on major mode, as RD_CAS is not issued during WMM (with the exception of underfills).", + "UMask": "0xc1", + "Unit": "iMC" + }, + { + "BriefDescription": "DRAM underfill read CAS commands issued", + "EventCode": "0x05", + "EventName": "UNC_M_CAS_COUNT.RD_UNDERFILL", + "PerPkg": "1", + "PublicDescription": "DRAM RD_CAS and WR_CAS Commands. : Underfill Read Issued : DRAM RD_CAS and WR_CAS Commands", + "UMask": "0xc4", + "Unit": "iMC" + }, + { + "BriefDescription": "All DRAM write CAS commands issued", + "EventCode": "0x05", + "EventName": "UNC_M_CAS_COUNT.WR", + "PerPkg": "1", + "PublicDescription": "DRAM RD_CAS and WR_CAS Commands : Counts the total number of DRAM Write CAS commands issued on this channel.", + "UMask": "0xf0", + "Unit": "iMC" + }, + { + "BriefDescription": "DRAM RD_CAS and WR_CAS Commands. : DRAM WR_CAS commands w/o auto-pre", + "EventCode": "0x05", + "EventName": "UNC_M_CAS_COUNT.WR_NONPRE", + "PerPkg": "1", + "PublicDescription": "DRAM RD_CAS and WR_CAS Commands. : DRAM WR_CAS commands w/o auto-pre : DRAM RD_CAS and WR_CAS Commands", + "UMask": "0xd0", + "Unit": "iMC" + }, + { + "BriefDescription": "DRAM RD_CAS and WR_CAS Commands.", + "EventCode": "0x05", + "EventName": "UNC_M_CAS_COUNT.WR_PRE", + "PerPkg": "1", + "PublicDescription": "DRAM RD_CAS and WR_CAS Commands. : DRAM RD_CAS and WR_CAS Commands", + "UMask": "0xe0", + "Unit": "iMC" + }, + { + "BriefDescription": "Pseudo Channel 0", + "EventCode": "0x06", + "EventName": "UNC_M_CAS_ISSUED_REQ_LEN.PCH0", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "iMC" + }, + { + "BriefDescription": "Pseudo Channel 1", + "EventCode": "0x06", + "EventName": "UNC_M_CAS_ISSUED_REQ_LEN.PCH1", + "PerPkg": "1", + "UMask": "0x80", + "Unit": "iMC" + }, + { + "BriefDescription": "Read CAS Command in Interleaved Mode (32B)", + "EventCode": "0x06", + "EventName": "UNC_M_CAS_ISSUED_REQ_LEN.RD_32B", + "PerPkg": "1", + "UMask": "0xc8", + "Unit": "iMC" + }, + { + "BriefDescription": "Read CAS Command in Regular Mode (64B) in Pseudochannel 0", + "EventCode": "0x06", + "EventName": "UNC_M_CAS_ISSUED_REQ_LEN.RD_64B", + "PerPkg": "1", + "UMask": "0xc1", + "Unit": "iMC" + }, + { + "BriefDescription": "Underfill Read CAS Command in Interleaved Mode (32B)", + "EventCode": "0x06", + "EventName": "UNC_M_CAS_ISSUED_REQ_LEN.RD_UFILL_32B", + "PerPkg": "1", + "UMask": "0xd0", + "Unit": "iMC" + }, + { + "BriefDescription": "Underfill Read CAS Command in Regular Mode (64B) in Pseudochannel 1", + "EventCode": "0x06", + "EventName": "UNC_M_CAS_ISSUED_REQ_LEN.RD_UFILL_64B", + "PerPkg": "1", + "UMask": "0xc2", + "Unit": "iMC" + }, + { + "BriefDescription": "Write CAS Command in Interleaved Mode (32B)", + "EventCode": "0x06", + "EventName": "UNC_M_CAS_ISSUED_REQ_LEN.WR_32B", + "PerPkg": "1", + "UMask": "0xe0", + "Unit": "iMC" + }, + { + "BriefDescription": "Write CAS Command in Regular Mode (64B) in Pseudochannel 0", + "EventCode": "0x06", + "EventName": "UNC_M_CAS_ISSUED_REQ_LEN.WR_64B", + "PerPkg": "1", + "UMask": "0xc4", + "Unit": "iMC" + }, + { + "BriefDescription": "IMC Clockticks at DCLK frequency", + "EventCode": "0x01", + "EventName": "UNC_M_CLOCKTICKS", + "PerPkg": "1", + "PublicDescription": "Number of DRAM DCLK clock cycles while the event is enabled", + "UMask": "0x1", + "Unit": "iMC" + }, + { + "BriefDescription": "DRAM Precharge All Commands", + "EventCode": "0x44", + "EventName": "UNC_M_DRAM_PRE_ALL", + "PerPkg": "1", + "PublicDescription": "DRAM Precharge All Commands : Counts the number of times that the precharge all command was sent.", + "UMask": "0x3", + "Unit": "iMC" + }, + { + "BriefDescription": "IMC Clockticks at HCLK frequency", + "EventCode": "0x01", + "EventName": "UNC_M_HCLOCKTICKS", + "PerPkg": "1", + "PublicDescription": "Number of DRAM HCLK clock cycles while the event is enabled", + "Unit": "iMC" + }, + { + "BriefDescription": "UNC_M_PCLS.RD", + "EventCode": "0xa0", + "EventName": "UNC_M_PCLS.RD", + "PerPkg": "1", + "UMask": "0x5", + "Unit": "iMC" + }, + { + "BriefDescription": "UNC_M_PCLS.TOTAL", + "EventCode": "0xa0", + "EventName": "UNC_M_PCLS.TOTAL", + "PerPkg": "1", + "UMask": "0xf", + "Unit": "iMC" + }, + { + "BriefDescription": "UNC_M_PCLS.WR", + "EventCode": "0xa0", + "EventName": "UNC_M_PCLS.WR", + "PerPkg": "1", + "UMask": "0xa", + "Unit": "iMC" + }, + { + "BriefDescription": "PMM Read Pending Queue inserts", + "EventCode": "0xe3", + "EventName": "UNC_M_PMM_RPQ_INSERTS", + "PerPkg": "1", + "PublicDescription": "Counts number of read requests allocated in the PMM Read Pending Queue.", + "Unit": "iMC" + }, + { + "BriefDescription": "PMM Read Pending Queue occupancy", + "EventCode": "0xe0", + "EventName": "UNC_M_PMM_RPQ_OCCUPANCY.ALL_SCH0", + "PerPkg": "1", + "PublicDescription": "Accumulates the per cycle occupancy of the PMM Read Pending Queue.", + "UMask": "0x1", + "Unit": "iMC" + }, + { + "BriefDescription": "PMM Read Pending Queue occupancy", + "EventCode": "0xe0", + "EventName": "UNC_M_PMM_RPQ_OCCUPANCY.ALL_SCH1", + "PerPkg": "1", + "PublicDescription": "Accumulates the per cycle occupancy of the PMM Read Pending Queue.", + "UMask": "0x2", + "Unit": "iMC" + }, + { + "BriefDescription": "PMM Read Pending Queue Occupancy", + "EventCode": "0xE0", + "EventName": "UNC_M_PMM_RPQ_OCCUPANCY.GNT_WAIT_SCH0", + "PerPkg": "1", + "PublicDescription": "PMM Read Pending Queue Occupancy : Accumulates the per cycle occupancy of the PMM Read Pending Queue.", + "UMask": "0x10", + "Unit": "iMC" + }, + { + "BriefDescription": "PMM Read Pending Queue Occupancy", + "EventCode": "0xE0", + "EventName": "UNC_M_PMM_RPQ_OCCUPANCY.GNT_WAIT_SCH1", + "PerPkg": "1", + "PublicDescription": "PMM Read Pending Queue Occupancy : Accumulates the per cycle occupancy of the PMM Read Pending Queue.", + "UMask": "0x20", + "Unit": "iMC" + }, + { + "BriefDescription": "PMM Read Pending Queue Occupancy", + "EventCode": "0xe0", + "EventName": "UNC_M_PMM_RPQ_OCCUPANCY.NO_GNT_SCH0", + "PerPkg": "1", + "PublicDescription": "Accumulates the per cycle occupancy of the PMM Read Pending Queue.", + "UMask": "0x4", + "Unit": "iMC" + }, + { + "BriefDescription": "PMM Read Pending Queue Occupancy", + "EventCode": "0xe0", + "EventName": "UNC_M_PMM_RPQ_OCCUPANCY.NO_GNT_SCH1", + "PerPkg": "1", + "PublicDescription": "Accumulates the per cycle occupancy of the PMM Read Pending Queue.", + "UMask": "0x8", + "Unit": "iMC" + }, + { + "BriefDescription": "PMM (for IXP) Write Queue Cycles Not Empty", + "EventCode": "0xe5", + "EventName": "UNC_M_PMM_WPQ_CYCLES_NE", + "PerPkg": "1", + "Unit": "iMC" + }, + { + "BriefDescription": "PMM Write Pending Queue inserts", + "EventCode": "0xe7", + "EventName": "UNC_M_PMM_WPQ_INSERTS", + "PerPkg": "1", + "PublicDescription": "Counts number of write requests allocated in the PMM Write Pending Queue.", + "Unit": "iMC" + }, + { + "BriefDescription": "PMM Write Pending Queue Occupancy", + "EventCode": "0xe4", + "EventName": "UNC_M_PMM_WPQ_OCCUPANCY.ALL", + "PerPkg": "1", + "PublicDescription": "PMM Write Pending Queue Occupancy : Accumulates the per cycle occupancy of the Write Pending Queue to the PMM DIMM.", + "UMask": "0x3", + "Unit": "iMC" + }, + { + "BriefDescription": "PMM Write Pending Queue Occupancy", + "EventCode": "0xE4", + "EventName": "UNC_M_PMM_WPQ_OCCUPANCY.ALL_SCH0", + "PerPkg": "1", + "PublicDescription": "PMM Write Pending Queue Occupancy : Accumulates the per cycle occupancy of the PMM Write Pending Queue.", + "UMask": "0x1", + "Unit": "iMC" + }, + { + "BriefDescription": "PMM Write Pending Queue Occupancy", + "EventCode": "0xE4", + "EventName": "UNC_M_PMM_WPQ_OCCUPANCY.ALL_SCH1", + "PerPkg": "1", + "PublicDescription": "PMM Write Pending Queue Occupancy : Accumulates the per cycle occupancy of the PMM Write Pending Queue.", + "UMask": "0x2", + "Unit": "iMC" + }, + { + "BriefDescription": "PMM (for IXP) Write Pending Queue Occupancy", + "EventCode": "0xe4", + "EventName": "UNC_M_PMM_WPQ_OCCUPANCY.CAS", + "PerPkg": "1", + "PublicDescription": "PMM (for IXP) Write Pending Queue Occupancy : Accumulates the per cycle occupancy of the Write Pending Queue to the IXP DIMM.", + "UMask": "0xc", + "Unit": "iMC" + }, + { + "BriefDescription": "PMM (for IXP) Write Pending Queue Occupancy", + "EventCode": "0xe4", + "EventName": "UNC_M_PMM_WPQ_OCCUPANCY.PWR", + "PerPkg": "1", + "PublicDescription": "PMM (for IXP) Write Pending Queue Occupancy : Accumulates the per cycle occupancy of the Write Pending Queue to the IXP DIMM.", + "UMask": "0x30", + "Unit": "iMC" + }, + { + "BriefDescription": "Channel PPD Cycles", + "EventCode": "0x85", + "EventName": "UNC_M_POWER_CHANNEL_PPD", + "PerPkg": "1", + "PublicDescription": "Channel PPD Cycles : Number of cycles when all the ranks in the channel are in PPD mode. If IBT=off is enabled, then this can be used to count those cycles. If it is not enabled, then this can count the number of cycles when that could have been taken advantage of.", + "Unit": "iMC" + }, + { + "BriefDescription": "CKE_ON_CYCLES by Rank : DIMM ID", + "EventCode": "0x47", + "EventName": "UNC_M_POWER_CKE_CYCLES.LOW_0", + "PerPkg": "1", + "PublicDescription": "CKE_ON_CYCLES by Rank : DIMM ID : Number of cycles spent in CKE ON mode. The filter allows you to select a rank to monitor. If multiple ranks are in CKE ON mode at one time, the counter will ONLY increment by one rather than doing accumulation. Multiple counters will need to be used to track multiple ranks simultaneously. There is no distinction between the different CKE modes (APD, PPDS, PPDF). This can be determined based on the system programming. These events should commonly be used with Invert to get the number of cycles in power saving mode. Edge Detect is also useful here. Make sure that you do NOT use Invert with Edge Detect (this just confuses the system and is not necessary).", + "UMask": "0x1", + "Unit": "iMC" + }, + { + "BriefDescription": "CKE_ON_CYCLES by Rank : DIMM ID", + "EventCode": "0x47", + "EventName": "UNC_M_POWER_CKE_CYCLES.LOW_1", + "PerPkg": "1", + "PublicDescription": "CKE_ON_CYCLES by Rank : DIMM ID : Number of cycles spent in CKE ON mode. The filter allows you to select a rank to monitor. If multiple ranks are in CKE ON mode at one time, the counter will ONLY increment by one rather than doing accumulation. Multiple counters will need to be used to track multiple ranks simultaneously. There is no distinction between the different CKE modes (APD, PPDS, PPDF). This can be determined based on the system programming. These events should commonly be used with Invert to get the number of cycles in power saving mode. Edge Detect is also useful here. Make sure that you do NOT use Invert with Edge Detect (this just confuses the system and is not necessary).", + "UMask": "0x2", + "Unit": "iMC" + }, + { + "BriefDescription": "CKE_ON_CYCLES by Rank : DIMM ID", + "EventCode": "0x47", + "EventName": "UNC_M_POWER_CKE_CYCLES.LOW_2", + "PerPkg": "1", + "PublicDescription": "CKE_ON_CYCLES by Rank : DIMM ID : Number of cycles spent in CKE ON mode. The filter allows you to select a rank to monitor. If multiple ranks are in CKE ON mode at one time, the counter will ONLY increment by one rather than doing accumulation. Multiple counters will need to be used to track multiple ranks simultaneously. There is no distinction between the different CKE modes (APD, PPDS, PPDF). This can be determined based on the system programming. These events should commonly be used with Invert to get the number of cycles in power saving mode. Edge Detect is also useful here. Make sure that you do NOT use Invert with Edge Detect (this just confuses the system and is not necessary).", + "UMask": "0x4", + "Unit": "iMC" + }, + { + "BriefDescription": "CKE_ON_CYCLES by Rank : DIMM ID", + "EventCode": "0x47", + "EventName": "UNC_M_POWER_CKE_CYCLES.LOW_3", + "PerPkg": "1", + "PublicDescription": "CKE_ON_CYCLES by Rank : DIMM ID : Number of cycles spent in CKE ON mode. The filter allows you to select a rank to monitor. If multiple ranks are in CKE ON mode at one time, the counter will ONLY increment by one rather than doing accumulation. Multiple counters will need to be used to track multiple ranks simultaneously. There is no distinction between the different CKE modes (APD, PPDS, PPDF). This can be determined based on the system programming. These events should commonly be used with Invert to get the number of cycles in power saving mode. Edge Detect is also useful here. Make sure that you do NOT use Invert with Edge Detect (this just confuses the system and is not necessary).", + "UMask": "0x8", + "Unit": "iMC" + }, + { + "BriefDescription": "Throttle Cycles for Rank 0", + "EventCode": "0x86", + "EventName": "UNC_M_POWER_CRIT_THROTTLE_CYCLES.SLOT0", + "PerPkg": "1", + "PublicDescription": "Throttle Cycles for Rank 0 : Counts the number of cycles while the iMC is being throttled by either thermal constraints or by the PCU throttling. It is not possible to distinguish between the two. This can be filtered by rank. If multiple ranks are selected and are being throttled at the same time, the counter will only increment by 1. : Thermal throttling is performed per DIMM. We support 3 DIMMs per channel. This ID allows us to filter by ID.", + "UMask": "0x1", + "Unit": "iMC" + }, + { + "BriefDescription": "Throttle Cycles for Rank 0", + "EventCode": "0x86", + "EventName": "UNC_M_POWER_CRIT_THROTTLE_CYCLES.SLOT1", + "PerPkg": "1", + "PublicDescription": "Throttle Cycles for Rank 0 : Counts the number of cycles while the iMC is being throttled by either thermal constraints or by the PCU throttling. It is not possible to distinguish between the two. This can be filtered by rank. If multiple ranks are selected and are being throttled at the same time, the counter will only increment by 1.", + "UMask": "0x2", + "Unit": "iMC" + }, + { + "BriefDescription": "Clock-Enabled Self-Refresh", + "EventCode": "0x43", + "EventName": "UNC_M_POWER_SELF_REFRESH", + "PerPkg": "1", + "PublicDescription": "Clock-Enabled Self-Refresh : Counts the number of cycles when the iMC is in self-refresh and the iMC still has a clock. This happens in some package C-states. For example, the PCU may ask the iMC to enter self-refresh even though some of the cores are still processing. One use of this is for Monroe technology. Self-refresh is required during package C3 and C6, but there is no clock in the iMC at this time, so it is not possible to count these cases.", + "Unit": "iMC" + }, + { + "BriefDescription": "Precharge due to read, write, underfill, or PGT.", + "EventCode": "0x03", + "EventName": "UNC_M_PRE_COUNT.ALL", + "PerPkg": "1", + "PublicDescription": "DRAM Precharge commands. : Counts the number of DRAM Precharge commands sent on this channel.", + "UMask": "0xff", + "Unit": "iMC" + }, + { + "BriefDescription": "DRAM Precharge commands", + "EventCode": "0x03", + "EventName": "UNC_M_PRE_COUNT.PGT", + "PerPkg": "1", + "PublicDescription": "DRAM Precharge commands. Counts the number of DRAM Precharge commands sent on this channel.", + "UMask": "0x88", + "Unit": "iMC" + }, + { + "BriefDescription": "DRAM Precharge commands. : Precharges from Page Table", + "EventCode": "0x03", + "EventName": "UNC_M_PRE_COUNT.PGT_PCH0", + "PerPkg": "1", + "PublicDescription": "DRAM Precharge commands. : Precharges from Page Table : Counts the number of DRAM Precharge commands sent on this channel. : Equivalent to PAGE_EMPTY", + "UMask": "0x8", + "Unit": "iMC" + }, + { + "BriefDescription": "DRAM Precharge commands.", + "EventCode": "0x03", + "EventName": "UNC_M_PRE_COUNT.PGT_PCH1", + "PerPkg": "1", + "PublicDescription": "DRAM Precharge commands. : Counts the number of DRAM Precharge commands sent on this channel.", + "UMask": "0x80", + "Unit": "iMC" + }, + { + "BriefDescription": "Precharge due to read on page miss", + "EventCode": "0x03", + "EventName": "UNC_M_PRE_COUNT.RD", + "PerPkg": "1", + "PublicDescription": "DRAM Precharge commands. : Counts the number of DRAM Precharge commands sent on this channel.", + "UMask": "0x11", + "Unit": "iMC" + }, + { + "BriefDescription": "DRAM Precharge commands. : Precharge due to read", + "EventCode": "0x03", + "EventName": "UNC_M_PRE_COUNT.RD_PCH0", + "PerPkg": "1", + "PublicDescription": "DRAM Precharge commands. : Precharge due to read : Counts the number of DRAM Precharge commands sent on this channel. : Precharge from read bank scheduler", + "UMask": "0x1", + "Unit": "iMC" + }, + { + "BriefDescription": "DRAM Precharge commands.", + "EventCode": "0x03", + "EventName": "UNC_M_PRE_COUNT.RD_PCH1", + "PerPkg": "1", + "PublicDescription": "DRAM Precharge commands. : Counts the number of DRAM Precharge commands sent on this channel.", + "UMask": "0x10", + "Unit": "iMC" + }, + { + "BriefDescription": "DRAM Precharge commands.", + "EventCode": "0x03", + "EventName": "UNC_M_PRE_COUNT.UFILL", + "PerPkg": "1", + "PublicDescription": "DRAM Precharge commands. : Counts the number of DRAM Precharge commands sent on this channel.", + "UMask": "0x44", + "Unit": "iMC" + }, + { + "BriefDescription": "DRAM Precharge commands.", + "EventCode": "0x03", + "EventName": "UNC_M_PRE_COUNT.UFILL_PCH0", + "PerPkg": "1", + "PublicDescription": "DRAM Precharge commands. : Counts the number of DRAM Precharge commands sent on this channel.", + "UMask": "0x4", + "Unit": "iMC" + }, + { + "BriefDescription": "DRAM Precharge commands.", + "EventCode": "0x03", + "EventName": "UNC_M_PRE_COUNT.UFILL_PCH1", + "PerPkg": "1", + "PublicDescription": "DRAM Precharge commands. : Counts the number of DRAM Precharge commands sent on this channel.", + "UMask": "0x40", + "Unit": "iMC" + }, + { + "BriefDescription": "Precharge due to write on page miss", + "EventCode": "0x03", + "EventName": "UNC_M_PRE_COUNT.WR", + "PerPkg": "1", + "PublicDescription": "DRAM Precharge commands. : Counts the number of DRAM Precharge commands sent on this channel.", + "UMask": "0x22", + "Unit": "iMC" + }, + { + "BriefDescription": "DRAM Precharge commands. : Precharge due to write", + "EventCode": "0x03", + "EventName": "UNC_M_PRE_COUNT.WR_PCH0", + "PerPkg": "1", + "PublicDescription": "DRAM Precharge commands. : Precharge due to write : Counts the number of DRAM Precharge commands sent on this channel. : Precharge from write bank scheduler", + "UMask": "0x2", + "Unit": "iMC" + }, + { + "BriefDescription": "DRAM Precharge commands.", + "EventCode": "0x03", + "EventName": "UNC_M_PRE_COUNT.WR_PCH1", + "PerPkg": "1", + "PublicDescription": "DRAM Precharge commands. : Counts the number of DRAM Precharge commands sent on this channel.", + "UMask": "0x20", + "Unit": "iMC" + }, + { + "BriefDescription": "Counts the number of cycles where the read buffer has greater than UMASK elements. This includes reads to both DDR and PMEM. NOTE: Umask must be set to the maximum number of elements in the queue (24 entries for SPR).", + "EventCode": "0x19", + "EventName": "UNC_M_RDB_FULL", + "PerPkg": "1", + "Unit": "iMC" + }, + { + "BriefDescription": "Counts the number of inserts into the read buffer destined for DDR. Does not count reads destined for PMEM.", + "EventCode": "0x17", + "EventName": "UNC_M_RDB_INSERTS", + "PerPkg": "1", + "UMask": "0x3", + "Unit": "iMC" + }, + { + "BriefDescription": "Read Data Buffer Inserts", + "EventCode": "0x17", + "EventName": "UNC_M_RDB_INSERTS.PCH0", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "iMC" + }, + { + "BriefDescription": "Read Data Buffer Inserts", + "EventCode": "0x17", + "EventName": "UNC_M_RDB_INSERTS.PCH1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "iMC" + }, + { + "BriefDescription": "Counts the number of cycles where there's at least one element in the read buffer. This includes reads to both DDR and PMEM.", + "EventCode": "0x18", + "EventName": "UNC_M_RDB_NE", + "PerPkg": "1", + "UMask": "0x3", + "Unit": "iMC" + }, + { + "BriefDescription": "Read Data Buffer Not Empty", + "EventCode": "0x18", + "EventName": "UNC_M_RDB_NE.PCH0", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "iMC" + }, + { + "BriefDescription": "Read Data Buffer Not Empty", + "EventCode": "0x18", + "EventName": "UNC_M_RDB_NE.PCH1", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "iMC" + }, + { + "BriefDescription": "Counts the number of cycles where there's at least one element in the read buffer. This includes reads to both DDR and PMEM.", + "EventCode": "0x18", + "EventName": "UNC_M_RDB_NOT_EMPTY", + "PerPkg": "1", + "UMask": "0x3", + "Unit": "iMC" + }, + { + "BriefDescription": "Counts the number of elements in the read buffer, including reads to both DDR and PMEM.", + "EventCode": "0x1a", + "EventName": "UNC_M_RDB_OCCUPANCY", + "PerPkg": "1", + "Unit": "iMC" + }, + { + "BriefDescription": "Read Pending Queue Allocations", + "EventCode": "0x10", + "EventName": "UNC_M_RPQ_INSERTS.PCH0", + "PerPkg": "1", + "PublicDescription": "Read Pending Queue Allocations : Counts the number of allocations into the Read Pending Queue. This queue is used to schedule reads out to the memory controller and to track the requests. Requests allocate into the RPQ soon after they enter the memory controller, and need credits for an entry in this buffer before being sent from the HA to the iMC. They deallocate after the CAS command has been issued to memory. This includes both ISOCH and non-ISOCH requests.", + "UMask": "0x1", + "Unit": "iMC" + }, + { + "BriefDescription": "Read Pending Queue Allocations", + "EventCode": "0x10", + "EventName": "UNC_M_RPQ_INSERTS.PCH1", + "PerPkg": "1", + "PublicDescription": "Read Pending Queue Allocations : Counts the number of allocations into the Read Pending Queue. This queue is used to schedule reads out to the memory controller and to track the requests. Requests allocate into the RPQ soon after they enter the memory controller, and need credits for an entry in this buffer before being sent from the HA to the iMC. They deallocate after the CAS command has been issued to memory. This includes both ISOCH and non-ISOCH requests.", + "UMask": "0x2", + "Unit": "iMC" + }, + { + "BriefDescription": "Read Pending Queue Occupancy", + "EventCode": "0x80", + "EventName": "UNC_M_RPQ_OCCUPANCY_PCH0", + "PerPkg": "1", + "PublicDescription": "Read Pending Queue Occupancy : Accumulates the occupancies of the Read Pending Queue each cycle. This can then be used to calculate both the average occupancy (in conjunction with the number of cycles not empty) and the average latency (in conjunction with the number of allocations). The RPQ is used to schedule reads out to the memory controller and to track the requests. Requests allocate into the RPQ soon after they enter the memory controller, and need credits for an entry in this buffer before being sent from the HA to the iMC. They deallocate after the CAS command has been issued to memory.", + "Unit": "iMC" + }, + { + "BriefDescription": "Read Pending Queue Occupancy", + "EventCode": "0x81", + "EventName": "UNC_M_RPQ_OCCUPANCY_PCH1", + "PerPkg": "1", + "PublicDescription": "Read Pending Queue Occupancy : Accumulates the occupancies of the Read Pending Queue each cycle. This can then be used to calculate both the average occupancy (in conjunction with the number of cycles not empty) and the average latency (in conjunction with the number of allocations). The RPQ is used to schedule reads out to the memory controller and to track the requests. Requests allocate into the RPQ soon after they enter the memory controller, and need credits for an entry in this buffer before being sent from the HA to the iMC. They deallocate after the CAS command has been issued to memory.", + "Unit": "iMC" + }, + { + "BriefDescription": "Scoreboard accepts", + "EventCode": "0xd2", + "EventName": "UNC_M_SB_ACCESSES.ACCEPTS", + "PerPkg": "1", + "UMask": "0x5", + "Unit": "iMC" + }, + { + "BriefDescription": "Scoreboard Accesses : Write Accepts", + "EventCode": "0xd2", + "EventName": "UNC_M_SB_ACCESSES.FM_RD_CMPS", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "iMC" + }, + { + "BriefDescription": "Scoreboard Accesses : Write Rejects", + "EventCode": "0xd2", + "EventName": "UNC_M_SB_ACCESSES.FM_WR_CMPS", + "PerPkg": "1", + "UMask": "0x80", + "Unit": "iMC" + }, + { + "BriefDescription": "Scoreboard Accesses : FM read completions", + "EventCode": "0xd2", + "EventName": "UNC_M_SB_ACCESSES.NM_RD_CMPS", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "iMC" + }, + { + "BriefDescription": "Scoreboard Accesses : FM write completions", + "EventCode": "0xd2", + "EventName": "UNC_M_SB_ACCESSES.NM_WR_CMPS", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "iMC" + }, + { + "BriefDescription": "Scoreboard Accesses : Read Accepts", + "EventCode": "0xd2", + "EventName": "UNC_M_SB_ACCESSES.RD_ACCEPTS", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "iMC" + }, + { + "BriefDescription": "Scoreboard Accesses : Read Rejects", + "EventCode": "0xd2", + "EventName": "UNC_M_SB_ACCESSES.RD_REJECTS", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "iMC" + }, + { + "BriefDescription": "Scoreboard rejects", + "EventCode": "0xd2", + "EventName": "UNC_M_SB_ACCESSES.REJECTS", + "PerPkg": "1", + "UMask": "0xa", + "Unit": "iMC" + }, + { + "BriefDescription": "Scoreboard Accesses : NM read completions", + "EventCode": "0xd2", + "EventName": "UNC_M_SB_ACCESSES.WR_ACCEPTS", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "iMC" + }, + { + "BriefDescription": "Scoreboard Accesses : NM write completions", + "EventCode": "0xd2", + "EventName": "UNC_M_SB_ACCESSES.WR_REJECTS", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "iMC" + }, + { + "BriefDescription": ": Alloc", + "EventCode": "0xd9", + "EventName": "UNC_M_SB_CANARY.ALLOC", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "iMC" + }, + { + "BriefDescription": ": Dealloc", + "EventCode": "0xd9", + "EventName": "UNC_M_SB_CANARY.DEALLOC", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "iMC" + }, + { + "BriefDescription": ": Near Mem Write Starved", + "EventCode": "0xd9", + "EventName": "UNC_M_SB_CANARY.FM_RD_STARVED", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "iMC" + }, + { + "BriefDescription": ": Far Mem Write Starved", + "EventCode": "0xd9", + "EventName": "UNC_M_SB_CANARY.FM_TGR_WR_STARVED", + "PerPkg": "1", + "UMask": "0x80", + "Unit": "iMC" + }, + { + "BriefDescription": ": Far Mem Read Starved", + "EventCode": "0xd9", + "EventName": "UNC_M_SB_CANARY.FM_WR_STARVED", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "iMC" + }, + { + "BriefDescription": ": Valid", + "EventCode": "0xd9", + "EventName": "UNC_M_SB_CANARY.NM_RD_STARVED", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "iMC" + }, + { + "BriefDescription": ": Near Mem Read Starved", + "EventCode": "0xd9", + "EventName": "UNC_M_SB_CANARY.NM_WR_STARVED", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "iMC" + }, + { + "BriefDescription": ": Reject", + "EventCode": "0xd9", + "EventName": "UNC_M_SB_CANARY.VLD", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "iMC" + }, + { + "BriefDescription": "Scoreboard Cycles Full", + "EventCode": "0xd1", + "EventName": "UNC_M_SB_CYCLES_FULL", + "PerPkg": "1", + "Unit": "iMC" + }, + { + "BriefDescription": "Scoreboard Cycles Not-Empty", + "EventCode": "0xd0", + "EventName": "UNC_M_SB_CYCLES_NE", + "PerPkg": "1", + "Unit": "iMC" + }, + { + "BriefDescription": "Scoreboard Inserts : Block region reads", + "EventCode": "0xd6", + "EventName": "UNC_M_SB_INSERTS.BLOCK_RDS", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "iMC" + }, + { + "BriefDescription": "Scoreboard Inserts : Block region writes", + "EventCode": "0xd6", + "EventName": "UNC_M_SB_INSERTS.BLOCK_WRS", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "iMC" + }, + { + "BriefDescription": "Scoreboard Inserts : Persistent Mem reads", + "EventCode": "0xd6", + "EventName": "UNC_M_SB_INSERTS.PMM_RDS", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "iMC" + }, + { + "BriefDescription": "Scoreboard Inserts : Persistent Mem writes", + "EventCode": "0xd6", + "EventName": "UNC_M_SB_INSERTS.PMM_WRS", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "iMC" + }, + { + "BriefDescription": "Scoreboard Inserts : Reads", + "EventCode": "0xd6", + "EventName": "UNC_M_SB_INSERTS.RDS", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "iMC" + }, + { + "BriefDescription": "Scoreboard Inserts : Writes", + "EventCode": "0xd6", + "EventName": "UNC_M_SB_INSERTS.WRS", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "iMC" + }, + { + "BriefDescription": "Scoreboard Occupancy : Block region reads", + "EventCode": "0xd5", + "EventName": "UNC_M_SB_OCCUPANCY.BLOCK_RDS", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "iMC" + }, + { + "BriefDescription": "Scoreboard Occupancy : Block region writes", + "EventCode": "0xd5", + "EventName": "UNC_M_SB_OCCUPANCY.BLOCK_WRS", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "iMC" + }, + { + "BriefDescription": "Scoreboard Occupancy : Persistent Mem reads", + "EventCode": "0xd5", + "EventName": "UNC_M_SB_OCCUPANCY.PMM_RDS", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "iMC" + }, + { + "BriefDescription": "Scoreboard Occupancy : Persistent Mem writes", + "EventCode": "0xd5", + "EventName": "UNC_M_SB_OCCUPANCY.PMM_WRS", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "iMC" + }, + { + "BriefDescription": "Scoreboard Occupancy : Reads", + "EventCode": "0xd5", + "EventName": "UNC_M_SB_OCCUPANCY.RDS", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "iMC" + }, + { + "BriefDescription": "Scoreboard Prefetch Inserts : All", + "EventCode": "0xda", + "EventName": "UNC_M_SB_PREF_INSERTS.ALL", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "iMC" + }, + { + "BriefDescription": "Scoreboard Prefetch Inserts : DDR4", + "EventCode": "0xda", + "EventName": "UNC_M_SB_PREF_INSERTS.DDR", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "iMC" + }, + { + "BriefDescription": "Scoreboard Prefetch Inserts : PMM", + "EventCode": "0xda", + "EventName": "UNC_M_SB_PREF_INSERTS.PMM", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "iMC" + }, + { + "BriefDescription": "Scoreboard Prefetch Occupancy : All", + "EventCode": "0xdb", + "EventName": "UNC_M_SB_PREF_OCCUPANCY.ALL", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "iMC" + }, + { + "BriefDescription": "Scoreboard Prefetch Occupancy : DDR4", + "EventCode": "0xdb", + "EventName": "UNC_M_SB_PREF_OCCUPANCY.DDR", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "iMC" + }, + { + "BriefDescription": "Scoreboard Prefetch Occupancy : Persistent Mem", + "EventCode": "0xDB", + "EventName": "UNC_M_SB_PREF_OCCUPANCY.PMM", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "UMask": "0x4", + "Unit": "iMC" + }, + { + "BriefDescription": "Number of Scoreboard Requests Rejected", + "EventCode": "0xd4", + "EventName": "UNC_M_SB_REJECT.CANARY", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "iMC" + }, + { + "BriefDescription": "Number of Scoreboard Requests Rejected", + "EventCode": "0xd4", + "EventName": "UNC_M_SB_REJECT.DDR_EARLY_CMP", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "iMC" + }, + { + "BriefDescription": "Number of Scoreboard Requests Rejected : FM requests rejected due to full address conflict", + "EventCode": "0xd4", + "EventName": "UNC_M_SB_REJECT.FM_ADDR_CNFLT", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "iMC" + }, + { + "BriefDescription": "Number of Scoreboard Requests Rejected : NM requests rejected due to set conflict", + "EventCode": "0xd4", + "EventName": "UNC_M_SB_REJECT.NM_SET_CNFLT", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "iMC" + }, + { + "BriefDescription": "Number of Scoreboard Requests Rejected : Patrol requests rejected due to set conflict", + "EventCode": "0xd4", + "EventName": "UNC_M_SB_REJECT.PATROL_SET_CNFLT", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "iMC" + }, + { + "BriefDescription": ": Far Mem Read - Set", + "EventCode": "0xd7", + "EventName": "UNC_M_SB_STRV_ALLOC.FM_RD", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "iMC" + }, + { + "BriefDescription": ": Near Mem Read - Clear", + "EventCode": "0xd7", + "EventName": "UNC_M_SB_STRV_ALLOC.FM_TGR", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "iMC" + }, + { + "BriefDescription": ": Far Mem Write - Set", + "EventCode": "0xd7", + "EventName": "UNC_M_SB_STRV_ALLOC.FM_WR", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "iMC" + }, + { + "BriefDescription": ": Near Mem Read - Set", + "EventCode": "0xd7", + "EventName": "UNC_M_SB_STRV_ALLOC.NM_RD", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "iMC" + }, + { + "BriefDescription": ": Near Mem Write - Set", + "EventCode": "0xd7", + "EventName": "UNC_M_SB_STRV_ALLOC.NM_WR", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "iMC" + }, + { + "BriefDescription": ": Far Mem Read - Set", + "EventCode": "0xde", + "EventName": "UNC_M_SB_STRV_DEALLOC.FM_RD", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "iMC" + }, + { + "BriefDescription": ": Near Mem Read - Clear", + "EventCode": "0xde", + "EventName": "UNC_M_SB_STRV_DEALLOC.FM_TGR", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "iMC" + }, + { + "BriefDescription": ": Far Mem Write - Set", + "EventCode": "0xde", + "EventName": "UNC_M_SB_STRV_DEALLOC.FM_WR", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "iMC" + }, + { + "BriefDescription": ": Near Mem Read - Set", + "EventCode": "0xde", + "EventName": "UNC_M_SB_STRV_DEALLOC.NM_RD", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "iMC" + }, + { + "BriefDescription": ": Near Mem Write - Set", + "EventCode": "0xde", + "EventName": "UNC_M_SB_STRV_DEALLOC.NM_WR", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "iMC" + }, + { + "BriefDescription": ": Far Mem Read", + "EventCode": "0xd8", + "EventName": "UNC_M_SB_STRV_OCC.FM_RD", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "iMC" + }, + { + "BriefDescription": ": Near Mem Read - Clear", + "EventCode": "0xd8", + "EventName": "UNC_M_SB_STRV_OCC.FM_TGR", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "iMC" + }, + { + "BriefDescription": ": Far Mem Write", + "EventCode": "0xd8", + "EventName": "UNC_M_SB_STRV_OCC.FM_WR", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "iMC" + }, + { + "BriefDescription": ": Near Mem Read", + "EventCode": "0xd8", + "EventName": "UNC_M_SB_STRV_OCC.NM_RD", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "iMC" + }, + { + "BriefDescription": ": Near Mem Write", + "EventCode": "0xd8", + "EventName": "UNC_M_SB_STRV_OCC.NM_WR", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "iMC" + }, + { + "BriefDescription": "UNC_M_SB_TAGGED.DDR4_CMP", + "EventCode": "0xdd", + "EventName": "UNC_M_SB_TAGGED.DDR4_CMP", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "iMC" + }, + { + "BriefDescription": "UNC_M_SB_TAGGED.NEW", + "EventCode": "0xdd", + "EventName": "UNC_M_SB_TAGGED.NEW", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "iMC" + }, + { + "BriefDescription": "UNC_M_SB_TAGGED.OCC", + "EventCode": "0xdd", + "EventName": "UNC_M_SB_TAGGED.OCC", + "PerPkg": "1", + "UMask": "0x80", + "Unit": "iMC" + }, + { + "BriefDescription": "UNC_M_SB_TAGGED.PMM0_CMP", + "EventCode": "0xdd", + "EventName": "UNC_M_SB_TAGGED.PMM0_CMP", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "iMC" + }, + { + "BriefDescription": "UNC_M_SB_TAGGED.PMM1_CMP", + "EventCode": "0xdd", + "EventName": "UNC_M_SB_TAGGED.PMM1_CMP", + "PerPkg": "1", + "UMask": "0x20", + "Unit": "iMC" + }, + { + "BriefDescription": "UNC_M_SB_TAGGED.PMM2_CMP", + "EventCode": "0xdd", + "EventName": "UNC_M_SB_TAGGED.PMM2_CMP", + "PerPkg": "1", + "UMask": "0x40", + "Unit": "iMC" + }, + { + "BriefDescription": "UNC_M_SB_TAGGED.RD_HIT", + "EventCode": "0xdd", + "EventName": "UNC_M_SB_TAGGED.RD_HIT", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "iMC" + }, + { + "BriefDescription": "UNC_M_SB_TAGGED.RD_MISS", + "EventCode": "0xdd", + "EventName": "UNC_M_SB_TAGGED.RD_MISS", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "iMC" + }, + { + "BriefDescription": "2LM Tag check hit in near memory cache (DDR4)", + "EventCode": "0xd3", + "EventName": "UNC_M_TAGCHK.HIT", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "iMC" + }, + { + "BriefDescription": "2LM Tag check miss, no data at this line", + "EventCode": "0xd3", + "EventName": "UNC_M_TAGCHK.MISS_CLEAN", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "iMC" + }, + { + "BriefDescription": "2LM Tag check miss, existing data may be evicted to PMM", + "EventCode": "0xd3", + "EventName": "UNC_M_TAGCHK.MISS_DIRTY", + "PerPkg": "1", + "UMask": "0x4", + "Unit": "iMC" + }, + { + "BriefDescription": "2LM Tag check hit due to memory read", + "EventCode": "0xd3", + "EventName": "UNC_M_TAGCHK.NM_RD_HIT", + "PerPkg": "1", + "UMask": "0x8", + "Unit": "iMC" + }, + { + "BriefDescription": "2LM Tag check hit due to memory write", + "EventCode": "0xd3", + "EventName": "UNC_M_TAGCHK.NM_WR_HIT", + "PerPkg": "1", + "UMask": "0x10", + "Unit": "iMC" + }, + { + "BriefDescription": "Write Pending Queue Allocations", + "EventCode": "0x20", + "EventName": "UNC_M_WPQ_INSERTS.PCH0", + "PerPkg": "1", + "PublicDescription": "Write Pending Queue Allocations : Counts the number of allocations into the Write Pending Queue. This can then be used to calculate the average queuing latency (in conjunction with the WPQ occupancy count). The WPQ is used to schedule write out to the memory controller and to track the writes. Requests allocate into the WPQ soon after they enter the memory controller, and need credits for an entry in this buffer before being sent from the CHA to the iMC. They deallocate after being issued to DRAM. Write requests themselves are able to complete (from the perspective of the rest of the system) as soon they have posted to the iMC.", + "UMask": "0x1", + "Unit": "iMC" + }, + { + "BriefDescription": "Write Pending Queue Allocations", + "EventCode": "0x20", + "EventName": "UNC_M_WPQ_INSERTS.PCH1", + "PerPkg": "1", + "PublicDescription": "Write Pending Queue Allocations : Counts the number of allocations into the Write Pending Queue. This can then be used to calculate the average queuing latency (in conjunction with the WPQ occupancy count). The WPQ is used to schedule write out to the memory controller and to track the writes. Requests allocate into the WPQ soon after they enter the memory controller, and need credits for an entry in this buffer before being sent from the CHA to the iMC. They deallocate after being issued to DRAM. Write requests themselves are able to complete (from the perspective of the rest of the system) as soon they have posted to the iMC.", + "UMask": "0x2", + "Unit": "iMC" + }, + { + "BriefDescription": "Write Pending Queue Occupancy", + "EventCode": "0x82", + "EventName": "UNC_M_WPQ_OCCUPANCY_PCH0", + "PerPkg": "1", + "PublicDescription": "Write Pending Queue Occupancy : Accumulates the occupancies of the Write Pending Queue each cycle. This can then be used to calculate both the average queue occupancy (in conjunction with the number of cycles not empty) and the average latency (in conjunction with the number of allocations). The WPQ is used to schedule write out to the memory controller and to track the writes. Requests allocate into the WPQ soon after they enter the memory controller, and need credits for an entry in this buffer before being sent from the HA to the iMC. They deallocate after being issued to DRAM. Write requests themselves are able to complete (from the perspective of the rest of the system) as soon they have posted to the iMC. This is not to be confused with actually performing the write to DRAM. Therefore, the average latency for this queue is actually not useful for deconstruction intermediate write latencies. So, we provide filtering based on if the request has posted or not. By using the not posted filter, we can track how long writes spent in the iMC before completions were sent to the HA. The posted filter, on the other hand, provides information about how much queueing is actually happening in the iMC for writes before they are actually issued to memory. High average occupancies will generally coincide with high write major mode counts.", + "Unit": "iMC" + }, + { + "BriefDescription": "Write Pending Queue Occupancy", + "EventCode": "0x83", + "EventName": "UNC_M_WPQ_OCCUPANCY_PCH1", + "PerPkg": "1", + "PublicDescription": "Write Pending Queue Occupancy : Accumulates the occupancies of the Write Pending Queue each cycle. This can then be used to calculate both the average queue occupancy (in conjunction with the number of cycles not empty) and the average latency (in conjunction with the number of allocations). The WPQ is used to schedule write out to the memory controller and to track the writes. Requests allocate into the WPQ soon after they enter the memory controller, and need credits for an entry in this buffer before being sent from the HA to the iMC. They deallocate after being issued to DRAM. Write requests themselves are able to complete (from the perspective of the rest of the system) as soon they have posted to the iMC. This is not to be confused with actually performing the write to DRAM. Therefore, the average latency for this queue is actually not useful for deconstruction intermediate write latencies. So, we provide filtering based on if the request has posted or not. By using the not posted filter, we can track how long writes spent in the iMC before completions were sent to the HA. The posted filter, on the other hand, provides information about how much queueing is actually happening in the iMC for writes before they are actually issued to memory. High average occupancies will generally coincide with high write major mode counts.", + "Unit": "iMC" + }, + { + "BriefDescription": "Write Pending Queue CAM Match", + "EventCode": "0x23", + "EventName": "UNC_M_WPQ_READ_HIT", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "PublicDescription": "Counts the number of times a request hits in the WPQ (write-pending queue). The iMC allows writes and reads to pass up other writes to different addresses. Before a read or a write is issued, it will first CAM the WPQ to see if there is a write pending to that address. When reads hit, they are able to directly pull their data from the WPQ instead of going to memory. Writes that hit will overwrite the existing data. Partial writes that hit will not need to do underfill reads and will simply update their relevant sections.", + "Unit": "iMC" + }, + { + "BriefDescription": "Write Pending Queue CAM Match", + "EventCode": "0x24", + "EventName": "UNC_M_WPQ_WRITE_HIT", + "FCMask": "0x00000000", + "PerPkg": "1", + "PortMask": "0x00000000", + "PublicDescription": "Counts the number of times a request hits in the WPQ (write-pending queue). The iMC allows writes and reads to pass up other writes to different addresses. Before a read or a write is issued, it will first CAM the WPQ to see if there is a write pending to that address. When reads hit, they are able to directly pull their data from the WPQ instead of going to memory. Writes that hit will overwrite the existing data. Partial writes that hit will not need to do underfill reads and will simply update their relevant sections.", + "Unit": "iMC" + } +] diff --git a/tools/perf/pmu-events/arch/x86/emeraldrapids/uncore-power.json b/tools/perf/pmu-events/arch/x86/emeraldrapids/uncore-power.json new file mode 100644 index 000000000000..8948e85074f0 --- /dev/null +++ b/tools/perf/pmu-events/arch/x86/emeraldrapids/uncore-power.json @@ -0,0 +1,197 @@ +[ + { + "BriefDescription": "PCU PCLK Clockticks", + "EventCode": "0x01", + "EventName": "UNC_P_CLOCKTICKS", + "PerPkg": "1", + "PublicDescription": "Number of PCU PCLK Clock cycles while the event is enabled", + "Unit": "PCU" + }, + { + "BriefDescription": "UNC_P_CORE_TRANSITION_CYCLES", + "EventCode": "0x60", + "EventName": "UNC_P_CORE_TRANSITION_CYCLES", + "PerPkg": "1", + "Unit": "PCU" + }, + { + "BriefDescription": "UNC_P_DEMOTIONS", + "EventCode": "0x30", + "EventName": "UNC_P_DEMOTIONS", + "PerPkg": "1", + "Unit": "PCU" + }, + { + "BriefDescription": "Phase Shed 0 Cycles", + "EventCode": "0x75", + "EventName": "UNC_P_FIVR_PS_PS0_CYCLES", + "PerPkg": "1", + "PublicDescription": "Phase Shed 0 Cycles : Cycles spent in phase-shedding power state 0", + "Unit": "PCU" + }, + { + "BriefDescription": "Phase Shed 1 Cycles", + "EventCode": "0x76", + "EventName": "UNC_P_FIVR_PS_PS1_CYCLES", + "PerPkg": "1", + "PublicDescription": "Phase Shed 1 Cycles : Cycles spent in phase-shedding power state 1", + "Unit": "PCU" + }, + { + "BriefDescription": "Phase Shed 2 Cycles", + "EventCode": "0x77", + "EventName": "UNC_P_FIVR_PS_PS2_CYCLES", + "PerPkg": "1", + "PublicDescription": "Phase Shed 2 Cycles : Cycles spent in phase-shedding power state 2", + "Unit": "PCU" + }, + { + "BriefDescription": "Phase Shed 3 Cycles", + "EventCode": "0x78", + "EventName": "UNC_P_FIVR_PS_PS3_CYCLES", + "PerPkg": "1", + "PublicDescription": "Phase Shed 3 Cycles : Cycles spent in phase-shedding power state 3", + "Unit": "PCU" + }, + { + "BriefDescription": "AVX256 Frequency Clipping", + "EventCode": "0x49", + "EventName": "UNC_P_FREQ_CLIP_AVX256", + "PerPkg": "1", + "Unit": "PCU" + }, + { + "BriefDescription": "AVX512 Frequency Clipping", + "EventCode": "0x4a", + "EventName": "UNC_P_FREQ_CLIP_AVX512", + "PerPkg": "1", + "Unit": "PCU" + }, + { + "BriefDescription": "Thermal Strongest Upper Limit Cycles", + "EventCode": "0x04", + "EventName": "UNC_P_FREQ_MAX_LIMIT_THERMAL_CYCLES", + "PerPkg": "1", + "PublicDescription": "Thermal Strongest Upper Limit Cycles : Number of cycles any frequency is reduced due to a thermal limit. Count only if throttling is occurring.", + "Unit": "PCU" + }, + { + "BriefDescription": "Power Strongest Upper Limit Cycles", + "EventCode": "0x05", + "EventName": "UNC_P_FREQ_MAX_POWER_CYCLES", + "PerPkg": "1", + "PublicDescription": "Power Strongest Upper Limit Cycles : Counts the number of cycles when power is the upper limit on frequency.", + "Unit": "PCU" + }, + { + "BriefDescription": "IO P Limit Strongest Lower Limit Cycles", + "EventCode": "0x73", + "EventName": "UNC_P_FREQ_MIN_IO_P_CYCLES", + "PerPkg": "1", + "PublicDescription": "IO P Limit Strongest Lower Limit Cycles : Counts the number of cycles when IO P Limit is preventing us from dropping the frequency lower. This algorithm monitors the needs to the IO subsystem on both local and remote sockets and will maintain a frequency high enough to maintain good IO BW. This is necessary for when all the IA cores on a socket are idle but a user still would like to maintain high IO Bandwidth.", + "Unit": "PCU" + }, + { + "BriefDescription": "Cycles spent changing Frequency", + "EventCode": "0x74", + "EventName": "UNC_P_FREQ_TRANS_CYCLES", + "PerPkg": "1", + "PublicDescription": "Cycles spent changing Frequency : Counts the number of cycles when the system is changing frequency. This can not be filtered by thread ID. One can also use it with the occupancy counter that monitors number of threads in C0 to estimate the performance impact that frequency transitions had on the system.", + "Unit": "PCU" + }, + { + "BriefDescription": "Memory Phase Shedding Cycles", + "EventCode": "0x2f", + "EventName": "UNC_P_MEMORY_PHASE_SHEDDING_CYCLES", + "PerPkg": "1", + "PublicDescription": "Memory Phase Shedding Cycles : Counts the number of cycles that the PCU has triggered memory phase shedding. This is a mode that can be run in the iMC physicals that saves power at the expense of additional latency.", + "Unit": "PCU" + }, + { + "BriefDescription": "Package C State Residency - C0", + "EventCode": "0x2a", + "EventName": "UNC_P_PKG_RESIDENCY_C0_CYCLES", + "PerPkg": "1", + "PublicDescription": "Package C State Residency - C0 : Counts the number of cycles when the package was in C0. This event can be used in conjunction with edge detect to count C0 entrances (or exits using invert). Residency events do not include transition times.", + "Unit": "PCU" + }, + { + "BriefDescription": "Package C State Residency - C2E", + "EventCode": "0x2b", + "EventName": "UNC_P_PKG_RESIDENCY_C2E_CYCLES", + "PerPkg": "1", + "PublicDescription": "Package C State Residency - C2E : Counts the number of cycles when the package was in C2E. This event can be used in conjunction with edge detect to count C2E entrances (or exits using invert). Residency events do not include transition times.", + "Unit": "PCU" + }, + { + "BriefDescription": "Package C State Residency - C6", + "EventCode": "0x2d", + "EventName": "UNC_P_PKG_RESIDENCY_C6_CYCLES", + "PerPkg": "1", + "PublicDescription": "Package C State Residency - C6 : Counts the number of cycles when the package was in C6. This event can be used in conjunction with edge detect to count C6 entrances (or exits using invert). Residency events do not include transition times.", + "Unit": "PCU" + }, + { + "BriefDescription": "UNC_P_PMAX_THROTTLED_CYCLES", + "EventCode": "0x06", + "EventName": "UNC_P_PMAX_THROTTLED_CYCLES", + "PerPkg": "1", + "Unit": "PCU" + }, + { + "BriefDescription": "Number of cores in C0", + "EventCode": "0x35", + "EventName": "UNC_P_POWER_STATE_OCCUPANCY_CORES_C0", + "PerPkg": "1", + "PublicDescription": "Number of cores in C0 : This is an occupancy event that tracks the number of cores that are in the chosen C-State. It can be used by itself to get the average number of cores in that C-state with thresholding to generate histograms, or with other PCU events and occupancy triggering to capture other details.", + "Unit": "PCU" + }, + { + "BriefDescription": "Number of cores in C3", + "EventCode": "0x36", + "EventName": "UNC_P_POWER_STATE_OCCUPANCY_CORES_C3", + "PerPkg": "1", + "PublicDescription": "Number of cores in C3 : This is an occupancy event that tracks the number of cores that are in the chosen C-State. It can be used by itself to get the average number of cores in that C-state with thresholding to generate histograms, or with other PCU events and occupancy triggering to capture other details.", + "Unit": "PCU" + }, + { + "BriefDescription": "Number of cores in C6", + "EventCode": "0x37", + "EventName": "UNC_P_POWER_STATE_OCCUPANCY_CORES_C6", + "PerPkg": "1", + "PublicDescription": "Number of cores in C6 : This is an occupancy event that tracks the number of cores that are in the chosen C-State. It can be used by itself to get the average number of cores in that C-state with thresholding to generate histograms, or with other PCU events and occupancy triggering to capture other details.", + "Unit": "PCU" + }, + { + "BriefDescription": "External Prochot", + "EventCode": "0x0a", + "EventName": "UNC_P_PROCHOT_EXTERNAL_CYCLES", + "PerPkg": "1", + "PublicDescription": "External Prochot : Counts the number of cycles that we are in external PROCHOT mode. This mode is triggered when a sensor off the die determines that something off-die (like DRAM) is too hot and must throttle to avoid damaging the chip.", + "Unit": "PCU" + }, + { + "BriefDescription": "Internal Prochot", + "EventCode": "0x09", + "EventName": "UNC_P_PROCHOT_INTERNAL_CYCLES", + "PerPkg": "1", + "PublicDescription": "Internal Prochot : Counts the number of cycles that we are in Internal PROCHOT mode. This mode is triggered when a sensor on the die determines that we are too hot and must throttle to avoid damaging the chip.", + "Unit": "PCU" + }, + { + "BriefDescription": "Total Core C State Transition Cycles", + "EventCode": "0x72", + "EventName": "UNC_P_TOTAL_TRANSITION_CYCLES", + "PerPkg": "1", + "PublicDescription": "Total Core C State Transition Cycles : Number of cycles spent performing core C state transitions across all cores.", + "Unit": "PCU" + }, + { + "BriefDescription": "VR Hot", + "EventCode": "0x42", + "EventName": "UNC_P_VR_HOT_CYCLES", + "PerPkg": "1", + "PublicDescription": "VR Hot : Number of cycles that a CPU SVID VR is hot. Does not cover DRAM VRs", + "Unit": "PCU" + } +] diff --git a/tools/perf/pmu-events/arch/x86/emeraldrapids/virtual-memory.json b/tools/perf/pmu-events/arch/x86/emeraldrapids/virtual-memory.json new file mode 100644 index 000000000000..a1e3b8d2ebe7 --- /dev/null +++ b/tools/perf/pmu-events/arch/x86/emeraldrapids/virtual-memory.json @@ -0,0 +1,165 @@ +[ + { + "BriefDescription": "Loads that miss the DTLB and hit the STLB.", + "EventCode": "0x12", + "EventName": "DTLB_LOAD_MISSES.STLB_HIT", + "PublicDescription": "Counts loads that miss the DTLB (Data TLB) and hit the STLB (Second level TLB).", + "SampleAfterValue": "100003", + "UMask": "0x20" + }, + { + "BriefDescription": "Cycles when at least one PMH is busy with a page walk for a demand load.", + "CounterMask": "1", + "EventCode": "0x12", + "EventName": "DTLB_LOAD_MISSES.WALK_ACTIVE", + "PublicDescription": "Counts cycles when at least one PMH (Page Miss Handler) is busy with a page walk for a demand load.", + "SampleAfterValue": "100003", + "UMask": "0x10" + }, + { + "BriefDescription": "Load miss in all TLB levels causes a page walk that completes. (All page sizes)", + "EventCode": "0x12", + "EventName": "DTLB_LOAD_MISSES.WALK_COMPLETED", + "PublicDescription": "Counts completed page walks (all page sizes) caused by demand data loads. This implies it missed in the DTLB and further levels of TLB. The page walk can end with or without a fault.", + "SampleAfterValue": "100003", + "UMask": "0xe" + }, + { + "BriefDescription": "Page walks completed due to a demand data load to a 1G page.", + "EventCode": "0x12", + "EventName": "DTLB_LOAD_MISSES.WALK_COMPLETED_1G", + "PublicDescription": "Counts completed page walks (1G sizes) caused by demand data loads. This implies address translations missed in the DTLB and further levels of TLB. The page walk can end with or without a fault.", + "SampleAfterValue": "100003", + "UMask": "0x8" + }, + { + "BriefDescription": "Page walks completed due to a demand data load to a 2M/4M page.", + "EventCode": "0x12", + "EventName": "DTLB_LOAD_MISSES.WALK_COMPLETED_2M_4M", + "PublicDescription": "Counts completed page walks (2M/4M sizes) caused by demand data loads. This implies address translations missed in the DTLB and further levels of TLB. The page walk can end with or without a fault.", + "SampleAfterValue": "100003", + "UMask": "0x4" + }, + { + "BriefDescription": "Page walks completed due to a demand data load to a 4K page.", + "EventCode": "0x12", + "EventName": "DTLB_LOAD_MISSES.WALK_COMPLETED_4K", + "PublicDescription": "Counts completed page walks (4K sizes) caused by demand data loads. This implies address translations missed in the DTLB and further levels of TLB. The page walk can end with or without a fault.", + "SampleAfterValue": "100003", + "UMask": "0x2" + }, + { + "BriefDescription": "Number of page walks outstanding for a demand load in the PMH each cycle.", + "EventCode": "0x12", + "EventName": "DTLB_LOAD_MISSES.WALK_PENDING", + "PublicDescription": "Counts the number of page walks outstanding for a demand load in the PMH (Page Miss Handler) each cycle.", + "SampleAfterValue": "100003", + "UMask": "0x10" + }, + { + "BriefDescription": "Stores that miss the DTLB and hit the STLB.", + "EventCode": "0x13", + "EventName": "DTLB_STORE_MISSES.STLB_HIT", + "PublicDescription": "Counts stores that miss the DTLB (Data TLB) and hit the STLB (2nd Level TLB).", + "SampleAfterValue": "100003", + "UMask": "0x20" + }, + { + "BriefDescription": "Cycles when at least one PMH is busy with a page walk for a store.", + "CounterMask": "1", + "EventCode": "0x13", + "EventName": "DTLB_STORE_MISSES.WALK_ACTIVE", + "PublicDescription": "Counts cycles when at least one PMH (Page Miss Handler) is busy with a page walk for a store.", + "SampleAfterValue": "100003", + "UMask": "0x10" + }, + { + "BriefDescription": "Store misses in all TLB levels causes a page walk that completes. (All page sizes)", + "EventCode": "0x13", + "EventName": "DTLB_STORE_MISSES.WALK_COMPLETED", + "PublicDescription": "Counts completed page walks (all page sizes) caused by demand data stores. This implies it missed in the DTLB and further levels of TLB. The page walk can end with or without a fault.", + "SampleAfterValue": "100003", + "UMask": "0xe" + }, + { + "BriefDescription": "Page walks completed due to a demand data store to a 1G page.", + "EventCode": "0x13", + "EventName": "DTLB_STORE_MISSES.WALK_COMPLETED_1G", + "PublicDescription": "Counts completed page walks (1G sizes) caused by demand data stores. This implies address translations missed in the DTLB and further levels of TLB. The page walk can end with or without a fault.", + "SampleAfterValue": "100003", + "UMask": "0x8" + }, + { + "BriefDescription": "Page walks completed due to a demand data store to a 2M/4M page.", + "EventCode": "0x13", + "EventName": "DTLB_STORE_MISSES.WALK_COMPLETED_2M_4M", + "PublicDescription": "Counts completed page walks (2M/4M sizes) caused by demand data stores. This implies address translations missed in the DTLB and further levels of TLB. The page walk can end with or without a fault.", + "SampleAfterValue": "100003", + "UMask": "0x4" + }, + { + "BriefDescription": "Page walks completed due to a demand data store to a 4K page.", + "EventCode": "0x13", + "EventName": "DTLB_STORE_MISSES.WALK_COMPLETED_4K", + "PublicDescription": "Counts completed page walks (4K sizes) caused by demand data stores. This implies address translations missed in the DTLB and further levels of TLB. The page walk can end with or without a fault.", + "SampleAfterValue": "100003", + "UMask": "0x2" + }, + { + "BriefDescription": "Number of page walks outstanding for a store in the PMH each cycle.", + "EventCode": "0x13", + "EventName": "DTLB_STORE_MISSES.WALK_PENDING", + "PublicDescription": "Counts the number of page walks outstanding for a store in the PMH (Page Miss Handler) each cycle.", + "SampleAfterValue": "100003", + "UMask": "0x10" + }, + { + "BriefDescription": "Instruction fetch requests that miss the ITLB and hit the STLB.", + "EventCode": "0x11", + "EventName": "ITLB_MISSES.STLB_HIT", + "PublicDescription": "Counts instruction fetch requests that miss the ITLB (Instruction TLB) and hit the STLB (Second-level TLB).", + "SampleAfterValue": "100003", + "UMask": "0x20" + }, + { + "BriefDescription": "Cycles when at least one PMH is busy with a page walk for code (instruction fetch) request.", + "CounterMask": "1", + "EventCode": "0x11", + "EventName": "ITLB_MISSES.WALK_ACTIVE", + "PublicDescription": "Counts cycles when at least one PMH (Page Miss Handler) is busy with a page walk for a code (instruction fetch) request.", + "SampleAfterValue": "100003", + "UMask": "0x10" + }, + { + "BriefDescription": "Code miss in all TLB levels causes a page walk that completes. (All page sizes)", + "EventCode": "0x11", + "EventName": "ITLB_MISSES.WALK_COMPLETED", + "PublicDescription": "Counts completed page walks (all page sizes) caused by a code fetch. This implies it missed in the ITLB (Instruction TLB) and further levels of TLB. The page walk can end with or without a fault.", + "SampleAfterValue": "100003", + "UMask": "0xe" + }, + { + "BriefDescription": "Code miss in all TLB levels causes a page walk that completes. (2M/4M)", + "EventCode": "0x11", + "EventName": "ITLB_MISSES.WALK_COMPLETED_2M_4M", + "PublicDescription": "Counts completed page walks (2M/4M page sizes) caused by a code fetch. This implies it missed in the ITLB (Instruction TLB) and further levels of TLB. The page walk can end with or without a fault.", + "SampleAfterValue": "100003", + "UMask": "0x4" + }, + { + "BriefDescription": "Code miss in all TLB levels causes a page walk that completes. (4K)", + "EventCode": "0x11", + "EventName": "ITLB_MISSES.WALK_COMPLETED_4K", + "PublicDescription": "Counts completed page walks (4K page sizes) caused by a code fetch. This implies it missed in the ITLB (Instruction TLB) and further levels of TLB. The page walk can end with or without a fault.", + "SampleAfterValue": "100003", + "UMask": "0x2" + }, + { + "BriefDescription": "Number of page walks outstanding for an outstanding code request in the PMH each cycle.", + "EventCode": "0x11", + "EventName": "ITLB_MISSES.WALK_PENDING", + "PublicDescription": "Counts the number of page walks outstanding for an outstanding code (instruction fetch) request in the PMH (Page Miss Handler) each cycle.", + "SampleAfterValue": "100003", + "UMask": "0x10" + } +] diff --git a/tools/perf/pmu-events/arch/x86/mapfile.csv b/tools/perf/pmu-events/arch/x86/mapfile.csv index de9582f183cb..dca2133473e3 100644 --- a/tools/perf/pmu-events/arch/x86/mapfile.csv +++ b/tools/perf/pmu-events/arch/x86/mapfile.csv @@ -7,6 +7,7 @@ GenuineIntel-6-56,v10,broadwellde,core GenuineIntel-6-4F,v21,broadwellx,core GenuineIntel-6-55-[56789ABCDEF],v1.19,cascadelakex,core GenuineIntel-6-9[6C],v1.04,elkhartlake,core +GenuineIntel-6-CF,v1.00,emeraldrapids,core GenuineIntel-6-5[CF],v13,goldmont,core GenuineIntel-6-7A,v1.01,goldmontplus,core GenuineIntel-6-B6,v1.00,grandridge,core @@ -25,7 +26,7 @@ GenuineIntel-6-1[AEF],v3,nehalemep,core GenuineIntel-6-2E,v3,nehalemex,core GenuineIntel-6-A7,v1.01,rocketlake,core GenuineIntel-6-2A,v19,sandybridge,core -GenuineIntel-6-(8F|CF),v1.15,sapphirerapids,core +GenuineIntel-6-8F,v1.16,sapphirerapids,core GenuineIntel-6-AF,v1.00,sierraforest,core GenuineIntel-6-(37|4A|4C|4D|5A),v15,silvermont,core GenuineIntel-6-(4E|5E|8E|9E|A5|A6),v57,skylake,core diff --git a/tools/perf/pmu-events/arch/x86/sapphirerapids/frontend.json b/tools/perf/pmu-events/arch/x86/sapphirerapids/frontend.json index 860a415e5e79..9e53da55d0c1 100644 --- a/tools/perf/pmu-events/arch/x86/sapphirerapids/frontend.json +++ b/tools/perf/pmu-events/arch/x86/sapphirerapids/frontend.json @@ -333,29 +333,56 @@ "UMask": "0x20" }, { - "BriefDescription": "Uops not delivered by IDQ when backend of the machine is not stalled", + "BriefDescription": "Uops not delivered by IDQ when backend of the machine is not stalled [This event is alias to IDQ_UOPS_NOT_DELIVERED.CORE]", "EventCode": "0x9c", - "EventName": "IDQ_UOPS_NOT_DELIVERED.CORE", - "PublicDescription": "Counts the number of uops not delivered to by the Instruction Decode Queue (IDQ) to the back-end of the pipeline when there was no back-end stalls. This event counts for one SMT thread in a given cycle.", + "EventName": "IDQ_BUBBLES.CORE", + "PublicDescription": "Counts the number of uops not delivered to by the Instruction Decode Queue (IDQ) to the back-end of the pipeline when there was no back-end stalls. This event counts for one SMT thread in a given cycle. [This event is alias to IDQ_UOPS_NOT_DELIVERED.CORE]", "SampleAfterValue": "1000003", "UMask": "0x1" }, { - "BriefDescription": "Cycles when no uops are not delivered by the IDQ when backend of the machine is not stalled", + "BriefDescription": "Cycles when no uops are not delivered by the IDQ when backend of the machine is not stalled [This event is alias to IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE]", + "CounterMask": "6", + "EventCode": "0x9c", + "EventName": "IDQ_BUBBLES.CYCLES_0_UOPS_DELIV.CORE", + "PublicDescription": "Counts the number of cycles when no uops were delivered by the Instruction Decode Queue (IDQ) to the back-end of the pipeline when there was no back-end stalls. This event counts for one SMT thread in a given cycle. [This event is alias to IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE]", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Cycles when optimal number of uops was delivered to the back-end when the back-end is not stalled [This event is alias to IDQ_UOPS_NOT_DELIVERED.CYCLES_FE_WAS_OK]", + "CounterMask": "1", + "EventCode": "0x9c", + "EventName": "IDQ_BUBBLES.CYCLES_FE_WAS_OK", + "Invert": "1", + "PublicDescription": "Counts the number of cycles when the optimal number of uops were delivered by the Instruction Decode Queue (IDQ) to the back-end of the pipeline when there was no back-end stalls. This event counts for one SMT thread in a given cycle. [This event is alias to IDQ_UOPS_NOT_DELIVERED.CYCLES_FE_WAS_OK]", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Uops not delivered by IDQ when backend of the machine is not stalled [This event is alias to IDQ_BUBBLES.CORE]", + "EventCode": "0x9c", + "EventName": "IDQ_UOPS_NOT_DELIVERED.CORE", + "PublicDescription": "Counts the number of uops not delivered to by the Instruction Decode Queue (IDQ) to the back-end of the pipeline when there was no back-end stalls. This event counts for one SMT thread in a given cycle. [This event is alias to IDQ_BUBBLES.CORE]", + "SampleAfterValue": "1000003", + "UMask": "0x1" + }, + { + "BriefDescription": "Cycles when no uops are not delivered by the IDQ when backend of the machine is not stalled [This event is alias to IDQ_BUBBLES.CYCLES_0_UOPS_DELIV.CORE]", "CounterMask": "6", "EventCode": "0x9c", "EventName": "IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE", - "PublicDescription": "Counts the number of cycles when no uops were delivered by the Instruction Decode Queue (IDQ) to the back-end of the pipeline when there was no back-end stalls. This event counts for one SMT thread in a given cycle.", + "PublicDescription": "Counts the number of cycles when no uops were delivered by the Instruction Decode Queue (IDQ) to the back-end of the pipeline when there was no back-end stalls. This event counts for one SMT thread in a given cycle. [This event is alias to IDQ_BUBBLES.CYCLES_0_UOPS_DELIV.CORE]", "SampleAfterValue": "1000003", "UMask": "0x1" }, { - "BriefDescription": "Cycles when optimal number of uops was delivered to the back-end when the back-end is not stalled", + "BriefDescription": "Cycles when optimal number of uops was delivered to the back-end when the back-end is not stalled [This event is alias to IDQ_BUBBLES.CYCLES_FE_WAS_OK]", "CounterMask": "1", "EventCode": "0x9c", "EventName": "IDQ_UOPS_NOT_DELIVERED.CYCLES_FE_WAS_OK", "Invert": "1", - "PublicDescription": "Counts the number of cycles when the optimal number of uops were delivered by the Instruction Decode Queue (IDQ) to the back-end of the pipeline when there was no back-end stalls. This event counts for one SMT thread in a given cycle.", + "PublicDescription": "Counts the number of cycles when the optimal number of uops were delivered by the Instruction Decode Queue (IDQ) to the back-end of the pipeline when there was no back-end stalls. This event counts for one SMT thread in a given cycle. [This event is alias to IDQ_BUBBLES.CYCLES_FE_WAS_OK]", "SampleAfterValue": "1000003", "UMask": "0x1" } diff --git a/tools/perf/pmu-events/arch/x86/sapphirerapids/uncore-cache.json b/tools/perf/pmu-events/arch/x86/sapphirerapids/uncore-cache.json index 3fa660694bc7..cf6fa70f37c1 100644 --- a/tools/perf/pmu-events/arch/x86/sapphirerapids/uncore-cache.json +++ b/tools/perf/pmu-events/arch/x86/sapphirerapids/uncore-cache.json @@ -4591,7 +4591,7 @@ }, { "BriefDescription": "UNC_CHA_TOR_OCCUPANCY.IA_HIT_CXL_ACC_LOCAL", - "EventCode": "0x35", + "EventCode": "0x36", "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_HIT_CXL_ACC_LOCAL", "PerPkg": "1", "PortMask": "0x000", @@ -4825,7 +4825,7 @@ }, { "BriefDescription": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_CXL_ACC_LOCAL", - "EventCode": "0x35", + "EventCode": "0x36", "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_CXL_ACC_LOCAL", "PerPkg": "1", "PortMask": "0x000", @@ -4870,7 +4870,7 @@ }, { "BriefDescription": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_CXL_ACC_LOCAL", - "EventCode": "0x35", + "EventCode": "0x36", "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_CXL_ACC_LOCAL", "PerPkg": "1", "PortMask": "0x000", @@ -4924,7 +4924,7 @@ }, { "BriefDescription": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_OPT_CXL_ACC_LOCAL", - "EventCode": "0x35", + "EventCode": "0x36", "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_OPT_CXL_ACC_LOCAL", "PerPkg": "1", "PortMask": "0x000", @@ -4942,7 +4942,7 @@ }, { "BriefDescription": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_OPT_PREF_CXL_ACC_LOCAL", - "EventCode": "0x35", + "EventCode": "0x36", "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_OPT_PREF_CXL_ACC_LOCAL", "PerPkg": "1", "PortMask": "0x000", @@ -4977,7 +4977,7 @@ }, { "BriefDescription": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PREF_CXL_ACC_LOCAL", - "EventCode": "0x35", + "EventCode": "0x36", "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PREF_CXL_ACC_LOCAL", "PerPkg": "1", "PortMask": "0x000", @@ -5128,7 +5128,7 @@ }, { "BriefDescription": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_LLCPREFDATA_CXL_ACC_LOCAL", - "EventCode": "0x35", + "EventCode": "0x36", "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_LLCPREFDATA_CXL_ACC_LOCAL", "PerPkg": "1", "PortMask": "0x000", @@ -5154,7 +5154,7 @@ }, { "BriefDescription": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_LLCPREFRFO_CXL_ACC_LOCAL", - "EventCode": "0x35", + "EventCode": "0x36", "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_LLCPREFRFO_CXL_ACC_LOCAL", "PerPkg": "1", "PortMask": "0x000", @@ -5260,7 +5260,7 @@ }, { "BriefDescription": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_RFO_CXL_ACC_LOCAL", - "EventCode": "0x35", + "EventCode": "0x36", "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_RFO_CXL_ACC_LOCAL", "PerPkg": "1", "PortMask": "0x000", @@ -5295,7 +5295,7 @@ }, { "BriefDescription": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_RFO_PREF_CXL_ACC_LOCAL", - "EventCode": "0x35", + "EventCode": "0x36", "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_RFO_PREF_CXL_ACC_LOCAL", "PerPkg": "1", "PortMask": "0x000", From b333067ff3b0ee63e226a06633f3df12491028ab Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Mon, 28 Aug 2023 17:17:30 -0700 Subject: [PATCH 036/475] perf vendor events intel: Fix spelling mistakes Update perf JSON files with spelling fixes by Colin Ian King contributed in: https://github.com/intel/perfmon/pull/96 "Fix various spelling mistakes and typos as found using codespell #96" This is added on top of the spelling mistakes and release number updates in: https://github.com/intel/perfmon/pull/98 "EMR, SPR, CLX, SKX, BDX, HSX, BDW-DE, WSM-EP*, NHM-*, JKT, IVT : Release event updates" Some additional spelling fixes reported by Edward Baker are added on top of this. Signed-off-by: Ian Rogers Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Alexandre Torgue Cc: Colin Ian King Cc: Edward Baker Cc: Ingo Molnar Cc: Jiri Olsa Cc: Kan Liang Cc: Mark Rutland Cc: Maxime Coquelin Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Xing Zhengjun Link: https://lore.kernel.org/r/20230829001730.1352769-3-irogers@google.com Signed-off-by: Arnaldo Carvalho de Melo --- .../arch/x86/alderlake/adl-metrics.json | 6 ++--- .../arch/x86/alderlake/pipeline.json | 2 +- .../arch/x86/alderlaken/adln-metrics.json | 6 ++--- .../pmu-events/arch/x86/bonnell/frontend.json | 2 +- .../x86/broadwellde/uncore-interconnect.json | 18 +++++++-------- .../x86/broadwellx/uncore-interconnect.json | 18 +++++++-------- .../pmu-events/arch/x86/haswell/memory.json | 2 +- .../pmu-events/arch/x86/haswellx/memory.json | 2 +- .../x86/haswellx/uncore-interconnect.json | 18 +++++++-------- .../pmu-events/arch/x86/ivybridge/cache.json | 8 +++---- .../arch/x86/ivybridge/frontend.json | 18 +++++++-------- .../pmu-events/arch/x86/ivytown/cache.json | 4 ++-- .../arch/x86/ivytown/uncore-interconnect.json | 20 ++++++++--------- .../x86/jaketown/uncore-interconnect.json | 18 +++++++-------- tools/perf/pmu-events/arch/x86/mapfile.csv | 22 +++++++++---------- .../pmu-events/arch/x86/nehalemep/cache.json | 2 +- .../arch/x86/nehalemep/floating-point.json | 2 +- .../pmu-events/arch/x86/nehalemex/cache.json | 2 +- .../arch/x86/nehalemex/floating-point.json | 2 +- .../x86/westmereep-dp/floating-point.json | 2 +- .../arch/x86/westmereep-sp/cache.json | 2 +- .../x86/westmereep-sp/floating-point.json | 2 +- .../pmu-events/arch/x86/westmereex/cache.json | 2 +- .../arch/x86/westmereex/floating-point.json | 2 +- 24 files changed, 91 insertions(+), 91 deletions(-) diff --git a/tools/perf/pmu-events/arch/x86/alderlake/adl-metrics.json b/tools/perf/pmu-events/arch/x86/alderlake/adl-metrics.json index c6780d5c456b..8b6bed3bc766 100644 --- a/tools/perf/pmu-events/arch/x86/alderlake/adl-metrics.json +++ b/tools/perf/pmu-events/arch/x86/alderlake/adl-metrics.json @@ -395,13 +395,13 @@ "Unit": "cpu_atom" }, { - "BriefDescription": "Instructions per Branch (lower number means higher occurance rate)", + "BriefDescription": "Instructions per Branch (lower number means higher occurrence rate)", "MetricExpr": "INST_RETIRED.ANY / BR_INST_RETIRED.ALL_BRANCHES", "MetricName": "tma_info_inst_mix_ipbranch", "Unit": "cpu_atom" }, { - "BriefDescription": "Instruction per (near) call (lower number means higher occurance rate)", + "BriefDescription": "Instruction per (near) call (lower number means higher occurrence rate)", "MetricExpr": "INST_RETIRED.ANY / BR_INST_RETIRED.CALL", "MetricName": "tma_info_inst_mix_ipcall", "Unit": "cpu_atom" @@ -726,7 +726,7 @@ "Unit": "cpu_atom" }, { - "BriefDescription": "Counts the numer of issue slots that result in retirement slots.", + "BriefDescription": "Counts the number of issue slots that result in retirement slots.", "DefaultMetricgroupName": "TopdownL1", "MetricExpr": "TOPDOWN_RETIRING.ALL / tma_info_core_slots", "MetricGroup": "Default;TopdownL1;tma_L1_group", diff --git a/tools/perf/pmu-events/arch/x86/alderlake/pipeline.json b/tools/perf/pmu-events/arch/x86/alderlake/pipeline.json index cb5b8611064b..a92013cdf136 100644 --- a/tools/perf/pmu-events/arch/x86/alderlake/pipeline.json +++ b/tools/perf/pmu-events/arch/x86/alderlake/pipeline.json @@ -1145,7 +1145,7 @@ "BriefDescription": "TMA slots wasted due to incorrect speculation by branch mispredictions", "EventCode": "0xa4", "EventName": "TOPDOWN.BR_MISPREDICT_SLOTS", - "PublicDescription": "Number of TMA slots that were wasted due to incorrect speculation by (any type of) branch mispredictions. This event estimates number of specualtive operations that were issued but not retired as well as the out-of-order engine recovery past a branch misprediction.", + "PublicDescription": "Number of TMA slots that were wasted due to incorrect speculation by (any type of) branch mispredictions. This event estimates number of speculative operations that were issued but not retired as well as the out-of-order engine recovery past a branch misprediction.", "SampleAfterValue": "10000003", "UMask": "0x8", "Unit": "cpu_core" diff --git a/tools/perf/pmu-events/arch/x86/alderlaken/adln-metrics.json b/tools/perf/pmu-events/arch/x86/alderlaken/adln-metrics.json index 06e67e34e1bf..c150c14ac6ed 100644 --- a/tools/perf/pmu-events/arch/x86/alderlaken/adln-metrics.json +++ b/tools/perf/pmu-events/arch/x86/alderlaken/adln-metrics.json @@ -328,12 +328,12 @@ "MetricName": "tma_info_inst_mix_idiv_uop_ratio" }, { - "BriefDescription": "Instructions per Branch (lower number means higher occurance rate)", + "BriefDescription": "Instructions per Branch (lower number means higher occurrence rate)", "MetricExpr": "INST_RETIRED.ANY / BR_INST_RETIRED.ALL_BRANCHES", "MetricName": "tma_info_inst_mix_ipbranch" }, { - "BriefDescription": "Instruction per (near) call (lower number means higher occurance rate)", + "BriefDescription": "Instruction per (near) call (lower number means higher occurrence rate)", "MetricExpr": "INST_RETIRED.ANY / BR_INST_RETIRED.CALL", "MetricName": "tma_info_inst_mix_ipcall" }, @@ -616,7 +616,7 @@ "ScaleUnit": "100%" }, { - "BriefDescription": "Counts the numer of issue slots that result in retirement slots.", + "BriefDescription": "Counts the number of issue slots that result in retirement slots.", "DefaultMetricgroupName": "TopdownL1", "MetricExpr": "TOPDOWN_RETIRING.ALL / tma_info_core_slots", "MetricGroup": "Default;TopdownL1;tma_L1_group", diff --git a/tools/perf/pmu-events/arch/x86/bonnell/frontend.json b/tools/perf/pmu-events/arch/x86/bonnell/frontend.json index 8d2f4edfb597..42284c02c11d 100644 --- a/tools/perf/pmu-events/arch/x86/bonnell/frontend.json +++ b/tools/perf/pmu-events/arch/x86/bonnell/frontend.json @@ -63,7 +63,7 @@ "UMask": "0x2" }, { - "BriefDescription": "Non-CISC nacro instructions decoded", + "BriefDescription": "Non-CISC macro instructions decoded", "EventCode": "0xAA", "EventName": "MACRO_INSTS.NON_CISC_DECODED", "SampleAfterValue": "2000000", diff --git a/tools/perf/pmu-events/arch/x86/broadwellde/uncore-interconnect.json b/tools/perf/pmu-events/arch/x86/broadwellde/uncore-interconnect.json index 8a327e0f1441..910395977a6e 100644 --- a/tools/perf/pmu-events/arch/x86/broadwellde/uncore-interconnect.json +++ b/tools/perf/pmu-events/arch/x86/broadwellde/uncore-interconnect.json @@ -253,7 +253,7 @@ "EventCode": "0x4", "EventName": "UNC_I_RxR_BL_DRS_CYCLES_FULL", "PerPkg": "1", - "PublicDescription": "Counts the number of cycles when the BL Ingress is full. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Counts the number of cycles when the BL Ingress is full. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { @@ -261,7 +261,7 @@ "EventCode": "0x1", "EventName": "UNC_I_RxR_BL_DRS_INSERTS", "PerPkg": "1", - "PublicDescription": "Counts the number of allocations into the BL Ingress. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Counts the number of allocations into the BL Ingress. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { @@ -269,7 +269,7 @@ "EventCode": "0x7", "EventName": "UNC_I_RxR_BL_DRS_OCCUPANCY", "PerPkg": "1", - "PublicDescription": "Accumulates the occupancy of the BL Ingress in each cycles. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Accumulates the occupancy of the BL Ingress in each cycles. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { @@ -277,7 +277,7 @@ "EventCode": "0x5", "EventName": "UNC_I_RxR_BL_NCB_CYCLES_FULL", "PerPkg": "1", - "PublicDescription": "Counts the number of cycles when the BL Ingress is full. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Counts the number of cycles when the BL Ingress is full. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { @@ -285,7 +285,7 @@ "EventCode": "0x2", "EventName": "UNC_I_RxR_BL_NCB_INSERTS", "PerPkg": "1", - "PublicDescription": "Counts the number of allocations into the BL Ingress. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Counts the number of allocations into the BL Ingress. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { @@ -293,7 +293,7 @@ "EventCode": "0x8", "EventName": "UNC_I_RxR_BL_NCB_OCCUPANCY", "PerPkg": "1", - "PublicDescription": "Accumulates the occupancy of the BL Ingress in each cycles. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Accumulates the occupancy of the BL Ingress in each cycles. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { @@ -301,7 +301,7 @@ "EventCode": "0x6", "EventName": "UNC_I_RxR_BL_NCS_CYCLES_FULL", "PerPkg": "1", - "PublicDescription": "Counts the number of cycles when the BL Ingress is full. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Counts the number of cycles when the BL Ingress is full. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { @@ -309,7 +309,7 @@ "EventCode": "0x3", "EventName": "UNC_I_RxR_BL_NCS_INSERTS", "PerPkg": "1", - "PublicDescription": "Counts the number of allocations into the BL Ingress. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Counts the number of allocations into the BL Ingress. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { @@ -317,7 +317,7 @@ "EventCode": "0x9", "EventName": "UNC_I_RxR_BL_NCS_OCCUPANCY", "PerPkg": "1", - "PublicDescription": "Accumulates the occupancy of the BL Ingress in each cycles. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Accumulates the occupancy of the BL Ingress in each cycles. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { diff --git a/tools/perf/pmu-events/arch/x86/broadwellx/uncore-interconnect.json b/tools/perf/pmu-events/arch/x86/broadwellx/uncore-interconnect.json index e61a23f68899..b9fb216bee16 100644 --- a/tools/perf/pmu-events/arch/x86/broadwellx/uncore-interconnect.json +++ b/tools/perf/pmu-events/arch/x86/broadwellx/uncore-interconnect.json @@ -271,7 +271,7 @@ "EventCode": "0x4", "EventName": "UNC_I_RxR_BL_DRS_CYCLES_FULL", "PerPkg": "1", - "PublicDescription": "Counts the number of cycles when the BL Ingress is full. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Counts the number of cycles when the BL Ingress is full. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { @@ -279,7 +279,7 @@ "EventCode": "0x1", "EventName": "UNC_I_RxR_BL_DRS_INSERTS", "PerPkg": "1", - "PublicDescription": "Counts the number of allocations into the BL Ingress. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Counts the number of allocations into the BL Ingress. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { @@ -287,7 +287,7 @@ "EventCode": "0x7", "EventName": "UNC_I_RxR_BL_DRS_OCCUPANCY", "PerPkg": "1", - "PublicDescription": "Accumulates the occupancy of the BL Ingress in each cycles. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Accumulates the occupancy of the BL Ingress in each cycles. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { @@ -295,7 +295,7 @@ "EventCode": "0x5", "EventName": "UNC_I_RxR_BL_NCB_CYCLES_FULL", "PerPkg": "1", - "PublicDescription": "Counts the number of cycles when the BL Ingress is full. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Counts the number of cycles when the BL Ingress is full. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { @@ -303,7 +303,7 @@ "EventCode": "0x2", "EventName": "UNC_I_RxR_BL_NCB_INSERTS", "PerPkg": "1", - "PublicDescription": "Counts the number of allocations into the BL Ingress. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Counts the number of allocations into the BL Ingress. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { @@ -311,7 +311,7 @@ "EventCode": "0x8", "EventName": "UNC_I_RxR_BL_NCB_OCCUPANCY", "PerPkg": "1", - "PublicDescription": "Accumulates the occupancy of the BL Ingress in each cycles. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Accumulates the occupancy of the BL Ingress in each cycles. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { @@ -319,7 +319,7 @@ "EventCode": "0x6", "EventName": "UNC_I_RxR_BL_NCS_CYCLES_FULL", "PerPkg": "1", - "PublicDescription": "Counts the number of cycles when the BL Ingress is full. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Counts the number of cycles when the BL Ingress is full. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { @@ -327,7 +327,7 @@ "EventCode": "0x3", "EventName": "UNC_I_RxR_BL_NCS_INSERTS", "PerPkg": "1", - "PublicDescription": "Counts the number of allocations into the BL Ingress. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Counts the number of allocations into the BL Ingress. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { @@ -335,7 +335,7 @@ "EventCode": "0x9", "EventName": "UNC_I_RxR_BL_NCS_OCCUPANCY", "PerPkg": "1", - "PublicDescription": "Accumulates the occupancy of the BL Ingress in each cycles. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Accumulates the occupancy of the BL Ingress in each cycles. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { diff --git a/tools/perf/pmu-events/arch/x86/haswell/memory.json b/tools/perf/pmu-events/arch/x86/haswell/memory.json index 2fc25e22a42a..df44c28efeeb 100644 --- a/tools/perf/pmu-events/arch/x86/haswell/memory.json +++ b/tools/perf/pmu-events/arch/x86/haswell/memory.json @@ -62,7 +62,7 @@ "BriefDescription": "Counts the number of machine clears due to memory order conflicts.", "EventCode": "0xC3", "EventName": "MACHINE_CLEARS.MEMORY_ORDERING", - "PublicDescription": "This event counts the number of memory ordering machine clears detected. Memory ordering machine clears can result from memory address aliasing or snoops from another hardware thread or core to data inflight in the pipeline. Machine clears can have a significant performance impact if they are happening frequently.", + "PublicDescription": "This event counts the number of memory ordering machine clears detected. Memory ordering machine clears can result from memory address aliasing or snoops from another hardware thread or core to data in-flight in the pipeline. Machine clears can have a significant performance impact if they are happening frequently.", "SampleAfterValue": "100003", "UMask": "0x2" }, diff --git a/tools/perf/pmu-events/arch/x86/haswellx/memory.json b/tools/perf/pmu-events/arch/x86/haswellx/memory.json index 2d212cf59e92..d66e465ce41a 100644 --- a/tools/perf/pmu-events/arch/x86/haswellx/memory.json +++ b/tools/perf/pmu-events/arch/x86/haswellx/memory.json @@ -62,7 +62,7 @@ "BriefDescription": "Counts the number of machine clears due to memory order conflicts.", "EventCode": "0xC3", "EventName": "MACHINE_CLEARS.MEMORY_ORDERING", - "PublicDescription": "This event counts the number of memory ordering machine clears detected. Memory ordering machine clears can result from memory address aliasing or snoops from another hardware thread or core to data inflight in the pipeline. Machine clears can have a significant performance impact if they are happening frequently.", + "PublicDescription": "This event counts the number of memory ordering machine clears detected. Memory ordering machine clears can result from memory address aliasing or snoops from another hardware thread or core to data in-flight in the pipeline. Machine clears can have a significant performance impact if they are happening frequently.", "SampleAfterValue": "100003", "UMask": "0x2" }, diff --git a/tools/perf/pmu-events/arch/x86/haswellx/uncore-interconnect.json b/tools/perf/pmu-events/arch/x86/haswellx/uncore-interconnect.json index 954e8198c7a5..bef1f5ef6f31 100644 --- a/tools/perf/pmu-events/arch/x86/haswellx/uncore-interconnect.json +++ b/tools/perf/pmu-events/arch/x86/haswellx/uncore-interconnect.json @@ -271,7 +271,7 @@ "EventCode": "0x4", "EventName": "UNC_I_RxR_BL_DRS_CYCLES_FULL", "PerPkg": "1", - "PublicDescription": "Counts the number of cycles when the BL Ingress is full. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Counts the number of cycles when the BL Ingress is full. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { @@ -279,7 +279,7 @@ "EventCode": "0x1", "EventName": "UNC_I_RxR_BL_DRS_INSERTS", "PerPkg": "1", - "PublicDescription": "Counts the number of allocations into the BL Ingress. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Counts the number of allocations into the BL Ingress. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { @@ -287,7 +287,7 @@ "EventCode": "0x7", "EventName": "UNC_I_RxR_BL_DRS_OCCUPANCY", "PerPkg": "1", - "PublicDescription": "Accumulates the occupancy of the BL Ingress in each cycles. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Accumulates the occupancy of the BL Ingress in each cycles. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { @@ -295,7 +295,7 @@ "EventCode": "0x5", "EventName": "UNC_I_RxR_BL_NCB_CYCLES_FULL", "PerPkg": "1", - "PublicDescription": "Counts the number of cycles when the BL Ingress is full. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Counts the number of cycles when the BL Ingress is full. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { @@ -303,7 +303,7 @@ "EventCode": "0x2", "EventName": "UNC_I_RxR_BL_NCB_INSERTS", "PerPkg": "1", - "PublicDescription": "Counts the number of allocations into the BL Ingress. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Counts the number of allocations into the BL Ingress. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { @@ -311,7 +311,7 @@ "EventCode": "0x8", "EventName": "UNC_I_RxR_BL_NCB_OCCUPANCY", "PerPkg": "1", - "PublicDescription": "Accumulates the occupancy of the BL Ingress in each cycles. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Accumulates the occupancy of the BL Ingress in each cycles. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { @@ -319,7 +319,7 @@ "EventCode": "0x6", "EventName": "UNC_I_RxR_BL_NCS_CYCLES_FULL", "PerPkg": "1", - "PublicDescription": "Counts the number of cycles when the BL Ingress is full. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Counts the number of cycles when the BL Ingress is full. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { @@ -327,7 +327,7 @@ "EventCode": "0x3", "EventName": "UNC_I_RxR_BL_NCS_INSERTS", "PerPkg": "1", - "PublicDescription": "Counts the number of allocations into the BL Ingress. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Counts the number of allocations into the BL Ingress. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { @@ -335,7 +335,7 @@ "EventCode": "0x9", "EventName": "UNC_I_RxR_BL_NCS_OCCUPANCY", "PerPkg": "1", - "PublicDescription": "Accumulates the occupancy of the BL Ingress in each cycles. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Accumulates the occupancy of the BL Ingress in each cycles. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { diff --git a/tools/perf/pmu-events/arch/x86/ivybridge/cache.json b/tools/perf/pmu-events/arch/x86/ivybridge/cache.json index 6ddc7d1c61d5..46570b522095 100644 --- a/tools/perf/pmu-events/arch/x86/ivybridge/cache.json +++ b/tools/perf/pmu-events/arch/x86/ivybridge/cache.json @@ -8,16 +8,16 @@ "UMask": "0x1" }, { - "BriefDescription": "Cycles a demand request was blocked due to Fill Buffers inavailability", + "BriefDescription": "Cycles a demand request was blocked due to Fill Buffers unavailability", "CounterMask": "1", "EventCode": "0x48", "EventName": "L1D_PEND_MISS.FB_FULL", - "PublicDescription": "Cycles a demand request was blocked due to Fill Buffers inavailability.", + "PublicDescription": "Cycles a demand request was blocked due to Fill Buffers unavailability.", "SampleAfterValue": "2000003", "UMask": "0x2" }, { - "BriefDescription": "L1D miss oustandings duration in cycles", + "BriefDescription": "L1D miss outstanding duration in cycles", "EventCode": "0x48", "EventName": "L1D_PEND_MISS.PENDING", "PublicDescription": "Increments the number of outstanding L1D misses every cycle. Set Cmask = 1 and Edge =1 to count occurrences.", @@ -506,7 +506,7 @@ "UMask": "0x8" }, { - "BriefDescription": "Cacheable and noncachaeble code read requests", + "BriefDescription": "Cacheable and noncacheable code read requests", "EventCode": "0xB0", "EventName": "OFFCORE_REQUESTS.DEMAND_CODE_RD", "PublicDescription": "Demand code read requests sent to uncore.", diff --git a/tools/perf/pmu-events/arch/x86/ivybridge/frontend.json b/tools/perf/pmu-events/arch/x86/ivybridge/frontend.json index 89004a6c9ed1..4ee100024ca9 100644 --- a/tools/perf/pmu-events/arch/x86/ivybridge/frontend.json +++ b/tools/perf/pmu-events/arch/x86/ivybridge/frontend.json @@ -142,35 +142,35 @@ "UMask": "0x4" }, { - "BriefDescription": "Cycles when uops are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy", + "BriefDescription": "Cycles when uops are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy", "CounterMask": "1", "EventCode": "0x79", "EventName": "IDQ.MS_CYCLES", - "PublicDescription": "Cycles when uops are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy.", + "PublicDescription": "Cycles when uops are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy.", "SampleAfterValue": "2000003", "UMask": "0x30" }, { - "BriefDescription": "Cycles when uops initiated by Decode Stream Buffer (DSB) are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy", + "BriefDescription": "Cycles when uops initiated by Decode Stream Buffer (DSB) are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy", "CounterMask": "1", "EventCode": "0x79", "EventName": "IDQ.MS_DSB_CYCLES", - "PublicDescription": "Cycles when uops initiated by Decode Stream Buffer (DSB) are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy.", + "PublicDescription": "Cycles when uops initiated by Decode Stream Buffer (DSB) are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy.", "SampleAfterValue": "2000003", "UMask": "0x10" }, { - "BriefDescription": "Deliveries to Instruction Decode Queue (IDQ) initiated by Decode Stream Buffer (DSB) while Microcode Sequenser (MS) is busy", + "BriefDescription": "Deliveries to Instruction Decode Queue (IDQ) initiated by Decode Stream Buffer (DSB) while Microcode Sequencer (MS) is busy", "CounterMask": "1", "EdgeDetect": "1", "EventCode": "0x79", "EventName": "IDQ.MS_DSB_OCCUR", - "PublicDescription": "Deliveries to Instruction Decode Queue (IDQ) initiated by Decode Stream Buffer (DSB) while Microcode Sequenser (MS) is busy.", + "PublicDescription": "Deliveries to Instruction Decode Queue (IDQ) initiated by Decode Stream Buffer (DSB) while Microcode Sequencer (MS) is busy.", "SampleAfterValue": "2000003", "UMask": "0x10" }, { - "BriefDescription": "Uops initiated by Decode Stream Buffer (DSB) that are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy", + "BriefDescription": "Uops initiated by Decode Stream Buffer (DSB) that are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy", "EventCode": "0x79", "EventName": "IDQ.MS_DSB_UOPS", "PublicDescription": "Increment each cycle # of uops delivered to IDQ when MS_busy by DSB. Set Cmask = 1 to count cycles. Add Edge=1 to count # of delivery.", @@ -178,7 +178,7 @@ "UMask": "0x10" }, { - "BriefDescription": "Uops initiated by MITE and delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy", + "BriefDescription": "Uops initiated by MITE and delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy", "EventCode": "0x79", "EventName": "IDQ.MS_MITE_UOPS", "PublicDescription": "Increment each cycle # of uops delivered to IDQ when MS_busy by MITE. Set Cmask = 1 to count cycles.", @@ -196,7 +196,7 @@ "UMask": "0x30" }, { - "BriefDescription": "Uops delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy", + "BriefDescription": "Uops delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy", "EventCode": "0x79", "EventName": "IDQ.MS_UOPS", "PublicDescription": "Increment each cycle # of uops delivered to IDQ from MS by either DSB or MITE. Set Cmask = 1 to count cycles.", diff --git a/tools/perf/pmu-events/arch/x86/ivytown/cache.json b/tools/perf/pmu-events/arch/x86/ivytown/cache.json index c8f7d5e66504..0e8e77253978 100644 --- a/tools/perf/pmu-events/arch/x86/ivytown/cache.json +++ b/tools/perf/pmu-events/arch/x86/ivytown/cache.json @@ -8,11 +8,11 @@ "UMask": "0x1" }, { - "BriefDescription": "Cycles a demand request was blocked due to Fill Buffers inavailability", + "BriefDescription": "Cycles a demand request was blocked due to Fill Buffers unavailability", "CounterMask": "1", "EventCode": "0x48", "EventName": "L1D_PEND_MISS.FB_FULL", - "PublicDescription": "Cycles a demand request was blocked due to Fill Buffers inavailability.", + "PublicDescription": "Cycles a demand request was blocked due to Fill Buffers unavailability.", "SampleAfterValue": "2000003", "UMask": "0x2" }, diff --git a/tools/perf/pmu-events/arch/x86/ivytown/uncore-interconnect.json b/tools/perf/pmu-events/arch/x86/ivytown/uncore-interconnect.json index ccf451534d16..914d2cfb3d3d 100644 --- a/tools/perf/pmu-events/arch/x86/ivytown/uncore-interconnect.json +++ b/tools/perf/pmu-events/arch/x86/ivytown/uncore-interconnect.json @@ -140,7 +140,7 @@ "EventCode": "0x4", "EventName": "UNC_I_RxR_BL_DRS_CYCLES_FULL", "PerPkg": "1", - "PublicDescription": "Counts the number of cycles when the BL Ingress is full. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Counts the number of cycles when the BL Ingress is full. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { @@ -148,21 +148,21 @@ "EventCode": "0x1", "EventName": "UNC_I_RxR_BL_DRS_INSERTS", "PerPkg": "1", - "PublicDescription": "Counts the number of allocations into the BL Ingress. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Counts the number of allocations into the BL Ingress. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { "EventCode": "0x7", "EventName": "UNC_I_RxR_BL_DRS_OCCUPANCY", "PerPkg": "1", - "PublicDescription": "Accumulates the occupancy of the BL Ingress in each cycles. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Accumulates the occupancy of the BL Ingress in each cycles. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { "EventCode": "0x5", "EventName": "UNC_I_RxR_BL_NCB_CYCLES_FULL", "PerPkg": "1", - "PublicDescription": "Counts the number of cycles when the BL Ingress is full. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Counts the number of cycles when the BL Ingress is full. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { @@ -170,21 +170,21 @@ "EventCode": "0x2", "EventName": "UNC_I_RxR_BL_NCB_INSERTS", "PerPkg": "1", - "PublicDescription": "Counts the number of allocations into the BL Ingress. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Counts the number of allocations into the BL Ingress. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { "EventCode": "0x8", "EventName": "UNC_I_RxR_BL_NCB_OCCUPANCY", "PerPkg": "1", - "PublicDescription": "Accumulates the occupancy of the BL Ingress in each cycles. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Accumulates the occupancy of the BL Ingress in each cycles. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { "EventCode": "0x6", "EventName": "UNC_I_RxR_BL_NCS_CYCLES_FULL", "PerPkg": "1", - "PublicDescription": "Counts the number of cycles when the BL Ingress is full. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Counts the number of cycles when the BL Ingress is full. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { @@ -192,14 +192,14 @@ "EventCode": "0x3", "EventName": "UNC_I_RxR_BL_NCS_INSERTS", "PerPkg": "1", - "PublicDescription": "Counts the number of allocations into the BL Ingress. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Counts the number of allocations into the BL Ingress. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { "EventCode": "0x9", "EventName": "UNC_I_RxR_BL_NCS_OCCUPANCY", "PerPkg": "1", - "PublicDescription": "Accumulates the occupancy of the BL Ingress in each cycles. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Accumulates the occupancy of the BL Ingress in each cycles. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { @@ -293,7 +293,7 @@ "EventCode": "0xd", "EventName": "UNC_I_TxR_REQUEST_OCCUPANCY", "PerPkg": "1", - "PublicDescription": "Accumultes the number of outstanding outbound requests from the IRP to the switch (towards the devices). This can be used in conjuection with the allocations event in order to calculate average latency of outbound requests.", + "PublicDescription": "Accumulates the number of outstanding outbound requests from the IRP to the switch (towards the devices). This can be used in conjunction with the allocations event in order to calculate average latency of outbound requests.", "Unit": "IRP" }, { diff --git a/tools/perf/pmu-events/arch/x86/jaketown/uncore-interconnect.json b/tools/perf/pmu-events/arch/x86/jaketown/uncore-interconnect.json index 874f15ea8228..0fc907e5cf3c 100644 --- a/tools/perf/pmu-events/arch/x86/jaketown/uncore-interconnect.json +++ b/tools/perf/pmu-events/arch/x86/jaketown/uncore-interconnect.json @@ -140,7 +140,7 @@ "EventCode": "0x4", "EventName": "UNC_I_RxR_BL_DRS_CYCLES_FULL", "PerPkg": "1", - "PublicDescription": "Counts the number of cycles when the BL Ingress is full. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Counts the number of cycles when the BL Ingress is full. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { @@ -148,21 +148,21 @@ "EventCode": "0x1", "EventName": "UNC_I_RxR_BL_DRS_INSERTS", "PerPkg": "1", - "PublicDescription": "Counts the number of allocations into the BL Ingress. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Counts the number of allocations into the BL Ingress. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { "EventCode": "0x7", "EventName": "UNC_I_RxR_BL_DRS_OCCUPANCY", "PerPkg": "1", - "PublicDescription": "Accumulates the occupancy of the BL Ingress in each cycles. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Accumulates the occupancy of the BL Ingress in each cycles. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { "EventCode": "0x5", "EventName": "UNC_I_RxR_BL_NCB_CYCLES_FULL", "PerPkg": "1", - "PublicDescription": "Counts the number of cycles when the BL Ingress is full. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Counts the number of cycles when the BL Ingress is full. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { @@ -170,21 +170,21 @@ "EventCode": "0x2", "EventName": "UNC_I_RxR_BL_NCB_INSERTS", "PerPkg": "1", - "PublicDescription": "Counts the number of allocations into the BL Ingress. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Counts the number of allocations into the BL Ingress. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { "EventCode": "0x8", "EventName": "UNC_I_RxR_BL_NCB_OCCUPANCY", "PerPkg": "1", - "PublicDescription": "Accumulates the occupancy of the BL Ingress in each cycles. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Accumulates the occupancy of the BL Ingress in each cycles. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { "EventCode": "0x6", "EventName": "UNC_I_RxR_BL_NCS_CYCLES_FULL", "PerPkg": "1", - "PublicDescription": "Counts the number of cycles when the BL Ingress is full. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Counts the number of cycles when the BL Ingress is full. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { @@ -192,14 +192,14 @@ "EventCode": "0x3", "EventName": "UNC_I_RxR_BL_NCS_INSERTS", "PerPkg": "1", - "PublicDescription": "Counts the number of allocations into the BL Ingress. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Counts the number of allocations into the BL Ingress. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { "EventCode": "0x9", "EventName": "UNC_I_RxR_BL_NCS_OCCUPANCY", "PerPkg": "1", - "PublicDescription": "Accumulates the occupancy of the BL Ingress in each cycles. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requets as well as outbound MMIO writes.", + "PublicDescription": "Accumulates the occupancy of the BL Ingress in each cycles. This queue is where the IRP receives data from R2PCIe (the ring). It is used for data returns from read requests as well as outbound MMIO writes.", "Unit": "IRP" }, { diff --git a/tools/perf/pmu-events/arch/x86/mapfile.csv b/tools/perf/pmu-events/arch/x86/mapfile.csv index dca2133473e3..57ba7e814563 100644 --- a/tools/perf/pmu-events/arch/x86/mapfile.csv +++ b/tools/perf/pmu-events/arch/x86/mapfile.csv @@ -3,9 +3,9 @@ GenuineIntel-6-(97|9A|B7|BA|BF),v1.21,alderlake,core GenuineIntel-6-BE,v1.21,alderlaken,core GenuineIntel-6-(1C|26|27|35|36),v4,bonnell,core GenuineIntel-6-(3D|47),v28,broadwell,core -GenuineIntel-6-56,v10,broadwellde,core -GenuineIntel-6-4F,v21,broadwellx,core -GenuineIntel-6-55-[56789ABCDEF],v1.19,cascadelakex,core +GenuineIntel-6-56,v11,broadwellde,core +GenuineIntel-6-4F,v22,broadwellx,core +GenuineIntel-6-55-[56789ABCDEF],v1.20,cascadelakex,core GenuineIntel-6-9[6C],v1.04,elkhartlake,core GenuineIntel-6-CF,v1.00,emeraldrapids,core GenuineIntel-6-5[CF],v13,goldmont,core @@ -13,28 +13,28 @@ GenuineIntel-6-7A,v1.01,goldmontplus,core GenuineIntel-6-B6,v1.00,grandridge,core GenuineIntel-6-A[DE],v1.01,graniterapids,core GenuineIntel-6-(3C|45|46),v33,haswell,core -GenuineIntel-6-3F,v27,haswellx,core +GenuineIntel-6-3F,v28,haswellx,core GenuineIntel-6-7[DE],v1.19,icelake,core GenuineIntel-6-6[AC],v1.21,icelakex,core GenuineIntel-6-3A,v24,ivybridge,core -GenuineIntel-6-3E,v23,ivytown,core -GenuineIntel-6-2D,v23,jaketown,core +GenuineIntel-6-3E,v24,ivytown,core +GenuineIntel-6-2D,v24,jaketown,core GenuineIntel-6-(57|85),v10,knightslanding,core GenuineIntel-6-BD,v1.00,lunarlake,core GenuineIntel-6-A[AC],v1.04,meteorlake,core -GenuineIntel-6-1[AEF],v3,nehalemep,core -GenuineIntel-6-2E,v3,nehalemex,core +GenuineIntel-6-1[AEF],v4,nehalemep,core +GenuineIntel-6-2E,v4,nehalemex,core GenuineIntel-6-A7,v1.01,rocketlake,core GenuineIntel-6-2A,v19,sandybridge,core GenuineIntel-6-8F,v1.16,sapphirerapids,core GenuineIntel-6-AF,v1.00,sierraforest,core GenuineIntel-6-(37|4A|4C|4D|5A),v15,silvermont,core GenuineIntel-6-(4E|5E|8E|9E|A5|A6),v57,skylake,core -GenuineIntel-6-55-[01234],v1.31,skylakex,core +GenuineIntel-6-55-[01234],v1.32,skylakex,core GenuineIntel-6-86,v1.21,snowridgex,core GenuineIntel-6-8[CD],v1.13,tigerlake,core -GenuineIntel-6-2C,v4,westmereep-dp,core -GenuineIntel-6-25,v3,westmereep-sp,core +GenuineIntel-6-2C,v5,westmereep-dp,core +GenuineIntel-6-25,v4,westmereep-sp,core GenuineIntel-6-2F,v3,westmereex,core AuthenticAMD-23-([12][0-9A-F]|[0-9A-F]),v2,amdzen1,core AuthenticAMD-23-[[:xdigit:]]+,v1,amdzen2,core diff --git a/tools/perf/pmu-events/arch/x86/nehalemep/cache.json b/tools/perf/pmu-events/arch/x86/nehalemep/cache.json index 1a132fcda964..5113a4e059e4 100644 --- a/tools/perf/pmu-events/arch/x86/nehalemep/cache.json +++ b/tools/perf/pmu-events/arch/x86/nehalemep/cache.json @@ -287,7 +287,7 @@ "UMask": "0x20" }, { - "BriefDescription": "L2 lines alloacated", + "BriefDescription": "L2 lines allocated", "EventCode": "0xF1", "EventName": "L2_LINES_IN.ANY", "SampleAfterValue": "100000", diff --git a/tools/perf/pmu-events/arch/x86/nehalemep/floating-point.json b/tools/perf/pmu-events/arch/x86/nehalemep/floating-point.json index c03f8990fa82..196ae1d9b157 100644 --- a/tools/perf/pmu-events/arch/x86/nehalemep/floating-point.json +++ b/tools/perf/pmu-events/arch/x86/nehalemep/floating-point.json @@ -8,7 +8,7 @@ "UMask": "0x1" }, { - "BriefDescription": "X87 Floating poiint assists for invalid input value (Precise Event)", + "BriefDescription": "X87 Floating point assists for invalid input value (Precise Event)", "EventCode": "0xF7", "EventName": "FP_ASSIST.INPUT", "PEBS": "1", diff --git a/tools/perf/pmu-events/arch/x86/nehalemex/cache.json b/tools/perf/pmu-events/arch/x86/nehalemex/cache.json index a4142cd2ca86..0042e53fdc78 100644 --- a/tools/perf/pmu-events/arch/x86/nehalemex/cache.json +++ b/tools/perf/pmu-events/arch/x86/nehalemex/cache.json @@ -287,7 +287,7 @@ "UMask": "0x20" }, { - "BriefDescription": "L2 lines alloacated", + "BriefDescription": "L2 lines allocated", "EventCode": "0xF1", "EventName": "L2_LINES_IN.ANY", "SampleAfterValue": "100000", diff --git a/tools/perf/pmu-events/arch/x86/nehalemex/floating-point.json b/tools/perf/pmu-events/arch/x86/nehalemex/floating-point.json index c03f8990fa82..196ae1d9b157 100644 --- a/tools/perf/pmu-events/arch/x86/nehalemex/floating-point.json +++ b/tools/perf/pmu-events/arch/x86/nehalemex/floating-point.json @@ -8,7 +8,7 @@ "UMask": "0x1" }, { - "BriefDescription": "X87 Floating poiint assists for invalid input value (Precise Event)", + "BriefDescription": "X87 Floating point assists for invalid input value (Precise Event)", "EventCode": "0xF7", "EventName": "FP_ASSIST.INPUT", "PEBS": "1", diff --git a/tools/perf/pmu-events/arch/x86/westmereep-dp/floating-point.json b/tools/perf/pmu-events/arch/x86/westmereep-dp/floating-point.json index c03f8990fa82..196ae1d9b157 100644 --- a/tools/perf/pmu-events/arch/x86/westmereep-dp/floating-point.json +++ b/tools/perf/pmu-events/arch/x86/westmereep-dp/floating-point.json @@ -8,7 +8,7 @@ "UMask": "0x1" }, { - "BriefDescription": "X87 Floating poiint assists for invalid input value (Precise Event)", + "BriefDescription": "X87 Floating point assists for invalid input value (Precise Event)", "EventCode": "0xF7", "EventName": "FP_ASSIST.INPUT", "PEBS": "1", diff --git a/tools/perf/pmu-events/arch/x86/westmereep-sp/cache.json b/tools/perf/pmu-events/arch/x86/westmereep-sp/cache.json index e00c301640f3..d025e2c0cf1c 100644 --- a/tools/perf/pmu-events/arch/x86/westmereep-sp/cache.json +++ b/tools/perf/pmu-events/arch/x86/westmereep-sp/cache.json @@ -182,7 +182,7 @@ "UMask": "0x20" }, { - "BriefDescription": "L2 lines alloacated", + "BriefDescription": "L2 lines allocated", "EventCode": "0xF1", "EventName": "L2_LINES_IN.ANY", "SampleAfterValue": "100000", diff --git a/tools/perf/pmu-events/arch/x86/westmereep-sp/floating-point.json b/tools/perf/pmu-events/arch/x86/westmereep-sp/floating-point.json index c03f8990fa82..196ae1d9b157 100644 --- a/tools/perf/pmu-events/arch/x86/westmereep-sp/floating-point.json +++ b/tools/perf/pmu-events/arch/x86/westmereep-sp/floating-point.json @@ -8,7 +8,7 @@ "UMask": "0x1" }, { - "BriefDescription": "X87 Floating poiint assists for invalid input value (Precise Event)", + "BriefDescription": "X87 Floating point assists for invalid input value (Precise Event)", "EventCode": "0xF7", "EventName": "FP_ASSIST.INPUT", "PEBS": "1", diff --git a/tools/perf/pmu-events/arch/x86/westmereex/cache.json b/tools/perf/pmu-events/arch/x86/westmereex/cache.json index 6c7c52733dda..18d61d43e4c9 100644 --- a/tools/perf/pmu-events/arch/x86/westmereex/cache.json +++ b/tools/perf/pmu-events/arch/x86/westmereex/cache.json @@ -182,7 +182,7 @@ "UMask": "0x20" }, { - "BriefDescription": "L2 lines alloacated", + "BriefDescription": "L2 lines allocated", "EventCode": "0xF1", "EventName": "L2_LINES_IN.ANY", "SampleAfterValue": "100000", diff --git a/tools/perf/pmu-events/arch/x86/westmereex/floating-point.json b/tools/perf/pmu-events/arch/x86/westmereex/floating-point.json index c03f8990fa82..196ae1d9b157 100644 --- a/tools/perf/pmu-events/arch/x86/westmereex/floating-point.json +++ b/tools/perf/pmu-events/arch/x86/westmereex/floating-point.json @@ -8,7 +8,7 @@ "UMask": "0x1" }, { - "BriefDescription": "X87 Floating poiint assists for invalid input value (Precise Event)", + "BriefDescription": "X87 Floating point assists for invalid input value (Precise Event)", "EventCode": "0xF7", "EventName": "FP_ASSIST.INPUT", "PEBS": "1", From f6ff1c760431be34e4daaa44f242be911becd998 Mon Sep 17 00:00:00 2001 From: Yang Jihong Date: Mon, 4 Sep 2023 02:33:35 +0000 Subject: [PATCH 037/475] perf evlist: Add perf_evlist__go_system_wide() helper For dummy events that keep tracking, we may need to modify its cpu_maps. For example, change the cpu_maps to record sideband events for all CPUS. Add perf_evlist__go_system_wide() helper to support this scenario. Signed-off-by: Yang Jihong Acked-by: Adrian Hunter Tested-by: Ravi Bangoria Cc: Alexander Shishkin Cc: Andi Kleen Cc: Anshuman Khandual Cc: Ian Rogers Cc: Ingo Molnar Cc: James Clark Cc: Jiri Olsa Cc: Kan Liang Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Thomas Richter Link: https://lore.kernel.org/r/20230904023340.12707-2-yangjihong1@huawei.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/lib/perf/evlist.c | 9 +++++++++ tools/lib/perf/include/internal/evlist.h | 2 ++ 2 files changed, 11 insertions(+) diff --git a/tools/lib/perf/evlist.c b/tools/lib/perf/evlist.c index b8b066d0dc5e..3acbbccc1901 100644 --- a/tools/lib/perf/evlist.c +++ b/tools/lib/perf/evlist.c @@ -738,3 +738,12 @@ int perf_evlist__nr_groups(struct perf_evlist *evlist) } return nr_groups; } + +void perf_evlist__go_system_wide(struct perf_evlist *evlist, struct perf_evsel *evsel) +{ + if (!evsel->system_wide) { + evsel->system_wide = true; + if (evlist->needs_map_propagation) + __perf_evlist__propagate_maps(evlist, evsel); + } +} diff --git a/tools/lib/perf/include/internal/evlist.h b/tools/lib/perf/include/internal/evlist.h index 3339bc2f1765..d86ffe8ed483 100644 --- a/tools/lib/perf/include/internal/evlist.h +++ b/tools/lib/perf/include/internal/evlist.h @@ -135,4 +135,6 @@ int perf_evlist__id_add_fd(struct perf_evlist *evlist, void perf_evlist__reset_id_hash(struct perf_evlist *evlist); void __perf_evlist__set_leader(struct list_head *list, struct perf_evsel *leader); + +void perf_evlist__go_system_wide(struct perf_evlist *evlist, struct perf_evsel *evsel); #endif /* __LIBPERF_INTERNAL_EVLIST_H */ From 9c95e4ef065723496442898614d09a9a916eab81 Mon Sep 17 00:00:00 2001 From: Yang Jihong Date: Mon, 4 Sep 2023 02:33:36 +0000 Subject: [PATCH 038/475] perf evlist: Add evlist__findnew_tracking_event() helper Currently, intel-bts, intel-pt, and arm-spe may add tracking event to the evlist. We may need to search for the tracking event for some settings. Therefore, add evlist__findnew_tracking_event() helper. If system_wide is true, evlist__findnew_tracking_event() set the cpu map of the evsel to all online CPUs. Signed-off-by: Yang Jihong Acked-by: Adrian Hunter Tested-by: Ravi Bangoria Cc: Alexander Shishkin Cc: Andi Kleen Cc: Anshuman Khandual Cc: Ian Rogers Cc: Ingo Molnar Cc: James Clark Cc: Jiri Olsa Cc: Kan Liang Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Thomas Richter Link: https://lore.kernel.org/r/20230904023340.12707-3-yangjihong1@huawei.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/builtin-record.c | 11 +++-------- tools/perf/util/evlist.c | 18 ++++++++++++++++++ tools/perf/util/evlist.h | 1 + 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/tools/perf/builtin-record.c b/tools/perf/builtin-record.c index 34bb31f08bb5..12edad8392cc 100644 --- a/tools/perf/builtin-record.c +++ b/tools/perf/builtin-record.c @@ -1293,14 +1293,9 @@ static int record__open(struct record *rec) */ if (opts->target.initial_delay || target__has_cpu(&opts->target) || perf_pmus__num_core_pmus() > 1) { - pos = evlist__get_tracking_event(evlist); - if (!evsel__is_dummy_event(pos)) { - /* Set up dummy event. */ - if (evlist__add_dummy(evlist)) - return -ENOMEM; - pos = evlist__last(evlist); - evlist__set_tracking_event(evlist, pos); - } + pos = evlist__findnew_tracking_event(evlist, false); + if (!pos) + return -ENOMEM; /* * Enable the dummy event when the process is forked for diff --git a/tools/perf/util/evlist.c b/tools/perf/util/evlist.c index 7ef43f72098e..25c3ebe2c2f5 100644 --- a/tools/perf/util/evlist.c +++ b/tools/perf/util/evlist.c @@ -1694,6 +1694,24 @@ void evlist__set_tracking_event(struct evlist *evlist, struct evsel *tracking_ev tracking_evsel->tracking = true; } +struct evsel *evlist__findnew_tracking_event(struct evlist *evlist, bool system_wide) +{ + struct evsel *evsel; + + evsel = evlist__get_tracking_event(evlist); + if (!evsel__is_dummy_event(evsel)) { + evsel = evlist__add_aux_dummy(evlist, system_wide); + if (!evsel) + return NULL; + + evlist__set_tracking_event(evlist, evsel); + } else if (system_wide) { + perf_evlist__go_system_wide(&evlist->core, &evsel->core); + } + + return evsel; +} + struct evsel *evlist__find_evsel_by_str(struct evlist *evlist, const char *str) { struct evsel *evsel; diff --git a/tools/perf/util/evlist.h b/tools/perf/util/evlist.h index 664c6bf7b3e0..98e7ddb2bd30 100644 --- a/tools/perf/util/evlist.h +++ b/tools/perf/util/evlist.h @@ -387,6 +387,7 @@ bool evlist_cpu_iterator__end(const struct evlist_cpu_iterator *evlist_cpu_itr); struct evsel *evlist__get_tracking_event(struct evlist *evlist); void evlist__set_tracking_event(struct evlist *evlist, struct evsel *tracking_evsel); +struct evsel *evlist__findnew_tracking_event(struct evlist *evlist, bool system_wide); struct evsel *evlist__find_evsel_by_str(struct evlist *evlist, const char *str); From 1285ab300d598ead593b190af65a16f4b0843c68 Mon Sep 17 00:00:00 2001 From: Yang Jihong Date: Mon, 4 Sep 2023 02:33:37 +0000 Subject: [PATCH 039/475] perf record: Move setting tracking events before record__init_thread_masks() User space tasks can migrate between CPUs, so when tracing selected CPUs, sideband for all CPUs is needed. In this case set the cpu map of the evsel to all online CPUs. This may modify the original cpu map of the evlist. Therefore, need to check whether the preceding scenario exists before record__init_thread_masks(). Dummy tracking has been set in record__open(), move it before record__init_thread_masks() and add a helper for unified processing. The sys_perf_event_open invoked is as follows: # perf --debug verbose=3 record -e cpu-clock -D 100 true Opening: cpu-clock ------------------------------------------------------------ perf_event_attr: type 1 (PERF_TYPE_SOFTWARE) size 136 config 0 (PERF_COUNT_SW_CPU_CLOCK) { sample_period, sample_freq } 4000 sample_type IP|TID|TIME|PERIOD|IDENTIFIER read_format ID|LOST disabled 1 inherit 1 freq 1 sample_id_all 1 exclude_guest 1 ------------------------------------------------------------ sys_perf_event_open: pid 10318 cpu 0 group_fd -1 flags 0x8 = 5 sys_perf_event_open: pid 10318 cpu 1 group_fd -1 flags 0x8 = 6 sys_perf_event_open: pid 10318 cpu 2 group_fd -1 flags 0x8 = 7 sys_perf_event_open: pid 10318 cpu 3 group_fd -1 flags 0x8 = 9 sys_perf_event_open: pid 10318 cpu 4 group_fd -1 flags 0x8 = 10 sys_perf_event_open: pid 10318 cpu 5 group_fd -1 flags 0x8 = 11 sys_perf_event_open: pid 10318 cpu 6 group_fd -1 flags 0x8 = 12 sys_perf_event_open: pid 10318 cpu 7 group_fd -1 flags 0x8 = 13 Opening: dummy:u ------------------------------------------------------------ perf_event_attr: type 1 (PERF_TYPE_SOFTWARE) size 136 config 0x9 (PERF_COUNT_SW_DUMMY) { sample_period, sample_freq } 1 sample_type IP|TID|TIME|IDENTIFIER read_format ID|LOST disabled 1 inherit 1 exclude_kernel 1 exclude_hv 1 mmap 1 comm 1 enable_on_exec 1 task 1 sample_id_all 1 exclude_guest 1 mmap2 1 comm_exec 1 ksymbol 1 bpf_event 1 ------------------------------------------------------------ sys_perf_event_open: pid 10318 cpu 0 group_fd -1 flags 0x8 = 14 sys_perf_event_open: pid 10318 cpu 1 group_fd -1 flags 0x8 = 15 sys_perf_event_open: pid 10318 cpu 2 group_fd -1 flags 0x8 = 16 sys_perf_event_open: pid 10318 cpu 3 group_fd -1 flags 0x8 = 17 sys_perf_event_open: pid 10318 cpu 4 group_fd -1 flags 0x8 = 18 sys_perf_event_open: pid 10318 cpu 5 group_fd -1 flags 0x8 = 19 sys_perf_event_open: pid 10318 cpu 6 group_fd -1 flags 0x8 = 20 sys_perf_event_open: pid 10318 cpu 7 group_fd -1 flags 0x8 = 21 'perf test' needs to update base-record & system-wide-dummy attr expected values for test-record-C0: 1. Because a dummy sideband event is added to the sampling of specified CPUs. When evlist contains evsel of different sample_type, evlist__config() will change the default PERF_SAMPLE_ID bit to PERF_SAMPLE_IDENTIFICATION bit. The attr sample_type expected value of base-record and system-wide-dummy in test-record-C0 needs to be updated. 2. The perf record uses evlist__add_aux_dummy() instead of evlist__add_dummy() to add a dummy event. The expected value of system-wide-dummy attr needs to be updated. The 'perf test' result is as follows: # ./perf test list 2>&1 | grep 'Setup struct perf_event_attr' 17: Setup struct perf_event_attr # ./perf test 17 17: Setup struct perf_event_attr : Ok Signed-off-by: Yang Jihong Acked-by: Adrian Hunter Tested-by: Ravi Bangoria Cc: Alexander Shishkin Cc: Andi Kleen Cc: Anshuman Khandual Cc: Ian Rogers Cc: Ingo Molnar Cc: James Clark Cc: Jiri Olsa Cc: Kan Liang Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Thomas Richter Link: https://lore.kernel.org/r/20230904023340.12707-4-yangjihong1@huawei.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/builtin-record.c | 59 ++++++++++++++++--------- tools/perf/tests/attr/system-wide-dummy | 14 +++--- tools/perf/tests/attr/test-record-C0 | 4 +- 3 files changed, 47 insertions(+), 30 deletions(-) diff --git a/tools/perf/builtin-record.c b/tools/perf/builtin-record.c index 12edad8392cc..83bd1f117191 100644 --- a/tools/perf/builtin-record.c +++ b/tools/perf/builtin-record.c @@ -906,6 +906,37 @@ static int record__config_off_cpu(struct record *rec) return off_cpu_prepare(rec->evlist, &rec->opts.target, &rec->opts); } +static int record__config_tracking_events(struct record *rec) +{ + struct record_opts *opts = &rec->opts; + struct evlist *evlist = rec->evlist; + struct evsel *evsel; + + /* + * For initial_delay, system wide or a hybrid system, we need to add + * tracking event so that we can track PERF_RECORD_MMAP to cover the + * delay of waiting or event synthesis. + */ + if (opts->target.initial_delay || target__has_cpu(&opts->target) || + perf_pmus__num_core_pmus() > 1) { + evsel = evlist__findnew_tracking_event(evlist, false); + if (!evsel) + return -ENOMEM; + + /* + * Enable the tracking event when the process is forked for + * initial_delay, immediately for system wide. + */ + if (opts->target.initial_delay && !evsel->immediate && + !target__has_cpu(&opts->target)) + evsel->core.attr.enable_on_exec = 1; + else + evsel->immediate = 1; + } + + return 0; +} + static bool record__kcore_readable(struct machine *machine) { char kcore[PATH_MAX]; @@ -1286,28 +1317,6 @@ static int record__open(struct record *rec) struct record_opts *opts = &rec->opts; int rc = 0; - /* - * For initial_delay, system wide or a hybrid system, we need to add a - * dummy event so that we can track PERF_RECORD_MMAP to cover the delay - * of waiting or event synthesis. - */ - if (opts->target.initial_delay || target__has_cpu(&opts->target) || - perf_pmus__num_core_pmus() > 1) { - pos = evlist__findnew_tracking_event(evlist, false); - if (!pos) - return -ENOMEM; - - /* - * Enable the dummy event when the process is forked for - * initial_delay, immediately for system wide. - */ - if (opts->target.initial_delay && !pos->immediate && - !target__has_cpu(&opts->target)) - pos->core.attr.enable_on_exec = 1; - else - pos->immediate = 1; - } - evlist__config(evlist, opts, &callchain_param); evlist__for_each_entry(evlist, pos) { @@ -4190,6 +4199,12 @@ int cmd_record(int argc, const char **argv) goto out; } + err = record__config_tracking_events(rec); + if (err) { + pr_err("record__config_tracking_events failed, error %d\n", err); + goto out; + } + err = record__init_thread_masks(rec); if (err) { pr_err("Failed to initialize parallel data streaming masks\n"); diff --git a/tools/perf/tests/attr/system-wide-dummy b/tools/perf/tests/attr/system-wide-dummy index 2f3e3eb728eb..a1e1d6a263bf 100644 --- a/tools/perf/tests/attr/system-wide-dummy +++ b/tools/perf/tests/attr/system-wide-dummy @@ -9,8 +9,10 @@ flags=8 type=1 size=136 config=9 -sample_period=4000 -sample_type=455 +sample_period=1 +# PERF_SAMPLE_IP | PERF_SAMPLE_TID | PERF_SAMPLE_TIME | +# PERF_SAMPLE_CPU | PERF_SAMPLE_IDENTIFIER +sample_type=65671 read_format=4|20 # Event will be enabled right away. disabled=0 @@ -18,12 +20,12 @@ inherit=1 pinned=0 exclusive=0 exclude_user=0 -exclude_kernel=0 -exclude_hv=0 +exclude_kernel=1 +exclude_hv=1 exclude_idle=0 mmap=1 comm=1 -freq=1 +freq=0 inherit_stat=0 enable_on_exec=0 task=1 @@ -32,7 +34,7 @@ precise_ip=0 mmap_data=0 sample_id_all=1 exclude_host=0 -exclude_guest=0 +exclude_guest=1 exclude_callchain_kernel=0 exclude_callchain_user=0 mmap2=1 diff --git a/tools/perf/tests/attr/test-record-C0 b/tools/perf/tests/attr/test-record-C0 index 317730b906dd..198e8429a1bf 100644 --- a/tools/perf/tests/attr/test-record-C0 +++ b/tools/perf/tests/attr/test-record-C0 @@ -10,9 +10,9 @@ cpu=0 enable_on_exec=0 # PERF_SAMPLE_IP | PERF_SAMPLE_TID | PERF_SAMPLE_TIME | -# PERF_SAMPLE_ID | PERF_SAMPLE_PERIOD +# PERF_SAMPLE_PERIOD | PERF_SAMPLE_IDENTIFIER # + PERF_SAMPLE_CPU added by -C 0 -sample_type=455 +sample_type=65927 # Dummy event handles mmaps, comm and task. mmap=0 From 74b4f3ecdf64b62446abfb36669b3d40a42d34eb Mon Sep 17 00:00:00 2001 From: Yang Jihong Date: Mon, 4 Sep 2023 02:33:38 +0000 Subject: [PATCH 040/475] perf record: Track sideband events for all CPUs when tracing selected CPUs User space tasks can migrate between CPUs, we need to track side-band events for all CPUs. The specific scenarios are as follows: CPU0 CPU1 perf record -C 0 start taskA starts to be created and executed -> PERF_RECORD_COMM and PERF_RECORD_MMAP events only deliver to CPU1 ...... | migrate to CPU0 | Running on CPU0 <----------/ ... perf record -C 0 stop Now perf samples the PC of taskA. However, perf does not record the PERF_RECORD_COMM and PERF_RECORD_MMAP events of taskA. Therefore, the comm and symbols of taskA cannot be parsed. The solution is to record sideband events for all CPUs when tracing selected CPUs. Because this modifies the default behavior, add related comments to the perf record man page. The sys_perf_event_open invoked is as follows: # perf --debug verbose=3 record -e cpu-clock -C 1 true Opening: cpu-clock ------------------------------------------------------------ perf_event_attr: type 1 (PERF_TYPE_SOFTWARE) size 136 config 0 (PERF_COUNT_SW_CPU_CLOCK) { sample_period, sample_freq } 4000 sample_type IP|TID|TIME|CPU|PERIOD|IDENTIFIER read_format ID|LOST disabled 1 inherit 1 freq 1 sample_id_all 1 exclude_guest 1 ------------------------------------------------------------ sys_perf_event_open: pid -1 cpu 1 group_fd -1 flags 0x8 = 5 Opening: dummy:u ------------------------------------------------------------ perf_event_attr: type 1 (PERF_TYPE_SOFTWARE) size 136 config 0x9 (PERF_COUNT_SW_DUMMY) { sample_period, sample_freq } 1 sample_type IP|TID|TIME|CPU|IDENTIFIER read_format ID|LOST inherit 1 exclude_kernel 1 exclude_hv 1 mmap 1 comm 1 task 1 sample_id_all 1 exclude_guest 1 mmap2 1 comm_exec 1 ksymbol 1 bpf_event 1 ------------------------------------------------------------ sys_perf_event_open: pid -1 cpu 0 group_fd -1 flags 0x8 = 6 sys_perf_event_open: pid -1 cpu 1 group_fd -1 flags 0x8 = 7 sys_perf_event_open: pid -1 cpu 2 group_fd -1 flags 0x8 = 9 sys_perf_event_open: pid -1 cpu 3 group_fd -1 flags 0x8 = 10 sys_perf_event_open: pid -1 cpu 4 group_fd -1 flags 0x8 = 11 sys_perf_event_open: pid -1 cpu 5 group_fd -1 flags 0x8 = 12 sys_perf_event_open: pid -1 cpu 6 group_fd -1 flags 0x8 = 13 sys_perf_event_open: pid -1 cpu 7 group_fd -1 flags 0x8 = 14 Signed-off-by: Yang Jihong Tested-by: Ravi Bangoria Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Andi Kleen Cc: Anshuman Khandual Cc: Ian Rogers Cc: Ingo Molnar Cc: James Clark Cc: Jiri Olsa Cc: Kan Liang Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Thomas Richter Link: https://lore.kernel.org/r/20230904023340.12707-5-yangjihong1@huawei.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/Documentation/perf-record.txt | 3 +++ tools/perf/builtin-record.c | 30 +++++++++++++++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/tools/perf/Documentation/perf-record.txt b/tools/perf/Documentation/perf-record.txt index d5217be012d7..1889f66addf2 100644 --- a/tools/perf/Documentation/perf-record.txt +++ b/tools/perf/Documentation/perf-record.txt @@ -374,6 +374,9 @@ comma-separated list with no space: 0,1. Ranges of CPUs are specified with -: 0- In per-thread mode with inheritance mode on (default), samples are captured only when the thread executes on the designated CPUs. Default is to monitor all CPUs. +User space tasks can migrate between CPUs, so when tracing selected CPUs, +a dummy event is created to track sideband for all CPUs. + -B:: --no-buildid:: Do not save the build ids of binaries in the perf.data files. This skips diff --git a/tools/perf/builtin-record.c b/tools/perf/builtin-record.c index 83bd1f117191..dcf288a4fb9a 100644 --- a/tools/perf/builtin-record.c +++ b/tools/perf/builtin-record.c @@ -906,10 +906,30 @@ static int record__config_off_cpu(struct record *rec) return off_cpu_prepare(rec->evlist, &rec->opts.target, &rec->opts); } +static bool record__tracking_system_wide(struct record *rec) +{ + struct evlist *evlist = rec->evlist; + struct evsel *evsel; + + /* + * If non-dummy evsel exists, system_wide sideband is need to + * help parse sample information. + * For example, PERF_EVENT_MMAP event to help parse symbol, + * and PERF_EVENT_COMM event to help parse task executable name. + */ + evlist__for_each_entry(evlist, evsel) { + if (!evsel__is_dummy_event(evsel)) + return true; + } + + return false; +} + static int record__config_tracking_events(struct record *rec) { struct record_opts *opts = &rec->opts; struct evlist *evlist = rec->evlist; + bool system_wide = false; struct evsel *evsel; /* @@ -919,7 +939,15 @@ static int record__config_tracking_events(struct record *rec) */ if (opts->target.initial_delay || target__has_cpu(&opts->target) || perf_pmus__num_core_pmus() > 1) { - evsel = evlist__findnew_tracking_event(evlist, false); + + /* + * User space tasks can migrate between CPUs, so when tracing + * selected CPUs, sideband for all CPUs is still needed. + */ + if (!!opts->target.cpu_list && record__tracking_system_wide(rec)) + system_wide = true; + + evsel = evlist__findnew_tracking_event(evlist, system_wide); if (!evsel) return -ENOMEM; From 23b97c7ee963f1d007c035e76ba7e3a4fd1259e6 Mon Sep 17 00:00:00 2001 From: Yang Jihong Date: Mon, 4 Sep 2023 02:33:39 +0000 Subject: [PATCH 041/475] perf test: Add test case for record sideband events Add a new test case to record sideband events for all CPUs when tracing selected CPUs Test result: # ./perf test list 2>&1 | grep 'perf record sideband tests' 95: perf record sideband tests # ./perf test 95 95: perf record sideband tests : Ok Signed-off-by: Yang Jihong Tested-by: Ravi Bangoria Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Andi Kleen Cc: Anshuman Khandual Cc: Ian Rogers Cc: Ingo Molnar Cc: James Clark Cc: Jiri Olsa Cc: Kan Liang Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Thomas Richter Link: https://lore.kernel.org/r/20230904023340.12707-6-yangjihong1@huawei.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/tests/shell/record_sideband.sh | 58 +++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100755 tools/perf/tests/shell/record_sideband.sh diff --git a/tools/perf/tests/shell/record_sideband.sh b/tools/perf/tests/shell/record_sideband.sh new file mode 100755 index 000000000000..5024a7ce0c51 --- /dev/null +++ b/tools/perf/tests/shell/record_sideband.sh @@ -0,0 +1,58 @@ +#!/bin/sh +# perf record sideband tests +# SPDX-License-Identifier: GPL-2.0 + +set -e + +err=0 +perfdata=$(mktemp /tmp/__perf_test.perf.data.XXXXX) + +cleanup() +{ + rm -rf ${perfdata} + trap - EXIT TERM INT +} + +trap_cleanup() +{ + cleanup + exit 1 +} +trap trap_cleanup EXIT TERM INT + +can_cpu_wide() +{ + if ! perf record -o ${perfdata} -BN --no-bpf-event -C $1 true 2>&1 >/dev/null + then + echo "record sideband test [Skipped cannot record cpu$1]" + err=2 + fi + + rm -f ${perfdata} + return $err +} + +test_system_wide_tracking() +{ + # Need CPU 0 and CPU 1 + can_cpu_wide 0 || return 0 + can_cpu_wide 1 || return 0 + + # Record on CPU 0 a task running on CPU 1 + perf record -BN --no-bpf-event -o ${perfdata} -C 0 -- taskset --cpu-list 1 true + + # Should get MMAP events from CPU 1 + mmap_cnt=`perf script -i ${perfdata} --show-mmap-events -C 1 2>/dev/null | grep MMAP | wc -l` + + if [ ${mmap_cnt} -gt 0 ] ; then + return 0 + fi + + echo "Failed to record MMAP events on CPU 1 when tracing CPU 0" + return 1 +} + +test_system_wide_tracking + +cleanup +exit $err From d50ad02cb39a5fe1d0c02b3b51e8a2a37464c54a Mon Sep 17 00:00:00 2001 From: Yang Jihong Date: Mon, 4 Sep 2023 02:33:40 +0000 Subject: [PATCH 042/475] perf test: Add perf_event_attr test for record dummy event If only dummy event is recorded, tracking event is not needed. Add this test scenario. Test result: # ./perf test list 2>&1 | grep 'Setup struct perf_event_attr' 17: Setup struct perf_event_attr # ./perf test 17 -v 17: Setup struct perf_event_attr : --- start --- test child forked, pid 720198 running './tests/attr/test-record-dummy-C0' test child finished with 0 ---- end ---- Setup struct perf_event_attr: Ok Signed-off-by: Yang Jihong Tested-by: Ravi Bangoria Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Andi Kleen Cc: Anshuman Khandual Cc: Ian Rogers Cc: Ingo Molnar Cc: James Clark Cc: Jiri Olsa Cc: Kan Liang Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Thomas Richter Link: https://lore.kernel.org/r/20230904023340.12707-7-yangjihong1@huawei.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/tests/attr/test-record-dummy-C0 | 55 ++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 tools/perf/tests/attr/test-record-dummy-C0 diff --git a/tools/perf/tests/attr/test-record-dummy-C0 b/tools/perf/tests/attr/test-record-dummy-C0 new file mode 100644 index 000000000000..83ca4e373acd --- /dev/null +++ b/tools/perf/tests/attr/test-record-dummy-C0 @@ -0,0 +1,55 @@ +[config] +command = record +args = --no-bpf-event -e dummy -C 0 kill >/dev/null 2>&1 +ret = 1 + +[event] +fd=1 +group_fd=-1 +cpu=0 +pid=-1 +flags=8 +type=1 +size=136 +config=9 +sample_period=4000 +# PERF_SAMPLE_IP | PERF_SAMPLE_TID | PERF_SAMPLE_TIME | +# PERF_SAMPLE_PERIOD +# + PERF_SAMPLE_CPU added by -C 0 +sample_type=391 +read_format=4 +disabled=0 +inherit=1 +pinned=0 +exclusive=0 +exclude_user=0 +exclude_kernel=0 +exclude_hv=0 +exclude_idle=0 +mmap=1 +comm=1 +freq=1 +inherit_stat=0 +enable_on_exec=0 +task=1 +watermark=0 +precise_ip=0 +mmap_data=0 +sample_id_all=1 +exclude_host=0 +exclude_guest=1 +exclude_callchain_kernel=0 +exclude_callchain_user=0 +mmap2=1 +comm_exec=1 +context_switch=0 +write_backward=0 +namespaces=0 +use_clockid=0 +wakeup_events=0 +bp_type=0 +config1=0 +config2=0 +branch_sample_type=0 +sample_regs_user=0 +sample_stack_user=0 From d39710088d82ef100b33cdf4a9de3546fb0bb5df Mon Sep 17 00:00:00 2001 From: Yang Jihong Date: Sat, 12 Aug 2023 08:49:02 +0000 Subject: [PATCH 043/475] perf kwork: Fix incorrect and missing free atom in work_push_atom() 1. Atoms are managed in page mode and should be released using atom_free() instead of free(). 2. When the event does not match, the atom needs to free. Fixes: f98919ec4fccdacf ("perf kwork: Implement 'report' subcommand") Reviewed-by: Ian Rogers Signed-off-by: Yang Jihong Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ingo Molnar Cc: Jiri Olsa Cc: Kan Liang Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Ravi Bangoria Cc: Sandipan Das Cc: Yang Jihong Link: https://lore.kernel.org/r/20230812084917.169338-2-yangjihong1@huawei.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/builtin-kwork.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/perf/builtin-kwork.c b/tools/perf/builtin-kwork.c index 14bf7a8429e7..73b5dc099a8a 100644 --- a/tools/perf/builtin-kwork.c +++ b/tools/perf/builtin-kwork.c @@ -406,12 +406,14 @@ static int work_push_atom(struct perf_kwork *kwork, work = work_findnew(&class->work_root, &key, &kwork->cmp_id); if (work == NULL) { - free(atom); + atom_free(atom); return -1; } - if (!profile_event_match(kwork, work, sample)) + if (!profile_event_match(kwork, work, sample)) { + atom_free(atom); return 0; + } if (dst_type < KWORK_TRACE_MAX) { dst_atom = list_last_entry_or_null(&work->atom_list[dst_type], From 76e0d8c821bbd952730799cc7af841f9de67b7f7 Mon Sep 17 00:00:00 2001 From: Yang Jihong Date: Sat, 12 Aug 2023 08:49:03 +0000 Subject: [PATCH 044/475] perf kwork: Add the supported subcommands to the document Add missing report, latency and timehist subcommands to the document. Fixes: f98919ec4fccdacf ("perf kwork: Implement 'report' subcommand") Fixes: ad3d9f7a929ab2df ("perf kwork: Implement perf kwork latency") Fixes: bcc8b3e88d6fa1a3 ("perf kwork: Implement perf kwork timehist") Reviewed-by: Ian Rogers Signed-off-by: Yang Jihong Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ingo Molnar Cc: Jiri Olsa Cc: Kan Liang Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Ravi Bangoria Cc: Sandipan Das Link: https://lore.kernel.org/r/20230812084917.169338-3-yangjihong1@huawei.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/Documentation/perf-kwork.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/perf/Documentation/perf-kwork.txt b/tools/perf/Documentation/perf-kwork.txt index 3c36324712b6..482d6c52e2ed 100644 --- a/tools/perf/Documentation/perf-kwork.txt +++ b/tools/perf/Documentation/perf-kwork.txt @@ -8,7 +8,7 @@ perf-kwork - Tool to trace/measure kernel work properties (latencies) SYNOPSIS -------- [verse] -'perf kwork' {record} +'perf kwork' {record|report|latency|timehist} DESCRIPTION ----------- From 0c526579a4b2b6ecd540472f2e34c2850cf70f76 Mon Sep 17 00:00:00 2001 From: Yang Jihong Date: Sat, 12 Aug 2023 08:49:04 +0000 Subject: [PATCH 045/475] perf kwork: Set ordered_events to true in 'struct perf_tool' 'perf kwork' processes data based on timestamps and needs to sort events. Fixes: f98919ec4fccdacf ("perf kwork: Implement 'report' subcommand") Reviewed-by: Ian Rogers Signed-off-by: Yang Jihong Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ingo Molnar Cc: Jiri Olsa Cc: Kan Liang Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Ravi Bangoria Cc: Sandipan Das Cc: Yang Jihong Link: https://lore.kernel.org/r/20230812084917.169338-4-yangjihong1@huawei.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/builtin-kwork.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/perf/builtin-kwork.c b/tools/perf/builtin-kwork.c index 73b5dc099a8a..de2fbb7c56c3 100644 --- a/tools/perf/builtin-kwork.c +++ b/tools/perf/builtin-kwork.c @@ -1694,9 +1694,10 @@ int cmd_kwork(int argc, const char **argv) static struct perf_kwork kwork = { .class_list = LIST_HEAD_INIT(kwork.class_list), .tool = { - .mmap = perf_event__process_mmap, - .mmap2 = perf_event__process_mmap2, - .sample = perf_kwork__process_tracepoint_sample, + .mmap = perf_event__process_mmap, + .mmap2 = perf_event__process_mmap2, + .sample = perf_kwork__process_tracepoint_sample, + .ordered_events = true, }, .atom_page_list = LIST_HEAD_INIT(kwork.atom_page_list), .sort_list = LIST_HEAD_INIT(kwork.sort_list), From 95064b3352250a199323b8161eb08aee428a8286 Mon Sep 17 00:00:00 2001 From: Yang Jihong Date: Sat, 12 Aug 2023 08:49:05 +0000 Subject: [PATCH 046/475] perf kwork: Add `kwork` and `src_type` to work_init() for 'struct kwork_class' To support different types of reports, two parameters `struct perf_kwork * kwork` and `enum kwork_trace_type src_type` are added to work_init() of struct kwork_class for initialization in different scenarios. No functional change intended. Reviewed-by: Ian Rogers Signed-off-by: Yang Jihong Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ingo Molnar Cc: Jiri Olsa Cc: Kan Liang Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Ravi Bangoria Cc: Sandipan Das Link: https://lore.kernel.org/r/20230812084917.169338-5-yangjihong1@huawei.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/builtin-kwork.c | 16 +++++++++++----- tools/perf/util/kwork.h | 6 ++++-- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/tools/perf/builtin-kwork.c b/tools/perf/builtin-kwork.c index de2fbb7c56c3..42ea59a957ae 100644 --- a/tools/perf/builtin-kwork.c +++ b/tools/perf/builtin-kwork.c @@ -398,7 +398,7 @@ static int work_push_atom(struct perf_kwork *kwork, struct kwork_work *work, key; BUG_ON(class->work_init == NULL); - class->work_init(class, &key, evsel, sample, machine); + class->work_init(kwork, class, &key, src_type, evsel, sample, machine); atom = atom_new(kwork, sample); if (atom == NULL) @@ -445,7 +445,7 @@ static struct kwork_atom *work_pop_atom(struct perf_kwork *kwork, struct kwork_work *work, key; BUG_ON(class->work_init == NULL); - class->work_init(class, &key, evsel, sample, machine); + class->work_init(kwork, class, &key, src_type, evsel, sample, machine); work = work_findnew(&class->work_root, &key, &kwork->cmp_id); if (ret_work != NULL) @@ -821,8 +821,10 @@ static int irq_class_init(struct kwork_class *class, return 0; } -static void irq_work_init(struct kwork_class *class, +static void irq_work_init(struct perf_kwork *kwork __maybe_unused, + struct kwork_class *class, struct kwork_work *work, + enum kwork_trace_type src_type __maybe_unused, struct evsel *evsel, struct perf_sample *sample, struct machine *machine __maybe_unused) @@ -940,8 +942,10 @@ static char *evsel__softirq_name(struct evsel *evsel, u64 num) return name; } -static void softirq_work_init(struct kwork_class *class, +static void softirq_work_init(struct perf_kwork *kwork __maybe_unused, + struct kwork_class *class, struct kwork_work *work, + enum kwork_trace_type src_type __maybe_unused, struct evsel *evsel, struct perf_sample *sample, struct machine *machine __maybe_unused) @@ -1031,8 +1035,10 @@ static int workqueue_class_init(struct kwork_class *class, return 0; } -static void workqueue_work_init(struct kwork_class *class, +static void workqueue_work_init(struct perf_kwork *kwork __maybe_unused, + struct kwork_class *class, struct kwork_work *work, + enum kwork_trace_type src_type __maybe_unused, struct evsel *evsel, struct perf_sample *sample, struct machine *machine) diff --git a/tools/perf/util/kwork.h b/tools/perf/util/kwork.h index 53b7327550b8..736c7a08fb19 100644 --- a/tools/perf/util/kwork.h +++ b/tools/perf/util/kwork.h @@ -91,6 +91,7 @@ struct kwork_atom_page { DECLARE_BITMAP(bitmap, NR_ATOM_PER_PAGE); }; +struct perf_kwork; struct kwork_class; struct kwork_work { /* @@ -142,8 +143,10 @@ struct kwork_class { int (*class_init)(struct kwork_class *class, struct perf_session *session); - void (*work_init)(struct kwork_class *class, + void (*work_init)(struct perf_kwork *kwork, + struct kwork_class *class, struct kwork_work *work, + enum kwork_trace_type src_type, struct evsel *evsel, struct perf_sample *sample, struct machine *machine); @@ -152,7 +155,6 @@ struct kwork_class { char *buf, int len); }; -struct perf_kwork; struct trace_kwork_handler { int (*raise_event)(struct perf_kwork *kwork, struct kwork_class *class, struct evsel *evsel, From 86c67c8af4bec82962ff711551e7f4d21f340fef Mon Sep 17 00:00:00 2001 From: Yang Jihong Date: Sat, 12 Aug 2023 08:49:06 +0000 Subject: [PATCH 047/475] perf kwork: Overwrite original atom in the list when a new atom is pushed. work_push_atom() supports nesting. Currently, all supported kworks are not nested. A `overwrite` parameter is added to overwrite the original atom in the list. Reviewed-by: Ian Rogers Signed-off-by: Yang Jihong Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ingo Molnar Cc: Jiri Olsa Cc: Kan Liang Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Ravi Bangoria Cc: Sandipan Das Link: https://lore.kernel.org/r/20230812084917.169338-6-yangjihong1@huawei.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/builtin-kwork.c | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/tools/perf/builtin-kwork.c b/tools/perf/builtin-kwork.c index 42ea59a957ae..f620911a26a2 100644 --- a/tools/perf/builtin-kwork.c +++ b/tools/perf/builtin-kwork.c @@ -392,9 +392,10 @@ static int work_push_atom(struct perf_kwork *kwork, struct evsel *evsel, struct perf_sample *sample, struct machine *machine, - struct kwork_work **ret_work) + struct kwork_work **ret_work, + bool overwrite) { - struct kwork_atom *atom, *dst_atom; + struct kwork_atom *atom, *dst_atom, *last_atom; struct kwork_work *work, key; BUG_ON(class->work_init == NULL); @@ -427,6 +428,17 @@ static int work_push_atom(struct perf_kwork *kwork, if (ret_work != NULL) *ret_work = work; + if (overwrite) { + last_atom = list_last_entry_or_null(&work->atom_list[src_type], + struct kwork_atom, list); + if (last_atom) { + atom_del(last_atom); + + kwork->nr_skipped_events[src_type]++; + kwork->nr_skipped_events[KWORK_TRACE_MAX]++; + } + } + list_add_tail(&atom->list, &work->atom_list[src_type]); return 0; @@ -502,7 +514,7 @@ static int report_entry_event(struct perf_kwork *kwork, { return work_push_atom(kwork, class, KWORK_TRACE_ENTRY, KWORK_TRACE_MAX, evsel, sample, - machine, NULL); + machine, NULL, true); } static int report_exit_event(struct perf_kwork *kwork, @@ -557,7 +569,7 @@ static int latency_raise_event(struct perf_kwork *kwork, { return work_push_atom(kwork, class, KWORK_TRACE_RAISE, KWORK_TRACE_MAX, evsel, sample, - machine, NULL); + machine, NULL, true); } static int latency_entry_event(struct perf_kwork *kwork, @@ -716,7 +728,7 @@ static int timehist_raise_event(struct perf_kwork *kwork, { return work_push_atom(kwork, class, KWORK_TRACE_RAISE, KWORK_TRACE_MAX, evsel, sample, - machine, NULL); + machine, NULL, true); } static int timehist_entry_event(struct perf_kwork *kwork, @@ -730,7 +742,7 @@ static int timehist_entry_event(struct perf_kwork *kwork, ret = work_push_atom(kwork, class, KWORK_TRACE_ENTRY, KWORK_TRACE_RAISE, evsel, sample, - machine, &work); + machine, &work, true); if (ret) return ret; From 26b7254ff100fdc9f06eb8f5e2cd7424605fce2b Mon Sep 17 00:00:00 2001 From: Yang Jihong Date: Sat, 12 Aug 2023 08:49:07 +0000 Subject: [PATCH 048/475] perf kwork: Set default events list if not specified in setup_event_list() Currently when no kwork event is specified, all events are configured by default. Now set to default event list string, which is more flexible and supports subsequent function extension. Also put setup_event_list() into each subcommand for different settings. Reviewed-by: Ian Rogers Signed-off-by: Yang Jihong Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ingo Molnar Cc: Jiri Olsa Cc: Kan Liang Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Ravi Bangoria Cc: Sandipan Das Link: https://lore.kernel.org/r/20230812084917.169338-7-yangjihong1@huawei.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/builtin-kwork.c | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/tools/perf/builtin-kwork.c b/tools/perf/builtin-kwork.c index f620911a26a2..e859c34b23f3 100644 --- a/tools/perf/builtin-kwork.c +++ b/tools/perf/builtin-kwork.c @@ -1623,8 +1623,11 @@ static void setup_event_list(struct perf_kwork *kwork, struct kwork_class *class; char *tmp, *tok, *str; + /* + * set default events list if not specified + */ if (kwork->event_list_str == NULL) - goto null_event_list_str; + kwork->event_list_str = "irq, softirq, workqueue"; str = strdup(kwork->event_list_str); for (tok = strtok_r(str, ", ", &tmp); @@ -1643,17 +1646,6 @@ static void setup_event_list(struct perf_kwork *kwork, } free(str); -null_event_list_str: - /* - * config all kwork events if not specified - */ - if (list_empty(&kwork->class_list)) { - for (i = 0; i < KWORK_CLASS_MAX; i++) { - list_add_tail(&kwork_class_supported_list[i]->list, - &kwork->class_list); - } - } - pr_debug("Config event list:"); list_for_each_entry(class, &kwork->class_list, list) pr_debug(" %s", class->name); @@ -1835,12 +1827,12 @@ int cmd_kwork(int argc, const char **argv) if (!argc) usage_with_options(kwork_usage, kwork_options); - setup_event_list(&kwork, kwork_options, kwork_usage); sort_dimension__add(&kwork, "id", &kwork.cmp_id); - if (strlen(argv[0]) > 2 && strstarts("record", argv[0])) + if (strlen(argv[0]) > 2 && strstarts("record", argv[0])) { + setup_event_list(&kwork, kwork_options, kwork_usage); return perf_kwork__record(&kwork, argc, argv); - else if (strlen(argv[0]) > 2 && strstarts("report", argv[0])) { + } else if (strlen(argv[0]) > 2 && strstarts("report", argv[0])) { kwork.sort_order = default_report_sort_order; if (argc > 1) { argc = parse_options(argc, argv, report_options, report_usage, 0); @@ -1849,6 +1841,7 @@ int cmd_kwork(int argc, const char **argv) } kwork.report = KWORK_REPORT_RUNTIME; setup_sorting(&kwork, report_options, report_usage); + setup_event_list(&kwork, kwork_options, kwork_usage); return perf_kwork__report(&kwork); } else if (strlen(argv[0]) > 2 && strstarts("latency", argv[0])) { kwork.sort_order = default_latency_sort_order; @@ -1859,6 +1852,7 @@ int cmd_kwork(int argc, const char **argv) } kwork.report = KWORK_REPORT_LATENCY; setup_sorting(&kwork, latency_options, latency_usage); + setup_event_list(&kwork, kwork_options, kwork_usage); return perf_kwork__report(&kwork); } else if (strlen(argv[0]) > 2 && strstarts("timehist", argv[0])) { if (argc > 1) { @@ -1867,6 +1861,7 @@ int cmd_kwork(int argc, const char **argv) usage_with_options(timehist_usage, timehist_options); } kwork.report = KWORK_REPORT_TIMEHIST; + setup_event_list(&kwork, kwork_options, kwork_usage); return perf_kwork__timehist(&kwork); } else usage_with_options(kwork_usage, kwork_options); From 38d8d013a525daf4f98a2b53065d8e89d5f810dd Mon Sep 17 00:00:00 2001 From: Yang Jihong Date: Sat, 12 Aug 2023 08:49:08 +0000 Subject: [PATCH 049/475] perf kwork: Add sched record support The kwork_class type of sched is added to support recording and parsing of sched_switch events. As follows: # perf kwork -h Usage: perf kwork [] {record|report|latency|timehist} -D, --dump-raw-trace dump raw trace in ASCII -f, --force don't complain, do it -k, --kwork list of kwork to profile (irq, softirq, workqueue, sched, etc) -v, --verbose be more verbose (show symbol address, etc) # perf kwork -k sched record true [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.083 MB perf.data (47 samples) ] # perf evlist sched:sched_switch dummy:HG # Tip: use 'perf evlist --trace-fields' to show fields for tracepoint events Reviewed-by: Ian Rogers Signed-off-by: Yang Jihong Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ingo Molnar Cc: Jiri Olsa Cc: Kan Liang Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Ravi Bangoria Cc: Sandipan Das Link: https://lore.kernel.org/r/20230812084917.169338-8-yangjihong1@huawei.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/Documentation/perf-kwork.txt | 2 +- tools/perf/builtin-kwork.c | 69 ++++++++++++++++++++++++- tools/perf/util/kwork.h | 5 ++ 3 files changed, 74 insertions(+), 2 deletions(-) diff --git a/tools/perf/Documentation/perf-kwork.txt b/tools/perf/Documentation/perf-kwork.txt index 482d6c52e2ed..2092ab916ea9 100644 --- a/tools/perf/Documentation/perf-kwork.txt +++ b/tools/perf/Documentation/perf-kwork.txt @@ -66,7 +66,7 @@ OPTIONS -k:: --kwork:: - List of kwork to profile (irq, softirq, workqueue, etc) + List of kwork to profile (irq, softirq, workqueue, sched, etc) -v:: --verbose:: diff --git a/tools/perf/builtin-kwork.c b/tools/perf/builtin-kwork.c index e859c34b23f3..4e2b9103b9e0 100644 --- a/tools/perf/builtin-kwork.c +++ b/tools/perf/builtin-kwork.c @@ -1084,10 +1084,77 @@ static struct kwork_class kwork_workqueue = { .work_name = workqueue_work_name, }; +static struct kwork_class kwork_sched; +static int process_sched_switch_event(struct perf_tool *tool, + struct evsel *evsel, + struct perf_sample *sample, + struct machine *machine) +{ + struct perf_kwork *kwork = container_of(tool, struct perf_kwork, tool); + + if (kwork->tp_handler->sched_switch_event) + return kwork->tp_handler->sched_switch_event(kwork, &kwork_sched, + evsel, sample, machine); + return 0; +} + +const struct evsel_str_handler sched_tp_handlers[] = { + { "sched:sched_switch", process_sched_switch_event, }, +}; + +static int sched_class_init(struct kwork_class *class, + struct perf_session *session) +{ + if (perf_session__set_tracepoints_handlers(session, + sched_tp_handlers)) { + pr_err("Failed to set sched tracepoints handlers\n"); + return -1; + } + + class->work_root = RB_ROOT_CACHED; + return 0; +} + +static void sched_work_init(struct perf_kwork *kwork __maybe_unused, + struct kwork_class *class, + struct kwork_work *work, + enum kwork_trace_type src_type, + struct evsel *evsel, + struct perf_sample *sample, + struct machine *machine __maybe_unused) +{ + work->class = class; + work->cpu = sample->cpu; + + if (src_type == KWORK_TRACE_EXIT) { + work->id = evsel__intval(evsel, sample, "prev_pid"); + work->name = strdup(evsel__strval(evsel, sample, "prev_comm")); + } else if (src_type == KWORK_TRACE_ENTRY) { + work->id = evsel__intval(evsel, sample, "next_pid"); + work->name = strdup(evsel__strval(evsel, sample, "next_comm")); + } +} + +static void sched_work_name(struct kwork_work *work, char *buf, int len) +{ + snprintf(buf, len, "%s", work->name); +} + +static struct kwork_class kwork_sched = { + .name = "sched", + .type = KWORK_CLASS_SCHED, + .nr_tracepoints = ARRAY_SIZE(sched_tp_handlers), + .tp_handlers = sched_tp_handlers, + .class_init = sched_class_init, + .work_init = sched_work_init, + .work_name = sched_work_name, +}; + static struct kwork_class *kwork_class_supported_list[KWORK_CLASS_MAX] = { [KWORK_CLASS_IRQ] = &kwork_irq, [KWORK_CLASS_SOFTIRQ] = &kwork_softirq, [KWORK_CLASS_WORKQUEUE] = &kwork_workqueue, + [KWORK_CLASS_SCHED] = &kwork_sched, }; static void print_separator(int len) @@ -1740,7 +1807,7 @@ int cmd_kwork(int argc, const char **argv) OPT_BOOLEAN('D', "dump-raw-trace", &dump_trace, "dump raw trace in ASCII"), OPT_STRING('k', "kwork", &kwork.event_list_str, "kwork", - "list of kwork to profile (irq, softirq, workqueue, etc)"), + "list of kwork to profile (irq, softirq, workqueue, sched, etc)"), OPT_BOOLEAN('f', "force", &kwork.force, "don't complain, do it"), OPT_END() }; diff --git a/tools/perf/util/kwork.h b/tools/perf/util/kwork.h index 736c7a08fb19..f8e9cdd1371a 100644 --- a/tools/perf/util/kwork.h +++ b/tools/perf/util/kwork.h @@ -16,6 +16,7 @@ enum kwork_class_type { KWORK_CLASS_IRQ, KWORK_CLASS_SOFTIRQ, KWORK_CLASS_WORKQUEUE, + KWORK_CLASS_SCHED, KWORK_CLASS_MAX, }; @@ -167,6 +168,10 @@ struct trace_kwork_handler { int (*exit_event)(struct perf_kwork *kwork, struct kwork_class *class, struct evsel *evsel, struct perf_sample *sample, struct machine *machine); + + int (*sched_switch_event)(struct perf_kwork *kwork, + struct kwork_class *class, struct evsel *evsel, + struct perf_sample *sample, struct machine *machine); }; struct perf_kwork { From b83b5071c018c74e53833d2fe7011d6ce583957d Mon Sep 17 00:00:00 2001 From: Yang Jihong Date: Sat, 12 Aug 2023 08:49:09 +0000 Subject: [PATCH 050/475] perf kwork: Add `root` parameter to work_sort() Add a `struct rb_root_cached *root` parameter to work_sort() to sort the specified rb tree elements. No functional change. Reviewed-by: Ian Rogers Signed-off-by: Yang Jihong Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ingo Molnar Cc: Jiri Olsa Cc: Kan Liang Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Ravi Bangoria Cc: Sandipan Das Link: https://lore.kernel.org/r/20230812084917.169338-9-yangjihong1@huawei.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/builtin-kwork.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/perf/builtin-kwork.c b/tools/perf/builtin-kwork.c index 4e2b9103b9e0..7d93951c2ed2 100644 --- a/tools/perf/builtin-kwork.c +++ b/tools/perf/builtin-kwork.c @@ -1378,11 +1378,11 @@ static void print_bad_events(struct perf_kwork *kwork) } } -static void work_sort(struct perf_kwork *kwork, struct kwork_class *class) +static void work_sort(struct perf_kwork *kwork, + struct kwork_class *class, struct rb_root_cached *root) { struct rb_node *node; struct kwork_work *data; - struct rb_root_cached *root = &class->work_root; pr_debug("Sorting %s ...\n", class->name); for (;;) { @@ -1402,7 +1402,7 @@ static void perf_kwork__sort(struct perf_kwork *kwork) struct kwork_class *class; list_for_each_entry(class, &kwork->class_list, list) - work_sort(kwork, class); + work_sort(kwork, class, &class->work_root); } static int perf_kwork__check_config(struct perf_kwork *kwork, From 55c40e505234588d049384aa45a6931438c3028d Mon Sep 17 00:00:00 2001 From: Yang Jihong Date: Sat, 12 Aug 2023 08:49:10 +0000 Subject: [PATCH 051/475] perf kwork top: Introduce new top utility Some common tools for collecting statistics on CPU usage, such as top, obtain statistics from timer interrupt sampling, and then periodically read statistics from /proc/stat. This method has some deviations: 1. In the tick interrupt, the time between the last tick and the current tick is counted in the current task. However, the task may be running only part of the time. 2. For each task, the top tool periodically reads the /proc/{PID}/status information. For tasks with a short life cycle, it may be missed. In conclusion, the top tool cannot accurately collect statistics on the CPU usage and running time of tasks. The statistical method based on sched_switch tracepoint can accurately calculate the CPU usage of all tasks. This method is applicable to scenarios where performance comparison data is of high precision. Example usage: # perf kwork Usage: perf kwork [] {record|report|latency|timehist|top} -D, --dump-raw-trace dump raw trace in ASCII -f, --force don't complain, do it -k, --kwork list of kwork to profile (irq, softirq, workqueue, sched, etc) -v, --verbose be more verbose (show symbol address, etc) # perf kwork -k sched record -- perf bench sched messaging -g 1 -l 10000 # Running 'sched/messaging' benchmark: # 20 sender and receiver processes per group # 1 groups == 40 processes run Total time: 14.074 [sec] [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 15.886 MB perf.data (129472 samples) ] # perf kwork top Total : 115708.178 ms, 8 cpus %Cpu(s): 9.78% id %Cpu0 [||||||||||||||||||||||||||| 90.55%] %Cpu1 [||||||||||||||||||||||||||| 90.51%] %Cpu2 [|||||||||||||||||||||||||| 88.57%] %Cpu3 [||||||||||||||||||||||||||| 91.18%] %Cpu4 [||||||||||||||||||||||||||| 91.09%] %Cpu5 [||||||||||||||||||||||||||| 90.88%] %Cpu6 [|||||||||||||||||||||||||| 88.64%] %Cpu7 [||||||||||||||||||||||||||| 90.28%] PID %CPU RUNTIME COMMMAND ---------------------------------------------------- 4113 22.23 3221.547 ms sched-messaging 4105 21.61 3131.495 ms sched-messaging 4119 21.53 3120.937 ms sched-messaging 4103 21.39 3101.614 ms sched-messaging 4106 21.37 3095.209 ms sched-messaging 4104 21.25 3077.269 ms sched-messaging 4115 21.21 3073.188 ms sched-messaging 4109 21.18 3069.022 ms sched-messaging 4111 20.78 3010.033 ms sched-messaging 4114 20.74 3007.073 ms sched-messaging 4108 20.73 3002.137 ms sched-messaging 4107 20.47 2967.292 ms sched-messaging 4117 20.39 2955.335 ms sched-messaging 4112 20.34 2947.080 ms sched-messaging 4118 20.32 2942.519 ms sched-messaging 4121 20.23 2929.865 ms sched-messaging 4110 20.22 2930.078 ms sched-messaging 4122 20.15 2919.542 ms sched-messaging 4120 19.77 2866.032 ms sched-messaging 4116 19.72 2857.660 ms sched-messaging 4127 16.19 2346.334 ms sched-messaging 4142 15.86 2297.600 ms sched-messaging 4141 15.62 2262.646 ms sched-messaging 4136 15.41 2231.408 ms sched-messaging 4130 15.38 2227.008 ms sched-messaging 4129 15.31 2217.692 ms sched-messaging 4126 15.21 2201.711 ms sched-messaging 4139 15.19 2200.722 ms sched-messaging 4137 15.10 2188.633 ms sched-messaging 4134 15.06 2182.082 ms sched-messaging 4132 15.02 2177.530 ms sched-messaging 4131 14.73 2131.973 ms sched-messaging 4125 14.68 2125.439 ms sched-messaging 4128 14.66 2122.255 ms sched-messaging 4123 14.65 2122.113 ms sched-messaging 4135 14.56 2107.144 ms sched-messaging 4133 14.51 2103.549 ms sched-messaging 4124 14.27 2066.671 ms sched-messaging 4140 14.17 2052.251 ms sched-messaging 4138 13.81 2000.361 ms sched-messaging 0 11.42 1652.009 ms swapper/2 0 11.35 1641.694 ms swapper/6 0 9.71 1405.108 ms swapper/7 0 9.48 1372.338 ms swapper/1 0 9.44 1366.013 ms swapper/0 0 9.11 1318.382 ms swapper/5 0 8.90 1287.582 ms swapper/4 0 8.81 1274.356 ms swapper/3 4100 2.61 379.328 ms perf 4101 1.16 169.487 ms perf-exec 151 0.65 94.741 ms systemd-resolve 249 0.36 53.030 ms sd-resolve 153 0.14 21.405 ms systemd-timesyn 1 0.10 16.200 ms systemd 16 0.09 15.785 ms rcu_preempt 4102 0.06 9.727 ms perf 4095 0.03 5.464 ms kworker/7:1 98 0.02 3.231 ms jbd2/sda-8 353 0.02 4.115 ms sshd 75 0.02 3.889 ms kworker/2:1 73 0.01 1.552 ms kworker/5:1 64 0.01 1.591 ms kworker/4:1 74 0.01 1.952 ms kworker/3:1 61 0.01 2.608 ms kcompactd0 397 0.01 1.602 ms kworker/1:1 69 0.01 1.817 ms kworker/1:1H 10 0.01 2.553 ms kworker/u16:0 2909 0.01 2.684 ms kworker/0:2 1211 0.00 0.426 ms kworker/7:0 97 0.00 0.153 ms kworker/7:1H 51 0.00 0.100 ms ksoftirqd/7 120 0.00 0.856 ms systemd-journal 76 0.00 1.414 ms kworker/6:1 46 0.00 0.246 ms ksoftirqd/6 45 0.00 0.164 ms migration/6 41 0.00 0.098 ms ksoftirqd/5 40 0.00 0.207 ms migration/5 86 0.00 1.339 ms kworker/4:1H 36 0.00 0.252 ms ksoftirqd/4 35 0.00 0.090 ms migration/4 31 0.00 0.156 ms ksoftirqd/3 30 0.00 0.073 ms migration/3 26 0.00 0.180 ms ksoftirqd/2 25 0.00 0.085 ms migration/2 21 0.00 0.106 ms ksoftirqd/1 20 0.00 0.118 ms migration/1 302 0.00 1.440 ms systemd-logind 17 0.00 0.132 ms migration/0 15 0.00 0.255 ms ksoftirqd/0 Reviewed-by: Ian Rogers Signed-off-by: Yang Jihong Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ingo Molnar Cc: Jiri Olsa Cc: Kan Liang Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Ravi Bangoria Cc: Sandipan Das Link: https://lore.kernel.org/r/20230812084917.169338-10-yangjihong1@huawei.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/Documentation/perf-kwork.txt | 5 +- tools/perf/builtin-kwork.c | 387 +++++++++++++++++++++++- tools/perf/util/kwork.h | 22 ++ 3 files changed, 412 insertions(+), 2 deletions(-) diff --git a/tools/perf/Documentation/perf-kwork.txt b/tools/perf/Documentation/perf-kwork.txt index 2092ab916ea9..0601fcb0feea 100644 --- a/tools/perf/Documentation/perf-kwork.txt +++ b/tools/perf/Documentation/perf-kwork.txt @@ -8,7 +8,7 @@ perf-kwork - Tool to trace/measure kernel work properties (latencies) SYNOPSIS -------- [verse] -'perf kwork' {record|report|latency|timehist} +'perf kwork' {record|report|latency|timehist|top} DESCRIPTION ----------- @@ -23,6 +23,8 @@ There are several variants of 'perf kwork': 'perf kwork timehist' provides an analysis of kernel work events. + 'perf kwork top' to report the task cpu usage. + Example usage: perf kwork record -- sleep 1 perf kwork report @@ -30,6 +32,7 @@ There are several variants of 'perf kwork': perf kwork latency perf kwork latency -b perf kwork timehist + perf kwork top By default it shows the individual work events such as irq, workqeueu, including the run time and delay (time between raise and actually entry): diff --git a/tools/perf/builtin-kwork.c b/tools/perf/builtin-kwork.c index 7d93951c2ed2..1b213af59135 100644 --- a/tools/perf/builtin-kwork.c +++ b/tools/perf/builtin-kwork.c @@ -45,6 +45,11 @@ #define PRINT_BRACKETPAIR_WIDTH 2 #define PRINT_TIME_UNIT_SEC_WIDTH 2 #define PRINT_TIME_UNIT_MESC_WIDTH 3 +#define PRINT_PID_WIDTH 7 +#define PRINT_TASK_NAME_WIDTH 16 +#define PRINT_CPU_USAGE_WIDTH 6 +#define PRINT_CPU_USAGE_DECIMAL_WIDTH 2 +#define PRINT_CPU_USAGE_HIST_WIDTH 30 #define PRINT_RUNTIME_HEADER_WIDTH (PRINT_RUNTIME_WIDTH + PRINT_TIME_UNIT_MESC_WIDTH) #define PRINT_LATENCY_HEADER_WIDTH (PRINT_LATENCY_WIDTH + PRINT_TIME_UNIT_MESC_WIDTH) #define PRINT_TIMEHIST_CPU_WIDTH (PRINT_CPU_WIDTH + PRINT_BRACKETPAIR_WIDTH) @@ -131,6 +136,16 @@ static int max_latency_cmp(struct kwork_work *l, struct kwork_work *r) return 0; } +static int cpu_usage_cmp(struct kwork_work *l, struct kwork_work *r) +{ + if (l->cpu_usage > r->cpu_usage) + return 1; + if (l->cpu_usage < r->cpu_usage) + return -1; + + return 0; +} + static int sort_dimension__add(struct perf_kwork *kwork __maybe_unused, const char *tok, struct list_head *list) { @@ -155,12 +170,17 @@ static int sort_dimension__add(struct perf_kwork *kwork __maybe_unused, .name = "avg", .cmp = avg_latency_cmp, }; + static struct sort_dimension rate_sort_dimension = { + .name = "rate", + .cmp = cpu_usage_cmp, + }; struct sort_dimension *available_sorts[] = { &id_sort_dimension, &max_sort_dimension, &count_sort_dimension, &runtime_sort_dimension, &avg_sort_dimension, + &rate_sort_dimension, }; if (kwork->report == KWORK_REPORT_LATENCY) @@ -485,6 +505,38 @@ static struct kwork_atom *work_pop_atom(struct perf_kwork *kwork, return NULL; } +static struct kwork_work *find_work_by_id(struct rb_root_cached *root, + u64 id, int cpu) +{ + struct rb_node *next; + struct kwork_work *work; + + next = rb_first_cached(root); + while (next) { + work = rb_entry(next, struct kwork_work, node); + if ((cpu != -1 && work->id == id && work->cpu == cpu) || + (cpu == -1 && work->id == id)) + return work; + + next = rb_next(next); + } + + return NULL; +} + +static struct kwork_class *get_kwork_class(struct perf_kwork *kwork, + enum kwork_class_type type) +{ + struct kwork_class *class; + + list_for_each_entry(class, &kwork->class_list, list) { + if (class->type == type) + return class; + } + + return NULL; +} + static void report_update_exit_event(struct kwork_work *work, struct kwork_atom *atom, struct perf_sample *sample) @@ -789,6 +841,54 @@ out: return ret; } +static void top_update_runtime(struct kwork_work *work, + struct kwork_atom *atom, + struct perf_sample *sample) +{ + u64 delta; + u64 exit_time = sample->time; + u64 entry_time = atom->time; + + if ((entry_time != 0) && (exit_time >= entry_time)) { + delta = exit_time - entry_time; + work->total_runtime += delta; + } +} + +static int top_entry_event(struct perf_kwork *kwork, + struct kwork_class *class, + struct evsel *evsel, + struct perf_sample *sample, + struct machine *machine) +{ + return work_push_atom(kwork, class, KWORK_TRACE_ENTRY, + KWORK_TRACE_MAX, evsel, sample, + machine, NULL, true); +} + +static int top_sched_switch_event(struct perf_kwork *kwork, + struct kwork_class *class, + struct evsel *evsel, + struct perf_sample *sample, + struct machine *machine) +{ + struct kwork_atom *atom; + struct kwork_work *work; + + atom = work_pop_atom(kwork, class, KWORK_TRACE_EXIT, + KWORK_TRACE_ENTRY, evsel, sample, + machine, &work); + if (!work) + return -1; + + if (atom) { + top_update_runtime(work, atom, sample); + atom_del(atom); + } + + return top_entry_event(kwork, class, evsel, sample, machine); +} + static struct kwork_class kwork_irq; static int process_irq_handler_entry_event(struct perf_tool *tool, struct evsel *evsel, @@ -1378,6 +1478,101 @@ static void print_bad_events(struct perf_kwork *kwork) } } +const char *graph_load = "||||||||||||||||||||||||||||||||||||||||||||||||"; +const char *graph_idle = " "; +static void top_print_per_cpu_load(struct perf_kwork *kwork) +{ + int i, load_width; + u64 total, load, load_ratio; + struct kwork_top_stat *stat = &kwork->top_stat; + + for (i = 0; i < MAX_NR_CPUS; i++) { + total = stat->cpus_runtime[i].total; + load = stat->cpus_runtime[i].load; + if (test_bit(i, stat->all_cpus_bitmap) && total) { + load_ratio = load * 10000 / total; + load_width = PRINT_CPU_USAGE_HIST_WIDTH * + load_ratio / 10000; + + printf("%%Cpu%-*d[%.*s%.*s %*.*f%%]\n", + PRINT_CPU_WIDTH, i, + load_width, graph_load, + PRINT_CPU_USAGE_HIST_WIDTH - load_width, + graph_idle, + PRINT_CPU_USAGE_WIDTH, + PRINT_CPU_USAGE_DECIMAL_WIDTH, + (double)load_ratio / 100); + } + } +} + +static void top_print_cpu_usage(struct perf_kwork *kwork) +{ + struct kwork_top_stat *stat = &kwork->top_stat; + u64 idle_time = stat->cpus_runtime[MAX_NR_CPUS].idle; + int cpus_nr = bitmap_weight(stat->all_cpus_bitmap, MAX_NR_CPUS); + u64 cpus_total_time = stat->cpus_runtime[MAX_NR_CPUS].total; + + printf("Total : %*.*f ms, %d cpus\n", + PRINT_RUNTIME_WIDTH, RPINT_DECIMAL_WIDTH, + (double)cpus_total_time / NSEC_PER_MSEC, + cpus_nr); + + printf("%%Cpu(s): %*.*f%% id\n", + PRINT_CPU_USAGE_WIDTH, PRINT_CPU_USAGE_DECIMAL_WIDTH, + cpus_total_time ? (double)idle_time * 100 / cpus_total_time : 0); + + top_print_per_cpu_load(kwork); +} + +static void top_print_header(struct perf_kwork *kwork __maybe_unused) +{ + int ret; + + printf("\n "); + ret = printf(" %*s %*s %*s %-*s", + PRINT_PID_WIDTH, "PID", + PRINT_CPU_USAGE_WIDTH, "%CPU", + PRINT_RUNTIME_HEADER_WIDTH + RPINT_DECIMAL_WIDTH, "RUNTIME", + PRINT_TASK_NAME_WIDTH, "COMMMAND"); + printf("\n "); + print_separator(ret); +} + +static int top_print_work(struct perf_kwork *kwork __maybe_unused, struct kwork_work *work) +{ + int ret = 0; + + printf(" "); + + /* + * pid + */ + ret += printf(" %*ld ", PRINT_PID_WIDTH, work->id); + + /* + * cpu usage + */ + ret += printf(" %*.*f ", + PRINT_CPU_USAGE_WIDTH, PRINT_CPU_USAGE_DECIMAL_WIDTH, + (double)work->cpu_usage / 100); + + /* + * total runtime + */ + ret += printf(" %*.*f ms ", + PRINT_RUNTIME_WIDTH + RPINT_DECIMAL_WIDTH, RPINT_DECIMAL_WIDTH, + (double)work->total_runtime / NSEC_PER_MSEC); + + /* + * command + */ + ret += printf(" %-*s", PRINT_TASK_NAME_WIDTH, work->name); + + printf("\n"); + return ret; +} + static void work_sort(struct perf_kwork *kwork, struct kwork_class *class, struct rb_root_cached *root) { @@ -1425,6 +1620,9 @@ static int perf_kwork__check_config(struct perf_kwork *kwork, .entry_event = timehist_entry_event, .exit_event = timehist_exit_event, }; + static struct trace_kwork_handler top_ops = { + .sched_switch_event = top_sched_switch_event, + }; switch (kwork->report) { case KWORK_REPORT_RUNTIME: @@ -1436,6 +1634,9 @@ static int perf_kwork__check_config(struct perf_kwork *kwork, case KWORK_REPORT_TIMEHIST: kwork->tp_handler = &timehist_ops; break; + case KWORK_REPORT_TOP: + kwork->tp_handler = &top_ops; + break; default: pr_debug("Invalid report type %d\n", kwork->report); return -1; @@ -1682,6 +1883,169 @@ static int perf_kwork__timehist(struct perf_kwork *kwork) return perf_kwork__read_events(kwork); } +static void top_calc_total_runtime(struct perf_kwork *kwork) +{ + struct kwork_class *class; + struct kwork_work *work; + struct rb_node *next; + struct kwork_top_stat *stat = &kwork->top_stat; + + class = get_kwork_class(kwork, KWORK_CLASS_SCHED); + if (!class) + return; + + next = rb_first_cached(&class->work_root); + while (next) { + work = rb_entry(next, struct kwork_work, node); + BUG_ON(work->cpu >= MAX_NR_CPUS); + stat->cpus_runtime[work->cpu].total += work->total_runtime; + stat->cpus_runtime[MAX_NR_CPUS].total += work->total_runtime; + next = rb_next(next); + } +} + +static void top_calc_idle_time(struct perf_kwork *kwork, + struct kwork_work *work) +{ + struct kwork_top_stat *stat = &kwork->top_stat; + + if (work->id == 0) { + stat->cpus_runtime[work->cpu].idle += work->total_runtime; + stat->cpus_runtime[MAX_NR_CPUS].idle += work->total_runtime; + } +} + +static void top_calc_cpu_usage(struct perf_kwork *kwork) +{ + struct kwork_class *class; + struct kwork_work *work; + struct rb_node *next; + struct kwork_top_stat *stat = &kwork->top_stat; + + class = get_kwork_class(kwork, KWORK_CLASS_SCHED); + if (!class) + return; + + next = rb_first_cached(&class->work_root); + while (next) { + work = rb_entry(next, struct kwork_work, node); + + if (work->total_runtime == 0) + goto next; + + __set_bit(work->cpu, stat->all_cpus_bitmap); + + work->cpu_usage = work->total_runtime * 10000 / + stat->cpus_runtime[work->cpu].total; + + top_calc_idle_time(kwork, work); +next: + next = rb_next(next); + } +} + +static void top_calc_load_runtime(struct perf_kwork *kwork, + struct kwork_work *work) +{ + struct kwork_top_stat *stat = &kwork->top_stat; + + if (work->id != 0) { + stat->cpus_runtime[work->cpu].load += work->total_runtime; + stat->cpus_runtime[MAX_NR_CPUS].load += work->total_runtime; + } +} + +static void top_merge_tasks(struct perf_kwork *kwork) +{ + struct kwork_work *merged_work, *data; + struct kwork_class *class; + struct rb_node *node; + int cpu; + struct rb_root_cached merged_root = RB_ROOT_CACHED; + + class = get_kwork_class(kwork, KWORK_CLASS_SCHED); + if (!class) + return; + + for (;;) { + node = rb_first_cached(&class->work_root); + if (!node) + break; + + rb_erase_cached(node, &class->work_root); + data = rb_entry(node, struct kwork_work, node); + + cpu = data->cpu; + merged_work = find_work_by_id(&merged_root, data->id, + data->id == 0 ? cpu : -1); + if (!merged_work) { + work_insert(&merged_root, data, &kwork->cmp_id); + } else { + merged_work->total_runtime += data->total_runtime; + merged_work->cpu_usage += data->cpu_usage; + } + + top_calc_load_runtime(kwork, data); + } + + work_sort(kwork, class, &merged_root); +} + +static void perf_kwork__top_report(struct perf_kwork *kwork) +{ + struct kwork_work *work; + struct rb_node *next; + + printf("\n"); + + top_print_cpu_usage(kwork); + top_print_header(kwork); + next = rb_first_cached(&kwork->sorted_work_root); + while (next) { + work = rb_entry(next, struct kwork_work, node); + process_skipped_events(kwork, work); + + if (work->total_runtime == 0) + goto next; + + top_print_work(kwork, work); + +next: + next = rb_next(next); + } + + printf("\n"); +} + +static int perf_kwork__top(struct perf_kwork *kwork) +{ + struct __top_cpus_runtime *cpus_runtime; + int ret = 0; + + cpus_runtime = zalloc(sizeof(struct __top_cpus_runtime) * (MAX_NR_CPUS + 1)); + if (!cpus_runtime) + return -1; + + kwork->top_stat.cpus_runtime = cpus_runtime; + bitmap_zero(kwork->top_stat.all_cpus_bitmap, MAX_NR_CPUS); + + ret = perf_kwork__read_events(kwork); + if (ret) + goto out; + + top_calc_total_runtime(kwork); + top_calc_cpu_usage(kwork); + top_merge_tasks(kwork); + + setup_pager(); + + perf_kwork__top_report(kwork); + +out: + free(kwork->top_stat.cpus_runtime); + return ret; +} + static void setup_event_list(struct perf_kwork *kwork, const struct option *options, const char * const usage_msg[]) @@ -1801,6 +2165,7 @@ int cmd_kwork(int argc, const char **argv) }; static const char default_report_sort_order[] = "runtime, max, count"; static const char default_latency_sort_order[] = "avg, max, count"; + static const char default_top_sort_order[] = "rate, runtime"; const struct option kwork_options[] = { OPT_INCR('v', "verbose", &verbose, "be more verbose (show symbol address, etc)"), @@ -1868,6 +2233,9 @@ int cmd_kwork(int argc, const char **argv) "input file name"), OPT_PARENT(kwork_options) }; + const struct option top_options[] = { + OPT_PARENT(kwork_options) + }; const char *kwork_usage[] = { NULL, NULL @@ -1884,8 +2252,12 @@ int cmd_kwork(int argc, const char **argv) "perf kwork timehist []", NULL }; + const char * const top_usage[] = { + "perf kwork top []", + NULL + }; const char *const kwork_subcommands[] = { - "record", "report", "latency", "timehist", NULL + "record", "report", "latency", "timehist", "top", NULL }; argc = parse_options_subcommand(argc, argv, kwork_options, @@ -1930,6 +2302,19 @@ int cmd_kwork(int argc, const char **argv) kwork.report = KWORK_REPORT_TIMEHIST; setup_event_list(&kwork, kwork_options, kwork_usage); return perf_kwork__timehist(&kwork); + } else if (strlen(argv[0]) > 2 && strstarts("top", argv[0])) { + kwork.sort_order = default_top_sort_order; + if (argc > 1) { + argc = parse_options(argc, argv, top_options, top_usage, 0); + if (argc) + usage_with_options(top_usage, top_options); + } + kwork.report = KWORK_REPORT_TOP; + if (!kwork.event_list_str) + kwork.event_list_str = "sched"; + setup_event_list(&kwork, kwork_options, kwork_usage); + setup_sorting(&kwork, top_options, top_usage); + return perf_kwork__top(&kwork); } else usage_with_options(kwork_usage, kwork_options); diff --git a/tools/perf/util/kwork.h b/tools/perf/util/kwork.h index f8e9cdd1371a..41ed193d5d8b 100644 --- a/tools/perf/util/kwork.h +++ b/tools/perf/util/kwork.h @@ -24,6 +24,7 @@ enum kwork_report_type { KWORK_REPORT_RUNTIME, KWORK_REPORT_LATENCY, KWORK_REPORT_TIMEHIST, + KWORK_REPORT_TOP, }; enum kwork_trace_type { @@ -129,6 +130,11 @@ struct kwork_work { u64 max_latency_start; u64 max_latency_end; u64 total_latency; + + /* + * top report + */ + u32 cpu_usage; }; struct kwork_class { @@ -174,6 +180,17 @@ struct trace_kwork_handler { struct perf_sample *sample, struct machine *machine); }; +struct __top_cpus_runtime { + u64 load; + u64 idle; + u64 total; +}; + +struct kwork_top_stat { + DECLARE_BITMAP(all_cpus_bitmap, MAX_NR_CPUS); + struct __top_cpus_runtime *cpus_runtime; +}; + struct perf_kwork { /* * metadata @@ -225,6 +242,11 @@ struct perf_kwork { u64 all_runtime; u64 all_count; u64 nr_skipped_events[KWORK_TRACE_MAX + 1]; + + /* + * perf kwork top data + */ + struct kwork_top_stat top_stat; }; struct kwork_work *perf_kwork_add_work(struct perf_kwork *kwork, From a8792242e4f2915a85bfa2d21a0ee2d9043de996 Mon Sep 17 00:00:00 2001 From: Yang Jihong Date: Sat, 12 Aug 2023 08:49:11 +0000 Subject: [PATCH 052/475] perf evsel: Add evsel__intval_common() helper Add evsel__intval_common() helper to search for common_field in tracepoint format. Reviewed-by: Ian Rogers Signed-off-by: Yang Jihong Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ingo Molnar Cc: Jiri Olsa Cc: Kan Liang Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Ravi Bangoria Cc: Sandipan Das Link: https://lore.kernel.org/r/20230812084917.169338-11-yangjihong1@huawei.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/util/evsel.c | 13 +++++++++++++ tools/perf/util/evsel.h | 2 ++ 2 files changed, 15 insertions(+) diff --git a/tools/perf/util/evsel.c b/tools/perf/util/evsel.c index a8a5ff87cc1f..d5363d23f5d3 100644 --- a/tools/perf/util/evsel.c +++ b/tools/perf/util/evsel.c @@ -2766,6 +2766,11 @@ struct tep_format_field *evsel__field(struct evsel *evsel, const char *name) return tep_find_field(evsel->tp_format, name); } +struct tep_format_field *evsel__common_field(struct evsel *evsel, const char *name) +{ + return tep_find_common_field(evsel->tp_format, name); +} + void *evsel__rawptr(struct evsel *evsel, struct perf_sample *sample, const char *name) { struct tep_format_field *field = evsel__field(evsel, name); @@ -2831,6 +2836,14 @@ u64 evsel__intval(struct evsel *evsel, struct perf_sample *sample, const char *n return field ? format_field__intval(field, sample, evsel->needs_swap) : 0; } + +u64 evsel__intval_common(struct evsel *evsel, struct perf_sample *sample, const char *name) +{ + struct tep_format_field *field = evsel__common_field(evsel, name); + + return field ? format_field__intval(field, sample, evsel->needs_swap) : 0; +} + #endif bool evsel__fallback(struct evsel *evsel, int err, char *msg, size_t msgsize) diff --git a/tools/perf/util/evsel.h b/tools/perf/util/evsel.h index 848534ec74fa..815be2491938 100644 --- a/tools/perf/util/evsel.h +++ b/tools/perf/util/evsel.h @@ -340,6 +340,7 @@ struct perf_sample; #ifdef HAVE_LIBTRACEEVENT void *evsel__rawptr(struct evsel *evsel, struct perf_sample *sample, const char *name); u64 evsel__intval(struct evsel *evsel, struct perf_sample *sample, const char *name); +u64 evsel__intval_common(struct evsel *evsel, struct perf_sample *sample, const char *name); static inline char *evsel__strval(struct evsel *evsel, struct perf_sample *sample, const char *name) { @@ -352,6 +353,7 @@ struct tep_format_field; u64 format_field__intval(struct tep_format_field *field, struct perf_sample *sample, bool needs_swap); struct tep_format_field *evsel__field(struct evsel *evsel, const char *name); +struct tep_format_field *evsel__common_field(struct evsel *evsel, const char *name); static inline bool __evsel__match(const struct evsel *evsel, u32 type, u64 config) { From 2f21f5e4b48d910cbbe234264527b23b7cfc49f2 Mon Sep 17 00:00:00 2001 From: Yang Jihong Date: Sat, 12 Aug 2023 08:49:12 +0000 Subject: [PATCH 053/475] perf kwork top: Add statistics on hardirq event support Calculate the runtime of the hardirq events and subtract it from the corresponding task runtime to improve the precision. Example usage: # perf kwork -k sched,irq record -- perf record -o perf_record.data -a sleep 10 [ perf record: Woken up 2 times to write data ] [ perf record: Captured and wrote 1.054 MB perf_record.data (18019 samples) ] [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 1.798 MB perf.data (16334 samples) ] # # perf kwork top Total : 139240.869 ms, 8 cpus %Cpu(s): 94.91% id, 0.05% hi %Cpu0 [ 0.05%] %Cpu1 [| 5.00%] %Cpu2 [ 0.43%] %Cpu3 [ 0.57%] %Cpu4 [ 1.19%] %Cpu5 [|||||| 20.46%] %Cpu6 [ 0.48%] %Cpu7 [||| 12.10%] PID %CPU RUNTIME COMMMAND ---------------------------------------------------- 0 99.54 17325.622 ms swapper/2 0 99.54 17327.527 ms swapper/0 0 99.51 17319.909 ms swapper/6 0 99.42 17304.934 ms swapper/3 0 98.80 17197.385 ms swapper/4 0 94.99 16534.991 ms swapper/1 0 87.89 15295.264 ms swapper/7 0 79.53 13843.182 ms swapper/5 4252 36.50 6361.768 ms perf 4256 1.17 205.215 ms bash 151 0.53 93.298 ms systemd-resolve 4254 0.39 69.468 ms perf 423 0.34 59.368 ms bash 412 0.29 51.204 ms sshd 249 0.20 35.288 ms sd-resolve 16 0.17 30.287 ms rcu_preempt 153 0.09 17.266 ms systemd-timesyn 1 0.09 17.078 ms systemd 4253 0.07 12.457 ms perf 4255 0.06 11.559 ms perf 4234 0.03 6.105 ms kworker/u16:1 69 0.03 6.259 ms kworker/1:1H 4251 0.02 4.615 ms perf 4095 0.02 4.890 ms kworker/7:1 61 0.02 4.005 ms kcompactd0 75 0.02 3.546 ms kworker/2:1 97 0.01 3.106 ms kworker/7:1H 98 0.01 1.995 ms jbd2/sda-8 4088 0.01 1.779 ms kworker/u16:3 2909 0.01 1.795 ms kworker/0:2 4246 0.00 1.117 ms kworker/7:2 51 0.00 0.327 ms ksoftirqd/7 50 0.00 0.369 ms migration/7 102 0.00 0.160 ms kworker/6:1H 76 0.00 0.609 ms kworker/6:1 45 0.00 0.779 ms migration/6 87 0.00 0.504 ms kworker/5:1H 73 0.00 1.130 ms kworker/5:1 41 0.00 0.152 ms ksoftirqd/5 40 0.00 0.702 ms migration/5 64 0.00 0.316 ms kworker/4:1 35 0.00 0.791 ms migration/4 353 0.00 2.211 ms sshd 74 0.00 0.272 ms kworker/3:1 30 0.00 0.819 ms migration/3 25 0.00 0.784 ms migration/2 397 0.00 0.539 ms kworker/1:1 21 0.00 1.600 ms ksoftirqd/1 20 0.00 0.773 ms migration/1 17 0.00 1.682 ms migration/0 15 0.00 0.076 ms ksoftirqd/0 Reviewed-by: Ian Rogers Signed-off-by: Yang Jihong Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ingo Molnar Cc: Jiri Olsa Cc: Kan Liang Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Ravi Bangoria Cc: Sandipan Das Link: https://lore.kernel.org/r/20230812084917.169338-12-yangjihong1@huawei.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/builtin-kwork.c | 93 +++++++++++++++++++++++++++++++++++--- tools/perf/util/kwork.h | 1 + 2 files changed, 88 insertions(+), 6 deletions(-) diff --git a/tools/perf/builtin-kwork.c b/tools/perf/builtin-kwork.c index 1b213af59135..66ad8c2e8020 100644 --- a/tools/perf/builtin-kwork.c +++ b/tools/perf/builtin-kwork.c @@ -866,6 +866,36 @@ static int top_entry_event(struct perf_kwork *kwork, machine, NULL, true); } +static int top_exit_event(struct perf_kwork *kwork, + struct kwork_class *class, + struct evsel *evsel, + struct perf_sample *sample, + struct machine *machine) +{ + struct kwork_work *work, *sched_work; + struct kwork_class *sched_class; + struct kwork_atom *atom; + + atom = work_pop_atom(kwork, class, KWORK_TRACE_EXIT, + KWORK_TRACE_ENTRY, evsel, sample, + machine, &work); + if (!work) + return -1; + + if (atom) { + sched_class = get_kwork_class(kwork, KWORK_CLASS_SCHED); + if (sched_class) { + sched_work = find_work_by_id(&sched_class->work_root, + work->id, work->cpu); + if (sched_work) + top_update_runtime(work, atom, sample); + } + atom_del(atom); + } + + return 0; +} + static int top_sched_switch_event(struct perf_kwork *kwork, struct kwork_class *class, struct evsel *evsel, @@ -933,7 +963,7 @@ static int irq_class_init(struct kwork_class *class, return 0; } -static void irq_work_init(struct perf_kwork *kwork __maybe_unused, +static void irq_work_init(struct perf_kwork *kwork, struct kwork_class *class, struct kwork_work *work, enum kwork_trace_type src_type __maybe_unused, @@ -943,8 +973,14 @@ static void irq_work_init(struct perf_kwork *kwork __maybe_unused, { work->class = class; work->cpu = sample->cpu; - work->id = evsel__intval(evsel, sample, "irq"); - work->name = evsel__strval(evsel, sample, "name"); + + if (kwork->report == KWORK_REPORT_TOP) { + work->id = evsel__intval_common(evsel, sample, "common_pid"); + work->name = NULL; + } else { + work->id = evsel__intval(evsel, sample, "irq"); + work->name = evsel__strval(evsel, sample, "name"); + } } static void irq_work_name(struct kwork_work *work, char *buf, int len) @@ -1510,6 +1546,7 @@ static void top_print_cpu_usage(struct perf_kwork *kwork) { struct kwork_top_stat *stat = &kwork->top_stat; u64 idle_time = stat->cpus_runtime[MAX_NR_CPUS].idle; + u64 hardirq_time = stat->cpus_runtime[MAX_NR_CPUS].irq; int cpus_nr = bitmap_weight(stat->all_cpus_bitmap, MAX_NR_CPUS); u64 cpus_total_time = stat->cpus_runtime[MAX_NR_CPUS].total; @@ -1518,9 +1555,12 @@ static void top_print_cpu_usage(struct perf_kwork *kwork) (double)cpus_total_time / NSEC_PER_MSEC, cpus_nr); - printf("%%Cpu(s): %*.*f%% id\n", + printf("%%Cpu(s): %*.*f%% id, %*.*f%% hi\n", PRINT_CPU_USAGE_WIDTH, PRINT_CPU_USAGE_DECIMAL_WIDTH, - cpus_total_time ? (double)idle_time * 100 / cpus_total_time : 0); + cpus_total_time ? (double)idle_time * 100 / cpus_total_time : 0, + + PRINT_CPU_USAGE_WIDTH, PRINT_CPU_USAGE_DECIMAL_WIDTH, + cpus_total_time ? (double)hardirq_time * 100 / cpus_total_time : 0); top_print_per_cpu_load(kwork); } @@ -1621,6 +1661,8 @@ static int perf_kwork__check_config(struct perf_kwork *kwork, .exit_event = timehist_exit_event, }; static struct trace_kwork_handler top_ops = { + .entry_event = timehist_entry_event, + .exit_event = top_exit_event, .sched_switch_event = top_sched_switch_event, }; @@ -1915,6 +1957,43 @@ static void top_calc_idle_time(struct perf_kwork *kwork, } } +static void top_calc_irq_runtime(struct perf_kwork *kwork, + enum kwork_class_type type, + struct kwork_work *work) +{ + struct kwork_top_stat *stat = &kwork->top_stat; + + if (type == KWORK_CLASS_IRQ) { + stat->cpus_runtime[work->cpu].irq += work->total_runtime; + stat->cpus_runtime[MAX_NR_CPUS].irq += work->total_runtime; + } +} + +static void top_subtract_irq_runtime(struct perf_kwork *kwork, + struct kwork_work *work) +{ + struct kwork_class *class; + struct kwork_work *data; + unsigned int i; + int irq_class_list[] = {KWORK_CLASS_IRQ}; + + for (i = 0; i < ARRAY_SIZE(irq_class_list); i++) { + class = get_kwork_class(kwork, irq_class_list[i]); + if (!class) + continue; + + data = find_work_by_id(&class->work_root, + work->id, work->cpu); + if (!data) + continue; + + if (work->total_runtime > data->total_runtime) { + work->total_runtime -= data->total_runtime; + top_calc_irq_runtime(kwork, irq_class_list[i], data); + } + } +} + static void top_calc_cpu_usage(struct perf_kwork *kwork) { struct kwork_class *class; @@ -1935,6 +2014,8 @@ static void top_calc_cpu_usage(struct perf_kwork *kwork) __set_bit(work->cpu, stat->all_cpus_bitmap); + top_subtract_irq_runtime(kwork, work); + work->cpu_usage = work->total_runtime * 10000 / stat->cpus_runtime[work->cpu].total; @@ -2311,7 +2392,7 @@ int cmd_kwork(int argc, const char **argv) } kwork.report = KWORK_REPORT_TOP; if (!kwork.event_list_str) - kwork.event_list_str = "sched"; + kwork.event_list_str = "sched, irq"; setup_event_list(&kwork, kwork_options, kwork_usage); setup_sorting(&kwork, top_options, top_usage); return perf_kwork__top(&kwork); diff --git a/tools/perf/util/kwork.h b/tools/perf/util/kwork.h index 41ed193d5d8b..b3e410d02043 100644 --- a/tools/perf/util/kwork.h +++ b/tools/perf/util/kwork.h @@ -183,6 +183,7 @@ struct trace_kwork_handler { struct __top_cpus_runtime { u64 load; u64 idle; + u64 irq; u64 total; }; From e29090d28cc1ac1479a062e07260fdd7aa56f057 Mon Sep 17 00:00:00 2001 From: Yang Jihong Date: Sat, 12 Aug 2023 08:49:13 +0000 Subject: [PATCH 054/475] perf kwork top: Add statistics on softirq event support Calculate the runtime of the softirq events and subtract it from the corresponding task runtime to improve the precision. Example usage: # perf kwork -k sched,irq,softirq record -- perf record -e cpu-clock -o perf_record.data -a sleep 10 [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.467 MB perf_record.data (7154 samples) ] [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 2.152 MB perf.data (22846 samples) ] # perf kwork top Total : 136601.588 ms, 8 cpus %Cpu(s): 95.66% id, 0.04% hi, 0.05% si %Cpu0 [ 0.02%] %Cpu1 [ 0.01%] %Cpu2 [| 4.61%] %Cpu3 [ 0.04%] %Cpu4 [ 0.01%] %Cpu5 [||||| 17.31%] %Cpu6 [ 0.51%] %Cpu7 [||| 11.42%] PID %CPU RUNTIME COMMMAND ---------------------------------------------------- 0 99.98 17073.515 ms swapper/4 0 99.98 17072.173 ms swapper/1 0 99.93 17064.229 ms swapper/3 0 99.62 17011.013 ms swapper/0 0 99.47 16985.180 ms swapper/6 0 95.17 16250.874 ms swapper/2 0 88.51 15111.684 ms swapper/7 0 82.62 14108.577 ms swapper/5 4342 33.00 5644.045 ms perf 4344 0.43 74.351 ms perf 16 0.13 22.296 ms rcu_preempt 4345 0.05 10.093 ms perf 4343 0.05 8.769 ms perf 4341 0.02 4.882 ms perf 4095 0.02 4.605 ms kworker/7:1 75 0.02 4.261 ms kworker/2:1 120 0.01 1.909 ms systemd-journal 98 0.01 2.540 ms jbd2/sda-8 61 0.01 3.404 ms kcompactd0 667 0.01 2.542 ms kworker/u16:2 4340 0.00 1.052 ms kworker/7:2 97 0.00 0.489 ms kworker/7:1H 51 0.00 0.209 ms ksoftirqd/7 50 0.00 0.646 ms migration/7 76 0.00 0.753 ms kworker/6:1 45 0.00 0.572 ms migration/6 87 0.00 0.145 ms kworker/5:1H 73 0.00 0.596 ms kworker/5:1 41 0.00 0.041 ms ksoftirqd/5 40 0.00 0.718 ms migration/5 64 0.00 0.115 ms kworker/4:1 35 0.00 0.556 ms migration/4 353 0.00 2.600 ms sshd 74 0.00 0.205 ms kworker/3:1 33 0.00 1.576 ms kworker/3:0H 30 0.00 0.996 ms migration/3 26 0.00 1.665 ms ksoftirqd/2 25 0.00 0.662 ms migration/2 397 0.00 0.057 ms kworker/1:1 20 0.00 1.005 ms migration/1 2909 0.00 1.053 ms kworker/0:2 17 0.00 0.720 ms migration/0 15 0.00 0.039 ms ksoftirqd/0 Reviewed-by: Ian Rogers Signed-off-by: Yang Jihong Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ingo Molnar Cc: Jiri Olsa Cc: Kan Liang Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Ravi Bangoria Cc: Sandipan Das Link: https://lore.kernel.org/r/20230812084917.169338-13-yangjihong1@huawei.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/builtin-kwork.c | 30 ++++++++++++++++++++++-------- tools/perf/util/kwork.h | 1 + 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/tools/perf/builtin-kwork.c b/tools/perf/builtin-kwork.c index 66ad8c2e8020..c741cc1a543f 100644 --- a/tools/perf/builtin-kwork.c +++ b/tools/perf/builtin-kwork.c @@ -1090,7 +1090,7 @@ static char *evsel__softirq_name(struct evsel *evsel, u64 num) return name; } -static void softirq_work_init(struct perf_kwork *kwork __maybe_unused, +static void softirq_work_init(struct perf_kwork *kwork, struct kwork_class *class, struct kwork_work *work, enum kwork_trace_type src_type __maybe_unused, @@ -1098,12 +1098,19 @@ static void softirq_work_init(struct perf_kwork *kwork __maybe_unused, struct perf_sample *sample, struct machine *machine __maybe_unused) { - u64 num = evsel__intval(evsel, sample, "vec"); + u64 num; - work->id = num; work->class = class; work->cpu = sample->cpu; - work->name = evsel__softirq_name(evsel, num); + + if (kwork->report == KWORK_REPORT_TOP) { + work->id = evsel__intval_common(evsel, sample, "common_pid"); + work->name = NULL; + } else { + num = evsel__intval(evsel, sample, "vec"); + work->id = num; + work->name = evsel__softirq_name(evsel, num); + } } static void softirq_work_name(struct kwork_work *work, char *buf, int len) @@ -1547,6 +1554,7 @@ static void top_print_cpu_usage(struct perf_kwork *kwork) struct kwork_top_stat *stat = &kwork->top_stat; u64 idle_time = stat->cpus_runtime[MAX_NR_CPUS].idle; u64 hardirq_time = stat->cpus_runtime[MAX_NR_CPUS].irq; + u64 softirq_time = stat->cpus_runtime[MAX_NR_CPUS].softirq; int cpus_nr = bitmap_weight(stat->all_cpus_bitmap, MAX_NR_CPUS); u64 cpus_total_time = stat->cpus_runtime[MAX_NR_CPUS].total; @@ -1555,12 +1563,15 @@ static void top_print_cpu_usage(struct perf_kwork *kwork) (double)cpus_total_time / NSEC_PER_MSEC, cpus_nr); - printf("%%Cpu(s): %*.*f%% id, %*.*f%% hi\n", + printf("%%Cpu(s): %*.*f%% id, %*.*f%% hi, %*.*f%% si\n", PRINT_CPU_USAGE_WIDTH, PRINT_CPU_USAGE_DECIMAL_WIDTH, cpus_total_time ? (double)idle_time * 100 / cpus_total_time : 0, PRINT_CPU_USAGE_WIDTH, PRINT_CPU_USAGE_DECIMAL_WIDTH, - cpus_total_time ? (double)hardirq_time * 100 / cpus_total_time : 0); + cpus_total_time ? (double)hardirq_time * 100 / cpus_total_time : 0, + + PRINT_CPU_USAGE_WIDTH, PRINT_CPU_USAGE_DECIMAL_WIDTH, + cpus_total_time ? (double)softirq_time * 100 / cpus_total_time : 0); top_print_per_cpu_load(kwork); } @@ -1966,6 +1977,9 @@ static void top_calc_irq_runtime(struct perf_kwork *kwork, if (type == KWORK_CLASS_IRQ) { stat->cpus_runtime[work->cpu].irq += work->total_runtime; stat->cpus_runtime[MAX_NR_CPUS].irq += work->total_runtime; + } else if (type == KWORK_CLASS_SOFTIRQ) { + stat->cpus_runtime[work->cpu].softirq += work->total_runtime; + stat->cpus_runtime[MAX_NR_CPUS].softirq += work->total_runtime; } } @@ -1975,7 +1989,7 @@ static void top_subtract_irq_runtime(struct perf_kwork *kwork, struct kwork_class *class; struct kwork_work *data; unsigned int i; - int irq_class_list[] = {KWORK_CLASS_IRQ}; + int irq_class_list[] = {KWORK_CLASS_IRQ, KWORK_CLASS_SOFTIRQ}; for (i = 0; i < ARRAY_SIZE(irq_class_list); i++) { class = get_kwork_class(kwork, irq_class_list[i]); @@ -2392,7 +2406,7 @@ int cmd_kwork(int argc, const char **argv) } kwork.report = KWORK_REPORT_TOP; if (!kwork.event_list_str) - kwork.event_list_str = "sched, irq"; + kwork.event_list_str = "sched, irq, softirq"; setup_event_list(&kwork, kwork_options, kwork_usage); setup_sorting(&kwork, top_options, top_usage); return perf_kwork__top(&kwork); diff --git a/tools/perf/util/kwork.h b/tools/perf/util/kwork.h index b3e410d02043..723b34385df6 100644 --- a/tools/perf/util/kwork.h +++ b/tools/perf/util/kwork.h @@ -184,6 +184,7 @@ struct __top_cpus_runtime { u64 load; u64 idle; u64 irq; + u64 softirq; u64 total; }; From aa172a5ad3159c1cc97da37253c972683681adaa Mon Sep 17 00:00:00 2001 From: Yang Jihong Date: Sat, 12 Aug 2023 08:49:14 +0000 Subject: [PATCH 055/475] perf kwork top: Add -C/--cpu -i/--input -n/--name -s/--sort --time options Provide the following options for perf kwork top: 1. -C, --cpu list of cpus to profile 2. -i, --input input file name 3. -n, --name event name to profile 4. -s, --sort sort by key(s): rate, runtime, tid 5. --time Time span for analysis (start,stop) Example usage: # perf kwork top -h Usage: perf kwork top [] -C, --cpu list of cpus to profile -i, --input input file name -n, --name event name to profile -s, --sort sort by key(s): rate, runtime, tid --time Time span for analysis (start,stop) # perf kwork top -C 2,4,5 Total : 51226.940 ms, 3 cpus %Cpu(s): 92.59% id, 0.00% hi, 0.09% si %Cpu2 [| 4.61%] %Cpu4 [ 0.01%] %Cpu5 [||||| 17.31%] PID %CPU RUNTIME COMMMAND ---------------------------------------------------- 0 99.98 17073.515 ms swapper/4 0 95.17 16250.874 ms swapper/2 0 82.62 14108.577 ms swapper/5 4342 21.70 3708.358 ms perf 16 0.13 22.296 ms rcu_preempt 75 0.02 4.261 ms kworker/2:1 98 0.01 2.540 ms jbd2/sda-8 61 0.01 3.404 ms kcompactd0 87 0.00 0.145 ms kworker/5:1H 73 0.00 0.596 ms kworker/5:1 41 0.00 0.041 ms ksoftirqd/5 40 0.00 0.718 ms migration/5 64 0.00 0.115 ms kworker/4:1 35 0.00 0.556 ms migration/4 353 0.00 1.143 ms sshd 26 0.00 1.665 ms ksoftirqd/2 25 0.00 0.662 ms migration/2 # perf kwork top -i perf.data Total : 136601.588 ms, 8 cpus %Cpu(s): 95.66% id, 0.04% hi, 0.05% si %Cpu0 [ 0.02%] %Cpu1 [ 0.01%] %Cpu2 [| 4.61%] %Cpu3 [ 0.04%] %Cpu4 [ 0.01%] %Cpu5 [||||| 17.31%] %Cpu6 [ 0.51%] %Cpu7 [||| 11.42%] PID %CPU RUNTIME COMMMAND ---------------------------------------------------- 0 99.98 17073.515 ms swapper/4 0 99.98 17072.173 ms swapper/1 0 99.93 17064.229 ms swapper/3 0 99.62 17011.013 ms swapper/0 0 99.47 16985.180 ms swapper/6 0 95.17 16250.874 ms swapper/2 0 88.51 15111.684 ms swapper/7 0 82.62 14108.577 ms swapper/5 4342 33.00 5644.045 ms perf 4344 0.43 74.351 ms perf 16 0.13 22.296 ms rcu_preempt 4345 0.05 10.093 ms perf 4343 0.05 8.769 ms perf 4341 0.02 4.882 ms perf 4095 0.02 4.605 ms kworker/7:1 75 0.02 4.261 ms kworker/2:1 120 0.01 1.909 ms systemd-journal 98 0.01 2.540 ms jbd2/sda-8 61 0.01 3.404 ms kcompactd0 667 0.01 2.542 ms kworker/u16:2 4340 0.00 1.052 ms kworker/7:2 97 0.00 0.489 ms kworker/7:1H 51 0.00 0.209 ms ksoftirqd/7 50 0.00 0.646 ms migration/7 76 0.00 0.753 ms kworker/6:1 45 0.00 0.572 ms migration/6 87 0.00 0.145 ms kworker/5:1H 73 0.00 0.596 ms kworker/5:1 41 0.00 0.041 ms ksoftirqd/5 40 0.00 0.718 ms migration/5 64 0.00 0.115 ms kworker/4:1 35 0.00 0.556 ms migration/4 353 0.00 2.600 ms sshd 74 0.00 0.205 ms kworker/3:1 33 0.00 1.576 ms kworker/3:0H 30 0.00 0.996 ms migration/3 26 0.00 1.665 ms ksoftirqd/2 25 0.00 0.662 ms migration/2 397 0.00 0.057 ms kworker/1:1 20 0.00 1.005 ms migration/1 2909 0.00 1.053 ms kworker/0:2 17 0.00 0.720 ms migration/0 15 0.00 0.039 ms ksoftirqd/0 # perf kwork top -n perf Total : 136601.588 ms, 8 cpus %Cpu(s): 95.66% id, 0.04% hi, 0.05% si %Cpu0 [ 0.01%] %Cpu1 [ 0.00%] %Cpu2 [| 4.44%] %Cpu3 [ 0.00%] %Cpu4 [ 0.00%] %Cpu5 [ 0.00%] %Cpu6 [ 0.49%] %Cpu7 [||| 11.38%] PID %CPU RUNTIME COMMMAND ---------------------------------------------------- 4342 15.74 2695.516 ms perf 4344 0.43 74.351 ms perf 4345 0.05 10.093 ms perf 4343 0.05 8.769 ms perf 4341 0.02 4.882 ms perf # perf kwork top -s tid Total : 136601.588 ms, 8 cpus %Cpu(s): 95.66% id, 0.04% hi, 0.05% si %Cpu0 [ 0.02%] %Cpu1 [ 0.01%] %Cpu2 [| 4.61%] %Cpu3 [ 0.04%] %Cpu4 [ 0.01%] %Cpu5 [||||| 17.31%] %Cpu6 [ 0.51%] %Cpu7 [||| 11.42%] PID %CPU RUNTIME COMMMAND ---------------------------------------------------- 0 99.62 17011.013 ms swapper/0 0 99.98 17072.173 ms swapper/1 0 95.17 16250.874 ms swapper/2 0 99.93 17064.229 ms swapper/3 0 99.98 17073.515 ms swapper/4 0 82.62 14108.577 ms swapper/5 0 99.47 16985.180 ms swapper/6 0 88.51 15111.684 ms swapper/7 15 0.00 0.039 ms ksoftirqd/0 16 0.13 22.296 ms rcu_preempt 17 0.00 0.720 ms migration/0 20 0.00 1.005 ms migration/1 25 0.00 0.662 ms migration/2 26 0.00 1.665 ms ksoftirqd/2 30 0.00 0.996 ms migration/3 33 0.00 1.576 ms kworker/3:0H 35 0.00 0.556 ms migration/4 40 0.00 0.718 ms migration/5 41 0.00 0.041 ms ksoftirqd/5 45 0.00 0.572 ms migration/6 50 0.00 0.646 ms migration/7 51 0.00 0.209 ms ksoftirqd/7 61 0.01 3.404 ms kcompactd0 64 0.00 0.115 ms kworker/4:1 73 0.00 0.596 ms kworker/5:1 74 0.00 0.205 ms kworker/3:1 75 0.02 4.261 ms kworker/2:1 76 0.00 0.753 ms kworker/6:1 87 0.00 0.145 ms kworker/5:1H 97 0.00 0.489 ms kworker/7:1H 98 0.01 2.540 ms jbd2/sda-8 120 0.01 1.909 ms systemd-journal 353 0.00 2.600 ms sshd 397 0.00 0.057 ms kworker/1:1 667 0.01 2.542 ms kworker/u16:2 2909 0.00 1.053 ms kworker/0:2 4095 0.02 4.605 ms kworker/7:1 4340 0.00 1.052 ms kworker/7:2 4341 0.02 4.882 ms perf 4342 33.00 5644.045 ms perf 4343 0.05 8.769 ms perf 4344 0.43 74.351 ms perf 4345 0.05 10.093 ms perf # perf kwork top --time 128800, Total : 53495.122 ms, 8 cpus %Cpu(s): 94.71% id, 0.09% hi, 0.09% si %Cpu0 [ 0.07%] %Cpu1 [ 0.04%] %Cpu2 [|| 8.49%] %Cpu3 [ 0.09%] %Cpu4 [ 0.02%] %Cpu5 [ 0.06%] %Cpu6 [ 0.12%] %Cpu7 [|||||| 21.24%] PID %CPU RUNTIME COMMMAND ---------------------------------------------------- 0 99.96 3981.363 ms swapper/4 0 99.94 3978.955 ms swapper/1 0 99.91 9329.375 ms swapper/5 0 99.87 4906.829 ms swapper/3 0 99.86 9028.064 ms swapper/6 0 98.67 3928.161 ms swapper/0 0 91.17 8388.432 ms swapper/2 0 78.65 7125.602 ms swapper/7 4342 29.42 2675.198 ms perf 16 0.18 16.817 ms rcu_preempt 4345 0.09 8.183 ms perf 4344 0.04 4.290 ms perf 4343 0.03 2.844 ms perf 353 0.03 2.600 ms sshd 4095 0.02 2.702 ms kworker/7:1 120 0.02 1.909 ms systemd-journal 98 0.02 2.540 ms jbd2/sda-8 61 0.02 1.886 ms kcompactd0 667 0.02 1.011 ms kworker/u16:2 75 0.02 2.693 ms kworker/2:1 4341 0.01 1.838 ms perf 30 0.01 0.788 ms migration/3 26 0.01 1.665 ms ksoftirqd/2 20 0.01 0.752 ms migration/1 2909 0.01 0.604 ms kworker/0:2 4340 0.00 0.635 ms kworker/7:2 97 0.00 0.214 ms kworker/7:1H 51 0.00 0.209 ms ksoftirqd/7 50 0.00 0.646 ms migration/7 76 0.00 0.602 ms kworker/6:1 45 0.00 0.366 ms migration/6 87 0.00 0.145 ms kworker/5:1H 40 0.00 0.446 ms migration/5 35 0.00 0.318 ms migration/4 74 0.00 0.205 ms kworker/3:1 33 0.00 0.080 ms kworker/3:0H 25 0.00 0.448 ms migration/2 397 0.00 0.057 ms kworker/1:1 17 0.00 0.365 ms migration/0 Reviewed-by: Ian Rogers Signed-off-by: Yang Jihong Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ingo Molnar Cc: Jiri Olsa Cc: Kan Liang Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Ravi Bangoria Cc: Sandipan Das Link: https://lore.kernel.org/r/20230812084917.169338-14-yangjihong1@huawei.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/Documentation/perf-kwork.txt | 26 +++++++++++ tools/perf/builtin-kwork.c | 57 +++++++++++++++++++++++-- 2 files changed, 80 insertions(+), 3 deletions(-) diff --git a/tools/perf/Documentation/perf-kwork.txt b/tools/perf/Documentation/perf-kwork.txt index 0601fcb0feea..34d6c285e527 100644 --- a/tools/perf/Documentation/perf-kwork.txt +++ b/tools/perf/Documentation/perf-kwork.txt @@ -178,6 +178,32 @@ OPTIONS for 'perf kwork timehist' stop time is not given (i.e, time string is 'x.y,') then analysis goes to end of file. +OPTIONS for 'perf kwork top' +--------------------------------- + +-C:: +--cpu:: + Only show events for the given CPU(s) (comma separated list). + +-i:: +--input:: + Input file name. (default: perf.data unless stdin is a fifo) + +-n:: +--name:: + Only show events for the given name. + +-s:: +--sort:: + Sort by key(s): rate, runtime, tid + +--time:: + Only analyze samples within given time window: ,. Times + have the format seconds.microseconds. If start is not given (i.e., time + string is ',x.y') then analysis starts at the beginning of the file. If + stop time is not given (i.e, time string is 'x.y,') then analysis goes + to end of file. + SEE ALSO -------- linkperf:perf-record[1] diff --git a/tools/perf/builtin-kwork.c b/tools/perf/builtin-kwork.c index c741cc1a543f..d5949ff4bd15 100644 --- a/tools/perf/builtin-kwork.c +++ b/tools/perf/builtin-kwork.c @@ -146,6 +146,24 @@ static int cpu_usage_cmp(struct kwork_work *l, struct kwork_work *r) return 0; } +static int id_or_cpu_r_cmp(struct kwork_work *l, struct kwork_work *r) +{ + if (l->id < r->id) + return 1; + if (l->id > r->id) + return -1; + + if (l->id != 0) + return 0; + + if (l->cpu < r->cpu) + return 1; + if (l->cpu > r->cpu) + return -1; + + return 0; +} + static int sort_dimension__add(struct perf_kwork *kwork __maybe_unused, const char *tok, struct list_head *list) { @@ -174,6 +192,10 @@ static int sort_dimension__add(struct perf_kwork *kwork __maybe_unused, .name = "rate", .cmp = cpu_usage_cmp, }; + static struct sort_dimension tid_sort_dimension = { + .name = "tid", + .cmp = id_or_cpu_r_cmp, + }; struct sort_dimension *available_sorts[] = { &id_sort_dimension, &max_sort_dimension, @@ -181,6 +203,7 @@ static int sort_dimension__add(struct perf_kwork *kwork __maybe_unused, &runtime_sort_dimension, &avg_sort_dimension, &rate_sort_dimension, + &tid_sort_dimension, }; if (kwork->report == KWORK_REPORT_LATENCY) @@ -381,6 +404,17 @@ static void profile_update_timespan(struct perf_kwork *kwork, kwork->timeend = sample->time; } +static bool profile_name_match(struct perf_kwork *kwork, + struct kwork_work *work) +{ + if (kwork->profile_name && work->name && + (strcmp(work->name, kwork->profile_name) != 0)) { + return false; + } + + return true; +} + static bool profile_event_match(struct perf_kwork *kwork, struct kwork_work *work, struct perf_sample *sample) @@ -396,10 +430,14 @@ static bool profile_event_match(struct perf_kwork *kwork, ((ptime->end != 0) && (ptime->end < time))) return false; - if ((kwork->profile_name != NULL) && - (work->name != NULL) && - (strcmp(work->name, kwork->profile_name) != 0)) + /* + * report top needs to collect the runtime of all tasks to + * calculate the load of each core. + */ + if ((kwork->report != KWORK_REPORT_TOP) && + !profile_name_match(kwork, work)) { return false; + } profile_update_timespan(kwork, sample); return true; @@ -2070,6 +2108,9 @@ static void top_merge_tasks(struct perf_kwork *kwork) rb_erase_cached(node, &class->work_root); data = rb_entry(node, struct kwork_work, node); + if (!profile_name_match(kwork, data)) + continue; + cpu = data->cpu; merged_work = find_work_by_id(&merged_root, data->id, data->id == 0 ? cpu : -1); @@ -2329,6 +2370,16 @@ int cmd_kwork(int argc, const char **argv) OPT_PARENT(kwork_options) }; const struct option top_options[] = { + OPT_STRING('s', "sort", &kwork.sort_order, "key[,key2...]", + "sort by key(s): rate, runtime, tid"), + OPT_STRING('C', "cpu", &kwork.cpu_list, "cpu", + "list of cpus to profile"), + OPT_STRING('n', "name", &kwork.profile_name, "name", + "event name to profile"), + OPT_STRING(0, "time", &kwork.time_str, "str", + "Time span for analysis (start,stop)"), + OPT_STRING('i', "input", &input_name, "file", + "input file name"), OPT_PARENT(kwork_options) }; const char *kwork_usage[] = { From 8c98420987cda501924b7a34f2413b982517d244 Mon Sep 17 00:00:00 2001 From: Yang Jihong Date: Sat, 12 Aug 2023 08:49:15 +0000 Subject: [PATCH 056/475] perf kwork top: Implements BPF-based cpu usage statistics Use BPF to collect statistics on the CPU usage based on perf BPF skeletons. Example usage: # perf kwork top -h Usage: perf kwork top [] -b, --use-bpf Use BPF to measure task cpu usage -C, --cpu list of cpus to profile -i, --input input file name -n, --name event name to profile -s, --sort sort by key(s): rate, runtime, tid --time Time span for analysis (start,stop) # # perf kwork -k sched top -b Starting trace, Hit to stop and report ^C Total : 160702.425 ms, 8 cpus %Cpu(s): 36.00% id, 0.00% hi, 0.00% si %Cpu0 [|||||||||||||||||| 61.66%] %Cpu1 [|||||||||||||||||| 61.27%] %Cpu2 [||||||||||||||||||| 66.40%] %Cpu3 [|||||||||||||||||| 61.28%] %Cpu4 [|||||||||||||||||| 61.82%] %Cpu5 [||||||||||||||||||||||| 77.41%] %Cpu6 [|||||||||||||||||| 61.73%] %Cpu7 [|||||||||||||||||| 63.25%] PID SPID %CPU RUNTIME COMMMAND ------------------------------------------------------------- 0 0 38.72 8089.463 ms [swapper/1] 0 0 38.71 8084.547 ms [swapper/3] 0 0 38.33 8007.532 ms [swapper/0] 0 0 38.26 7992.985 ms [swapper/6] 0 0 38.17 7971.865 ms [swapper/4] 0 0 36.74 7447.765 ms [swapper/7] 0 0 33.59 6486.942 ms [swapper/2] 0 0 22.58 3771.268 ms [swapper/5] 9545 9351 2.48 447.136 ms sched-messaging 9574 9351 2.09 418.583 ms sched-messaging 9724 9351 2.05 372.407 ms sched-messaging 9531 9351 2.01 368.804 ms sched-messaging 9512 9351 2.00 362.250 ms sched-messaging 9514 9351 1.95 357.767 ms sched-messaging 9538 9351 1.86 384.476 ms sched-messaging 9712 9351 1.84 386.490 ms sched-messaging 9723 9351 1.83 380.021 ms sched-messaging 9722 9351 1.82 382.738 ms sched-messaging 9517 9351 1.81 354.794 ms sched-messaging 9559 9351 1.79 344.305 ms sched-messaging 9725 9351 1.77 365.315 ms sched-messaging # perf kwork -k sched top -b -n perf Starting trace, Hit to stop and report ^C Total : 151563.332 ms, 8 cpus %Cpu(s): 26.49% id, 0.00% hi, 0.00% si %Cpu0 [ 0.01%] %Cpu1 [ 0.00%] %Cpu2 [ 0.00%] %Cpu3 [ 0.00%] %Cpu4 [ 0.00%] %Cpu5 [ 0.00%] %Cpu6 [ 0.00%] %Cpu7 [ 0.00%] PID SPID %CPU RUNTIME COMMMAND ------------------------------------------------------------- 9754 9754 0.01 2.303 ms perf # # perf kwork -k sched top -b -C 2,3,4 Starting trace, Hit to stop and report ^C Total : 48016.721 ms, 3 cpus %Cpu(s): 27.82% id, 0.00% hi, 0.00% si %Cpu2 [|||||||||||||||||||||| 74.68%] %Cpu3 [||||||||||||||||||||| 71.06%] %Cpu4 [||||||||||||||||||||| 70.91%] PID SPID %CPU RUNTIME COMMMAND ------------------------------------------------------------- 0 0 29.08 4734.998 ms [swapper/4] 0 0 28.93 4710.029 ms [swapper/3] 0 0 25.31 3912.363 ms [swapper/2] 10248 10158 1.62 264.931 ms sched-messaging 10253 10158 1.62 265.136 ms sched-messaging 10158 10158 1.60 263.013 ms bash 10360 10158 1.49 243.639 ms sched-messaging 10413 10158 1.48 238.604 ms sched-messaging 10531 10158 1.47 234.067 ms sched-messaging 10400 10158 1.47 240.631 ms sched-messaging 10355 10158 1.47 230.586 ms sched-messaging 10377 10158 1.43 234.835 ms sched-messaging 10526 10158 1.42 232.045 ms sched-messaging 10298 10158 1.41 222.396 ms sched-messaging 10410 10158 1.38 221.853 ms sched-messaging 10364 10158 1.38 226.042 ms sched-messaging 10480 10158 1.36 213.633 ms sched-messaging 10370 10158 1.36 223.620 ms sched-messaging 10553 10158 1.34 217.169 ms sched-messaging 10291 10158 1.34 211.516 ms sched-messaging 10251 10158 1.34 218.813 ms sched-messaging 10522 10158 1.33 218.498 ms sched-messaging 10288 10158 1.33 216.787 ms sched-messaging Reviewed-by: Ian Rogers Signed-off-by: Yang Jihong Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ingo Molnar Cc: Jiri Olsa Cc: Kan Liang Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Ravi Bangoria Cc: Sandipan Das Link: https://lore.kernel.org/r/20230812084917.169338-15-yangjihong1@huawei.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/Documentation/perf-kwork.txt | 5 + tools/perf/Makefile.perf | 1 + tools/perf/builtin-kwork.c | 62 ++++- tools/perf/util/Build | 1 + tools/perf/util/bpf_kwork_top.c | 286 +++++++++++++++++++++++ tools/perf/util/bpf_skel/kwork_top.bpf.c | 187 +++++++++++++++ tools/perf/util/kwork.h | 26 +++ 7 files changed, 565 insertions(+), 3 deletions(-) create mode 100644 tools/perf/util/bpf_kwork_top.c create mode 100644 tools/perf/util/bpf_skel/kwork_top.bpf.c diff --git a/tools/perf/Documentation/perf-kwork.txt b/tools/perf/Documentation/perf-kwork.txt index 34d6c285e527..109ace1d5e90 100644 --- a/tools/perf/Documentation/perf-kwork.txt +++ b/tools/perf/Documentation/perf-kwork.txt @@ -33,6 +33,7 @@ There are several variants of 'perf kwork': perf kwork latency -b perf kwork timehist perf kwork top + perf kwork top -b By default it shows the individual work events such as irq, workqeueu, including the run time and delay (time between raise and actually entry): @@ -181,6 +182,10 @@ OPTIONS for 'perf kwork timehist' OPTIONS for 'perf kwork top' --------------------------------- +-b:: +--use-bpf:: + Use BPF to measure task cpu usage. + -C:: --cpu:: Only show events for the given CPU(s) (comma separated list). diff --git a/tools/perf/Makefile.perf b/tools/perf/Makefile.perf index 37af6df7b978..cb0422debdb9 100644 --- a/tools/perf/Makefile.perf +++ b/tools/perf/Makefile.perf @@ -1032,6 +1032,7 @@ SKELETONS += $(SKEL_OUT)/bperf_leader.skel.h $(SKEL_OUT)/bperf_follower.skel.h SKELETONS += $(SKEL_OUT)/bperf_cgroup.skel.h $(SKEL_OUT)/func_latency.skel.h SKELETONS += $(SKEL_OUT)/off_cpu.skel.h $(SKEL_OUT)/lock_contention.skel.h SKELETONS += $(SKEL_OUT)/kwork_trace.skel.h $(SKEL_OUT)/sample_filter.skel.h +SKELETONS += $(SKEL_OUT)/kwork_top.skel.h SKELETONS += $(SKEL_OUT)/bench_uprobe.skel.h SKELETONS += $(SKEL_OUT)/augmented_raw_syscalls.skel.h diff --git a/tools/perf/builtin-kwork.c b/tools/perf/builtin-kwork.c index d5949ff4bd15..04b966801643 100644 --- a/tools/perf/builtin-kwork.c +++ b/tools/perf/builtin-kwork.c @@ -1619,8 +1619,14 @@ static void top_print_header(struct perf_kwork *kwork __maybe_unused) int ret; printf("\n "); - ret = printf(" %*s %*s %*s %-*s", + ret = printf(" %*s %s%*s%s %*s %*s %-*s", PRINT_PID_WIDTH, "PID", + + kwork->use_bpf ? " " : "", + kwork->use_bpf ? PRINT_PID_WIDTH : 0, + kwork->use_bpf ? "SPID" : "", + kwork->use_bpf ? " " : "", + PRINT_CPU_USAGE_WIDTH, "%CPU", PRINT_RUNTIME_HEADER_WIDTH + RPINT_DECIMAL_WIDTH, "RUNTIME", PRINT_TASK_NAME_WIDTH, "COMMMAND"); @@ -1639,6 +1645,12 @@ static int top_print_work(struct perf_kwork *kwork __maybe_unused, struct kwork_ */ ret += printf(" %*ld ", PRINT_PID_WIDTH, work->id); + /* + * tgid + */ + if (kwork->use_bpf) + ret += printf(" %*d ", PRINT_PID_WIDTH, work->tgid); + /* * cpu usage */ @@ -1656,7 +1668,13 @@ static int top_print_work(struct perf_kwork *kwork __maybe_unused, struct kwork_ /* * command */ - ret += printf(" %-*s", PRINT_TASK_NAME_WIDTH, work->name); + if (kwork->use_bpf) + ret += printf(" %s%s%s", + work->is_kthread ? "[" : "", + work->name, + work->is_kthread ? "]" : ""); + else + ret += printf(" %-*s", PRINT_TASK_NAME_WIDTH, work->name); printf("\n"); return ret; @@ -2153,6 +2171,36 @@ next: printf("\n"); } +static int perf_kwork__top_bpf(struct perf_kwork *kwork) +{ + int ret; + + signal(SIGINT, sig_handler); + signal(SIGTERM, sig_handler); + + ret = perf_kwork__top_prepare_bpf(kwork); + if (ret) + return -1; + + printf("Starting trace, Hit to stop and report\n"); + + perf_kwork__top_start(); + + /* + * a simple pause, wait here for stop signal + */ + pause(); + + perf_kwork__top_finish(); + + perf_kwork__top_read_bpf(kwork); + + perf_kwork__top_cleanup_bpf(); + + return 0; + +} + static int perf_kwork__top(struct perf_kwork *kwork) { struct __top_cpus_runtime *cpus_runtime; @@ -2165,7 +2213,11 @@ static int perf_kwork__top(struct perf_kwork *kwork) kwork->top_stat.cpus_runtime = cpus_runtime; bitmap_zero(kwork->top_stat.all_cpus_bitmap, MAX_NR_CPUS); - ret = perf_kwork__read_events(kwork); + if (kwork->use_bpf) + ret = perf_kwork__top_bpf(kwork); + else + ret = perf_kwork__read_events(kwork); + if (ret) goto out; @@ -2380,6 +2432,10 @@ int cmd_kwork(int argc, const char **argv) "Time span for analysis (start,stop)"), OPT_STRING('i', "input", &input_name, "file", "input file name"), +#ifdef HAVE_BPF_SKEL + OPT_BOOLEAN('b', "use-bpf", &kwork.use_bpf, + "Use BPF to measure task cpu usage"), +#endif OPT_PARENT(kwork_options) }; const char *kwork_usage[] = { diff --git a/tools/perf/util/Build b/tools/perf/util/Build index 6d657c9927f7..0ea5a9d368d4 100644 --- a/tools/perf/util/Build +++ b/tools/perf/util/Build @@ -165,6 +165,7 @@ endif ifeq ($(CONFIG_LIBTRACEEVENT),y) perf-$(CONFIG_PERF_BPF_SKEL) += bpf_kwork.o + perf-$(CONFIG_PERF_BPF_SKEL) += bpf_kwork_top.o endif perf-$(CONFIG_LIBELF) += symbol-elf.o diff --git a/tools/perf/util/bpf_kwork_top.c b/tools/perf/util/bpf_kwork_top.c new file mode 100644 index 000000000000..42897ea22c61 --- /dev/null +++ b/tools/perf/util/bpf_kwork_top.c @@ -0,0 +1,286 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * bpf_kwork_top.c + * + * Copyright (c) 2022 Huawei Inc, Yang Jihong + */ + +#include +#include +#include +#include +#include + +#include + +#include "util/debug.h" +#include "util/evsel.h" +#include "util/kwork.h" + +#include +#include + +#include "util/bpf_skel/kwork_top.skel.h" + +/* + * This should be in sync with "util/kwork_top.bpf.c" + */ +#define MAX_COMMAND_LEN 16 + +struct time_data { + __u64 timestamp; +}; + +struct work_data { + __u64 runtime; +}; + +struct task_data { + __u32 tgid; + __u32 is_kthread; + char comm[MAX_COMMAND_LEN]; +}; + +struct work_key { + __u32 type; + __u32 pid; + __u64 task_p; +}; + +struct task_key { + __u32 pid; + __u32 cpu; +}; + +struct kwork_class_bpf { + struct kwork_class *class; + void (*load_prepare)(void); +}; + +static struct kwork_top_bpf *skel; + +void perf_kwork__top_start(void) +{ + struct timespec ts; + + clock_gettime(CLOCK_MONOTONIC, &ts); + skel->bss->from_timestamp = (u64)ts.tv_sec * NSEC_PER_SEC + ts.tv_nsec; + skel->bss->enabled = 1; + pr_debug("perf kwork top start at: %lld\n", skel->bss->from_timestamp); +} + +void perf_kwork__top_finish(void) +{ + struct timespec ts; + + skel->bss->enabled = 0; + clock_gettime(CLOCK_MONOTONIC, &ts); + skel->bss->to_timestamp = (u64)ts.tv_sec * NSEC_PER_SEC + ts.tv_nsec; + pr_debug("perf kwork top finish at: %lld\n", skel->bss->to_timestamp); +} + +static void sched_load_prepare(void) +{ + bpf_program__set_autoload(skel->progs.on_switch, true); +} + +static struct kwork_class_bpf kwork_sched_bpf = { + .load_prepare = sched_load_prepare, +}; + +static struct kwork_class_bpf * +kwork_class_bpf_supported_list[KWORK_CLASS_MAX] = { + [KWORK_CLASS_SCHED] = &kwork_sched_bpf, +}; + +static bool valid_kwork_class_type(enum kwork_class_type type) +{ + return type >= 0 && type < KWORK_CLASS_MAX ? true : false; +} + +static int setup_filters(struct perf_kwork *kwork) +{ + u8 val = 1; + int i, nr_cpus, fd; + struct perf_cpu_map *map; + + if (kwork->cpu_list) { + fd = bpf_map__fd(skel->maps.kwork_top_cpu_filter); + if (fd < 0) { + pr_debug("Invalid cpu filter fd\n"); + return -1; + } + + map = perf_cpu_map__new(kwork->cpu_list); + if (!map) { + pr_debug("Invalid cpu_list\n"); + return -1; + } + + nr_cpus = libbpf_num_possible_cpus(); + for (i = 0; i < perf_cpu_map__nr(map); i++) { + struct perf_cpu cpu = perf_cpu_map__cpu(map, i); + + if (cpu.cpu >= nr_cpus) { + perf_cpu_map__put(map); + pr_err("Requested cpu %d too large\n", cpu.cpu); + return -1; + } + bpf_map_update_elem(fd, &cpu.cpu, &val, BPF_ANY); + } + perf_cpu_map__put(map); + + skel->bss->has_cpu_filter = 1; + } + + return 0; +} + +int perf_kwork__top_prepare_bpf(struct perf_kwork *kwork __maybe_unused) +{ + struct bpf_program *prog; + struct kwork_class *class; + struct kwork_class_bpf *class_bpf; + enum kwork_class_type type; + + skel = kwork_top_bpf__open(); + if (!skel) { + pr_debug("Failed to open kwork top skeleton\n"); + return -1; + } + + /* + * set all progs to non-autoload, + * then set corresponding progs according to config + */ + bpf_object__for_each_program(prog, skel->obj) + bpf_program__set_autoload(prog, false); + + list_for_each_entry(class, &kwork->class_list, list) { + type = class->type; + if (!valid_kwork_class_type(type) || + !kwork_class_bpf_supported_list[type]) { + pr_err("Unsupported bpf trace class %s\n", class->name); + goto out; + } + + class_bpf = kwork_class_bpf_supported_list[type]; + class_bpf->class = class; + + if (class_bpf->load_prepare) + class_bpf->load_prepare(); + } + + if (kwork_top_bpf__load(skel)) { + pr_debug("Failed to load kwork top skeleton\n"); + goto out; + } + + if (setup_filters(kwork)) + goto out; + + if (kwork_top_bpf__attach(skel)) { + pr_debug("Failed to attach kwork top skeleton\n"); + goto out; + } + + return 0; + +out: + kwork_top_bpf__destroy(skel); + return -1; +} + +static void read_task_info(struct kwork_work *work) +{ + int fd; + struct task_data data; + struct task_key key = { + .pid = work->id, + .cpu = work->cpu, + }; + + fd = bpf_map__fd(skel->maps.kwork_top_tasks); + if (fd < 0) { + pr_debug("Invalid top tasks map fd\n"); + return; + } + + if (!bpf_map_lookup_elem(fd, &key, &data)) { + work->tgid = data.tgid; + work->is_kthread = data.is_kthread; + work->name = strdup(data.comm); + } +} +static int add_work(struct perf_kwork *kwork, struct work_key *key, + struct work_data *data, int cpu) +{ + struct kwork_class_bpf *bpf_trace; + struct kwork_work *work; + struct kwork_work tmp = { + .id = key->pid, + .cpu = cpu, + .name = NULL, + }; + enum kwork_class_type type = key->type; + + if (!valid_kwork_class_type(type)) { + pr_debug("Invalid class type %d to add work\n", type); + return -1; + } + + bpf_trace = kwork_class_bpf_supported_list[type]; + tmp.class = bpf_trace->class; + + work = perf_kwork_add_work(kwork, tmp.class, &tmp); + if (!work) + return -1; + + work->total_runtime = data->runtime; + read_task_info(work); + + return 0; +} + +int perf_kwork__top_read_bpf(struct perf_kwork *kwork) +{ + int i, fd, nr_cpus; + struct work_data *data; + struct work_key key, prev; + + fd = bpf_map__fd(skel->maps.kwork_top_works); + if (fd < 0) { + pr_debug("Invalid top runtime fd\n"); + return -1; + } + + nr_cpus = libbpf_num_possible_cpus(); + data = calloc(nr_cpus, sizeof(struct work_data)); + if (!data) + return -1; + + memset(&prev, 0, sizeof(prev)); + while (!bpf_map_get_next_key(fd, &prev, &key)) { + if ((bpf_map_lookup_elem(fd, &key, data)) != 0) { + pr_debug("Failed to lookup top elem\n"); + return -1; + } + + for (i = 0; i < nr_cpus; i++) { + if (data[i].runtime == 0) + continue; + + if (add_work(kwork, &key, &data[i], i)) + return -1; + } + prev = key; + } + free(data); + + return 0; +} + +void perf_kwork__top_cleanup_bpf(void) +{ + kwork_top_bpf__destroy(skel); +} diff --git a/tools/perf/util/bpf_skel/kwork_top.bpf.c b/tools/perf/util/bpf_skel/kwork_top.bpf.c new file mode 100644 index 000000000000..47ad61608ec7 --- /dev/null +++ b/tools/perf/util/bpf_skel/kwork_top.bpf.c @@ -0,0 +1,187 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +// Copyright (c) 2022, Huawei + +#include "vmlinux.h" +#include +#include +#include + +/* + * This should be in sync with "util/kwork.h" + */ +enum kwork_class_type { + KWORK_CLASS_IRQ, + KWORK_CLASS_SOFTIRQ, + KWORK_CLASS_WORKQUEUE, + KWORK_CLASS_SCHED, + KWORK_CLASS_MAX, +}; + +#define MAX_ENTRIES 102400 +#define MAX_NR_CPUS 2048 +#define PF_KTHREAD 0x00200000 +#define MAX_COMMAND_LEN 16 + +struct time_data { + __u64 timestamp; +}; + +struct work_data { + __u64 runtime; +}; + +struct task_data { + __u32 tgid; + __u32 is_kthread; + char comm[MAX_COMMAND_LEN]; +}; + +struct work_key { + __u32 type; + __u32 pid; + __u64 task_p; +}; + +struct task_key { + __u32 pid; + __u32 cpu; +}; + +struct { + __uint(type, BPF_MAP_TYPE_TASK_STORAGE); + __uint(map_flags, BPF_F_NO_PREALLOC); + __type(key, int); + __type(value, struct time_data); +} kwork_top_task_time SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(struct task_key)); + __uint(value_size, sizeof(struct task_data)); + __uint(max_entries, MAX_ENTRIES); +} kwork_top_tasks SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_HASH); + __uint(key_size, sizeof(struct work_key)); + __uint(value_size, sizeof(struct work_data)); + __uint(max_entries, MAX_ENTRIES); +} kwork_top_works SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(u32)); + __uint(value_size, sizeof(u8)); + __uint(max_entries, MAX_NR_CPUS); +} kwork_top_cpu_filter SEC(".maps"); + +int enabled = 0; + +int has_cpu_filter = 0; + +__u64 from_timestamp = 0; +__u64 to_timestamp = 0; + +static __always_inline int cpu_is_filtered(__u32 cpu) +{ + __u8 *cpu_val; + + if (has_cpu_filter) { + cpu_val = bpf_map_lookup_elem(&kwork_top_cpu_filter, &cpu); + if (!cpu_val) + return 1; + } + + return 0; +} + +static __always_inline void update_task_info(struct task_struct *task, __u32 cpu) +{ + struct task_key key = { + .pid = task->pid, + .cpu = cpu, + }; + + if (!bpf_map_lookup_elem(&kwork_top_tasks, &key)) { + struct task_data data = { + .tgid = task->tgid, + .is_kthread = task->flags & PF_KTHREAD ? 1 : 0, + }; + BPF_CORE_READ_STR_INTO(&data.comm, task, comm); + + bpf_map_update_elem(&kwork_top_tasks, &key, &data, BPF_ANY); + } +} + +static __always_inline void update_work(struct work_key *key, __u64 delta) +{ + struct work_data *data; + + data = bpf_map_lookup_elem(&kwork_top_works, key); + if (data) { + data->runtime += delta; + } else { + struct work_data new_data = { + .runtime = delta, + }; + + bpf_map_update_elem(&kwork_top_works, key, &new_data, BPF_ANY); + } +} + +static void on_sched_out(struct task_struct *task, __u64 ts, __u32 cpu) +{ + __u64 delta; + struct time_data *pelem; + + pelem = bpf_task_storage_get(&kwork_top_task_time, task, NULL, 0); + if (pelem) + delta = ts - pelem->timestamp; + else + delta = ts - from_timestamp; + + struct work_key key = { + .type = KWORK_CLASS_SCHED, + .pid = task->pid, + .task_p = (__u64)task, + }; + + update_work(&key, delta); + update_task_info(task, cpu); +} + +static void on_sched_in(struct task_struct *task, __u64 ts) +{ + struct time_data *pelem; + + pelem = bpf_task_storage_get(&kwork_top_task_time, task, NULL, + BPF_LOCAL_STORAGE_GET_F_CREATE); + if (pelem) + pelem->timestamp = ts; +} + +SEC("tp_btf/sched_switch") +int on_switch(u64 *ctx) +{ + struct task_struct *prev, *next; + + prev = (struct task_struct *)ctx[1]; + next = (struct task_struct *)ctx[2]; + + if (!enabled) + return 0; + + __u32 cpu = bpf_get_smp_processor_id(); + + if (cpu_is_filtered(cpu)) + return 0; + + __u64 ts = bpf_ktime_get_ns(); + + on_sched_out(prev, ts, cpu); + on_sched_in(next, ts); + + return 0; +} + +char LICENSE[] SEC("license") = "Dual BSD/GPL"; diff --git a/tools/perf/util/kwork.h b/tools/perf/util/kwork.h index 723b34385df6..76fe2a821bcf 100644 --- a/tools/perf/util/kwork.h +++ b/tools/perf/util/kwork.h @@ -135,6 +135,8 @@ struct kwork_work { * top report */ u32 cpu_usage; + u32 tgid; + bool is_kthread; }; struct kwork_class { @@ -264,6 +266,13 @@ void perf_kwork__report_cleanup_bpf(void); void perf_kwork__trace_start(void); void perf_kwork__trace_finish(void); +int perf_kwork__top_prepare_bpf(struct perf_kwork *kwork); +int perf_kwork__top_read_bpf(struct perf_kwork *kwork); +void perf_kwork__top_cleanup_bpf(void); + +void perf_kwork__top_start(void); +void perf_kwork__top_finish(void); + #else /* !HAVE_BPF_SKEL */ static inline int @@ -283,6 +292,23 @@ static inline void perf_kwork__report_cleanup_bpf(void) {} static inline void perf_kwork__trace_start(void) {} static inline void perf_kwork__trace_finish(void) {} +static inline int +perf_kwork__top_prepare_bpf(struct perf_kwork *kwork __maybe_unused) +{ + return -1; +} + +static inline int +perf_kwork__top_read_bpf(struct perf_kwork *kwork __maybe_unused) +{ + return -1; +} + +static inline void perf_kwork__top_cleanup_bpf(void) {} + +static inline void perf_kwork__top_start(void) {} +static inline void perf_kwork__top_finish(void) {} + #endif /* HAVE_BPF_SKEL */ #endif /* PERF_UTIL_KWORK_H */ From d2956b3acf869336e30d903dcaf23645c82d06f6 Mon Sep 17 00:00:00 2001 From: Yang Jihong Date: Sat, 12 Aug 2023 08:49:16 +0000 Subject: [PATCH 057/475] perf kwork top: Add BPF-based statistics on hardirq event support Use BPF to collect statistics on hardirq events based on perf BPF skeletons. Example usage: # perf kwork top -k sched,irq -b Starting trace, Hit to stop and report ^C Total : 136717.945 ms, 8 cpus %Cpu(s): 17.10% id, 0.01% hi, 0.00% si %Cpu0 [||||||||||||||||||||||||| 84.26%] %Cpu1 [||||||||||||||||||||||||| 84.77%] %Cpu2 [|||||||||||||||||||||||| 83.22%] %Cpu3 [|||||||||||||||||||||||| 80.37%] %Cpu4 [|||||||||||||||||||||||| 81.49%] %Cpu5 [||||||||||||||||||||||||| 84.68%] %Cpu6 [||||||||||||||||||||||||| 84.48%] %Cpu7 [|||||||||||||||||||||||| 80.21%] PID SPID %CPU RUNTIME COMMMAND ------------------------------------------------------------- 0 0 19.78 3482.833 ms [swapper/7] 0 0 19.62 3454.219 ms [swapper/3] 0 0 18.50 3258.339 ms [swapper/4] 0 0 16.76 2842.749 ms [swapper/2] 0 0 15.71 2627.905 ms [swapper/0] 0 0 15.51 2598.206 ms [swapper/6] 0 0 15.31 2561.820 ms [swapper/5] 0 0 15.22 2548.708 ms [swapper/1] 13253 13018 2.95 513.108 ms sched-messaging 13092 13018 2.67 454.167 ms sched-messaging 13401 13018 2.66 454.790 ms sched-messaging 13240 13018 2.64 454.587 ms sched-messaging 13251 13018 2.61 442.273 ms sched-messaging 13075 13018 2.61 438.932 ms sched-messaging 13220 13018 2.60 443.245 ms sched-messaging 13235 13018 2.59 443.268 ms sched-messaging 13222 13018 2.50 426.344 ms sched-messaging 13410 13018 2.49 426.191 ms sched-messaging 13228 13018 2.46 425.121 ms sched-messaging 13379 13018 2.38 409.950 ms sched-messaging 13236 13018 2.37 413.159 ms sched-messaging 13095 13018 2.36 396.572 ms sched-messaging 13325 13018 2.35 408.089 ms sched-messaging 13242 13018 2.32 394.750 ms sched-messaging 13386 13018 2.31 396.997 ms sched-messaging 13046 13018 2.29 383.833 ms sched-messaging 13109 13018 2.28 388.482 ms sched-messaging 13388 13018 2.28 393.576 ms sched-messaging 13238 13018 2.26 388.487 ms sched-messaging Reviewed-by: Ian Rogers Signed-off-by: Yang Jihong Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ingo Molnar Cc: Jiri Olsa Cc: Kan Liang Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Ravi Bangoria Cc: Sandipan Das Link: https://lore.kernel.org/r/20230812084917.169338-16-yangjihong1@huawei.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/util/bpf_kwork_top.c | 11 ++++ tools/perf/util/bpf_skel/kwork_top.bpf.c | 79 ++++++++++++++++++++++++ 2 files changed, 90 insertions(+) diff --git a/tools/perf/util/bpf_kwork_top.c b/tools/perf/util/bpf_kwork_top.c index 42897ea22c61..3998bd2a938f 100644 --- a/tools/perf/util/bpf_kwork_top.c +++ b/tools/perf/util/bpf_kwork_top.c @@ -79,6 +79,16 @@ void perf_kwork__top_finish(void) pr_debug("perf kwork top finish at: %lld\n", skel->bss->to_timestamp); } +static void irq_load_prepare(void) +{ + bpf_program__set_autoload(skel->progs.on_irq_handler_entry, true); + bpf_program__set_autoload(skel->progs.on_irq_handler_exit, true); +} + +static struct kwork_class_bpf kwork_irq_bpf = { + .load_prepare = irq_load_prepare, +}; + static void sched_load_prepare(void) { bpf_program__set_autoload(skel->progs.on_switch, true); @@ -90,6 +100,7 @@ static struct kwork_class_bpf kwork_sched_bpf = { static struct kwork_class_bpf * kwork_class_bpf_supported_list[KWORK_CLASS_MAX] = { + [KWORK_CLASS_IRQ] = &kwork_irq_bpf, [KWORK_CLASS_SCHED] = &kwork_sched_bpf, }; diff --git a/tools/perf/util/bpf_skel/kwork_top.bpf.c b/tools/perf/util/bpf_skel/kwork_top.bpf.c index 47ad61608ec7..9c7dc62386c7 100644 --- a/tools/perf/util/bpf_skel/kwork_top.bpf.c +++ b/tools/perf/util/bpf_skel/kwork_top.bpf.c @@ -54,6 +54,13 @@ struct { __type(value, struct time_data); } kwork_top_task_time SEC(".maps"); +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_HASH); + __uint(key_size, sizeof(struct work_key)); + __uint(value_size, sizeof(struct time_data)); + __uint(max_entries, MAX_ENTRIES); +} kwork_top_irq_time SEC(".maps"); + struct { __uint(type, BPF_MAP_TYPE_HASH); __uint(key_size, sizeof(struct task_key)); @@ -184,4 +191,76 @@ int on_switch(u64 *ctx) return 0; } +SEC("tp_btf/irq_handler_entry") +int on_irq_handler_entry(u64 *cxt) +{ + struct task_struct *task; + + if (!enabled) + return 0; + + __u32 cpu = bpf_get_smp_processor_id(); + + if (cpu_is_filtered(cpu)) + return 0; + + __u64 ts = bpf_ktime_get_ns(); + + task = (struct task_struct *)bpf_get_current_task(); + if (!task) + return 0; + + struct work_key key = { + .type = KWORK_CLASS_IRQ, + .pid = BPF_CORE_READ(task, pid), + .task_p = (__u64)task, + }; + + struct time_data data = { + .timestamp = ts, + }; + + bpf_map_update_elem(&kwork_top_irq_time, &key, &data, BPF_ANY); + + return 0; +} + +SEC("tp_btf/irq_handler_exit") +int on_irq_handler_exit(u64 *cxt) +{ + __u64 delta; + struct task_struct *task; + struct time_data *pelem; + + if (!enabled) + return 0; + + __u32 cpu = bpf_get_smp_processor_id(); + + if (cpu_is_filtered(cpu)) + return 0; + + __u64 ts = bpf_ktime_get_ns(); + + task = (struct task_struct *)bpf_get_current_task(); + if (!task) + return 0; + + struct work_key key = { + .type = KWORK_CLASS_IRQ, + .pid = BPF_CORE_READ(task, pid), + .task_p = (__u64)task, + }; + + pelem = bpf_map_lookup_elem(&kwork_top_irq_time, &key); + if (pelem && pelem->timestamp != 0) + delta = ts - pelem->timestamp; + else + delta = ts - from_timestamp; + + update_work(&key, delta); + + return 0; +} + char LICENSE[] SEC("license") = "Dual BSD/GPL"; From 36019dff30f7aacb8a26b2b9564fe04b097d502e Mon Sep 17 00:00:00 2001 From: Yang Jihong Date: Sat, 12 Aug 2023 08:49:17 +0000 Subject: [PATCH 058/475] perf kwork top: Add BPF-based statistics on softirq event support Use BPF to collect statistics on softirq events based on perf BPF skeletons. Example usage: # perf kwork top -b Starting trace, Hit to stop and report ^C Total : 135445.704 ms, 8 cpus %Cpu(s): 28.35% id, 0.00% hi, 0.25% si %Cpu0 [|||||||||||||||||||| 69.85%] %Cpu1 [|||||||||||||||||||||| 74.10%] %Cpu2 [||||||||||||||||||||| 71.18%] %Cpu3 [|||||||||||||||||||| 69.61%] %Cpu4 [|||||||||||||||||||||| 74.05%] %Cpu5 [|||||||||||||||||||| 69.33%] %Cpu6 [|||||||||||||||||||| 69.71%] %Cpu7 [|||||||||||||||||||||| 73.77%] PID SPID %CPU RUNTIME COMMMAND ------------------------------------------------------------- 0 0 30.43 5271.005 ms [swapper/5] 0 0 30.17 5226.644 ms [swapper/3] 0 0 30.08 5210.257 ms [swapper/6] 0 0 29.89 5177.177 ms [swapper/0] 0 0 28.51 4938.672 ms [swapper/2] 0 0 25.93 4223.464 ms [swapper/7] 0 0 25.69 4181.411 ms [swapper/4] 0 0 25.63 4173.804 ms [swapper/1] 16665 16265 2.16 360.600 ms sched-messaging 16537 16265 2.05 356.275 ms sched-messaging 16503 16265 2.01 343.063 ms sched-messaging 16424 16265 1.97 336.876 ms sched-messaging 16580 16265 1.94 323.658 ms sched-messaging 16515 16265 1.92 321.616 ms sched-messaging 16659 16265 1.91 325.538 ms sched-messaging 16634 16265 1.88 327.766 ms sched-messaging 16454 16265 1.87 326.843 ms sched-messaging 16382 16265 1.87 322.591 ms sched-messaging 16642 16265 1.86 320.506 ms sched-messaging 16582 16265 1.86 320.164 ms sched-messaging 16315 16265 1.86 326.872 ms sched-messaging 16637 16265 1.85 323.766 ms sched-messaging 16506 16265 1.82 311.688 ms sched-messaging 16512 16265 1.81 304.643 ms sched-messaging 16560 16265 1.80 314.751 ms sched-messaging 16320 16265 1.80 313.405 ms sched-messaging 16442 16265 1.80 314.403 ms sched-messaging 16626 16265 1.78 295.380 ms sched-messaging 16600 16265 1.77 309.444 ms sched-messaging 16550 16265 1.76 301.161 ms sched-messaging 16525 16265 1.75 296.560 ms sched-messaging 16314 16265 1.75 298.338 ms sched-messaging 16595 16265 1.74 304.390 ms sched-messaging 16555 16265 1.74 287.564 ms sched-messaging 16520 16265 1.74 295.734 ms sched-messaging 16507 16265 1.73 293.956 ms sched-messaging 16593 16265 1.72 296.443 ms sched-messaging 16531 16265 1.72 299.950 ms sched-messaging 16281 16265 1.72 301.339 ms sched-messaging Reviewed-by: Ian Rogers Signed-off-by: Yang Jihong Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ingo Molnar Cc: Jiri Olsa Cc: Kan Liang Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Ravi Bangoria Cc: Sandipan Das Link: https://lore.kernel.org/r/20230812084917.169338-17-yangjihong1@huawei.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/util/bpf_kwork_top.c | 11 ++++ tools/perf/util/bpf_skel/kwork_top.bpf.c | 72 ++++++++++++++++++++++++ 2 files changed, 83 insertions(+) diff --git a/tools/perf/util/bpf_kwork_top.c b/tools/perf/util/bpf_kwork_top.c index 3998bd2a938f..1a607b94f44d 100644 --- a/tools/perf/util/bpf_kwork_top.c +++ b/tools/perf/util/bpf_kwork_top.c @@ -89,6 +89,16 @@ static struct kwork_class_bpf kwork_irq_bpf = { .load_prepare = irq_load_prepare, }; +static void softirq_load_prepare(void) +{ + bpf_program__set_autoload(skel->progs.on_softirq_entry, true); + bpf_program__set_autoload(skel->progs.on_softirq_exit, true); +} + +static struct kwork_class_bpf kwork_softirq_bpf = { + .load_prepare = softirq_load_prepare, +}; + static void sched_load_prepare(void) { bpf_program__set_autoload(skel->progs.on_switch, true); @@ -101,6 +111,7 @@ static struct kwork_class_bpf kwork_sched_bpf = { static struct kwork_class_bpf * kwork_class_bpf_supported_list[KWORK_CLASS_MAX] = { [KWORK_CLASS_IRQ] = &kwork_irq_bpf, + [KWORK_CLASS_SOFTIRQ] = &kwork_softirq_bpf, [KWORK_CLASS_SCHED] = &kwork_sched_bpf, }; diff --git a/tools/perf/util/bpf_skel/kwork_top.bpf.c b/tools/perf/util/bpf_skel/kwork_top.bpf.c index 9c7dc62386c7..84c15ccbab44 100644 --- a/tools/perf/util/bpf_skel/kwork_top.bpf.c +++ b/tools/perf/util/bpf_skel/kwork_top.bpf.c @@ -263,4 +263,76 @@ int on_irq_handler_exit(u64 *cxt) return 0; } +SEC("tp_btf/softirq_entry") +int on_softirq_entry(u64 *cxt) +{ + struct task_struct *task; + + if (!enabled) + return 0; + + __u32 cpu = bpf_get_smp_processor_id(); + + if (cpu_is_filtered(cpu)) + return 0; + + __u64 ts = bpf_ktime_get_ns(); + + task = (struct task_struct *)bpf_get_current_task(); + if (!task) + return 0; + + struct work_key key = { + .type = KWORK_CLASS_SOFTIRQ, + .pid = BPF_CORE_READ(task, pid), + .task_p = (__u64)task, + }; + + struct time_data data = { + .timestamp = ts, + }; + + bpf_map_update_elem(&kwork_top_irq_time, &key, &data, BPF_ANY); + + return 0; +} + +SEC("tp_btf/softirq_exit") +int on_softirq_exit(u64 *cxt) +{ + __u64 delta; + struct task_struct *task; + struct time_data *pelem; + + if (!enabled) + return 0; + + __u32 cpu = bpf_get_smp_processor_id(); + + if (cpu_is_filtered(cpu)) + return 0; + + __u64 ts = bpf_ktime_get_ns(); + + task = (struct task_struct *)bpf_get_current_task(); + if (!task) + return 0; + + struct work_key key = { + .type = KWORK_CLASS_SOFTIRQ, + .pid = BPF_CORE_READ(task, pid), + .task_p = (__u64)task, + }; + + pelem = bpf_map_lookup_elem(&kwork_top_irq_time, &key); + if (pelem) + delta = ts - pelem->timestamp; + else + delta = ts - from_timestamp; + + update_work(&key, delta); + + return 0; +} + char LICENSE[] SEC("license") = "Dual BSD/GPL"; From 2bc12abce8efb9f755bdbc14d426a0fc80ce74f4 Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Wed, 6 Sep 2023 10:48:59 -0700 Subject: [PATCH 059/475] perf tools: Add read_all_cgroups() and __cgroup_find() The read_all_cgroups() is to build a tree of cgroups in the system and users can look up a cgroup using __cgroup_find(). Committer notes: Had to do this to cover that #else block: -static inline u64 __read_cgroup_id(const char *path) { return -1ULL; } +static inline u64 __read_cgroup_id(const char *path __maybe_unused) { return -1ULL; } Reviewed-by: Ian Rogers Signed-off-by: Namhyung Kim Cc: Adrian Hunter Cc: Hao Luo Cc: Ingo Molnar Cc: Jiri Olsa Cc: Peter Zijlstra Cc: Song Liu Cc: bpf@vger.kernel.org Link: https://lore.kernel.org/r/20230906174903.346486-2-namhyung@kernel.org Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/util/cgroup.c | 61 ++++++++++++++++++++++++++++++++++------ tools/perf/util/cgroup.h | 4 +++ 2 files changed, 57 insertions(+), 8 deletions(-) diff --git a/tools/perf/util/cgroup.c b/tools/perf/util/cgroup.c index bfb13306d82c..d03651098dde 100644 --- a/tools/perf/util/cgroup.c +++ b/tools/perf/util/cgroup.c @@ -48,28 +48,36 @@ static int open_cgroup(const char *name) } #ifdef HAVE_FILE_HANDLE -int read_cgroup_id(struct cgroup *cgrp) +static u64 __read_cgroup_id(const char *path) { - char path[PATH_MAX + 1]; - char mnt[PATH_MAX + 1]; struct { struct file_handle fh; uint64_t cgroup_id; } handle; int mount_id; + handle.fh.handle_bytes = sizeof(handle.cgroup_id); + if (name_to_handle_at(AT_FDCWD, path, &handle.fh, &mount_id, 0) < 0) + return -1ULL; + + return handle.cgroup_id; +} + +int read_cgroup_id(struct cgroup *cgrp) +{ + char path[PATH_MAX + 1]; + char mnt[PATH_MAX + 1]; + if (cgroupfs_find_mountpoint(mnt, PATH_MAX + 1, "perf_event")) return -1; scnprintf(path, PATH_MAX, "%s/%s", mnt, cgrp->name); - handle.fh.handle_bytes = sizeof(handle.cgroup_id); - if (name_to_handle_at(AT_FDCWD, path, &handle.fh, &mount_id, 0) < 0) - return -1; - - cgrp->id = handle.cgroup_id; + cgrp->id = __read_cgroup_id(path); return 0; } +#else +static inline u64 __read_cgroup_id(const char *path __maybe_unused) { return -1ULL; } #endif /* HAVE_FILE_HANDLE */ #ifndef CGROUP2_SUPER_MAGIC @@ -562,6 +570,11 @@ struct cgroup *cgroup__findnew(struct perf_env *env, uint64_t id, return cgrp; } +struct cgroup *__cgroup__find(struct rb_root *root, uint64_t id) +{ + return __cgroup__findnew(root, id, /*create=*/false, /*path=*/NULL); +} + struct cgroup *cgroup__find(struct perf_env *env, uint64_t id) { struct cgroup *cgrp; @@ -587,3 +600,35 @@ void perf_env__purge_cgroups(struct perf_env *env) } up_write(&env->cgroups.lock); } + +void read_all_cgroups(struct rb_root *root) +{ + char mnt[PATH_MAX]; + struct cgroup_name *cn; + int prefix_len; + + if (cgroupfs_find_mountpoint(mnt, sizeof(mnt), "perf_event")) + return; + + /* cgroup_name will have a full path, skip the root directory */ + prefix_len = strlen(mnt); + + /* collect all cgroups in the cgroup_list */ + if (nftw(mnt, add_cgroup_name, 20, 0) < 0) + return; + + list_for_each_entry(cn, &cgroup_list, list) { + const char *name; + u64 cgrp_id; + + /* cgroup_name might have a full path, skip the prefix */ + name = cn->name + prefix_len; + if (name[0] == '\0') + name = "/"; + + cgrp_id = __read_cgroup_id(cn->name); + __cgroup__findnew(root, cgrp_id, /*create=*/true, name); + } + + release_cgroup_list(); +} diff --git a/tools/perf/util/cgroup.h b/tools/perf/util/cgroup.h index 12256b78608c..beb6fe1012ed 100644 --- a/tools/perf/util/cgroup.h +++ b/tools/perf/util/cgroup.h @@ -37,6 +37,7 @@ int parse_cgroups(const struct option *opt, const char *str, int unset); struct cgroup *cgroup__findnew(struct perf_env *env, uint64_t id, const char *path); struct cgroup *cgroup__find(struct perf_env *env, uint64_t id); +struct cgroup *__cgroup__find(struct rb_root *root, uint64_t id); void perf_env__purge_cgroups(struct perf_env *env); @@ -49,6 +50,9 @@ static inline int read_cgroup_id(struct cgroup *cgrp __maybe_unused) } #endif /* HAVE_FILE_HANDLE */ +/* read all cgroups in the system and save them in the rbtree */ +void read_all_cgroups(struct rb_root *root); + int cgroup_is_v2(const char *subsys); #endif /* __CGROUP_H__ */ From d0c502e46e9796b19e3b0fd28750cb5b679f7c4b Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Wed, 6 Sep 2023 10:49:00 -0700 Subject: [PATCH 060/475] perf lock contention: Prepare to handle cgroups Save cgroup info and display cgroup names if requested. This is a preparation for the next patch. Reviewed-by: Ian Rogers Signed-off-by: Namhyung Kim Cc: Adrian Hunter Cc: Hao Luo Cc: Ingo Molnar Cc: Jiri Olsa Cc: Peter Zijlstra Cc: Song Liu Cc: bpf@vger.kernel.org Link: https://lore.kernel.org/r/20230906174903.346486-3-namhyung@kernel.org Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/builtin-lock.c | 3 ++- tools/perf/util/bpf_lock_contention.c | 26 +++++++++++++++++++++++++- tools/perf/util/lock-contention.h | 9 +++++++-- 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/tools/perf/builtin-lock.c b/tools/perf/builtin-lock.c index b141f2134274..06430980dfd7 100644 --- a/tools/perf/builtin-lock.c +++ b/tools/perf/builtin-lock.c @@ -2040,6 +2040,7 @@ static int __cmd_contention(int argc, const char **argv) .filters = &filters, .save_callstack = needs_callstack(), .owner = show_lock_owner, + .cgroups = RB_ROOT, }; lockhash_table = calloc(LOCKHASH_SIZE, sizeof(*lockhash_table)); @@ -2158,7 +2159,7 @@ static int __cmd_contention(int argc, const char **argv) out_delete: lock_filter_finish(); evlist__delete(con.evlist); - lock_contention_finish(); + lock_contention_finish(&con); perf_session__delete(session); zfree(&lockhash_table); return err; diff --git a/tools/perf/util/bpf_lock_contention.c b/tools/perf/util/bpf_lock_contention.c index e7dddf0127bc..c6bd7c9b2d57 100644 --- a/tools/perf/util/bpf_lock_contention.c +++ b/tools/perf/util/bpf_lock_contention.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +#include "util/cgroup.h" #include "util/debug.h" #include "util/evlist.h" #include "util/machine.h" @@ -151,6 +152,10 @@ int lock_contention_prepare(struct lock_contention *con) skel->bss->needs_callstack = con->save_callstack; skel->bss->lock_owner = con->owner; + if (con->use_cgroup) { + read_all_cgroups(&con->cgroups); + } + bpf_program__set_autoload(skel->progs.collect_lock_syms, false); lock_contention_bpf__attach(skel); @@ -222,6 +227,17 @@ static const char *lock_contention_get_name(struct lock_contention *con, return ""; } + if (con->use_cgroup) { + u64 cgrp_id = key->lock_addr; + struct cgroup *cgrp = __cgroup__find(&con->cgroups, cgrp_id); + + if (cgrp) + return cgrp->name; + + snprintf(name_buf, sizeof(name_buf), "cgroup:%lu", cgrp_id); + return name_buf; + } + /* LOCK_AGGR_CALLER: skip lock internal functions */ while (machine__is_lock_function(machine, stack_trace[idx]) && idx < con->max_stack - 1) @@ -364,12 +380,20 @@ next: return err; } -int lock_contention_finish(void) +int lock_contention_finish(struct lock_contention *con) { if (skel) { skel->bss->enabled = 0; lock_contention_bpf__destroy(skel); } + while (!RB_EMPTY_ROOT(&con->cgroups)) { + struct rb_node *node = rb_first(&con->cgroups); + struct cgroup *cgrp = rb_entry(node, struct cgroup, node); + + rb_erase(node, &con->cgroups); + cgroup__put(cgrp); + } + return 0; } diff --git a/tools/perf/util/lock-contention.h b/tools/perf/util/lock-contention.h index fa16532c971c..70423966d778 100644 --- a/tools/perf/util/lock-contention.h +++ b/tools/perf/util/lock-contention.h @@ -136,6 +136,7 @@ struct lock_contention { struct hlist_head *result; struct lock_filter *filters; struct lock_contention_fails fails; + struct rb_root cgroups; unsigned long map_nr_entries; int max_stack; int stack_skip; @@ -143,6 +144,7 @@ struct lock_contention { int owner; int nr_filtered; bool save_callstack; + bool use_cgroup; }; #ifdef HAVE_BPF_SKEL @@ -151,7 +153,7 @@ int lock_contention_prepare(struct lock_contention *con); int lock_contention_start(void); int lock_contention_stop(void); int lock_contention_read(struct lock_contention *con); -int lock_contention_finish(void); +int lock_contention_finish(struct lock_contention *con); #else /* !HAVE_BPF_SKEL */ @@ -162,7 +164,10 @@ static inline int lock_contention_prepare(struct lock_contention *con __maybe_un static inline int lock_contention_start(void) { return 0; } static inline int lock_contention_stop(void) { return 0; } -static inline int lock_contention_finish(void) { return 0; } +static inline int lock_contention_finish(struct lock_contention *con __maybe_unused) +{ + return 0; +} static inline int lock_contention_read(struct lock_contention *con __maybe_unused) { From 4d1792d0a2564caf1620a2cc9aa6f12284b3a56f Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Wed, 6 Sep 2023 10:49:01 -0700 Subject: [PATCH 061/475] perf lock contention: Add --lock-cgroup option The --lock-cgroup option shows lock contention stats break down by cgroups. Add LOCK_AGGR_CGROUP mode and use it instead of use_cgroup field. $ sudo ./perf lock con -ab --lock-cgroup sleep 1 contended total wait max wait avg wait cgroup 8 15.70 us 6.34 us 1.96 us / 2 1.48 us 747 ns 738 ns /user.slice/.../app.slice/app-gnome-google\x2dchrome-6442.scope 1 848 ns 848 ns 848 ns /user.slice/.../session.slice/org.gnome.Shell@x11.service 1 220 ns 220 ns 220 ns /user.slice/.../session.slice/pipewire-pulse.service For now, the cgroup mode only works with BPF (-b). Committer notes: Remove -g as it is used in the other tools with a clear meaning of collect/show callchains. As agreed with Namhyung off list. Reviewed-by: Ian Rogers Signed-off-by: Namhyung Kim Tested-by: Arnaldo Carvalho de Melo Cc: Adrian Hunter Cc: Hao Luo Cc: Ingo Molnar Cc: Jiri Olsa Cc: Peter Zijlstra Cc: Song Liu Cc: bpf@vger.kernel.org Link: https://lore.kernel.org/r/20230906174903.346486-4-namhyung@kernel.org Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/Documentation/perf-lock.txt | 3 ++ tools/perf/builtin-lock.c | 40 ++++++++++++++++++- tools/perf/util/bpf_lock_contention.c | 16 +++++--- .../perf/util/bpf_skel/lock_contention.bpf.c | 31 +++++++++++++- tools/perf/util/bpf_skel/lock_data.h | 3 +- tools/perf/util/lock-contention.h | 1 - 6 files changed, 84 insertions(+), 10 deletions(-) diff --git a/tools/perf/Documentation/perf-lock.txt b/tools/perf/Documentation/perf-lock.txt index 30eea576721f..4b5cdf4b33c6 100644 --- a/tools/perf/Documentation/perf-lock.txt +++ b/tools/perf/Documentation/perf-lock.txt @@ -208,6 +208,9 @@ CONTENTION OPTIONS Show results using a CSV-style output to make it easy to import directly into spreadsheets. Columns are separated by the string specified in SEP. +--lock-cgroup:: + Show lock contention stat by cgroup. Requires --use-bpf. + SEE ALSO -------- diff --git a/tools/perf/builtin-lock.c b/tools/perf/builtin-lock.c index 06430980dfd7..03d82f5a9afc 100644 --- a/tools/perf/builtin-lock.c +++ b/tools/perf/builtin-lock.c @@ -60,6 +60,7 @@ static bool combine_locks; static bool show_thread_stats; static bool show_lock_addrs; static bool show_lock_owner; +static bool show_lock_cgroups; static bool use_bpf; static unsigned long bpf_map_entries = MAX_ENTRIES; static int max_stack_depth = CONTENTION_STACK_DEPTH; @@ -619,6 +620,7 @@ static int get_key_by_aggr_mode_simple(u64 *key, u64 addr, u32 tid) *key = tid; break; case LOCK_AGGR_CALLER: + case LOCK_AGGR_CGROUP: default: pr_err("Invalid aggregation mode: %d\n", aggr_mode); return -EINVAL; @@ -1103,6 +1105,7 @@ static int report_lock_contention_begin_event(struct evsel *evsel, if (lock_contention_caller(evsel, sample, buf, sizeof(buf)) < 0) name = "Unknown"; break; + case LOCK_AGGR_CGROUP: case LOCK_AGGR_TASK: default: break; @@ -1653,6 +1656,9 @@ static void print_header_stdio(void) case LOCK_AGGR_ADDR: fprintf(lock_output, " %16s %s\n\n", "address", "symbol"); break; + case LOCK_AGGR_CGROUP: + fprintf(lock_output, " %s\n\n", "cgroup"); + break; default: break; } @@ -1680,6 +1686,9 @@ static void print_header_csv(const char *sep) case LOCK_AGGR_ADDR: fprintf(lock_output, "%s%s %s%s %s\n", "address", sep, "symbol", sep, "type"); break; + case LOCK_AGGR_CGROUP: + fprintf(lock_output, "%s\n", "cgroup"); + break; default: break; } @@ -1720,6 +1729,9 @@ static void print_lock_stat_stdio(struct lock_contention *con, struct lock_stat fprintf(lock_output, " %016llx %s (%s)\n", (unsigned long long)st->addr, st->name, get_type_name(st->flags)); break; + case LOCK_AGGR_CGROUP: + fprintf(lock_output, " %s\n", st->name); + break; default: break; } @@ -1770,6 +1782,9 @@ static void print_lock_stat_csv(struct lock_contention *con, struct lock_stat *s fprintf(lock_output, "%llx%s %s%s %s\n", (unsigned long long)st->addr, sep, st->name, sep, get_type_name(st->flags)); break; + case LOCK_AGGR_CGROUP: + fprintf(lock_output, "%s\n",st->name); + break; default: break; } @@ -1999,6 +2014,27 @@ static int check_lock_contention_options(const struct option *options, return -1; } + if (show_lock_cgroups && !use_bpf) { + pr_err("Cgroups are available only with BPF\n"); + parse_options_usage(usage, options, "lock-cgroup", 0); + parse_options_usage(NULL, options, "use-bpf", 0); + return -1; + } + + if (show_lock_cgroups && show_lock_addrs) { + pr_err("Cannot use cgroup and addr mode together\n"); + parse_options_usage(usage, options, "lock-cgroup", 0); + parse_options_usage(NULL, options, "lock-addr", 0); + return -1; + } + + if (show_lock_cgroups && show_thread_stats) { + pr_err("Cannot use cgroup and thread mode together\n"); + parse_options_usage(usage, options, "lock-cgroup", 0); + parse_options_usage(NULL, options, "threads", 0); + return -1; + } + if (symbol_conf.field_sep) { if (strstr(symbol_conf.field_sep, ":") || /* part of type flags */ strstr(symbol_conf.field_sep, "+") || /* part of caller offset */ @@ -2060,7 +2096,8 @@ static int __cmd_contention(int argc, const char **argv) con.machine = &session->machines.host; con.aggr_mode = aggr_mode = show_thread_stats ? LOCK_AGGR_TASK : - show_lock_addrs ? LOCK_AGGR_ADDR : LOCK_AGGR_CALLER; + show_lock_addrs ? LOCK_AGGR_ADDR : + show_lock_cgroups ? LOCK_AGGR_CGROUP : LOCK_AGGR_CALLER; if (con.aggr_mode == LOCK_AGGR_CALLER) con.save_callstack = true; @@ -2524,6 +2561,7 @@ int cmd_lock(int argc, const char **argv) OPT_BOOLEAN('o', "lock-owner", &show_lock_owner, "show lock owners instead of waiters"), OPT_STRING_NOEMPTY('x', "field-separator", &symbol_conf.field_sep, "separator", "print result in CSV format with custom separator"), + OPT_BOOLEAN(0, "lock-cgroup", &show_lock_cgroups, "show lock stats by cgroup"), OPT_PARENT(lock_options) }; diff --git a/tools/perf/util/bpf_lock_contention.c b/tools/perf/util/bpf_lock_contention.c index c6bd7c9b2d57..42753a0dfdc5 100644 --- a/tools/perf/util/bpf_lock_contention.c +++ b/tools/perf/util/bpf_lock_contention.c @@ -152,7 +152,10 @@ int lock_contention_prepare(struct lock_contention *con) skel->bss->needs_callstack = con->save_callstack; skel->bss->lock_owner = con->owner; - if (con->use_cgroup) { + if (con->aggr_mode == LOCK_AGGR_CGROUP) { + if (cgroup_is_v2("perf_event")) + skel->bss->use_cgroup_v2 = 1; + read_all_cgroups(&con->cgroups); } @@ -214,12 +217,12 @@ static const char *lock_contention_get_name(struct lock_contention *con, return "siglock"; /* global locks with symbols */ - sym = machine__find_kernel_symbol(machine, key->lock_addr, &kmap); + sym = machine__find_kernel_symbol(machine, key->lock_addr_or_cgroup, &kmap); if (sym) return sym->name; /* try semi-global locks collected separately */ - if (!bpf_map_lookup_elem(lock_fd, &key->lock_addr, &flags)) { + if (!bpf_map_lookup_elem(lock_fd, &key->lock_addr_or_cgroup, &flags)) { if (flags == LOCK_CLASS_RQLOCK) return "rq_lock"; } @@ -227,8 +230,8 @@ static const char *lock_contention_get_name(struct lock_contention *con, return ""; } - if (con->use_cgroup) { - u64 cgrp_id = key->lock_addr; + if (con->aggr_mode == LOCK_AGGR_CGROUP) { + u64 cgrp_id = key->lock_addr_or_cgroup; struct cgroup *cgrp = __cgroup__find(&con->cgroups, cgrp_id); if (cgrp) @@ -329,7 +332,8 @@ int lock_contention_read(struct lock_contention *con) ls_key = key.pid; break; case LOCK_AGGR_ADDR: - ls_key = key.lock_addr; + case LOCK_AGGR_CGROUP: + ls_key = key.lock_addr_or_cgroup; break; default: goto next; diff --git a/tools/perf/util/bpf_skel/lock_contention.bpf.c b/tools/perf/util/bpf_skel/lock_contention.bpf.c index 8d3cfbb3cc65..823354999022 100644 --- a/tools/perf/util/bpf_skel/lock_contention.bpf.c +++ b/tools/perf/util/bpf_skel/lock_contention.bpf.c @@ -118,6 +118,9 @@ int needs_callstack; int stack_skip; int lock_owner; +int use_cgroup_v2; +int perf_subsys_id = -1; + /* determine the key of lock stat */ int aggr_mode; @@ -130,6 +133,29 @@ int data_fail; int task_map_full; int data_map_full; +static inline __u64 get_current_cgroup_id(void) +{ + struct task_struct *task; + struct cgroup *cgrp; + + if (use_cgroup_v2) + return bpf_get_current_cgroup_id(); + + task = bpf_get_current_task_btf(); + + if (perf_subsys_id == -1) { +#if __has_builtin(__builtin_preserve_enum_value) + perf_subsys_id = bpf_core_enum_value(enum cgroup_subsys_id, + perf_event_cgrp_id); +#else + perf_subsys_id = perf_event_cgrp_id; +#endif + } + + cgrp = BPF_CORE_READ(task, cgroups, subsys[perf_subsys_id], cgroup); + return BPF_CORE_READ(cgrp, kn, id); +} + static inline int can_record(u64 *ctx) { if (has_cpu) { @@ -364,10 +390,13 @@ int contention_end(u64 *ctx) key.stack_id = pelem->stack_id; break; case LOCK_AGGR_ADDR: - key.lock_addr = pelem->lock; + key.lock_addr_or_cgroup = pelem->lock; if (needs_callstack) key.stack_id = pelem->stack_id; break; + case LOCK_AGGR_CGROUP: + key.lock_addr_or_cgroup = get_current_cgroup_id(); + break; default: /* should not happen */ return 0; diff --git a/tools/perf/util/bpf_skel/lock_data.h b/tools/perf/util/bpf_skel/lock_data.h index 260062a9f2ab..08482daf61be 100644 --- a/tools/perf/util/bpf_skel/lock_data.h +++ b/tools/perf/util/bpf_skel/lock_data.h @@ -6,7 +6,7 @@ struct contention_key { u32 stack_id; u32 pid; - u64 lock_addr; + u64 lock_addr_or_cgroup; }; #define TASK_COMM_LEN 16 @@ -39,6 +39,7 @@ enum lock_aggr_mode { LOCK_AGGR_ADDR = 0, LOCK_AGGR_TASK, LOCK_AGGR_CALLER, + LOCK_AGGR_CGROUP, }; enum lock_class_sym { diff --git a/tools/perf/util/lock-contention.h b/tools/perf/util/lock-contention.h index 70423966d778..a073cc6a82d2 100644 --- a/tools/perf/util/lock-contention.h +++ b/tools/perf/util/lock-contention.h @@ -144,7 +144,6 @@ struct lock_contention { int owner; int nr_filtered; bool save_callstack; - bool use_cgroup; }; #ifdef HAVE_BPF_SKEL From 4fd06bd2dcc893d922985cc4e9082477f6d115e6 Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Wed, 6 Sep 2023 10:49:02 -0700 Subject: [PATCH 062/475] perf lock contention: Add -G/--cgroup-filter option The -G/--cgroup-filter is to limit lock contention collection on the tasks in the specific cgroups only. $ sudo ./perf lock con -abt -G /user.slice/.../vte-spawn-52221fb8-b33f-4a52-b5c3-e35d1e6fc0e0.scope \ ./perf bench sched messaging # Running 'sched/messaging' benchmark: # 20 sender and receiver processes per group # 10 groups == 400 processes run Total time: 0.174 [sec] contended total wait max wait avg wait pid comm 4 114.45 us 60.06 us 28.61 us 214847 sched-messaging 2 111.40 us 60.84 us 55.70 us 214848 sched-messaging 2 106.09 us 59.42 us 53.04 us 214837 sched-messaging 1 81.70 us 81.70 us 81.70 us 214709 sched-messaging 68 78.44 us 6.83 us 1.15 us 214633 sched-messaging 69 73.71 us 2.69 us 1.07 us 214632 sched-messaging 4 72.62 us 60.83 us 18.15 us 214850 sched-messaging 2 71.75 us 67.60 us 35.88 us 214840 sched-messaging 2 69.29 us 67.53 us 34.65 us 214804 sched-messaging 2 69.00 us 68.23 us 34.50 us 214826 sched-messaging ... Export cgroup__new() function as it's needed from outside. Reviewed-by: Ian Rogers Signed-off-by: Namhyung Kim Tested-by: Arnaldo Carvalho de Melo Cc: Adrian Hunter Cc: Hao Luo Cc: Ingo Molnar Cc: Jiri Olsa Cc: Peter Zijlstra Cc: Song Liu Cc: bpf@vger.kernel.org Link: https://lore.kernel.org/r/20230906174903.346486-5-namhyung@kernel.org Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/Documentation/perf-lock.txt | 4 ++ tools/perf/builtin-lock.c | 56 +++++++++++++++++++ tools/perf/util/bpf_lock_contention.c | 15 ++++- .../perf/util/bpf_skel/lock_contention.bpf.c | 17 ++++++ tools/perf/util/cgroup.c | 2 +- tools/perf/util/cgroup.h | 1 + tools/perf/util/lock-contention.h | 2 + 7 files changed, 95 insertions(+), 2 deletions(-) diff --git a/tools/perf/Documentation/perf-lock.txt b/tools/perf/Documentation/perf-lock.txt index 4b5cdf4b33c6..503abcba1438 100644 --- a/tools/perf/Documentation/perf-lock.txt +++ b/tools/perf/Documentation/perf-lock.txt @@ -211,6 +211,10 @@ CONTENTION OPTIONS --lock-cgroup:: Show lock contention stat by cgroup. Requires --use-bpf. +-G:: +--cgroup-filter=:: + Show lock contention only in the given cgroups (comma separated list). + SEE ALSO -------- diff --git a/tools/perf/builtin-lock.c b/tools/perf/builtin-lock.c index 03d82f5a9afc..d4b22313e5fc 100644 --- a/tools/perf/builtin-lock.c +++ b/tools/perf/builtin-lock.c @@ -10,6 +10,7 @@ #include "util/thread.h" #include "util/header.h" #include "util/target.h" +#include "util/cgroup.h" #include "util/callchain.h" #include "util/lock-contention.h" #include "util/bpf_skel/lock_data.h" @@ -1631,6 +1632,9 @@ static void lock_filter_finish(void) zfree(&filters.syms); filters.nr_syms = 0; + + zfree(&filters.cgrps); + filters.nr_cgrps = 0; } static void sort_contention_result(void) @@ -2488,6 +2492,56 @@ static int parse_output(const struct option *opt __maybe_unused, const char *str return 0; } +static bool add_lock_cgroup(char *name) +{ + u64 *tmp; + struct cgroup *cgrp; + + cgrp = cgroup__new(name, /*do_open=*/false); + if (cgrp == NULL) { + pr_err("Failed to create cgroup: %s\n", name); + return false; + } + + if (read_cgroup_id(cgrp) < 0) { + pr_err("Failed to read cgroup id for %s\n", name); + cgroup__put(cgrp); + return false; + } + + tmp = realloc(filters.cgrps, (filters.nr_cgrps + 1) * sizeof(*filters.cgrps)); + if (tmp == NULL) { + pr_err("Memory allocation failure\n"); + return false; + } + + tmp[filters.nr_cgrps++] = cgrp->id; + filters.cgrps = tmp; + cgroup__put(cgrp); + return true; +} + +static int parse_cgroup_filter(const struct option *opt __maybe_unused, const char *str, + int unset __maybe_unused) +{ + char *s, *tmp, *tok; + int ret = 0; + + s = strdup(str); + if (s == NULL) + return -1; + + for (tok = strtok_r(s, ", ", &tmp); tok; tok = strtok_r(NULL, ", ", &tmp)) { + if (!add_lock_cgroup(tok)) { + ret = -1; + break; + } + } + + free(s); + return ret; +} + int cmd_lock(int argc, const char **argv) { const struct option lock_options[] = { @@ -2562,6 +2616,8 @@ int cmd_lock(int argc, const char **argv) OPT_STRING_NOEMPTY('x', "field-separator", &symbol_conf.field_sep, "separator", "print result in CSV format with custom separator"), OPT_BOOLEAN(0, "lock-cgroup", &show_lock_cgroups, "show lock stats by cgroup"), + OPT_CALLBACK('G', "cgroup-filter", NULL, "CGROUPS", + "Filter specific cgroups", parse_cgroup_filter), OPT_PARENT(lock_options) }; diff --git a/tools/perf/util/bpf_lock_contention.c b/tools/perf/util/bpf_lock_contention.c index 42753a0dfdc5..e105245eb905 100644 --- a/tools/perf/util/bpf_lock_contention.c +++ b/tools/perf/util/bpf_lock_contention.c @@ -21,7 +21,7 @@ static struct lock_contention_bpf *skel; int lock_contention_prepare(struct lock_contention *con) { int i, fd; - int ncpus = 1, ntasks = 1, ntypes = 1, naddrs = 1; + int ncpus = 1, ntasks = 1, ntypes = 1, naddrs = 1, ncgrps = 1; struct evlist *evlist = con->evlist; struct target *target = con->target; @@ -51,6 +51,8 @@ int lock_contention_prepare(struct lock_contention *con) ntasks = perf_thread_map__nr(evlist->core.threads); if (con->filters->nr_types) ntypes = con->filters->nr_types; + if (con->filters->nr_cgrps) + ncgrps = con->filters->nr_cgrps; /* resolve lock name filters to addr */ if (con->filters->nr_syms) { @@ -85,6 +87,7 @@ int lock_contention_prepare(struct lock_contention *con) bpf_map__set_max_entries(skel->maps.task_filter, ntasks); bpf_map__set_max_entries(skel->maps.type_filter, ntypes); bpf_map__set_max_entries(skel->maps.addr_filter, naddrs); + bpf_map__set_max_entries(skel->maps.cgroup_filter, ncgrps); if (lock_contention_bpf__load(skel) < 0) { pr_err("Failed to load lock-contention BPF skeleton\n"); @@ -146,6 +149,16 @@ int lock_contention_prepare(struct lock_contention *con) bpf_map_update_elem(fd, &con->filters->addrs[i], &val, BPF_ANY); } + if (con->filters->nr_cgrps) { + u8 val = 1; + + skel->bss->has_cgroup = 1; + fd = bpf_map__fd(skel->maps.cgroup_filter); + + for (i = 0; i < con->filters->nr_cgrps; i++) + bpf_map_update_elem(fd, &con->filters->cgrps[i], &val, BPF_ANY); + } + /* these don't work well if in the rodata section */ skel->bss->stack_skip = con->stack_skip; skel->bss->aggr_mode = con->aggr_mode; diff --git a/tools/perf/util/bpf_skel/lock_contention.bpf.c b/tools/perf/util/bpf_skel/lock_contention.bpf.c index 823354999022..4900a5dfb4a4 100644 --- a/tools/perf/util/bpf_skel/lock_contention.bpf.c +++ b/tools/perf/util/bpf_skel/lock_contention.bpf.c @@ -92,6 +92,13 @@ struct { __uint(max_entries, 1); } addr_filter SEC(".maps"); +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u64)); + __uint(value_size, sizeof(__u8)); + __uint(max_entries, 1); +} cgroup_filter SEC(".maps"); + struct rw_semaphore___old { struct task_struct *owner; } __attribute__((preserve_access_index)); @@ -114,6 +121,7 @@ int has_cpu; int has_task; int has_type; int has_addr; +int has_cgroup; int needs_callstack; int stack_skip; int lock_owner; @@ -194,6 +202,15 @@ static inline int can_record(u64 *ctx) return 0; } + if (has_cgroup) { + __u8 *ok; + __u64 cgrp = get_current_cgroup_id(); + + ok = bpf_map_lookup_elem(&cgroup_filter, &cgrp); + if (!ok) + return 0; + } + return 1; } diff --git a/tools/perf/util/cgroup.c b/tools/perf/util/cgroup.c index d03651098dde..fcb509058499 100644 --- a/tools/perf/util/cgroup.c +++ b/tools/perf/util/cgroup.c @@ -114,7 +114,7 @@ static struct cgroup *evlist__find_cgroup(struct evlist *evlist, const char *str return NULL; } -static struct cgroup *cgroup__new(const char *name, bool do_open) +struct cgroup *cgroup__new(const char *name, bool do_open) { struct cgroup *cgroup = zalloc(sizeof(*cgroup)); diff --git a/tools/perf/util/cgroup.h b/tools/perf/util/cgroup.h index beb6fe1012ed..de8882d6e8d3 100644 --- a/tools/perf/util/cgroup.h +++ b/tools/perf/util/cgroup.h @@ -26,6 +26,7 @@ void cgroup__put(struct cgroup *cgroup); struct evlist; struct rblist; +struct cgroup *cgroup__new(const char *name, bool do_open); struct cgroup *evlist__findnew_cgroup(struct evlist *evlist, const char *name); int evlist__expand_cgroup(struct evlist *evlist, const char *cgroups, struct rblist *metric_events, bool open_cgroup); diff --git a/tools/perf/util/lock-contention.h b/tools/perf/util/lock-contention.h index a073cc6a82d2..1a7248ff3889 100644 --- a/tools/perf/util/lock-contention.h +++ b/tools/perf/util/lock-contention.h @@ -9,9 +9,11 @@ struct lock_filter { int nr_types; int nr_addrs; int nr_syms; + int nr_cgrps; unsigned int *types; unsigned long *addrs; char **syms; + u64 *cgrps; }; struct lock_stat { From e44b47b93132fb757e52fd4547cbfd2243685cb5 Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Wed, 6 Sep 2023 10:49:03 -0700 Subject: [PATCH 063/475] perf test shell lock_contention: Add cgroup aggregation and filter tests Add cgroup aggregation and filter tests. $ sudo ./perf test -v contention 84: kernel lock contention analysis test : --- start --- test child forked, pid 222423 Testing perf lock record and perf lock contention Testing perf lock contention --use-bpf Testing perf lock record and perf lock contention at the same time Testing perf lock contention --threads Testing perf lock contention --lock-addr Testing perf lock contention --lock-cgroup Testing perf lock contention --type-filter (w/ spinlock) Testing perf lock contention --lock-filter (w/ tasklist_lock) Testing perf lock contention --callstack-filter (w/ unix_stream) Testing perf lock contention --callstack-filter with task aggregation Testing perf lock contention --cgroup-filter Testing perf lock contention CSV output test child finished with 0 ---- end ---- kernel lock contention analysis test: Ok Committer testing: [root@quaco ~]# uname -a Linux quaco 6.4.10-200.fc38.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Aug 11 12:20:29 UTC 2023 x86_64 GNU/Linux [root@quaco ~]# perf test -v contention 84: kernel lock contention analysis test : --- start --- test child forked, pid 452625 Testing perf lock record and perf lock contention Testing perf lock contention --use-bpf Testing perf lock record and perf lock contention at the same time Testing perf lock contention --threads Testing perf lock contention --lock-addr Testing perf lock contention --lock-cgroup Testing perf lock contention --type-filter (w/ spinlock) Testing perf lock contention --lock-filter (w/ tasklist_lock) Testing perf lock contention --callstack-filter (w/ unix_stream) Testing perf lock contention --callstack-filter with task aggregation Testing perf lock contention --cgroup-filter Testing perf lock contention CSV output test child finished with 0 ---- end ---- kernel lock contention analysis test: Ok [root@quaco ~]# Reviewed-by: Ian Rogers Signed-off-by: Namhyung Kim Tested-by: Arnaldo Carvalho de Melo Cc: Adrian Hunter Cc: Hao Luo Cc: Ingo Molnar Cc: Jiri Olsa Cc: Peter Zijlstra Cc: Song Liu Cc: bpf@vger.kernel.org Link: https://lore.kernel.org/r/20230906174903.346486-6-namhyung@kernel.org Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/tests/shell/lock_contention.sh | 45 +++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/tools/perf/tests/shell/lock_contention.sh b/tools/perf/tests/shell/lock_contention.sh index d120e83db7d9..966e67db75f3 100755 --- a/tools/perf/tests/shell/lock_contention.sh +++ b/tools/perf/tests/shell/lock_contention.sh @@ -123,6 +123,24 @@ test_aggr_addr() fi } +test_aggr_cgroup() +{ + echo "Testing perf lock contention --lock-cgroup" + + if ! perf lock con -b true > /dev/null 2>&1 ; then + echo "[Skip] No BPF support" + return + fi + + # the perf lock contention output goes to the stderr + perf lock con -a -b -g -E 1 -q -- perf bench sched messaging > /dev/null 2> ${result} + if [ "$(cat "${result}" | wc -l)" != "1" ]; then + echo "[Fail] BPF result count is not 1:" "$(cat "${result}" | wc -l)" + err=1 + exit + fi +} + test_type_filter() { echo "Testing perf lock contention --type-filter (w/ spinlock)" @@ -232,6 +250,31 @@ test_aggr_task_stack_filter() exit fi } +test_cgroup_filter() +{ + echo "Testing perf lock contention --cgroup-filter" + + if ! perf lock con -b true > /dev/null 2>&1 ; then + echo "[Skip] No BPF support" + return + fi + + perf lock con -a -b -g -E 1 -F wait_total -q -- perf bench sched messaging > /dev/null 2> ${result} + if [ "$(cat "${result}" | wc -l)" != "1" ]; then + echo "[Fail] BPF result should have a cgroup result:" "$(cat "${result}")" + err=1 + exit + fi + + cgroup=$(cat "${result}" | awk '{ print $3 }') + perf lock con -a -b -g -E 1 -G "${cgroup}" -q -- perf bench sched messaging > /dev/null 2> ${result} + if [ "$(cat "${result}" | wc -l)" != "1" ]; then + echo "[Fail] BPF result should have a result with cgroup filter:" "$(cat "${cgroup}")" + err=1 + exit + fi +} + test_csv_output() { @@ -275,10 +318,12 @@ test_bpf test_record_concurrent test_aggr_task test_aggr_addr +test_aggr_cgroup test_type_filter test_lock_filter test_stack_filter test_aggr_task_stack_filter +test_cgroup_filter test_csv_output exit ${err} From 4f19fc1839e54d2c0d1e449a29a7578d029846d1 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Wed, 6 Sep 2023 16:44:15 -0700 Subject: [PATCH 064/475] perf list: Avoid a hardcoded cpu PMU name Use the first core PMU instead. On a Raspberry Pi, before: $ perf list ... cpu/t1=v1[,t2=v2,t3 ...]/modifier [Raw hardware event descriptor] [(see 'man perf-list' on how to encode it)] ... After: $ perf list ... armv8_cortex_a72/t1=v1[,t2=v2,t3 ...]/modifier [Raw hardware event descriptor] [(see 'man perf-list' on how to encode it)] ... ``` Signed-off-by: Ian Rogers Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ingo Molnar Cc: Jiri Olsa Cc: Kan Liang Cc: Kang Minchul Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Thomas Richter Cc: Xing Zhengjun Link: https://lore.kernel.org/r/20230906234416.3472339-1-irogers@google.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/util/print-events.c | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/tools/perf/util/print-events.c b/tools/perf/util/print-events.c index a7566edc86a3..b0fc48be623f 100644 --- a/tools/perf/util/print-events.c +++ b/tools/perf/util/print-events.c @@ -395,6 +395,8 @@ void print_symbol_events(const struct print_callbacks *print_cb, void *print_sta */ void print_events(const struct print_callbacks *print_cb, void *print_state) { + char *tmp; + print_symbol_events(print_cb, print_state, PERF_TYPE_HARDWARE, event_symbols_hw, PERF_COUNT_HW_MAX); print_symbol_events(print_cb, print_state, PERF_TYPE_SOFTWARE, @@ -418,17 +420,21 @@ void print_events(const struct print_callbacks *print_cb, void *print_state) /*long_desc=*/NULL, /*encoding_desc=*/NULL); - print_cb->print_event(print_state, - /*topic=*/NULL, - /*pmu_name=*/NULL, - "cpu/t1=v1[,t2=v2,t3 ...]/modifier", - /*event_alias=*/NULL, - /*scale_unit=*/NULL, - /*deprecated=*/false, - event_type_descriptors[PERF_TYPE_RAW], - "(see 'man perf-list' on how to encode it)", - /*long_desc=*/NULL, - /*encoding_desc=*/NULL); + if (asprintf(&tmp, "%s/t1=v1[,t2=v2,t3 ...]/modifier", + perf_pmus__scan_core(/*pmu=*/NULL)->name) > 0) { + print_cb->print_event(print_state, + /*topic=*/NULL, + /*pmu_name=*/NULL, + tmp, + /*event_alias=*/NULL, + /*scale_unit=*/NULL, + /*deprecated=*/false, + event_type_descriptors[PERF_TYPE_RAW], + "(see 'man perf-list' on how to encode it)", + /*long_desc=*/NULL, + /*encoding_desc=*/NULL); + free(tmp); + } print_cb->print_event(print_state, /*topic=*/NULL, From 6bd8c2ea6b93aabcb4a046ca7a6ad353b574d0a3 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Wed, 6 Sep 2023 16:44:16 -0700 Subject: [PATCH 065/475] perf list pfm: Retry supported test with exclude_kernel With paranoia set at 2 evsel__open will fail with EACCES for non-root users. To avoid this stopping libpfm4 events from being printed, retry with exclude_kernel enabled - copying the regular is_event_supported test. Signed-off-by: Ian Rogers Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ingo Molnar Cc: Jiri Olsa Cc: Kan Liang Cc: Kang Minchul Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Thomas Richter Cc: Xing Zhengjun Link: https://lore.kernel.org/r/20230906234416.3472339-2-irogers@google.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/util/pfm.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tools/perf/util/pfm.c b/tools/perf/util/pfm.c index 862e4a689868..5ccfe4b64cdf 100644 --- a/tools/perf/util/pfm.c +++ b/tools/perf/util/pfm.c @@ -145,7 +145,20 @@ static bool is_libpfm_event_supported(const char *name, struct perf_cpu_map *cpu evsel->is_libpfm_event = true; - if (evsel__open(evsel, cpus, threads) < 0) + ret = evsel__open(evsel, cpus, threads); + if (ret == -EACCES) { + /* + * This happens if the paranoid value + * /proc/sys/kernel/perf_event_paranoid is set to 2 + * Re-run with exclude_kernel set; we don't do that + * by default as some ARM machines do not support it. + * + */ + evsel->core.attr.exclude_kernel = 1; + ret = evsel__open(evsel, cpus, threads); + + } + if (ret < 0) result = false; evsel__close(evsel); From d19a353cdd0d3021e31e00df17811ab68472ad77 Mon Sep 17 00:00:00 2001 From: James Clark Date: Mon, 4 Sep 2023 10:50:43 +0100 Subject: [PATCH 066/475] perf test: Check result of has_event(cycles) test Currently the function always returns 0, so even when the has_event() test fails, the test still passes. Fix it by returning ret instead. Reviewed-by: Ian Rogers Signed-off-by: James Clark Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Chen Zhongjin Cc: Eduard Zingerman Cc: Haixin Yu Cc: Ingo Molnar Cc: Jing Zhang Cc: Jiri Olsa Cc: John Garry Cc: Kajol Jain Cc: Kan Liang Cc: Leo Yan Cc: Liam Howlett Cc: Madhavan Srinivasan Cc: Mark Rutland Cc: Miguel Ojeda Cc: Mike Leach Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Ravi Bangoria Cc: Will Deacon Cc: Yang Jihong Cc: linux-arm-kernel@lists.infradead.org Link: https://lore.kernel.org/r/20230904095104.1162928-2-james.clark@arm.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/tests/expr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/perf/tests/expr.c b/tools/perf/tests/expr.c index 81229fa4f1e9..39be7f3b3a53 100644 --- a/tools/perf/tests/expr.c +++ b/tools/perf/tests/expr.c @@ -263,7 +263,7 @@ static int test__expr(struct test_suite *t __maybe_unused, int subtest __maybe_u expr__ctx_free(ctx); - return 0; + return ret; } DEFINE_SUITE("Simple expression parser", expr); From f561fc78c5713aae3f90f943fd5aff347ba3d4ee Mon Sep 17 00:00:00 2001 From: James Clark Date: Mon, 4 Sep 2023 10:50:44 +0100 Subject: [PATCH 067/475] perf jevents: Remove unused keyword 'cpuid_not_more_than' was the working title of the new 'strcmp_cpuid_str' keyword and was accidentally left in. It was never used so tidying it up has no effect. Reviewed-by: Ian Rogers Signed-off-by: James Clark Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Chen Zhongjin Cc: Eduard Zingerman Cc: Haixin Yu Cc: Ingo Molnar Cc: Jing Zhang Cc: Jiri Olsa Cc: John Garry Cc: Kajol Jain Cc: Kan Liang Cc: Leo Yan Cc: Liam Howlett Cc: Madhavan Srinivasan Cc: Mark Rutland Cc: Miguel Ojeda Cc: Mike Leach Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Ravi Bangoria Cc: Will Deacon Cc: Yang Jihong Cc: linux-arm-kernel@lists.infradead.org Link: https://lore.kernel.org/r/20230904095104.1162928-3-james.clark@arm.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/pmu-events/metric.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/perf/pmu-events/metric.py b/tools/perf/pmu-events/metric.py index 0e9ec65d92ae..1bbb89102901 100644 --- a/tools/perf/pmu-events/metric.py +++ b/tools/perf/pmu-events/metric.py @@ -558,8 +558,7 @@ def ParsePerfJson(orig: str) -> Expression: # Convert accidentally converted scientific notation constants back py = re.sub(r'([0-9]+)Event\(r"(e[0-9]+)"\)', r'\1\2', py) # Convert all the known keywords back from events to just the keyword - keywords = ['if', 'else', 'min', 'max', 'd_ratio', 'source_count', 'has_event', 'strcmp_cpuid_str', - 'cpuid_not_more_than'] + keywords = ['if', 'else', 'min', 'max', 'd_ratio', 'source_count', 'has_event', 'strcmp_cpuid_str'] for kw in keywords: py = re.sub(rf'Event\(r"{kw}"\)', kw, py) try: From 8a55c1e2c9e123b399b272a7db23f09dbb74af21 Mon Sep 17 00:00:00 2001 From: James Clark Date: Mon, 4 Sep 2023 10:50:45 +0100 Subject: [PATCH 068/475] perf util: Add a function for replacing characters in a string It finds all occurrences of a single character and replaces them with a multi character string. This will be used in a test in a following commit. Reviewed-by: Ian Rogers Signed-off-by: James Clark Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Chen Zhongjin Cc: Eduard Zingerman Cc: Haixin Yu Cc: Ingo Molnar Cc: Jing Zhang Cc: Jiri Olsa Cc: John Garry Cc: Kajol Jain Cc: Kan Liang Cc: Leo Yan Cc: Liam Howlett Cc: Madhavan Srinivasan Cc: Mark Rutland Cc: Mike Leach Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Ravi Bangoria Cc: Will Deacon Cc: Yang Jihong Cc: linux-arm-kernel@lists.infradead.org Link: https://lore.kernel.org/r/20230904095104.1162928-4-james.clark@arm.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/tests/Build | 1 + tools/perf/tests/builtin-test.c | 1 + tools/perf/tests/tests.h | 1 + tools/perf/tests/util.c | 31 +++++++++++++++++++++ tools/perf/util/string.c | 48 +++++++++++++++++++++++++++++++++ tools/perf/util/string2.h | 1 + 6 files changed, 83 insertions(+) create mode 100644 tools/perf/tests/util.c diff --git a/tools/perf/tests/Build b/tools/perf/tests/Build index 63d5e6d5f165..2b45ffa462a6 100644 --- a/tools/perf/tests/Build +++ b/tools/perf/tests/Build @@ -66,6 +66,7 @@ perf-y += dlfilter-test.o perf-y += sigtrap.o perf-y += event_groups.o perf-y += symbols.o +perf-y += util.o ifeq ($(SRCARCH),$(filter $(SRCARCH),x86 arm arm64 powerpc)) perf-$(CONFIG_DWARF_UNWIND) += dwarf-unwind.o diff --git a/tools/perf/tests/builtin-test.c b/tools/perf/tests/builtin-test.c index 0ad18cf6dd22..cb6f1dd00dc4 100644 --- a/tools/perf/tests/builtin-test.c +++ b/tools/perf/tests/builtin-test.c @@ -123,6 +123,7 @@ static struct test_suite *generic_tests[] = { &suite__sigtrap, &suite__event_groups, &suite__symbols, + &suite__util, NULL, }; diff --git a/tools/perf/tests/tests.h b/tools/perf/tests/tests.h index f33cfc3c19a4..b394f3ac2d66 100644 --- a/tools/perf/tests/tests.h +++ b/tools/perf/tests/tests.h @@ -145,6 +145,7 @@ DECLARE_SUITE(dlfilter); DECLARE_SUITE(sigtrap); DECLARE_SUITE(event_groups); DECLARE_SUITE(symbols); +DECLARE_SUITE(util); /* * PowerPC and S390 do not support creation of instruction breakpoints using the diff --git a/tools/perf/tests/util.c b/tools/perf/tests/util.c new file mode 100644 index 000000000000..6366db5cbf8c --- /dev/null +++ b/tools/perf/tests/util.c @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "tests.h" +#include "util/debug.h" + +#include +#include +#include + +static int test_strreplace(char needle, const char *haystack, + const char *replace, const char *expected) +{ + char *new = strreplace_chars(needle, haystack, replace); + int ret = strcmp(new, expected); + + free(new); + return ret == 0; +} + +static int test__util(struct test_suite *t __maybe_unused, int subtest __maybe_unused) +{ + TEST_ASSERT_VAL("empty string", test_strreplace(' ', "", "123", "")); + TEST_ASSERT_VAL("no match", test_strreplace('5', "123", "4", "123")); + TEST_ASSERT_VAL("replace 1", test_strreplace('3', "123", "4", "124")); + TEST_ASSERT_VAL("replace 2", test_strreplace('a', "abcabc", "ef", "efbcefbc")); + TEST_ASSERT_VAL("replace long", test_strreplace('a', "abcabc", "longlong", + "longlongbclonglongbc")); + + return 0; +} + +DEFINE_SUITE("util", util); diff --git a/tools/perf/util/string.c b/tools/perf/util/string.c index cf05b0b56c57..116a642ad99d 100644 --- a/tools/perf/util/string.c +++ b/tools/perf/util/string.c @@ -301,3 +301,51 @@ unsigned int hex(char c) return c - 'a' + 10; return c - 'A' + 10; } + +/* + * Replace all occurrences of character 'needle' in string 'haystack' with + * string 'replace' + * + * The new string could be longer so a new string is returned which must be + * freed. + */ +char *strreplace_chars(char needle, const char *haystack, const char *replace) +{ + int replace_len = strlen(replace); + char *new_s, *to; + const char *loc = strchr(haystack, needle); + const char *from = haystack; + int num = 0; + + /* Count occurrences */ + while (loc) { + loc = strchr(loc + 1, needle); + num++; + } + + /* Allocate enough space for replacements and reset first location */ + new_s = malloc(strlen(haystack) + (num * (replace_len - 1) + 1)); + if (!new_s) + return NULL; + loc = strchr(haystack, needle); + to = new_s; + + while (loc) { + /* Copy original string up to found char and update positions */ + memcpy(to, from, 1 + loc - from); + to += loc - from; + from = loc + 1; + + /* Copy replacement string and update positions */ + memcpy(to, replace, replace_len); + to += replace_len; + + /* needle next occurrence or end of string */ + loc = strchr(from, needle); + } + + /* Copy any remaining chars + null */ + strcpy(to, from); + + return new_s; +} diff --git a/tools/perf/util/string2.h b/tools/perf/util/string2.h index 56c30fef9682..52cb8ba057c7 100644 --- a/tools/perf/util/string2.h +++ b/tools/perf/util/string2.h @@ -39,5 +39,6 @@ char *strpbrk_esc(char *str, const char *stopset); char *strdup_esc(const char *str); unsigned int hex(char c); +char *strreplace_chars(char needle, const char *haystack, const char *replace); #endif /* PERF_STRING_H */ From a1ebf7718ee31501d2d2ee3af1716e0084c81926 Mon Sep 17 00:00:00 2001 From: James Clark Date: Mon, 4 Sep 2023 10:50:46 +0100 Subject: [PATCH 069/475] perf test: Add a test for strcmp_cpuid_str() expression Test that the new expression builtin returns a match when the current escaped CPU ID is given, and that it doesn't match when "0x0" is given. The CPU ID in test__expr() has to be changed to perf_pmu__getcpuid() which returns the CPU ID string, rather than the raw CPU ID that get_cpuid() returns because that can't be used with strcmp_cpuid_str(). It doesn't affect the is_intel test because both versions contain "Intel". Reviewed-by: Ian Rogers Signed-off-by: James Clark Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Chen Zhongjin Cc: Eduard Zingerman Cc: Haixin Yu Cc: Ingo Molnar Cc: Jing Zhang Cc: Jiri Olsa Cc: John Garry Cc: Kajol Jain Cc: Kan Liang Cc: Leo Yan Cc: Liam Howlett Cc: Madhavan Srinivasan Cc: Mark Rutland Cc: Mike Leach Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Ravi Bangoria Cc: Will Deacon Cc: Yang Jihong Cc: linux-arm-kernel@lists.infradead.org Link: https://lore.kernel.org/r/20230904095104.1162928-5-james.clark@arm.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/tests/expr.c | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/tools/perf/tests/expr.c b/tools/perf/tests/expr.c index 39be7f3b3a53..78544092ef0c 100644 --- a/tools/perf/tests/expr.c +++ b/tools/perf/tests/expr.c @@ -9,6 +9,7 @@ #include #include #include +#include #include static int test_ids_union(void) @@ -74,10 +75,13 @@ static int test__expr(struct test_suite *t __maybe_unused, int subtest __maybe_u int ret; struct expr_parse_ctx *ctx; bool is_intel = false; - char buf[128]; + char strcmp_cpuid_buf[256]; + struct perf_pmu *pmu = pmu__find_core_pmu(); + char *cpuid = perf_pmu__getcpuid(pmu); + char *escaped_cpuid1, *escaped_cpuid2; - if (!get_cpuid(buf, sizeof(buf))) - is_intel = strstr(buf, "Intel") != NULL; + TEST_ASSERT_VAL("get_cpuid", cpuid); + is_intel = strstr(cpuid, "Intel") != NULL; TEST_ASSERT_EQUAL("ids_union", test_ids_union(), 0); @@ -257,9 +261,28 @@ static int test__expr(struct test_suite *t __maybe_unused, int subtest __maybe_u TEST_ASSERT_VAL("source count", hashmap__size(ctx->ids) == 1); TEST_ASSERT_VAL("source count", hashmap__find(ctx->ids, "EVENT1", &val_ptr)); + + /* Test no cpuid match */ + ret = test(ctx, "strcmp_cpuid_str(0x0)", 0); + + /* + * Test cpuid match with current cpuid. Special chars have to be + * escaped. + */ + escaped_cpuid1 = strreplace_chars('-', cpuid, "\\-"); + free(cpuid); + escaped_cpuid2 = strreplace_chars(',', escaped_cpuid1, "\\,"); + free(escaped_cpuid1); + escaped_cpuid1 = strreplace_chars('=', escaped_cpuid2, "\\="); + free(escaped_cpuid2); + scnprintf(strcmp_cpuid_buf, sizeof(strcmp_cpuid_buf), + "strcmp_cpuid_str(%s)", escaped_cpuid1); + free(escaped_cpuid1); + ret |= test(ctx, strcmp_cpuid_buf, 1); + /* has_event returns 1 when an event exists. */ expr__add_id_val(ctx, strdup("cycles"), 2); - ret = test(ctx, "has_event(cycles)", 1); + ret |= test(ctx, "has_event(cycles)", 1); expr__ctx_free(ctx); From a484e64580bdb792ad613ee45bc36efd24f77191 Mon Sep 17 00:00:00 2001 From: James Clark Date: Thu, 31 Aug 2023 17:15:31 +0100 Subject: [PATCH 070/475] perf vendor events arm64: Update V1 events using Arm telemetry repo The new data [1] includes descriptions that may have product specific details and new groupings that will be consistent with other products. The following command was used to generate the jsons: $ telemetry-solution/tools/perf_json_generator/generate.py \ linux/tools/perf/ --telemetry-files \ telemetry-solution/data/pmu/cpu/neoverse/neoverse-v1.json [1]: https://gitlab.arm.com/telemetry-solution/telemetry-solution/-/blob/main/data/pmu/cpu/neoverse/neoverse-v1.json Signed-off-by: James Clark Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ian Rogers Cc: Ingo Molnar Cc: Jiri Olsa Cc: John Garry Cc: Leo Yan Cc: Mark Rutland Cc: Mike Leach Cc: Namhyung Kim Cc: Nick Forrington Cc: Peter Zijlstra Cc: Will Deacon Cc: linux-arm-kernel@lists.infradead.org Link: https://lore.kernel.org/r/20230831161618.134738-2-james.clark@arm.com Signed-off-by: Arnaldo Carvalho de Melo --- .../arch/arm64/arm/neoverse-v1/branch.json | 8 - .../arch/arm64/arm/neoverse-v1/bus.json | 18 +- .../arch/arm64/arm/neoverse-v1/cache.json | 155 ------------------ .../arch/arm64/arm/neoverse-v1/exception.json | 45 +++-- .../arm64/arm/neoverse-v1/fp_operation.json | 10 ++ .../arch/arm64/arm/neoverse-v1/general.json | 10 ++ .../arm64/arm/neoverse-v1/instruction.json | 119 -------------- .../arch/arm64/arm/neoverse-v1/l1d_cache.json | 54 ++++++ .../arch/arm64/arm/neoverse-v1/l1i_cache.json | 14 ++ .../arch/arm64/arm/neoverse-v1/l2_cache.json | 50 ++++++ .../arch/arm64/arm/neoverse-v1/l3_cache.json | 22 +++ .../arch/arm64/arm/neoverse-v1/ll_cache.json | 10 ++ .../arch/arm64/arm/neoverse-v1/memory.json | 21 ++- .../arch/arm64/arm/neoverse-v1/pipeline.json | 23 --- .../arch/arm64/arm/neoverse-v1/retired.json | 30 ++++ .../arch/arm64/arm/neoverse-v1/spe.json | 18 ++ .../arm64/arm/neoverse-v1/spec_operation.json | 110 +++++++++++++ .../arch/arm64/arm/neoverse-v1/stall.json | 30 ++++ .../arch/arm64/arm/neoverse-v1/sve.json | 30 ++++ .../arch/arm64/arm/neoverse-v1/tlb.json | 66 ++++++++ 20 files changed, 502 insertions(+), 341 deletions(-) delete mode 100644 tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/branch.json delete mode 100644 tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/cache.json create mode 100644 tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/fp_operation.json create mode 100644 tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/general.json delete mode 100644 tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/instruction.json create mode 100644 tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/l1d_cache.json create mode 100644 tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/l1i_cache.json create mode 100644 tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/l2_cache.json create mode 100644 tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/l3_cache.json create mode 100644 tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/ll_cache.json delete mode 100644 tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/pipeline.json create mode 100644 tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/retired.json create mode 100644 tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/spe.json create mode 100644 tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/spec_operation.json create mode 100644 tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/stall.json create mode 100644 tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/sve.json create mode 100644 tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/tlb.json diff --git a/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/branch.json b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/branch.json deleted file mode 100644 index 79f2016c53b0..000000000000 --- a/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/branch.json +++ /dev/null @@ -1,8 +0,0 @@ -[ - { - "ArchStdEvent": "BR_MIS_PRED" - }, - { - "ArchStdEvent": "BR_PRED" - } -] diff --git a/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/bus.json b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/bus.json index 579c1c993d17..2e11a8c4a484 100644 --- a/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/bus.json +++ b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/bus.json @@ -1,20 +1,18 @@ [ { - "ArchStdEvent": "CPU_CYCLES" + "ArchStdEvent": "BUS_ACCESS", + "PublicDescription": "Counts memory transactions issued by the CPU to the external bus, including snoop requests and snoop responses. Each beat of data is counted individually." }, { - "ArchStdEvent": "BUS_ACCESS" + "ArchStdEvent": "BUS_CYCLES", + "PublicDescription": "Counts bus cycles in the CPU. Bus cycles represent a clock cycle in which a transaction could be sent or received on the interface from the CPU to the external bus. Since that interface is driven at the same clock speed as the CPU, this event is a duplicate of CPU_CYCLES." }, { - "ArchStdEvent": "BUS_CYCLES" + "ArchStdEvent": "BUS_ACCESS_RD", + "PublicDescription": "Counts memory read transactions seen on the external bus. Each beat of data is counted individually." }, { - "ArchStdEvent": "BUS_ACCESS_RD" - }, - { - "ArchStdEvent": "BUS_ACCESS_WR" - }, - { - "ArchStdEvent": "CNT_CYCLES" + "ArchStdEvent": "BUS_ACCESS_WR", + "PublicDescription": "Counts memory write transactions seen on the external bus. Each beat of data is counted individually." } ] diff --git a/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/cache.json b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/cache.json deleted file mode 100644 index 0141f749bff3..000000000000 --- a/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/cache.json +++ /dev/null @@ -1,155 +0,0 @@ -[ - { - "ArchStdEvent": "L1I_CACHE_REFILL" - }, - { - "ArchStdEvent": "L1I_TLB_REFILL" - }, - { - "ArchStdEvent": "L1D_CACHE_REFILL" - }, - { - "ArchStdEvent": "L1D_CACHE" - }, - { - "ArchStdEvent": "L1D_TLB_REFILL" - }, - { - "ArchStdEvent": "L1I_CACHE" - }, - { - "ArchStdEvent": "L1D_CACHE_WB" - }, - { - "ArchStdEvent": "L2D_CACHE" - }, - { - "ArchStdEvent": "L2D_CACHE_REFILL" - }, - { - "ArchStdEvent": "L2D_CACHE_WB" - }, - { - "ArchStdEvent": "L2D_CACHE_ALLOCATE" - }, - { - "ArchStdEvent": "L1D_TLB" - }, - { - "ArchStdEvent": "L1I_TLB" - }, - { - "ArchStdEvent": "L3D_CACHE_ALLOCATE" - }, - { - "ArchStdEvent": "L3D_CACHE_REFILL" - }, - { - "ArchStdEvent": "L3D_CACHE" - }, - { - "ArchStdEvent": "L2D_TLB_REFILL" - }, - { - "ArchStdEvent": "L2D_TLB" - }, - { - "ArchStdEvent": "DTLB_WALK" - }, - { - "ArchStdEvent": "ITLB_WALK" - }, - { - "ArchStdEvent": "LL_CACHE_RD" - }, - { - "ArchStdEvent": "LL_CACHE_MISS_RD" - }, - { - "ArchStdEvent": "L1D_CACHE_LMISS_RD" - }, - { - "ArchStdEvent": "L1D_CACHE_RD" - }, - { - "ArchStdEvent": "L1D_CACHE_WR" - }, - { - "ArchStdEvent": "L1D_CACHE_REFILL_RD" - }, - { - "ArchStdEvent": "L1D_CACHE_REFILL_WR" - }, - { - "ArchStdEvent": "L1D_CACHE_REFILL_INNER" - }, - { - "ArchStdEvent": "L1D_CACHE_REFILL_OUTER" - }, - { - "ArchStdEvent": "L1D_CACHE_WB_VICTIM" - }, - { - "ArchStdEvent": "L1D_CACHE_WB_CLEAN" - }, - { - "ArchStdEvent": "L1D_CACHE_INVAL" - }, - { - "ArchStdEvent": "L1D_TLB_REFILL_RD" - }, - { - "ArchStdEvent": "L1D_TLB_REFILL_WR" - }, - { - "ArchStdEvent": "L1D_TLB_RD" - }, - { - "ArchStdEvent": "L1D_TLB_WR" - }, - { - "ArchStdEvent": "L2D_CACHE_RD" - }, - { - "ArchStdEvent": "L2D_CACHE_WR" - }, - { - "ArchStdEvent": "L2D_CACHE_REFILL_RD" - }, - { - "ArchStdEvent": "L2D_CACHE_REFILL_WR" - }, - { - "ArchStdEvent": "L2D_CACHE_WB_VICTIM" - }, - { - "ArchStdEvent": "L2D_CACHE_WB_CLEAN" - }, - { - "ArchStdEvent": "L2D_CACHE_INVAL" - }, - { - "ArchStdEvent": "L2D_TLB_REFILL_RD" - }, - { - "ArchStdEvent": "L2D_TLB_REFILL_WR" - }, - { - "ArchStdEvent": "L2D_TLB_RD" - }, - { - "ArchStdEvent": "L2D_TLB_WR" - }, - { - "ArchStdEvent": "L3D_CACHE_RD" - }, - { - "ArchStdEvent": "L1I_CACHE_LMISS" - }, - { - "ArchStdEvent": "L2D_CACHE_LMISS_RD" - }, - { - "ArchStdEvent": "L3D_CACHE_LMISS_RD" - } -] diff --git a/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/exception.json b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/exception.json index 344a2d552ad5..4404b8e91690 100644 --- a/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/exception.json +++ b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/exception.json @@ -1,47 +1,62 @@ [ { - "ArchStdEvent": "EXC_TAKEN" + "ArchStdEvent": "EXC_TAKEN", + "PublicDescription": "Counts any taken architecturally visible exceptions such as IRQ, FIQ, SError, and other synchronous exceptions. Exceptions are counted whether or not they are taken locally." }, { - "ArchStdEvent": "MEMORY_ERROR" + "ArchStdEvent": "EXC_RETURN", + "PublicDescription": "Counts any architecturally executed exception return instructions. Eg: AArch64: ERET" }, { - "ArchStdEvent": "EXC_UNDEF" + "ArchStdEvent": "EXC_UNDEF", + "PublicDescription": "Counts the number of synchronous exceptions which are taken locally that are due to attempting to execute an instruction that is UNDEFINED. Attempting to execute instruction bit patterns that have not been allocated. Attempting to execute instructions when they are disabled. Attempting to execute instructions at an inappropriate Exception level. Attempting to execute an instruction when the value of PSTATE.IL is 1." }, { - "ArchStdEvent": "EXC_SVC" + "ArchStdEvent": "EXC_SVC", + "PublicDescription": "Counts SVC exceptions taken locally." }, { - "ArchStdEvent": "EXC_PABORT" + "ArchStdEvent": "EXC_PABORT", + "PublicDescription": "Counts synchronous exceptions that are taken locally and caused by Instruction Aborts." }, { - "ArchStdEvent": "EXC_DABORT" + "ArchStdEvent": "EXC_DABORT", + "PublicDescription": "Counts exceptions that are taken locally and are caused by data aborts or SErrors. Conditions that could cause those exceptions are attempting to read or write memory where the MMU generates a fault, attempting to read or write memory with a misaligned address, interrupts from the nSEI inputs and internally generated SErrors." }, { - "ArchStdEvent": "EXC_IRQ" + "ArchStdEvent": "EXC_IRQ", + "PublicDescription": "Counts IRQ exceptions including the virtual IRQs that are taken locally." }, { - "ArchStdEvent": "EXC_FIQ" + "ArchStdEvent": "EXC_FIQ", + "PublicDescription": "Counts FIQ exceptions including the virtual FIQs that are taken locally." }, { - "ArchStdEvent": "EXC_SMC" + "ArchStdEvent": "EXC_SMC", + "PublicDescription": "Counts SMC exceptions take to EL3." }, { - "ArchStdEvent": "EXC_HVC" + "ArchStdEvent": "EXC_HVC", + "PublicDescription": "Counts HVC exceptions taken to EL2." }, { - "ArchStdEvent": "EXC_TRAP_PABORT" + "ArchStdEvent": "EXC_TRAP_PABORT", + "PublicDescription": "Counts exceptions which are traps not taken locally and are caused by Instruction Aborts. For example, attempting to execute an instruction with a misaligned PC." }, { - "ArchStdEvent": "EXC_TRAP_DABORT" + "ArchStdEvent": "EXC_TRAP_DABORT", + "PublicDescription": "Counts exceptions which are traps not taken locally and are caused by Data Aborts or SError interrupts. Conditions that could cause those exceptions are:\n\n1. Attempting to read or write memory where the MMU generates a fault,\n2. Attempting to read or write memory with a misaligned address,\n3. Interrupts from the SEI input.\n4. internally generated SErrors." }, { - "ArchStdEvent": "EXC_TRAP_OTHER" + "ArchStdEvent": "EXC_TRAP_OTHER", + "PublicDescription": "Counts the number of synchronous trap exceptions which are not taken locally and are not SVC, SMC, HVC, data aborts, Instruction Aborts, or interrupts." }, { - "ArchStdEvent": "EXC_TRAP_IRQ" + "ArchStdEvent": "EXC_TRAP_IRQ", + "PublicDescription": "Counts IRQ exceptions including the virtual IRQs that are not taken locally." }, { - "ArchStdEvent": "EXC_TRAP_FIQ" + "ArchStdEvent": "EXC_TRAP_FIQ", + "PublicDescription": "Counts FIQs which are not taken locally but taken from EL0, EL1,\n or EL2 to EL3 (which would be the normal behavior for FIQs when not executing\n in EL3)." } ] diff --git a/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/fp_operation.json b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/fp_operation.json new file mode 100644 index 000000000000..a09e226e7138 --- /dev/null +++ b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/fp_operation.json @@ -0,0 +1,10 @@ +[ + { + "ArchStdEvent": "FP_SCALE_OPS_SPEC", + "PublicDescription": "Counts speculatively executed scalable single precision floating point operations." + }, + { + "ArchStdEvent": "FP_FIXED_OPS_SPEC", + "PublicDescription": "Counts speculatively executed non-scalable single precision floating point operations." + } +] diff --git a/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/general.json b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/general.json new file mode 100644 index 000000000000..428810f855b8 --- /dev/null +++ b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/general.json @@ -0,0 +1,10 @@ +[ + { + "ArchStdEvent": "CPU_CYCLES", + "PublicDescription": "Counts CPU clock cycles (not timer cycles). The clock measured by this event is defined as the physical clock driving the CPU logic." + }, + { + "ArchStdEvent": "CNT_CYCLES", + "PublicDescription": "Counts constant frequency cycles" + } +] diff --git a/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/instruction.json b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/instruction.json deleted file mode 100644 index e29b88fb7f24..000000000000 --- a/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/instruction.json +++ /dev/null @@ -1,119 +0,0 @@ -[ - { - "ArchStdEvent": "SW_INCR" - }, - { - "ArchStdEvent": "INST_RETIRED" - }, - { - "ArchStdEvent": "EXC_RETURN" - }, - { - "ArchStdEvent": "CID_WRITE_RETIRED" - }, - { - "ArchStdEvent": "INST_SPEC" - }, - { - "ArchStdEvent": "TTBR_WRITE_RETIRED" - }, - { - "ArchStdEvent": "BR_RETIRED" - }, - { - "ArchStdEvent": "BR_MIS_PRED_RETIRED" - }, - { - "ArchStdEvent": "OP_RETIRED" - }, - { - "ArchStdEvent": "OP_SPEC" - }, - { - "ArchStdEvent": "LDREX_SPEC" - }, - { - "ArchStdEvent": "STREX_PASS_SPEC" - }, - { - "ArchStdEvent": "STREX_FAIL_SPEC" - }, - { - "ArchStdEvent": "STREX_SPEC" - }, - { - "ArchStdEvent": "LD_SPEC" - }, - { - "ArchStdEvent": "ST_SPEC" - }, - { - "ArchStdEvent": "DP_SPEC" - }, - { - "ArchStdEvent": "ASE_SPEC" - }, - { - "ArchStdEvent": "VFP_SPEC" - }, - { - "ArchStdEvent": "PC_WRITE_SPEC" - }, - { - "ArchStdEvent": "CRYPTO_SPEC" - }, - { - "ArchStdEvent": "BR_IMMED_SPEC" - }, - { - "ArchStdEvent": "BR_RETURN_SPEC" - }, - { - "ArchStdEvent": "BR_INDIRECT_SPEC" - }, - { - "ArchStdEvent": "ISB_SPEC" - }, - { - "ArchStdEvent": "DSB_SPEC" - }, - { - "ArchStdEvent": "DMB_SPEC" - }, - { - "ArchStdEvent": "RC_LD_SPEC" - }, - { - "ArchStdEvent": "RC_ST_SPEC" - }, - { - "ArchStdEvent": "ASE_INST_SPEC" - }, - { - "ArchStdEvent": "SVE_INST_SPEC" - }, - { - "ArchStdEvent": "SVE_PRED_SPEC" - }, - { - "ArchStdEvent": "SVE_PRED_EMPTY_SPEC" - }, - { - "ArchStdEvent": "SVE_PRED_FULL_SPEC" - }, - { - "ArchStdEvent": "SVE_PRED_PARTIAL_SPEC" - }, - { - "ArchStdEvent": "SVE_LDFF_SPEC" - }, - { - "ArchStdEvent": "SVE_LDFF_FAULT_SPEC" - }, - { - "ArchStdEvent": "FP_SCALE_OPS_SPEC" - }, - { - "ArchStdEvent": "FP_FIXED_OPS_SPEC" - } -] diff --git a/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/l1d_cache.json b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/l1d_cache.json new file mode 100644 index 000000000000..ed83e1c5affe --- /dev/null +++ b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/l1d_cache.json @@ -0,0 +1,54 @@ +[ + { + "ArchStdEvent": "L1D_CACHE_REFILL", + "PublicDescription": "Counts level 1 data cache refills caused by speculatively executed load or store operations that missed in the level 1 data cache. This event only counts one event per cache line. This event does not count cache line allocations from preload instructions or from hardware cache prefetching." + }, + { + "ArchStdEvent": "L1D_CACHE", + "PublicDescription": "Counts level 1 data cache accesses from any load/store operations. Atomic operations that resolve in the CPUs caches (near atomic operations) counts as both a write access and read access. Each access to a cache line is counted including the multiple accesses caused by single instructions such as LDM or STM. Each access to other level 1 data or unified memory structures, for example refill buffers, write buffers, and write-back buffers, are also counted." + }, + { + "ArchStdEvent": "L1D_CACHE_WB", + "PublicDescription": "Counts write-backs of dirty data from the L1 data cache to the L2 cache. This occurs when either a dirty cache line is evicted from L1 data cache and allocated in the L2 cache or dirty data is written to the L2 and possibly to the next level of cache. This event counts both victim cache line evictions and cache write-backs from snoops or cache maintenance operations. The following cache operations are not counted:\n\n1. Invalidations which do not result in data being transferred out of the L1 (such as evictions of clean data),\n2. Full line writes which write to L2 without writing L1, such as write streaming mode." + }, + { + "ArchStdEvent": "L1D_CACHE_LMISS_RD", + "PublicDescription": "Counts cache line refills into the level 1 data cache from any memory read operations, that incurred additional latency." + }, + { + "ArchStdEvent": "L1D_CACHE_RD", + "PublicDescription": "Counts level 1 data cache accesses from any load operation. Atomic load operations that resolve in the CPUs caches counts as both a write access and read access." + }, + { + "ArchStdEvent": "L1D_CACHE_WR", + "PublicDescription": "Counts level 1 data cache accesses generated by store operations. This event also counts accesses caused by a DC ZVA (data cache zero, specified by virtual address) instruction. Near atomic operations that resolve in the CPUs caches count as a write access and read access." + }, + { + "ArchStdEvent": "L1D_CACHE_REFILL_RD", + "PublicDescription": "Counts level 1 data cache refills caused by speculatively executed load instructions where the memory read operation misses in the level 1 data cache. This event only counts one event per cache line." + }, + { + "ArchStdEvent": "L1D_CACHE_REFILL_WR", + "PublicDescription": "Counts level 1 data cache refills caused by speculatively executed store instructions where the memory write operation misses in the level 1 data cache. This event only counts one event per cache line." + }, + { + "ArchStdEvent": "L1D_CACHE_REFILL_INNER", + "PublicDescription": "Counts level 1 data cache refills where the cache line data came from caches inside the immediate cluster of the core." + }, + { + "ArchStdEvent": "L1D_CACHE_REFILL_OUTER", + "PublicDescription": "Counts level 1 data cache refills for which the cache line data came from outside the immediate cluster of the core, like an SLC in the system interconnect or DRAM." + }, + { + "ArchStdEvent": "L1D_CACHE_WB_VICTIM", + "PublicDescription": "Counts dirty cache line evictions from the level 1 data cache caused by a new cache line allocation. This event does not count evictions caused by cache maintenance operations." + }, + { + "ArchStdEvent": "L1D_CACHE_WB_CLEAN", + "PublicDescription": "Counts write-backs from the level 1 data cache that are a result of a coherency operation made by another CPU. Event count includes cache maintenance operations." + }, + { + "ArchStdEvent": "L1D_CACHE_INVAL", + "PublicDescription": "Counts each explicit invalidation of a cache line in the level 1 data cache caused by:\n\n- Cache Maintenance Operations (CMO) that operate by a virtual address.\n- Broadcast cache coherency operations from another CPU in the system.\n\nThis event does not count for the following conditions:\n\n1. A cache refill invalidates a cache line.\n2. A CMO which is executed on that CPU and invalidates a cache line specified by set/way.\n\nNote that CMOs that operate by set/way cannot be broadcast from one CPU to another." + } +] diff --git a/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/l1i_cache.json b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/l1i_cache.json new file mode 100644 index 000000000000..633f1030359d --- /dev/null +++ b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/l1i_cache.json @@ -0,0 +1,14 @@ +[ + { + "ArchStdEvent": "L1I_CACHE_REFILL", + "PublicDescription": "Counts cache line refills in the level 1 instruction cache caused by a missed instruction fetch. Instruction fetches may include accessing multiple instructions, but the single cache line allocation is counted once." + }, + { + "ArchStdEvent": "L1I_CACHE", + "PublicDescription": "Counts instruction fetches which access the level 1 instruction cache. Instruction cache accesses caused by cache maintenance operations are not counted." + }, + { + "ArchStdEvent": "L1I_CACHE_LMISS", + "PublicDescription": "Counts cache line refills into the level 1 instruction cache, that incurred additional latency." + } +] diff --git a/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/l2_cache.json b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/l2_cache.json new file mode 100644 index 000000000000..0e31d0daf88b --- /dev/null +++ b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/l2_cache.json @@ -0,0 +1,50 @@ +[ + { + "ArchStdEvent": "L2D_CACHE", + "PublicDescription": "Counts level 2 cache accesses. level 2 cache is a unified cache for data and instruction accesses. Accesses are for misses in the first level caches or translation resolutions due to accesses. This event also counts write back of dirty data from level 1 data cache to the L2 cache." + }, + { + "ArchStdEvent": "L2D_CACHE_REFILL", + "PublicDescription": "Counts cache line refills into the level 2 cache. level 2 cache is a unified cache for data and instruction accesses. Accesses are for misses in the level 1 caches or translation resolutions due to accesses." + }, + { + "ArchStdEvent": "L2D_CACHE_WB", + "PublicDescription": "Counts write-backs of data from the L2 cache to outside the CPU. This includes snoops to the L2 (from other CPUs) which return data even if the snoops cause an invalidation. L2 cache line invalidations which do not write data outside the CPU and snoops which return data from an L1 cache are not counted. Data would not be written outside the cache when invalidating a clean cache line." + }, + { + "ArchStdEvent": "L2D_CACHE_ALLOCATE", + "PublicDescription": "TBD" + }, + { + "ArchStdEvent": "L2D_CACHE_RD", + "PublicDescription": "Counts level 2 cache accesses due to memory read operations. level 2 cache is a unified cache for data and instruction accesses, accesses are for misses in the level 1 caches or translation resolutions due to accesses." + }, + { + "ArchStdEvent": "L2D_CACHE_WR", + "PublicDescription": "Counts level 2 cache accesses due to memory write operations. level 2 cache is a unified cache for data and instruction accesses, accesses are for misses in the level 1 caches or translation resolutions due to accesses." + }, + { + "ArchStdEvent": "L2D_CACHE_REFILL_RD", + "PublicDescription": "Counts refills for memory accesses due to memory read operation counted by L2D_CACHE_RD. level 2 cache is a unified cache for data and instruction accesses, accesses are for misses in the level 1 caches or translation resolutions due to accesses." + }, + { + "ArchStdEvent": "L2D_CACHE_REFILL_WR", + "PublicDescription": "Counts refills for memory accesses due to memory write operation counted by L2D_CACHE_WR. level 2 cache is a unified cache for data and instruction accesses, accesses are for misses in the level 1 caches or translation resolutions due to accesses." + }, + { + "ArchStdEvent": "L2D_CACHE_WB_VICTIM", + "PublicDescription": "Counts evictions from the level 2 cache because of a line being allocated into the L2 cache." + }, + { + "ArchStdEvent": "L2D_CACHE_WB_CLEAN", + "PublicDescription": "Counts write-backs from the level 2 cache that are a result of either:\n\n1. Cache maintenance operations,\n\n2. Snoop responses or,\n\n3. Direct cache transfers to another CPU due to a forwarding snoop request." + }, + { + "ArchStdEvent": "L2D_CACHE_INVAL", + "PublicDescription": "Counts each explicit invalidation of a cache line in the level 2 cache by cache maintenance operations that operate by a virtual address, or by external coherency operations. This event does not count if either:\n\n1. A cache refill invalidates a cache line or,\n2. A Cache Maintenance Operation (CMO), which invalidates a cache line specified by set/way, is executed on that CPU.\n\nCMOs that operate by set/way cannot be broadcast from one CPU to another." + }, + { + "ArchStdEvent": "L2D_CACHE_LMISS_RD", + "PublicDescription": "Counts cache line refills into the level 2 unified cache from any memory read operations that incurred additional latency." + } +] diff --git a/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/l3_cache.json b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/l3_cache.json new file mode 100644 index 000000000000..45bfba532df7 --- /dev/null +++ b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/l3_cache.json @@ -0,0 +1,22 @@ +[ + { + "ArchStdEvent": "L3D_CACHE_ALLOCATE", + "PublicDescription": "Counts level 3 cache line allocates that do not fetch data from outside the level 3 data or unified cache. For example, allocates due to streaming stores." + }, + { + "ArchStdEvent": "L3D_CACHE_REFILL", + "PublicDescription": "Counts level 3 accesses that receive data from outside the L3 cache." + }, + { + "ArchStdEvent": "L3D_CACHE", + "PublicDescription": "Counts level 3 cache accesses. level 3 cache is a unified cache for data and instruction accesses. Accesses are for misses in the lower level caches or translation resolutions due to accesses." + }, + { + "ArchStdEvent": "L3D_CACHE_RD", + "PublicDescription": "TBD" + }, + { + "ArchStdEvent": "L3D_CACHE_LMISS_RD", + "PublicDescription": "Counts any cache line refill into the level 3 cache from memory read operations that incurred additional latency." + } +] diff --git a/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/ll_cache.json b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/ll_cache.json new file mode 100644 index 000000000000..bb712d57d58a --- /dev/null +++ b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/ll_cache.json @@ -0,0 +1,10 @@ +[ + { + "ArchStdEvent": "LL_CACHE_RD", + "PublicDescription": "Counts read transactions that were returned from outside the core cluster. This event counts when the system register CPUECTLR.EXTLLC bit is set. This event counts read transactions returned from outside the core if those transactions are either hit in the system level cache or missed in the SLC and are returned from any other external sources." + }, + { + "ArchStdEvent": "LL_CACHE_MISS_RD", + "PublicDescription": "Counts read transactions that were returned from outside the core cluster but missed in the system level cache. This event counts when the system register CPUECTLR.EXTLLC bit is set. This event counts read transactions returned from outside the core if those transactions are missed in the System level Cache. The data source of the transaction is indicated by a field in the CHI transaction returning to the CPU. This event does not count reads caused by cache maintenance operations." + } +] diff --git a/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/memory.json b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/memory.json index 5aff6e93c1ad..9041f6e0befb 100644 --- a/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/memory.json +++ b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/memory.json @@ -1,23 +1,22 @@ [ { - "ArchStdEvent": "MEM_ACCESS" + "ArchStdEvent": "MEM_ACCESS", + "PublicDescription": "Counts memory accesses issued by the CPU load store unit, where those accesses are issued due to load or store operations. This event counts memory accesses no matter whether the data is received from any level of cache hierarchy or external memory. If memory accesses are broken up into smaller transactions than what were specified in the load or store instructions, then the event counts those smaller memory transactions." }, { - "ArchStdEvent": "REMOTE_ACCESS" + "ArchStdEvent": "MEMORY_ERROR", + "PublicDescription": "Counts any detected correctable or uncorrectable physical memory errors (ECC or parity) in protected CPUs RAMs. On the core, this event counts errors in the caches (including data and tag rams). Any detected memory error (from either a speculative and abandoned access, or an architecturally executed access) is counted. Note that errors are only detected when the actual protected memory is accessed by an operation." }, { - "ArchStdEvent": "MEM_ACCESS_RD" + "ArchStdEvent": "REMOTE_ACCESS", + "PublicDescription": "Counts accesses to another chip, which is implemented as a different CMN mesh in the system. If the CHI bus response back to the core indicates that the data source is from another chip (mesh), then the counter is updated. If no data is returned, even if the system snoops another chip/mesh, then the counter is not updated." }, { - "ArchStdEvent": "MEM_ACCESS_WR" + "ArchStdEvent": "MEM_ACCESS_RD", + "PublicDescription": "Counts memory accesses issued by the CPU due to load operations. The event counts any memory load access, no matter whether the data is received from any level of cache hierarchy or external memory. The event also counts atomic load operations. If memory accesses are broken up by the load/store unit into smaller transactions that are issued by the bus interface, then the event counts those smaller transactions." }, { - "ArchStdEvent": "UNALIGNED_LD_SPEC" - }, - { - "ArchStdEvent": "UNALIGNED_ST_SPEC" - }, - { - "ArchStdEvent": "UNALIGNED_LDST_SPEC" + "ArchStdEvent": "MEM_ACCESS_WR", + "PublicDescription": "Counts memory accesses issued by the CPU due to store operations. The event counts any memory store access, no matter whether the data is located in any level of cache or external memory. The event also counts atomic load and store operations. If memory accesses are broken up by the load/store unit into smaller transactions that are issued by the bus interface, then the event counts those smaller transactions." } ] diff --git a/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/pipeline.json b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/pipeline.json deleted file mode 100644 index f9fae15f7555..000000000000 --- a/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/pipeline.json +++ /dev/null @@ -1,23 +0,0 @@ -[ - { - "ArchStdEvent": "STALL_FRONTEND" - }, - { - "ArchStdEvent": "STALL_BACKEND" - }, - { - "ArchStdEvent": "STALL" - }, - { - "ArchStdEvent": "STALL_SLOT_BACKEND" - }, - { - "ArchStdEvent": "STALL_SLOT_FRONTEND" - }, - { - "ArchStdEvent": "STALL_SLOT" - }, - { - "ArchStdEvent": "STALL_BACKEND_MEM" - } -] diff --git a/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/retired.json b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/retired.json new file mode 100644 index 000000000000..f297b049b62f --- /dev/null +++ b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/retired.json @@ -0,0 +1,30 @@ +[ + { + "ArchStdEvent": "SW_INCR", + "PublicDescription": "Counts software writes to the PMSWINC_EL0 (software PMU increment) register. The PMSWINC_EL0 register is a manually updated counter for use by application software.\n\nThis event could be used to measure any user program event, such as accesses to a particular data structure (by writing to the PMSWINC_EL0 register each time the data structure is accessed).\n\nTo use the PMSWINC_EL0 register and event, developers must insert instructions that write to the PMSWINC_EL0 register into the source code.\n\nSince the SW_INCR event records writes to the PMSWINC_EL0 register, there is no need to do a read/increment/write sequence to the PMSWINC_EL0 register." + }, + { + "ArchStdEvent": "INST_RETIRED", + "PublicDescription": "Counts instructions that have been architecturally executed." + }, + { + "ArchStdEvent": "CID_WRITE_RETIRED", + "PublicDescription": "Counts architecturally executed writes to the CONTEXTIDR register, which usually contain the kernel PID and can be output with hardware trace." + }, + { + "ArchStdEvent": "TTBR_WRITE_RETIRED", + "PublicDescription": "Counts architectural writes to TTBR0/1_EL1. If virtualization host extensions are enabled (by setting the HCR_EL2.E2H bit to 1), then accesses to TTBR0/1_EL1 that are redirected to TTBR0/1_EL2, or accesses to TTBR0/1_EL12, are counted. TTBRn registers are typically updated when the kernel is swapping user-space threads or applications." + }, + { + "ArchStdEvent": "BR_RETIRED", + "PublicDescription": "Counts architecturally executed branches, whether the branch is taken or not. Instructions that explicitly write to the PC are also counted." + }, + { + "ArchStdEvent": "BR_MIS_PRED_RETIRED", + "PublicDescription": "Counts branches counted by BR_RETIRED which were mispredicted and caused a pipeline flush." + }, + { + "ArchStdEvent": "OP_RETIRED", + "PublicDescription": "Counts micro-operations that are architecturally executed. This is a count of number of micro-operations retired from the commit queue in a single cycle." + } +] diff --git a/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/spe.json b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/spe.json new file mode 100644 index 000000000000..5de8b0f3a440 --- /dev/null +++ b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/spe.json @@ -0,0 +1,18 @@ +[ + { + "ArchStdEvent": "SAMPLE_POP", + "PublicDescription": "Counts statistical profiling sample population, the count of all operations that could be sampled but may or may not be chosen for sampling." + }, + { + "ArchStdEvent": "SAMPLE_FEED", + "PublicDescription": "Counts statistical profiling samples taken for sampling." + }, + { + "ArchStdEvent": "SAMPLE_FILTRATE", + "PublicDescription": "Counts statistical profiling samples taken which are not removed by filtering." + }, + { + "ArchStdEvent": "SAMPLE_COLLISION", + "PublicDescription": "Counts statistical profiling samples that have collided with a previous sample and so therefore not taken." + } +] diff --git a/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/spec_operation.json b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/spec_operation.json new file mode 100644 index 000000000000..1af961f8a6c8 --- /dev/null +++ b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/spec_operation.json @@ -0,0 +1,110 @@ +[ + { + "ArchStdEvent": "BR_MIS_PRED", + "PublicDescription": "Counts branches which are speculatively executed and mispredicted." + }, + { + "ArchStdEvent": "BR_PRED", + "PublicDescription": "Counts branches speculatively executed and were predicted right." + }, + { + "ArchStdEvent": "INST_SPEC", + "PublicDescription": "Counts operations that have been speculatively executed." + }, + { + "ArchStdEvent": "OP_SPEC", + "PublicDescription": "Counts micro-operations speculatively executed. This is the count of the number of micro-operations dispatched in a cycle." + }, + { + "ArchStdEvent": "UNALIGNED_LD_SPEC", + "PublicDescription": "Counts unaligned memory read operations issued by the CPU. This event counts unaligned accesses (as defined by the actual instruction), even if they are subsequently issued as multiple aligned accesses. The event does not count preload operations (PLD, PLI)." + }, + { + "ArchStdEvent": "UNALIGNED_ST_SPEC", + "PublicDescription": "Counts unaligned memory write operations issued by the CPU. This event counts unaligned accesses (as defined by the actual instruction), even if they are subsequently issued as multiple aligned accesses." + }, + { + "ArchStdEvent": "UNALIGNED_LDST_SPEC", + "PublicDescription": "Counts unaligned memory operations issued by the CPU. This event counts unaligned accesses (as defined by the actual instruction), even if they are subsequently issued as multiple aligned accesses." + }, + { + "ArchStdEvent": "LDREX_SPEC", + "PublicDescription": "Counts Load-Exclusive operations that have been speculatively executed. Eg: LDREX, LDX" + }, + { + "ArchStdEvent": "STREX_PASS_SPEC", + "PublicDescription": "Counts store-exclusive operations that have been speculatively executed and have successfully completed the store operation." + }, + { + "ArchStdEvent": "STREX_FAIL_SPEC", + "PublicDescription": "Counts store-exclusive operations that have been speculatively executed and have not successfully completed the store operation." + }, + { + "ArchStdEvent": "STREX_SPEC", + "PublicDescription": "Counts store-exclusive operations that have been speculatively executed." + }, + { + "ArchStdEvent": "LD_SPEC", + "PublicDescription": "Counts speculatively executed load operations including Single Instruction Multiple Data (SIMD) load operations." + }, + { + "ArchStdEvent": "ST_SPEC", + "PublicDescription": "Counts speculatively executed store operations including Single Instruction Multiple Data (SIMD) store operations." + }, + { + "ArchStdEvent": "DP_SPEC", + "PublicDescription": "Counts speculatively executed logical or arithmetic instructions such as MOV/MVN operations." + }, + { + "ArchStdEvent": "ASE_SPEC", + "PublicDescription": "Counts speculatively executed Advanced SIMD operations excluding load, store and move micro-operations that move data to or from SIMD (vector) registers." + }, + { + "ArchStdEvent": "VFP_SPEC", + "PublicDescription": "Counts speculatively executed floating point operations. This event does not count operations that move data to or from floating point (vector) registers." + }, + { + "ArchStdEvent": "PC_WRITE_SPEC", + "PublicDescription": "Counts speculatively executed operations which cause software changes of the PC. Those operations include all taken branch operations." + }, + { + "ArchStdEvent": "CRYPTO_SPEC", + "PublicDescription": "Counts speculatively executed cryptographic operations except for PMULL and VMULL operations." + }, + { + "ArchStdEvent": "BR_IMMED_SPEC", + "PublicDescription": "Counts immediate branch operations which are speculatively executed." + }, + { + "ArchStdEvent": "BR_RETURN_SPEC", + "PublicDescription": "Counts procedure return operations (RET) which are speculatively executed." + }, + { + "ArchStdEvent": "BR_INDIRECT_SPEC", + "PublicDescription": "Counts indirect branch operations including procedure returns, which are speculatively executed. This includes operations that force a software change of the PC, other than exception-generating operations. Eg: BR Xn, RET" + }, + { + "ArchStdEvent": "ISB_SPEC", + "PublicDescription": "Counts ISB operations that are executed." + }, + { + "ArchStdEvent": "DSB_SPEC", + "PublicDescription": "Counts DSB operations that are speculatively issued to Load/Store unit in the CPU." + }, + { + "ArchStdEvent": "DMB_SPEC", + "PublicDescription": "Counts DMB operations that are speculatively issued to the Load/Store unit in the CPU. This event does not count implied barriers from load acquire/store release operations." + }, + { + "ArchStdEvent": "RC_LD_SPEC", + "PublicDescription": "Counts any load acquire operations that are speculatively executed. Eg: LDAR, LDARH, LDARB" + }, + { + "ArchStdEvent": "RC_ST_SPEC", + "PublicDescription": "Counts any store release operations that are speculatively executed. Eg: STLR, STLRH, STLRB'" + }, + { + "ArchStdEvent": "ASE_INST_SPEC", + "PublicDescription": "Counts speculatively executed Advanced SIMD operations." + } +] diff --git a/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/stall.json b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/stall.json new file mode 100644 index 000000000000..bbbebc805034 --- /dev/null +++ b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/stall.json @@ -0,0 +1,30 @@ +[ + { + "ArchStdEvent": "STALL_FRONTEND", + "PublicDescription": "Counts cycles when frontend could not send any micro-operations to the rename stage because of frontend resource stalls caused by fetch memory latency or branch prediction flow stalls. All the frontend slots were empty during the cycle when this event counts." + }, + { + "ArchStdEvent": "STALL_BACKEND", + "PublicDescription": "Counts cycles whenever the rename unit is unable to send any micro-operations to the backend of the pipeline because of backend resource constraints. Backend resource constraints can include issue stage fullness, execution stage fullness, or other internal pipeline resource fullness. All the backend slots were empty during the cycle when this event counts." + }, + { + "ArchStdEvent": "STALL", + "PublicDescription": "Counts cycles when no operations are sent to the rename unit from the frontend or from the rename unit to the backend for any reason (either frontend or backend stall)." + }, + { + "ArchStdEvent": "STALL_SLOT_BACKEND", + "PublicDescription": "Counts slots per cycle in which no operations are sent from the rename unit to the backend due to backend resource constraints." + }, + { + "ArchStdEvent": "STALL_SLOT_FRONTEND", + "PublicDescription": "Counts slots per cycle in which no operations are sent to the rename unit from the frontend due to frontend resource constraints." + }, + { + "ArchStdEvent": "STALL_SLOT", + "PublicDescription": "Counts slots per cycle in which no operations are sent to the rename unit from the frontend or from the rename unit to the backend for any reason (either frontend or backend stall)." + }, + { + "ArchStdEvent": "STALL_BACKEND_MEM", + "PublicDescription": "Counts cycles when the backend is stalled because there is a pending demand load request in progress in the last level core cache." + } +] diff --git a/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/sve.json b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/sve.json new file mode 100644 index 000000000000..5137e2075a76 --- /dev/null +++ b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/sve.json @@ -0,0 +1,30 @@ +[ + { + "ArchStdEvent": "SVE_INST_SPEC", + "PublicDescription": "Counts speculatively executed operations that are SVE operations." + }, + { + "ArchStdEvent": "SVE_PRED_SPEC", + "PublicDescription": "Counts speculatively executed predicated SVE operations." + }, + { + "ArchStdEvent": "SVE_PRED_EMPTY_SPEC", + "PublicDescription": "Counts speculatively executed predicated SVE operations with no active predicate elements." + }, + { + "ArchStdEvent": "SVE_PRED_FULL_SPEC", + "PublicDescription": "Counts speculatively executed predicated SVE operations with all predicate elements active." + }, + { + "ArchStdEvent": "SVE_PRED_PARTIAL_SPEC", + "PublicDescription": "Counts speculatively executed predicated SVE operations with at least one but not all active predicate elements." + }, + { + "ArchStdEvent": "SVE_LDFF_SPEC", + "PublicDescription": "Counts speculatively executed SVE first fault or non-fault load operations." + }, + { + "ArchStdEvent": "SVE_LDFF_FAULT_SPEC", + "PublicDescription": "Counts speculatively executed SVE first fault or non-fault load operations that clear at least one bit in the FFR." + } +] diff --git a/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/tlb.json b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/tlb.json new file mode 100644 index 000000000000..b550af1831f5 --- /dev/null +++ b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/tlb.json @@ -0,0 +1,66 @@ +[ + { + "ArchStdEvent": "L1I_TLB_REFILL", + "PublicDescription": "Counts level 1 instruction TLB refills from any Instruction fetch. If there are multiple misses in the TLB that are resolved by the refill, then this event only counts once. This event will not count if the translation table walk results in a fault (such as a translation or access fault), since there is no new translation created for the TLB." + }, + { + "ArchStdEvent": "L1D_TLB_REFILL", + "PublicDescription": "Counts level 1 data TLB accesses that resulted in TLB refills. If there are multiple misses in the TLB that are resolved by the refill, then this event only counts once. This event counts for refills caused by preload instructions or hardware prefetch accesses. This event counts regardless of whether the miss hits in L2 or results in a translation table walk. This event will not count if the translation table walk results in a fault (such as a translation or access fault), since there is no new translation created for the TLB. This event will not count on an access from an AT(address translation) instruction." + }, + { + "ArchStdEvent": "L1D_TLB", + "PublicDescription": "Counts level 1 data TLB accesses caused by any memory load or store operation. Note that load or store instructions can be broken up into multiple memory operations. This event does not count TLB maintenance operations." + }, + { + "ArchStdEvent": "L1I_TLB", + "PublicDescription": "Counts level 1 instruction TLB accesses, whether the access hits or misses in the TLB. This event counts both demand accesses and prefetch or preload generated accesses." + }, + { + "ArchStdEvent": "L2D_TLB_REFILL", + "PublicDescription": "Counts level 2 TLB refills caused by memory operations from both data and instruction fetch, except for those caused by TLB maintenance operations and hardware prefetches." + }, + { + "ArchStdEvent": "L2D_TLB", + "PublicDescription": "Counts level 2 TLB accesses except those caused by TLB maintenance operations." + }, + { + "ArchStdEvent": "DTLB_WALK", + "PublicDescription": "Counts data memory translation table walks caused by a miss in the L2 TLB driven by a memory access. Note that partial translations that also cause a table walk are counted. This event does not count table walks caused by TLB maintenance operations." + }, + { + "ArchStdEvent": "ITLB_WALK", + "PublicDescription": "Counts instruction memory translation table walks caused by a miss in the L2 TLB driven by a memory access. Partial translations that also cause a table walk are counted. This event does not count table walks caused by TLB maintenance operations." + }, + { + "ArchStdEvent": "L1D_TLB_REFILL_RD", + "PublicDescription": "Counts level 1 data TLB refills caused by memory read operations. If there are multiple misses in the TLB that are resolved by the refill, then this event only counts once. This event counts for refills caused by preload instructions or hardware prefetch accesses. This event counts regardless of whether the miss hits in L2 or results in a translation table walk. This event will not count if the translation table walk results in a fault (such as a translation or access fault), since there is no new translation created for the TLB. This event will not count on an access from an Address Translation (AT) instruction." + }, + { + "ArchStdEvent": "L1D_TLB_REFILL_WR", + "PublicDescription": "Counts level 1 data TLB refills caused by data side memory write operations. If there are multiple misses in the TLB that are resolved by the refill, then this event only counts once. This event counts for refills caused by preload instructions or hardware prefetch accesses. This event counts regardless of whether the miss hits in L2 or results in a translation table walk. This event will not count if the table walk results in a fault (such as a translation or access fault), since there is no new translation created for the TLB. This event will not count with an access from an Address Translation (AT) instruction." + }, + { + "ArchStdEvent": "L1D_TLB_RD", + "PublicDescription": "Counts level 1 data TLB accesses caused by memory read operations. This event counts whether the access hits or misses in the TLB. This event does not count TLB maintenance operations." + }, + { + "ArchStdEvent": "L1D_TLB_WR", + "PublicDescription": "Counts any L1 data side TLB accesses caused by memory write operations. This event counts whether the access hits or misses in the TLB. This event does not count TLB maintenance operations." + }, + { + "ArchStdEvent": "L2D_TLB_REFILL_RD", + "PublicDescription": "Counts level 2 TLB refills caused by memory read operations from both data and instruction fetch except for those caused by TLB maintenance operations or hardware prefetches." + }, + { + "ArchStdEvent": "L2D_TLB_REFILL_WR", + "PublicDescription": "Counts level 2 TLB refills caused by memory write operations from both data and instruction fetch except for those caused by TLB maintenance operations." + }, + { + "ArchStdEvent": "L2D_TLB_RD", + "PublicDescription": "Counts level 2 TLB accesses caused by memory read operations from both data and instruction fetch except for those caused by TLB maintenance operations." + }, + { + "ArchStdEvent": "L2D_TLB_WR", + "PublicDescription": "Counts level 2 TLB accesses caused by memory write operations from both data and instruction fetch except for those caused by TLB maintenance operations." + } +] From 5cdb51baf34e3d531f726c47a7ae7db80e2edef7 Mon Sep 17 00:00:00 2001 From: James Clark Date: Thu, 31 Aug 2023 17:15:32 +0100 Subject: [PATCH 071/475] perf vendor events arm64: Add V1 metrics using Arm telemetry repo Metrics for V1 weren't previously included in the Perf Jsons, so add them using the telemetry source [1]. After generation any parts identical to the default metrics in sbsa.json were manually removed. [1]: https://gitlab.arm.com/telemetry-solution/telemetry-solution/-/blob/main/data/pmu/cpu/neoverse/neoverse-v1.json Signed-off-by: James Clark Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ian Rogers Cc: Ingo Molnar Cc: Jiri Olsa Cc: John Garry Cc: Leo Yan Cc: Mark Rutland Cc: Mike Leach Cc: Namhyung Kim Cc: Nick Forrington Cc: Peter Zijlstra Cc: Will Deacon Cc: linux-arm-kernel@lists.infradead.org Link: https://lore.kernel.org/r/20230831161618.134738-3-james.clark@arm.com Signed-off-by: Arnaldo Carvalho de Melo --- .../arch/arm64/arm/neoverse-v1/metrics.json | 233 ++++++++++++++++++ 1 file changed, 233 insertions(+) create mode 100644 tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/metrics.json diff --git a/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/metrics.json b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/metrics.json new file mode 100644 index 000000000000..3fd8fefec46a --- /dev/null +++ b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/metrics.json @@ -0,0 +1,233 @@ +[ + { + "ArchStdEvent": "backend_bound" + }, + { + "MetricName": "backend_stalled_cycles", + "MetricExpr": "((STALL_BACKEND / CPU_CYCLES) * 100)", + "BriefDescription": "This metric is the percentage of cycles that were stalled due to resource constraints in the backend unit of the processor.", + "MetricGroup": "Cycle_Accounting", + "ScaleUnit": "1percent of cycles" + }, + { + "ArchStdEvent": "bad_speculation", + "MetricExpr": "(100 * (((1 - (OP_RETIRED / OP_SPEC)) * (1 - (STALL_SLOT / (CPU_CYCLES * 8)))) + ((BR_MIS_PRED * 4) / CPU_CYCLES)))" + }, + { + "MetricName": "branch_misprediction_ratio", + "MetricExpr": "(BR_MIS_PRED_RETIRED / BR_RETIRED)", + "BriefDescription": "This metric measures the ratio of branches mispredicted to the total number of branches architecturally executed. This gives an indication of the effectiveness of the branch prediction unit.", + "MetricGroup": "Miss_Ratio;Branch_Effectiveness", + "ScaleUnit": "1per branch" + }, + { + "MetricName": "branch_mpki", + "MetricExpr": "((BR_MIS_PRED_RETIRED / INST_RETIRED) * 1000)", + "BriefDescription": "This metric measures the number of branch mispredictions per thousand instructions executed.", + "MetricGroup": "MPKI;Branch_Effectiveness", + "ScaleUnit": "1MPKI" + }, + { + "MetricName": "branch_percentage", + "MetricExpr": "(((BR_IMMED_SPEC + BR_INDIRECT_SPEC) / INST_SPEC) * 100)", + "BriefDescription": "This metric measures branch operations as a percentage of operations speculatively executed.", + "MetricGroup": "Operation_Mix", + "ScaleUnit": "1percent of operations" + }, + { + "MetricName": "crypto_percentage", + "MetricExpr": "((CRYPTO_SPEC / INST_SPEC) * 100)", + "BriefDescription": "This metric measures crypto operations as a percentage of operations speculatively executed.", + "MetricGroup": "Operation_Mix", + "ScaleUnit": "1percent of operations" + }, + { + "MetricName": "dtlb_mpki", + "MetricExpr": "((DTLB_WALK / INST_RETIRED) * 1000)", + "BriefDescription": "This metric measures the number of data TLB Walks per thousand instructions executed.", + "MetricGroup": "MPKI;DTLB_Effectiveness", + "ScaleUnit": "1MPKI" + }, + { + "MetricName": "dtlb_walk_ratio", + "MetricExpr": "(DTLB_WALK / L1D_TLB)", + "BriefDescription": "This metric measures the ratio of data TLB Walks to the total number of data TLB accesses. This gives an indication of the effectiveness of the data TLB accesses.", + "MetricGroup": "Miss_Ratio;DTLB_Effectiveness", + "ScaleUnit": "1per TLB access" + }, + { + "ArchStdEvent": "frontend_bound", + "MetricExpr": "(100 * ((STALL_SLOT_FRONTEND / (CPU_CYCLES * 8)) - ((BR_MIS_PRED * 4) / CPU_CYCLES)))" + }, + { + "MetricName": "frontend_stalled_cycles", + "MetricExpr": "((STALL_FRONTEND / CPU_CYCLES) * 100)", + "BriefDescription": "This metric is the percentage of cycles that were stalled due to resource constraints in the frontend unit of the processor.", + "MetricGroup": "Cycle_Accounting", + "ScaleUnit": "1percent of cycles" + }, + { + "MetricName": "integer_dp_percentage", + "MetricExpr": "((DP_SPEC / INST_SPEC) * 100)", + "BriefDescription": "This metric measures scalar integer operations as a percentage of operations speculatively executed.", + "MetricGroup": "Operation_Mix", + "ScaleUnit": "1percent of operations" + }, + { + "MetricName": "ipc", + "MetricExpr": "(INST_RETIRED / CPU_CYCLES)", + "BriefDescription": "This metric measures the number of instructions retired per cycle.", + "MetricGroup": "General", + "ScaleUnit": "1per cycle" + }, + { + "MetricName": "itlb_mpki", + "MetricExpr": "((ITLB_WALK / INST_RETIRED) * 1000)", + "BriefDescription": "This metric measures the number of instruction TLB Walks per thousand instructions executed.", + "MetricGroup": "MPKI;ITLB_Effectiveness", + "ScaleUnit": "1MPKI" + }, + { + "MetricName": "itlb_walk_ratio", + "MetricExpr": "(ITLB_WALK / L1I_TLB)", + "BriefDescription": "This metric measures the ratio of instruction TLB Walks to the total number of instruction TLB accesses. This gives an indication of the effectiveness of the instruction TLB accesses.", + "MetricGroup": "Miss_Ratio;ITLB_Effectiveness", + "ScaleUnit": "1per TLB access" + }, + { + "MetricName": "l1d_cache_miss_ratio", + "MetricExpr": "(L1D_CACHE_REFILL / L1D_CACHE)", + "BriefDescription": "This metric measures the ratio of level 1 data cache accesses missed to the total number of level 1 data cache accesses. This gives an indication of the effectiveness of the level 1 data cache.", + "MetricGroup": "Miss_Ratio;L1D_Cache_Effectiveness", + "ScaleUnit": "1per cache access" + }, + { + "MetricName": "l1d_cache_mpki", + "MetricExpr": "((L1D_CACHE_REFILL / INST_RETIRED) * 1000)", + "BriefDescription": "This metric measures the number of level 1 data cache accesses missed per thousand instructions executed.", + "MetricGroup": "MPKI;L1D_Cache_Effectiveness", + "ScaleUnit": "1MPKI" + }, + { + "MetricName": "l1d_tlb_miss_ratio", + "MetricExpr": "(L1D_TLB_REFILL / L1D_TLB)", + "BriefDescription": "This metric measures the ratio of level 1 data TLB accesses missed to the total number of level 1 data TLB accesses. This gives an indication of the effectiveness of the level 1 data TLB.", + "MetricGroup": "Miss_Ratio;DTLB_Effectiveness", + "ScaleUnit": "1per TLB access" + }, + { + "MetricName": "l1d_tlb_mpki", + "MetricExpr": "((L1D_TLB_REFILL / INST_RETIRED) * 1000)", + "BriefDescription": "This metric measures the number of level 1 instruction TLB accesses missed per thousand instructions executed.", + "MetricGroup": "MPKI;DTLB_Effectiveness", + "ScaleUnit": "1MPKI" + }, + { + "MetricName": "l1i_cache_miss_ratio", + "MetricExpr": "(L1I_CACHE_REFILL / L1I_CACHE)", + "BriefDescription": "This metric measures the ratio of level 1 instruction cache accesses missed to the total number of level 1 instruction cache accesses. This gives an indication of the effectiveness of the level 1 instruction cache.", + "MetricGroup": "Miss_Ratio;L1I_Cache_Effectiveness", + "ScaleUnit": "1per cache access" + }, + { + "MetricName": "l1i_cache_mpki", + "MetricExpr": "((L1I_CACHE_REFILL / INST_RETIRED) * 1000)", + "BriefDescription": "This metric measures the number of level 1 instruction cache accesses missed per thousand instructions executed.", + "MetricGroup": "MPKI;L1I_Cache_Effectiveness", + "ScaleUnit": "1MPKI" + }, + { + "MetricName": "l1i_tlb_miss_ratio", + "MetricExpr": "(L1I_TLB_REFILL / L1I_TLB)", + "BriefDescription": "This metric measures the ratio of level 1 instruction TLB accesses missed to the total number of level 1 instruction TLB accesses. This gives an indication of the effectiveness of the level 1 instruction TLB.", + "MetricGroup": "Miss_Ratio;ITLB_Effectiveness", + "ScaleUnit": "1per TLB access" + }, + { + "MetricName": "l1i_tlb_mpki", + "MetricExpr": "((L1I_TLB_REFILL / INST_RETIRED) * 1000)", + "BriefDescription": "This metric measures the number of level 1 instruction TLB accesses missed per thousand instructions executed.", + "MetricGroup": "MPKI;ITLB_Effectiveness", + "ScaleUnit": "1MPKI" + }, + { + "MetricName": "l2_cache_miss_ratio", + "MetricExpr": "(L2D_CACHE_REFILL / L2D_CACHE)", + "BriefDescription": "This metric measures the ratio of level 2 cache accesses missed to the total number of level 2 cache accesses. This gives an indication of the effectiveness of the level 2 cache, which is a unified cache that stores both data and instruction. Note that cache accesses in this cache are either data memory access or instruction fetch as this is a unified cache.", + "MetricGroup": "Miss_Ratio;L2_Cache_Effectiveness", + "ScaleUnit": "1per cache access" + }, + { + "MetricName": "l2_cache_mpki", + "MetricExpr": "((L2D_CACHE_REFILL / INST_RETIRED) * 1000)", + "BriefDescription": "This metric measures the number of level 2 unified cache accesses missed per thousand instructions executed. Note that cache accesses in this cache are either data memory access or instruction fetch as this is a unified cache.", + "MetricGroup": "MPKI;L2_Cache_Effectiveness", + "ScaleUnit": "1MPKI" + }, + { + "MetricName": "l2_tlb_miss_ratio", + "MetricExpr": "(L2D_TLB_REFILL / L2D_TLB)", + "BriefDescription": "This metric measures the ratio of level 2 unified TLB accesses missed to the total number of level 2 unified TLB accesses. This gives an indication of the effectiveness of the level 2 TLB.", + "MetricGroup": "Miss_Ratio;ITLB_Effectiveness;DTLB_Effectiveness", + "ScaleUnit": "1per TLB access" + }, + { + "MetricName": "l2_tlb_mpki", + "MetricExpr": "((L2D_TLB_REFILL / INST_RETIRED) * 1000)", + "BriefDescription": "This metric measures the number of level 2 unified TLB accesses missed per thousand instructions executed.", + "MetricGroup": "MPKI;ITLB_Effectiveness;DTLB_Effectiveness", + "ScaleUnit": "1MPKI" + }, + { + "MetricName": "ll_cache_read_hit_ratio", + "MetricExpr": "((LL_CACHE_RD - LL_CACHE_MISS_RD) / LL_CACHE_RD)", + "BriefDescription": "This metric measures the ratio of last level cache read accesses hit in the cache to the total number of last level cache accesses. This gives an indication of the effectiveness of the last level cache for read traffic. Note that cache accesses in this cache are either data memory access or instruction fetch as this is a system level cache.", + "MetricGroup": "LL_Cache_Effectiveness", + "ScaleUnit": "1per cache access" + }, + { + "MetricName": "ll_cache_read_miss_ratio", + "MetricExpr": "(LL_CACHE_MISS_RD / LL_CACHE_RD)", + "BriefDescription": "This metric measures the ratio of last level cache read accesses missed to the total number of last level cache accesses. This gives an indication of the effectiveness of the last level cache for read traffic. Note that cache accesses in this cache are either data memory access or instruction fetch as this is a system level cache.", + "MetricGroup": "Miss_Ratio;LL_Cache_Effectiveness", + "ScaleUnit": "1per cache access" + }, + { + "MetricName": "ll_cache_read_mpki", + "MetricExpr": "((LL_CACHE_MISS_RD / INST_RETIRED) * 1000)", + "BriefDescription": "This metric measures the number of last level cache read accesses missed per thousand instructions executed.", + "MetricGroup": "MPKI;LL_Cache_Effectiveness", + "ScaleUnit": "1MPKI" + }, + { + "MetricName": "load_percentage", + "MetricExpr": "((LD_SPEC / INST_SPEC) * 100)", + "BriefDescription": "This metric measures load operations as a percentage of operations speculatively executed.", + "MetricGroup": "Operation_Mix", + "ScaleUnit": "1percent of operations" + }, + { + "ArchStdEvent": "retiring" + }, + { + "MetricName": "scalar_fp_percentage", + "MetricExpr": "((VFP_SPEC / INST_SPEC) * 100)", + "BriefDescription": "This metric measures scalar floating point operations as a percentage of operations speculatively executed.", + "MetricGroup": "Operation_Mix", + "ScaleUnit": "1percent of operations" + }, + { + "MetricName": "simd_percentage", + "MetricExpr": "((ASE_SPEC / INST_SPEC) * 100)", + "BriefDescription": "This metric measures advanced SIMD operations as a percentage of total operations speculatively executed.", + "MetricGroup": "Operation_Mix", + "ScaleUnit": "1percent of operations" + }, + { + "MetricName": "store_percentage", + "MetricExpr": "((ST_SPEC / INST_SPEC) * 100)", + "BriefDescription": "This metric measures store operations as a percentage of operations speculatively executed.", + "MetricGroup": "Operation_Mix", + "ScaleUnit": "1percent of operations" + } +] From bb350847965daf9217e884dc00639f5d2ea89045 Mon Sep 17 00:00:00 2001 From: Ruidong Tian Date: Mon, 11 Sep 2023 14:55:41 +0800 Subject: [PATCH 072/475] perf test: Update cs_etm testcase for Arm ETE Add ETE as one of the supported device types in perf cs_etm testcase. Reviewed-by: James Clark Signed-off-by: Ruidong Tian Cc: Alexander Shishkin Cc: Mike Leach Cc: Suzuki Poulouse Cc: coresight@lists.linaro.org Cc: linux-arm-kernel@lists.infradead.org Link: https://lore.kernel.org/r/20230911065541.91293-1-tianruidong@linux.alibaba.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/tests/shell/test_arm_coresight.sh | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tools/perf/tests/shell/test_arm_coresight.sh b/tools/perf/tests/shell/test_arm_coresight.sh index f1bf5621160f..fe78c4626e45 100755 --- a/tools/perf/tests/shell/test_arm_coresight.sh +++ b/tools/perf/tests/shell/test_arm_coresight.sh @@ -11,6 +11,19 @@ glb_err=0 +cs_etm_dev_name() { + cs_etm_path=$(find /sys/bus/event_source/devices/cs_etm/ -name cpu* -print -quit) + trcdevarch=$(cat ${cs_etm_path}/mgmt/trcdevarch) + archhver=$((($trcdevarch >> 12) & 0xf)) + archpart=$(($trcdevarch & 0xfff)) + + if [ $archhver -eq 5 -a "$(printf "0x%X\n" $archpart)" = "0xA13" ] ; then + echo "ete" + else + echo "etm" + fi +} + skip_if_no_cs_etm_event() { perf list | grep -q 'cs_etm//' && return 0 @@ -136,7 +149,7 @@ arm_cs_iterate_devices() { arm_cs_etm_traverse_path_test() { # Iterate for every ETM device - for dev in /sys/bus/coresight/devices/etm*; do + for dev in /sys/bus/coresight/devices/$(cs_etm_dev_name)*; do # Find the ETM device belonging to which CPU cpu=`cat $dev/cpu` From f5d98b8bdcd3c39b9ec3954573a7be1a5531215f Mon Sep 17 00:00:00 2001 From: Athira Rajeev Date: Tue, 12 Sep 2023 12:08:07 +0530 Subject: [PATCH 073/475] perf tools: Add includes for detected configs in Makefile.perf Makefile.perf uses "CONFIG_*" checks in the code. Example the config for libtraceevent is used to set PYTHON_EXT_SRCS ifeq ($(CONFIG_LIBTRACEEVENT),y) PYTHON_EXT_SRCS := $(shell grep -v ^\# util/python-ext-sources) else PYTHON_EXT_SRCS := $(shell grep -v '^\#\|util/trace-event.c' util/python-ext-sources) endif But this is not picking the value for CONFIG_LIBTRACEEVENT that is set using the settings in Makefile.config. Include the file ".config-detected" so that make will use the system detected configuration in the CONFIG checks. This will fix isues that could arise when other "CONFIG_*" checks are added to Makefile.perf in future as well. Reviewed-by: Ian Rogers Signed-off-by: Athira Rajeev Cc: Disha Goel Cc: Jiri Olsa Cc: Kajol Jain Cc: Madhavan Srinivasan Cc: Namhyung Kim Cc: linuxppc-dev@lists.ozlabs.org Link: https://lore.kernel.org/r/20230912063807.74250-1-atrajeev@linux.vnet.ibm.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/Makefile.perf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/perf/Makefile.perf b/tools/perf/Makefile.perf index cb0422debdb9..8d0f6d2bbc7a 100644 --- a/tools/perf/Makefile.perf +++ b/tools/perf/Makefile.perf @@ -351,6 +351,9 @@ export PYTHON_EXTBUILD_LIB PYTHON_EXTBUILD_TMP python-clean := $(call QUIET_CLEAN, python) $(RM) -r $(PYTHON_EXTBUILD) $(OUTPUT)python/perf*.so +# Use the detected configuration +-include $(OUTPUT).config-detected + ifeq ($(CONFIG_LIBTRACEEVENT),y) PYTHON_EXT_SRCS := $(shell grep -v ^\# util/python-ext-sources) else From dc2cfef9a9afe04a1b7683a999065b544d7b54f7 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Mon, 11 Sep 2023 10:05:55 -0700 Subject: [PATCH 074/475] perf parse-events: Remove unused header files The fnmatch header is now used in the PMU matching logic in pmu.c. Signed-off-by: Ian Rogers Acked-by: Namhyung Kim Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Gaosheng Cui Cc: Ingo Molnar Cc: James Clark Cc: Jiri Olsa Cc: Kan Liang Cc: Mark Rutland Cc: Peter Zijlstra Cc: Rob Herring Cc: bpf@vger.kernel.org Link: https://lore.kernel.org/r/20230911170559.4037734-1-irogers@google.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/util/parse-events.y | 3 --- 1 file changed, 3 deletions(-) diff --git a/tools/perf/util/parse-events.y b/tools/perf/util/parse-events.y index bf74d6564eba..dcadf08f840a 100644 --- a/tools/perf/util/parse-events.y +++ b/tools/perf/util/parse-events.y @@ -9,11 +9,8 @@ #define YYDEBUG 1 #include -#include -#include #include #include -#include #include "pmu.h" #include "pmus.h" #include "evsel.h" From d4ce60190e08d84f88937019defa5e3d23409ac1 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Mon, 11 Sep 2023 10:05:56 -0700 Subject: [PATCH 075/475] perf parse-events: Make YYDEBUG dependent on doing a debug build YYDEBUG enables line numbers and other error helpers in the generated parse-events-bison.c. These shouldn't be generated when debugging isn't enabled. Signed-off-by: Ian Rogers Acked-by: Namhyung Kim Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Gaosheng Cui Cc: Ingo Molnar Cc: James Clark Cc: Jiri Olsa Cc: Kan Liang Cc: Mark Rutland Cc: Peter Zijlstra Cc: Rob Herring Cc: bpf@vger.kernel.org Link: https://lore.kernel.org/r/20230911170559.4037734-2-irogers@google.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/util/parse-events.y | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/perf/util/parse-events.y b/tools/perf/util/parse-events.y index dcadf08f840a..c4ce888a2550 100644 --- a/tools/perf/util/parse-events.y +++ b/tools/perf/util/parse-events.y @@ -6,7 +6,9 @@ %{ +#ifndef NDEBUG #define YYDEBUG 1 +#endif #include #include From 1344a7077d78bb2ba72e539fc9119321d98fd06e Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Mon, 11 Sep 2023 10:05:57 -0700 Subject: [PATCH 076/475] perf expr: Make YYDEBUG dependent on doing a debug build YYDEBUG enables line numbers and other error helpers in the generated expr-bison.c. These shouldn't be generated when debugging isn't enabled. Signed-off-by: Ian Rogers Acked-by: Namhyung Kim Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Gaosheng Cui Cc: Ingo Molnar Cc: James Clark Cc: Jiri Olsa Cc: Kan Liang Cc: Mark Rutland Cc: Peter Zijlstra Cc: Rob Herring Cc: bpf@vger.kernel.org Link: https://lore.kernel.org/r/20230911170559.4037734-3-irogers@google.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/util/expr.y | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/perf/util/expr.y b/tools/perf/util/expr.y index 6c93b358cc2d..e364790babb5 100644 --- a/tools/perf/util/expr.y +++ b/tools/perf/util/expr.y @@ -1,6 +1,8 @@ /* Simple expression parser */ %{ +#ifndef NDEBUG #define YYDEBUG 1 +#endif #include #include #include From f0f4cd1003a03b0ae86be8c9ec8e88b0df384001 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Mon, 11 Sep 2023 10:05:58 -0700 Subject: [PATCH 077/475] perf pmu: Add YYDEBUG YYDEBUG enables line numbers and other error helpers in the generated pmu-bison.c. Conditionally enabled only for debug builds. Signed-off-by: Ian Rogers Acked-by: Namhyung Kim Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Gaosheng Cui Cc: Ingo Molnar Cc: James Clark Cc: Jiri Olsa Cc: Kan Liang Cc: Mark Rutland Cc: Peter Zijlstra Cc: Rob Herring Cc: bpf@vger.kernel.org Link: https://lore.kernel.org/r/20230911170559.4037734-4-irogers@google.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/util/pmu.y | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/perf/util/pmu.y b/tools/perf/util/pmu.y index 600c8c158c8e..198907a8a48a 100644 --- a/tools/perf/util/pmu.y +++ b/tools/perf/util/pmu.y @@ -5,6 +5,10 @@ %{ +#ifndef NDEBUG +#define YYDEBUG 1 +#endif + #include #include #include From 999b81b907ea92fa759e426591068244d9635496 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Mon, 11 Sep 2023 10:05:59 -0700 Subject: [PATCH 078/475] perf bpf-filter: Add YYDEBUG YYDEBUG enables line numbers and other error helpers in the generated bpf-filter-bison.c. Conditionally enabled only for debug builds. Signed-off-by: Ian Rogers Acked-by: Namhyung Kim Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Gaosheng Cui Cc: Ingo Molnar Cc: James Clark Cc: Jiri Olsa Cc: Kan Liang Cc: Mark Rutland Cc: Peter Zijlstra Cc: Rob Herring Cc: bpf@vger.kernel.org Link: https://lore.kernel.org/r/20230911170559.4037734-5-irogers@google.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/util/bpf-filter.y | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/perf/util/bpf-filter.y b/tools/perf/util/bpf-filter.y index 5dfa948fc986..0e4d6de3c2ad 100644 --- a/tools/perf/util/bpf-filter.y +++ b/tools/perf/util/bpf-filter.y @@ -3,6 +3,10 @@ %{ +#ifndef NDEBUG +#define YYDEBUG 1 +#endif + #include #include #include From 0e85b7df9cb0c65f840109159ef6754c783e07a0 Mon Sep 17 00:00:00 2001 From: Jordan Niethe Date: Thu, 14 Sep 2023 13:05:50 +1000 Subject: [PATCH 079/475] KVM: PPC: Always use the GPR accessors Always use the GPR accessor functions. This will be important later for Nested APIv2 support which requires additional functionality for accessing and modifying VCPU state. Signed-off-by: Jordan Niethe Signed-off-by: Michael Ellerman Link: https://msgid.link/20230914030600.16993-2-jniethe5@gmail.com --- arch/powerpc/kvm/book3s_64_vio.c | 4 ++-- arch/powerpc/kvm/book3s_hv.c | 8 ++++++-- arch/powerpc/kvm/book3s_hv_builtin.c | 6 +++++- arch/powerpc/kvm/book3s_hv_rm_mmu.c | 8 ++++---- arch/powerpc/kvm/book3s_hv_rm_xics.c | 4 ++-- arch/powerpc/kvm/book3s_xive.c | 4 ++-- 6 files changed, 21 insertions(+), 13 deletions(-) diff --git a/arch/powerpc/kvm/book3s_64_vio.c b/arch/powerpc/kvm/book3s_64_vio.c index 93b695b289e9..4ba048f272f2 100644 --- a/arch/powerpc/kvm/book3s_64_vio.c +++ b/arch/powerpc/kvm/book3s_64_vio.c @@ -786,12 +786,12 @@ long kvmppc_h_get_tce(struct kvm_vcpu *vcpu, unsigned long liobn, idx = (ioba >> stt->page_shift) - stt->offset; page = stt->pages[idx / TCES_PER_PAGE]; if (!page) { - vcpu->arch.regs.gpr[4] = 0; + kvmppc_set_gpr(vcpu, 4, 0); return H_SUCCESS; } tbl = (u64 *)page_address(page); - vcpu->arch.regs.gpr[4] = tbl[idx % TCES_PER_PAGE]; + kvmppc_set_gpr(vcpu, 4, tbl[idx % TCES_PER_PAGE]); return H_SUCCESS; } diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c index 130bafdb1430..4af5b68cf7f8 100644 --- a/arch/powerpc/kvm/book3s_hv.c +++ b/arch/powerpc/kvm/book3s_hv.c @@ -1267,10 +1267,14 @@ int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu) return RESUME_HOST; break; #endif - case H_RANDOM: - if (!arch_get_random_seed_longs(&vcpu->arch.regs.gpr[4], 1)) + case H_RANDOM: { + unsigned long rand; + + if (!arch_get_random_seed_longs(&rand, 1)) ret = H_HARDWARE; + kvmppc_set_gpr(vcpu, 4, rand); break; + } case H_RPT_INVALIDATE: ret = kvmppc_h_rpt_invalidate(vcpu, kvmppc_get_gpr(vcpu, 4), kvmppc_get_gpr(vcpu, 5), diff --git a/arch/powerpc/kvm/book3s_hv_builtin.c b/arch/powerpc/kvm/book3s_hv_builtin.c index 0f5b021fa559..f3afe194e616 100644 --- a/arch/powerpc/kvm/book3s_hv_builtin.c +++ b/arch/powerpc/kvm/book3s_hv_builtin.c @@ -182,9 +182,13 @@ EXPORT_SYMBOL_GPL(kvmppc_hwrng_present); long kvmppc_rm_h_random(struct kvm_vcpu *vcpu) { + unsigned long rand; + if (ppc_md.get_random_seed && - ppc_md.get_random_seed(&vcpu->arch.regs.gpr[4])) + ppc_md.get_random_seed(&rand)) { + kvmppc_set_gpr(vcpu, 4, rand); return H_SUCCESS; + } return H_HARDWARE; } diff --git a/arch/powerpc/kvm/book3s_hv_rm_mmu.c b/arch/powerpc/kvm/book3s_hv_rm_mmu.c index 9182324dbef9..17cb75a127b0 100644 --- a/arch/powerpc/kvm/book3s_hv_rm_mmu.c +++ b/arch/powerpc/kvm/book3s_hv_rm_mmu.c @@ -776,8 +776,8 @@ long kvmppc_h_read(struct kvm_vcpu *vcpu, unsigned long flags, r = rev[i].guest_rpte | (r & (HPTE_R_R | HPTE_R_C)); r &= ~HPTE_GR_RESERVED; } - vcpu->arch.regs.gpr[4 + i * 2] = v; - vcpu->arch.regs.gpr[5 + i * 2] = r; + kvmppc_set_gpr(vcpu, 4 + i * 2, v); + kvmppc_set_gpr(vcpu, 5 + i * 2, r); } return H_SUCCESS; } @@ -824,7 +824,7 @@ long kvmppc_h_clear_ref(struct kvm_vcpu *vcpu, unsigned long flags, } } } - vcpu->arch.regs.gpr[4] = gr; + kvmppc_set_gpr(vcpu, 4, gr); ret = H_SUCCESS; out: unlock_hpte(hpte, v & ~HPTE_V_HVLOCK); @@ -872,7 +872,7 @@ long kvmppc_h_clear_mod(struct kvm_vcpu *vcpu, unsigned long flags, kvmppc_set_dirty_from_hpte(kvm, v, gr); } } - vcpu->arch.regs.gpr[4] = gr; + kvmppc_set_gpr(vcpu, 4, gr); ret = H_SUCCESS; out: unlock_hpte(hpte, v & ~HPTE_V_HVLOCK); diff --git a/arch/powerpc/kvm/book3s_hv_rm_xics.c b/arch/powerpc/kvm/book3s_hv_rm_xics.c index e165bfa842bf..e42984878503 100644 --- a/arch/powerpc/kvm/book3s_hv_rm_xics.c +++ b/arch/powerpc/kvm/book3s_hv_rm_xics.c @@ -481,7 +481,7 @@ static void icp_rm_down_cppr(struct kvmppc_xics *xics, struct kvmppc_icp *icp, unsigned long xics_rm_h_xirr_x(struct kvm_vcpu *vcpu) { - vcpu->arch.regs.gpr[5] = get_tb(); + kvmppc_set_gpr(vcpu, 5, get_tb()); return xics_rm_h_xirr(vcpu); } @@ -518,7 +518,7 @@ unsigned long xics_rm_h_xirr(struct kvm_vcpu *vcpu) } while (!icp_rm_try_update(icp, old_state, new_state)); /* Return the result in GPR4 */ - vcpu->arch.regs.gpr[4] = xirr; + kvmppc_set_gpr(vcpu, 4, xirr); return check_too_hard(xics, icp); } diff --git a/arch/powerpc/kvm/book3s_xive.c b/arch/powerpc/kvm/book3s_xive.c index f4115819e738..48d11baf1f16 100644 --- a/arch/powerpc/kvm/book3s_xive.c +++ b/arch/powerpc/kvm/book3s_xive.c @@ -328,7 +328,7 @@ static unsigned long xive_vm_h_xirr(struct kvm_vcpu *vcpu) */ /* Return interrupt and old CPPR in GPR4 */ - vcpu->arch.regs.gpr[4] = hirq | (old_cppr << 24); + kvmppc_set_gpr(vcpu, 4, hirq | (old_cppr << 24)); return H_SUCCESS; } @@ -364,7 +364,7 @@ static unsigned long xive_vm_h_ipoll(struct kvm_vcpu *vcpu, unsigned long server hirq = xive_vm_scan_interrupts(xc, pending, scan_poll); /* Return interrupt and old CPPR in GPR4 */ - vcpu->arch.regs.gpr[4] = hirq | (xc->cppr << 24); + kvmppc_set_gpr(vcpu, 4, hirq | (xc->cppr << 24)); return H_SUCCESS; } From 52425a3b3c11cec58cf66e4c897fc1504f3911a9 Mon Sep 17 00:00:00 2001 From: Jordan Niethe Date: Thu, 14 Sep 2023 13:05:51 +1000 Subject: [PATCH 080/475] KVM: PPC: Introduce FPR/VR accessor functions Introduce accessor functions for floating point and vector registers like the ones that exist for GPRs. Use these to replace the existing FPR and VR accessor macros. This will be important later for Nested APIv2 support which requires additional functionality for accessing and modifying VCPU state. Signed-off-by: Gautam Menghani Signed-off-by: Jordan Niethe Signed-off-by: Michael Ellerman Link: https://msgid.link/20230914030600.16993-3-jniethe5@gmail.com --- arch/powerpc/include/asm/kvm_book3s.h | 55 ++++++++++++++++++++ arch/powerpc/include/asm/kvm_booke.h | 10 ++++ arch/powerpc/kvm/book3s.c | 16 +++--- arch/powerpc/kvm/emulate_loadstore.c | 2 +- arch/powerpc/kvm/powerpc.c | 72 +++++++++++++-------------- 5 files changed, 110 insertions(+), 45 deletions(-) diff --git a/arch/powerpc/include/asm/kvm_book3s.h b/arch/powerpc/include/asm/kvm_book3s.h index bbf5e2c5fe09..109a5f56767a 100644 --- a/arch/powerpc/include/asm/kvm_book3s.h +++ b/arch/powerpc/include/asm/kvm_book3s.h @@ -403,6 +403,61 @@ static inline ulong kvmppc_get_fault_dar(struct kvm_vcpu *vcpu) return vcpu->arch.fault_dar; } +static inline u64 kvmppc_get_fpr(struct kvm_vcpu *vcpu, int i) +{ + return vcpu->arch.fp.fpr[i][TS_FPROFFSET]; +} + +static inline void kvmppc_set_fpr(struct kvm_vcpu *vcpu, int i, u64 val) +{ + vcpu->arch.fp.fpr[i][TS_FPROFFSET] = val; +} + +static inline u64 kvmppc_get_fpscr(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.fp.fpscr; +} + +static inline void kvmppc_set_fpscr(struct kvm_vcpu *vcpu, u64 val) +{ + vcpu->arch.fp.fpscr = val; +} + + +static inline u64 kvmppc_get_vsx_fpr(struct kvm_vcpu *vcpu, int i, int j) +{ + return vcpu->arch.fp.fpr[i][j]; +} + +static inline void kvmppc_set_vsx_fpr(struct kvm_vcpu *vcpu, int i, int j, + u64 val) +{ + vcpu->arch.fp.fpr[i][j] = val; +} + +#ifdef CONFIG_ALTIVEC +static inline void kvmppc_get_vsx_vr(struct kvm_vcpu *vcpu, int i, vector128 *v) +{ + *v = vcpu->arch.vr.vr[i]; +} + +static inline void kvmppc_set_vsx_vr(struct kvm_vcpu *vcpu, int i, + vector128 *val) +{ + vcpu->arch.vr.vr[i] = *val; +} + +static inline u32 kvmppc_get_vscr(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.vr.vscr.u[3]; +} + +static inline void kvmppc_set_vscr(struct kvm_vcpu *vcpu, u32 val) +{ + vcpu->arch.vr.vscr.u[3] = val; +} +#endif + /* Expiry time of vcpu DEC relative to host TB */ static inline u64 kvmppc_dec_expires_host_tb(struct kvm_vcpu *vcpu) { diff --git a/arch/powerpc/include/asm/kvm_booke.h b/arch/powerpc/include/asm/kvm_booke.h index 0c3401b2e19e..7c3291aa8922 100644 --- a/arch/powerpc/include/asm/kvm_booke.h +++ b/arch/powerpc/include/asm/kvm_booke.h @@ -89,6 +89,16 @@ static inline ulong kvmppc_get_pc(struct kvm_vcpu *vcpu) return vcpu->arch.regs.nip; } +static inline void kvmppc_set_fpr(struct kvm_vcpu *vcpu, int i, u64 val) +{ + vcpu->arch.fp.fpr[i][TS_FPROFFSET] = val; +} + +static inline u64 kvmppc_get_fpr(struct kvm_vcpu *vcpu, int i) +{ + return vcpu->arch.fp.fpr[i][TS_FPROFFSET]; +} + #ifdef CONFIG_BOOKE static inline ulong kvmppc_get_fault_dar(struct kvm_vcpu *vcpu) { diff --git a/arch/powerpc/kvm/book3s.c b/arch/powerpc/kvm/book3s.c index 686d8d9eda3e..c080dd2e96ac 100644 --- a/arch/powerpc/kvm/book3s.c +++ b/arch/powerpc/kvm/book3s.c @@ -636,17 +636,17 @@ int kvmppc_get_one_reg(struct kvm_vcpu *vcpu, u64 id, break; case KVM_REG_PPC_FPR0 ... KVM_REG_PPC_FPR31: i = id - KVM_REG_PPC_FPR0; - *val = get_reg_val(id, VCPU_FPR(vcpu, i)); + *val = get_reg_val(id, kvmppc_get_fpr(vcpu, i)); break; case KVM_REG_PPC_FPSCR: - *val = get_reg_val(id, vcpu->arch.fp.fpscr); + *val = get_reg_val(id, kvmppc_get_fpscr(vcpu)); break; #ifdef CONFIG_VSX case KVM_REG_PPC_VSR0 ... KVM_REG_PPC_VSR31: if (cpu_has_feature(CPU_FTR_VSX)) { i = id - KVM_REG_PPC_VSR0; - val->vsxval[0] = vcpu->arch.fp.fpr[i][0]; - val->vsxval[1] = vcpu->arch.fp.fpr[i][1]; + val->vsxval[0] = kvmppc_get_vsx_fpr(vcpu, i, 0); + val->vsxval[1] = kvmppc_get_vsx_fpr(vcpu, i, 1); } else { r = -ENXIO; } @@ -724,7 +724,7 @@ int kvmppc_set_one_reg(struct kvm_vcpu *vcpu, u64 id, break; case KVM_REG_PPC_FPR0 ... KVM_REG_PPC_FPR31: i = id - KVM_REG_PPC_FPR0; - VCPU_FPR(vcpu, i) = set_reg_val(id, *val); + kvmppc_set_fpr(vcpu, i, set_reg_val(id, *val)); break; case KVM_REG_PPC_FPSCR: vcpu->arch.fp.fpscr = set_reg_val(id, *val); @@ -733,8 +733,8 @@ int kvmppc_set_one_reg(struct kvm_vcpu *vcpu, u64 id, case KVM_REG_PPC_VSR0 ... KVM_REG_PPC_VSR31: if (cpu_has_feature(CPU_FTR_VSX)) { i = id - KVM_REG_PPC_VSR0; - vcpu->arch.fp.fpr[i][0] = val->vsxval[0]; - vcpu->arch.fp.fpr[i][1] = val->vsxval[1]; + kvmppc_set_vsx_fpr(vcpu, i, 0, val->vsxval[0]); + kvmppc_set_vsx_fpr(vcpu, i, 1, val->vsxval[1]); } else { r = -ENXIO; } @@ -765,7 +765,7 @@ int kvmppc_set_one_reg(struct kvm_vcpu *vcpu, u64 id, break; #endif /* CONFIG_KVM_XIVE */ case KVM_REG_PPC_FSCR: - vcpu->arch.fscr = set_reg_val(id, *val); + kvmppc_set_fpscr(vcpu, set_reg_val(id, *val)); break; case KVM_REG_PPC_TAR: vcpu->arch.tar = set_reg_val(id, *val); diff --git a/arch/powerpc/kvm/emulate_loadstore.c b/arch/powerpc/kvm/emulate_loadstore.c index 059c08ae0340..e6e66c3792f8 100644 --- a/arch/powerpc/kvm/emulate_loadstore.c +++ b/arch/powerpc/kvm/emulate_loadstore.c @@ -250,7 +250,7 @@ int kvmppc_emulate_loadstore(struct kvm_vcpu *vcpu) vcpu->arch.mmio_sp64_extend = 1; emulated = kvmppc_handle_store(vcpu, - VCPU_FPR(vcpu, op.reg), size, 1); + kvmppc_get_fpr(vcpu, op.reg), size, 1); if ((op.type & UPDATE) && (emulated != EMULATE_FAIL)) kvmppc_set_gpr(vcpu, op.update_reg, op.ea); diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c index 7197c8256668..2e72e5cd7532 100644 --- a/arch/powerpc/kvm/powerpc.c +++ b/arch/powerpc/kvm/powerpc.c @@ -934,11 +934,11 @@ static inline void kvmppc_set_vsr_dword(struct kvm_vcpu *vcpu, return; if (index >= 32) { - val.vval = VCPU_VSX_VR(vcpu, index - 32); + kvmppc_get_vsx_vr(vcpu, index - 32, &val.vval); val.vsxval[offset] = gpr; - VCPU_VSX_VR(vcpu, index - 32) = val.vval; + kvmppc_set_vsx_vr(vcpu, index - 32, &val.vval); } else { - VCPU_VSX_FPR(vcpu, index, offset) = gpr; + kvmppc_set_vsx_fpr(vcpu, index, offset, gpr); } } @@ -949,13 +949,13 @@ static inline void kvmppc_set_vsr_dword_dump(struct kvm_vcpu *vcpu, int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK; if (index >= 32) { - val.vval = VCPU_VSX_VR(vcpu, index - 32); + kvmppc_get_vsx_vr(vcpu, index - 32, &val.vval); val.vsxval[0] = gpr; val.vsxval[1] = gpr; - VCPU_VSX_VR(vcpu, index - 32) = val.vval; + kvmppc_set_vsx_vr(vcpu, index - 32, &val.vval); } else { - VCPU_VSX_FPR(vcpu, index, 0) = gpr; - VCPU_VSX_FPR(vcpu, index, 1) = gpr; + kvmppc_set_vsx_fpr(vcpu, index, 0, gpr); + kvmppc_set_vsx_fpr(vcpu, index, 1, gpr); } } @@ -970,12 +970,12 @@ static inline void kvmppc_set_vsr_word_dump(struct kvm_vcpu *vcpu, val.vsx32val[1] = gpr; val.vsx32val[2] = gpr; val.vsx32val[3] = gpr; - VCPU_VSX_VR(vcpu, index - 32) = val.vval; + kvmppc_set_vsx_vr(vcpu, index - 32, &val.vval); } else { val.vsx32val[0] = gpr; val.vsx32val[1] = gpr; - VCPU_VSX_FPR(vcpu, index, 0) = val.vsxval[0]; - VCPU_VSX_FPR(vcpu, index, 1) = val.vsxval[0]; + kvmppc_set_vsx_fpr(vcpu, index, 0, val.vsxval[0]); + kvmppc_set_vsx_fpr(vcpu, index, 1, val.vsxval[0]); } } @@ -991,15 +991,15 @@ static inline void kvmppc_set_vsr_word(struct kvm_vcpu *vcpu, return; if (index >= 32) { - val.vval = VCPU_VSX_VR(vcpu, index - 32); + kvmppc_get_vsx_vr(vcpu, index - 32, &val.vval); val.vsx32val[offset] = gpr32; - VCPU_VSX_VR(vcpu, index - 32) = val.vval; + kvmppc_set_vsx_vr(vcpu, index - 32, &val.vval); } else { dword_offset = offset / 2; word_offset = offset % 2; - val.vsxval[0] = VCPU_VSX_FPR(vcpu, index, dword_offset); + val.vsxval[0] = kvmppc_get_vsx_fpr(vcpu, index, dword_offset); val.vsx32val[word_offset] = gpr32; - VCPU_VSX_FPR(vcpu, index, dword_offset) = val.vsxval[0]; + kvmppc_set_vsx_fpr(vcpu, index, dword_offset, val.vsxval[0]); } } #endif /* CONFIG_VSX */ @@ -1058,9 +1058,9 @@ static inline void kvmppc_set_vmx_dword(struct kvm_vcpu *vcpu, if (offset == -1) return; - val.vval = VCPU_VSX_VR(vcpu, index); + kvmppc_get_vsx_vr(vcpu, index, &val.vval); val.vsxval[offset] = gpr; - VCPU_VSX_VR(vcpu, index) = val.vval; + kvmppc_set_vsx_vr(vcpu, index, &val.vval); } static inline void kvmppc_set_vmx_word(struct kvm_vcpu *vcpu, @@ -1074,9 +1074,9 @@ static inline void kvmppc_set_vmx_word(struct kvm_vcpu *vcpu, if (offset == -1) return; - val.vval = VCPU_VSX_VR(vcpu, index); + kvmppc_get_vsx_vr(vcpu, index, &val.vval); val.vsx32val[offset] = gpr32; - VCPU_VSX_VR(vcpu, index) = val.vval; + kvmppc_set_vsx_vr(vcpu, index, &val.vval); } static inline void kvmppc_set_vmx_hword(struct kvm_vcpu *vcpu, @@ -1090,9 +1090,9 @@ static inline void kvmppc_set_vmx_hword(struct kvm_vcpu *vcpu, if (offset == -1) return; - val.vval = VCPU_VSX_VR(vcpu, index); + kvmppc_get_vsx_vr(vcpu, index, &val.vval); val.vsx16val[offset] = gpr16; - VCPU_VSX_VR(vcpu, index) = val.vval; + kvmppc_set_vsx_vr(vcpu, index, &val.vval); } static inline void kvmppc_set_vmx_byte(struct kvm_vcpu *vcpu, @@ -1106,9 +1106,9 @@ static inline void kvmppc_set_vmx_byte(struct kvm_vcpu *vcpu, if (offset == -1) return; - val.vval = VCPU_VSX_VR(vcpu, index); + kvmppc_get_vsx_vr(vcpu, index, &val.vval); val.vsx8val[offset] = gpr8; - VCPU_VSX_VR(vcpu, index) = val.vval; + kvmppc_set_vsx_vr(vcpu, index, &val.vval); } #endif /* CONFIG_ALTIVEC */ @@ -1194,14 +1194,14 @@ static void kvmppc_complete_mmio_load(struct kvm_vcpu *vcpu) if (vcpu->kvm->arch.kvm_ops->giveup_ext) vcpu->kvm->arch.kvm_ops->giveup_ext(vcpu, MSR_FP); - VCPU_FPR(vcpu, vcpu->arch.io_gpr & KVM_MMIO_REG_MASK) = gpr; + kvmppc_set_fpr(vcpu, vcpu->arch.io_gpr & KVM_MMIO_REG_MASK, gpr); break; #ifdef CONFIG_PPC_BOOK3S case KVM_MMIO_REG_QPR: vcpu->arch.qpr[vcpu->arch.io_gpr & KVM_MMIO_REG_MASK] = gpr; break; case KVM_MMIO_REG_FQPR: - VCPU_FPR(vcpu, vcpu->arch.io_gpr & KVM_MMIO_REG_MASK) = gpr; + kvmppc_set_fpr(vcpu, vcpu->arch.io_gpr & KVM_MMIO_REG_MASK, gpr); vcpu->arch.qpr[vcpu->arch.io_gpr & KVM_MMIO_REG_MASK] = gpr; break; #endif @@ -1419,9 +1419,9 @@ static inline int kvmppc_get_vsr_data(struct kvm_vcpu *vcpu, int rs, u64 *val) } if (rs < 32) { - *val = VCPU_VSX_FPR(vcpu, rs, vsx_offset); + *val = kvmppc_get_vsx_fpr(vcpu, rs, vsx_offset); } else { - reg.vval = VCPU_VSX_VR(vcpu, rs - 32); + kvmppc_get_vsx_vr(vcpu, rs - 32, ®.vval); *val = reg.vsxval[vsx_offset]; } break; @@ -1438,10 +1438,10 @@ static inline int kvmppc_get_vsr_data(struct kvm_vcpu *vcpu, int rs, u64 *val) if (rs < 32) { dword_offset = vsx_offset / 2; word_offset = vsx_offset % 2; - reg.vsxval[0] = VCPU_VSX_FPR(vcpu, rs, dword_offset); + reg.vsxval[0] = kvmppc_get_vsx_fpr(vcpu, rs, dword_offset); *val = reg.vsx32val[word_offset]; } else { - reg.vval = VCPU_VSX_VR(vcpu, rs - 32); + kvmppc_get_vsx_vr(vcpu, rs - 32, ®.vval); *val = reg.vsx32val[vsx_offset]; } break; @@ -1556,7 +1556,7 @@ static int kvmppc_get_vmx_dword(struct kvm_vcpu *vcpu, int index, u64 *val) if (vmx_offset == -1) return -1; - reg.vval = VCPU_VSX_VR(vcpu, index); + kvmppc_get_vsx_vr(vcpu, index, ®.vval); *val = reg.vsxval[vmx_offset]; return result; @@ -1574,7 +1574,7 @@ static int kvmppc_get_vmx_word(struct kvm_vcpu *vcpu, int index, u64 *val) if (vmx_offset == -1) return -1; - reg.vval = VCPU_VSX_VR(vcpu, index); + kvmppc_get_vsx_vr(vcpu, index, ®.vval); *val = reg.vsx32val[vmx_offset]; return result; @@ -1592,7 +1592,7 @@ static int kvmppc_get_vmx_hword(struct kvm_vcpu *vcpu, int index, u64 *val) if (vmx_offset == -1) return -1; - reg.vval = VCPU_VSX_VR(vcpu, index); + kvmppc_get_vsx_vr(vcpu, index, ®.vval); *val = reg.vsx16val[vmx_offset]; return result; @@ -1610,7 +1610,7 @@ static int kvmppc_get_vmx_byte(struct kvm_vcpu *vcpu, int index, u64 *val) if (vmx_offset == -1) return -1; - reg.vval = VCPU_VSX_VR(vcpu, index); + kvmppc_get_vsx_vr(vcpu, index, ®.vval); *val = reg.vsx8val[vmx_offset]; return result; @@ -1719,14 +1719,14 @@ int kvm_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu, struct kvm_one_reg *reg) r = -ENXIO; break; } - val.vval = vcpu->arch.vr.vr[reg->id - KVM_REG_PPC_VR0]; + kvmppc_get_vsx_vr(vcpu, reg->id - KVM_REG_PPC_VR0, &val.vval); break; case KVM_REG_PPC_VSCR: if (!cpu_has_feature(CPU_FTR_ALTIVEC)) { r = -ENXIO; break; } - val = get_reg_val(reg->id, vcpu->arch.vr.vscr.u[3]); + val = get_reg_val(reg->id, kvmppc_get_vscr(vcpu)); break; case KVM_REG_PPC_VRSAVE: val = get_reg_val(reg->id, vcpu->arch.vrsave); @@ -1770,14 +1770,14 @@ int kvm_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu, struct kvm_one_reg *reg) r = -ENXIO; break; } - vcpu->arch.vr.vr[reg->id - KVM_REG_PPC_VR0] = val.vval; + kvmppc_set_vsx_vr(vcpu, reg->id - KVM_REG_PPC_VR0, &val.vval); break; case KVM_REG_PPC_VSCR: if (!cpu_has_feature(CPU_FTR_ALTIVEC)) { r = -ENXIO; break; } - vcpu->arch.vr.vscr.u[3] = set_reg_val(reg->id, val); + kvmppc_set_vscr(vcpu, set_reg_val(reg->id, val)); break; case KVM_REG_PPC_VRSAVE: if (!cpu_has_feature(CPU_FTR_ALTIVEC)) { From 2a64bc673133346a7a3bd163f2e6048bd1788727 Mon Sep 17 00:00:00 2001 From: Jordan Niethe Date: Thu, 14 Sep 2023 13:05:52 +1000 Subject: [PATCH 081/475] KVM: PPC: Rename accessor generator macros More "wrapper" style accessor generating macros will be introduced for the nestedv2 guest support. Rename the existing macros with more descriptive names now so there is a consistent naming convention. Reviewed-by: Nicholas Piggin Signed-off-by: Jordan Niethe Signed-off-by: Michael Ellerman Link: https://msgid.link/20230914030600.16993-4-jniethe5@gmail.com --- arch/powerpc/include/asm/kvm_ppc.h | 60 +++++++++++++++--------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/arch/powerpc/include/asm/kvm_ppc.h b/arch/powerpc/include/asm/kvm_ppc.h index b4da8514af43..004998b267e6 100644 --- a/arch/powerpc/include/asm/kvm_ppc.h +++ b/arch/powerpc/include/asm/kvm_ppc.h @@ -927,19 +927,19 @@ static inline bool kvmppc_shared_big_endian(struct kvm_vcpu *vcpu) #endif } -#define SPRNG_WRAPPER_GET(reg, bookehv_spr) \ +#define KVMPPC_BOOKE_HV_SPRNG_ACCESSOR_GET(reg, bookehv_spr) \ static inline ulong kvmppc_get_##reg(struct kvm_vcpu *vcpu) \ { \ return mfspr(bookehv_spr); \ } \ -#define SPRNG_WRAPPER_SET(reg, bookehv_spr) \ +#define KVMPPC_BOOKE_HV_SPRNG_ACCESSOR_SET(reg, bookehv_spr) \ static inline void kvmppc_set_##reg(struct kvm_vcpu *vcpu, ulong val) \ { \ mtspr(bookehv_spr, val); \ } \ -#define SHARED_WRAPPER_GET(reg, size) \ +#define KVMPPC_VCPU_SHARED_REGS_ACCESSOR_GET(reg, size) \ static inline u##size kvmppc_get_##reg(struct kvm_vcpu *vcpu) \ { \ if (kvmppc_shared_big_endian(vcpu)) \ @@ -948,7 +948,7 @@ static inline u##size kvmppc_get_##reg(struct kvm_vcpu *vcpu) \ return le##size##_to_cpu(vcpu->arch.shared->reg); \ } \ -#define SHARED_WRAPPER_SET(reg, size) \ +#define KVMPPC_VCPU_SHARED_REGS_ACCESSOR_SET(reg, size) \ static inline void kvmppc_set_##reg(struct kvm_vcpu *vcpu, u##size val) \ { \ if (kvmppc_shared_big_endian(vcpu)) \ @@ -957,36 +957,36 @@ static inline void kvmppc_set_##reg(struct kvm_vcpu *vcpu, u##size val) \ vcpu->arch.shared->reg = cpu_to_le##size(val); \ } \ -#define SHARED_WRAPPER(reg, size) \ - SHARED_WRAPPER_GET(reg, size) \ - SHARED_WRAPPER_SET(reg, size) \ +#define KVMPPC_VCPU_SHARED_REGS_ACCESSOR(reg, size) \ + KVMPPC_VCPU_SHARED_REGS_ACCESSOR_GET(reg, size) \ + KVMPPC_VCPU_SHARED_REGS_ACCESSOR_SET(reg, size) \ -#define SPRNG_WRAPPER(reg, bookehv_spr) \ - SPRNG_WRAPPER_GET(reg, bookehv_spr) \ - SPRNG_WRAPPER_SET(reg, bookehv_spr) \ +#define KVMPPC_BOOKE_HV_SPRNG_ACCESSOR(reg, bookehv_spr) \ + KVMPPC_BOOKE_HV_SPRNG_ACCESSOR_GET(reg, bookehv_spr) \ + KVMPPC_BOOKE_HV_SPRNG_ACCESSOR_SET(reg, bookehv_spr) \ #ifdef CONFIG_KVM_BOOKE_HV -#define SHARED_SPRNG_WRAPPER(reg, size, bookehv_spr) \ - SPRNG_WRAPPER(reg, bookehv_spr) \ +#define KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(reg, size, bookehv_spr) \ + KVMPPC_BOOKE_HV_SPRNG_ACCESSOR(reg, bookehv_spr) \ #else -#define SHARED_SPRNG_WRAPPER(reg, size, bookehv_spr) \ - SHARED_WRAPPER(reg, size) \ +#define KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(reg, size, bookehv_spr) \ + KVMPPC_VCPU_SHARED_REGS_ACCESSOR(reg, size) \ #endif -SHARED_WRAPPER(critical, 64) -SHARED_SPRNG_WRAPPER(sprg0, 64, SPRN_GSPRG0) -SHARED_SPRNG_WRAPPER(sprg1, 64, SPRN_GSPRG1) -SHARED_SPRNG_WRAPPER(sprg2, 64, SPRN_GSPRG2) -SHARED_SPRNG_WRAPPER(sprg3, 64, SPRN_GSPRG3) -SHARED_SPRNG_WRAPPER(srr0, 64, SPRN_GSRR0) -SHARED_SPRNG_WRAPPER(srr1, 64, SPRN_GSRR1) -SHARED_SPRNG_WRAPPER(dar, 64, SPRN_GDEAR) -SHARED_SPRNG_WRAPPER(esr, 64, SPRN_GESR) -SHARED_WRAPPER_GET(msr, 64) +KVMPPC_VCPU_SHARED_REGS_ACCESSOR(critical, 64) +KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(sprg0, 64, SPRN_GSPRG0) +KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(sprg1, 64, SPRN_GSPRG1) +KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(sprg2, 64, SPRN_GSPRG2) +KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(sprg3, 64, SPRN_GSPRG3) +KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(srr0, 64, SPRN_GSRR0) +KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(srr1, 64, SPRN_GSRR1) +KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(dar, 64, SPRN_GDEAR) +KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(esr, 64, SPRN_GESR) +KVMPPC_VCPU_SHARED_REGS_ACCESSOR_GET(msr, 64) static inline void kvmppc_set_msr_fast(struct kvm_vcpu *vcpu, u64 val) { if (kvmppc_shared_big_endian(vcpu)) @@ -994,12 +994,12 @@ static inline void kvmppc_set_msr_fast(struct kvm_vcpu *vcpu, u64 val) else vcpu->arch.shared->msr = cpu_to_le64(val); } -SHARED_WRAPPER(dsisr, 32) -SHARED_WRAPPER(int_pending, 32) -SHARED_WRAPPER(sprg4, 64) -SHARED_WRAPPER(sprg5, 64) -SHARED_WRAPPER(sprg6, 64) -SHARED_WRAPPER(sprg7, 64) +KVMPPC_VCPU_SHARED_REGS_ACCESSOR(dsisr, 32) +KVMPPC_VCPU_SHARED_REGS_ACCESSOR(int_pending, 32) +KVMPPC_VCPU_SHARED_REGS_ACCESSOR(sprg4, 64) +KVMPPC_VCPU_SHARED_REGS_ACCESSOR(sprg5, 64) +KVMPPC_VCPU_SHARED_REGS_ACCESSOR(sprg6, 64) +KVMPPC_VCPU_SHARED_REGS_ACCESSOR(sprg7, 64) static inline u32 kvmppc_get_sr(struct kvm_vcpu *vcpu, int nr) { From 7028ac8d174f28220f0e2de0cb3346cd3c31976d Mon Sep 17 00:00:00 2001 From: Jordan Niethe Date: Thu, 14 Sep 2023 13:05:53 +1000 Subject: [PATCH 082/475] KVM: PPC: Use accessors for VCPU registers Introduce accessor generator macros for VCPU registers. Use the accessor functions to replace direct accesses to this registers. This will be important later for Nested APIv2 support which requires additional functionality for accessing and modifying VCPU state. Signed-off-by: Jordan Niethe Signed-off-by: Michael Ellerman Link: https://msgid.link/20230914030600.16993-5-jniethe5@gmail.com --- arch/powerpc/include/asm/kvm_book3s.h | 37 +++++++++++++++++++++++++- arch/powerpc/kvm/book3s.c | 22 +++++++-------- arch/powerpc/kvm/book3s_64_mmu_radix.c | 4 +-- arch/powerpc/kvm/book3s_hv.c | 12 ++++----- arch/powerpc/kvm/book3s_hv_p9_entry.c | 4 +-- arch/powerpc/kvm/powerpc.c | 4 +-- 6 files changed, 59 insertions(+), 24 deletions(-) diff --git a/arch/powerpc/include/asm/kvm_book3s.h b/arch/powerpc/include/asm/kvm_book3s.h index 109a5f56767a..1a220cd63227 100644 --- a/arch/powerpc/include/asm/kvm_book3s.h +++ b/arch/powerpc/include/asm/kvm_book3s.h @@ -458,10 +458,45 @@ static inline void kvmppc_set_vscr(struct kvm_vcpu *vcpu, u32 val) } #endif +#define KVMPPC_BOOK3S_VCPU_ACCESSOR_SET(reg, size) \ +static inline void kvmppc_set_##reg(struct kvm_vcpu *vcpu, u##size val) \ +{ \ + \ + vcpu->arch.reg = val; \ +} + +#define KVMPPC_BOOK3S_VCPU_ACCESSOR_GET(reg, size) \ +static inline u##size kvmppc_get_##reg(struct kvm_vcpu *vcpu) \ +{ \ + return vcpu->arch.reg; \ +} + +#define KVMPPC_BOOK3S_VCPU_ACCESSOR(reg, size) \ + KVMPPC_BOOK3S_VCPU_ACCESSOR_SET(reg, size) \ + KVMPPC_BOOK3S_VCPU_ACCESSOR_GET(reg, size) \ + +KVMPPC_BOOK3S_VCPU_ACCESSOR(pid, 32) +KVMPPC_BOOK3S_VCPU_ACCESSOR(tar, 64) +KVMPPC_BOOK3S_VCPU_ACCESSOR(ebbhr, 64) +KVMPPC_BOOK3S_VCPU_ACCESSOR(ebbrr, 64) +KVMPPC_BOOK3S_VCPU_ACCESSOR(bescr, 64) +KVMPPC_BOOK3S_VCPU_ACCESSOR(ic, 64) +KVMPPC_BOOK3S_VCPU_ACCESSOR(vrsave, 64) + +static inline u64 kvmppc_get_dec_expires(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.dec_expires; +} + +static inline void kvmppc_set_dec_expires(struct kvm_vcpu *vcpu, u64 val) +{ + vcpu->arch.dec_expires = val; +} + /* Expiry time of vcpu DEC relative to host TB */ static inline u64 kvmppc_dec_expires_host_tb(struct kvm_vcpu *vcpu) { - return vcpu->arch.dec_expires - vcpu->arch.vcore->tb_offset; + return kvmppc_get_dec_expires(vcpu) - vcpu->arch.vcore->tb_offset; } static inline bool is_kvmppc_resume_guest(int r) diff --git a/arch/powerpc/kvm/book3s.c b/arch/powerpc/kvm/book3s.c index c080dd2e96ac..6cd20ab9e94e 100644 --- a/arch/powerpc/kvm/book3s.c +++ b/arch/powerpc/kvm/book3s.c @@ -565,7 +565,7 @@ int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) regs->msr = kvmppc_get_msr(vcpu); regs->srr0 = kvmppc_get_srr0(vcpu); regs->srr1 = kvmppc_get_srr1(vcpu); - regs->pid = vcpu->arch.pid; + regs->pid = kvmppc_get_pid(vcpu); regs->sprg0 = kvmppc_get_sprg0(vcpu); regs->sprg1 = kvmppc_get_sprg1(vcpu); regs->sprg2 = kvmppc_get_sprg2(vcpu); @@ -683,19 +683,19 @@ int kvmppc_get_one_reg(struct kvm_vcpu *vcpu, u64 id, *val = get_reg_val(id, vcpu->arch.fscr); break; case KVM_REG_PPC_TAR: - *val = get_reg_val(id, vcpu->arch.tar); + *val = get_reg_val(id, kvmppc_get_tar(vcpu)); break; case KVM_REG_PPC_EBBHR: - *val = get_reg_val(id, vcpu->arch.ebbhr); + *val = get_reg_val(id, kvmppc_get_ebbhr(vcpu)); break; case KVM_REG_PPC_EBBRR: - *val = get_reg_val(id, vcpu->arch.ebbrr); + *val = get_reg_val(id, kvmppc_get_ebbrr(vcpu)); break; case KVM_REG_PPC_BESCR: - *val = get_reg_val(id, vcpu->arch.bescr); + *val = get_reg_val(id, kvmppc_get_bescr(vcpu)); break; case KVM_REG_PPC_IC: - *val = get_reg_val(id, vcpu->arch.ic); + *val = get_reg_val(id, kvmppc_get_ic(vcpu)); break; default: r = -EINVAL; @@ -768,19 +768,19 @@ int kvmppc_set_one_reg(struct kvm_vcpu *vcpu, u64 id, kvmppc_set_fpscr(vcpu, set_reg_val(id, *val)); break; case KVM_REG_PPC_TAR: - vcpu->arch.tar = set_reg_val(id, *val); + kvmppc_set_tar(vcpu, set_reg_val(id, *val)); break; case KVM_REG_PPC_EBBHR: - vcpu->arch.ebbhr = set_reg_val(id, *val); + kvmppc_set_ebbhr(vcpu, set_reg_val(id, *val)); break; case KVM_REG_PPC_EBBRR: - vcpu->arch.ebbrr = set_reg_val(id, *val); + kvmppc_set_ebbrr(vcpu, set_reg_val(id, *val)); break; case KVM_REG_PPC_BESCR: - vcpu->arch.bescr = set_reg_val(id, *val); + kvmppc_set_bescr(vcpu, set_reg_val(id, *val)); break; case KVM_REG_PPC_IC: - vcpu->arch.ic = set_reg_val(id, *val); + kvmppc_set_ic(vcpu, set_reg_val(id, *val)); break; default: r = -EINVAL; diff --git a/arch/powerpc/kvm/book3s_64_mmu_radix.c b/arch/powerpc/kvm/book3s_64_mmu_radix.c index 572707858d65..5c71d6ae3a7b 100644 --- a/arch/powerpc/kvm/book3s_64_mmu_radix.c +++ b/arch/powerpc/kvm/book3s_64_mmu_radix.c @@ -96,7 +96,7 @@ static long kvmhv_copy_tofrom_guest_radix(struct kvm_vcpu *vcpu, gva_t eaddr, void *to, void *from, unsigned long n) { int lpid = vcpu->kvm->arch.lpid; - int pid = vcpu->arch.pid; + int pid = kvmppc_get_pid(vcpu); /* This would cause a data segment intr so don't allow the access */ if (eaddr & (0x3FFUL << 52)) @@ -270,7 +270,7 @@ int kvmppc_mmu_radix_xlate(struct kvm_vcpu *vcpu, gva_t eaddr, /* Work out effective PID */ switch (eaddr >> 62) { case 0: - pid = vcpu->arch.pid; + pid = kvmppc_get_pid(vcpu); break; case 3: pid = 0; diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c index 4af5b68cf7f8..27faecad1e3b 100644 --- a/arch/powerpc/kvm/book3s_hv.c +++ b/arch/powerpc/kvm/book3s_hv.c @@ -2310,7 +2310,7 @@ static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, *val = get_reg_val(id, vcpu->arch.tcscr); break; case KVM_REG_PPC_PID: - *val = get_reg_val(id, vcpu->arch.pid); + *val = get_reg_val(id, kvmppc_get_pid(vcpu)); break; case KVM_REG_PPC_ACOP: *val = get_reg_val(id, vcpu->arch.acop); @@ -2421,7 +2421,7 @@ static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, *val = get_reg_val(id, vcpu->arch.vcore->arch_compat); break; case KVM_REG_PPC_DEC_EXPIRY: - *val = get_reg_val(id, vcpu->arch.dec_expires); + *val = get_reg_val(id, kvmppc_get_dec_expires(vcpu)); break; case KVM_REG_PPC_ONLINE: *val = get_reg_val(id, vcpu->arch.online); @@ -2553,7 +2553,7 @@ static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, vcpu->arch.tcscr = set_reg_val(id, *val); break; case KVM_REG_PPC_PID: - vcpu->arch.pid = set_reg_val(id, *val); + kvmppc_set_pid(vcpu, set_reg_val(id, *val)); break; case KVM_REG_PPC_ACOP: vcpu->arch.acop = set_reg_val(id, *val); @@ -2606,8 +2606,8 @@ static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, * decrementer, which is better than a large one that * causes a hang. */ - if (!vcpu->arch.dec_expires && tb_offset) - vcpu->arch.dec_expires = get_tb() + tb_offset; + if (!kvmppc_get_dec_expires(vcpu) && tb_offset) + kvmppc_set_dec_expires(vcpu, get_tb() + tb_offset); vcpu->arch.vcore->tb_offset = tb_offset; break; @@ -2690,7 +2690,7 @@ static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, r = kvmppc_set_arch_compat(vcpu, set_reg_val(id, *val)); break; case KVM_REG_PPC_DEC_EXPIRY: - vcpu->arch.dec_expires = set_reg_val(id, *val); + kvmppc_set_dec_expires(vcpu, set_reg_val(id, *val)); break; case KVM_REG_PPC_ONLINE: i = set_reg_val(id, *val); diff --git a/arch/powerpc/kvm/book3s_hv_p9_entry.c b/arch/powerpc/kvm/book3s_hv_p9_entry.c index 34f1db212824..34bc0a8a1288 100644 --- a/arch/powerpc/kvm/book3s_hv_p9_entry.c +++ b/arch/powerpc/kvm/book3s_hv_p9_entry.c @@ -305,7 +305,7 @@ static void switch_mmu_to_guest_radix(struct kvm *kvm, struct kvm_vcpu *vcpu, u6 u32 pid; lpid = nested ? nested->shadow_lpid : kvm->arch.lpid; - pid = vcpu->arch.pid; + pid = kvmppc_get_pid(vcpu); /* * Prior memory accesses to host PID Q3 must be completed before we @@ -330,7 +330,7 @@ static void switch_mmu_to_guest_hpt(struct kvm *kvm, struct kvm_vcpu *vcpu, u64 int i; lpid = kvm->arch.lpid; - pid = vcpu->arch.pid; + pid = kvmppc_get_pid(vcpu); /* * See switch_mmu_to_guest_radix. ptesync should not be required here diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c index 2e72e5cd7532..f6af752698d0 100644 --- a/arch/powerpc/kvm/powerpc.c +++ b/arch/powerpc/kvm/powerpc.c @@ -1729,7 +1729,7 @@ int kvm_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu, struct kvm_one_reg *reg) val = get_reg_val(reg->id, kvmppc_get_vscr(vcpu)); break; case KVM_REG_PPC_VRSAVE: - val = get_reg_val(reg->id, vcpu->arch.vrsave); + val = get_reg_val(reg->id, kvmppc_get_vrsave(vcpu)); break; #endif /* CONFIG_ALTIVEC */ default: @@ -1784,7 +1784,7 @@ int kvm_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu, struct kvm_one_reg *reg) r = -ENXIO; break; } - vcpu->arch.vrsave = set_reg_val(reg->id, val); + kvmppc_set_vrsave(vcpu, set_reg_val(reg->id, val)); break; #endif /* CONFIG_ALTIVEC */ default: From c8ae9b3c6e7f22a4b71e42edb0fc3942aa7a7c42 Mon Sep 17 00:00:00 2001 From: Jordan Niethe Date: Thu, 14 Sep 2023 13:05:54 +1000 Subject: [PATCH 083/475] KVM: PPC: Use accessors for VCORE registers Introduce accessor generator macros for VCORE registers. Use the accessor functions to replace direct accesses to this registers. This will be important later for Nested APIv2 support which requires additional functionality for accessing and modifying VCPU state. Signed-off-by: Jordan Niethe Signed-off-by: Michael Ellerman Link: https://msgid.link/20230914030600.16993-6-jniethe5@gmail.com --- arch/powerpc/include/asm/kvm_book3s.h | 25 ++++++++++++++++++++++++- arch/powerpc/kvm/book3s_hv.c | 24 ++++++++++++------------ arch/powerpc/kvm/book3s_hv_ras.c | 4 ++-- arch/powerpc/kvm/book3s_xive.c | 4 +--- 4 files changed, 39 insertions(+), 18 deletions(-) diff --git a/arch/powerpc/include/asm/kvm_book3s.h b/arch/powerpc/include/asm/kvm_book3s.h index 1a220cd63227..4c6558d5fefe 100644 --- a/arch/powerpc/include/asm/kvm_book3s.h +++ b/arch/powerpc/include/asm/kvm_book3s.h @@ -483,6 +483,29 @@ KVMPPC_BOOK3S_VCPU_ACCESSOR(bescr, 64) KVMPPC_BOOK3S_VCPU_ACCESSOR(ic, 64) KVMPPC_BOOK3S_VCPU_ACCESSOR(vrsave, 64) + +#define KVMPPC_BOOK3S_VCORE_ACCESSOR_SET(reg, size) \ +static inline void kvmppc_set_##reg(struct kvm_vcpu *vcpu, u##size val) \ +{ \ + vcpu->arch.vcore->reg = val; \ +} + +#define KVMPPC_BOOK3S_VCORE_ACCESSOR_GET(reg, size) \ +static inline u##size kvmppc_get_##reg(struct kvm_vcpu *vcpu) \ +{ \ + return vcpu->arch.vcore->reg; \ +} + +#define KVMPPC_BOOK3S_VCORE_ACCESSOR(reg, size) \ + KVMPPC_BOOK3S_VCORE_ACCESSOR_SET(reg, size) \ + KVMPPC_BOOK3S_VCORE_ACCESSOR_GET(reg, size) \ + + +KVMPPC_BOOK3S_VCORE_ACCESSOR(vtb, 64) +KVMPPC_BOOK3S_VCORE_ACCESSOR(tb_offset, 64) +KVMPPC_BOOK3S_VCORE_ACCESSOR_GET(arch_compat, 32) +KVMPPC_BOOK3S_VCORE_ACCESSOR_GET(lpcr, 64) + static inline u64 kvmppc_get_dec_expires(struct kvm_vcpu *vcpu) { return vcpu->arch.dec_expires; @@ -496,7 +519,7 @@ static inline void kvmppc_set_dec_expires(struct kvm_vcpu *vcpu, u64 val) /* Expiry time of vcpu DEC relative to host TB */ static inline u64 kvmppc_dec_expires_host_tb(struct kvm_vcpu *vcpu) { - return kvmppc_get_dec_expires(vcpu) - vcpu->arch.vcore->tb_offset; + return kvmppc_get_dec_expires(vcpu) - kvmppc_get_tb_offset(vcpu); } static inline bool is_kvmppc_resume_guest(int r) diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c index 27faecad1e3b..73d9a9eb376f 100644 --- a/arch/powerpc/kvm/book3s_hv.c +++ b/arch/powerpc/kvm/book3s_hv.c @@ -794,7 +794,7 @@ static void kvmppc_update_vpa_dispatch(struct kvm_vcpu *vcpu, vpa->enqueue_dispatch_tb = cpu_to_be64(be64_to_cpu(vpa->enqueue_dispatch_tb) + stolen); - __kvmppc_create_dtl_entry(vcpu, vpa, vc->pcpu, now + vc->tb_offset, stolen); + __kvmppc_create_dtl_entry(vcpu, vpa, vc->pcpu, now + kvmppc_get_tb_offset(vcpu), stolen); vcpu->arch.vpa.dirty = true; } @@ -845,9 +845,9 @@ static bool kvmppc_doorbell_pending(struct kvm_vcpu *vcpu) static bool kvmppc_power8_compatible(struct kvm_vcpu *vcpu) { - if (vcpu->arch.vcore->arch_compat >= PVR_ARCH_207) + if (kvmppc_get_arch_compat(vcpu) >= PVR_ARCH_207) return true; - if ((!vcpu->arch.vcore->arch_compat) && + if ((!kvmppc_get_arch_compat(vcpu)) && cpu_has_feature(CPU_FTR_ARCH_207S)) return true; return false; @@ -2283,7 +2283,7 @@ static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, *val = get_reg_val(id, vcpu->arch.vcore->dpdes); break; case KVM_REG_PPC_VTB: - *val = get_reg_val(id, vcpu->arch.vcore->vtb); + *val = get_reg_val(id, kvmppc_get_vtb(vcpu)); break; case KVM_REG_PPC_DAWR: *val = get_reg_val(id, vcpu->arch.dawr0); @@ -2342,11 +2342,11 @@ static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, spin_unlock(&vcpu->arch.vpa_update_lock); break; case KVM_REG_PPC_TB_OFFSET: - *val = get_reg_val(id, vcpu->arch.vcore->tb_offset); + *val = get_reg_val(id, kvmppc_get_tb_offset(vcpu)); break; case KVM_REG_PPC_LPCR: case KVM_REG_PPC_LPCR_64: - *val = get_reg_val(id, vcpu->arch.vcore->lpcr); + *val = get_reg_val(id, kvmppc_get_lpcr(vcpu)); break; case KVM_REG_PPC_PPR: *val = get_reg_val(id, vcpu->arch.ppr); @@ -2418,7 +2418,7 @@ static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, break; #endif case KVM_REG_PPC_ARCH_COMPAT: - *val = get_reg_val(id, vcpu->arch.vcore->arch_compat); + *val = get_reg_val(id, kvmppc_get_arch_compat(vcpu)); break; case KVM_REG_PPC_DEC_EXPIRY: *val = get_reg_val(id, kvmppc_get_dec_expires(vcpu)); @@ -2523,7 +2523,7 @@ static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, vcpu->arch.vcore->dpdes = set_reg_val(id, *val); break; case KVM_REG_PPC_VTB: - vcpu->arch.vcore->vtb = set_reg_val(id, *val); + kvmppc_set_vtb(vcpu, set_reg_val(id, *val)); break; case KVM_REG_PPC_DAWR: vcpu->arch.dawr0 = set_reg_val(id, *val); @@ -2606,10 +2606,11 @@ static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, * decrementer, which is better than a large one that * causes a hang. */ + kvmppc_set_tb_offset(vcpu, tb_offset); if (!kvmppc_get_dec_expires(vcpu) && tb_offset) kvmppc_set_dec_expires(vcpu, get_tb() + tb_offset); - vcpu->arch.vcore->tb_offset = tb_offset; + kvmppc_set_tb_offset(vcpu, tb_offset); break; } case KVM_REG_PPC_LPCR: @@ -4042,7 +4043,6 @@ static void vcpu_vpa_increment_dispatch(struct kvm_vcpu *vcpu) /* call our hypervisor to load up HV regs and go */ static int kvmhv_vcpu_entry_p9_nested(struct kvm_vcpu *vcpu, u64 time_limit, unsigned long lpcr, u64 *tb) { - struct kvmppc_vcore *vc = vcpu->arch.vcore; unsigned long host_psscr; unsigned long msr; struct hv_guest_state hvregs; @@ -4122,7 +4122,7 @@ static int kvmhv_vcpu_entry_p9_nested(struct kvm_vcpu *vcpu, u64 time_limit, uns if (!(lpcr & LPCR_LD)) /* Sign extend if not using large decrementer */ dec = (s32) dec; *tb = mftb(); - vcpu->arch.dec_expires = dec + (*tb + vc->tb_offset); + vcpu->arch.dec_expires = dec + (*tb + kvmppc_get_tb_offset(vcpu)); timer_rearm_host_dec(*tb); @@ -4681,7 +4681,7 @@ int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit, tb = mftb(); - kvmppc_update_vpa_dispatch_p9(vcpu, vc, tb + vc->tb_offset); + kvmppc_update_vpa_dispatch_p9(vcpu, vc, tb + kvmppc_get_tb_offset(vcpu)); trace_kvm_guest_enter(vcpu); diff --git a/arch/powerpc/kvm/book3s_hv_ras.c b/arch/powerpc/kvm/book3s_hv_ras.c index 82be6d87514b..9012acadbca8 100644 --- a/arch/powerpc/kvm/book3s_hv_ras.c +++ b/arch/powerpc/kvm/book3s_hv_ras.c @@ -174,14 +174,14 @@ long kvmppc_p9_realmode_hmi_handler(struct kvm_vcpu *vcpu) ppc_md.hmi_exception_early(NULL); out: - if (vc->tb_offset) { + if (kvmppc_get_tb_offset(vcpu)) { u64 new_tb = mftb() + vc->tb_offset; mtspr(SPRN_TBU40, new_tb); if ((mftb() & 0xffffff) < (new_tb & 0xffffff)) { new_tb += 0x1000000; mtspr(SPRN_TBU40, new_tb); } - vc->tb_offset_applied = vc->tb_offset; + vc->tb_offset_applied = kvmppc_get_tb_offset(vcpu); } return ret; diff --git a/arch/powerpc/kvm/book3s_xive.c b/arch/powerpc/kvm/book3s_xive.c index 48d11baf1f16..24d8378824a2 100644 --- a/arch/powerpc/kvm/book3s_xive.c +++ b/arch/powerpc/kvm/book3s_xive.c @@ -2779,8 +2779,6 @@ static int kvmppc_xive_create(struct kvm_device *dev, u32 type) int kvmppc_xive_xics_hcall(struct kvm_vcpu *vcpu, u32 req) { - struct kvmppc_vcore *vc = vcpu->arch.vcore; - /* The VM should have configured XICS mode before doing XICS hcalls. */ if (!kvmppc_xics_enabled(vcpu)) return H_TOO_HARD; @@ -2799,7 +2797,7 @@ int kvmppc_xive_xics_hcall(struct kvm_vcpu *vcpu, u32 req) return xive_vm_h_ipoll(vcpu, kvmppc_get_gpr(vcpu, 4)); case H_XIRR_X: xive_vm_h_xirr(vcpu); - kvmppc_set_gpr(vcpu, 5, get_tb() + vc->tb_offset); + kvmppc_set_gpr(vcpu, 5, get_tb() + kvmppc_get_tb_offset(vcpu)); return H_SUCCESS; } From ebc88ea7a6ad0ea349df9c765357d3aa4e662aa9 Mon Sep 17 00:00:00 2001 From: Jordan Niethe Date: Thu, 14 Sep 2023 13:05:55 +1000 Subject: [PATCH 084/475] KVM: PPC: Book3S HV: Use accessors for VCPU registers Introduce accessor generator macros for Book3S HV VCPU registers. Use the accessor functions to replace direct accesses to this registers. This will be important later for Nested APIv2 support which requires additional functionality for accessing and modifying VCPU state. Signed-off-by: Jordan Niethe Signed-off-by: Michael Ellerman Link: https://msgid.link/20230914030600.16993-7-jniethe5@gmail.com --- arch/powerpc/kvm/book3s_64_mmu_radix.c | 5 +- arch/powerpc/kvm/book3s_hv.c | 148 +++++++++++++------------ arch/powerpc/kvm/book3s_hv.h | 58 ++++++++++ 3 files changed, 139 insertions(+), 72 deletions(-) diff --git a/arch/powerpc/kvm/book3s_64_mmu_radix.c b/arch/powerpc/kvm/book3s_64_mmu_radix.c index 5c71d6ae3a7b..ab646f59afd7 100644 --- a/arch/powerpc/kvm/book3s_64_mmu_radix.c +++ b/arch/powerpc/kvm/book3s_64_mmu_radix.c @@ -15,6 +15,7 @@ #include #include +#include "book3s_hv.h" #include #include #include @@ -294,9 +295,9 @@ int kvmppc_mmu_radix_xlate(struct kvm_vcpu *vcpu, gva_t eaddr, } else { if (!(pte & _PAGE_PRIVILEGED)) { /* Check AMR/IAMR to see if strict mode is in force */ - if (vcpu->arch.amr & (1ul << 62)) + if (kvmppc_get_amr_hv(vcpu) & (1ul << 62)) gpte->may_read = 0; - if (vcpu->arch.amr & (1ul << 63)) + if (kvmppc_get_amr_hv(vcpu) & (1ul << 63)) gpte->may_write = 0; if (vcpu->arch.iamr & (1ul << 62)) gpte->may_execute = 0; diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c index 73d9a9eb376f..25025f6c4cce 100644 --- a/arch/powerpc/kvm/book3s_hv.c +++ b/arch/powerpc/kvm/book3s_hv.c @@ -868,7 +868,7 @@ static int kvmppc_h_set_mode(struct kvm_vcpu *vcpu, unsigned long mflags, /* Guests can't breakpoint the hypervisor */ if ((value1 & CIABR_PRIV) == CIABR_PRIV_HYPER) return H_P3; - vcpu->arch.ciabr = value1; + kvmppc_set_ciabr_hv(vcpu, value1); return H_SUCCESS; case H_SET_MODE_RESOURCE_SET_DAWR0: if (!kvmppc_power8_compatible(vcpu)) @@ -879,8 +879,8 @@ static int kvmppc_h_set_mode(struct kvm_vcpu *vcpu, unsigned long mflags, return H_UNSUPPORTED_FLAG_START; if (value2 & DABRX_HYP) return H_P4; - vcpu->arch.dawr0 = value1; - vcpu->arch.dawrx0 = value2; + kvmppc_set_dawr0_hv(vcpu, value1); + kvmppc_set_dawrx0_hv(vcpu, value2); return H_SUCCESS; case H_SET_MODE_RESOURCE_SET_DAWR1: if (!kvmppc_power8_compatible(vcpu)) @@ -895,8 +895,8 @@ static int kvmppc_h_set_mode(struct kvm_vcpu *vcpu, unsigned long mflags, return H_UNSUPPORTED_FLAG_START; if (value2 & DABRX_HYP) return H_P4; - vcpu->arch.dawr1 = value1; - vcpu->arch.dawrx1 = value2; + kvmppc_set_dawr1_hv(vcpu, value1); + kvmppc_set_dawrx1_hv(vcpu, value2); return H_SUCCESS; case H_SET_MODE_RESOURCE_ADDR_TRANS_MODE: /* @@ -1548,7 +1548,7 @@ static int kvmppc_pmu_unavailable(struct kvm_vcpu *vcpu) if (!(vcpu->arch.hfscr_permitted & HFSCR_PM)) return EMULATE_FAIL; - vcpu->arch.hfscr |= HFSCR_PM; + kvmppc_set_hfscr_hv(vcpu, kvmppc_get_hfscr_hv(vcpu) | HFSCR_PM); return RESUME_GUEST; } @@ -1558,7 +1558,7 @@ static int kvmppc_ebb_unavailable(struct kvm_vcpu *vcpu) if (!(vcpu->arch.hfscr_permitted & HFSCR_EBB)) return EMULATE_FAIL; - vcpu->arch.hfscr |= HFSCR_EBB; + kvmppc_set_hfscr_hv(vcpu, kvmppc_get_hfscr_hv(vcpu) | HFSCR_EBB); return RESUME_GUEST; } @@ -1568,7 +1568,7 @@ static int kvmppc_tm_unavailable(struct kvm_vcpu *vcpu) if (!(vcpu->arch.hfscr_permitted & HFSCR_TM)) return EMULATE_FAIL; - vcpu->arch.hfscr |= HFSCR_TM; + kvmppc_set_hfscr_hv(vcpu, kvmppc_get_hfscr_hv(vcpu) | HFSCR_TM); return RESUME_GUEST; } @@ -1867,7 +1867,7 @@ static int kvmppc_handle_exit_hv(struct kvm_vcpu *vcpu, * Otherwise, we just generate a program interrupt to the guest. */ case BOOK3S_INTERRUPT_H_FAC_UNAVAIL: { - u64 cause = vcpu->arch.hfscr >> 56; + u64 cause = kvmppc_get_hfscr_hv(vcpu) >> 56; r = EMULATE_FAIL; if (cpu_has_feature(CPU_FTR_ARCH_300)) { @@ -2211,64 +2211,64 @@ static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, *val = get_reg_val(id, vcpu->arch.dabrx); break; case KVM_REG_PPC_DSCR: - *val = get_reg_val(id, vcpu->arch.dscr); + *val = get_reg_val(id, kvmppc_get_dscr_hv(vcpu)); break; case KVM_REG_PPC_PURR: - *val = get_reg_val(id, vcpu->arch.purr); + *val = get_reg_val(id, kvmppc_get_purr_hv(vcpu)); break; case KVM_REG_PPC_SPURR: - *val = get_reg_val(id, vcpu->arch.spurr); + *val = get_reg_val(id, kvmppc_get_spurr_hv(vcpu)); break; case KVM_REG_PPC_AMR: - *val = get_reg_val(id, vcpu->arch.amr); + *val = get_reg_val(id, kvmppc_get_amr_hv(vcpu)); break; case KVM_REG_PPC_UAMOR: - *val = get_reg_val(id, vcpu->arch.uamor); + *val = get_reg_val(id, kvmppc_get_uamor_hv(vcpu)); break; case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCR1: i = id - KVM_REG_PPC_MMCR0; - *val = get_reg_val(id, vcpu->arch.mmcr[i]); + *val = get_reg_val(id, kvmppc_get_mmcr_hv(vcpu, i)); break; case KVM_REG_PPC_MMCR2: - *val = get_reg_val(id, vcpu->arch.mmcr[2]); + *val = get_reg_val(id, kvmppc_get_mmcr_hv(vcpu, 2)); break; case KVM_REG_PPC_MMCRA: - *val = get_reg_val(id, vcpu->arch.mmcra); + *val = get_reg_val(id, kvmppc_get_mmcra_hv(vcpu)); break; case KVM_REG_PPC_MMCRS: *val = get_reg_val(id, vcpu->arch.mmcrs); break; case KVM_REG_PPC_MMCR3: - *val = get_reg_val(id, vcpu->arch.mmcr[3]); + *val = get_reg_val(id, kvmppc_get_mmcr_hv(vcpu, 3)); break; case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8: i = id - KVM_REG_PPC_PMC1; - *val = get_reg_val(id, vcpu->arch.pmc[i]); + *val = get_reg_val(id, kvmppc_get_pmc_hv(vcpu, i)); break; case KVM_REG_PPC_SPMC1 ... KVM_REG_PPC_SPMC2: i = id - KVM_REG_PPC_SPMC1; *val = get_reg_val(id, vcpu->arch.spmc[i]); break; case KVM_REG_PPC_SIAR: - *val = get_reg_val(id, vcpu->arch.siar); + *val = get_reg_val(id, kvmppc_get_siar_hv(vcpu)); break; case KVM_REG_PPC_SDAR: - *val = get_reg_val(id, vcpu->arch.sdar); + *val = get_reg_val(id, kvmppc_get_siar_hv(vcpu)); break; case KVM_REG_PPC_SIER: - *val = get_reg_val(id, vcpu->arch.sier[0]); + *val = get_reg_val(id, kvmppc_get_sier_hv(vcpu, 0)); break; case KVM_REG_PPC_SIER2: - *val = get_reg_val(id, vcpu->arch.sier[1]); + *val = get_reg_val(id, kvmppc_get_sier_hv(vcpu, 1)); break; case KVM_REG_PPC_SIER3: - *val = get_reg_val(id, vcpu->arch.sier[2]); + *val = get_reg_val(id, kvmppc_get_sier_hv(vcpu, 2)); break; case KVM_REG_PPC_IAMR: - *val = get_reg_val(id, vcpu->arch.iamr); + *val = get_reg_val(id, kvmppc_get_iamr_hv(vcpu)); break; case KVM_REG_PPC_PSPB: - *val = get_reg_val(id, vcpu->arch.pspb); + *val = get_reg_val(id, kvmppc_get_pspb_hv(vcpu)); break; case KVM_REG_PPC_DPDES: /* @@ -2286,19 +2286,19 @@ static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, *val = get_reg_val(id, kvmppc_get_vtb(vcpu)); break; case KVM_REG_PPC_DAWR: - *val = get_reg_val(id, vcpu->arch.dawr0); + *val = get_reg_val(id, kvmppc_get_dawr0_hv(vcpu)); break; case KVM_REG_PPC_DAWRX: - *val = get_reg_val(id, vcpu->arch.dawrx0); + *val = get_reg_val(id, kvmppc_get_dawrx0_hv(vcpu)); break; case KVM_REG_PPC_DAWR1: - *val = get_reg_val(id, vcpu->arch.dawr1); + *val = get_reg_val(id, kvmppc_get_dawr1_hv(vcpu)); break; case KVM_REG_PPC_DAWRX1: - *val = get_reg_val(id, vcpu->arch.dawrx1); + *val = get_reg_val(id, kvmppc_get_dawrx1_hv(vcpu)); break; case KVM_REG_PPC_CIABR: - *val = get_reg_val(id, vcpu->arch.ciabr); + *val = get_reg_val(id, kvmppc_get_ciabr_hv(vcpu)); break; case KVM_REG_PPC_CSIGR: *val = get_reg_val(id, vcpu->arch.csigr); @@ -2316,7 +2316,7 @@ static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, *val = get_reg_val(id, vcpu->arch.acop); break; case KVM_REG_PPC_WORT: - *val = get_reg_val(id, vcpu->arch.wort); + *val = get_reg_val(id, kvmppc_get_wort_hv(vcpu)); break; case KVM_REG_PPC_TIDR: *val = get_reg_val(id, vcpu->arch.tid); @@ -2349,7 +2349,7 @@ static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, *val = get_reg_val(id, kvmppc_get_lpcr(vcpu)); break; case KVM_REG_PPC_PPR: - *val = get_reg_val(id, vcpu->arch.ppr); + *val = get_reg_val(id, kvmppc_get_ppr_hv(vcpu)); break; #ifdef CONFIG_PPC_TRANSACTIONAL_MEM case KVM_REG_PPC_TFHAR: @@ -2429,6 +2429,9 @@ static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, case KVM_REG_PPC_PTCR: *val = get_reg_val(id, vcpu->kvm->arch.l1_ptcr); break; + case KVM_REG_PPC_FSCR: + *val = get_reg_val(id, kvmppc_get_fscr_hv(vcpu)); + break; default: r = -EINVAL; break; @@ -2457,29 +2460,29 @@ static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, vcpu->arch.dabrx = set_reg_val(id, *val) & ~DABRX_HYP; break; case KVM_REG_PPC_DSCR: - vcpu->arch.dscr = set_reg_val(id, *val); + kvmppc_set_dscr_hv(vcpu, set_reg_val(id, *val)); break; case KVM_REG_PPC_PURR: - vcpu->arch.purr = set_reg_val(id, *val); + kvmppc_set_purr_hv(vcpu, set_reg_val(id, *val)); break; case KVM_REG_PPC_SPURR: - vcpu->arch.spurr = set_reg_val(id, *val); + kvmppc_set_spurr_hv(vcpu, set_reg_val(id, *val)); break; case KVM_REG_PPC_AMR: - vcpu->arch.amr = set_reg_val(id, *val); + kvmppc_set_amr_hv(vcpu, set_reg_val(id, *val)); break; case KVM_REG_PPC_UAMOR: - vcpu->arch.uamor = set_reg_val(id, *val); + kvmppc_set_uamor_hv(vcpu, set_reg_val(id, *val)); break; case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCR1: i = id - KVM_REG_PPC_MMCR0; - vcpu->arch.mmcr[i] = set_reg_val(id, *val); + kvmppc_set_mmcr_hv(vcpu, i, set_reg_val(id, *val)); break; case KVM_REG_PPC_MMCR2: - vcpu->arch.mmcr[2] = set_reg_val(id, *val); + kvmppc_set_mmcr_hv(vcpu, 2, set_reg_val(id, *val)); break; case KVM_REG_PPC_MMCRA: - vcpu->arch.mmcra = set_reg_val(id, *val); + kvmppc_set_mmcra_hv(vcpu, set_reg_val(id, *val)); break; case KVM_REG_PPC_MMCRS: vcpu->arch.mmcrs = set_reg_val(id, *val); @@ -2489,32 +2492,32 @@ static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, break; case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8: i = id - KVM_REG_PPC_PMC1; - vcpu->arch.pmc[i] = set_reg_val(id, *val); + kvmppc_set_pmc_hv(vcpu, i, set_reg_val(id, *val)); break; case KVM_REG_PPC_SPMC1 ... KVM_REG_PPC_SPMC2: i = id - KVM_REG_PPC_SPMC1; vcpu->arch.spmc[i] = set_reg_val(id, *val); break; case KVM_REG_PPC_SIAR: - vcpu->arch.siar = set_reg_val(id, *val); + kvmppc_set_siar_hv(vcpu, set_reg_val(id, *val)); break; case KVM_REG_PPC_SDAR: - vcpu->arch.sdar = set_reg_val(id, *val); + kvmppc_set_sdar_hv(vcpu, set_reg_val(id, *val)); break; case KVM_REG_PPC_SIER: - vcpu->arch.sier[0] = set_reg_val(id, *val); + kvmppc_set_sier_hv(vcpu, 0, set_reg_val(id, *val)); break; case KVM_REG_PPC_SIER2: - vcpu->arch.sier[1] = set_reg_val(id, *val); + kvmppc_set_sier_hv(vcpu, 1, set_reg_val(id, *val)); break; case KVM_REG_PPC_SIER3: - vcpu->arch.sier[2] = set_reg_val(id, *val); + kvmppc_set_sier_hv(vcpu, 2, set_reg_val(id, *val)); break; case KVM_REG_PPC_IAMR: - vcpu->arch.iamr = set_reg_val(id, *val); + kvmppc_set_iamr_hv(vcpu, set_reg_val(id, *val)); break; case KVM_REG_PPC_PSPB: - vcpu->arch.pspb = set_reg_val(id, *val); + kvmppc_set_pspb_hv(vcpu, set_reg_val(id, *val)); break; case KVM_REG_PPC_DPDES: if (cpu_has_feature(CPU_FTR_ARCH_300)) @@ -2526,22 +2529,22 @@ static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, kvmppc_set_vtb(vcpu, set_reg_val(id, *val)); break; case KVM_REG_PPC_DAWR: - vcpu->arch.dawr0 = set_reg_val(id, *val); + kvmppc_set_dawr0_hv(vcpu, set_reg_val(id, *val)); break; case KVM_REG_PPC_DAWRX: - vcpu->arch.dawrx0 = set_reg_val(id, *val) & ~DAWRX_HYP; + kvmppc_set_dawrx0_hv(vcpu, set_reg_val(id, *val) & ~DAWRX_HYP); break; case KVM_REG_PPC_DAWR1: - vcpu->arch.dawr1 = set_reg_val(id, *val); + kvmppc_set_dawr1_hv(vcpu, set_reg_val(id, *val)); break; case KVM_REG_PPC_DAWRX1: - vcpu->arch.dawrx1 = set_reg_val(id, *val) & ~DAWRX_HYP; + kvmppc_set_dawrx1_hv(vcpu, set_reg_val(id, *val) & ~DAWRX_HYP); break; case KVM_REG_PPC_CIABR: - vcpu->arch.ciabr = set_reg_val(id, *val); + kvmppc_set_ciabr_hv(vcpu, set_reg_val(id, *val)); /* Don't allow setting breakpoints in hypervisor code */ - if ((vcpu->arch.ciabr & CIABR_PRIV) == CIABR_PRIV_HYPER) - vcpu->arch.ciabr &= ~CIABR_PRIV; /* disable */ + if ((kvmppc_get_ciabr_hv(vcpu) & CIABR_PRIV) == CIABR_PRIV_HYPER) + kvmppc_set_ciabr_hv(vcpu, kvmppc_get_ciabr_hv(vcpu) & ~CIABR_PRIV); break; case KVM_REG_PPC_CSIGR: vcpu->arch.csigr = set_reg_val(id, *val); @@ -2559,7 +2562,7 @@ static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, vcpu->arch.acop = set_reg_val(id, *val); break; case KVM_REG_PPC_WORT: - vcpu->arch.wort = set_reg_val(id, *val); + kvmppc_set_wort_hv(vcpu, set_reg_val(id, *val)); break; case KVM_REG_PPC_TIDR: vcpu->arch.tid = set_reg_val(id, *val); @@ -2620,7 +2623,7 @@ static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, kvmppc_set_lpcr(vcpu, set_reg_val(id, *val), false); break; case KVM_REG_PPC_PPR: - vcpu->arch.ppr = set_reg_val(id, *val); + kvmppc_set_ppr_hv(vcpu, set_reg_val(id, *val)); break; #ifdef CONFIG_PPC_TRANSACTIONAL_MEM case KVM_REG_PPC_TFHAR: @@ -2704,6 +2707,9 @@ static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, case KVM_REG_PPC_PTCR: vcpu->kvm->arch.l1_ptcr = set_reg_val(id, *val); break; + case KVM_REG_PPC_FSCR: + kvmppc_set_fscr_hv(vcpu, set_reg_val(id, *val)); + break; default: r = -EINVAL; break; @@ -2921,13 +2927,14 @@ static int kvmppc_core_vcpu_create_hv(struct kvm_vcpu *vcpu) vcpu->arch.shared_big_endian = false; #endif #endif - vcpu->arch.mmcr[0] = MMCR0_FC; + kvmppc_set_mmcr_hv(vcpu, 0, MMCR0_FC); + if (cpu_has_feature(CPU_FTR_ARCH_31)) { - vcpu->arch.mmcr[0] |= MMCR0_PMCCEXT; - vcpu->arch.mmcra = MMCRA_BHRB_DISABLE; + kvmppc_set_mmcr_hv(vcpu, 0, kvmppc_get_mmcr_hv(vcpu, 0) | MMCR0_PMCCEXT); + kvmppc_set_mmcra_hv(vcpu, MMCRA_BHRB_DISABLE); } - vcpu->arch.ctrl = CTRL_RUNLATCH; + kvmppc_set_ctrl_hv(vcpu, CTRL_RUNLATCH); /* default to host PVR, since we can't spoof it */ kvmppc_set_pvr_hv(vcpu, mfspr(SPRN_PVR)); spin_lock_init(&vcpu->arch.vpa_update_lock); @@ -2943,29 +2950,30 @@ static int kvmppc_core_vcpu_create_hv(struct kvm_vcpu *vcpu) * don't set the HFSCR_MSGP bit, and that causes those instructions * to trap and then we emulate them. */ - vcpu->arch.hfscr = HFSCR_TAR | HFSCR_EBB | HFSCR_PM | HFSCR_BHRB | - HFSCR_DSCR | HFSCR_VECVSX | HFSCR_FP; + kvmppc_set_hfscr_hv(vcpu, HFSCR_TAR | HFSCR_EBB | HFSCR_PM | HFSCR_BHRB | + HFSCR_DSCR | HFSCR_VECVSX | HFSCR_FP); /* On POWER10 and later, allow prefixed instructions */ if (cpu_has_feature(CPU_FTR_ARCH_31)) - vcpu->arch.hfscr |= HFSCR_PREFIX; + kvmppc_set_hfscr_hv(vcpu, kvmppc_get_hfscr_hv(vcpu) | HFSCR_PREFIX); if (cpu_has_feature(CPU_FTR_HVMODE)) { - vcpu->arch.hfscr &= mfspr(SPRN_HFSCR); + kvmppc_set_hfscr_hv(vcpu, kvmppc_get_hfscr_hv(vcpu) & mfspr(SPRN_HFSCR)); + #ifdef CONFIG_PPC_TRANSACTIONAL_MEM if (cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) - vcpu->arch.hfscr |= HFSCR_TM; + kvmppc_set_hfscr_hv(vcpu, kvmppc_get_hfscr_hv(vcpu) | HFSCR_TM); #endif } if (cpu_has_feature(CPU_FTR_TM_COMP)) vcpu->arch.hfscr |= HFSCR_TM; - vcpu->arch.hfscr_permitted = vcpu->arch.hfscr; + vcpu->arch.hfscr_permitted = kvmppc_get_hfscr_hv(vcpu); /* * PM, EBB, TM are demand-faulted so start with it clear. */ - vcpu->arch.hfscr &= ~(HFSCR_PM | HFSCR_EBB | HFSCR_TM); + kvmppc_set_hfscr_hv(vcpu, kvmppc_get_hfscr_hv(vcpu) & ~(HFSCR_PM | HFSCR_EBB | HFSCR_TM)); kvmppc_mmu_book3s_hv_init(vcpu); @@ -4848,7 +4856,7 @@ static int kvmppc_vcpu_run_hv(struct kvm_vcpu *vcpu) msr |= MSR_VSX; if ((cpu_has_feature(CPU_FTR_TM) || cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST)) && - (vcpu->arch.hfscr & HFSCR_TM)) + (kvmppc_get_hfscr_hv(vcpu) & HFSCR_TM)) msr |= MSR_TM; msr = msr_check_and_set(msr); diff --git a/arch/powerpc/kvm/book3s_hv.h b/arch/powerpc/kvm/book3s_hv.h index 2f2e59d7d433..acd9a7a95bbf 100644 --- a/arch/powerpc/kvm/book3s_hv.h +++ b/arch/powerpc/kvm/book3s_hv.h @@ -50,3 +50,61 @@ void accumulate_time(struct kvm_vcpu *vcpu, struct kvmhv_tb_accumulator *next); #define start_timing(vcpu, next) do {} while (0) #define end_timing(vcpu) do {} while (0) #endif + +#define KVMPPC_BOOK3S_HV_VCPU_ACCESSOR_SET(reg, size) \ +static inline void kvmppc_set_##reg ##_hv(struct kvm_vcpu *vcpu, u##size val) \ +{ \ + vcpu->arch.reg = val; \ +} + +#define KVMPPC_BOOK3S_HV_VCPU_ACCESSOR_GET(reg, size) \ +static inline u##size kvmppc_get_##reg ##_hv(struct kvm_vcpu *vcpu) \ +{ \ + return vcpu->arch.reg; \ +} + +#define KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(reg, size) \ + KVMPPC_BOOK3S_HV_VCPU_ACCESSOR_SET(reg, size) \ + KVMPPC_BOOK3S_HV_VCPU_ACCESSOR_GET(reg, size) \ + +#define KVMPPC_BOOK3S_HV_VCPU_ARRAY_ACCESSOR_SET(reg, size) \ +static inline void kvmppc_set_##reg ##_hv(struct kvm_vcpu *vcpu, int i, u##size val) \ +{ \ + vcpu->arch.reg[i] = val; \ +} + +#define KVMPPC_BOOK3S_HV_VCPU_ARRAY_ACCESSOR_GET(reg, size) \ +static inline u##size kvmppc_get_##reg ##_hv(struct kvm_vcpu *vcpu, int i) \ +{ \ + return vcpu->arch.reg[i]; \ +} + +#define KVMPPC_BOOK3S_HV_VCPU_ARRAY_ACCESSOR(reg, size) \ + KVMPPC_BOOK3S_HV_VCPU_ARRAY_ACCESSOR_SET(reg, size) \ + KVMPPC_BOOK3S_HV_VCPU_ARRAY_ACCESSOR_GET(reg, size) \ + +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(mmcra, 64) +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(hfscr, 64) +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(fscr, 64) +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(dscr, 64) +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(purr, 64) +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(spurr, 64) +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(amr, 64) +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(uamor, 64) +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(siar, 64) +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(sdar, 64) +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(iamr, 64) +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(dawr0, 64) +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(dawr1, 64) +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(dawrx0, 64) +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(dawrx1, 64) +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(ciabr, 64) +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(wort, 64) +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(ppr, 64) +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(ctrl, 64) + +KVMPPC_BOOK3S_HV_VCPU_ARRAY_ACCESSOR(mmcr, 64) +KVMPPC_BOOK3S_HV_VCPU_ARRAY_ACCESSOR(sier, 64) +KVMPPC_BOOK3S_HV_VCPU_ARRAY_ACCESSOR(pmc, 32) + +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(pspb, 32) From 6de2e837babb411cfb3cdb570581c3a65576ddaf Mon Sep 17 00:00:00 2001 From: Jordan Niethe Date: Thu, 14 Sep 2023 13:05:56 +1000 Subject: [PATCH 085/475] KVM: PPC: Book3S HV: Introduce low level MSR accessor kvmppc_get_msr() and kvmppc_set_msr_fast() serve as accessors for the MSR. However because the MSR is kept in the shared regs they include a conditional check for kvmppc_shared_big_endian() and endian conversion. Within the Book3S HV specific code there are direct reads and writes of shregs::msr. In preparation for Nested APIv2 these accesses need to be replaced with accessor functions so it is possible to extend their behavior. However, using the kvmppc_get_msr() and kvmppc_set_msr_fast() functions is undesirable because it would introduce a conditional branch and endian conversion that is not currently present. kvmppc_set_msr_hv() already exists, it is used for the kvmppc_ops::set_msr callback. Introduce a low level accessor __kvmppc_{s,g}et_msr_hv() that simply gets and sets shregs::msr. This will be extend for Nested APIv2 support. Signed-off-by: Jordan Niethe Signed-off-by: Michael Ellerman Link: https://msgid.link/20230914030600.16993-8-jniethe5@gmail.com --- arch/powerpc/kvm/book3s_64_mmu_hv.c | 5 ++-- arch/powerpc/kvm/book3s_hv.c | 34 ++++++++++++++-------------- arch/powerpc/kvm/book3s_hv.h | 10 ++++++++ arch/powerpc/kvm/book3s_hv_builtin.c | 5 ++-- 4 files changed, 33 insertions(+), 21 deletions(-) diff --git a/arch/powerpc/kvm/book3s_64_mmu_hv.c b/arch/powerpc/kvm/book3s_64_mmu_hv.c index efd0ebf70a5e..fdfc2a62dd67 100644 --- a/arch/powerpc/kvm/book3s_64_mmu_hv.c +++ b/arch/powerpc/kvm/book3s_64_mmu_hv.c @@ -28,6 +28,7 @@ #include #include "book3s.h" +#include "book3s_hv.h" #include "trace_hv.h" //#define DEBUG_RESIZE_HPT 1 @@ -347,7 +348,7 @@ static int kvmppc_mmu_book3s_64_hv_xlate(struct kvm_vcpu *vcpu, gva_t eaddr, unsigned long v, orig_v, gr; __be64 *hptep; long int index; - int virtmode = vcpu->arch.shregs.msr & (data ? MSR_DR : MSR_IR); + int virtmode = __kvmppc_get_msr_hv(vcpu) & (data ? MSR_DR : MSR_IR); if (kvm_is_radix(vcpu->kvm)) return kvmppc_mmu_radix_xlate(vcpu, eaddr, gpte, data, iswrite); @@ -385,7 +386,7 @@ static int kvmppc_mmu_book3s_64_hv_xlate(struct kvm_vcpu *vcpu, gva_t eaddr, /* Get PP bits and key for permission check */ pp = gr & (HPTE_R_PP0 | HPTE_R_PP); - key = (vcpu->arch.shregs.msr & MSR_PR) ? SLB_VSID_KP : SLB_VSID_KS; + key = (__kvmppc_get_msr_hv(vcpu) & MSR_PR) ? SLB_VSID_KP : SLB_VSID_KS; key &= slb_v; /* Calculate permissions */ diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c index 25025f6c4cce..5743f32bf45e 100644 --- a/arch/powerpc/kvm/book3s_hv.c +++ b/arch/powerpc/kvm/book3s_hv.c @@ -1374,7 +1374,7 @@ int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu) */ static void kvmppc_cede(struct kvm_vcpu *vcpu) { - vcpu->arch.shregs.msr |= MSR_EE; + __kvmppc_set_msr_hv(vcpu, __kvmppc_get_msr_hv(vcpu) | MSR_EE); vcpu->arch.ceded = 1; smp_mb(); if (vcpu->arch.prodded) { @@ -1589,7 +1589,7 @@ static int kvmppc_handle_exit_hv(struct kvm_vcpu *vcpu, * That can happen due to a bug, or due to a machine check * occurring at just the wrong time. */ - if (vcpu->arch.shregs.msr & MSR_HV) { + if (__kvmppc_get_msr_hv(vcpu) & MSR_HV) { printk(KERN_EMERG "KVM trap in HV mode!\n"); printk(KERN_EMERG "trap=0x%x | pc=0x%lx | msr=0x%llx\n", vcpu->arch.trap, kvmppc_get_pc(vcpu), @@ -1640,7 +1640,7 @@ static int kvmppc_handle_exit_hv(struct kvm_vcpu *vcpu, * so that it knows that the machine check occurred. */ if (!vcpu->kvm->arch.fwnmi_enabled) { - ulong flags = (vcpu->arch.shregs.msr & 0x083c0000) | + ulong flags = (__kvmppc_get_msr_hv(vcpu) & 0x083c0000) | (kvmppc_get_msr(vcpu) & SRR1_PREFIXED); kvmppc_core_queue_machine_check(vcpu, flags); r = RESUME_GUEST; @@ -1670,7 +1670,7 @@ static int kvmppc_handle_exit_hv(struct kvm_vcpu *vcpu, * as a result of a hypervisor emulation interrupt * (e40) getting turned into a 700 by BML RTAS. */ - flags = (vcpu->arch.shregs.msr & 0x1f0000ull) | + flags = (__kvmppc_get_msr_hv(vcpu) & 0x1f0000ull) | (kvmppc_get_msr(vcpu) & SRR1_PREFIXED); kvmppc_core_queue_program(vcpu, flags); r = RESUME_GUEST; @@ -1680,7 +1680,7 @@ static int kvmppc_handle_exit_hv(struct kvm_vcpu *vcpu, { int i; - if (unlikely(vcpu->arch.shregs.msr & MSR_PR)) { + if (unlikely(__kvmppc_get_msr_hv(vcpu) & MSR_PR)) { /* * Guest userspace executed sc 1. This can only be * reached by the P9 path because the old path @@ -1758,7 +1758,7 @@ static int kvmppc_handle_exit_hv(struct kvm_vcpu *vcpu, break; } - if (!(vcpu->arch.shregs.msr & MSR_DR)) + if (!(__kvmppc_get_msr_hv(vcpu) & MSR_DR)) vsid = vcpu->kvm->arch.vrma_slb_v; else vsid = vcpu->arch.fault_gpa; @@ -1782,7 +1782,7 @@ static int kvmppc_handle_exit_hv(struct kvm_vcpu *vcpu, long err; vcpu->arch.fault_dar = kvmppc_get_pc(vcpu); - vcpu->arch.fault_dsisr = vcpu->arch.shregs.msr & + vcpu->arch.fault_dsisr = __kvmppc_get_msr_hv(vcpu) & DSISR_SRR1_MATCH_64S; if (kvm_is_radix(vcpu->kvm) || !cpu_has_feature(CPU_FTR_ARCH_300)) { /* @@ -1791,7 +1791,7 @@ static int kvmppc_handle_exit_hv(struct kvm_vcpu *vcpu, * hash fault handling below is v3 only (it uses ASDR * via fault_gpa). */ - if (vcpu->arch.shregs.msr & HSRR1_HISI_WRITE) + if (__kvmppc_get_msr_hv(vcpu) & HSRR1_HISI_WRITE) vcpu->arch.fault_dsisr |= DSISR_ISSTORE; r = RESUME_PAGE_FAULT; break; @@ -1805,7 +1805,7 @@ static int kvmppc_handle_exit_hv(struct kvm_vcpu *vcpu, break; } - if (!(vcpu->arch.shregs.msr & MSR_IR)) + if (!(__kvmppc_get_msr_hv(vcpu) & MSR_IR)) vsid = vcpu->kvm->arch.vrma_slb_v; else vsid = vcpu->arch.fault_gpa; @@ -1895,7 +1895,7 @@ static int kvmppc_handle_exit_hv(struct kvm_vcpu *vcpu, kvmppc_dump_regs(vcpu); printk(KERN_EMERG "trap=0x%x | pc=0x%lx | msr=0x%llx\n", vcpu->arch.trap, kvmppc_get_pc(vcpu), - vcpu->arch.shregs.msr); + __kvmppc_get_msr_hv(vcpu)); run->hw.hardware_exit_reason = vcpu->arch.trap; r = RESUME_HOST; break; @@ -1919,11 +1919,11 @@ static int kvmppc_handle_nested_exit(struct kvm_vcpu *vcpu) * That can happen due to a bug, or due to a machine check * occurring at just the wrong time. */ - if (vcpu->arch.shregs.msr & MSR_HV) { + if (__kvmppc_get_msr_hv(vcpu) & MSR_HV) { pr_emerg("KVM trap in HV mode while nested!\n"); pr_emerg("trap=0x%x | pc=0x%lx | msr=0x%llx\n", vcpu->arch.trap, kvmppc_get_pc(vcpu), - vcpu->arch.shregs.msr); + __kvmppc_get_msr_hv(vcpu)); kvmppc_dump_regs(vcpu); return RESUME_HOST; } @@ -1980,7 +1980,7 @@ static int kvmppc_handle_nested_exit(struct kvm_vcpu *vcpu) vcpu->arch.fault_dar = kvmppc_get_pc(vcpu); vcpu->arch.fault_dsisr = kvmppc_get_msr(vcpu) & DSISR_SRR1_MATCH_64S; - if (vcpu->arch.shregs.msr & HSRR1_HISI_WRITE) + if (__kvmppc_get_msr_hv(vcpu) & HSRR1_HISI_WRITE) vcpu->arch.fault_dsisr |= DSISR_ISSTORE; srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); r = kvmhv_nested_page_fault(vcpu); @@ -2940,7 +2940,7 @@ static int kvmppc_core_vcpu_create_hv(struct kvm_vcpu *vcpu) spin_lock_init(&vcpu->arch.vpa_update_lock); spin_lock_init(&vcpu->arch.tbacct_lock); vcpu->arch.busy_preempt = TB_NIL; - vcpu->arch.shregs.msr = MSR_ME; + __kvmppc_set_msr_hv(vcpu, MSR_ME); vcpu->arch.intr_msr = MSR_SF | MSR_ME; /* @@ -4188,7 +4188,7 @@ static int kvmhv_p9_guest_entry(struct kvm_vcpu *vcpu, u64 time_limit, __this_cpu_write(cpu_in_guest, NULL); if (trap == BOOK3S_INTERRUPT_SYSCALL && - !(vcpu->arch.shregs.msr & MSR_PR)) { + !(__kvmppc_get_msr_hv(vcpu) & MSR_PR)) { unsigned long req = kvmppc_get_gpr(vcpu, 3); /* @@ -4667,7 +4667,7 @@ int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit, if (!nested) { kvmppc_core_prepare_to_enter(vcpu); - if (vcpu->arch.shregs.msr & MSR_EE) { + if (__kvmppc_get_msr_hv(vcpu) & MSR_EE) { if (xive_interrupt_pending(vcpu)) kvmppc_inject_interrupt_hv(vcpu, BOOK3S_INTERRUPT_EXTERNAL, 0); @@ -4880,7 +4880,7 @@ static int kvmppc_vcpu_run_hv(struct kvm_vcpu *vcpu) if (run->exit_reason == KVM_EXIT_PAPR_HCALL) { accumulate_time(vcpu, &vcpu->arch.hcall); - if (WARN_ON_ONCE(vcpu->arch.shregs.msr & MSR_PR)) { + if (WARN_ON_ONCE(__kvmppc_get_msr_hv(vcpu) & MSR_PR)) { /* * These should have been caught reflected * into the guest by now. Final sanity check: diff --git a/arch/powerpc/kvm/book3s_hv.h b/arch/powerpc/kvm/book3s_hv.h index acd9a7a95bbf..95241764dfb4 100644 --- a/arch/powerpc/kvm/book3s_hv.h +++ b/arch/powerpc/kvm/book3s_hv.h @@ -51,6 +51,16 @@ void accumulate_time(struct kvm_vcpu *vcpu, struct kvmhv_tb_accumulator *next); #define end_timing(vcpu) do {} while (0) #endif +static inline void __kvmppc_set_msr_hv(struct kvm_vcpu *vcpu, u64 val) +{ + vcpu->arch.shregs.msr = val; +} + +static inline u64 __kvmppc_get_msr_hv(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.shregs.msr; +} + #define KVMPPC_BOOK3S_HV_VCPU_ACCESSOR_SET(reg, size) \ static inline void kvmppc_set_##reg ##_hv(struct kvm_vcpu *vcpu, u##size val) \ { \ diff --git a/arch/powerpc/kvm/book3s_hv_builtin.c b/arch/powerpc/kvm/book3s_hv_builtin.c index f3afe194e616..fa0e3a22cac0 100644 --- a/arch/powerpc/kvm/book3s_hv_builtin.c +++ b/arch/powerpc/kvm/book3s_hv_builtin.c @@ -32,6 +32,7 @@ #include "book3s_xics.h" #include "book3s_xive.h" +#include "book3s_hv.h" /* * Hash page table alignment on newer cpus(CPU_FTR_ARCH_206) @@ -514,7 +515,7 @@ void kvmppc_set_msr_hv(struct kvm_vcpu *vcpu, u64 msr) */ if ((msr & MSR_TS_MASK) == MSR_TS_MASK) msr &= ~MSR_TS_MASK; - vcpu->arch.shregs.msr = msr; + __kvmppc_set_msr_hv(vcpu, msr); kvmppc_end_cede(vcpu); } EXPORT_SYMBOL_GPL(kvmppc_set_msr_hv); @@ -552,7 +553,7 @@ static void inject_interrupt(struct kvm_vcpu *vcpu, int vec, u64 srr1_flags) kvmppc_set_srr0(vcpu, pc); kvmppc_set_srr1(vcpu, (msr & SRR1_MSR_BITS) | srr1_flags); kvmppc_set_pc(vcpu, new_pc); - vcpu->arch.shregs.msr = new_msr; + __kvmppc_set_msr_hv(vcpu, new_msr); } void kvmppc_inject_interrupt_hv(struct kvm_vcpu *vcpu, int vec, u64 srr1_flags) From 6ccbbc33f06adaf79acde18571c6543ad1cb4be6 Mon Sep 17 00:00:00 2001 From: Jordan Niethe Date: Thu, 14 Sep 2023 13:05:57 +1000 Subject: [PATCH 086/475] KVM: PPC: Add helper library for Guest State Buffers The PAPR "Nestedv2" guest API introduces the concept of a Guest State Buffer for communication about L2 guests between L1 and L0 hosts. In the new API, the L0 manages the L2 on behalf of the L1. This means that if the L1 needs to change L2 state (e.g. GPRs, SPRs, partition table...), it must request the L0 perform the modification. If the nested host needs to read L2 state likewise this request must go through the L0. The Guest State Buffer is a Type-Length-Value style data format defined in the PAPR which assigns all relevant partition state a unique identity. Unlike a typical TLV format the length is redundant as the length of each identity is fixed but is included for checking correctness. A guest state buffer consists of an element count followed by a stream of elements, where elements are composed of an ID number, data length, then the data: Header: <---4 bytes---> +----------------+----- | Element Count | Elements... +----------------+----- Element: <----2 bytes---> <-2 bytes-> <-Length bytes-> +----------------+-----------+----------------+ | Guest State ID | Length | Data | +----------------+-----------+----------------+ Guest State IDs have other attributes defined in the PAPR such as whether they are per thread or per guest, or read-only. Introduce a library for using guest state buffers. This includes support for actions such as creating buffers, adding elements to buffers, reading the value of elements and parsing buffers. This will be used later by the nestedv2 guest support. Signed-off-by: Jordan Niethe Signed-off-by: Michael Ellerman Link: https://msgid.link/20230914030600.16993-9-jniethe5@gmail.com --- arch/powerpc/Kconfig.debug | 12 + arch/powerpc/include/asm/guest-state-buffer.h | 904 ++++++++++++++++++ arch/powerpc/kvm/Makefile | 3 + arch/powerpc/kvm/guest-state-buffer.c | 571 +++++++++++ arch/powerpc/kvm/test-guest-state-buffer.c | 328 +++++++ 5 files changed, 1818 insertions(+) create mode 100644 arch/powerpc/include/asm/guest-state-buffer.h create mode 100644 arch/powerpc/kvm/guest-state-buffer.c create mode 100644 arch/powerpc/kvm/test-guest-state-buffer.c diff --git a/arch/powerpc/Kconfig.debug b/arch/powerpc/Kconfig.debug index 2a54fadbeaf5..339c3a5f56f1 100644 --- a/arch/powerpc/Kconfig.debug +++ b/arch/powerpc/Kconfig.debug @@ -82,6 +82,18 @@ config MSI_BITMAP_SELFTEST bool "Run self-tests of the MSI bitmap code" depends on DEBUG_KERNEL +config GUEST_STATE_BUFFER_TEST + def_tristate n + prompt "Enable Guest State Buffer unit tests" + depends on KUNIT + depends on KVM_BOOK3S_HV_POSSIBLE + default KUNIT_ALL_TESTS + help + The Guest State Buffer is a data format specified in the PAPR. + It is by hcalls to communicate the state of L2 guests between + the L1 and L0 hypervisors. Enable unit tests for the library + used to create and use guest state buffers. + config PPC_IRQ_SOFT_MASK_DEBUG bool "Include extra checks for powerpc irq soft masking" depends on PPC64 diff --git a/arch/powerpc/include/asm/guest-state-buffer.h b/arch/powerpc/include/asm/guest-state-buffer.h new file mode 100644 index 000000000000..aaefe1075fc4 --- /dev/null +++ b/arch/powerpc/include/asm/guest-state-buffer.h @@ -0,0 +1,904 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Interface based on include/net/netlink.h + */ +#ifndef _ASM_POWERPC_GUEST_STATE_BUFFER_H +#define _ASM_POWERPC_GUEST_STATE_BUFFER_H + +#include +#include +#include + +/************************************************************************** + * Guest State Buffer Constants + **************************************************************************/ +/* Element without a value and any length */ +#define KVMPPC_GSID_BLANK 0x0000 +/* Size required for the L0's internal VCPU representation */ +#define KVMPPC_GSID_HOST_STATE_SIZE 0x0001 + /* Minimum size for the H_GUEST_RUN_VCPU output buffer */ +#define KVMPPC_GSID_RUN_OUTPUT_MIN_SIZE 0x0002 + /* "Logical" PVR value as defined in the PAPR */ +#define KVMPPC_GSID_LOGICAL_PVR 0x0003 + /* L0 relative timebase offset */ +#define KVMPPC_GSID_TB_OFFSET 0x0004 + /* Partition Scoped Page Table Info */ +#define KVMPPC_GSID_PARTITION_TABLE 0x0005 + /* Process Table Info */ +#define KVMPPC_GSID_PROCESS_TABLE 0x0006 + +/* H_GUEST_RUN_VCPU input buffer Info */ +#define KVMPPC_GSID_RUN_INPUT 0x0C00 +/* H_GUEST_RUN_VCPU output buffer Info */ +#define KVMPPC_GSID_RUN_OUTPUT 0x0C01 +#define KVMPPC_GSID_VPA 0x0C02 + +#define KVMPPC_GSID_GPR(x) (0x1000 + (x)) +#define KVMPPC_GSID_HDEC_EXPIRY_TB 0x1020 +#define KVMPPC_GSID_NIA 0x1021 +#define KVMPPC_GSID_MSR 0x1022 +#define KVMPPC_GSID_LR 0x1023 +#define KVMPPC_GSID_XER 0x1024 +#define KVMPPC_GSID_CTR 0x1025 +#define KVMPPC_GSID_CFAR 0x1026 +#define KVMPPC_GSID_SRR0 0x1027 +#define KVMPPC_GSID_SRR1 0x1028 +#define KVMPPC_GSID_DAR 0x1029 +#define KVMPPC_GSID_DEC_EXPIRY_TB 0x102A +#define KVMPPC_GSID_VTB 0x102B +#define KVMPPC_GSID_LPCR 0x102C +#define KVMPPC_GSID_HFSCR 0x102D +#define KVMPPC_GSID_FSCR 0x102E +#define KVMPPC_GSID_FPSCR 0x102F +#define KVMPPC_GSID_DAWR0 0x1030 +#define KVMPPC_GSID_DAWR1 0x1031 +#define KVMPPC_GSID_CIABR 0x1032 +#define KVMPPC_GSID_PURR 0x1033 +#define KVMPPC_GSID_SPURR 0x1034 +#define KVMPPC_GSID_IC 0x1035 +#define KVMPPC_GSID_SPRG0 0x1036 +#define KVMPPC_GSID_SPRG1 0x1037 +#define KVMPPC_GSID_SPRG2 0x1038 +#define KVMPPC_GSID_SPRG3 0x1039 +#define KVMPPC_GSID_PPR 0x103A +#define KVMPPC_GSID_MMCR(x) (0x103B + (x)) +#define KVMPPC_GSID_MMCRA 0x103F +#define KVMPPC_GSID_SIER(x) (0x1040 + (x)) +#define KVMPPC_GSID_BESCR 0x1043 +#define KVMPPC_GSID_EBBHR 0x1044 +#define KVMPPC_GSID_EBBRR 0x1045 +#define KVMPPC_GSID_AMR 0x1046 +#define KVMPPC_GSID_IAMR 0x1047 +#define KVMPPC_GSID_AMOR 0x1048 +#define KVMPPC_GSID_UAMOR 0x1049 +#define KVMPPC_GSID_SDAR 0x104A +#define KVMPPC_GSID_SIAR 0x104B +#define KVMPPC_GSID_DSCR 0x104C +#define KVMPPC_GSID_TAR 0x104D +#define KVMPPC_GSID_DEXCR 0x104E +#define KVMPPC_GSID_HDEXCR 0x104F +#define KVMPPC_GSID_HASHKEYR 0x1050 +#define KVMPPC_GSID_HASHPKEYR 0x1051 +#define KVMPPC_GSID_CTRL 0x1052 + +#define KVMPPC_GSID_CR 0x2000 +#define KVMPPC_GSID_PIDR 0x2001 +#define KVMPPC_GSID_DSISR 0x2002 +#define KVMPPC_GSID_VSCR 0x2003 +#define KVMPPC_GSID_VRSAVE 0x2004 +#define KVMPPC_GSID_DAWRX0 0x2005 +#define KVMPPC_GSID_DAWRX1 0x2006 +#define KVMPPC_GSID_PMC(x) (0x2007 + (x)) +#define KVMPPC_GSID_WORT 0x200D +#define KVMPPC_GSID_PSPB 0x200E + +#define KVMPPC_GSID_VSRS(x) (0x3000 + (x)) + +#define KVMPPC_GSID_HDAR 0xF000 +#define KVMPPC_GSID_HDSISR 0xF001 +#define KVMPPC_GSID_HEIR 0xF002 +#define KVMPPC_GSID_ASDR 0xF003 + +#define KVMPPC_GSE_GUESTWIDE_START KVMPPC_GSID_BLANK +#define KVMPPC_GSE_GUESTWIDE_END KVMPPC_GSID_PROCESS_TABLE +#define KVMPPC_GSE_GUESTWIDE_COUNT \ + (KVMPPC_GSE_GUESTWIDE_END - KVMPPC_GSE_GUESTWIDE_START + 1) + +#define KVMPPC_GSE_META_START KVMPPC_GSID_RUN_INPUT +#define KVMPPC_GSE_META_END KVMPPC_GSID_VPA +#define KVMPPC_GSE_META_COUNT (KVMPPC_GSE_META_END - KVMPPC_GSE_META_START + 1) + +#define KVMPPC_GSE_DW_REGS_START KVMPPC_GSID_GPR(0) +#define KVMPPC_GSE_DW_REGS_END KVMPPC_GSID_CTRL +#define KVMPPC_GSE_DW_REGS_COUNT \ + (KVMPPC_GSE_DW_REGS_END - KVMPPC_GSE_DW_REGS_START + 1) + +#define KVMPPC_GSE_W_REGS_START KVMPPC_GSID_CR +#define KVMPPC_GSE_W_REGS_END KVMPPC_GSID_PSPB +#define KVMPPC_GSE_W_REGS_COUNT \ + (KVMPPC_GSE_W_REGS_END - KVMPPC_GSE_W_REGS_START + 1) + +#define KVMPPC_GSE_VSRS_START KVMPPC_GSID_VSRS(0) +#define KVMPPC_GSE_VSRS_END KVMPPC_GSID_VSRS(63) +#define KVMPPC_GSE_VSRS_COUNT (KVMPPC_GSE_VSRS_END - KVMPPC_GSE_VSRS_START + 1) + +#define KVMPPC_GSE_INTR_REGS_START KVMPPC_GSID_HDAR +#define KVMPPC_GSE_INTR_REGS_END KVMPPC_GSID_ASDR +#define KVMPPC_GSE_INTR_REGS_COUNT \ + (KVMPPC_GSE_INTR_REGS_END - KVMPPC_GSE_INTR_REGS_START + 1) + +#define KVMPPC_GSE_IDEN_COUNT \ + (KVMPPC_GSE_GUESTWIDE_COUNT + KVMPPC_GSE_META_COUNT + \ + KVMPPC_GSE_DW_REGS_COUNT + KVMPPC_GSE_W_REGS_COUNT + \ + KVMPPC_GSE_VSRS_COUNT + KVMPPC_GSE_INTR_REGS_COUNT) + +/** + * Ranges of guest state buffer elements + */ +enum { + KVMPPC_GS_CLASS_GUESTWIDE = 0x01, + KVMPPC_GS_CLASS_META = 0x02, + KVMPPC_GS_CLASS_DWORD_REG = 0x04, + KVMPPC_GS_CLASS_WORD_REG = 0x08, + KVMPPC_GS_CLASS_VECTOR = 0x10, + KVMPPC_GS_CLASS_INTR = 0x20, +}; + +/** + * Types of guest state buffer elements + */ +enum { + KVMPPC_GSE_BE32, + KVMPPC_GSE_BE64, + KVMPPC_GSE_VEC128, + KVMPPC_GSE_PARTITION_TABLE, + KVMPPC_GSE_PROCESS_TABLE, + KVMPPC_GSE_BUFFER, + __KVMPPC_GSE_TYPE_MAX, +}; + +/** + * Flags for guest state elements + */ +enum { + KVMPPC_GS_FLAGS_WIDE = 0x01, +}; + +/** + * struct kvmppc_gs_part_table - deserialized partition table information + * element + * @address: start of the partition table + * @ea_bits: number of bits in the effective address + * @gpd_size: root page directory size + */ +struct kvmppc_gs_part_table { + u64 address; + u64 ea_bits; + u64 gpd_size; +}; + +/** + * struct kvmppc_gs_proc_table - deserialized process table information element + * @address: start of the process table + * @gpd_size: process table size + */ +struct kvmppc_gs_proc_table { + u64 address; + u64 gpd_size; +}; + +/** + * struct kvmppc_gs_buff_info - deserialized meta guest state buffer information + * @address: start of the guest state buffer + * @size: size of the guest state buffer + */ +struct kvmppc_gs_buff_info { + u64 address; + u64 size; +}; + +/** + * struct kvmppc_gs_header - serialized guest state buffer header + * @nelem: count of guest state elements in the buffer + * @data: start of the stream of elements in the buffer + */ +struct kvmppc_gs_header { + __be32 nelems; + char data[]; +} __packed; + +/** + * struct kvmppc_gs_elem - serialized guest state buffer element + * @iden: Guest State ID + * @len: length of data + * @data: the guest state buffer element's value + */ +struct kvmppc_gs_elem { + __be16 iden; + __be16 len; + char data[]; +} __packed; + +/** + * struct kvmppc_gs_buff - a guest state buffer with metadata. + * @capacity: total length of the buffer + * @len: current length of the elements and header + * @guest_id: guest id associated with the buffer + * @vcpu_id: vcpu_id associated with the buffer + * @hdr: the serialised guest state buffer + */ +struct kvmppc_gs_buff { + size_t capacity; + size_t len; + unsigned long guest_id; + unsigned long vcpu_id; + struct kvmppc_gs_header *hdr; +}; + +/** + * struct kvmppc_gs_bitmap - a bitmap for element ids + * @bitmap: a bitmap large enough for all Guest State IDs + */ +struct kvmppc_gs_bitmap { + /* private: */ + DECLARE_BITMAP(bitmap, KVMPPC_GSE_IDEN_COUNT); +}; + +/** + * struct kvmppc_gs_parser - a map of element ids to locations in a buffer + * @iterator: bitmap used for iterating + * @gses: contains the pointers to elements + * + * A guest state parser is used for deserialising a guest state buffer. + * Given a buffer, it then allows looking up guest state elements using + * a guest state id. + */ +struct kvmppc_gs_parser { + /* private: */ + struct kvmppc_gs_bitmap iterator; + struct kvmppc_gs_elem *gses[KVMPPC_GSE_IDEN_COUNT]; +}; + +enum { + GSM_GUEST_WIDE = 0x1, + GSM_SEND = 0x2, + GSM_RECEIVE = 0x4, + GSM_GSB_OWNER = 0x8, +}; + +struct kvmppc_gs_msg; + +/** + * struct kvmppc_gs_msg_ops - guest state message behavior + * @get_size: maximum size required for the message data + * @fill_info: serializes to the guest state buffer format + * @refresh_info: dserializes from the guest state buffer format + */ +struct kvmppc_gs_msg_ops { + size_t (*get_size)(struct kvmppc_gs_msg *gsm); + int (*fill_info)(struct kvmppc_gs_buff *gsb, struct kvmppc_gs_msg *gsm); + int (*refresh_info)(struct kvmppc_gs_msg *gsm, + struct kvmppc_gs_buff *gsb); +}; + +/** + * struct kvmppc_gs_msg - a guest state message + * @bitmap: the guest state ids that should be included + * @ops: modify message behavior for reading and writing to buffers + * @flags: guest wide or thread wide + * @data: location where buffer data will be written to or from. + * + * A guest state message is allows flexibility in sending in receiving data + * in a guest state buffer format. + */ +struct kvmppc_gs_msg { + struct kvmppc_gs_bitmap bitmap; + struct kvmppc_gs_msg_ops *ops; + unsigned long flags; + void *data; +}; + +/************************************************************************** + * Guest State IDs + **************************************************************************/ + +u16 kvmppc_gsid_size(u16 iden); +unsigned long kvmppc_gsid_flags(u16 iden); +u64 kvmppc_gsid_mask(u16 iden); + +/************************************************************************** + * Guest State Buffers + **************************************************************************/ +struct kvmppc_gs_buff *kvmppc_gsb_new(size_t size, unsigned long guest_id, + unsigned long vcpu_id, gfp_t flags); +void kvmppc_gsb_free(struct kvmppc_gs_buff *gsb); +void *kvmppc_gsb_put(struct kvmppc_gs_buff *gsb, size_t size); + +/** + * kvmppc_gsb_header() - the header of a guest state buffer + * @gsb: guest state buffer + * + * Returns a pointer to the buffer header. + */ +static inline struct kvmppc_gs_header * +kvmppc_gsb_header(struct kvmppc_gs_buff *gsb) +{ + return gsb->hdr; +} + +/** + * kvmppc_gsb_data() - the elements of a guest state buffer + * @gsb: guest state buffer + * + * Returns a pointer to the first element of the buffer data. + */ +static inline struct kvmppc_gs_elem *kvmppc_gsb_data(struct kvmppc_gs_buff *gsb) +{ + return (struct kvmppc_gs_elem *)kvmppc_gsb_header(gsb)->data; +} + +/** + * kvmppc_gsb_len() - the current length of a guest state buffer + * @gsb: guest state buffer + * + * Returns the length including the header of a buffer. + */ +static inline size_t kvmppc_gsb_len(struct kvmppc_gs_buff *gsb) +{ + return gsb->len; +} + +/** + * kvmppc_gsb_capacity() - the capacity of a guest state buffer + * @gsb: guest state buffer + * + * Returns the capacity of a buffer. + */ +static inline size_t kvmppc_gsb_capacity(struct kvmppc_gs_buff *gsb) +{ + return gsb->capacity; +} + +/** + * kvmppc_gsb_paddress() - the physical address of buffer + * @gsb: guest state buffer + * + * Returns the physical address of the buffer. + */ +static inline u64 kvmppc_gsb_paddress(struct kvmppc_gs_buff *gsb) +{ + return __pa(kvmppc_gsb_header(gsb)); +} + +/** + * kvmppc_gsb_nelems() - the number of elements in a buffer + * @gsb: guest state buffer + * + * Returns the number of elements in a buffer + */ +static inline u32 kvmppc_gsb_nelems(struct kvmppc_gs_buff *gsb) +{ + return be32_to_cpu(kvmppc_gsb_header(gsb)->nelems); +} + +/** + * kvmppc_gsb_reset() - empty a guest state buffer + * @gsb: guest state buffer + * + * Reset the number of elements and length of buffer to empty. + */ +static inline void kvmppc_gsb_reset(struct kvmppc_gs_buff *gsb) +{ + kvmppc_gsb_header(gsb)->nelems = cpu_to_be32(0); + gsb->len = sizeof(struct kvmppc_gs_header); +} + +/** + * kvmppc_gsb_data_len() - the length of a buffer excluding the header + * @gsb: guest state buffer + * + * Returns the length of a buffer excluding the header + */ +static inline size_t kvmppc_gsb_data_len(struct kvmppc_gs_buff *gsb) +{ + return gsb->len - sizeof(struct kvmppc_gs_header); +} + +/** + * kvmppc_gsb_data_cap() - the capacity of a buffer excluding the header + * @gsb: guest state buffer + * + * Returns the capacity of a buffer excluding the header + */ +static inline size_t kvmppc_gsb_data_cap(struct kvmppc_gs_buff *gsb) +{ + return gsb->capacity - sizeof(struct kvmppc_gs_header); +} + +/** + * kvmppc_gsb_for_each_elem - iterate over the elements in a buffer + * @i: loop counter + * @pos: set to current element + * @gsb: guest state buffer + * @rem: initialized to buffer capacity, holds bytes currently remaining in + * stream + */ +#define kvmppc_gsb_for_each_elem(i, pos, gsb, rem) \ + kvmppc_gse_for_each_elem(i, kvmppc_gsb_nelems(gsb), pos, \ + kvmppc_gsb_data(gsb), \ + kvmppc_gsb_data_cap(gsb), rem) + +/************************************************************************** + * Guest State Elements + **************************************************************************/ + +/** + * kvmppc_gse_iden() - guest state ID of element + * @gse: guest state element + * + * Return the guest state ID in host endianness. + */ +static inline u16 kvmppc_gse_iden(const struct kvmppc_gs_elem *gse) +{ + return be16_to_cpu(gse->iden); +} + +/** + * kvmppc_gse_len() - length of guest state element data + * @gse: guest state element + * + * Returns the length of guest state element data + */ +static inline u16 kvmppc_gse_len(const struct kvmppc_gs_elem *gse) +{ + return be16_to_cpu(gse->len); +} + +/** + * kvmppc_gse_total_len() - total length of guest state element + * @gse: guest state element + * + * Returns the length of the data plus the ID and size header. + */ +static inline u16 kvmppc_gse_total_len(const struct kvmppc_gs_elem *gse) +{ + return be16_to_cpu(gse->len) + sizeof(*gse); +} + +/** + * kvmppc_gse_total_size() - space needed for a given data length + * @size: data length + * + * Returns size plus the space needed for the ID and size header. + */ +static inline u16 kvmppc_gse_total_size(u16 size) +{ + return sizeof(struct kvmppc_gs_elem) + size; +} + +/** + * kvmppc_gse_data() - pointer to data of a guest state element + * @gse: guest state element + * + * Returns a pointer to the beginning of guest state element data. + */ +static inline void *kvmppc_gse_data(const struct kvmppc_gs_elem *gse) +{ + return (void *)gse->data; +} + +/** + * kvmppc_gse_ok() - checks space exists for guest state element + * @gse: guest state element + * @remaining: bytes of space remaining + * + * Returns true if the guest state element can fit in remaining space. + */ +static inline bool kvmppc_gse_ok(const struct kvmppc_gs_elem *gse, + int remaining) +{ + return remaining >= kvmppc_gse_total_len(gse); +} + +/** + * kvmppc_gse_next() - iterate to the next guest state element in a stream + * @gse: stream of guest state elements + * @remaining: length of the guest element stream + * + * Returns the next guest state element in a stream of elements. The length of + * the stream is updated in remaining. + */ +static inline struct kvmppc_gs_elem * +kvmppc_gse_next(const struct kvmppc_gs_elem *gse, int *remaining) +{ + int len = sizeof(*gse) + kvmppc_gse_len(gse); + + *remaining -= len; + return (struct kvmppc_gs_elem *)(gse->data + kvmppc_gse_len(gse)); +} + +/** + * kvmppc_gse_for_each_elem - iterate over a stream of guest state elements + * @i: loop counter + * @max: number of elements + * @pos: set to current element + * @head: head of elements + * @len: length of the stream + * @rem: initialized to len, holds bytes currently remaining elements + */ +#define kvmppc_gse_for_each_elem(i, max, pos, head, len, rem) \ + for (i = 0, pos = head, rem = len; kvmppc_gse_ok(pos, rem) && i < max; \ + pos = kvmppc_gse_next(pos, &(rem)), i++) + +int __kvmppc_gse_put(struct kvmppc_gs_buff *gsb, u16 iden, u16 size, + const void *data); +int kvmppc_gse_parse(struct kvmppc_gs_parser *gsp, struct kvmppc_gs_buff *gsb); + +/** + * kvmppc_gse_put_be32() - add a be32 guest state element to a buffer + * @gsb: guest state buffer to add element to + * @iden: guest state ID + * @val: big endian value + */ +static inline int kvmppc_gse_put_be32(struct kvmppc_gs_buff *gsb, u16 iden, + __be32 val) +{ + __be32 tmp; + + tmp = val; + return __kvmppc_gse_put(gsb, iden, sizeof(__be32), &tmp); +} + +/** + * kvmppc_gse_put_u32() - add a host endian 32bit int guest state element to a + * buffer + * @gsb: guest state buffer to add element to + * @iden: guest state ID + * @val: host endian value + */ +static inline int kvmppc_gse_put_u32(struct kvmppc_gs_buff *gsb, u16 iden, + u32 val) +{ + __be32 tmp; + + val &= kvmppc_gsid_mask(iden); + tmp = cpu_to_be32(val); + return kvmppc_gse_put_be32(gsb, iden, tmp); +} + +/** + * kvmppc_gse_put_be64() - add a be64 guest state element to a buffer + * @gsb: guest state buffer to add element to + * @iden: guest state ID + * @val: big endian value + */ +static inline int kvmppc_gse_put_be64(struct kvmppc_gs_buff *gsb, u16 iden, + __be64 val) +{ + __be64 tmp; + + tmp = val; + return __kvmppc_gse_put(gsb, iden, sizeof(__be64), &tmp); +} + +/** + * kvmppc_gse_put_u64() - add a host endian 64bit guest state element to a + * buffer + * @gsb: guest state buffer to add element to + * @iden: guest state ID + * @val: host endian value + */ +static inline int kvmppc_gse_put_u64(struct kvmppc_gs_buff *gsb, u16 iden, + u64 val) +{ + __be64 tmp; + + val &= kvmppc_gsid_mask(iden); + tmp = cpu_to_be64(val); + return kvmppc_gse_put_be64(gsb, iden, tmp); +} + +/** + * __kvmppc_gse_put_reg() - add a register type guest state element to a buffer + * @gsb: guest state buffer to add element to + * @iden: guest state ID + * @val: host endian value + * + * Adds a register type guest state element. Uses the guest state ID for + * determining the length of the guest element. If the guest state ID has + * bits that can not be set they will be cleared. + */ +static inline int __kvmppc_gse_put_reg(struct kvmppc_gs_buff *gsb, u16 iden, + u64 val) +{ + val &= kvmppc_gsid_mask(iden); + if (kvmppc_gsid_size(iden) == sizeof(u64)) + return kvmppc_gse_put_u64(gsb, iden, val); + + if (kvmppc_gsid_size(iden) == sizeof(u32)) { + u32 tmp; + + tmp = (u32)val; + if (tmp != val) + return -EINVAL; + + return kvmppc_gse_put_u32(gsb, iden, tmp); + } + return -EINVAL; +} + +/** + * kvmppc_gse_put_vector128() - add a vector guest state element to a buffer + * @gsb: guest state buffer to add element to + * @iden: guest state ID + * @val: 16 byte vector value + */ +static inline int kvmppc_gse_put_vector128(struct kvmppc_gs_buff *gsb, u16 iden, + vector128 *val) +{ + __be64 tmp[2] = { 0 }; + union { + __vector128 v; + u64 dw[2]; + } u; + + u.v = *val; + tmp[0] = cpu_to_be64(u.dw[TS_FPROFFSET]); +#ifdef CONFIG_VSX + tmp[1] = cpu_to_be64(u.dw[TS_VSRLOWOFFSET]); +#endif + return __kvmppc_gse_put(gsb, iden, sizeof(tmp), &tmp); +} + +/** + * kvmppc_gse_put_part_table() - add a partition table guest state element to a + * buffer + * @gsb: guest state buffer to add element to + * @iden: guest state ID + * @val: partition table value + */ +static inline int kvmppc_gse_put_part_table(struct kvmppc_gs_buff *gsb, + u16 iden, + struct kvmppc_gs_part_table val) +{ + __be64 tmp[3]; + + tmp[0] = cpu_to_be64(val.address); + tmp[1] = cpu_to_be64(val.ea_bits); + tmp[2] = cpu_to_be64(val.gpd_size); + return __kvmppc_gse_put(gsb, KVMPPC_GSID_PARTITION_TABLE, sizeof(tmp), + &tmp); +} + +/** + * kvmppc_gse_put_proc_table() - add a process table guest state element to a + * buffer + * @gsb: guest state buffer to add element to + * @iden: guest state ID + * @val: process table value + */ +static inline int kvmppc_gse_put_proc_table(struct kvmppc_gs_buff *gsb, + u16 iden, + struct kvmppc_gs_proc_table val) +{ + __be64 tmp[2]; + + tmp[0] = cpu_to_be64(val.address); + tmp[1] = cpu_to_be64(val.gpd_size); + return __kvmppc_gse_put(gsb, KVMPPC_GSID_PROCESS_TABLE, sizeof(tmp), + &tmp); +} + +/** + * kvmppc_gse_put_buff_info() - adds a GSB description guest state element to a + * buffer + * @gsb: guest state buffer to add element to + * @iden: guest state ID + * @val: guest state buffer description value + */ +static inline int kvmppc_gse_put_buff_info(struct kvmppc_gs_buff *gsb, u16 iden, + struct kvmppc_gs_buff_info val) +{ + __be64 tmp[2]; + + tmp[0] = cpu_to_be64(val.address); + tmp[1] = cpu_to_be64(val.size); + return __kvmppc_gse_put(gsb, iden, sizeof(tmp), &tmp); +} + +int __kvmppc_gse_put(struct kvmppc_gs_buff *gsb, u16 iden, u16 size, + const void *data); + +/** + * kvmppc_gse_get_be32() - return the data of a be32 element + * @gse: guest state element + */ +static inline __be32 kvmppc_gse_get_be32(const struct kvmppc_gs_elem *gse) +{ + if (WARN_ON(kvmppc_gse_len(gse) != sizeof(__be32))) + return 0; + return *(__be32 *)kvmppc_gse_data(gse); +} + +/** + * kvmppc_gse_get_u32() - return the data of a be32 element in host endianness + * @gse: guest state element + */ +static inline u32 kvmppc_gse_get_u32(const struct kvmppc_gs_elem *gse) +{ + return be32_to_cpu(kvmppc_gse_get_be32(gse)); +} + +/** + * kvmppc_gse_get_be64() - return the data of a be64 element + * @gse: guest state element + */ +static inline __be64 kvmppc_gse_get_be64(const struct kvmppc_gs_elem *gse) +{ + if (WARN_ON(kvmppc_gse_len(gse) != sizeof(__be64))) + return 0; + return *(__be64 *)kvmppc_gse_data(gse); +} + +/** + * kvmppc_gse_get_u64() - return the data of a be64 element in host endianness + * @gse: guest state element + */ +static inline u64 kvmppc_gse_get_u64(const struct kvmppc_gs_elem *gse) +{ + return be64_to_cpu(kvmppc_gse_get_be64(gse)); +} + +/** + * kvmppc_gse_get_vector128() - return the data of a vector element + * @gse: guest state element + */ +static inline void kvmppc_gse_get_vector128(const struct kvmppc_gs_elem *gse, + vector128 *v) +{ + union { + __vector128 v; + u64 dw[2]; + } u = { 0 }; + __be64 *src; + + if (WARN_ON(kvmppc_gse_len(gse) != sizeof(__vector128))) + *v = u.v; + + src = (__be64 *)kvmppc_gse_data(gse); + u.dw[TS_FPROFFSET] = be64_to_cpu(src[0]); +#ifdef CONFIG_VSX + u.dw[TS_VSRLOWOFFSET] = be64_to_cpu(src[1]); +#endif + *v = u.v; +} + +/************************************************************************** + * Guest State Bitmap + **************************************************************************/ + +bool kvmppc_gsbm_test(struct kvmppc_gs_bitmap *gsbm, u16 iden); +void kvmppc_gsbm_set(struct kvmppc_gs_bitmap *gsbm, u16 iden); +void kvmppc_gsbm_clear(struct kvmppc_gs_bitmap *gsbm, u16 iden); +u16 kvmppc_gsbm_next(struct kvmppc_gs_bitmap *gsbm, u16 prev); + +/** + * kvmppc_gsbm_zero - zero the entire bitmap + * @gsbm: guest state buffer bitmap + */ +static inline void kvmppc_gsbm_zero(struct kvmppc_gs_bitmap *gsbm) +{ + bitmap_zero(gsbm->bitmap, KVMPPC_GSE_IDEN_COUNT); +} + +/** + * kvmppc_gsbm_fill - fill the entire bitmap + * @gsbm: guest state buffer bitmap + */ +static inline void kvmppc_gsbm_fill(struct kvmppc_gs_bitmap *gsbm) +{ + bitmap_fill(gsbm->bitmap, KVMPPC_GSE_IDEN_COUNT); + clear_bit(0, gsbm->bitmap); +} + +/** + * kvmppc_gsbm_for_each - iterate the present guest state IDs + * @gsbm: guest state buffer bitmap + * @iden: current guest state ID + */ +#define kvmppc_gsbm_for_each(gsbm, iden) \ + for (iden = kvmppc_gsbm_next(gsbm, 0); iden != 0; \ + iden = kvmppc_gsbm_next(gsbm, iden)) + +/************************************************************************** + * Guest State Parser + **************************************************************************/ + +void kvmppc_gsp_insert(struct kvmppc_gs_parser *gsp, u16 iden, + struct kvmppc_gs_elem *gse); +struct kvmppc_gs_elem *kvmppc_gsp_lookup(struct kvmppc_gs_parser *gsp, + u16 iden); + +/** + * kvmppc_gsp_for_each - iterate the + * pairs + * @gsp: guest state buffer bitmap + * @iden: current guest state ID + * @gse: guest state element + */ +#define kvmppc_gsp_for_each(gsp, iden, gse) \ + for (iden = kvmppc_gsbm_next(&(gsp)->iterator, 0), \ + gse = kvmppc_gsp_lookup((gsp), iden); \ + iden != 0; iden = kvmppc_gsbm_next(&(gsp)->iterator, iden), \ + gse = kvmppc_gsp_lookup((gsp), iden)) + +/************************************************************************** + * Guest State Message + **************************************************************************/ + +/** + * kvmppc_gsm_for_each - iterate the guest state IDs included in a guest state + * message + * @gsp: guest state buffer bitmap + * @iden: current guest state ID + * @gse: guest state element + */ +#define kvmppc_gsm_for_each(gsm, iden) \ + for (iden = kvmppc_gsbm_next(&gsm->bitmap, 0); iden != 0; \ + iden = kvmppc_gsbm_next(&gsm->bitmap, iden)) + +int kvmppc_gsm_init(struct kvmppc_gs_msg *mgs, struct kvmppc_gs_msg_ops *ops, + void *data, unsigned long flags); + +struct kvmppc_gs_msg *kvmppc_gsm_new(struct kvmppc_gs_msg_ops *ops, void *data, + unsigned long flags, gfp_t gfp_flags); +void kvmppc_gsm_free(struct kvmppc_gs_msg *gsm); +size_t kvmppc_gsm_size(struct kvmppc_gs_msg *gsm); +int kvmppc_gsm_fill_info(struct kvmppc_gs_msg *gsm, struct kvmppc_gs_buff *gsb); +int kvmppc_gsm_refresh_info(struct kvmppc_gs_msg *gsm, + struct kvmppc_gs_buff *gsb); + +/** + * kvmppc_gsm_include - indicate a guest state ID should be included when + * serializing + * @gsm: guest state message + * @iden: guest state ID + */ +static inline void kvmppc_gsm_include(struct kvmppc_gs_msg *gsm, u16 iden) +{ + kvmppc_gsbm_set(&gsm->bitmap, iden); +} + +/** + * kvmppc_gsm_includes - check if a guest state ID will be included when + * serializing + * @gsm: guest state message + * @iden: guest state ID + */ +static inline bool kvmppc_gsm_includes(struct kvmppc_gs_msg *gsm, u16 iden) +{ + return kvmppc_gsbm_test(&gsm->bitmap, iden); +} + +/** + * kvmppc_gsm_includes - indicate all guest state IDs should be included when + * serializing + * @gsm: guest state message + * @iden: guest state ID + */ +static inline void kvmppc_gsm_include_all(struct kvmppc_gs_msg *gsm) +{ + kvmppc_gsbm_fill(&gsm->bitmap); +} + +/** + * kvmppc_gsm_include - clear the guest state IDs that should be included when + * serializing + * @gsm: guest state message + */ +static inline void kvmppc_gsm_reset(struct kvmppc_gs_msg *gsm) +{ + kvmppc_gsbm_zero(&gsm->bitmap); +} + +#endif /* _ASM_POWERPC_GUEST_STATE_BUFFER_H */ diff --git a/arch/powerpc/kvm/Makefile b/arch/powerpc/kvm/Makefile index 5319d889b184..eb8445e71c14 100644 --- a/arch/powerpc/kvm/Makefile +++ b/arch/powerpc/kvm/Makefile @@ -87,8 +87,11 @@ kvm-book3s_64-builtin-objs-$(CONFIG_KVM_BOOK3S_64_HANDLER) += \ book3s_hv_ras.o \ book3s_hv_builtin.o \ book3s_hv_p9_perf.o \ + guest-state-buffer.o \ $(kvm-book3s_64-builtin-tm-objs-y) \ $(kvm-book3s_64-builtin-xics-objs-y) + +obj-$(CONFIG_GUEST_STATE_BUFFER_TEST) += test-guest-state-buffer.o endif kvm-book3s_64-objs-$(CONFIG_KVM_XICS) += \ diff --git a/arch/powerpc/kvm/guest-state-buffer.c b/arch/powerpc/kvm/guest-state-buffer.c new file mode 100644 index 000000000000..2b22a2f8c67e --- /dev/null +++ b/arch/powerpc/kvm/guest-state-buffer.c @@ -0,0 +1,571 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "asm/hvcall.h" +#include +#include +#include + +static const u16 kvmppc_gse_iden_len[__KVMPPC_GSE_TYPE_MAX] = { + [KVMPPC_GSE_BE32] = sizeof(__be32), + [KVMPPC_GSE_BE64] = sizeof(__be64), + [KVMPPC_GSE_VEC128] = sizeof(vector128), + [KVMPPC_GSE_PARTITION_TABLE] = sizeof(struct kvmppc_gs_part_table), + [KVMPPC_GSE_PROCESS_TABLE] = sizeof(struct kvmppc_gs_proc_table), + [KVMPPC_GSE_BUFFER] = sizeof(struct kvmppc_gs_buff_info), +}; + +/** + * kvmppc_gsb_new() - create a new guest state buffer + * @size: total size of the guest state buffer (includes header) + * @guest_id: guest_id + * @vcpu_id: vcpu_id + * @flags: GFP flags + * + * Returns a guest state buffer. + */ +struct kvmppc_gs_buff *kvmppc_gsb_new(size_t size, unsigned long guest_id, + unsigned long vcpu_id, gfp_t flags) +{ + struct kvmppc_gs_buff *gsb; + + gsb = kzalloc(sizeof(*gsb), flags); + if (!gsb) + return NULL; + + size = roundup_pow_of_two(size); + gsb->hdr = kzalloc(size, GFP_KERNEL); + if (!gsb->hdr) + goto free; + + gsb->capacity = size; + gsb->len = sizeof(struct kvmppc_gs_header); + gsb->vcpu_id = vcpu_id; + gsb->guest_id = guest_id; + + gsb->hdr->nelems = cpu_to_be32(0); + + return gsb; + +free: + kfree(gsb); + return NULL; +} +EXPORT_SYMBOL_GPL(kvmppc_gsb_new); + +/** + * kvmppc_gsb_free() - free a guest state buffer + * @gsb: guest state buffer + */ +void kvmppc_gsb_free(struct kvmppc_gs_buff *gsb) +{ + kfree(gsb->hdr); + kfree(gsb); +} +EXPORT_SYMBOL_GPL(kvmppc_gsb_free); + +/** + * kvmppc_gsb_put() - allocate space in a guest state buffer + * @gsb: buffer to allocate in + * @size: amount of space to allocate + * + * Returns a pointer to the amount of space requested within the buffer and + * increments the count of elements in the buffer. + * + * Does not check if there is enough space in the buffer. + */ +void *kvmppc_gsb_put(struct kvmppc_gs_buff *gsb, size_t size) +{ + u32 nelems = kvmppc_gsb_nelems(gsb); + void *p; + + p = (void *)kvmppc_gsb_header(gsb) + kvmppc_gsb_len(gsb); + gsb->len += size; + + kvmppc_gsb_header(gsb)->nelems = cpu_to_be32(nelems + 1); + return p; +} +EXPORT_SYMBOL_GPL(kvmppc_gsb_put); + +static int kvmppc_gsid_class(u16 iden) +{ + if ((iden >= KVMPPC_GSE_GUESTWIDE_START) && + (iden <= KVMPPC_GSE_GUESTWIDE_END)) + return KVMPPC_GS_CLASS_GUESTWIDE; + + if ((iden >= KVMPPC_GSE_META_START) && (iden <= KVMPPC_GSE_META_END)) + return KVMPPC_GS_CLASS_META; + + if ((iden >= KVMPPC_GSE_DW_REGS_START) && + (iden <= KVMPPC_GSE_DW_REGS_END)) + return KVMPPC_GS_CLASS_DWORD_REG; + + if ((iden >= KVMPPC_GSE_W_REGS_START) && + (iden <= KVMPPC_GSE_W_REGS_END)) + return KVMPPC_GS_CLASS_WORD_REG; + + if ((iden >= KVMPPC_GSE_VSRS_START) && (iden <= KVMPPC_GSE_VSRS_END)) + return KVMPPC_GS_CLASS_VECTOR; + + if ((iden >= KVMPPC_GSE_INTR_REGS_START) && + (iden <= KVMPPC_GSE_INTR_REGS_END)) + return KVMPPC_GS_CLASS_INTR; + + return -1; +} + +static int kvmppc_gsid_type(u16 iden) +{ + int type = -1; + + switch (kvmppc_gsid_class(iden)) { + case KVMPPC_GS_CLASS_GUESTWIDE: + switch (iden) { + case KVMPPC_GSID_HOST_STATE_SIZE: + case KVMPPC_GSID_RUN_OUTPUT_MIN_SIZE: + case KVMPPC_GSID_TB_OFFSET: + type = KVMPPC_GSE_BE64; + break; + case KVMPPC_GSID_PARTITION_TABLE: + type = KVMPPC_GSE_PARTITION_TABLE; + break; + case KVMPPC_GSID_PROCESS_TABLE: + type = KVMPPC_GSE_PROCESS_TABLE; + break; + case KVMPPC_GSID_LOGICAL_PVR: + type = KVMPPC_GSE_BE32; + break; + } + break; + case KVMPPC_GS_CLASS_META: + switch (iden) { + case KVMPPC_GSID_RUN_INPUT: + case KVMPPC_GSID_RUN_OUTPUT: + type = KVMPPC_GSE_BUFFER; + break; + case KVMPPC_GSID_VPA: + type = KVMPPC_GSE_BE64; + break; + } + break; + case KVMPPC_GS_CLASS_DWORD_REG: + type = KVMPPC_GSE_BE64; + break; + case KVMPPC_GS_CLASS_WORD_REG: + type = KVMPPC_GSE_BE32; + break; + case KVMPPC_GS_CLASS_VECTOR: + type = KVMPPC_GSE_VEC128; + break; + case KVMPPC_GS_CLASS_INTR: + switch (iden) { + case KVMPPC_GSID_HDAR: + case KVMPPC_GSID_ASDR: + case KVMPPC_GSID_HEIR: + type = KVMPPC_GSE_BE64; + break; + case KVMPPC_GSID_HDSISR: + type = KVMPPC_GSE_BE32; + break; + } + break; + } + + return type; +} + +/** + * kvmppc_gsid_flags() - the flags for a guest state ID + * @iden: guest state ID + * + * Returns any flags for the guest state ID. + */ +unsigned long kvmppc_gsid_flags(u16 iden) +{ + unsigned long flags = 0; + + switch (kvmppc_gsid_class(iden)) { + case KVMPPC_GS_CLASS_GUESTWIDE: + flags = KVMPPC_GS_FLAGS_WIDE; + break; + case KVMPPC_GS_CLASS_META: + case KVMPPC_GS_CLASS_DWORD_REG: + case KVMPPC_GS_CLASS_WORD_REG: + case KVMPPC_GS_CLASS_VECTOR: + case KVMPPC_GS_CLASS_INTR: + break; + } + + return flags; +} +EXPORT_SYMBOL_GPL(kvmppc_gsid_flags); + +/** + * kvmppc_gsid_size() - the size of a guest state ID + * @iden: guest state ID + * + * Returns the size of guest state ID. + */ +u16 kvmppc_gsid_size(u16 iden) +{ + int type; + + type = kvmppc_gsid_type(iden); + if (type == -1) + return 0; + + if (type >= __KVMPPC_GSE_TYPE_MAX) + return 0; + + return kvmppc_gse_iden_len[type]; +} +EXPORT_SYMBOL_GPL(kvmppc_gsid_size); + +/** + * kvmppc_gsid_mask() - the settable bits of a guest state ID + * @iden: guest state ID + * + * Returns a mask of settable bits for a guest state ID. + */ +u64 kvmppc_gsid_mask(u16 iden) +{ + u64 mask = ~0ull; + + switch (iden) { + case KVMPPC_GSID_LPCR: + mask = LPCR_DPFD | LPCR_ILE | LPCR_AIL | LPCR_LD | LPCR_MER | + LPCR_GTSE; + break; + case KVMPPC_GSID_MSR: + mask = ~(MSR_HV | MSR_S | MSR_ME); + break; + } + + return mask; +} +EXPORT_SYMBOL_GPL(kvmppc_gsid_mask); + +/** + * __kvmppc_gse_put() - add a guest state element to a buffer + * @gsb: buffer to the element to + * @iden: guest state ID + * @size: length of data + * @data: pointer to data + */ +int __kvmppc_gse_put(struct kvmppc_gs_buff *gsb, u16 iden, u16 size, + const void *data) +{ + struct kvmppc_gs_elem *gse; + u16 total_size; + + total_size = sizeof(*gse) + size; + if (total_size + kvmppc_gsb_len(gsb) > kvmppc_gsb_capacity(gsb)) + return -ENOMEM; + + if (kvmppc_gsid_size(iden) != size) + return -EINVAL; + + gse = kvmppc_gsb_put(gsb, total_size); + gse->iden = cpu_to_be16(iden); + gse->len = cpu_to_be16(size); + memcpy(gse->data, data, size); + + return 0; +} +EXPORT_SYMBOL_GPL(__kvmppc_gse_put); + +/** + * kvmppc_gse_parse() - create a parse map from a guest state buffer + * @gsp: guest state parser + * @gsb: guest state buffer + */ +int kvmppc_gse_parse(struct kvmppc_gs_parser *gsp, struct kvmppc_gs_buff *gsb) +{ + struct kvmppc_gs_elem *curr; + int rem, i; + + kvmppc_gsb_for_each_elem(i, curr, gsb, rem) { + if (kvmppc_gse_len(curr) != + kvmppc_gsid_size(kvmppc_gse_iden(curr))) + return -EINVAL; + kvmppc_gsp_insert(gsp, kvmppc_gse_iden(curr), curr); + } + + if (kvmppc_gsb_nelems(gsb) != i) + return -EINVAL; + return 0; +} +EXPORT_SYMBOL_GPL(kvmppc_gse_parse); + +static inline int kvmppc_gse_flatten_iden(u16 iden) +{ + int bit = 0; + int class; + + class = kvmppc_gsid_class(iden); + + if (class == KVMPPC_GS_CLASS_GUESTWIDE) { + bit += iden - KVMPPC_GSE_GUESTWIDE_START; + return bit; + } + + bit += KVMPPC_GSE_GUESTWIDE_COUNT; + + if (class == KVMPPC_GS_CLASS_META) { + bit += iden - KVMPPC_GSE_META_START; + return bit; + } + + bit += KVMPPC_GSE_META_COUNT; + + if (class == KVMPPC_GS_CLASS_DWORD_REG) { + bit += iden - KVMPPC_GSE_DW_REGS_START; + return bit; + } + + bit += KVMPPC_GSE_DW_REGS_COUNT; + + if (class == KVMPPC_GS_CLASS_WORD_REG) { + bit += iden - KVMPPC_GSE_W_REGS_START; + return bit; + } + + bit += KVMPPC_GSE_W_REGS_COUNT; + + if (class == KVMPPC_GS_CLASS_VECTOR) { + bit += iden - KVMPPC_GSE_VSRS_START; + return bit; + } + + bit += KVMPPC_GSE_VSRS_COUNT; + + if (class == KVMPPC_GS_CLASS_INTR) { + bit += iden - KVMPPC_GSE_INTR_REGS_START; + return bit; + } + + return 0; +} + +static inline u16 kvmppc_gse_unflatten_iden(int bit) +{ + u16 iden; + + if (bit < KVMPPC_GSE_GUESTWIDE_COUNT) { + iden = KVMPPC_GSE_GUESTWIDE_START + bit; + return iden; + } + bit -= KVMPPC_GSE_GUESTWIDE_COUNT; + + if (bit < KVMPPC_GSE_META_COUNT) { + iden = KVMPPC_GSE_META_START + bit; + return iden; + } + bit -= KVMPPC_GSE_META_COUNT; + + if (bit < KVMPPC_GSE_DW_REGS_COUNT) { + iden = KVMPPC_GSE_DW_REGS_START + bit; + return iden; + } + bit -= KVMPPC_GSE_DW_REGS_COUNT; + + if (bit < KVMPPC_GSE_W_REGS_COUNT) { + iden = KVMPPC_GSE_W_REGS_START + bit; + return iden; + } + bit -= KVMPPC_GSE_W_REGS_COUNT; + + if (bit < KVMPPC_GSE_VSRS_COUNT) { + iden = KVMPPC_GSE_VSRS_START + bit; + return iden; + } + bit -= KVMPPC_GSE_VSRS_COUNT; + + if (bit < KVMPPC_GSE_IDEN_COUNT) { + iden = KVMPPC_GSE_INTR_REGS_START + bit; + return iden; + } + + return 0; +} + +/** + * kvmppc_gsp_insert() - add a mapping from an guest state ID to an element + * @gsp: guest state parser + * @iden: guest state id (key) + * @gse: guest state element (value) + */ +void kvmppc_gsp_insert(struct kvmppc_gs_parser *gsp, u16 iden, + struct kvmppc_gs_elem *gse) +{ + int i; + + i = kvmppc_gse_flatten_iden(iden); + kvmppc_gsbm_set(&gsp->iterator, iden); + gsp->gses[i] = gse; +} +EXPORT_SYMBOL_GPL(kvmppc_gsp_insert); + +/** + * kvmppc_gsp_lookup() - lookup an element from a guest state ID + * @gsp: guest state parser + * @iden: guest state ID (key) + * + * Returns the guest state element if present. + */ +struct kvmppc_gs_elem *kvmppc_gsp_lookup(struct kvmppc_gs_parser *gsp, u16 iden) +{ + int i; + + i = kvmppc_gse_flatten_iden(iden); + return gsp->gses[i]; +} +EXPORT_SYMBOL_GPL(kvmppc_gsp_lookup); + +/** + * kvmppc_gsbm_set() - set the guest state ID + * @gsbm: guest state bitmap + * @iden: guest state ID + */ +void kvmppc_gsbm_set(struct kvmppc_gs_bitmap *gsbm, u16 iden) +{ + set_bit(kvmppc_gse_flatten_iden(iden), gsbm->bitmap); +} +EXPORT_SYMBOL_GPL(kvmppc_gsbm_set); + +/** + * kvmppc_gsbm_clear() - clear the guest state ID + * @gsbm: guest state bitmap + * @iden: guest state ID + */ +void kvmppc_gsbm_clear(struct kvmppc_gs_bitmap *gsbm, u16 iden) +{ + clear_bit(kvmppc_gse_flatten_iden(iden), gsbm->bitmap); +} +EXPORT_SYMBOL_GPL(kvmppc_gsbm_clear); + +/** + * kvmppc_gsbm_test() - test the guest state ID + * @gsbm: guest state bitmap + * @iden: guest state ID + */ +bool kvmppc_gsbm_test(struct kvmppc_gs_bitmap *gsbm, u16 iden) +{ + return test_bit(kvmppc_gse_flatten_iden(iden), gsbm->bitmap); +} +EXPORT_SYMBOL_GPL(kvmppc_gsbm_test); + +/** + * kvmppc_gsbm_next() - return the next set guest state ID + * @gsbm: guest state bitmap + * @prev: last guest state ID + */ +u16 kvmppc_gsbm_next(struct kvmppc_gs_bitmap *gsbm, u16 prev) +{ + int bit, pbit; + + pbit = prev ? kvmppc_gse_flatten_iden(prev) + 1 : 0; + bit = find_next_bit(gsbm->bitmap, KVMPPC_GSE_IDEN_COUNT, pbit); + + if (bit < KVMPPC_GSE_IDEN_COUNT) + return kvmppc_gse_unflatten_iden(bit); + return 0; +} +EXPORT_SYMBOL_GPL(kvmppc_gsbm_next); + +/** + * kvmppc_gsm_init() - initialize a guest state message + * @gsm: guest state message + * @ops: callbacks + * @data: private data + * @flags: guest wide or thread wide + */ +int kvmppc_gsm_init(struct kvmppc_gs_msg *gsm, struct kvmppc_gs_msg_ops *ops, + void *data, unsigned long flags) +{ + memset(gsm, 0, sizeof(*gsm)); + gsm->ops = ops; + gsm->data = data; + gsm->flags = flags; + + return 0; +} +EXPORT_SYMBOL_GPL(kvmppc_gsm_init); + +/** + * kvmppc_gsm_new() - creates a new guest state message + * @ops: callbacks + * @data: private data + * @flags: guest wide or thread wide + * @gfp_flags: GFP allocation flags + * + * Returns an initialized guest state message. + */ +struct kvmppc_gs_msg *kvmppc_gsm_new(struct kvmppc_gs_msg_ops *ops, void *data, + unsigned long flags, gfp_t gfp_flags) +{ + struct kvmppc_gs_msg *gsm; + + gsm = kzalloc(sizeof(*gsm), gfp_flags); + if (!gsm) + return NULL; + + kvmppc_gsm_init(gsm, ops, data, flags); + + return gsm; +} +EXPORT_SYMBOL_GPL(kvmppc_gsm_new); + +/** + * kvmppc_gsm_size() - creates a new guest state message + * @gsm: self + * + * Returns the size required for the message. + */ +size_t kvmppc_gsm_size(struct kvmppc_gs_msg *gsm) +{ + if (gsm->ops->get_size) + return gsm->ops->get_size(gsm); + return 0; +} +EXPORT_SYMBOL_GPL(kvmppc_gsm_size); + +/** + * kvmppc_gsm_free() - free guest state message + * @gsm: guest state message + * + * Returns the size required for the message. + */ +void kvmppc_gsm_free(struct kvmppc_gs_msg *gsm) +{ + kfree(gsm); +} +EXPORT_SYMBOL_GPL(kvmppc_gsm_free); + +/** + * kvmppc_gsm_fill_info() - serialises message to guest state buffer format + * @gsm: self + * @gsb: buffer to serialise into + */ +int kvmppc_gsm_fill_info(struct kvmppc_gs_msg *gsm, struct kvmppc_gs_buff *gsb) +{ + if (!gsm->ops->fill_info) + return -EINVAL; + + return gsm->ops->fill_info(gsb, gsm); +} +EXPORT_SYMBOL_GPL(kvmppc_gsm_fill_info); + +/** + * kvmppc_gsm_refresh_info() - deserialises from guest state buffer + * @gsm: self + * @gsb: buffer to serialise from + */ +int kvmppc_gsm_refresh_info(struct kvmppc_gs_msg *gsm, + struct kvmppc_gs_buff *gsb) +{ + if (!gsm->ops->fill_info) + return -EINVAL; + + return gsm->ops->refresh_info(gsm, gsb); +} +EXPORT_SYMBOL_GPL(kvmppc_gsm_refresh_info); diff --git a/arch/powerpc/kvm/test-guest-state-buffer.c b/arch/powerpc/kvm/test-guest-state-buffer.c new file mode 100644 index 000000000000..4720b8dc8837 --- /dev/null +++ b/arch/powerpc/kvm/test-guest-state-buffer.c @@ -0,0 +1,328 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +#include +#include +#include + +#include + +static void test_creating_buffer(struct kunit *test) +{ + struct kvmppc_gs_buff *gsb; + size_t size = 0x100; + + gsb = kvmppc_gsb_new(size, 0, 0, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, gsb); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, gsb->hdr); + + KUNIT_EXPECT_EQ(test, gsb->capacity, roundup_pow_of_two(size)); + KUNIT_EXPECT_EQ(test, gsb->len, sizeof(__be32)); + + kvmppc_gsb_free(gsb); +} + +static void test_adding_element(struct kunit *test) +{ + const struct kvmppc_gs_elem *head, *curr; + union { + __vector128 v; + u64 dw[2]; + } u; + int rem; + struct kvmppc_gs_buff *gsb; + size_t size = 0x1000; + int i, rc; + u64 data; + + gsb = kvmppc_gsb_new(size, 0, 0, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, gsb); + + /* Single elements, direct use of __kvmppc_gse_put() */ + data = 0xdeadbeef; + rc = __kvmppc_gse_put(gsb, KVMPPC_GSID_GPR(0), 8, &data); + KUNIT_EXPECT_GE(test, rc, 0); + + head = kvmppc_gsb_data(gsb); + KUNIT_EXPECT_EQ(test, kvmppc_gse_iden(head), KVMPPC_GSID_GPR(0)); + KUNIT_EXPECT_EQ(test, kvmppc_gse_len(head), 8); + data = 0; + memcpy(&data, kvmppc_gse_data(head), 8); + KUNIT_EXPECT_EQ(test, data, 0xdeadbeef); + + /* Multiple elements, simple wrapper */ + rc = kvmppc_gse_put_u64(gsb, KVMPPC_GSID_GPR(1), 0xcafef00d); + KUNIT_EXPECT_GE(test, rc, 0); + + u.dw[0] = 0x1; + u.dw[1] = 0x2; + rc = kvmppc_gse_put_vector128(gsb, KVMPPC_GSID_VSRS(0), &u.v); + KUNIT_EXPECT_GE(test, rc, 0); + u.dw[0] = 0x0; + u.dw[1] = 0x0; + + kvmppc_gsb_for_each_elem(i, curr, gsb, rem) { + switch (i) { + case 0: + KUNIT_EXPECT_EQ(test, kvmppc_gse_iden(curr), + KVMPPC_GSID_GPR(0)); + KUNIT_EXPECT_EQ(test, kvmppc_gse_len(curr), 8); + KUNIT_EXPECT_EQ(test, kvmppc_gse_get_be64(curr), + 0xdeadbeef); + break; + case 1: + KUNIT_EXPECT_EQ(test, kvmppc_gse_iden(curr), + KVMPPC_GSID_GPR(1)); + KUNIT_EXPECT_EQ(test, kvmppc_gse_len(curr), 8); + KUNIT_EXPECT_EQ(test, kvmppc_gse_get_u64(curr), + 0xcafef00d); + break; + case 2: + KUNIT_EXPECT_EQ(test, kvmppc_gse_iden(curr), + KVMPPC_GSID_VSRS(0)); + KUNIT_EXPECT_EQ(test, kvmppc_gse_len(curr), 16); + kvmppc_gse_get_vector128(curr, &u.v); + KUNIT_EXPECT_EQ(test, u.dw[0], 0x1); + KUNIT_EXPECT_EQ(test, u.dw[1], 0x2); + break; + } + } + KUNIT_EXPECT_EQ(test, i, 3); + + kvmppc_gsb_reset(gsb); + KUNIT_EXPECT_EQ(test, kvmppc_gsb_nelems(gsb), 0); + KUNIT_EXPECT_EQ(test, kvmppc_gsb_len(gsb), + sizeof(struct kvmppc_gs_header)); + + kvmppc_gsb_free(gsb); +} + +static void test_gs_parsing(struct kunit *test) +{ + struct kvmppc_gs_elem *gse; + struct kvmppc_gs_parser gsp = { 0 }; + struct kvmppc_gs_buff *gsb; + size_t size = 0x1000; + u64 tmp1, tmp2; + + gsb = kvmppc_gsb_new(size, 0, 0, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, gsb); + + tmp1 = 0xdeadbeefull; + kvmppc_gse_put_u64(gsb, KVMPPC_GSID_GPR(0), tmp1); + + KUNIT_EXPECT_GE(test, kvmppc_gse_parse(&gsp, gsb), 0); + + gse = kvmppc_gsp_lookup(&gsp, KVMPPC_GSID_GPR(0)); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, gse); + + tmp2 = kvmppc_gse_get_u64(gse); + KUNIT_EXPECT_EQ(test, tmp2, 0xdeadbeefull); + + kvmppc_gsb_free(gsb); +} + +static void test_gs_bitmap(struct kunit *test) +{ + struct kvmppc_gs_bitmap gsbm = { 0 }; + struct kvmppc_gs_bitmap gsbm1 = { 0 }; + struct kvmppc_gs_bitmap gsbm2 = { 0 }; + u16 iden; + int i, j; + + i = 0; + for (u16 iden = KVMPPC_GSID_HOST_STATE_SIZE; + iden <= KVMPPC_GSID_PROCESS_TABLE; iden++) { + kvmppc_gsbm_set(&gsbm, iden); + kvmppc_gsbm_set(&gsbm1, iden); + KUNIT_EXPECT_TRUE(test, kvmppc_gsbm_test(&gsbm, iden)); + kvmppc_gsbm_clear(&gsbm, iden); + KUNIT_EXPECT_FALSE(test, kvmppc_gsbm_test(&gsbm, iden)); + i++; + } + + for (u16 iden = KVMPPC_GSID_RUN_INPUT; iden <= KVMPPC_GSID_VPA; + iden++) { + kvmppc_gsbm_set(&gsbm, iden); + kvmppc_gsbm_set(&gsbm1, iden); + KUNIT_EXPECT_TRUE(test, kvmppc_gsbm_test(&gsbm, iden)); + kvmppc_gsbm_clear(&gsbm, iden); + KUNIT_EXPECT_FALSE(test, kvmppc_gsbm_test(&gsbm, iden)); + i++; + } + + for (u16 iden = KVMPPC_GSID_GPR(0); iden <= KVMPPC_GSID_CTRL; iden++) { + kvmppc_gsbm_set(&gsbm, iden); + kvmppc_gsbm_set(&gsbm1, iden); + KUNIT_EXPECT_TRUE(test, kvmppc_gsbm_test(&gsbm, iden)); + kvmppc_gsbm_clear(&gsbm, iden); + KUNIT_EXPECT_FALSE(test, kvmppc_gsbm_test(&gsbm, iden)); + i++; + } + + for (u16 iden = KVMPPC_GSID_CR; iden <= KVMPPC_GSID_PSPB; iden++) { + kvmppc_gsbm_set(&gsbm, iden); + kvmppc_gsbm_set(&gsbm1, iden); + KUNIT_EXPECT_TRUE(test, kvmppc_gsbm_test(&gsbm, iden)); + kvmppc_gsbm_clear(&gsbm, iden); + KUNIT_EXPECT_FALSE(test, kvmppc_gsbm_test(&gsbm, iden)); + i++; + } + + for (u16 iden = KVMPPC_GSID_VSRS(0); iden <= KVMPPC_GSID_VSRS(63); + iden++) { + kvmppc_gsbm_set(&gsbm, iden); + kvmppc_gsbm_set(&gsbm1, iden); + KUNIT_EXPECT_TRUE(test, kvmppc_gsbm_test(&gsbm, iden)); + kvmppc_gsbm_clear(&gsbm, iden); + KUNIT_EXPECT_FALSE(test, kvmppc_gsbm_test(&gsbm, iden)); + i++; + } + + for (u16 iden = KVMPPC_GSID_HDAR; iden <= KVMPPC_GSID_ASDR; iden++) { + kvmppc_gsbm_set(&gsbm, iden); + kvmppc_gsbm_set(&gsbm1, iden); + KUNIT_EXPECT_TRUE(test, kvmppc_gsbm_test(&gsbm, iden)); + kvmppc_gsbm_clear(&gsbm, iden); + KUNIT_EXPECT_FALSE(test, kvmppc_gsbm_test(&gsbm, iden)); + i++; + } + + j = 0; + kvmppc_gsbm_for_each(&gsbm1, iden) + { + kvmppc_gsbm_set(&gsbm2, iden); + j++; + } + KUNIT_EXPECT_EQ(test, i, j); + KUNIT_EXPECT_MEMEQ(test, &gsbm1, &gsbm2, sizeof(gsbm1)); +} + +struct kvmppc_gs_msg_test1_data { + u64 a; + u32 b; + struct kvmppc_gs_part_table c; + struct kvmppc_gs_proc_table d; + struct kvmppc_gs_buff_info e; +}; + +static size_t test1_get_size(struct kvmppc_gs_msg *gsm) +{ + size_t size = 0; + u16 ids[] = { + KVMPPC_GSID_PARTITION_TABLE, + KVMPPC_GSID_PROCESS_TABLE, + KVMPPC_GSID_RUN_INPUT, + KVMPPC_GSID_GPR(0), + KVMPPC_GSID_CR, + }; + + for (int i = 0; i < ARRAY_SIZE(ids); i++) + size += kvmppc_gse_total_size(kvmppc_gsid_size(ids[i])); + return size; +} + +static int test1_fill_info(struct kvmppc_gs_buff *gsb, + struct kvmppc_gs_msg *gsm) +{ + struct kvmppc_gs_msg_test1_data *data = gsm->data; + + if (kvmppc_gsm_includes(gsm, KVMPPC_GSID_GPR(0))) + kvmppc_gse_put_u64(gsb, KVMPPC_GSID_GPR(0), data->a); + + if (kvmppc_gsm_includes(gsm, KVMPPC_GSID_CR)) + kvmppc_gse_put_u32(gsb, KVMPPC_GSID_CR, data->b); + + if (kvmppc_gsm_includes(gsm, KVMPPC_GSID_PARTITION_TABLE)) + kvmppc_gse_put_part_table(gsb, KVMPPC_GSID_PARTITION_TABLE, + data->c); + + if (kvmppc_gsm_includes(gsm, KVMPPC_GSID_PROCESS_TABLE)) + kvmppc_gse_put_proc_table(gsb, KVMPPC_GSID_PARTITION_TABLE, + data->d); + + if (kvmppc_gsm_includes(gsm, KVMPPC_GSID_RUN_INPUT)) + kvmppc_gse_put_buff_info(gsb, KVMPPC_GSID_RUN_INPUT, data->e); + + return 0; +} + +static int test1_refresh_info(struct kvmppc_gs_msg *gsm, + struct kvmppc_gs_buff *gsb) +{ + struct kvmppc_gs_parser gsp = { 0 }; + struct kvmppc_gs_msg_test1_data *data = gsm->data; + struct kvmppc_gs_elem *gse; + int rc; + + rc = kvmppc_gse_parse(&gsp, gsb); + if (rc < 0) + return rc; + + gse = kvmppc_gsp_lookup(&gsp, KVMPPC_GSID_GPR(0)); + if (gse) + data->a = kvmppc_gse_get_u64(gse); + + gse = kvmppc_gsp_lookup(&gsp, KVMPPC_GSID_CR); + if (gse) + data->b = kvmppc_gse_get_u32(gse); + + return 0; +} + +static struct kvmppc_gs_msg_ops gs_msg_test1_ops = { + .get_size = test1_get_size, + .fill_info = test1_fill_info, + .refresh_info = test1_refresh_info, +}; + +static void test_gs_msg(struct kunit *test) +{ + struct kvmppc_gs_msg_test1_data test1_data = { + .a = 0xdeadbeef, + .b = 0x1, + }; + struct kvmppc_gs_msg *gsm; + struct kvmppc_gs_buff *gsb; + + gsm = kvmppc_gsm_new(&gs_msg_test1_ops, &test1_data, GSM_SEND, + GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, gsm); + + gsb = kvmppc_gsb_new(kvmppc_gsm_size(gsm), 0, 0, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, gsb); + + kvmppc_gsm_include(gsm, KVMPPC_GSID_PARTITION_TABLE); + kvmppc_gsm_include(gsm, KVMPPC_GSID_PROCESS_TABLE); + kvmppc_gsm_include(gsm, KVMPPC_GSID_RUN_INPUT); + kvmppc_gsm_include(gsm, KVMPPC_GSID_GPR(0)); + kvmppc_gsm_include(gsm, KVMPPC_GSID_CR); + + kvmppc_gsm_fill_info(gsm, gsb); + + memset(&test1_data, 0, sizeof(test1_data)); + + kvmppc_gsm_refresh_info(gsm, gsb); + KUNIT_EXPECT_EQ(test, test1_data.a, 0xdeadbeef); + KUNIT_EXPECT_EQ(test, test1_data.b, 0x1); + + kvmppc_gsm_free(gsm); +} + +static struct kunit_case guest_state_buffer_testcases[] = { + KUNIT_CASE(test_creating_buffer), + KUNIT_CASE(test_adding_element), + KUNIT_CASE(test_gs_bitmap), + KUNIT_CASE(test_gs_parsing), + KUNIT_CASE(test_gs_msg), + {} +}; + +static struct kunit_suite guest_state_buffer_test_suite = { + .name = "guest_state_buffer_test", + .test_cases = guest_state_buffer_testcases, +}; + +kunit_test_suites(&guest_state_buffer_test_suite); + +MODULE_LICENSE("GPL"); From dfcaacc8f970c6b4ea4e32d2186f2bea4a1d5255 Mon Sep 17 00:00:00 2001 From: Jordan Niethe Date: Thu, 14 Sep 2023 13:05:58 +1000 Subject: [PATCH 087/475] KVM: PPC: Book3s HV: Hold LPIDs in an unsigned long The LPID register is 32 bits long. The host keeps the lpids for each guest in an unsigned word struct kvm_arch. Currently, LPIDs are already limited by mmu_lpid_bits and KVM_MAX_NESTED_GUESTS_SHIFT. The nestedv2 API returns a 64 bit "Guest ID" to be used be the L1 host for each L2 guest. This value is used as an lpid, e.g. it is the parameter used by H_RPT_INVALIDATE. To minimize needless special casing it makes sense to keep this "Guest ID" in struct kvm_arch::lpid. This means that struct kvm_arch::lpid is too small so prepare for this and make it an unsigned long. This is not a problem for the KVM-HV and nestedv1 cases as their lpid values are already limited to valid ranges so in those contexts the lpid can be used as an unsigned word safely as needed. In the PAPR, the H_RPT_INVALIDATE pid/lpid parameter is already specified as an unsigned long so change pseries_rpt_invalidate() to match that. Update the callers of pseries_rpt_invalidate() to also take an unsigned long if they take an lpid value. Signed-off-by: Jordan Niethe Signed-off-by: Michael Ellerman Link: https://msgid.link/20230914030600.16993-10-jniethe5@gmail.com --- arch/powerpc/include/asm/kvm_book3s.h | 10 +++++----- arch/powerpc/include/asm/kvm_book3s_64.h | 2 +- arch/powerpc/include/asm/kvm_host.h | 2 +- arch/powerpc/include/asm/plpar_wrappers.h | 4 ++-- arch/powerpc/kvm/book3s_64_mmu_hv.c | 2 +- arch/powerpc/kvm/book3s_64_mmu_radix.c | 22 +++++++++++----------- arch/powerpc/kvm/book3s_hv_nested.c | 4 ++-- arch/powerpc/kvm/book3s_hv_uvmem.c | 2 +- arch/powerpc/kvm/book3s_xive.c | 4 ++-- 9 files changed, 26 insertions(+), 26 deletions(-) diff --git a/arch/powerpc/include/asm/kvm_book3s.h b/arch/powerpc/include/asm/kvm_book3s.h index 4c6558d5fefe..831c23e4f121 100644 --- a/arch/powerpc/include/asm/kvm_book3s.h +++ b/arch/powerpc/include/asm/kvm_book3s.h @@ -191,14 +191,14 @@ extern int kvmppc_mmu_radix_translate_table(struct kvm_vcpu *vcpu, gva_t eaddr, extern int kvmppc_mmu_radix_xlate(struct kvm_vcpu *vcpu, gva_t eaddr, struct kvmppc_pte *gpte, bool data, bool iswrite); extern void kvmppc_radix_tlbie_page(struct kvm *kvm, unsigned long addr, - unsigned int pshift, unsigned int lpid); + unsigned int pshift, u64 lpid); extern void kvmppc_unmap_pte(struct kvm *kvm, pte_t *pte, unsigned long gpa, unsigned int shift, const struct kvm_memory_slot *memslot, - unsigned int lpid); + u64 lpid); extern bool kvmppc_hv_handle_set_rc(struct kvm *kvm, bool nested, bool writing, unsigned long gpa, - unsigned int lpid); + u64 lpid); extern int kvmppc_book3s_instantiate_page(struct kvm_vcpu *vcpu, unsigned long gpa, struct kvm_memory_slot *memslot, @@ -207,7 +207,7 @@ extern int kvmppc_book3s_instantiate_page(struct kvm_vcpu *vcpu, extern int kvmppc_init_vm_radix(struct kvm *kvm); extern void kvmppc_free_radix(struct kvm *kvm); extern void kvmppc_free_pgtable_radix(struct kvm *kvm, pgd_t *pgd, - unsigned int lpid); + u64 lpid); extern int kvmppc_radix_init(void); extern void kvmppc_radix_exit(void); extern void kvm_unmap_radix(struct kvm *kvm, struct kvm_memory_slot *memslot, @@ -300,7 +300,7 @@ void kvmhv_nested_exit(void); void kvmhv_vm_nested_init(struct kvm *kvm); long kvmhv_set_partition_table(struct kvm_vcpu *vcpu); long kvmhv_copy_tofrom_guest_nested(struct kvm_vcpu *vcpu); -void kvmhv_set_ptbl_entry(unsigned int lpid, u64 dw0, u64 dw1); +void kvmhv_set_ptbl_entry(u64 lpid, u64 dw0, u64 dw1); void kvmhv_release_all_nested(struct kvm *kvm); long kvmhv_enter_nested_guest(struct kvm_vcpu *vcpu); long kvmhv_do_nested_tlbie(struct kvm_vcpu *vcpu); diff --git a/arch/powerpc/include/asm/kvm_book3s_64.h b/arch/powerpc/include/asm/kvm_book3s_64.h index d49065af08e9..572f9bbf1a25 100644 --- a/arch/powerpc/include/asm/kvm_book3s_64.h +++ b/arch/powerpc/include/asm/kvm_book3s_64.h @@ -624,7 +624,7 @@ static inline void copy_to_checkpoint(struct kvm_vcpu *vcpu) extern int kvmppc_create_pte(struct kvm *kvm, pgd_t *pgtable, pte_t pte, unsigned long gpa, unsigned int level, - unsigned long mmu_seq, unsigned int lpid, + unsigned long mmu_seq, u64 lpid, unsigned long *rmapp, struct rmap_nested **n_rmap); extern void kvmhv_insert_nest_rmap(struct kvm *kvm, unsigned long *rmapp, struct rmap_nested **n_rmap); diff --git a/arch/powerpc/include/asm/kvm_host.h b/arch/powerpc/include/asm/kvm_host.h index 14ee0dece853..429b53bc1773 100644 --- a/arch/powerpc/include/asm/kvm_host.h +++ b/arch/powerpc/include/asm/kvm_host.h @@ -276,7 +276,7 @@ struct kvm_resize_hpt; #define KVMPPC_SECURE_INIT_ABORT 0x4 /* H_SVM_INIT_ABORT issued */ struct kvm_arch { - unsigned int lpid; + u64 lpid; unsigned int smt_mode; /* # vcpus per virtual core */ unsigned int emul_smt_mode; /* emualted SMT mode, on P9 */ #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE diff --git a/arch/powerpc/include/asm/plpar_wrappers.h b/arch/powerpc/include/asm/plpar_wrappers.h index fe3d0ea0058a..8d26f2537586 100644 --- a/arch/powerpc/include/asm/plpar_wrappers.h +++ b/arch/powerpc/include/asm/plpar_wrappers.h @@ -355,7 +355,7 @@ static inline long plpar_get_cpu_characteristics(struct h_cpu_char_result *p) * error recovery of killing the process/guest will be eventually * needed. */ -static inline long pseries_rpt_invalidate(u32 pid, u64 target, u64 type, +static inline long pseries_rpt_invalidate(u64 pid, u64 target, u64 type, u64 page_sizes, u64 start, u64 end) { long rc; @@ -401,7 +401,7 @@ static inline long plpar_pte_read_4(unsigned long flags, unsigned long ptex, return 0; } -static inline long pseries_rpt_invalidate(u32 pid, u64 target, u64 type, +static inline long pseries_rpt_invalidate(u64 pid, u64 target, u64 type, u64 page_sizes, u64 start, u64 end) { return 0; diff --git a/arch/powerpc/kvm/book3s_64_mmu_hv.c b/arch/powerpc/kvm/book3s_64_mmu_hv.c index fdfc2a62dd67..2b1f0cdd8c18 100644 --- a/arch/powerpc/kvm/book3s_64_mmu_hv.c +++ b/arch/powerpc/kvm/book3s_64_mmu_hv.c @@ -121,7 +121,7 @@ void kvmppc_set_hpt(struct kvm *kvm, struct kvm_hpt_info *info) kvm->arch.hpt = *info; kvm->arch.sdr1 = __pa(info->virt) | (info->order - 18); - pr_debug("KVM guest htab at %lx (order %ld), LPID %x\n", + pr_debug("KVM guest htab at %lx (order %ld), LPID %llx\n", info->virt, (long)info->order, kvm->arch.lpid); } diff --git a/arch/powerpc/kvm/book3s_64_mmu_radix.c b/arch/powerpc/kvm/book3s_64_mmu_radix.c index ab646f59afd7..175a8eb2681f 100644 --- a/arch/powerpc/kvm/book3s_64_mmu_radix.c +++ b/arch/powerpc/kvm/book3s_64_mmu_radix.c @@ -308,7 +308,7 @@ int kvmppc_mmu_radix_xlate(struct kvm_vcpu *vcpu, gva_t eaddr, } void kvmppc_radix_tlbie_page(struct kvm *kvm, unsigned long addr, - unsigned int pshift, unsigned int lpid) + unsigned int pshift, u64 lpid) { unsigned long psize = PAGE_SIZE; int psi; @@ -345,7 +345,7 @@ void kvmppc_radix_tlbie_page(struct kvm *kvm, unsigned long addr, pr_err("KVM: TLB page invalidation hcall failed, rc=%ld\n", rc); } -static void kvmppc_radix_flush_pwc(struct kvm *kvm, unsigned int lpid) +static void kvmppc_radix_flush_pwc(struct kvm *kvm, u64 lpid) { long rc; @@ -418,7 +418,7 @@ static void kvmppc_pmd_free(pmd_t *pmdp) void kvmppc_unmap_pte(struct kvm *kvm, pte_t *pte, unsigned long gpa, unsigned int shift, const struct kvm_memory_slot *memslot, - unsigned int lpid) + u64 lpid) { unsigned long old; @@ -469,7 +469,7 @@ void kvmppc_unmap_pte(struct kvm *kvm, pte_t *pte, unsigned long gpa, * (or 4kB) mappings (of sub-pages of the same 2MB page). */ static void kvmppc_unmap_free_pte(struct kvm *kvm, pte_t *pte, bool full, - unsigned int lpid) + u64 lpid) { if (full) { memset(pte, 0, sizeof(long) << RADIX_PTE_INDEX_SIZE); @@ -490,7 +490,7 @@ static void kvmppc_unmap_free_pte(struct kvm *kvm, pte_t *pte, bool full, } static void kvmppc_unmap_free_pmd(struct kvm *kvm, pmd_t *pmd, bool full, - unsigned int lpid) + u64 lpid) { unsigned long im; pmd_t *p = pmd; @@ -519,7 +519,7 @@ static void kvmppc_unmap_free_pmd(struct kvm *kvm, pmd_t *pmd, bool full, } static void kvmppc_unmap_free_pud(struct kvm *kvm, pud_t *pud, - unsigned int lpid) + u64 lpid) { unsigned long iu; pud_t *p = pud; @@ -540,7 +540,7 @@ static void kvmppc_unmap_free_pud(struct kvm *kvm, pud_t *pud, pud_free(kvm->mm, pud); } -void kvmppc_free_pgtable_radix(struct kvm *kvm, pgd_t *pgd, unsigned int lpid) +void kvmppc_free_pgtable_radix(struct kvm *kvm, pgd_t *pgd, u64 lpid) { unsigned long ig; @@ -567,7 +567,7 @@ void kvmppc_free_radix(struct kvm *kvm) } static void kvmppc_unmap_free_pmd_entry_table(struct kvm *kvm, pmd_t *pmd, - unsigned long gpa, unsigned int lpid) + unsigned long gpa, u64 lpid) { pte_t *pte = pte_offset_kernel(pmd, 0); @@ -583,7 +583,7 @@ static void kvmppc_unmap_free_pmd_entry_table(struct kvm *kvm, pmd_t *pmd, } static void kvmppc_unmap_free_pud_entry_table(struct kvm *kvm, pud_t *pud, - unsigned long gpa, unsigned int lpid) + unsigned long gpa, u64 lpid) { pmd_t *pmd = pmd_offset(pud, 0); @@ -609,7 +609,7 @@ static void kvmppc_unmap_free_pud_entry_table(struct kvm *kvm, pud_t *pud, int kvmppc_create_pte(struct kvm *kvm, pgd_t *pgtable, pte_t pte, unsigned long gpa, unsigned int level, - unsigned long mmu_seq, unsigned int lpid, + unsigned long mmu_seq, u64 lpid, unsigned long *rmapp, struct rmap_nested **n_rmap) { pgd_t *pgd; @@ -786,7 +786,7 @@ int kvmppc_create_pte(struct kvm *kvm, pgd_t *pgtable, pte_t pte, } bool kvmppc_hv_handle_set_rc(struct kvm *kvm, bool nested, bool writing, - unsigned long gpa, unsigned int lpid) + unsigned long gpa, u64 lpid) { unsigned long pgflags; unsigned int shift; diff --git a/arch/powerpc/kvm/book3s_hv_nested.c b/arch/powerpc/kvm/book3s_hv_nested.c index 377d0b4a05ee..9b63fae27eba 100644 --- a/arch/powerpc/kvm/book3s_hv_nested.c +++ b/arch/powerpc/kvm/book3s_hv_nested.c @@ -478,7 +478,7 @@ void kvmhv_nested_exit(void) } } -static void kvmhv_flush_lpid(unsigned int lpid) +static void kvmhv_flush_lpid(u64 lpid) { long rc; @@ -500,7 +500,7 @@ static void kvmhv_flush_lpid(unsigned int lpid) pr_err("KVM: TLB LPID invalidation hcall failed, rc=%ld\n", rc); } -void kvmhv_set_ptbl_entry(unsigned int lpid, u64 dw0, u64 dw1) +void kvmhv_set_ptbl_entry(u64 lpid, u64 dw0, u64 dw1) { if (!kvmhv_on_pseries()) { mmu_partition_table_set_entry(lpid, dw0, dw1, true); diff --git a/arch/powerpc/kvm/book3s_hv_uvmem.c b/arch/powerpc/kvm/book3s_hv_uvmem.c index e2d6f9327f77..92f33115144b 100644 --- a/arch/powerpc/kvm/book3s_hv_uvmem.c +++ b/arch/powerpc/kvm/book3s_hv_uvmem.c @@ -858,7 +858,7 @@ unsigned long kvmppc_h_svm_init_done(struct kvm *kvm) } kvm->arch.secure_guest |= KVMPPC_SECURE_INIT_DONE; - pr_info("LPID %d went secure\n", kvm->arch.lpid); + pr_info("LPID %lld went secure\n", kvm->arch.lpid); out: srcu_read_unlock(&kvm->srcu, srcu_idx); diff --git a/arch/powerpc/kvm/book3s_xive.c b/arch/powerpc/kvm/book3s_xive.c index 24d8378824a2..29a382249770 100644 --- a/arch/powerpc/kvm/book3s_xive.c +++ b/arch/powerpc/kvm/book3s_xive.c @@ -884,10 +884,10 @@ int kvmppc_xive_attach_escalation(struct kvm_vcpu *vcpu, u8 prio, } if (single_escalation) - name = kasprintf(GFP_KERNEL, "kvm-%d-%d", + name = kasprintf(GFP_KERNEL, "kvm-%lld-%d", vcpu->kvm->arch.lpid, xc->server_num); else - name = kasprintf(GFP_KERNEL, "kvm-%d-%d-%d", + name = kasprintf(GFP_KERNEL, "kvm-%lld-%d-%d", vcpu->kvm->arch.lpid, xc->server_num, prio); if (!name) { pr_err("Failed to allocate escalation irq name for queue %d of VCPU %d\n", From 19d31c5f115754c369c0995df47479c384757f82 Mon Sep 17 00:00:00 2001 From: Jordan Niethe Date: Thu, 14 Sep 2023 13:05:59 +1000 Subject: [PATCH 088/475] KVM: PPC: Add support for nestedv2 guests A series of hcalls have been added to the PAPR which allow a regular guest partition to create and manage guest partitions of its own. KVM already had an interface that allowed this on powernv platforms. This existing interface will now be called "nestedv1". The newly added PAPR interface will be called "nestedv2". PHYP will support the nestedv2 interface. At this time the host side of the nestedv2 interface has not been implemented on powernv but there is no technical reason why it could not be added. The nestedv1 interface is still supported. Add support to KVM to utilize these hcalls to enable running nested guests as a pseries guest on PHYP. Overview of the new hcall usage: - L1 and L0 negotiate capabilities with H_GUEST_{G,S}ET_CAPABILITIES() - L1 requests the L0 create a L2 with H_GUEST_CREATE() and receives a handle to use in future hcalls - L1 requests the L0 create a L2 vCPU with H_GUEST_CREATE_VCPU() - L1 sets up the L2 using H_GUEST_SET and the H_GUEST_VCPU_RUN input buffer - L1 requests the L0 runs the L2 vCPU using H_GUEST_VCPU_RUN() - L2 returns to L1 with an exit reason and L1 reads the H_GUEST_VCPU_RUN output buffer populated by the L0 - L1 handles the exit using H_GET_STATE if necessary - L1 reruns L2 vCPU with H_GUEST_VCPU_RUN - L1 frees the L2 in the L0 with H_GUEST_DELETE() Support for the new API is determined by trying H_GUEST_GET_CAPABILITIES. On a successful return, use the nestedv2 interface. Use the vcpu register state setters for tracking modified guest state elements and copy the thread wide values into the H_GUEST_VCPU_RUN input buffer immediately before running a L2. The guest wide elements can not be added to the input buffer so send them with a separate H_GUEST_SET call if necessary. Make the vcpu register getter load the corresponding value from the real host with H_GUEST_GET. To avoid unnecessarily calling H_GUEST_GET, track which values have already been loaded between H_GUEST_VCPU_RUN calls. If an element is present in the H_GUEST_VCPU_RUN output buffer it also does not need to be loaded again. Tested-by: Sachin Sant Signed-off-by: Vaibhav Jain Signed-off-by: Gautam Menghani Signed-off-by: Kautuk Consul Signed-off-by: Amit Machhiwal Signed-off-by: Jordan Niethe Signed-off-by: Michael Ellerman Link: https://msgid.link/20230914030600.16993-11-jniethe5@gmail.com --- arch/powerpc/include/asm/guest-state-buffer.h | 91 ++ arch/powerpc/include/asm/hvcall.h | 30 + arch/powerpc/include/asm/kvm_book3s.h | 137 ++- arch/powerpc/include/asm/kvm_book3s_64.h | 6 + arch/powerpc/include/asm/kvm_host.h | 20 + arch/powerpc/include/asm/kvm_ppc.h | 90 +- arch/powerpc/include/asm/plpar_wrappers.h | 263 +++++ arch/powerpc/kvm/Makefile | 1 + arch/powerpc/kvm/book3s_hv.c | 134 ++- arch/powerpc/kvm/book3s_hv.h | 74 +- arch/powerpc/kvm/book3s_hv_nested.c | 40 +- arch/powerpc/kvm/book3s_hv_nestedv2.c | 994 ++++++++++++++++++ arch/powerpc/kvm/emulate_loadstore.c | 4 +- arch/powerpc/kvm/guest-state-buffer.c | 50 + 14 files changed, 1840 insertions(+), 94 deletions(-) create mode 100644 arch/powerpc/kvm/book3s_hv_nestedv2.c diff --git a/arch/powerpc/include/asm/guest-state-buffer.h b/arch/powerpc/include/asm/guest-state-buffer.h index aaefe1075fc4..808149f31576 100644 --- a/arch/powerpc/include/asm/guest-state-buffer.h +++ b/arch/powerpc/include/asm/guest-state-buffer.h @@ -5,6 +5,7 @@ #ifndef _ASM_POWERPC_GUEST_STATE_BUFFER_H #define _ASM_POWERPC_GUEST_STATE_BUFFER_H +#include "asm/hvcall.h" #include #include #include @@ -313,6 +314,8 @@ struct kvmppc_gs_buff *kvmppc_gsb_new(size_t size, unsigned long guest_id, unsigned long vcpu_id, gfp_t flags); void kvmppc_gsb_free(struct kvmppc_gs_buff *gsb); void *kvmppc_gsb_put(struct kvmppc_gs_buff *gsb, size_t size); +int kvmppc_gsb_send(struct kvmppc_gs_buff *gsb, unsigned long flags); +int kvmppc_gsb_recv(struct kvmppc_gs_buff *gsb, unsigned long flags); /** * kvmppc_gsb_header() - the header of a guest state buffer @@ -901,4 +904,92 @@ static inline void kvmppc_gsm_reset(struct kvmppc_gs_msg *gsm) kvmppc_gsbm_zero(&gsm->bitmap); } +/** + * kvmppc_gsb_receive_data - flexibly update values from a guest state buffer + * @gsb: guest state buffer + * @gsm: guest state message + * + * Requests updated values for the guest state values included in the guest + * state message. The guest state message will then deserialize the guest state + * buffer. + */ +static inline int kvmppc_gsb_receive_data(struct kvmppc_gs_buff *gsb, + struct kvmppc_gs_msg *gsm) +{ + int rc; + + kvmppc_gsb_reset(gsb); + rc = kvmppc_gsm_fill_info(gsm, gsb); + if (rc < 0) + return rc; + + rc = kvmppc_gsb_recv(gsb, gsm->flags); + if (rc < 0) + return rc; + + rc = kvmppc_gsm_refresh_info(gsm, gsb); + if (rc < 0) + return rc; + return 0; +} + +/** + * kvmppc_gsb_recv - receive a single guest state ID + * @gsb: guest state buffer + * @gsm: guest state message + * @iden: guest state identity + */ +static inline int kvmppc_gsb_receive_datum(struct kvmppc_gs_buff *gsb, + struct kvmppc_gs_msg *gsm, u16 iden) +{ + int rc; + + kvmppc_gsm_include(gsm, iden); + rc = kvmppc_gsb_receive_data(gsb, gsm); + if (rc < 0) + return rc; + kvmppc_gsm_reset(gsm); + return 0; +} + +/** + * kvmppc_gsb_send_data - flexibly send values from a guest state buffer + * @gsb: guest state buffer + * @gsm: guest state message + * + * Sends the guest state values included in the guest state message. + */ +static inline int kvmppc_gsb_send_data(struct kvmppc_gs_buff *gsb, + struct kvmppc_gs_msg *gsm) +{ + int rc; + + kvmppc_gsb_reset(gsb); + rc = kvmppc_gsm_fill_info(gsm, gsb); + if (rc < 0) + return rc; + rc = kvmppc_gsb_send(gsb, gsm->flags); + + return rc; +} + +/** + * kvmppc_gsb_recv - send a single guest state ID + * @gsb: guest state buffer + * @gsm: guest state message + * @iden: guest state identity + */ +static inline int kvmppc_gsb_send_datum(struct kvmppc_gs_buff *gsb, + struct kvmppc_gs_msg *gsm, u16 iden) +{ + int rc; + + kvmppc_gsm_include(gsm, iden); + rc = kvmppc_gsb_send_data(gsb, gsm); + if (rc < 0) + return rc; + kvmppc_gsm_reset(gsm); + return 0; +} + #endif /* _ASM_POWERPC_GUEST_STATE_BUFFER_H */ diff --git a/arch/powerpc/include/asm/hvcall.h b/arch/powerpc/include/asm/hvcall.h index c099780385dd..ddb99e982917 100644 --- a/arch/powerpc/include/asm/hvcall.h +++ b/arch/powerpc/include/asm/hvcall.h @@ -100,6 +100,18 @@ #define H_COP_HW -74 #define H_STATE -75 #define H_IN_USE -77 + +#define H_INVALID_ELEMENT_ID -79 +#define H_INVALID_ELEMENT_SIZE -80 +#define H_INVALID_ELEMENT_VALUE -81 +#define H_INPUT_BUFFER_NOT_DEFINED -82 +#define H_INPUT_BUFFER_TOO_SMALL -83 +#define H_OUTPUT_BUFFER_NOT_DEFINED -84 +#define H_OUTPUT_BUFFER_TOO_SMALL -85 +#define H_PARTITION_PAGE_TABLE_NOT_DEFINED -86 +#define H_GUEST_VCPU_STATE_NOT_HV_OWNED -87 + + #define H_UNSUPPORTED_FLAG_START -256 #define H_UNSUPPORTED_FLAG_END -511 #define H_MULTI_THREADS_ACTIVE -9005 @@ -381,6 +393,15 @@ #define H_ENTER_NESTED 0xF804 #define H_TLB_INVALIDATE 0xF808 #define H_COPY_TOFROM_GUEST 0xF80C +#define H_GUEST_GET_CAPABILITIES 0x460 +#define H_GUEST_SET_CAPABILITIES 0x464 +#define H_GUEST_CREATE 0x470 +#define H_GUEST_CREATE_VCPU 0x474 +#define H_GUEST_GET_STATE 0x478 +#define H_GUEST_SET_STATE 0x47C +#define H_GUEST_RUN_VCPU 0x480 +#define H_GUEST_COPY_MEMORY 0x484 +#define H_GUEST_DELETE 0x488 /* Flags for H_SVM_PAGE_IN */ #define H_PAGE_IN_SHARED 0x1 @@ -467,6 +488,15 @@ #define H_RPTI_PAGE_1G 0x08 #define H_RPTI_PAGE_ALL (-1UL) +/* Flags for H_GUEST_{S,G}_STATE */ +#define H_GUEST_FLAGS_WIDE (1UL<<(63-0)) + +/* Flag values used for H_{S,G}SET_GUEST_CAPABILITIES */ +#define H_GUEST_CAP_COPY_MEM (1UL<<(63-0)) +#define H_GUEST_CAP_POWER9 (1UL<<(63-1)) +#define H_GUEST_CAP_POWER10 (1UL<<(63-2)) +#define H_GUEST_CAP_BITMAP2 (1UL<<(63-63)) + #ifndef __ASSEMBLY__ #include diff --git a/arch/powerpc/include/asm/kvm_book3s.h b/arch/powerpc/include/asm/kvm_book3s.h index 831c23e4f121..4f527d09c92b 100644 --- a/arch/powerpc/include/asm/kvm_book3s.h +++ b/arch/powerpc/include/asm/kvm_book3s.h @@ -12,6 +12,7 @@ #include #include #include +#include struct kvmppc_bat { u64 raw; @@ -295,6 +296,7 @@ static inline void kvmppc_save_tm_sprs(struct kvm_vcpu *vcpu) {} static inline void kvmppc_restore_tm_sprs(struct kvm_vcpu *vcpu) {} #endif +extern unsigned long nested_capabilities; long kvmhv_nested_init(void); void kvmhv_nested_exit(void); void kvmhv_vm_nested_init(struct kvm *kvm); @@ -316,6 +318,69 @@ long int kvmhv_nested_page_fault(struct kvm_vcpu *vcpu); void kvmppc_giveup_fac(struct kvm_vcpu *vcpu, ulong fac); + +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE + +extern struct static_key_false __kvmhv_is_nestedv2; + +static inline bool kvmhv_is_nestedv2(void) +{ + return static_branch_unlikely(&__kvmhv_is_nestedv2); +} + +static inline bool kvmhv_is_nestedv1(void) +{ + return !static_branch_likely(&__kvmhv_is_nestedv2); +} + +#else + +static inline bool kvmhv_is_nestedv2(void) +{ + return false; +} + +static inline bool kvmhv_is_nestedv1(void) +{ + return false; +} + +#endif + +int __kvmhv_nestedv2_reload_ptregs(struct kvm_vcpu *vcpu, struct pt_regs *regs); +int __kvmhv_nestedv2_mark_dirty_ptregs(struct kvm_vcpu *vcpu, struct pt_regs *regs); +int __kvmhv_nestedv2_mark_dirty(struct kvm_vcpu *vcpu, u16 iden); +int __kvmhv_nestedv2_cached_reload(struct kvm_vcpu *vcpu, u16 iden); + +static inline int kvmhv_nestedv2_reload_ptregs(struct kvm_vcpu *vcpu, + struct pt_regs *regs) +{ + if (kvmhv_is_nestedv2()) + return __kvmhv_nestedv2_reload_ptregs(vcpu, regs); + return 0; +} +static inline int kvmhv_nestedv2_mark_dirty_ptregs(struct kvm_vcpu *vcpu, + struct pt_regs *regs) +{ + if (kvmhv_is_nestedv2()) + return __kvmhv_nestedv2_mark_dirty_ptregs(vcpu, regs); + return 0; +} + +static inline int kvmhv_nestedv2_mark_dirty(struct kvm_vcpu *vcpu, u16 iden) +{ + if (kvmhv_is_nestedv2()) + return __kvmhv_nestedv2_mark_dirty(vcpu, iden); + return 0; +} + +static inline int kvmhv_nestedv2_cached_reload(struct kvm_vcpu *vcpu, u16 iden) +{ + if (kvmhv_is_nestedv2()) + return __kvmhv_nestedv2_cached_reload(vcpu, iden); + return 0; +} + extern int kvm_irq_bypass; static inline struct kvmppc_vcpu_book3s *to_book3s(struct kvm_vcpu *vcpu) @@ -335,60 +400,72 @@ static inline struct kvmppc_vcpu_book3s *to_book3s(struct kvm_vcpu *vcpu) static inline void kvmppc_set_gpr(struct kvm_vcpu *vcpu, int num, ulong val) { vcpu->arch.regs.gpr[num] = val; + kvmhv_nestedv2_mark_dirty(vcpu, KVMPPC_GSID_GPR(num)); } static inline ulong kvmppc_get_gpr(struct kvm_vcpu *vcpu, int num) { + WARN_ON(kvmhv_nestedv2_cached_reload(vcpu, KVMPPC_GSID_GPR(num)) < 0); return vcpu->arch.regs.gpr[num]; } static inline void kvmppc_set_cr(struct kvm_vcpu *vcpu, u32 val) { vcpu->arch.regs.ccr = val; + kvmhv_nestedv2_mark_dirty(vcpu, KVMPPC_GSID_CR); } static inline u32 kvmppc_get_cr(struct kvm_vcpu *vcpu) { + WARN_ON(kvmhv_nestedv2_cached_reload(vcpu, KVMPPC_GSID_CR) < 0); return vcpu->arch.regs.ccr; } static inline void kvmppc_set_xer(struct kvm_vcpu *vcpu, ulong val) { vcpu->arch.regs.xer = val; + kvmhv_nestedv2_mark_dirty(vcpu, KVMPPC_GSID_XER); } static inline ulong kvmppc_get_xer(struct kvm_vcpu *vcpu) { + WARN_ON(kvmhv_nestedv2_cached_reload(vcpu, KVMPPC_GSID_XER) < 0); return vcpu->arch.regs.xer; } static inline void kvmppc_set_ctr(struct kvm_vcpu *vcpu, ulong val) { vcpu->arch.regs.ctr = val; + kvmhv_nestedv2_mark_dirty(vcpu, KVMPPC_GSID_CTR); } static inline ulong kvmppc_get_ctr(struct kvm_vcpu *vcpu) { + WARN_ON(kvmhv_nestedv2_cached_reload(vcpu, KVMPPC_GSID_CTR) < 0); return vcpu->arch.regs.ctr; } static inline void kvmppc_set_lr(struct kvm_vcpu *vcpu, ulong val) { vcpu->arch.regs.link = val; + kvmhv_nestedv2_mark_dirty(vcpu, KVMPPC_GSID_LR); } static inline ulong kvmppc_get_lr(struct kvm_vcpu *vcpu) { + WARN_ON(kvmhv_nestedv2_cached_reload(vcpu, KVMPPC_GSID_LR) < 0); return vcpu->arch.regs.link; } static inline void kvmppc_set_pc(struct kvm_vcpu *vcpu, ulong val) { vcpu->arch.regs.nip = val; + kvmhv_nestedv2_mark_dirty(vcpu, KVMPPC_GSID_NIA); } static inline ulong kvmppc_get_pc(struct kvm_vcpu *vcpu) { + WARN_ON(kvmhv_nestedv2_cached_reload(vcpu, KVMPPC_GSID_NIA) < 0); return vcpu->arch.regs.nip; } @@ -405,27 +482,32 @@ static inline ulong kvmppc_get_fault_dar(struct kvm_vcpu *vcpu) static inline u64 kvmppc_get_fpr(struct kvm_vcpu *vcpu, int i) { + WARN_ON(kvmhv_nestedv2_cached_reload(vcpu, KVMPPC_GSID_VSRS(i)) < 0); return vcpu->arch.fp.fpr[i][TS_FPROFFSET]; } static inline void kvmppc_set_fpr(struct kvm_vcpu *vcpu, int i, u64 val) { vcpu->arch.fp.fpr[i][TS_FPROFFSET] = val; + kvmhv_nestedv2_mark_dirty(vcpu, KVMPPC_GSID_VSRS(i)); } static inline u64 kvmppc_get_fpscr(struct kvm_vcpu *vcpu) { + WARN_ON(kvmhv_nestedv2_cached_reload(vcpu, KVMPPC_GSID_FPSCR) < 0); return vcpu->arch.fp.fpscr; } static inline void kvmppc_set_fpscr(struct kvm_vcpu *vcpu, u64 val) { vcpu->arch.fp.fpscr = val; + kvmhv_nestedv2_mark_dirty(vcpu, KVMPPC_GSID_FPSCR); } static inline u64 kvmppc_get_vsx_fpr(struct kvm_vcpu *vcpu, int i, int j) { + WARN_ON(kvmhv_nestedv2_cached_reload(vcpu, KVMPPC_GSID_VSRS(i)) < 0); return vcpu->arch.fp.fpr[i][j]; } @@ -433,11 +515,13 @@ static inline void kvmppc_set_vsx_fpr(struct kvm_vcpu *vcpu, int i, int j, u64 val) { vcpu->arch.fp.fpr[i][j] = val; + kvmhv_nestedv2_mark_dirty(vcpu, KVMPPC_GSID_VSRS(i)); } #ifdef CONFIG_ALTIVEC static inline void kvmppc_get_vsx_vr(struct kvm_vcpu *vcpu, int i, vector128 *v) { + WARN_ON(kvmhv_nestedv2_cached_reload(vcpu, KVMPPC_GSID_VSRS(32 + i)) < 0); *v = vcpu->arch.vr.vr[i]; } @@ -445,75 +529,86 @@ static inline void kvmppc_set_vsx_vr(struct kvm_vcpu *vcpu, int i, vector128 *val) { vcpu->arch.vr.vr[i] = *val; + kvmhv_nestedv2_mark_dirty(vcpu, KVMPPC_GSID_VSRS(32 + i)); } static inline u32 kvmppc_get_vscr(struct kvm_vcpu *vcpu) { + WARN_ON(kvmhv_nestedv2_cached_reload(vcpu, KVMPPC_GSID_VSCR) < 0); return vcpu->arch.vr.vscr.u[3]; } static inline void kvmppc_set_vscr(struct kvm_vcpu *vcpu, u32 val) { vcpu->arch.vr.vscr.u[3] = val; + kvmhv_nestedv2_mark_dirty(vcpu, KVMPPC_GSID_VSCR); } #endif -#define KVMPPC_BOOK3S_VCPU_ACCESSOR_SET(reg, size) \ +#define KVMPPC_BOOK3S_VCPU_ACCESSOR_SET(reg, size, iden) \ static inline void kvmppc_set_##reg(struct kvm_vcpu *vcpu, u##size val) \ { \ \ vcpu->arch.reg = val; \ + kvmhv_nestedv2_mark_dirty(vcpu, iden); \ } -#define KVMPPC_BOOK3S_VCPU_ACCESSOR_GET(reg, size) \ +#define KVMPPC_BOOK3S_VCPU_ACCESSOR_GET(reg, size, iden) \ static inline u##size kvmppc_get_##reg(struct kvm_vcpu *vcpu) \ { \ + WARN_ON(kvmhv_nestedv2_cached_reload(vcpu, iden) < 0); \ return vcpu->arch.reg; \ } -#define KVMPPC_BOOK3S_VCPU_ACCESSOR(reg, size) \ - KVMPPC_BOOK3S_VCPU_ACCESSOR_SET(reg, size) \ - KVMPPC_BOOK3S_VCPU_ACCESSOR_GET(reg, size) \ +#define KVMPPC_BOOK3S_VCPU_ACCESSOR(reg, size, iden) \ + KVMPPC_BOOK3S_VCPU_ACCESSOR_SET(reg, size, iden) \ + KVMPPC_BOOK3S_VCPU_ACCESSOR_GET(reg, size, iden) \ -KVMPPC_BOOK3S_VCPU_ACCESSOR(pid, 32) -KVMPPC_BOOK3S_VCPU_ACCESSOR(tar, 64) -KVMPPC_BOOK3S_VCPU_ACCESSOR(ebbhr, 64) -KVMPPC_BOOK3S_VCPU_ACCESSOR(ebbrr, 64) -KVMPPC_BOOK3S_VCPU_ACCESSOR(bescr, 64) -KVMPPC_BOOK3S_VCPU_ACCESSOR(ic, 64) -KVMPPC_BOOK3S_VCPU_ACCESSOR(vrsave, 64) +KVMPPC_BOOK3S_VCPU_ACCESSOR(pid, 32, KVMPPC_GSID_PIDR) +KVMPPC_BOOK3S_VCPU_ACCESSOR(tar, 64, KVMPPC_GSID_TAR) +KVMPPC_BOOK3S_VCPU_ACCESSOR(ebbhr, 64, KVMPPC_GSID_EBBHR) +KVMPPC_BOOK3S_VCPU_ACCESSOR(ebbrr, 64, KVMPPC_GSID_EBBRR) +KVMPPC_BOOK3S_VCPU_ACCESSOR(bescr, 64, KVMPPC_GSID_BESCR) +KVMPPC_BOOK3S_VCPU_ACCESSOR(ic, 64, KVMPPC_GSID_IC) +KVMPPC_BOOK3S_VCPU_ACCESSOR(vrsave, 64, KVMPPC_GSID_VRSAVE) -#define KVMPPC_BOOK3S_VCORE_ACCESSOR_SET(reg, size) \ +#define KVMPPC_BOOK3S_VCORE_ACCESSOR_SET(reg, size, iden) \ static inline void kvmppc_set_##reg(struct kvm_vcpu *vcpu, u##size val) \ { \ vcpu->arch.vcore->reg = val; \ + kvmhv_nestedv2_mark_dirty(vcpu, iden); \ } -#define KVMPPC_BOOK3S_VCORE_ACCESSOR_GET(reg, size) \ +#define KVMPPC_BOOK3S_VCORE_ACCESSOR_GET(reg, size, iden) \ static inline u##size kvmppc_get_##reg(struct kvm_vcpu *vcpu) \ { \ + WARN_ON(kvmhv_nestedv2_cached_reload(vcpu, iden) < 0); \ return vcpu->arch.vcore->reg; \ } -#define KVMPPC_BOOK3S_VCORE_ACCESSOR(reg, size) \ - KVMPPC_BOOK3S_VCORE_ACCESSOR_SET(reg, size) \ - KVMPPC_BOOK3S_VCORE_ACCESSOR_GET(reg, size) \ +#define KVMPPC_BOOK3S_VCORE_ACCESSOR(reg, size, iden) \ + KVMPPC_BOOK3S_VCORE_ACCESSOR_SET(reg, size, iden) \ + KVMPPC_BOOK3S_VCORE_ACCESSOR_GET(reg, size, iden) \ -KVMPPC_BOOK3S_VCORE_ACCESSOR(vtb, 64) -KVMPPC_BOOK3S_VCORE_ACCESSOR(tb_offset, 64) -KVMPPC_BOOK3S_VCORE_ACCESSOR_GET(arch_compat, 32) -KVMPPC_BOOK3S_VCORE_ACCESSOR_GET(lpcr, 64) +KVMPPC_BOOK3S_VCORE_ACCESSOR(vtb, 64, KVMPPC_GSID_VTB) +KVMPPC_BOOK3S_VCORE_ACCESSOR(tb_offset, 64, KVMPPC_GSID_TB_OFFSET) +KVMPPC_BOOK3S_VCORE_ACCESSOR_GET(arch_compat, 32, KVMPPC_GSID_LOGICAL_PVR) +KVMPPC_BOOK3S_VCORE_ACCESSOR_GET(lpcr, 64, KVMPPC_GSID_LPCR) static inline u64 kvmppc_get_dec_expires(struct kvm_vcpu *vcpu) { + WARN_ON(kvmhv_nestedv2_cached_reload(vcpu, KVMPPC_GSID_TB_OFFSET) < 0); + WARN_ON(kvmhv_nestedv2_cached_reload(vcpu, KVMPPC_GSID_DEC_EXPIRY_TB) < 0); return vcpu->arch.dec_expires; } static inline void kvmppc_set_dec_expires(struct kvm_vcpu *vcpu, u64 val) { vcpu->arch.dec_expires = val; + WARN_ON(kvmhv_nestedv2_cached_reload(vcpu, KVMPPC_GSID_TB_OFFSET) < 0); + kvmhv_nestedv2_mark_dirty(vcpu, KVMPPC_GSID_DEC_EXPIRY_TB); } /* Expiry time of vcpu DEC relative to host TB */ diff --git a/arch/powerpc/include/asm/kvm_book3s_64.h b/arch/powerpc/include/asm/kvm_book3s_64.h index 572f9bbf1a25..2477021bff54 100644 --- a/arch/powerpc/include/asm/kvm_book3s_64.h +++ b/arch/powerpc/include/asm/kvm_book3s_64.h @@ -677,6 +677,12 @@ static inline pte_t *find_kvm_host_pte(struct kvm *kvm, unsigned long mmu_seq, extern pte_t *find_kvm_nested_guest_pte(struct kvm *kvm, unsigned long lpid, unsigned long ea, unsigned *hshift); +int kvmhv_nestedv2_vcpu_create(struct kvm_vcpu *vcpu, struct kvmhv_nestedv2_io *io); +void kvmhv_nestedv2_vcpu_free(struct kvm_vcpu *vcpu, struct kvmhv_nestedv2_io *io); +int kvmhv_nestedv2_flush_vcpu(struct kvm_vcpu *vcpu, u64 time_limit); +int kvmhv_nestedv2_set_ptbl_entry(unsigned long lpid, u64 dw0, u64 dw1); +int kvmhv_nestedv2_parse_output(struct kvm_vcpu *vcpu); + #endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */ #endif /* __ASM_KVM_BOOK3S_64_H__ */ diff --git a/arch/powerpc/include/asm/kvm_host.h b/arch/powerpc/include/asm/kvm_host.h index 429b53bc1773..8799b37be295 100644 --- a/arch/powerpc/include/asm/kvm_host.h +++ b/arch/powerpc/include/asm/kvm_host.h @@ -25,6 +25,7 @@ #include #include #include +#include #define __KVM_HAVE_ARCH_VCPU_DEBUGFS @@ -509,6 +510,23 @@ union xive_tma_w01 { __be64 w01; }; + /* Nestedv2 H_GUEST_RUN_VCPU configuration */ +struct kvmhv_nestedv2_config { + struct kvmppc_gs_buff_info vcpu_run_output_cfg; + struct kvmppc_gs_buff_info vcpu_run_input_cfg; + u64 vcpu_run_output_size; +}; + + /* Nestedv2 L1<->L0 communication state */ +struct kvmhv_nestedv2_io { + struct kvmhv_nestedv2_config cfg; + struct kvmppc_gs_buff *vcpu_run_output; + struct kvmppc_gs_buff *vcpu_run_input; + struct kvmppc_gs_msg *vcpu_message; + struct kvmppc_gs_msg *vcore_message; + struct kvmppc_gs_bitmap valids; +}; + struct kvm_vcpu_arch { ulong host_stack; u32 host_pid; @@ -829,6 +847,8 @@ struct kvm_vcpu_arch { u64 nested_hfscr; /* HFSCR that the L1 requested for the nested guest */ u32 nested_vcpu_id; gpa_t nested_io_gpr; + /* For nested APIv2 guests*/ + struct kvmhv_nestedv2_io nestedv2_io; #endif #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING diff --git a/arch/powerpc/include/asm/kvm_ppc.h b/arch/powerpc/include/asm/kvm_ppc.h index 004998b267e6..db8bfaaa4536 100644 --- a/arch/powerpc/include/asm/kvm_ppc.h +++ b/arch/powerpc/include/asm/kvm_ppc.h @@ -615,6 +615,42 @@ static inline bool kvmhv_on_pseries(void) { return false; } + +#endif + +#ifndef CONFIG_PPC_BOOK3S + +static inline bool kvmhv_is_nestedv2(void) +{ + return false; +} + +static inline bool kvmhv_is_nestedv1(void) +{ + return false; +} + +static inline int kvmhv_nestedv2_reload_ptregs(struct kvm_vcpu *vcpu, + struct pt_regs *regs) +{ + return 0; +} +static inline int kvmhv_nestedv2_mark_dirty_ptregs(struct kvm_vcpu *vcpu, + struct pt_regs *regs) +{ + return 0; +} + +static inline int kvmhv_nestedv2_mark_dirty(struct kvm_vcpu *vcpu, u16 iden) +{ + return 0; +} + +static inline int kvmhv_nestedv2_cached_reload(struct kvm_vcpu *vcpu, u16 iden) +{ + return 0; +} + #endif #ifdef CONFIG_KVM_XICS @@ -939,27 +975,32 @@ static inline void kvmppc_set_##reg(struct kvm_vcpu *vcpu, ulong val) \ mtspr(bookehv_spr, val); \ } \ -#define KVMPPC_VCPU_SHARED_REGS_ACCESSOR_GET(reg, size) \ +#define KVMPPC_VCPU_SHARED_REGS_ACCESSOR_GET(reg, size, iden) \ static inline u##size kvmppc_get_##reg(struct kvm_vcpu *vcpu) \ { \ + if (iden) \ + WARN_ON(kvmhv_nestedv2_cached_reload(vcpu, iden) < 0); \ if (kvmppc_shared_big_endian(vcpu)) \ return be##size##_to_cpu(vcpu->arch.shared->reg); \ else \ return le##size##_to_cpu(vcpu->arch.shared->reg); \ } \ -#define KVMPPC_VCPU_SHARED_REGS_ACCESSOR_SET(reg, size) \ +#define KVMPPC_VCPU_SHARED_REGS_ACCESSOR_SET(reg, size, iden) \ static inline void kvmppc_set_##reg(struct kvm_vcpu *vcpu, u##size val) \ { \ if (kvmppc_shared_big_endian(vcpu)) \ vcpu->arch.shared->reg = cpu_to_be##size(val); \ else \ vcpu->arch.shared->reg = cpu_to_le##size(val); \ + \ + if (iden) \ + kvmhv_nestedv2_mark_dirty(vcpu, iden); \ } \ -#define KVMPPC_VCPU_SHARED_REGS_ACCESSOR(reg, size) \ - KVMPPC_VCPU_SHARED_REGS_ACCESSOR_GET(reg, size) \ - KVMPPC_VCPU_SHARED_REGS_ACCESSOR_SET(reg, size) \ +#define KVMPPC_VCPU_SHARED_REGS_ACCESSOR(reg, size, iden) \ + KVMPPC_VCPU_SHARED_REGS_ACCESSOR_GET(reg, size, iden) \ + KVMPPC_VCPU_SHARED_REGS_ACCESSOR_SET(reg, size, iden) \ #define KVMPPC_BOOKE_HV_SPRNG_ACCESSOR(reg, bookehv_spr) \ KVMPPC_BOOKE_HV_SPRNG_ACCESSOR_GET(reg, bookehv_spr) \ @@ -967,39 +1008,40 @@ static inline void kvmppc_set_##reg(struct kvm_vcpu *vcpu, u##size val) \ #ifdef CONFIG_KVM_BOOKE_HV -#define KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(reg, size, bookehv_spr) \ +#define KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(reg, size, bookehv_spr, iden) \ KVMPPC_BOOKE_HV_SPRNG_ACCESSOR(reg, bookehv_spr) \ #else -#define KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(reg, size, bookehv_spr) \ - KVMPPC_VCPU_SHARED_REGS_ACCESSOR(reg, size) \ +#define KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(reg, size, bookehv_spr, iden) \ + KVMPPC_VCPU_SHARED_REGS_ACCESSOR(reg, size, iden) \ #endif -KVMPPC_VCPU_SHARED_REGS_ACCESSOR(critical, 64) -KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(sprg0, 64, SPRN_GSPRG0) -KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(sprg1, 64, SPRN_GSPRG1) -KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(sprg2, 64, SPRN_GSPRG2) -KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(sprg3, 64, SPRN_GSPRG3) -KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(srr0, 64, SPRN_GSRR0) -KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(srr1, 64, SPRN_GSRR1) -KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(dar, 64, SPRN_GDEAR) -KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(esr, 64, SPRN_GESR) -KVMPPC_VCPU_SHARED_REGS_ACCESSOR_GET(msr, 64) +KVMPPC_VCPU_SHARED_REGS_ACCESSOR(critical, 64, 0) +KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(sprg0, 64, SPRN_GSPRG0, KVMPPC_GSID_SPRG0) +KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(sprg1, 64, SPRN_GSPRG1, KVMPPC_GSID_SPRG1) +KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(sprg2, 64, SPRN_GSPRG2, KVMPPC_GSID_SPRG2) +KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(sprg3, 64, SPRN_GSPRG3, KVMPPC_GSID_SPRG3) +KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(srr0, 64, SPRN_GSRR0, KVMPPC_GSID_SRR0) +KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(srr1, 64, SPRN_GSRR1, KVMPPC_GSID_SRR1) +KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(dar, 64, SPRN_GDEAR, KVMPPC_GSID_DAR) +KVMPPC_BOOKE_HV_SPRNG_OR_VCPU_SHARED_REGS_ACCESSOR(esr, 64, SPRN_GESR, 0) +KVMPPC_VCPU_SHARED_REGS_ACCESSOR_GET(msr, 64, KVMPPC_GSID_MSR) static inline void kvmppc_set_msr_fast(struct kvm_vcpu *vcpu, u64 val) { if (kvmppc_shared_big_endian(vcpu)) vcpu->arch.shared->msr = cpu_to_be64(val); else vcpu->arch.shared->msr = cpu_to_le64(val); + kvmhv_nestedv2_mark_dirty(vcpu, KVMPPC_GSID_MSR); } -KVMPPC_VCPU_SHARED_REGS_ACCESSOR(dsisr, 32) -KVMPPC_VCPU_SHARED_REGS_ACCESSOR(int_pending, 32) -KVMPPC_VCPU_SHARED_REGS_ACCESSOR(sprg4, 64) -KVMPPC_VCPU_SHARED_REGS_ACCESSOR(sprg5, 64) -KVMPPC_VCPU_SHARED_REGS_ACCESSOR(sprg6, 64) -KVMPPC_VCPU_SHARED_REGS_ACCESSOR(sprg7, 64) +KVMPPC_VCPU_SHARED_REGS_ACCESSOR(dsisr, 32, KVMPPC_GSID_DSISR) +KVMPPC_VCPU_SHARED_REGS_ACCESSOR(int_pending, 32, 0) +KVMPPC_VCPU_SHARED_REGS_ACCESSOR(sprg4, 64, 0) +KVMPPC_VCPU_SHARED_REGS_ACCESSOR(sprg5, 64, 0) +KVMPPC_VCPU_SHARED_REGS_ACCESSOR(sprg6, 64, 0) +KVMPPC_VCPU_SHARED_REGS_ACCESSOR(sprg7, 64, 0) static inline u32 kvmppc_get_sr(struct kvm_vcpu *vcpu, int nr) { diff --git a/arch/powerpc/include/asm/plpar_wrappers.h b/arch/powerpc/include/asm/plpar_wrappers.h index 8d26f2537586..b3ee44a40c2f 100644 --- a/arch/powerpc/include/asm/plpar_wrappers.h +++ b/arch/powerpc/include/asm/plpar_wrappers.h @@ -6,6 +6,7 @@ #include #include +#include #include #include @@ -343,6 +344,212 @@ static inline long plpar_get_cpu_characteristics(struct h_cpu_char_result *p) return rc; } +static inline long plpar_guest_create(unsigned long flags, unsigned long *guest_id) +{ + unsigned long retbuf[PLPAR_HCALL_BUFSIZE]; + unsigned long token; + long rc; + + token = -1UL; + do { + rc = plpar_hcall(H_GUEST_CREATE, retbuf, flags, token); + if (rc == H_SUCCESS) + *guest_id = retbuf[0]; + + if (rc == H_BUSY) { + token = retbuf[0]; + cond_resched(); + } + + if (H_IS_LONG_BUSY(rc)) { + token = retbuf[0]; + msleep(get_longbusy_msecs(rc)); + rc = H_BUSY; + } + + } while (rc == H_BUSY); + + return rc; +} + +static inline long plpar_guest_create_vcpu(unsigned long flags, + unsigned long guest_id, + unsigned long vcpu_id) +{ + long rc; + + do { + rc = plpar_hcall_norets(H_GUEST_CREATE_VCPU, 0, guest_id, vcpu_id); + + if (rc == H_BUSY) + cond_resched(); + + if (H_IS_LONG_BUSY(rc)) { + msleep(get_longbusy_msecs(rc)); + rc = H_BUSY; + } + + } while (rc == H_BUSY); + + return rc; +} + +static inline long plpar_guest_set_state(unsigned long flags, + unsigned long guest_id, + unsigned long vcpu_id, + unsigned long data_buffer, + unsigned long data_size, + unsigned long *failed_index) +{ + unsigned long retbuf[PLPAR_HCALL_BUFSIZE]; + long rc; + + while (true) { + rc = plpar_hcall(H_GUEST_SET_STATE, retbuf, flags, guest_id, + vcpu_id, data_buffer, data_size); + + if (rc == H_BUSY) { + cpu_relax(); + continue; + } + + if (H_IS_LONG_BUSY(rc)) { + mdelay(get_longbusy_msecs(rc)); + continue; + } + + if (rc == H_INVALID_ELEMENT_ID) + *failed_index = retbuf[0]; + else if (rc == H_INVALID_ELEMENT_SIZE) + *failed_index = retbuf[0]; + else if (rc == H_INVALID_ELEMENT_VALUE) + *failed_index = retbuf[0]; + + break; + } + + return rc; +} + +static inline long plpar_guest_get_state(unsigned long flags, + unsigned long guest_id, + unsigned long vcpu_id, + unsigned long data_buffer, + unsigned long data_size, + unsigned long *failed_index) +{ + unsigned long retbuf[PLPAR_HCALL_BUFSIZE]; + long rc; + + while (true) { + rc = plpar_hcall(H_GUEST_GET_STATE, retbuf, flags, guest_id, + vcpu_id, data_buffer, data_size); + + if (rc == H_BUSY) { + cpu_relax(); + continue; + } + + if (H_IS_LONG_BUSY(rc)) { + mdelay(get_longbusy_msecs(rc)); + continue; + } + + if (rc == H_INVALID_ELEMENT_ID) + *failed_index = retbuf[0]; + else if (rc == H_INVALID_ELEMENT_SIZE) + *failed_index = retbuf[0]; + else if (rc == H_INVALID_ELEMENT_VALUE) + *failed_index = retbuf[0]; + + break; + } + + return rc; +} + +static inline long plpar_guest_run_vcpu(unsigned long flags, unsigned long guest_id, + unsigned long vcpu_id, int *trap, + unsigned long *failed_index) +{ + unsigned long retbuf[PLPAR_HCALL_BUFSIZE]; + long rc; + + rc = plpar_hcall(H_GUEST_RUN_VCPU, retbuf, flags, guest_id, vcpu_id); + if (rc == H_SUCCESS) + *trap = retbuf[0]; + else if (rc == H_INVALID_ELEMENT_ID) + *failed_index = retbuf[0]; + else if (rc == H_INVALID_ELEMENT_SIZE) + *failed_index = retbuf[0]; + else if (rc == H_INVALID_ELEMENT_VALUE) + *failed_index = retbuf[0]; + + return rc; +} + +static inline long plpar_guest_delete(unsigned long flags, u64 guest_id) +{ + long rc; + + do { + rc = plpar_hcall_norets(H_GUEST_DELETE, flags, guest_id); + if (rc == H_BUSY) + cond_resched(); + + if (H_IS_LONG_BUSY(rc)) { + msleep(get_longbusy_msecs(rc)); + rc = H_BUSY; + } + + } while (rc == H_BUSY); + + return rc; +} + +static inline long plpar_guest_set_capabilities(unsigned long flags, + unsigned long capabilities) +{ + unsigned long retbuf[PLPAR_HCALL_BUFSIZE]; + long rc; + + do { + rc = plpar_hcall(H_GUEST_SET_CAPABILITIES, retbuf, flags, capabilities); + if (rc == H_BUSY) + cond_resched(); + + if (H_IS_LONG_BUSY(rc)) { + msleep(get_longbusy_msecs(rc)); + rc = H_BUSY; + } + } while (rc == H_BUSY); + + return rc; +} + +static inline long plpar_guest_get_capabilities(unsigned long flags, + unsigned long *capabilities) +{ + unsigned long retbuf[PLPAR_HCALL_BUFSIZE]; + long rc; + + do { + rc = plpar_hcall(H_GUEST_GET_CAPABILITIES, retbuf, flags); + if (rc == H_BUSY) + cond_resched(); + + if (H_IS_LONG_BUSY(rc)) { + msleep(get_longbusy_msecs(rc)); + rc = H_BUSY; + } + } while (rc == H_BUSY); + + if (rc == H_SUCCESS) + *capabilities = retbuf[0]; + + return rc; +} + /* * Wrapper to H_RPT_INVALIDATE hcall that handles return values appropriately * @@ -407,6 +614,62 @@ static inline long pseries_rpt_invalidate(u64 pid, u64 target, u64 type, return 0; } +static inline long plpar_guest_create_vcpu(unsigned long flags, + unsigned long guest_id, + unsigned long vcpu_id) +{ + return 0; +} + +static inline long plpar_guest_get_state(unsigned long flags, + unsigned long guest_id, + unsigned long vcpu_id, + unsigned long data_buffer, + unsigned long data_size, + unsigned long *failed_index) +{ + return 0; +} + +static inline long plpar_guest_set_state(unsigned long flags, + unsigned long guest_id, + unsigned long vcpu_id, + unsigned long data_buffer, + unsigned long data_size, + unsigned long *failed_index) +{ + return 0; +} + +static inline long plpar_guest_run_vcpu(unsigned long flags, unsigned long guest_id, + unsigned long vcpu_id, int *trap, + unsigned long *failed_index) +{ + return 0; +} + +static inline long plpar_guest_create(unsigned long flags, unsigned long *guest_id) +{ + return 0; +} + +static inline long plpar_guest_delete(unsigned long flags, u64 guest_id) +{ + return 0; +} + +static inline long plpar_guest_get_capabilities(unsigned long flags, + unsigned long *capabilities) +{ + return 0; +} + +static inline long plpar_guest_set_capabilities(unsigned long flags, + unsigned long capabilities) +{ + return 0; +} + #endif /* CONFIG_PPC_PSERIES */ #endif /* _ASM_POWERPC_PLPAR_WRAPPERS_H */ diff --git a/arch/powerpc/kvm/Makefile b/arch/powerpc/kvm/Makefile index eb8445e71c14..4bd9d1230869 100644 --- a/arch/powerpc/kvm/Makefile +++ b/arch/powerpc/kvm/Makefile @@ -87,6 +87,7 @@ kvm-book3s_64-builtin-objs-$(CONFIG_KVM_BOOK3S_64_HANDLER) += \ book3s_hv_ras.o \ book3s_hv_builtin.o \ book3s_hv_p9_perf.o \ + book3s_hv_nestedv2.o \ guest-state-buffer.o \ $(kvm-book3s_64-builtin-tm-objs-y) \ $(kvm-book3s_64-builtin-xics-objs-y) diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c index 5743f32bf45e..1ed6ec140701 100644 --- a/arch/powerpc/kvm/book3s_hv.c +++ b/arch/powerpc/kvm/book3s_hv.c @@ -393,7 +393,7 @@ static void kvmppc_set_pvr_hv(struct kvm_vcpu *vcpu, u32 pvr) static int kvmppc_set_arch_compat(struct kvm_vcpu *vcpu, u32 arch_compat) { - unsigned long host_pcr_bit = 0, guest_pcr_bit = 0; + unsigned long host_pcr_bit = 0, guest_pcr_bit = 0, cap = 0; struct kvmppc_vcore *vc = vcpu->arch.vcore; /* We can (emulate) our own architecture version and anything older */ @@ -424,9 +424,11 @@ static int kvmppc_set_arch_compat(struct kvm_vcpu *vcpu, u32 arch_compat) break; case PVR_ARCH_300: guest_pcr_bit = PCR_ARCH_300; + cap = H_GUEST_CAP_POWER9; break; case PVR_ARCH_31: guest_pcr_bit = PCR_ARCH_31; + cap = H_GUEST_CAP_POWER10; break; default: return -EINVAL; @@ -437,8 +439,14 @@ static int kvmppc_set_arch_compat(struct kvm_vcpu *vcpu, u32 arch_compat) if (guest_pcr_bit > host_pcr_bit) return -EINVAL; + if (kvmhv_on_pseries() && kvmhv_is_nestedv2()) { + if (!(cap & nested_capabilities)) + return -EINVAL; + } + spin_lock(&vc->lock); vc->arch_compat = arch_compat; + kvmhv_nestedv2_mark_dirty(vcpu, KVMPPC_GSID_LOGICAL_PVR); /* * Set all PCR bits for which guest_pcr_bit <= bit < host_pcr_bit * Also set all reserved PCR bits @@ -2187,6 +2195,7 @@ static void kvmppc_set_lpcr(struct kvm_vcpu *vcpu, u64 new_lpcr, } vc->lpcr = new_lpcr; + kvmhv_nestedv2_mark_dirty(vcpu, KVMPPC_GSID_LPCR); spin_unlock(&vc->lock); } @@ -2927,8 +2936,14 @@ static int kvmppc_core_vcpu_create_hv(struct kvm_vcpu *vcpu) vcpu->arch.shared_big_endian = false; #endif #endif - kvmppc_set_mmcr_hv(vcpu, 0, MMCR0_FC); + if (kvmhv_is_nestedv2()) { + err = kvmhv_nestedv2_vcpu_create(vcpu, &vcpu->arch.nestedv2_io); + if (err < 0) + return err; + } + + kvmppc_set_mmcr_hv(vcpu, 0, MMCR0_FC); if (cpu_has_feature(CPU_FTR_ARCH_31)) { kvmppc_set_mmcr_hv(vcpu, 0, kvmppc_get_mmcr_hv(vcpu, 0) | MMCR0_PMCCEXT); kvmppc_set_mmcra_hv(vcpu, MMCRA_BHRB_DISABLE); @@ -3084,6 +3099,8 @@ static void kvmppc_core_vcpu_free_hv(struct kvm_vcpu *vcpu) unpin_vpa(vcpu->kvm, &vcpu->arch.slb_shadow); unpin_vpa(vcpu->kvm, &vcpu->arch.vpa); spin_unlock(&vcpu->arch.vpa_update_lock); + if (kvmhv_is_nestedv2()) + kvmhv_nestedv2_vcpu_free(vcpu, &vcpu->arch.nestedv2_io); } static int kvmppc_core_check_requests_hv(struct kvm_vcpu *vcpu) @@ -4048,6 +4065,55 @@ static void vcpu_vpa_increment_dispatch(struct kvm_vcpu *vcpu) } } +static int kvmhv_vcpu_entry_nestedv2(struct kvm_vcpu *vcpu, u64 time_limit, + unsigned long lpcr, u64 *tb) +{ + struct kvmhv_nestedv2_io *io; + unsigned long msr, i; + int trap; + long rc; + + io = &vcpu->arch.nestedv2_io; + + msr = mfmsr(); + kvmppc_msr_hard_disable_set_facilities(vcpu, msr); + if (lazy_irq_pending()) + return 0; + + rc = kvmhv_nestedv2_flush_vcpu(vcpu, time_limit); + if (rc < 0) + return -EINVAL; + + accumulate_time(vcpu, &vcpu->arch.in_guest); + rc = plpar_guest_run_vcpu(0, vcpu->kvm->arch.lpid, vcpu->vcpu_id, + &trap, &i); + + if (rc != H_SUCCESS) { + pr_err("KVM Guest Run VCPU hcall failed\n"); + if (rc == H_INVALID_ELEMENT_ID) + pr_err("KVM: Guest Run VCPU invalid element id at %ld\n", i); + else if (rc == H_INVALID_ELEMENT_SIZE) + pr_err("KVM: Guest Run VCPU invalid element size at %ld\n", i); + else if (rc == H_INVALID_ELEMENT_VALUE) + pr_err("KVM: Guest Run VCPU invalid element value at %ld\n", i); + return -EINVAL; + } + accumulate_time(vcpu, &vcpu->arch.guest_exit); + + *tb = mftb(); + kvmppc_gsm_reset(io->vcpu_message); + kvmppc_gsm_reset(io->vcore_message); + kvmppc_gsbm_zero(&io->valids); + + rc = kvmhv_nestedv2_parse_output(vcpu); + if (rc < 0) + return -EINVAL; + + timer_rearm_host_dec(*tb); + + return trap; +} + /* call our hypervisor to load up HV regs and go */ static int kvmhv_vcpu_entry_p9_nested(struct kvm_vcpu *vcpu, u64 time_limit, unsigned long lpcr, u64 *tb) { @@ -4165,7 +4231,10 @@ static int kvmhv_p9_guest_entry(struct kvm_vcpu *vcpu, u64 time_limit, vcpu_vpa_increment_dispatch(vcpu); if (kvmhv_on_pseries()) { - trap = kvmhv_vcpu_entry_p9_nested(vcpu, time_limit, lpcr, tb); + if (kvmhv_is_nestedv1()) + trap = kvmhv_vcpu_entry_p9_nested(vcpu, time_limit, lpcr, tb); + else + trap = kvmhv_vcpu_entry_nestedv2(vcpu, time_limit, lpcr, tb); /* H_CEDE has to be handled now, not later */ if (trap == BOOK3S_INTERRUPT_SYSCALL && !nested && @@ -5145,6 +5214,14 @@ void kvmppc_update_lpcr(struct kvm *kvm, unsigned long lpcr, unsigned long mask) if (++cores_done >= kvm->arch.online_vcores) break; } + + if (kvmhv_is_nestedv2()) { + struct kvm_vcpu *vcpu; + + kvm_for_each_vcpu(i, vcpu, kvm) { + kvmhv_nestedv2_mark_dirty(vcpu, KVMPPC_GSID_LPCR); + } + } } void kvmppc_setup_partition_table(struct kvm *kvm) @@ -5411,15 +5488,43 @@ static int kvmppc_core_init_vm_hv(struct kvm *kvm) /* Allocate the guest's logical partition ID */ - lpid = kvmppc_alloc_lpid(); - if ((long)lpid < 0) - return -ENOMEM; - kvm->arch.lpid = lpid; + if (!kvmhv_is_nestedv2()) { + lpid = kvmppc_alloc_lpid(); + if ((long)lpid < 0) + return -ENOMEM; + kvm->arch.lpid = lpid; + } kvmppc_alloc_host_rm_ops(); kvmhv_vm_nested_init(kvm); + if (kvmhv_is_nestedv2()) { + long rc; + unsigned long guest_id; + + rc = plpar_guest_create(0, &guest_id); + + if (rc != H_SUCCESS) + pr_err("KVM: Create Guest hcall failed, rc=%ld\n", rc); + + switch (rc) { + case H_PARAMETER: + case H_FUNCTION: + case H_STATE: + return -EINVAL; + case H_NOT_ENOUGH_RESOURCES: + case H_ABORTED: + return -ENOMEM; + case H_AUTHORITY: + return -EPERM; + case H_NOT_AVAILABLE: + return -EBUSY; + } + kvm->arch.lpid = guest_id; + } + + /* * Since we don't flush the TLB when tearing down a VM, * and this lpid might have previously been used, @@ -5489,7 +5594,10 @@ static int kvmppc_core_init_vm_hv(struct kvm *kvm) lpcr |= LPCR_HAIL; ret = kvmppc_init_vm_radix(kvm); if (ret) { - kvmppc_free_lpid(kvm->arch.lpid); + if (kvmhv_is_nestedv2()) + plpar_guest_delete(0, kvm->arch.lpid); + else + kvmppc_free_lpid(kvm->arch.lpid); return ret; } kvmppc_setup_partition_table(kvm); @@ -5579,10 +5687,14 @@ static void kvmppc_core_destroy_vm_hv(struct kvm *kvm) kvm->arch.process_table = 0; if (kvm->arch.secure_guest) uv_svm_terminate(kvm->arch.lpid); - kvmhv_set_ptbl_entry(kvm->arch.lpid, 0, 0); + if (!kvmhv_is_nestedv2()) + kvmhv_set_ptbl_entry(kvm->arch.lpid, 0, 0); } - kvmppc_free_lpid(kvm->arch.lpid); + if (kvmhv_is_nestedv2()) + plpar_guest_delete(0, kvm->arch.lpid); + else + kvmppc_free_lpid(kvm->arch.lpid); kvmppc_free_pimap(kvm); } @@ -5994,6 +6106,8 @@ static int kvmhv_enable_nested(struct kvm *kvm) return -ENODEV; if (!radix_enabled()) return -ENODEV; + if (kvmhv_is_nestedv2()) + return -ENODEV; /* kvm == NULL means the caller is testing if the capability exists */ if (kvm) diff --git a/arch/powerpc/kvm/book3s_hv.h b/arch/powerpc/kvm/book3s_hv.h index 95241764dfb4..47b2c815641e 100644 --- a/arch/powerpc/kvm/book3s_hv.h +++ b/arch/powerpc/kvm/book3s_hv.h @@ -3,6 +3,8 @@ /* * Privileged (non-hypervisor) host registers to save. */ +#include "asm/guest-state-buffer.h" + struct p9_host_os_sprs { unsigned long iamr; unsigned long amr; @@ -54,67 +56,73 @@ void accumulate_time(struct kvm_vcpu *vcpu, struct kvmhv_tb_accumulator *next); static inline void __kvmppc_set_msr_hv(struct kvm_vcpu *vcpu, u64 val) { vcpu->arch.shregs.msr = val; + kvmhv_nestedv2_mark_dirty(vcpu, KVMPPC_GSID_MSR); } static inline u64 __kvmppc_get_msr_hv(struct kvm_vcpu *vcpu) { + WARN_ON(kvmhv_nestedv2_cached_reload(vcpu, KVMPPC_GSID_MSR) < 0); return vcpu->arch.shregs.msr; } -#define KVMPPC_BOOK3S_HV_VCPU_ACCESSOR_SET(reg, size) \ +#define KVMPPC_BOOK3S_HV_VCPU_ACCESSOR_SET(reg, size, iden) \ static inline void kvmppc_set_##reg ##_hv(struct kvm_vcpu *vcpu, u##size val) \ { \ vcpu->arch.reg = val; \ + kvmhv_nestedv2_mark_dirty(vcpu, iden); \ } -#define KVMPPC_BOOK3S_HV_VCPU_ACCESSOR_GET(reg, size) \ +#define KVMPPC_BOOK3S_HV_VCPU_ACCESSOR_GET(reg, size, iden) \ static inline u##size kvmppc_get_##reg ##_hv(struct kvm_vcpu *vcpu) \ { \ + kvmhv_nestedv2_cached_reload(vcpu, iden); \ return vcpu->arch.reg; \ } -#define KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(reg, size) \ - KVMPPC_BOOK3S_HV_VCPU_ACCESSOR_SET(reg, size) \ - KVMPPC_BOOK3S_HV_VCPU_ACCESSOR_GET(reg, size) \ +#define KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(reg, size, iden) \ + KVMPPC_BOOK3S_HV_VCPU_ACCESSOR_SET(reg, size, iden) \ + KVMPPC_BOOK3S_HV_VCPU_ACCESSOR_GET(reg, size, iden) \ -#define KVMPPC_BOOK3S_HV_VCPU_ARRAY_ACCESSOR_SET(reg, size) \ +#define KVMPPC_BOOK3S_HV_VCPU_ARRAY_ACCESSOR_SET(reg, size, iden) \ static inline void kvmppc_set_##reg ##_hv(struct kvm_vcpu *vcpu, int i, u##size val) \ { \ vcpu->arch.reg[i] = val; \ + kvmhv_nestedv2_mark_dirty(vcpu, iden(i)); \ } -#define KVMPPC_BOOK3S_HV_VCPU_ARRAY_ACCESSOR_GET(reg, size) \ +#define KVMPPC_BOOK3S_HV_VCPU_ARRAY_ACCESSOR_GET(reg, size, iden) \ static inline u##size kvmppc_get_##reg ##_hv(struct kvm_vcpu *vcpu, int i) \ { \ + WARN_ON(kvmhv_nestedv2_cached_reload(vcpu, iden(i)) < 0); \ return vcpu->arch.reg[i]; \ } -#define KVMPPC_BOOK3S_HV_VCPU_ARRAY_ACCESSOR(reg, size) \ - KVMPPC_BOOK3S_HV_VCPU_ARRAY_ACCESSOR_SET(reg, size) \ - KVMPPC_BOOK3S_HV_VCPU_ARRAY_ACCESSOR_GET(reg, size) \ +#define KVMPPC_BOOK3S_HV_VCPU_ARRAY_ACCESSOR(reg, size, iden) \ + KVMPPC_BOOK3S_HV_VCPU_ARRAY_ACCESSOR_SET(reg, size, iden) \ + KVMPPC_BOOK3S_HV_VCPU_ARRAY_ACCESSOR_GET(reg, size, iden) \ -KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(mmcra, 64) -KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(hfscr, 64) -KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(fscr, 64) -KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(dscr, 64) -KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(purr, 64) -KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(spurr, 64) -KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(amr, 64) -KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(uamor, 64) -KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(siar, 64) -KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(sdar, 64) -KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(iamr, 64) -KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(dawr0, 64) -KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(dawr1, 64) -KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(dawrx0, 64) -KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(dawrx1, 64) -KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(ciabr, 64) -KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(wort, 64) -KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(ppr, 64) -KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(ctrl, 64) +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(mmcra, 64, KVMPPC_GSID_MMCRA) +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(hfscr, 64, KVMPPC_GSID_HFSCR) +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(fscr, 64, KVMPPC_GSID_FSCR) +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(dscr, 64, KVMPPC_GSID_DSCR) +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(purr, 64, KVMPPC_GSID_PURR) +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(spurr, 64, KVMPPC_GSID_SPURR) +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(amr, 64, KVMPPC_GSID_AMR) +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(uamor, 64, KVMPPC_GSID_UAMOR) +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(siar, 64, KVMPPC_GSID_SIAR) +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(sdar, 64, KVMPPC_GSID_SDAR) +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(iamr, 64, KVMPPC_GSID_IAMR) +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(dawr0, 64, KVMPPC_GSID_DAWR0) +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(dawr1, 64, KVMPPC_GSID_DAWR1) +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(dawrx0, 64, KVMPPC_GSID_DAWRX0) +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(dawrx1, 64, KVMPPC_GSID_DAWRX1) +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(ciabr, 64, KVMPPC_GSID_CIABR) +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(wort, 64, KVMPPC_GSID_WORT) +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(ppr, 64, KVMPPC_GSID_PPR) +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(ctrl, 64, KVMPPC_GSID_CTRL); -KVMPPC_BOOK3S_HV_VCPU_ARRAY_ACCESSOR(mmcr, 64) -KVMPPC_BOOK3S_HV_VCPU_ARRAY_ACCESSOR(sier, 64) -KVMPPC_BOOK3S_HV_VCPU_ARRAY_ACCESSOR(pmc, 32) +KVMPPC_BOOK3S_HV_VCPU_ARRAY_ACCESSOR(mmcr, 64, KVMPPC_GSID_MMCR) +KVMPPC_BOOK3S_HV_VCPU_ARRAY_ACCESSOR(sier, 64, KVMPPC_GSID_SIER) +KVMPPC_BOOK3S_HV_VCPU_ARRAY_ACCESSOR(pmc, 32, KVMPPC_GSID_PMC) -KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(pspb, 32) +KVMPPC_BOOK3S_HV_VCPU_ACCESSOR(pspb, 32, KVMPPC_GSID_PSPB) diff --git a/arch/powerpc/kvm/book3s_hv_nested.c b/arch/powerpc/kvm/book3s_hv_nested.c index 9b63fae27eba..3b658b8696bc 100644 --- a/arch/powerpc/kvm/book3s_hv_nested.c +++ b/arch/powerpc/kvm/book3s_hv_nested.c @@ -428,10 +428,12 @@ long kvmhv_enter_nested_guest(struct kvm_vcpu *vcpu) return vcpu->arch.trap; } +unsigned long nested_capabilities; + long kvmhv_nested_init(void) { long int ptb_order; - unsigned long ptcr; + unsigned long ptcr, host_capabilities; long rc; if (!kvmhv_on_pseries()) @@ -439,6 +441,29 @@ long kvmhv_nested_init(void) if (!radix_enabled()) return -ENODEV; + rc = plpar_guest_get_capabilities(0, &host_capabilities); + if (rc == H_SUCCESS) { + unsigned long capabilities = 0; + + if (cpu_has_feature(CPU_FTR_ARCH_31)) + capabilities |= H_GUEST_CAP_POWER10; + if (cpu_has_feature(CPU_FTR_ARCH_300)) + capabilities |= H_GUEST_CAP_POWER9; + + nested_capabilities = capabilities & host_capabilities; + rc = plpar_guest_set_capabilities(0, nested_capabilities); + if (rc != H_SUCCESS) { + pr_err("kvm-hv: Could not configure parent hypervisor capabilities (rc=%ld)", + rc); + return -ENODEV; + } + + static_branch_enable(&__kvmhv_is_nestedv2); + return 0; + } + + pr_info("kvm-hv: nestedv2 get capabilities hcall failed, falling back to nestedv1 (rc=%ld)\n", + rc); /* Partition table entry is 1<<4 bytes in size, hence the 4. */ ptb_order = KVM_MAX_NESTED_GUESTS_SHIFT + 4; /* Minimum partition table size is 1<<12 bytes */ @@ -507,10 +532,15 @@ void kvmhv_set_ptbl_entry(u64 lpid, u64 dw0, u64 dw1) return; } - pseries_partition_tb[lpid].patb0 = cpu_to_be64(dw0); - pseries_partition_tb[lpid].patb1 = cpu_to_be64(dw1); - /* L0 will do the necessary barriers */ - kvmhv_flush_lpid(lpid); + if (kvmhv_is_nestedv1()) { + pseries_partition_tb[lpid].patb0 = cpu_to_be64(dw0); + pseries_partition_tb[lpid].patb1 = cpu_to_be64(dw1); + /* L0 will do the necessary barriers */ + kvmhv_flush_lpid(lpid); + } + + if (kvmhv_is_nestedv2()) + kvmhv_nestedv2_set_ptbl_entry(lpid, dw0, dw1); } static void kvmhv_set_nested_ptbl(struct kvm_nested_guest *gp) diff --git a/arch/powerpc/kvm/book3s_hv_nestedv2.c b/arch/powerpc/kvm/book3s_hv_nestedv2.c new file mode 100644 index 000000000000..fd3c4f2d9480 --- /dev/null +++ b/arch/powerpc/kvm/book3s_hv_nestedv2.c @@ -0,0 +1,994 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2023 Jordan Niethe, IBM Corp. + * + * Authors: + * Jordan Niethe + * + * Description: KVM functions specific to running on Book 3S + * processors as a NESTEDv2 guest. + * + */ + +#include "linux/blk-mq.h" +#include "linux/console.h" +#include "linux/gfp_types.h" +#include "linux/signal.h" +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include "trace_hv.h" + +struct static_key_false __kvmhv_is_nestedv2 __read_mostly; +EXPORT_SYMBOL_GPL(__kvmhv_is_nestedv2); + + +static size_t +gs_msg_ops_kvmhv_nestedv2_config_get_size(struct kvmppc_gs_msg *gsm) +{ + u16 ids[] = { + KVMPPC_GSID_RUN_OUTPUT_MIN_SIZE, + KVMPPC_GSID_RUN_INPUT, + KVMPPC_GSID_RUN_OUTPUT, + + }; + size_t size = 0; + + for (int i = 0; i < ARRAY_SIZE(ids); i++) + size += kvmppc_gse_total_size(kvmppc_gsid_size(ids[i])); + return size; +} + +static int +gs_msg_ops_kvmhv_nestedv2_config_fill_info(struct kvmppc_gs_buff *gsb, + struct kvmppc_gs_msg *gsm) +{ + struct kvmhv_nestedv2_config *cfg; + int rc; + + cfg = gsm->data; + + if (kvmppc_gsm_includes(gsm, KVMPPC_GSID_RUN_OUTPUT_MIN_SIZE)) { + rc = kvmppc_gse_put_u64(gsb, KVMPPC_GSID_RUN_OUTPUT_MIN_SIZE, + cfg->vcpu_run_output_size); + if (rc < 0) + return rc; + } + + if (kvmppc_gsm_includes(gsm, KVMPPC_GSID_RUN_INPUT)) { + rc = kvmppc_gse_put_buff_info(gsb, KVMPPC_GSID_RUN_INPUT, + cfg->vcpu_run_input_cfg); + if (rc < 0) + return rc; + } + + if (kvmppc_gsm_includes(gsm, KVMPPC_GSID_RUN_OUTPUT)) { + kvmppc_gse_put_buff_info(gsb, KVMPPC_GSID_RUN_OUTPUT, + cfg->vcpu_run_output_cfg); + if (rc < 0) + return rc; + } + + return 0; +} + +static int +gs_msg_ops_kvmhv_nestedv2_config_refresh_info(struct kvmppc_gs_msg *gsm, + struct kvmppc_gs_buff *gsb) +{ + struct kvmhv_nestedv2_config *cfg; + struct kvmppc_gs_parser gsp = { 0 }; + struct kvmppc_gs_elem *gse; + int rc; + + cfg = gsm->data; + + rc = kvmppc_gse_parse(&gsp, gsb); + if (rc < 0) + return rc; + + gse = kvmppc_gsp_lookup(&gsp, KVMPPC_GSID_RUN_OUTPUT_MIN_SIZE); + if (gse) + cfg->vcpu_run_output_size = kvmppc_gse_get_u64(gse); + return 0; +} + +static struct kvmppc_gs_msg_ops config_msg_ops = { + .get_size = gs_msg_ops_kvmhv_nestedv2_config_get_size, + .fill_info = gs_msg_ops_kvmhv_nestedv2_config_fill_info, + .refresh_info = gs_msg_ops_kvmhv_nestedv2_config_refresh_info, +}; + +static size_t gs_msg_ops_vcpu_get_size(struct kvmppc_gs_msg *gsm) +{ + struct kvmppc_gs_bitmap gsbm = { 0 }; + size_t size = 0; + u16 iden; + + kvmppc_gsbm_fill(&gsbm); + kvmppc_gsbm_for_each(&gsbm, iden) + { + switch (iden) { + case KVMPPC_GSID_HOST_STATE_SIZE: + case KVMPPC_GSID_RUN_OUTPUT_MIN_SIZE: + case KVMPPC_GSID_PARTITION_TABLE: + case KVMPPC_GSID_PROCESS_TABLE: + case KVMPPC_GSID_RUN_INPUT: + case KVMPPC_GSID_RUN_OUTPUT: + break; + default: + size += kvmppc_gse_total_size(kvmppc_gsid_size(iden)); + } + } + return size; +} + +static int gs_msg_ops_vcpu_fill_info(struct kvmppc_gs_buff *gsb, + struct kvmppc_gs_msg *gsm) +{ + struct kvm_vcpu *vcpu; + vector128 v; + int rc, i; + u16 iden; + + vcpu = gsm->data; + + kvmppc_gsm_for_each(gsm, iden) + { + rc = 0; + + if ((gsm->flags & KVMPPC_GS_FLAGS_WIDE) != + (kvmppc_gsid_flags(iden) & KVMPPC_GS_FLAGS_WIDE)) + continue; + + switch (iden) { + case KVMPPC_GSID_DSCR: + rc = kvmppc_gse_put_u64(gsb, iden, vcpu->arch.dscr); + break; + case KVMPPC_GSID_MMCRA: + rc = kvmppc_gse_put_u64(gsb, iden, vcpu->arch.mmcra); + break; + case KVMPPC_GSID_HFSCR: + rc = kvmppc_gse_put_u64(gsb, iden, vcpu->arch.hfscr); + break; + case KVMPPC_GSID_PURR: + rc = kvmppc_gse_put_u64(gsb, iden, vcpu->arch.purr); + break; + case KVMPPC_GSID_SPURR: + rc = kvmppc_gse_put_u64(gsb, iden, vcpu->arch.spurr); + break; + case KVMPPC_GSID_AMR: + rc = kvmppc_gse_put_u64(gsb, iden, vcpu->arch.amr); + break; + case KVMPPC_GSID_UAMOR: + rc = kvmppc_gse_put_u64(gsb, iden, vcpu->arch.uamor); + break; + case KVMPPC_GSID_SIAR: + rc = kvmppc_gse_put_u64(gsb, iden, vcpu->arch.siar); + break; + case KVMPPC_GSID_SDAR: + rc = kvmppc_gse_put_u64(gsb, iden, vcpu->arch.sdar); + break; + case KVMPPC_GSID_IAMR: + rc = kvmppc_gse_put_u64(gsb, iden, vcpu->arch.iamr); + break; + case KVMPPC_GSID_DAWR0: + rc = kvmppc_gse_put_u64(gsb, iden, vcpu->arch.dawr0); + break; + case KVMPPC_GSID_DAWR1: + rc = kvmppc_gse_put_u64(gsb, iden, vcpu->arch.dawr1); + break; + case KVMPPC_GSID_DAWRX0: + rc = kvmppc_gse_put_u32(gsb, iden, vcpu->arch.dawrx0); + break; + case KVMPPC_GSID_DAWRX1: + rc = kvmppc_gse_put_u32(gsb, iden, vcpu->arch.dawrx1); + break; + case KVMPPC_GSID_CIABR: + rc = kvmppc_gse_put_u64(gsb, iden, vcpu->arch.ciabr); + break; + case KVMPPC_GSID_WORT: + rc = kvmppc_gse_put_u32(gsb, iden, vcpu->arch.wort); + break; + case KVMPPC_GSID_PPR: + rc = kvmppc_gse_put_u64(gsb, iden, vcpu->arch.ppr); + break; + case KVMPPC_GSID_PSPB: + rc = kvmppc_gse_put_u32(gsb, iden, vcpu->arch.pspb); + break; + case KVMPPC_GSID_TAR: + rc = kvmppc_gse_put_u64(gsb, iden, vcpu->arch.tar); + break; + case KVMPPC_GSID_FSCR: + rc = kvmppc_gse_put_u64(gsb, iden, vcpu->arch.fscr); + break; + case KVMPPC_GSID_EBBHR: + rc = kvmppc_gse_put_u64(gsb, iden, vcpu->arch.ebbhr); + break; + case KVMPPC_GSID_EBBRR: + rc = kvmppc_gse_put_u64(gsb, iden, vcpu->arch.ebbrr); + break; + case KVMPPC_GSID_BESCR: + rc = kvmppc_gse_put_u64(gsb, iden, vcpu->arch.bescr); + break; + case KVMPPC_GSID_IC: + rc = kvmppc_gse_put_u64(gsb, iden, vcpu->arch.ic); + break; + case KVMPPC_GSID_CTRL: + rc = kvmppc_gse_put_u64(gsb, iden, vcpu->arch.ctrl); + break; + case KVMPPC_GSID_PIDR: + rc = kvmppc_gse_put_u32(gsb, iden, vcpu->arch.pid); + break; + case KVMPPC_GSID_AMOR: { + u64 amor = ~0; + + rc = kvmppc_gse_put_u64(gsb, iden, amor); + break; + } + case KVMPPC_GSID_VRSAVE: + rc = kvmppc_gse_put_u32(gsb, iden, vcpu->arch.vrsave); + break; + case KVMPPC_GSID_MMCR(0)... KVMPPC_GSID_MMCR(3): + i = iden - KVMPPC_GSID_MMCR(0); + rc = kvmppc_gse_put_u64(gsb, iden, vcpu->arch.mmcr[i]); + break; + case KVMPPC_GSID_SIER(0)... KVMPPC_GSID_SIER(2): + i = iden - KVMPPC_GSID_SIER(0); + rc = kvmppc_gse_put_u64(gsb, iden, vcpu->arch.sier[i]); + break; + case KVMPPC_GSID_PMC(0)... KVMPPC_GSID_PMC(5): + i = iden - KVMPPC_GSID_PMC(0); + rc = kvmppc_gse_put_u32(gsb, iden, vcpu->arch.pmc[i]); + break; + case KVMPPC_GSID_GPR(0)... KVMPPC_GSID_GPR(31): + i = iden - KVMPPC_GSID_GPR(0); + rc = kvmppc_gse_put_u64(gsb, iden, + vcpu->arch.regs.gpr[i]); + break; + case KVMPPC_GSID_CR: + rc = kvmppc_gse_put_u32(gsb, iden, vcpu->arch.regs.ccr); + break; + case KVMPPC_GSID_XER: + rc = kvmppc_gse_put_u64(gsb, iden, vcpu->arch.regs.xer); + break; + case KVMPPC_GSID_CTR: + rc = kvmppc_gse_put_u64(gsb, iden, vcpu->arch.regs.ctr); + break; + case KVMPPC_GSID_LR: + rc = kvmppc_gse_put_u64(gsb, iden, + vcpu->arch.regs.link); + break; + case KVMPPC_GSID_NIA: + rc = kvmppc_gse_put_u64(gsb, iden, vcpu->arch.regs.nip); + break; + case KVMPPC_GSID_SRR0: + rc = kvmppc_gse_put_u64(gsb, iden, + vcpu->arch.shregs.srr0); + break; + case KVMPPC_GSID_SRR1: + rc = kvmppc_gse_put_u64(gsb, iden, + vcpu->arch.shregs.srr1); + break; + case KVMPPC_GSID_SPRG0: + rc = kvmppc_gse_put_u64(gsb, iden, + vcpu->arch.shregs.sprg0); + break; + case KVMPPC_GSID_SPRG1: + rc = kvmppc_gse_put_u64(gsb, iden, + vcpu->arch.shregs.sprg1); + break; + case KVMPPC_GSID_SPRG2: + rc = kvmppc_gse_put_u64(gsb, iden, + vcpu->arch.shregs.sprg2); + break; + case KVMPPC_GSID_SPRG3: + rc = kvmppc_gse_put_u64(gsb, iden, + vcpu->arch.shregs.sprg3); + break; + case KVMPPC_GSID_DAR: + rc = kvmppc_gse_put_u64(gsb, iden, + vcpu->arch.shregs.dar); + break; + case KVMPPC_GSID_DSISR: + rc = kvmppc_gse_put_u32(gsb, iden, + vcpu->arch.shregs.dsisr); + break; + case KVMPPC_GSID_MSR: + rc = kvmppc_gse_put_u64(gsb, iden, + vcpu->arch.shregs.msr); + break; + case KVMPPC_GSID_VTB: + rc = kvmppc_gse_put_u64(gsb, iden, + vcpu->arch.vcore->vtb); + break; + case KVMPPC_GSID_LPCR: + rc = kvmppc_gse_put_u64(gsb, iden, + vcpu->arch.vcore->lpcr); + break; + case KVMPPC_GSID_TB_OFFSET: + rc = kvmppc_gse_put_u64(gsb, iden, + vcpu->arch.vcore->tb_offset); + break; + case KVMPPC_GSID_FPSCR: + rc = kvmppc_gse_put_u64(gsb, iden, vcpu->arch.fp.fpscr); + break; + case KVMPPC_GSID_VSRS(0)... KVMPPC_GSID_VSRS(31): + i = iden - KVMPPC_GSID_VSRS(0); + memcpy(&v, &vcpu->arch.fp.fpr[i], + sizeof(vcpu->arch.fp.fpr[i])); + rc = kvmppc_gse_put_vector128(gsb, iden, &v); + break; +#ifdef CONFIG_VSX + case KVMPPC_GSID_VSCR: + rc = kvmppc_gse_put_u32(gsb, iden, + vcpu->arch.vr.vscr.u[3]); + break; + case KVMPPC_GSID_VSRS(32)... KVMPPC_GSID_VSRS(63): + i = iden - KVMPPC_GSID_VSRS(32); + rc = kvmppc_gse_put_vector128(gsb, iden, + &vcpu->arch.vr.vr[i]); + break; +#endif + case KVMPPC_GSID_DEC_EXPIRY_TB: { + u64 dw; + + dw = vcpu->arch.dec_expires - + vcpu->arch.vcore->tb_offset; + rc = kvmppc_gse_put_u64(gsb, iden, dw); + break; + } + case KVMPPC_GSID_LOGICAL_PVR: + rc = kvmppc_gse_put_u32(gsb, iden, + vcpu->arch.vcore->arch_compat); + break; + } + + if (rc < 0) + return rc; + } + + return 0; +} + +static int gs_msg_ops_vcpu_refresh_info(struct kvmppc_gs_msg *gsm, + struct kvmppc_gs_buff *gsb) +{ + struct kvmppc_gs_parser gsp = { 0 }; + struct kvmhv_nestedv2_io *io; + struct kvmppc_gs_bitmap *valids; + struct kvm_vcpu *vcpu; + struct kvmppc_gs_elem *gse; + vector128 v; + int rc, i; + u16 iden; + + vcpu = gsm->data; + + rc = kvmppc_gse_parse(&gsp, gsb); + if (rc < 0) + return rc; + + io = &vcpu->arch.nestedv2_io; + valids = &io->valids; + + kvmppc_gsp_for_each(&gsp, iden, gse) + { + switch (iden) { + case KVMPPC_GSID_DSCR: + vcpu->arch.dscr = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_MMCRA: + vcpu->arch.mmcra = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_HFSCR: + vcpu->arch.hfscr = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_PURR: + vcpu->arch.purr = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_SPURR: + vcpu->arch.spurr = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_AMR: + vcpu->arch.amr = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_UAMOR: + vcpu->arch.uamor = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_SIAR: + vcpu->arch.siar = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_SDAR: + vcpu->arch.sdar = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_IAMR: + vcpu->arch.iamr = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_DAWR0: + vcpu->arch.dawr0 = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_DAWR1: + vcpu->arch.dawr1 = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_DAWRX0: + vcpu->arch.dawrx0 = kvmppc_gse_get_u32(gse); + break; + case KVMPPC_GSID_DAWRX1: + vcpu->arch.dawrx1 = kvmppc_gse_get_u32(gse); + break; + case KVMPPC_GSID_CIABR: + vcpu->arch.ciabr = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_WORT: + vcpu->arch.wort = kvmppc_gse_get_u32(gse); + break; + case KVMPPC_GSID_PPR: + vcpu->arch.ppr = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_PSPB: + vcpu->arch.pspb = kvmppc_gse_get_u32(gse); + break; + case KVMPPC_GSID_TAR: + vcpu->arch.tar = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_FSCR: + vcpu->arch.fscr = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_EBBHR: + vcpu->arch.ebbhr = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_EBBRR: + vcpu->arch.ebbrr = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_BESCR: + vcpu->arch.bescr = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_IC: + vcpu->arch.ic = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_CTRL: + vcpu->arch.ctrl = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_PIDR: + vcpu->arch.pid = kvmppc_gse_get_u32(gse); + break; + case KVMPPC_GSID_AMOR: + break; + case KVMPPC_GSID_VRSAVE: + vcpu->arch.vrsave = kvmppc_gse_get_u32(gse); + break; + case KVMPPC_GSID_MMCR(0)... KVMPPC_GSID_MMCR(3): + i = iden - KVMPPC_GSID_MMCR(0); + vcpu->arch.mmcr[i] = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_SIER(0)... KVMPPC_GSID_SIER(2): + i = iden - KVMPPC_GSID_SIER(0); + vcpu->arch.sier[i] = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_PMC(0)... KVMPPC_GSID_PMC(5): + i = iden - KVMPPC_GSID_PMC(0); + vcpu->arch.pmc[i] = kvmppc_gse_get_u32(gse); + break; + case KVMPPC_GSID_GPR(0)... KVMPPC_GSID_GPR(31): + i = iden - KVMPPC_GSID_GPR(0); + vcpu->arch.regs.gpr[i] = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_CR: + vcpu->arch.regs.ccr = kvmppc_gse_get_u32(gse); + break; + case KVMPPC_GSID_XER: + vcpu->arch.regs.xer = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_CTR: + vcpu->arch.regs.ctr = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_LR: + vcpu->arch.regs.link = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_NIA: + vcpu->arch.regs.nip = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_SRR0: + vcpu->arch.shregs.srr0 = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_SRR1: + vcpu->arch.shregs.srr1 = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_SPRG0: + vcpu->arch.shregs.sprg0 = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_SPRG1: + vcpu->arch.shregs.sprg1 = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_SPRG2: + vcpu->arch.shregs.sprg2 = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_SPRG3: + vcpu->arch.shregs.sprg3 = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_DAR: + vcpu->arch.shregs.dar = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_DSISR: + vcpu->arch.shregs.dsisr = kvmppc_gse_get_u32(gse); + break; + case KVMPPC_GSID_MSR: + vcpu->arch.shregs.msr = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_VTB: + vcpu->arch.vcore->vtb = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_LPCR: + vcpu->arch.vcore->lpcr = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_TB_OFFSET: + vcpu->arch.vcore->tb_offset = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_FPSCR: + vcpu->arch.fp.fpscr = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_VSRS(0)... KVMPPC_GSID_VSRS(31): + kvmppc_gse_get_vector128(gse, &v); + i = iden - KVMPPC_GSID_VSRS(0); + memcpy(&vcpu->arch.fp.fpr[i], &v, + sizeof(vcpu->arch.fp.fpr[i])); + break; +#ifdef CONFIG_VSX + case KVMPPC_GSID_VSCR: + vcpu->arch.vr.vscr.u[3] = kvmppc_gse_get_u32(gse); + break; + case KVMPPC_GSID_VSRS(32)... KVMPPC_GSID_VSRS(63): + i = iden - KVMPPC_GSID_VSRS(32); + kvmppc_gse_get_vector128(gse, &vcpu->arch.vr.vr[i]); + break; +#endif + case KVMPPC_GSID_HDAR: + vcpu->arch.fault_dar = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_HDSISR: + vcpu->arch.fault_dsisr = kvmppc_gse_get_u32(gse); + break; + case KVMPPC_GSID_ASDR: + vcpu->arch.fault_gpa = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_HEIR: + vcpu->arch.emul_inst = kvmppc_gse_get_u64(gse); + break; + case KVMPPC_GSID_DEC_EXPIRY_TB: { + u64 dw; + + dw = kvmppc_gse_get_u64(gse); + vcpu->arch.dec_expires = + dw + vcpu->arch.vcore->tb_offset; + break; + } + case KVMPPC_GSID_LOGICAL_PVR: + vcpu->arch.vcore->arch_compat = kvmppc_gse_get_u32(gse); + break; + default: + continue; + } + kvmppc_gsbm_set(valids, iden); + } + + return 0; +} + +static struct kvmppc_gs_msg_ops vcpu_message_ops = { + .get_size = gs_msg_ops_vcpu_get_size, + .fill_info = gs_msg_ops_vcpu_fill_info, + .refresh_info = gs_msg_ops_vcpu_refresh_info, +}; + +static int kvmhv_nestedv2_host_create(struct kvm_vcpu *vcpu, + struct kvmhv_nestedv2_io *io) +{ + struct kvmhv_nestedv2_config *cfg; + struct kvmppc_gs_buff *gsb, *vcpu_run_output, *vcpu_run_input; + unsigned long guest_id, vcpu_id; + struct kvmppc_gs_msg *gsm, *vcpu_message, *vcore_message; + int rc; + + cfg = &io->cfg; + guest_id = vcpu->kvm->arch.lpid; + vcpu_id = vcpu->vcpu_id; + + gsm = kvmppc_gsm_new(&config_msg_ops, cfg, KVMPPC_GS_FLAGS_WIDE, + GFP_KERNEL); + if (!gsm) { + rc = -ENOMEM; + goto err; + } + + gsb = kvmppc_gsb_new(kvmppc_gsm_size(gsm), guest_id, vcpu_id, + GFP_KERNEL); + if (!gsb) { + rc = -ENOMEM; + goto free_gsm; + } + + rc = kvmppc_gsb_receive_datum(gsb, gsm, + KVMPPC_GSID_RUN_OUTPUT_MIN_SIZE); + if (rc < 0) { + pr_err("KVM-NESTEDv2: couldn't get vcpu run output buffer minimum size\n"); + goto free_gsb; + } + + vcpu_run_output = kvmppc_gsb_new(cfg->vcpu_run_output_size, guest_id, + vcpu_id, GFP_KERNEL); + if (!vcpu_run_output) { + rc = -ENOMEM; + goto free_gsb; + } + + cfg->vcpu_run_output_cfg.address = kvmppc_gsb_paddress(vcpu_run_output); + cfg->vcpu_run_output_cfg.size = kvmppc_gsb_capacity(vcpu_run_output); + io->vcpu_run_output = vcpu_run_output; + + gsm->flags = 0; + rc = kvmppc_gsb_send_datum(gsb, gsm, KVMPPC_GSID_RUN_OUTPUT); + if (rc < 0) { + pr_err("KVM-NESTEDv2: couldn't set vcpu run output buffer\n"); + goto free_gs_out; + } + + vcpu_message = kvmppc_gsm_new(&vcpu_message_ops, vcpu, 0, GFP_KERNEL); + if (!vcpu_message) { + rc = -ENOMEM; + goto free_gs_out; + } + kvmppc_gsm_include_all(vcpu_message); + + io->vcpu_message = vcpu_message; + + vcpu_run_input = kvmppc_gsb_new(kvmppc_gsm_size(vcpu_message), guest_id, + vcpu_id, GFP_KERNEL); + if (!vcpu_run_input) { + rc = -ENOMEM; + goto free_vcpu_message; + } + + io->vcpu_run_input = vcpu_run_input; + cfg->vcpu_run_input_cfg.address = kvmppc_gsb_paddress(vcpu_run_input); + cfg->vcpu_run_input_cfg.size = kvmppc_gsb_capacity(vcpu_run_input); + rc = kvmppc_gsb_send_datum(gsb, gsm, KVMPPC_GSID_RUN_INPUT); + if (rc < 0) { + pr_err("KVM-NESTEDv2: couldn't set vcpu run input buffer\n"); + goto free_vcpu_run_input; + } + + vcore_message = kvmppc_gsm_new(&vcpu_message_ops, vcpu, + KVMPPC_GS_FLAGS_WIDE, GFP_KERNEL); + if (!vcore_message) { + rc = -ENOMEM; + goto free_vcpu_run_input; + } + + kvmppc_gsm_include_all(vcore_message); + kvmppc_gsbm_clear(&vcore_message->bitmap, KVMPPC_GSID_LOGICAL_PVR); + io->vcore_message = vcore_message; + + kvmppc_gsbm_fill(&io->valids); + kvmppc_gsm_free(gsm); + kvmppc_gsb_free(gsb); + return 0; + +free_vcpu_run_input: + kvmppc_gsb_free(vcpu_run_input); +free_vcpu_message: + kvmppc_gsm_free(vcpu_message); +free_gs_out: + kvmppc_gsb_free(vcpu_run_output); +free_gsb: + kvmppc_gsb_free(gsb); +free_gsm: + kvmppc_gsm_free(gsm); +err: + return rc; +} + +/** + * __kvmhv_nestedv2_mark_dirty() - mark a Guest State ID to be sent to the host + * @vcpu: vcpu + * @iden: guest state ID + * + * Mark a guest state ID as having been changed by the L1 host and thus + * the new value must be sent to the L0 hypervisor. See kvmhv_nestedv2_flush_vcpu() + */ +int __kvmhv_nestedv2_mark_dirty(struct kvm_vcpu *vcpu, u16 iden) +{ + struct kvmhv_nestedv2_io *io; + struct kvmppc_gs_bitmap *valids; + struct kvmppc_gs_msg *gsm; + + if (!iden) + return 0; + + io = &vcpu->arch.nestedv2_io; + valids = &io->valids; + gsm = io->vcpu_message; + kvmppc_gsm_include(gsm, iden); + gsm = io->vcore_message; + kvmppc_gsm_include(gsm, iden); + kvmppc_gsbm_set(valids, iden); + return 0; +} +EXPORT_SYMBOL_GPL(__kvmhv_nestedv2_mark_dirty); + +/** + * __kvmhv_nestedv2_cached_reload() - reload a Guest State ID from the host + * @vcpu: vcpu + * @iden: guest state ID + * + * Reload the value for the guest state ID from the L0 host into the L1 host. + * This is cached so that going out to the L0 host only happens if necessary. + */ +int __kvmhv_nestedv2_cached_reload(struct kvm_vcpu *vcpu, u16 iden) +{ + struct kvmhv_nestedv2_io *io; + struct kvmppc_gs_bitmap *valids; + struct kvmppc_gs_buff *gsb; + struct kvmppc_gs_msg gsm; + int rc; + + if (!iden) + return 0; + + io = &vcpu->arch.nestedv2_io; + valids = &io->valids; + if (kvmppc_gsbm_test(valids, iden)) + return 0; + + gsb = io->vcpu_run_input; + kvmppc_gsm_init(&gsm, &vcpu_message_ops, vcpu, kvmppc_gsid_flags(iden)); + rc = kvmppc_gsb_receive_datum(gsb, &gsm, iden); + if (rc < 0) { + pr_err("KVM-NESTEDv2: couldn't get GSID: 0x%x\n", iden); + return rc; + } + return 0; +} +EXPORT_SYMBOL_GPL(__kvmhv_nestedv2_cached_reload); + +/** + * kvmhv_nestedv2_flush_vcpu() - send modified Guest State IDs to the host + * @vcpu: vcpu + * @time_limit: hdec expiry tb + * + * Send the values marked by __kvmhv_nestedv2_mark_dirty() to the L0 host. + * Thread wide values are copied to the H_GUEST_RUN_VCPU input buffer. Guest + * wide values need to be sent with H_GUEST_SET first. + * + * The hdec tb offset is always sent to L0 host. + */ +int kvmhv_nestedv2_flush_vcpu(struct kvm_vcpu *vcpu, u64 time_limit) +{ + struct kvmhv_nestedv2_io *io; + struct kvmppc_gs_buff *gsb; + struct kvmppc_gs_msg *gsm; + int rc; + + io = &vcpu->arch.nestedv2_io; + gsb = io->vcpu_run_input; + gsm = io->vcore_message; + rc = kvmppc_gsb_send_data(gsb, gsm); + if (rc < 0) { + pr_err("KVM-NESTEDv2: couldn't set guest wide elements\n"); + return rc; + } + + gsm = io->vcpu_message; + kvmppc_gsb_reset(gsb); + rc = kvmppc_gsm_fill_info(gsm, gsb); + if (rc < 0) { + pr_err("KVM-NESTEDv2: couldn't fill vcpu run input buffer\n"); + return rc; + } + + rc = kvmppc_gse_put_u64(gsb, KVMPPC_GSID_HDEC_EXPIRY_TB, time_limit); + if (rc < 0) + return rc; + return 0; +} +EXPORT_SYMBOL_GPL(kvmhv_nestedv2_flush_vcpu); + +/** + * kvmhv_nestedv2_set_ptbl_entry() - send partition and process table state to + * L0 host + * @lpid: guest id + * @dw0: partition table double word + * @dw1: process table double word + */ +int kvmhv_nestedv2_set_ptbl_entry(unsigned long lpid, u64 dw0, u64 dw1) +{ + struct kvmppc_gs_part_table patbl; + struct kvmppc_gs_proc_table prtbl; + struct kvmppc_gs_buff *gsb; + size_t size; + int rc; + + size = kvmppc_gse_total_size( + kvmppc_gsid_size(KVMPPC_GSID_PARTITION_TABLE)) + + kvmppc_gse_total_size( + kvmppc_gsid_size(KVMPPC_GSID_PROCESS_TABLE)) + + sizeof(struct kvmppc_gs_header); + gsb = kvmppc_gsb_new(size, lpid, 0, GFP_KERNEL); + if (!gsb) + return -ENOMEM; + + patbl.address = dw0 & RPDB_MASK; + patbl.ea_bits = ((((dw0 & RTS1_MASK) >> (RTS1_SHIFT - 3)) | + ((dw0 & RTS2_MASK) >> RTS2_SHIFT)) + + 31); + patbl.gpd_size = 1ul << ((dw0 & RPDS_MASK) + 3); + rc = kvmppc_gse_put_part_table(gsb, KVMPPC_GSID_PARTITION_TABLE, patbl); + if (rc < 0) + goto free_gsb; + + prtbl.address = dw1 & PRTB_MASK; + prtbl.gpd_size = 1ul << ((dw1 & PRTS_MASK) + 12); + rc = kvmppc_gse_put_proc_table(gsb, KVMPPC_GSID_PROCESS_TABLE, prtbl); + if (rc < 0) + goto free_gsb; + + rc = kvmppc_gsb_send(gsb, KVMPPC_GS_FLAGS_WIDE); + if (rc < 0) { + pr_err("KVM-NESTEDv2: couldn't set the PATE\n"); + goto free_gsb; + } + + kvmppc_gsb_free(gsb); + return 0; + +free_gsb: + kvmppc_gsb_free(gsb); + return rc; +} +EXPORT_SYMBOL_GPL(kvmhv_nestedv2_set_ptbl_entry); + +/** + * kvmhv_nestedv2_parse_output() - receive values from H_GUEST_RUN_VCPU output + * @vcpu: vcpu + * + * Parse the output buffer from H_GUEST_RUN_VCPU to update vcpu. + */ +int kvmhv_nestedv2_parse_output(struct kvm_vcpu *vcpu) +{ + struct kvmhv_nestedv2_io *io; + struct kvmppc_gs_buff *gsb; + struct kvmppc_gs_msg gsm; + + io = &vcpu->arch.nestedv2_io; + gsb = io->vcpu_run_output; + + vcpu->arch.fault_dar = 0; + vcpu->arch.fault_dsisr = 0; + vcpu->arch.fault_gpa = 0; + vcpu->arch.emul_inst = KVM_INST_FETCH_FAILED; + + kvmppc_gsm_init(&gsm, &vcpu_message_ops, vcpu, 0); + return kvmppc_gsm_refresh_info(&gsm, gsb); +} +EXPORT_SYMBOL_GPL(kvmhv_nestedv2_parse_output); + +static void kvmhv_nestedv2_host_free(struct kvm_vcpu *vcpu, + struct kvmhv_nestedv2_io *io) +{ + kvmppc_gsm_free(io->vcpu_message); + kvmppc_gsm_free(io->vcore_message); + kvmppc_gsb_free(io->vcpu_run_input); + kvmppc_gsb_free(io->vcpu_run_output); +} + +int __kvmhv_nestedv2_reload_ptregs(struct kvm_vcpu *vcpu, struct pt_regs *regs) +{ + struct kvmhv_nestedv2_io *io; + struct kvmppc_gs_bitmap *valids; + struct kvmppc_gs_buff *gsb; + struct kvmppc_gs_msg gsm; + int rc = 0; + + + io = &vcpu->arch.nestedv2_io; + valids = &io->valids; + + gsb = io->vcpu_run_input; + kvmppc_gsm_init(&gsm, &vcpu_message_ops, vcpu, 0); + + for (int i = 0; i < 32; i++) { + if (!kvmppc_gsbm_test(valids, KVMPPC_GSID_GPR(i))) + kvmppc_gsm_include(&gsm, KVMPPC_GSID_GPR(i)); + } + + if (!kvmppc_gsbm_test(valids, KVMPPC_GSID_CR)) + kvmppc_gsm_include(&gsm, KVMPPC_GSID_CR); + + if (!kvmppc_gsbm_test(valids, KVMPPC_GSID_XER)) + kvmppc_gsm_include(&gsm, KVMPPC_GSID_XER); + + if (!kvmppc_gsbm_test(valids, KVMPPC_GSID_CTR)) + kvmppc_gsm_include(&gsm, KVMPPC_GSID_CTR); + + if (!kvmppc_gsbm_test(valids, KVMPPC_GSID_LR)) + kvmppc_gsm_include(&gsm, KVMPPC_GSID_LR); + + if (!kvmppc_gsbm_test(valids, KVMPPC_GSID_NIA)) + kvmppc_gsm_include(&gsm, KVMPPC_GSID_NIA); + + rc = kvmppc_gsb_receive_data(gsb, &gsm); + if (rc < 0) + pr_err("KVM-NESTEDv2: couldn't reload ptregs\n"); + + return rc; +} +EXPORT_SYMBOL_GPL(__kvmhv_nestedv2_reload_ptregs); + +int __kvmhv_nestedv2_mark_dirty_ptregs(struct kvm_vcpu *vcpu, + struct pt_regs *regs) +{ + for (int i = 0; i < 32; i++) + kvmhv_nestedv2_mark_dirty(vcpu, KVMPPC_GSID_GPR(i)); + + kvmhv_nestedv2_mark_dirty(vcpu, KVMPPC_GSID_CR); + kvmhv_nestedv2_mark_dirty(vcpu, KVMPPC_GSID_XER); + kvmhv_nestedv2_mark_dirty(vcpu, KVMPPC_GSID_CTR); + kvmhv_nestedv2_mark_dirty(vcpu, KVMPPC_GSID_LR); + kvmhv_nestedv2_mark_dirty(vcpu, KVMPPC_GSID_NIA); + + return 0; +} +EXPORT_SYMBOL_GPL(__kvmhv_nestedv2_mark_dirty_ptregs); + +/** + * kvmhv_nestedv2_vcpu_create() - create nested vcpu for the NESTEDv2 API + * @vcpu: vcpu + * @io: NESTEDv2 nested io state + * + * Parse the output buffer from H_GUEST_RUN_VCPU to update vcpu. + */ +int kvmhv_nestedv2_vcpu_create(struct kvm_vcpu *vcpu, + struct kvmhv_nestedv2_io *io) +{ + long rc; + + rc = plpar_guest_create_vcpu(0, vcpu->kvm->arch.lpid, vcpu->vcpu_id); + + if (rc != H_SUCCESS) { + pr_err("KVM: Create Guest vcpu hcall failed, rc=%ld\n", rc); + switch (rc) { + case H_NOT_ENOUGH_RESOURCES: + case H_ABORTED: + return -ENOMEM; + case H_AUTHORITY: + return -EPERM; + default: + return -EINVAL; + } + } + + rc = kvmhv_nestedv2_host_create(vcpu, io); + + return rc; +} +EXPORT_SYMBOL_GPL(kvmhv_nestedv2_vcpu_create); + +/** + * kvmhv_nestedv2_vcpu_free() - free the NESTEDv2 state + * @vcpu: vcpu + * @io: NESTEDv2 nested io state + */ +void kvmhv_nestedv2_vcpu_free(struct kvm_vcpu *vcpu, + struct kvmhv_nestedv2_io *io) +{ + kvmhv_nestedv2_host_free(vcpu, io); +} +EXPORT_SYMBOL_GPL(kvmhv_nestedv2_vcpu_free); diff --git a/arch/powerpc/kvm/emulate_loadstore.c b/arch/powerpc/kvm/emulate_loadstore.c index e6e66c3792f8..077fd88a0b68 100644 --- a/arch/powerpc/kvm/emulate_loadstore.c +++ b/arch/powerpc/kvm/emulate_loadstore.c @@ -92,7 +92,8 @@ int kvmppc_emulate_loadstore(struct kvm_vcpu *vcpu) vcpu->arch.mmio_host_swabbed = 0; emulated = EMULATE_FAIL; - vcpu->arch.regs.msr = vcpu->arch.shared->msr; + vcpu->arch.regs.msr = kvmppc_get_msr(vcpu); + kvmhv_nestedv2_reload_ptregs(vcpu, &vcpu->arch.regs); if (analyse_instr(&op, &vcpu->arch.regs, inst) == 0) { int type = op.type & INSTR_TYPE_MASK; int size = GETSIZE(op.type); @@ -357,6 +358,7 @@ int kvmppc_emulate_loadstore(struct kvm_vcpu *vcpu) } trace_kvm_ppc_instr(ppc_inst_val(inst), kvmppc_get_pc(vcpu), emulated); + kvmhv_nestedv2_mark_dirty_ptregs(vcpu, &vcpu->arch.regs); /* Advance past emulated instruction. */ if (emulated != EMULATE_FAIL) diff --git a/arch/powerpc/kvm/guest-state-buffer.c b/arch/powerpc/kvm/guest-state-buffer.c index 2b22a2f8c67e..b80dbc58621f 100644 --- a/arch/powerpc/kvm/guest-state-buffer.c +++ b/arch/powerpc/kvm/guest-state-buffer.c @@ -569,3 +569,53 @@ int kvmppc_gsm_refresh_info(struct kvmppc_gs_msg *gsm, return gsm->ops->refresh_info(gsm, gsb); } EXPORT_SYMBOL_GPL(kvmppc_gsm_refresh_info); + +/** + * kvmppc_gsb_send - send all elements in the buffer to the hypervisor. + * @gsb: guest state buffer + * @flags: guest wide or thread wide + * + * Performs the H_GUEST_SET_STATE hcall for the guest state buffer. + */ +int kvmppc_gsb_send(struct kvmppc_gs_buff *gsb, unsigned long flags) +{ + unsigned long hflags = 0; + unsigned long i; + int rc; + + if (kvmppc_gsb_nelems(gsb) == 0) + return 0; + + if (flags & KVMPPC_GS_FLAGS_WIDE) + hflags |= H_GUEST_FLAGS_WIDE; + + rc = plpar_guest_set_state(hflags, gsb->guest_id, gsb->vcpu_id, + __pa(gsb->hdr), gsb->capacity, &i); + return rc; +} +EXPORT_SYMBOL_GPL(kvmppc_gsb_send); + +/** + * kvmppc_gsb_recv - request all elements in the buffer have their value + * updated. + * @gsb: guest state buffer + * @flags: guest wide or thread wide + * + * Performs the H_GUEST_GET_STATE hcall for the guest state buffer. + * After returning from the hcall the guest state elements that were + * present in the buffer will have updated values from the hypervisor. + */ +int kvmppc_gsb_recv(struct kvmppc_gs_buff *gsb, unsigned long flags) +{ + unsigned long hflags = 0; + unsigned long i; + int rc; + + if (flags & KVMPPC_GS_FLAGS_WIDE) + hflags |= H_GUEST_FLAGS_WIDE; + + rc = plpar_guest_get_state(hflags, gsb->guest_id, gsb->vcpu_id, + __pa(gsb->hdr), gsb->capacity, &i); + return rc; +} +EXPORT_SYMBOL_GPL(kvmppc_gsb_recv); From 476652297f94a2e5e5ef29e734b0da37ade94110 Mon Sep 17 00:00:00 2001 From: Michael Neuling Date: Thu, 14 Sep 2023 13:06:00 +1000 Subject: [PATCH 089/475] docs: powerpc: Document nested KVM on POWER Document support for nested KVM on POWER using the existing API as well as the new PAPR API. This includes the new HCALL interface and how it used by KVM. Signed-off-by: Michael Neuling Signed-off-by: Jordan Niethe Signed-off-by: Michael Ellerman Link: https://msgid.link/20230914030600.16993-12-jniethe5@gmail.com --- Documentation/powerpc/index.rst | 1 + Documentation/powerpc/kvm-nested.rst | 634 +++++++++++++++++++++++++++ 2 files changed, 635 insertions(+) create mode 100644 Documentation/powerpc/kvm-nested.rst diff --git a/Documentation/powerpc/index.rst b/Documentation/powerpc/index.rst index a50834798454..9749f6dc258f 100644 --- a/Documentation/powerpc/index.rst +++ b/Documentation/powerpc/index.rst @@ -26,6 +26,7 @@ powerpc isa-versions kaslr-booke32 mpc52xx + kvm-nested papr_hcalls pci_iov_resource_on_powernv pmu-ebb diff --git a/Documentation/powerpc/kvm-nested.rst b/Documentation/powerpc/kvm-nested.rst new file mode 100644 index 000000000000..630602a8aa00 --- /dev/null +++ b/Documentation/powerpc/kvm-nested.rst @@ -0,0 +1,634 @@ +.. SPDX-License-Identifier: GPL-2.0 + +==================================== +Nested KVM on POWER +==================================== + +Introduction +============ + +This document explains how a guest operating system can act as a +hypervisor and run nested guests through the use of hypercalls, if the +hypervisor has implemented them. The terms L0, L1, and L2 are used to +refer to different software entities. L0 is the hypervisor mode entity +that would normally be called the "host" or "hypervisor". L1 is a +guest virtual machine that is directly run under L0 and is initiated +and controlled by L0. L2 is a guest virtual machine that is initiated +and controlled by L1 acting as a hypervisor. + +Existing API +============ + +Linux/KVM has had support for Nesting as an L0 or L1 since 2018 + +The L0 code was added:: + + commit 8e3f5fc1045dc49fd175b978c5457f5f51e7a2ce + Author: Paul Mackerras + Date: Mon Oct 8 16:31:03 2018 +1100 + KVM: PPC: Book3S HV: Framework and hcall stubs for nested virtualization + +The L1 code was added:: + + commit 360cae313702cdd0b90f82c261a8302fecef030a + Author: Paul Mackerras + Date: Mon Oct 8 16:31:04 2018 +1100 + KVM: PPC: Book3S HV: Nested guest entry via hypercall + +This API works primarily using a single hcall h_enter_nested(). This +call made by the L1 to tell the L0 to start an L2 vCPU with the given +state. The L0 then starts this L2 and runs until an L2 exit condition +is reached. Once the L2 exits, the state of the L2 is given back to +the L1 by the L0. The full L2 vCPU state is always transferred from +and to L1 when the L2 is run. The L0 doesn't keep any state on the L2 +vCPU (except in the short sequence in the L0 on L1 -> L2 entry and L2 +-> L1 exit). + +The only state kept by the L0 is the partition table. The L1 registers +it's partition table using the h_set_partition_table() hcall. All +other state held by the L0 about the L2s is cached state (such as +shadow page tables). + +The L1 may run any L2 or vCPU without first informing the L0. It +simply starts the vCPU using h_enter_nested(). The creation of L2s and +vCPUs is done implicitly whenever h_enter_nested() is called. + +In this document, we call this existing API the v1 API. + +New PAPR API +=============== + +The new PAPR API changes from the v1 API such that the creating L2 and +associated vCPUs is explicit. In this document, we call this the v2 +API. + +h_enter_nested() is replaced with H_GUEST_VCPU_RUN(). Before this can +be called the L1 must explicitly create the L2 using h_guest_create() +and any associated vCPUs() created with h_guest_create_vCPU(). Getting +and setting vCPU state can also be performed using h_guest_{g|s}et +hcall. + +The basic execution flow is for an L1 to create an L2, run it, and +delete it is: + +- L1 and L0 negotiate capabilities with H_GUEST_{G,S}ET_CAPABILITIES() + (normally at L1 boot time). + +- L1 requests the L0 create an L2 with H_GUEST_CREATE() and receives a token + +- L1 requests the L0 create an L2 vCPU with H_GUEST_CREATE_VCPU() + +- L1 and L0 communicate the vCPU state using the H_GUEST_{G,S}ET() hcall + +- L1 requests the L0 runs the vCPU running H_GUEST_VCPU_RUN() hcall + +- L1 deletes L2 with H_GUEST_DELETE() + +More details of the individual hcalls follows: + +HCALL Details +============= + +This documentation is provided to give an overall understating of the +API. It doesn't aim to provide all the details required to implement +an L1 or L0. Latest version of PAPR can be referred to for more details. + +All these HCALLs are made by the L1 to the L0. + +H_GUEST_GET_CAPABILITIES() +-------------------------- + +This is called to get the capabilities of the L0 nested +hypervisor. This includes capabilities such the CPU versions (eg +POWER9, POWER10) that are supported as L2s:: + + H_GUEST_GET_CAPABILITIES(uint64 flags) + + Parameters: + Input: + flags: Reserved + Output: + R3: Return code + R4: Hypervisor Supported Capabilities bitmap 1 + +H_GUEST_SET_CAPABILITIES() +-------------------------- + +This is called to inform the L0 of the capabilities of the L1 +hypervisor. The set of flags passed here are the same as +H_GUEST_GET_CAPABILITIES() + +Typically, GET will be called first and then SET will be called with a +subset of the flags returned from GET. This process allows the L0 and +L1 to negotiate an agreed set of capabilities:: + + H_GUEST_SET_CAPABILITIES(uint64 flags, + uint64 capabilitiesBitmap1) + Parameters: + Input: + flags: Reserved + capabilitiesBitmap1: Only capabilities advertised through + H_GUEST_GET_CAPABILITIES + Output: + R3: Return code + R4: If R3 = H_P2: The number of invalid bitmaps + R5: If R3 = H_P2: The index of first invalid bitmap + +H_GUEST_CREATE() +---------------- + +This is called to create an L2. A unique ID of the L2 created +(similar to an LPID) is returned, which can be used on subsequent HCALLs to +identify the L2:: + + H_GUEST_CREATE(uint64 flags, + uint64 continueToken); + Parameters: + Input: + flags: Reserved + continueToken: Initial call set to -1. Subsequent calls, + after H_Busy or H_LongBusyOrder has been + returned, value that was returned in R4. + Output: + R3: Return code. Notable: + H_Not_Enough_Resources: Unable to create Guest VCPU due to not + enough Hypervisor memory. See H_GUEST_CREATE_GET_STATE(flags = + takeOwnershipOfVcpuState) + R4: If R3 = H_Busy or_H_LongBusyOrder -> continueToken + +H_GUEST_CREATE_VCPU() +--------------------- + +This is called to create a vCPU associated with an L2. The L2 id +(returned from H_GUEST_CREATE()) should be passed it. Also passed in +is a unique (for this L2) vCPUid. This vCPUid is allocated by the +L1:: + + H_GUEST_CREATE_VCPU(uint64 flags, + uint64 guestId, + uint64 vcpuId); + Parameters: + Input: + flags: Reserved + guestId: ID obtained from H_GUEST_CREATE + vcpuId: ID of the vCPU to be created. This must be within the + range of 0 to 2047 + Output: + R3: Return code. Notable: + H_Not_Enough_Resources: Unable to create Guest VCPU due to not + enough Hypervisor memory. See H_GUEST_CREATE_GET_STATE(flags = + takeOwnershipOfVcpuState) + +H_GUEST_GET_STATE() +------------------- + +This is called to get state associated with an L2 (Guest-wide or vCPU specific). +This info is passed via the Guest State Buffer (GSB), a standard format as +explained later in this doc, necessary details below: + +This can get either L2 wide or vcpu specific information. Examples of +L2 wide is the timebase offset or process scoped page table +info. Examples of vCPU specific are GPRs or VSRs. A bit in the flags +parameter specifies if this call is L2 wide or vCPU specific and the +IDs in the GSB must match this. + +The L1 provides a pointer to the GSB as a parameter to this call. Also +provided is the L2 and vCPU IDs associated with the state to set. + +The L1 writes only the IDs and sizes in the GSB. L0 writes the +associated values for each ID in the GSB:: + + H_GUEST_GET_STATE(uint64 flags, + uint64 guestId, + uint64 vcpuId, + uint64 dataBuffer, + uint64 dataBufferSizeInBytes); + Parameters: + Input: + flags: + Bit 0: getGuestWideState: Request state of the Guest instead + of an individual VCPU. + Bit 1: takeOwnershipOfVcpuState Indicate the L1 is taking + over ownership of the VCPU state and that the L0 can free + the storage holding the state. The VCPU state will need to + be returned to the Hypervisor via H_GUEST_SET_STATE prior + to H_GUEST_RUN_VCPU being called for this VCPU. The data + returned in the dataBuffer is in a Hypervisor internal + format. + Bits 2-63: Reserved + guestId: ID obtained from H_GUEST_CREATE + vcpuId: ID of the vCPU pass to H_GUEST_CREATE_VCPU + dataBuffer: A L1 real address of the GSB. + If takeOwnershipOfVcpuState, size must be at least the size + returned by ID=0x0001 + dataBufferSizeInBytes: Size of dataBuffer + Output: + R3: Return code + R4: If R3 = H_Invalid_Element_Id: The array index of the bad + element ID. + If R3 = H_Invalid_Element_Size: The array index of the bad + element size. + If R3 = H_Invalid_Element_Value: The array index of the bad + element value. + +H_GUEST_SET_STATE() +------------------- + +This is called to set L2 wide or vCPU specific L2 state. This info is +passed via the Guest State Buffer (GSB), necessary details below: + +This can set either L2 wide or vcpu specific information. Examples of +L2 wide is the timebase offset or process scoped page table +info. Examples of vCPU specific are GPRs or VSRs. A bit in the flags +parameter specifies if this call is L2 wide or vCPU specific and the +IDs in the GSB must match this. + +The L1 provides a pointer to the GSB as a parameter to this call. Also +provided is the L2 and vCPU IDs associated with the state to set. + +The L1 writes all values in the GSB and the L0 only reads the GSB for +this call:: + + H_GUEST_SET_STATE(uint64 flags, + uint64 guestId, + uint64 vcpuId, + uint64 dataBuffer, + uint64 dataBufferSizeInBytes); + Parameters: + Input: + flags: + Bit 0: getGuestWideState: Request state of the Guest instead + of an individual VCPU. + Bit 1: returnOwnershipOfVcpuState Return Guest VCPU state. See + GET_STATE takeOwnershipOfVcpuState + Bits 2-63: Reserved + guestId: ID obtained from H_GUEST_CREATE + vcpuId: ID of the vCPU pass to H_GUEST_CREATE_VCPU + dataBuffer: A L1 real address of the GSB. + If takeOwnershipOfVcpuState, size must be at least the size + returned by ID=0x0001 + dataBufferSizeInBytes: Size of dataBuffer + Output: + R3: Return code + R4: If R3 = H_Invalid_Element_Id: The array index of the bad + element ID. + If R3 = H_Invalid_Element_Size: The array index of the bad + element size. + If R3 = H_Invalid_Element_Value: The array index of the bad + element value. + +H_GUEST_RUN_VCPU() +------------------ + +This is called to run an L2 vCPU. The L2 and vCPU IDs are passed in as +parameters. The vCPU runs with the state set previously using +H_GUEST_SET_STATE(). When the L2 exits, the L1 will resume from this +hcall. + +This hcall also has associated input and output GSBs. Unlike +H_GUEST_{S,G}ET_STATE(), these GSB pointers are not passed in as +parameters to the hcall (This was done in the interest of +performance). The locations of these GSBs must be preregistered using +the H_GUEST_SET_STATE() call with ID 0x0c00 and 0x0c01 (see table +below). + +The input GSB may contain only VCPU specific elements to be set. This +GSB may also contain zero elements (ie 0 in the first 4 bytes of the +GSB) if nothing needs to be set. + +On exit from the hcall, the output buffer is filled with elements +determined by the L0. The reason for the exit is contained in GPR4 (ie +NIP is put in GPR4). The elements returned depend on the exit +type. For example, if the exit reason is the L2 doing a hcall (GPR4 = +0xc00), then GPR3-12 are provided in the output GSB as this is the +state likely needed to service the hcall. If additional state is +needed, H_GUEST_GET_STATE() may be called by the L1. + +To synthesize interrupts in the L2, when calling H_GUEST_RUN_VCPU() +the L1 may set a flag (as a hcall parameter) and the L0 will +synthesize the interrupt in the L2. Alternatively, the L1 may +synthesize the interrupt itself using H_GUEST_SET_STATE() or the +H_GUEST_RUN_VCPU() input GSB to set the state appropriately:: + + H_GUEST_RUN_VCPU(uint64 flags, + uint64 guestId, + uint64 vcpuId, + uint64 dataBuffer, + uint64 dataBufferSizeInBytes); + Parameters: + Input: + flags: + Bit 0: generateExternalInterrupt: Generate an external interrupt + Bit 1: generatePrivilegedDoorbell: Generate a Privileged Doorbell + Bit 2: sendToSystemReset”: Generate a System Reset Interrupt + Bits 3-63: Reserved + guestId: ID obtained from H_GUEST_CREATE + vcpuId: ID of the vCPU pass to H_GUEST_CREATE_VCPU + Output: + R3: Return code + R4: If R3 = H_Success: The reason L1 VCPU exited (ie. NIA) + 0x000: The VCPU stopped running for an unspecified reason. An + example of this is the Hypervisor stopping a VCPU running + due to an outstanding interrupt for the Host Partition. + 0x980: HDEC + 0xC00: HCALL + 0xE00: HDSI + 0xE20: HISI + 0xE40: HEA + 0xF80: HV Fac Unavail + If R3 = H_Invalid_Element_Id, H_Invalid_Element_Size, or + H_Invalid_Element_Value: R4 is offset of the invalid element + in the input buffer. + +H_GUEST_DELETE() +---------------- + +This is called to delete an L2. All associated vCPUs are also +deleted. No specific vCPU delete call is provided. + +A flag may be provided to delete all guests. This is used to reset the +L0 in the case of kdump/kexec:: + + H_GUEST_DELETE(uint64 flags, + uint64 guestId) + Parameters: + Input: + flags: + Bit 0: deleteAllGuests: deletes all guests + Bits 1-63: Reserved + guestId: ID obtained from H_GUEST_CREATE + Output: + R3: Return code + +Guest State Buffer +================== + +The Guest State Buffer (GSB) is the main method of communicating state +about the L2 between the L1 and L0 via H_GUEST_{G,S}ET() and +H_GUEST_VCPU_RUN() calls. + +State may be associated with a whole L2 (eg timebase offset) or a +specific L2 vCPU (eg. GPR state). Only L2 VCPU state maybe be set by +H_GUEST_VCPU_RUN(). + +All data in the GSB is big endian (as is standard in PAPR) + +The Guest state buffer has a header which gives the number of +elements, followed by the GSB elements themselves. + +GSB header: + ++----------+----------+-------------------------------------------+ +| Offset | Size | Purpose | +| Bytes | Bytes | | ++==========+==========+===========================================+ +| 0 | 4 | Number of elements | ++----------+----------+-------------------------------------------+ +| 4 | | Guest state buffer elements | ++----------+----------+-------------------------------------------+ + +GSB element: + ++----------+----------+-------------------------------------------+ +| Offset | Size | Purpose | +| Bytes | Bytes | | ++==========+==========+===========================================+ +| 0 | 2 | ID | ++----------+----------+-------------------------------------------+ +| 2 | 2 | Size of Value | ++----------+----------+-------------------------------------------+ +| 4 | As above | Value | ++----------+----------+-------------------------------------------+ + +The ID in the GSB element specifies what is to be set. This includes +archtected state like GPRs, VSRs, SPRs, plus also some meta data about +the partition like the timebase offset and partition scoped page +table information. + ++--------+-------+----+--------+----------------------------------+ +| ID | Size | RW | Thread | Details | +| | Bytes | | Guest | | +| | | | Scope | | ++========+=======+====+========+==================================+ +| 0x0000 | | RW | TG | NOP element | ++--------+-------+----+--------+----------------------------------+ +| 0x0001 | 0x08 | R | G | Size of L0 vCPU state. See: | +| | | | | H_GUEST_GET_STATE: | +| | | | | flags = takeOwnershipOfVcpuState | ++--------+-------+----+--------+----------------------------------+ +| 0x0002 | 0x08 | R | G | Size Run vCPU out buffer | ++--------+-------+----+--------+----------------------------------+ +| 0x0003 | 0x04 | RW | G | Logical PVR | ++--------+-------+----+--------+----------------------------------+ +| 0x0004 | 0x08 | RW | G | TB Offset (L1 relative) | ++--------+-------+----+--------+----------------------------------+ +| 0x0005 | 0x18 | RW | G |Partition scoped page tbl info: | +| | | | | | +| | | | |- 0x00 Addr part scope table | +| | | | |- 0x08 Num addr bits | +| | | | |- 0x10 Size root dir | ++--------+-------+----+--------+----------------------------------+ +| 0x0006 | 0x10 | RW | G |Process Table Information: | +| | | | | | +| | | | |- 0x0 Addr proc scope table | +| | | | |- 0x8 Table size. | ++--------+-------+----+--------+----------------------------------+ +| 0x0007-| | | | Reserved | +| 0x0BFF | | | | | ++--------+-------+----+--------+----------------------------------+ +| 0x0C00 | 0x10 | RW | T |Run vCPU Input Buffer: | +| | | | | | +| | | | |- 0x0 Addr of buffer | +| | | | |- 0x8 Buffer Size. | ++--------+-------+----+--------+----------------------------------+ +| 0x0C01 | 0x10 | RW | T |Run vCPU Output Buffer: | +| | | | | | +| | | | |- 0x0 Addr of buffer | +| | | | |- 0x8 Buffer Size. | ++--------+-------+----+--------+----------------------------------+ +| 0x0C02 | 0x08 | RW | T | vCPU VPA Address | ++--------+-------+----+--------+----------------------------------+ +| 0x0C03-| | | | Reserved | +| 0x0FFF | | | | | ++--------+-------+----+--------+----------------------------------+ +| 0x1000-| 0x08 | RW | T | GPR 0-31 | +| 0x101F | | | | | ++--------+-------+----+--------+----------------------------------+ +| 0x1020 | 0x08 | T | T | HDEC expiry TB | ++--------+-------+----+--------+----------------------------------+ +| 0x1021 | 0x08 | RW | T | NIA | ++--------+-------+----+--------+----------------------------------+ +| 0x1022 | 0x08 | RW | T | MSR | ++--------+-------+----+--------+----------------------------------+ +| 0x1023 | 0x08 | RW | T | LR | ++--------+-------+----+--------+----------------------------------+ +| 0x1024 | 0x08 | RW | T | XER | ++--------+-------+----+--------+----------------------------------+ +| 0x1025 | 0x08 | RW | T | CTR | ++--------+-------+----+--------+----------------------------------+ +| 0x1026 | 0x08 | RW | T | CFAR | ++--------+-------+----+--------+----------------------------------+ +| 0x1027 | 0x08 | RW | T | SRR0 | ++--------+-------+----+--------+----------------------------------+ +| 0x1028 | 0x08 | RW | T | SRR1 | ++--------+-------+----+--------+----------------------------------+ +| 0x1029 | 0x08 | RW | T | DAR | ++--------+-------+----+--------+----------------------------------+ +| 0x102A | 0x08 | RW | T | DEC expiry TB | ++--------+-------+----+--------+----------------------------------+ +| 0x102B | 0x08 | RW | T | VTB | ++--------+-------+----+--------+----------------------------------+ +| 0x102C | 0x08 | RW | T | LPCR | ++--------+-------+----+--------+----------------------------------+ +| 0x102D | 0x08 | RW | T | HFSCR | ++--------+-------+----+--------+----------------------------------+ +| 0x102E | 0x08 | RW | T | FSCR | ++--------+-------+----+--------+----------------------------------+ +| 0x102F | 0x08 | RW | T | FPSCR | ++--------+-------+----+--------+----------------------------------+ +| 0x1030 | 0x08 | RW | T | DAWR0 | ++--------+-------+----+--------+----------------------------------+ +| 0x1031 | 0x08 | RW | T | DAWR1 | ++--------+-------+----+--------+----------------------------------+ +| 0x1032 | 0x08 | RW | T | CIABR | ++--------+-------+----+--------+----------------------------------+ +| 0x1033 | 0x08 | RW | T | PURR | ++--------+-------+----+--------+----------------------------------+ +| 0x1034 | 0x08 | RW | T | SPURR | ++--------+-------+----+--------+----------------------------------+ +| 0x1035 | 0x08 | RW | T | IC | ++--------+-------+----+--------+----------------------------------+ +| 0x1036-| 0x08 | RW | T | SPRG 0-3 | +| 0x1039 | | | | | ++--------+-------+----+--------+----------------------------------+ +| 0x103A | 0x08 | W | T | PPR | ++--------+-------+----+--------+----------------------------------+ +| 0x103B | 0x08 | RW | T | MMCR 0-3 | +| 0x103E | | | | | ++--------+-------+----+--------+----------------------------------+ +| 0x103F | 0x08 | RW | T | MMCRA | ++--------+-------+----+--------+----------------------------------+ +| 0x1040 | 0x08 | RW | T | SIER | ++--------+-------+----+--------+----------------------------------+ +| 0x1041 | 0x08 | RW | T | SIER 2 | ++--------+-------+----+--------+----------------------------------+ +| 0x1042 | 0x08 | RW | T | SIER 3 | ++--------+-------+----+--------+----------------------------------+ +| 0x1043 | 0x08 | RW | T | BESCR | ++--------+-------+----+--------+----------------------------------+ +| 0x1044 | 0x08 | RW | T | EBBHR | ++--------+-------+----+--------+----------------------------------+ +| 0x1045 | 0x08 | RW | T | EBBRR | ++--------+-------+----+--------+----------------------------------+ +| 0x1046 | 0x08 | RW | T | AMR | ++--------+-------+----+--------+----------------------------------+ +| 0x1047 | 0x08 | RW | T | IAMR | ++--------+-------+----+--------+----------------------------------+ +| 0x1048 | 0x08 | RW | T | AMOR | ++--------+-------+----+--------+----------------------------------+ +| 0x1049 | 0x08 | RW | T | UAMOR | ++--------+-------+----+--------+----------------------------------+ +| 0x104A | 0x08 | RW | T | SDAR | ++--------+-------+----+--------+----------------------------------+ +| 0x104B | 0x08 | RW | T | SIAR | ++--------+-------+----+--------+----------------------------------+ +| 0x104C | 0x08 | RW | T | DSCR | ++--------+-------+----+--------+----------------------------------+ +| 0x104D | 0x08 | RW | T | TAR | ++--------+-------+----+--------+----------------------------------+ +| 0x104E | 0x08 | RW | T | DEXCR | ++--------+-------+----+--------+----------------------------------+ +| 0x104F | 0x08 | RW | T | HDEXCR | ++--------+-------+----+--------+----------------------------------+ +| 0x1050 | 0x08 | RW | T | HASHKEYR | ++--------+-------+----+--------+----------------------------------+ +| 0x1051 | 0x08 | RW | T | HASHPKEYR | ++--------+-------+----+--------+----------------------------------+ +| 0x1052 | 0x08 | RW | T | CTRL | ++--------+-------+----+--------+----------------------------------+ +| 0x1053-| | | | Reserved | +| 0x1FFF | | | | | ++--------+-------+----+--------+----------------------------------+ +| 0x2000 | 0x04 | RW | T | CR | ++--------+-------+----+--------+----------------------------------+ +| 0x2001 | 0x04 | RW | T | PIDR | ++--------+-------+----+--------+----------------------------------+ +| 0x2002 | 0x04 | RW | T | DSISR | ++--------+-------+----+--------+----------------------------------+ +| 0x2003 | 0x04 | RW | T | VSCR | ++--------+-------+----+--------+----------------------------------+ +| 0x2004 | 0x04 | RW | T | VRSAVE | ++--------+-------+----+--------+----------------------------------+ +| 0x2005 | 0x04 | RW | T | DAWRX0 | ++--------+-------+----+--------+----------------------------------+ +| 0x2006 | 0x04 | RW | T | DAWRX1 | ++--------+-------+----+--------+----------------------------------+ +| 0x2007-| 0x04 | RW | T | PMC 1-6 | +| 0x200c | | | | | ++--------+-------+----+--------+----------------------------------+ +| 0x200D | 0x04 | RW | T | WORT | ++--------+-------+----+--------+----------------------------------+ +| 0x200E | 0x04 | RW | T | PSPB | ++--------+-------+----+--------+----------------------------------+ +| 0x200F-| | | | Reserved | +| 0x2FFF | | | | | ++--------+-------+----+--------+----------------------------------+ +| 0x3000-| 0x10 | RW | T | VSR 0-63 | +| 0x303F | | | | | ++--------+-------+----+--------+----------------------------------+ +| 0x3040-| | | | Reserved | +| 0xEFFF | | | | | ++--------+-------+----+--------+----------------------------------+ +| 0xF000 | 0x08 | R | T | HDAR | ++--------+-------+----+--------+----------------------------------+ +| 0xF001 | 0x04 | R | T | HDSISR | ++--------+-------+----+--------+----------------------------------+ +| 0xF002 | 0x04 | R | T | HEIR | ++--------+-------+----+--------+----------------------------------+ +| 0xF003 | 0x08 | R | T | ASDR | ++--------+-------+----+--------+----------------------------------+ + + +Miscellaneous info +================== + +State not in ptregs/hvregs +-------------------------- + +In the v1 API, some state is not in the ptregs/hvstate. This includes +the vector register and some SPRs. For the L1 to set this state for +the L2, the L1 loads up these hardware registers before the +h_enter_nested() call and the L0 ensures they end up as the L2 state +(by not touching them). + +The v2 API removes this and explicitly sets this state via the GSB. + +L1 Implementation details: Caching state +---------------------------------------- + +In the v1 API, all state is sent from the L1 to the L0 and vice versa +on every h_enter_nested() hcall. If the L0 is not currently running +any L2s, the L0 has no state information about them. The only +exception to this is the location of the partition table, registered +via h_set_partition_table(). + +The v2 API changes this so that the L0 retains the L2 state even when +it's vCPUs are no longer running. This means that the L1 only needs to +communicate with the L0 about L2 state when it needs to modify the L2 +state, or when it's value is out of date. This provides an opportunity +for performance optimisation. + +When a vCPU exits from a H_GUEST_RUN_VCPU() call, the L1 internally +marks all L2 state as invalid. This means that if the L1 wants to know +the L2 state (say via a kvm_get_one_reg() call), it needs call +H_GUEST_GET_STATE() to get that state. Once it's read, it's marked as +valid in L1 until the L2 is run again. + +Also, when an L1 modifies L2 vcpu state, it doesn't need to write it +to the L0 until that L2 vcpu runs again. Hence when the L1 updates +state (say via a kvm_set_one_reg() call), it writes to an internal L1 +copy and only flushes this copy to the L0 when the L2 runs again via +the H_GUEST_VCPU_RUN() input buffer. + +This lazy updating of state by the L1 avoids unnecessary +H_GUEST_{G|S}ET_STATE() calls. From 21ce931e55c19c1f74378b4836d9dae631da0e62 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Wed, 13 Sep 2023 21:42:32 -0700 Subject: [PATCH 090/475] perf symbol: Avoid an undefined behavior warning The node (nd) may be NULL and pointer arithmetic on NULL is undefined behavior. Move the computation of next below the NULL check on the node. Signed-off-by: Ian Rogers Cc: James Clark Link: https://lore.kernel.org/r/20230914044233.1550195-1-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/util/symbol.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/perf/util/symbol.c b/tools/perf/util/symbol.c index 3f36675b7c8f..5b54d2639df4 100644 --- a/tools/perf/util/symbol.c +++ b/tools/perf/util/symbol.c @@ -202,11 +202,10 @@ void symbols__fixup_duplicate(struct rb_root_cached *symbols) curr = rb_entry(nd, struct symbol, rb_node); again: nd = rb_next(&curr->rb_node); - next = rb_entry(nd, struct symbol, rb_node); - if (!nd) break; + next = rb_entry(nd, struct symbol, rb_node); if (curr->start != next->start) continue; From 3d0f5f456a5786573ba6a3358178c8db580e4b85 Mon Sep 17 00:00:00 2001 From: James Clark Date: Wed, 13 Sep 2023 16:33:48 +0100 Subject: [PATCH 091/475] perf pmu: Move pmu__find_core_pmu() to pmus.c pmu__find_core_pmu() more logically belongs in pmus.c because it iterates over all PMUs, so move it to pmus.c At the same time rename it to perf_pmus__find_core_pmu() to match the naming convention in this file. list_prepare_entry() can't be used in perf_pmus__scan_core() anymore now that it's called from the same compilation unit. This is with -O2 (specifically -O1 -ftree-vrp -finline-functions -finline-small-functions) which allow the bounds of the array access to be determined at compile time. list_prepare_entry() subtracts the offset of the 'list' member in struct perf_pmu from &core_pmus, which isn't a struct perf_pmu. The compiler sees that pmu results in &core_pmus - 8 and refuses to compile. At runtime this works because list_for_each_entry_continue() always adds the offset back again before dereferencing ->next, but it's technically undefined behavior. With -fsanitize=undefined an additional warning is generated. Using list_first_entry_or_null() to get the first entry here avoids doing &core_pmus - 8 but has the same result and fixes both the compile warning and the undefined behavior warning. There are other uses of list_prepare_entry() in pmus.c, but the compiler doesn't seem to be able to see that they can also be called with &core_pmus, so I won't change any at this time. Signed-off-by: James Clark Reviewed-by: Ian Rogers Reviewed-by: John Garry Cc: Ravi Bangoria Cc: Eduard Zingerman Cc: Will Deacon Cc: Leo Yan Cc: Mike Leach Cc: Jing Zhang Cc: Haixin Yu Cc: Kan Liang Cc: linux-arm-kernel@lists.infradead.org Link: https://lore.kernel.org/r/20230913153355.138331-2-james.clark@arm.com Signed-off-by: Namhyung Kim --- tools/perf/arch/arm64/util/pmu.c | 6 +++--- tools/perf/tests/expr.c | 2 +- tools/perf/util/expr.c | 2 +- tools/perf/util/pmu.c | 17 ----------------- tools/perf/util/pmu.h | 2 +- tools/perf/util/pmus.c | 20 +++++++++++++++++++- 6 files changed, 25 insertions(+), 24 deletions(-) diff --git a/tools/perf/arch/arm64/util/pmu.c b/tools/perf/arch/arm64/util/pmu.c index 615084eb88d8..3d9330feebd2 100644 --- a/tools/perf/arch/arm64/util/pmu.c +++ b/tools/perf/arch/arm64/util/pmu.c @@ -10,7 +10,7 @@ const struct pmu_metrics_table *pmu_metrics_table__find(void) { - struct perf_pmu *pmu = pmu__find_core_pmu(); + struct perf_pmu *pmu = perf_pmus__find_core_pmu(); if (pmu) return perf_pmu__find_metrics_table(pmu); @@ -20,7 +20,7 @@ const struct pmu_metrics_table *pmu_metrics_table__find(void) const struct pmu_events_table *pmu_events_table__find(void) { - struct perf_pmu *pmu = pmu__find_core_pmu(); + struct perf_pmu *pmu = perf_pmus__find_core_pmu(); if (pmu) return perf_pmu__find_events_table(pmu); @@ -32,7 +32,7 @@ double perf_pmu__cpu_slots_per_cycle(void) { char path[PATH_MAX]; unsigned long long slots = 0; - struct perf_pmu *pmu = pmu__find_core_pmu(); + struct perf_pmu *pmu = perf_pmus__find_core_pmu(); if (pmu) { perf_pmu__pathname_scnprintf(path, sizeof(path), diff --git a/tools/perf/tests/expr.c b/tools/perf/tests/expr.c index 78544092ef0c..e3aa9d4fcf3a 100644 --- a/tools/perf/tests/expr.c +++ b/tools/perf/tests/expr.c @@ -76,7 +76,7 @@ static int test__expr(struct test_suite *t __maybe_unused, int subtest __maybe_u struct expr_parse_ctx *ctx; bool is_intel = false; char strcmp_cpuid_buf[256]; - struct perf_pmu *pmu = pmu__find_core_pmu(); + struct perf_pmu *pmu = perf_pmus__find_core_pmu(); char *cpuid = perf_pmu__getcpuid(pmu); char *escaped_cpuid1, *escaped_cpuid2; diff --git a/tools/perf/util/expr.c b/tools/perf/util/expr.c index 4488f306de78..7be23b3ac082 100644 --- a/tools/perf/util/expr.c +++ b/tools/perf/util/expr.c @@ -509,7 +509,7 @@ double expr__strcmp_cpuid_str(const struct expr_parse_ctx *ctx __maybe_unused, bool compute_ids __maybe_unused, const char *test_id) { double ret; - struct perf_pmu *pmu = pmu__find_core_pmu(); + struct perf_pmu *pmu = perf_pmus__find_core_pmu(); char *cpuid = perf_pmu__getcpuid(pmu); if (!cpuid) diff --git a/tools/perf/util/pmu.c b/tools/perf/util/pmu.c index e2159854ab26..f50a5636633f 100644 --- a/tools/perf/util/pmu.c +++ b/tools/perf/util/pmu.c @@ -2050,20 +2050,3 @@ void perf_pmu__delete(struct perf_pmu *pmu) zfree(&pmu->id); free(pmu); } - -struct perf_pmu *pmu__find_core_pmu(void) -{ - struct perf_pmu *pmu = NULL; - - while ((pmu = perf_pmus__scan_core(pmu))) { - /* - * The cpumap should cover all CPUs. Otherwise, some CPUs may - * not support some events or have different event IDs. - */ - if (RC_CHK_ACCESS(pmu->cpus)->nr != cpu__max_cpu().cpu) - return NULL; - - return pmu; - } - return NULL; -} diff --git a/tools/perf/util/pmu.h b/tools/perf/util/pmu.h index bd5d804a6736..d7b46085642d 100644 --- a/tools/perf/util/pmu.h +++ b/tools/perf/util/pmu.h @@ -264,6 +264,6 @@ int perf_pmu__pathname_fd(int dirfd, const char *pmu_name, const char *filename, struct perf_pmu *perf_pmu__lookup(struct list_head *pmus, int dirfd, const char *lookup_name); struct perf_pmu *perf_pmu__create_placeholder_core_pmu(struct list_head *core_pmus); void perf_pmu__delete(struct perf_pmu *pmu); -struct perf_pmu *pmu__find_core_pmu(void); +struct perf_pmu *perf_pmus__find_core_pmu(void); #endif /* __PMU_H */ diff --git a/tools/perf/util/pmus.c b/tools/perf/util/pmus.c index 6631367c756f..cec869cbe163 100644 --- a/tools/perf/util/pmus.c +++ b/tools/perf/util/pmus.c @@ -10,6 +10,7 @@ #include #include #include +#include "cpumap.h" #include "debug.h" #include "evsel.h" #include "pmus.h" @@ -268,7 +269,7 @@ struct perf_pmu *perf_pmus__scan_core(struct perf_pmu *pmu) { if (!pmu) { pmu_read_sysfs(/*core_only=*/true); - pmu = list_prepare_entry(pmu, &core_pmus, list); + return list_first_entry_or_null(&core_pmus, typeof(*pmu), list); } list_for_each_entry_continue(pmu, &core_pmus, list) return pmu; @@ -592,3 +593,20 @@ struct perf_pmu *evsel__find_pmu(const struct evsel *evsel) } return pmu; } + +struct perf_pmu *perf_pmus__find_core_pmu(void) +{ + struct perf_pmu *pmu = NULL; + + while ((pmu = perf_pmus__scan_core(pmu))) { + /* + * The cpumap should cover all CPUs. Otherwise, some CPUs may + * not support some events or have different event IDs. + */ + if (RC_CHK_ACCESS(pmu->cpus)->nr != cpu__max_cpu().cpu) + return NULL; + + return pmu; + } + return NULL; +} From 105e5b433e5c743bcdb4956b00b772dfddbee455 Mon Sep 17 00:00:00 2001 From: James Clark Date: Wed, 13 Sep 2023 16:33:49 +0100 Subject: [PATCH 092/475] perf pmus: Simplify perf_pmus__find_core_pmu() Currently the while loop always either exits on the first iteration with a core PMU, or exits with NULL on heterogeneous systems or when not all CPUs are online. Both of the latter behaviors are undesirable for platforms other than Arm so simplify it to always return the first core PMU, or NULL if none exist. This behavior was depended on by the Arm version of pmu_metrics_table__find(), so the logic has been moved there instead. Signed-off-by: James Clark Suggested-by: Ian Rogers Reviewed-by: Ian Rogers Reviewed-by: John Garry Cc: Ravi Bangoria Cc: Eduard Zingerman Cc: Will Deacon Cc: Leo Yan Cc: Mike Leach Cc: Jing Zhang Cc: Haixin Yu Cc: Kan Liang Cc: linux-arm-kernel@lists.infradead.org Link: https://lore.kernel.org/r/20230913153355.138331-3-james.clark@arm.com Signed-off-by: Namhyung Kim --- tools/perf/arch/arm64/util/pmu.c | 8 +++++++- tools/perf/util/pmus.c | 14 +------------- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/tools/perf/arch/arm64/util/pmu.c b/tools/perf/arch/arm64/util/pmu.c index 3d9330feebd2..3099f5f448ba 100644 --- a/tools/perf/arch/arm64/util/pmu.c +++ b/tools/perf/arch/arm64/util/pmu.c @@ -10,8 +10,14 @@ const struct pmu_metrics_table *pmu_metrics_table__find(void) { - struct perf_pmu *pmu = perf_pmus__find_core_pmu(); + struct perf_pmu *pmu; + /* Metrics aren't currently supported on heterogeneous Arm systems */ + if (perf_pmus__num_core_pmus() > 1) + return NULL; + + /* Doesn't matter which one here because they'll all be the same */ + pmu = perf_pmus__find_core_pmu(); if (pmu) return perf_pmu__find_metrics_table(pmu); diff --git a/tools/perf/util/pmus.c b/tools/perf/util/pmus.c index cec869cbe163..64e798e68a2d 100644 --- a/tools/perf/util/pmus.c +++ b/tools/perf/util/pmus.c @@ -596,17 +596,5 @@ struct perf_pmu *evsel__find_pmu(const struct evsel *evsel) struct perf_pmu *perf_pmus__find_core_pmu(void) { - struct perf_pmu *pmu = NULL; - - while ((pmu = perf_pmus__scan_core(pmu))) { - /* - * The cpumap should cover all CPUs. Otherwise, some CPUs may - * not support some events or have different event IDs. - */ - if (RC_CHK_ACCESS(pmu->cpus)->nr != cpu__max_cpu().cpu) - return NULL; - - return pmu; - } - return NULL; + return perf_pmus__scan_core(NULL); } From 70360fad919b06100748764eb82fa399a9f37e1c Mon Sep 17 00:00:00 2001 From: James Clark Date: Wed, 13 Sep 2023 16:33:50 +0100 Subject: [PATCH 093/475] perf pmu: Remove unused function pmu_events_table__find() is no longer used so remove it and its Arm specific version. Signed-off-by: James Clark Reviewed-by: Ian Rogers Reviewed-by: John Garry Cc: Ravi Bangoria Cc: Eduard Zingerman Cc: Will Deacon Cc: Leo Yan Cc: Mike Leach Cc: Jing Zhang Cc: Haixin Yu Cc: Kan Liang Cc: linux-arm-kernel@lists.infradead.org Link: https://lore.kernel.org/r/20230913153355.138331-4-james.clark@arm.com Signed-off-by: Namhyung Kim --- tools/perf/arch/arm64/util/pmu.c | 10 ---------- tools/perf/util/pmu.c | 5 ----- tools/perf/util/pmu.h | 1 - 3 files changed, 16 deletions(-) diff --git a/tools/perf/arch/arm64/util/pmu.c b/tools/perf/arch/arm64/util/pmu.c index 3099f5f448ba..2a4eab2d160e 100644 --- a/tools/perf/arch/arm64/util/pmu.c +++ b/tools/perf/arch/arm64/util/pmu.c @@ -24,16 +24,6 @@ const struct pmu_metrics_table *pmu_metrics_table__find(void) return NULL; } -const struct pmu_events_table *pmu_events_table__find(void) -{ - struct perf_pmu *pmu = perf_pmus__find_core_pmu(); - - if (pmu) - return perf_pmu__find_events_table(pmu); - - return NULL; -} - double perf_pmu__cpu_slots_per_cycle(void) { char path[PATH_MAX]; diff --git a/tools/perf/util/pmu.c b/tools/perf/util/pmu.c index f50a5636633f..0d81c059c91c 100644 --- a/tools/perf/util/pmu.c +++ b/tools/perf/util/pmu.c @@ -776,11 +776,6 @@ char *perf_pmu__getcpuid(struct perf_pmu *pmu) return cpuid; } -__weak const struct pmu_events_table *pmu_events_table__find(void) -{ - return perf_pmu__find_events_table(NULL); -} - __weak const struct pmu_metrics_table *pmu_metrics_table__find(void) { return perf_pmu__find_metrics_table(NULL); diff --git a/tools/perf/util/pmu.h b/tools/perf/util/pmu.h index d7b46085642d..04b317b17d66 100644 --- a/tools/perf/util/pmu.h +++ b/tools/perf/util/pmu.h @@ -238,7 +238,6 @@ void pmu_add_cpu_aliases_table(struct perf_pmu *pmu, const struct pmu_events_table *table); char *perf_pmu__getcpuid(struct perf_pmu *pmu); -const struct pmu_events_table *pmu_events_table__find(void); const struct pmu_metrics_table *pmu_metrics_table__find(void); int perf_pmu__convert_scale(const char *scale, char **end, double *sval); From 486021e04b24f8dffe54ec282ce5d4cc3e71133f Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Thu, 7 Sep 2023 22:22:16 -0700 Subject: [PATCH 094/475] perf annotate: Add more x86 mov instruction cases Instructions with sign- and zero- extention like movsbl and movzwq were not handled properly. As it can check different size suffix (-b, -w, -l or -q) we can omit that and add the common parts even though some combinations are not possible. Reviewed-by: Ian Rogers Link: https://lore.kernel.org/r/20230908052216.566148-1-namhyung@kernel.org Signed-off-by: Namhyung Kim --- tools/perf/arch/x86/annotate/instructions.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/perf/arch/x86/annotate/instructions.c b/tools/perf/arch/x86/annotate/instructions.c index 5f4ac4fc7fcf..5cdf457f5cbe 100644 --- a/tools/perf/arch/x86/annotate/instructions.c +++ b/tools/perf/arch/x86/annotate/instructions.c @@ -74,12 +74,15 @@ static struct ins x86__instructions[] = { { .name = "movdqa", .ops = &mov_ops, }, { .name = "movdqu", .ops = &mov_ops, }, { .name = "movsd", .ops = &mov_ops, }, - { .name = "movslq", .ops = &mov_ops, }, { .name = "movss", .ops = &mov_ops, }, + { .name = "movsb", .ops = &mov_ops, }, + { .name = "movsw", .ops = &mov_ops, }, + { .name = "movsl", .ops = &mov_ops, }, { .name = "movupd", .ops = &mov_ops, }, { .name = "movups", .ops = &mov_ops, }, - { .name = "movzbl", .ops = &mov_ops, }, - { .name = "movzwl", .ops = &mov_ops, }, + { .name = "movzb", .ops = &mov_ops, }, + { .name = "movzw", .ops = &mov_ops, }, + { .name = "movzl", .ops = &mov_ops, }, { .name = "mulsd", .ops = &mov_ops, }, { .name = "mulss", .ops = &mov_ops, }, { .name = "nop", .ops = &nop_ops, }, From 79df8365e37d38a84d9a15cc2f6386f5587e211a Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Fri, 15 Sep 2023 10:09:10 +0100 Subject: [PATCH 095/475] perf kwork: Fix spelling mistake "COMMMAND" -> "COMMAND" There is a spelling mistake in a literal string. Fix it. Signed-off-by: Colin Ian King Reviewed-by: Ian Rogers Cc: kernel-janitors@vger.kernel.org Link: https://lore.kernel.org/r/20230915090910.30182-1-colin.i.king@gmail.com Signed-off-by: Namhyung Kim --- tools/perf/builtin-kwork.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/perf/builtin-kwork.c b/tools/perf/builtin-kwork.c index 04b966801643..7e8dd35d764b 100644 --- a/tools/perf/builtin-kwork.c +++ b/tools/perf/builtin-kwork.c @@ -1629,7 +1629,7 @@ static void top_print_header(struct perf_kwork *kwork __maybe_unused) PRINT_CPU_USAGE_WIDTH, "%CPU", PRINT_RUNTIME_HEADER_WIDTH + RPINT_DECIMAL_WIDTH, "RUNTIME", - PRINT_TASK_NAME_WIDTH, "COMMMAND"); + PRINT_TASK_NAME_WIDTH, "COMMAND"); printf("\n "); print_separator(ret); } From a132b784db68b543fd2745973cd8b5edf8e9bde4 Mon Sep 17 00:00:00 2001 From: Yang Jihong Date: Sat, 16 Sep 2023 09:16:41 +0000 Subject: [PATCH 096/475] perf test: Fix test-record-dummy-C0 failure for supported PERF_FORMAT_LOST feature kernel For kernel that supports PERF_FORMAT_LOST, attr->read_format has PERF_FORMAT_LOST bit. Update expected value of attr->read_format of test-record-dummy-C0 for this scenario. Before: # ./perf test 17 -vv 17: Setup struct perf_event_attr : --- start --- test child forked, pid 1609441 running './tests/attr/test-record-dummy-C0' 'PERF_TEST_ATTR=/tmp/tmpm3s60aji ./perf record -o /tmp/tmpm3s60aji/perf.data --no-bpf-event -e dummy -C 0 kill >/dev/null 2>&1' ret '1', expected '1' expected read_format=4, got 20 FAILED './tests/attr/test-record-dummy-C0' - match failure test child finished with -1 ---- end ---- Setup struct perf_event_attr: FAILED! After: # ./perf test 17 -vv 17: Setup struct perf_event_attr : --- start --- test child forked, pid 1609441 running './tests/attr/test-record-dummy-C0' 'PERF_TEST_ATTR=/tmp/tmppa9vxcb7 ./perf record -o /tmp/tmppa9vxcb7/perf.data --no-bpf-event -e dummy -C 0 kill >/dev/null 2>&1' ret '1', expected '1' test child finished with 0 ---- end ---- Setup struct perf_event_attr: Ok Reported-and-Tested-by: Namhyung Kim Signed-off-by: Yang Jihong Link: https://lore.kernel.org/r/20230916091641.776031-1-yangjihong1@huawei.com Signed-off-by: Namhyung Kim --- tools/perf/tests/attr/test-record-dummy-C0 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/perf/tests/attr/test-record-dummy-C0 b/tools/perf/tests/attr/test-record-dummy-C0 index 83ca4e373acd..576ec48b3aaf 100644 --- a/tools/perf/tests/attr/test-record-dummy-C0 +++ b/tools/perf/tests/attr/test-record-dummy-C0 @@ -17,7 +17,7 @@ sample_period=4000 # PERF_SAMPLE_PERIOD # + PERF_SAMPLE_CPU added by -C 0 sample_type=391 -read_format=4 +read_format=4|20 disabled=0 inherit=1 pinned=0 From a59e9eb25216eb1dc99e14fc31b76aa648d79540 Mon Sep 17 00:00:00 2001 From: Julia Lawall Date: Thu, 7 Sep 2023 11:55:13 +0200 Subject: [PATCH 097/475] powerpc/powermac: add missing of_node_put for_each_node_by_name performs an of_node_get on each iteration, so a break out of the loop requires an of_node_put. This was done using the Coccinelle semantic patch iterators/for_each_child.cocci Signed-off-by: Julia Lawall Signed-off-by: Michael Ellerman Link: https://msgid.link/20230907095521.14053-4-Julia.Lawall@inria.fr --- arch/powerpc/platforms/powermac/low_i2c.c | 4 +++- arch/powerpc/platforms/powermac/smp.c | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/arch/powerpc/platforms/powermac/low_i2c.c b/arch/powerpc/platforms/powermac/low_i2c.c index 40f3aa432fba..c097d591670e 100644 --- a/arch/powerpc/platforms/powermac/low_i2c.c +++ b/arch/powerpc/platforms/powermac/low_i2c.c @@ -925,8 +925,10 @@ static void __init smu_i2c_probe(void) sz = sizeof(struct pmac_i2c_bus) + sizeof(struct smu_i2c_cmd); bus = kzalloc(sz, GFP_KERNEL); - if (bus == NULL) + if (bus == NULL) { + of_node_put(busnode); return; + } bus->controller = controller; bus->busnode = of_node_get(busnode); diff --git a/arch/powerpc/platforms/powermac/smp.c b/arch/powerpc/platforms/powermac/smp.c index 8be71920e63c..c83d1e14077e 100644 --- a/arch/powerpc/platforms/powermac/smp.c +++ b/arch/powerpc/platforms/powermac/smp.c @@ -598,8 +598,10 @@ static void __init smp_core99_setup_i2c_hwsync(int ncpus) name = "Pulsar"; break; } - if (pmac_tb_freeze != NULL) + if (pmac_tb_freeze != NULL) { + of_node_put(cc); break; + } } if (pmac_tb_freeze != NULL) { /* Open i2c bus for synchronous access */ From 06b627c1236216ac1239c5e1afcc75359af3fb72 Mon Sep 17 00:00:00 2001 From: Julia Lawall Date: Thu, 7 Sep 2023 11:55:16 +0200 Subject: [PATCH 098/475] powerpc/kexec_file: add missing of_node_put for_each_node_with_property performs an of_node_get on each iteration, so a break out of the loop requires an of_node_put. This was done using the Coccinelle semantic patch iterators/for_each_child.cocci Signed-off-by: Julia Lawall Signed-off-by: Michael Ellerman Link: https://msgid.link/20230907095521.14053-7-Julia.Lawall@inria.fr --- arch/powerpc/kexec/file_load_64.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/arch/powerpc/kexec/file_load_64.c b/arch/powerpc/kexec/file_load_64.c index a3de5369d22c..09187aca3d1f 100644 --- a/arch/powerpc/kexec/file_load_64.c +++ b/arch/powerpc/kexec/file_load_64.c @@ -1138,11 +1138,15 @@ static int update_pci_dma_nodes(void *fdt, const char *dmapropname) continue; ret = copy_property(fdt, pci_offset, dn, "ibm,dma-window"); - if (ret < 0) + if (ret < 0) { + of_node_put(dn); break; + } ret = copy_property(fdt, pci_offset, dn, dmapropname); - if (ret < 0) + if (ret < 0) { + of_node_put(dn); break; + } } return ret; From a3ef2fef198c25c1d9ac6ff89fd50230e9507207 Mon Sep 17 00:00:00 2001 From: Yuan Tan Date: Thu, 14 Sep 2023 18:10:31 +0800 Subject: [PATCH 099/475] powerpc/32: Add dependencies of POWER_RESET for pmac32 pmac32's power off depends on ADB_CUDA to work. Enable it when POWER_RESET is set for convenience. Suggested-by: Zhangjin Wu Signed-off-by: Yuan Tan Signed-off-by: Michael Ellerman Link: https://msgid.link/0cca5d5afb6c4a1b78648e98339b4b7c9def46d5.1694685860.git.tanyuan@tinylab.org --- arch/powerpc/platforms/powermac/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/powerpc/platforms/powermac/Kconfig b/arch/powerpc/platforms/powermac/Kconfig index 130707ec9f99..8bdae0caf21e 100644 --- a/arch/powerpc/platforms/powermac/Kconfig +++ b/arch/powerpc/platforms/powermac/Kconfig @@ -2,6 +2,7 @@ config PPC_PMAC bool "Apple PowerMac based machines" depends on PPC_BOOK3S && CPU_BIG_ENDIAN + select ADB_CUDA if POWER_RESET && PPC32 select MPIC select FORCE_PCI select PPC_INDIRECT_PCI if PPC32 From f84b727d132c39c70208503e60149af6dd5a217f Mon Sep 17 00:00:00 2001 From: Yuan Tan Date: Thu, 14 Sep 2023 18:10:42 +0800 Subject: [PATCH 100/475] powerpc/32: Enable POWER_RESET in pmac32_defconfig MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enable POWER_RESET, and drop ADB_CUDA as it will now be selected. Suggested-by: Philippe Mathieu-Daudé Suggested-by: Christophe Leroy Signed-off-by: Yuan Tan Signed-off-by: Michael Ellerman Link: https://msgid.link/73716a14e9a24bcea4ca6c454aae308c419933c6.1694685860.git.tanyuan@tinylab.org --- arch/powerpc/configs/pmac32_defconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/powerpc/configs/pmac32_defconfig b/arch/powerpc/configs/pmac32_defconfig index a205da9ee5f2..57ded82c2840 100644 --- a/arch/powerpc/configs/pmac32_defconfig +++ b/arch/powerpc/configs/pmac32_defconfig @@ -138,7 +138,6 @@ CONFIG_DM_SNAPSHOT=m CONFIG_DM_MIRROR=m CONFIG_DM_ZERO=m CONFIG_ADB=y -CONFIG_ADB_CUDA=y CONFIG_ADB_PMU=y CONFIG_ADB_PMU_LED=y CONFIG_ADB_PMU_LED_DISK=y @@ -181,6 +180,7 @@ CONFIG_SERIAL_PMACZILOG_TTYS=y CONFIG_SERIAL_PMACZILOG_CONSOLE=y CONFIG_NVRAM=y CONFIG_I2C_CHARDEV=m +CONFIG_POWER_RESET=y CONFIG_APM_POWER=y CONFIG_BATTERY_PMU=y CONFIG_HWMON=m From 7135b921b32966d7602ede396b7286d372aee63f Mon Sep 17 00:00:00 2001 From: Aditya Gupta Date: Mon, 11 Sep 2023 14:44:09 +0530 Subject: [PATCH 101/475] powerpc: add `cur_cpu_spec` symbol to vmcoreinfo Presently, while reading a vmcore, makedumpfile uses `cur_cpu_spec.mmu_features` to decide whether the crashed system had RADIX MMU or not. Currently, makedumpfile fails to get the `cur_cpu_spec` symbol (unless a vmlinux is passed with the `-x` flag to makedumpfile), and hence assigns offsets and shifts (such as pgd_offset_l4) incorrecly considering MMU to be hash MMU. Add `cur_cpu_spec` symbol and offset of `mmu_features` in the `cpu_spec` struct, to VMCOREINFO, so that the symbol address and offset is accessible to makedumpfile, without needing the vmlinux file Signed-off-by: Aditya Gupta Reported-by: Sachin Sant Tested-by: Sachin Sant Signed-off-by: Michael Ellerman Link: https://msgid.link/20230911091409.415662-1-adityag@linux.ibm.com --- arch/powerpc/kexec/core.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/powerpc/kexec/core.c b/arch/powerpc/kexec/core.c index de64c7962991..369b8334a4f0 100644 --- a/arch/powerpc/kexec/core.c +++ b/arch/powerpc/kexec/core.c @@ -63,6 +63,8 @@ void arch_crash_save_vmcoreinfo(void) #ifndef CONFIG_NUMA VMCOREINFO_SYMBOL(contig_page_data); #endif + VMCOREINFO_SYMBOL(cur_cpu_spec); + VMCOREINFO_OFFSET(cpu_spec, mmu_features); #if defined(CONFIG_PPC64) && defined(CONFIG_SPARSEMEM_VMEMMAP) VMCOREINFO_SYMBOL(vmemmap_list); VMCOREINFO_SYMBOL(mmu_vmemmap_psize); From 86328b338c3996b814417dd68e3f899a1a649059 Mon Sep 17 00:00:00 2001 From: Hari Bathini Date: Tue, 12 Sep 2023 13:59:49 +0530 Subject: [PATCH 102/475] vmcore: remove dependency with is_kdump_kernel() for exporting vmcore Currently, is_kdump_kernel() returns true when elfcorehdr_addr is set. While elfcorehdr_addr is set for kexec based kernel dump mechanism, alternate dump capturing methods like fadump [1] also set it to export the vmcore. Since, is_kdump_kernel() is used to restrict resources in crash dump capture kernel and such restrictions may not be desirable for fadump, allow is_kdump_kernel() to be defined differently for such scenarios. With this, is_kdump_kernel() could be false while vmcore is usable. So, remove unnecessary dependency with is_kdump_kernel(), for exporting vmcore. [1] https://docs.kernel.org/powerpc/firmware-assisted-dump.html Suggested-by: Michael Ellerman Signed-off-by: Hari Bathini Acked-by: Baoquan He Signed-off-by: Michael Ellerman Link: https://msgid.link/20230912082950.856977-1-hbathini@linux.ibm.com --- include/linux/crash_dump.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/include/linux/crash_dump.h b/include/linux/crash_dump.h index 0f3a656293b0..acc55626afdc 100644 --- a/include/linux/crash_dump.h +++ b/include/linux/crash_dump.h @@ -50,6 +50,7 @@ void vmcore_cleanup(void); #define vmcore_elf64_check_arch(x) (elf_check_arch(x) || vmcore_elf_check_arch_cross(x)) #endif +#ifndef is_kdump_kernel /* * is_kdump_kernel() checks whether this kernel is booting after a panic of * previous kernel or not. This is determined by checking if previous kernel @@ -64,6 +65,7 @@ static inline bool is_kdump_kernel(void) { return elfcorehdr_addr != ELFCORE_ADDR_MAX; } +#endif /* is_vmcore_usable() checks if the kernel is booting after a panic and * the vmcore region is usable. @@ -75,7 +77,8 @@ static inline bool is_kdump_kernel(void) static inline int is_vmcore_usable(void) { - return is_kdump_kernel() && elfcorehdr_addr != ELFCORE_ADDR_ERR ? 1 : 0; + return elfcorehdr_addr != ELFCORE_ADDR_ERR && + elfcorehdr_addr != ELFCORE_ADDR_MAX ? 1 : 0; } /* vmcore_unusable() marks the vmcore as unusable, @@ -84,8 +87,7 @@ static inline int is_vmcore_usable(void) static inline void vmcore_unusable(void) { - if (is_kdump_kernel()) - elfcorehdr_addr = ELFCORE_ADDR_ERR; + elfcorehdr_addr = ELFCORE_ADDR_ERR; } /** From b098f1c32365304633077d73e4ae21c72d4241b3 Mon Sep 17 00:00:00 2001 From: Hari Bathini Date: Tue, 12 Sep 2023 13:59:50 +0530 Subject: [PATCH 103/475] powerpc/fadump: make is_kdump_kernel() return false when fadump is active Currently, is_kdump_kernel() returns true in crash dump capture kernel for both kdump and fadump crash dump capturing methods, as both these methods set elfcorehdr_addr. Some restrictions enforced for crash dump capture kernel, based on is_kdump_kernel(), are specifically meant for kdump case and not desirable for fadump - eg. IO queues restriction in device drivers. So, define is_kdump_kernel() to return false when f/w assisted dump is active. Signed-off-by: Hari Bathini Acked-by: Baoquan He Signed-off-by: Michael Ellerman Link: https://msgid.link/20230912082950.856977-2-hbathini@linux.ibm.com --- arch/powerpc/include/asm/kexec.h | 8 ++++++-- arch/powerpc/kernel/crash_dump.c | 12 ++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/arch/powerpc/include/asm/kexec.h b/arch/powerpc/include/asm/kexec.h index a1ddba01e7d1..e1b43aa12175 100644 --- a/arch/powerpc/include/asm/kexec.h +++ b/arch/powerpc/include/asm/kexec.h @@ -99,10 +99,14 @@ void relocate_new_kernel(unsigned long indirection_page, unsigned long reboot_co void kexec_copy_flush(struct kimage *image); -#if defined(CONFIG_CRASH_DUMP) && defined(CONFIG_PPC_RTAS) +#if defined(CONFIG_CRASH_DUMP) +bool is_kdump_kernel(void); +#define is_kdump_kernel is_kdump_kernel +#if defined(CONFIG_PPC_RTAS) void crash_free_reserved_phys_range(unsigned long begin, unsigned long end); #define crash_free_reserved_phys_range crash_free_reserved_phys_range -#endif +#endif /* CONFIG_PPC_RTAS */ +#endif /* CONFIG_CRASH_DUMP */ #ifdef CONFIG_KEXEC_FILE extern const struct kexec_file_ops kexec_elf64_ops; diff --git a/arch/powerpc/kernel/crash_dump.c b/arch/powerpc/kernel/crash_dump.c index 9a3b85bfc83f..2086fa6cdc25 100644 --- a/arch/powerpc/kernel/crash_dump.c +++ b/arch/powerpc/kernel/crash_dump.c @@ -19,6 +19,7 @@ #include #include #include +#include #ifdef DEBUG #include @@ -92,6 +93,17 @@ ssize_t copy_oldmem_page(struct iov_iter *iter, unsigned long pfn, return csize; } +/* + * Return true only when kexec based kernel dump capturing method is used. + * This ensures all restritions applied for kdump case are not automatically + * applied for fadump case. + */ +bool is_kdump_kernel(void) +{ + return !is_fadump_active() && elfcorehdr_addr != ELFCORE_ADDR_MAX; +} +EXPORT_SYMBOL_GPL(is_kdump_kernel); + #ifdef CONFIG_PPC_RTAS /* * The crashkernel region will almost always overlap the RTAS region, so From ff25ad0aa280012eda5699713a9f0b468a1d6e4e Mon Sep 17 00:00:00 2001 From: Benjamin Gray Date: Wed, 30 Aug 2023 14:42:38 +1000 Subject: [PATCH 104/475] powerpc/configs: Set more PPC debug configs Add more config options that wouldn't be done by the generic debug config in kernel/configs/debug.config CONFIG_JUMP_LABEL_FEATURE_CHECK_DEBUG Adds an initialized check on each (cpu|mmu)_has_feature() CONFIG_PPC_IRQ_SOFT_MASK_DEBUG Adds some extra checks around IRQ mask manipulation CONFIG_PPC_KUAP_DEBUG Adds some extra KAUP checks around interrupts/context switching CONFIG_PPC_RFI_SRR_DEBUG Adds some extra SSR checks around interrupts/syscalls Signed-off-by: Benjamin Gray Signed-off-by: Michael Ellerman Link: https://msgid.link/20230830044238.578840-1-bgray@linux.ibm.com --- arch/powerpc/configs/debug.config | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/powerpc/configs/debug.config b/arch/powerpc/configs/debug.config index a14ae1f20d60..bcc1fcf25e10 100644 --- a/arch/powerpc/configs/debug.config +++ b/arch/powerpc/configs/debug.config @@ -1 +1,5 @@ +CONFIG_JUMP_LABEL_FEATURE_CHECK_DEBUG=y +CONFIG_PPC_IRQ_SOFT_MASK_DEBUG=y +CONFIG_PPC_KUAP_DEBUG=y +CONFIG_PPC_RFI_SRR_DEBUG=y CONFIG_SCOM_DEBUGFS=y From 6b93bb41f6eaa1cc5c5f30ec3b687b380f116cd0 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Sat, 16 Sep 2023 21:26:00 +0206 Subject: [PATCH 105/475] printk: Add non-BKL (nbcon) console basic infrastructure The current console/printk subsystem is protected by a Big Kernel Lock, (aka console_lock) which has ill defined semantics and is more or less stateless. This puts severe limitations on the console subsystem and makes forced takeover and output in emergency and panic situations a fragile endeavour that is based on try and pray. The goal of non-BKL (nbcon) consoles is to break out of the console lock jail and to provide a new infrastructure that avoids the pitfalls and also allows console drivers to be gradually converted over. The proposed infrastructure aims for the following properties: - Per console locking instead of global locking - Per console state that allows to make informed decisions - Stateful handover and takeover As a first step, state is added to struct console. The per console state is an atomic_t using a 32bit bit field. Reserve state bits, which will be populated later in the series. Wire it up into the console register/unregister functionality. It was decided to use a bitfield because using a plain u32 with mask/shift operations resulted in uncomprehensible code. Co-developed-by: John Ogness Signed-off-by: John Ogness Signed-off-by: Thomas Gleixner (Intel) Reviewed-by: Petr Mladek Signed-off-by: Petr Mladek Link: https://lore.kernel.org/r/20230916192007.608398-2-john.ogness@linutronix.de --- include/linux/console.h | 31 ++++++++++++++++++ kernel/printk/Makefile | 2 +- kernel/printk/internal.h | 8 +++++ kernel/printk/nbcon.c | 70 ++++++++++++++++++++++++++++++++++++++++ kernel/printk/printk.c | 13 ++++++-- 5 files changed, 120 insertions(+), 4 deletions(-) create mode 100644 kernel/printk/nbcon.c diff --git a/include/linux/console.h b/include/linux/console.h index 7de11c763eb3..a2d37a7a98a8 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -156,6 +156,8 @@ static inline int con_debug_leave(void) * /dev/kmesg which requires a larger output buffer. * @CON_SUSPENDED: Indicates if a console is suspended. If true, the * printing callbacks must not be called. + * @CON_NBCON: Console can operate outside of the legacy style console_lock + * constraints. */ enum cons_flags { CON_PRINTBUFFER = BIT(0), @@ -166,8 +168,32 @@ enum cons_flags { CON_BRL = BIT(5), CON_EXTENDED = BIT(6), CON_SUSPENDED = BIT(7), + CON_NBCON = BIT(8), }; +/** + * struct nbcon_state - console state for nbcon consoles + * @atom: Compound of the state fields for atomic operations + * + * To be used for reading and preparing of the value stored in the nbcon + * state variable @console::nbcon_state. + */ +struct nbcon_state { + union { + unsigned int atom; + struct { + }; + }; +}; + +/* + * The nbcon_state struct is used to easily create and interpret values that + * are stored in the @console::nbcon_state variable. Ensure this struct stays + * within the size boundaries of the atomic variable's underlying type in + * order to avoid any accidental truncation. + */ +static_assert(sizeof(struct nbcon_state) <= sizeof(int)); + /** * struct console - The console descriptor structure * @name: The name of the console driver @@ -187,6 +213,8 @@ enum cons_flags { * @dropped: Number of unreported dropped ringbuffer records * @data: Driver private data * @node: hlist node for the console list + * + * @nbcon_state: State for nbcon consoles */ struct console { char name[16]; @@ -206,6 +234,9 @@ struct console { unsigned long dropped; void *data; struct hlist_node node; + + /* nbcon console specific members */ + atomic_t __private nbcon_state; }; #ifdef CONFIG_LOCKDEP diff --git a/kernel/printk/Makefile b/kernel/printk/Makefile index f5b388e810b9..39a2b61c7232 100644 --- a/kernel/printk/Makefile +++ b/kernel/printk/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only obj-y = printk.o -obj-$(CONFIG_PRINTK) += printk_safe.o +obj-$(CONFIG_PRINTK) += printk_safe.o nbcon.o obj-$(CONFIG_A11Y_BRAILLE_CONSOLE) += braille.o obj-$(CONFIG_PRINTK_INDEX) += index.o diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h index 7d4979d5c3ce..2ca0ab78802c 100644 --- a/kernel/printk/internal.h +++ b/kernel/printk/internal.h @@ -3,6 +3,7 @@ * internal.h - printk internal definitions */ #include +#include #if defined(CONFIG_PRINTK) && defined(CONFIG_SYSCTL) void __init printk_sysctl_init(void); @@ -61,6 +62,10 @@ void defer_console_output(void); u16 printk_parse_prefix(const char *text, int *level, enum printk_info_flags *flags); + +void nbcon_init(struct console *con); +void nbcon_cleanup(struct console *con); + #else #define PRINTK_PREFIX_MAX 0 @@ -76,6 +81,9 @@ u16 printk_parse_prefix(const char *text, int *level, #define printk_safe_exit_irqrestore(flags) local_irq_restore(flags) static inline bool printk_percpu_data_ready(void) { return false; } +static inline void nbcon_init(struct console *con) { } +static inline void nbcon_cleanup(struct console *con) { } + #endif /* CONFIG_PRINTK */ /** diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c new file mode 100644 index 000000000000..63d24ca62ac5 --- /dev/null +++ b/kernel/printk/nbcon.c @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2022 Linutronix GmbH, John Ogness +// Copyright (C) 2022 Intel, Thomas Gleixner + +#include +#include +#include "internal.h" +/* + * Printk console printing implementation for consoles which does not depend + * on the legacy style console_lock mechanism. + */ + +/** + * nbcon_state_set - Helper function to set the console state + * @con: Console to update + * @new: The new state to write + * + * Only to be used when the console is not yet or no longer visible in the + * system. Otherwise use nbcon_state_try_cmpxchg(). + */ +static inline void nbcon_state_set(struct console *con, struct nbcon_state *new) +{ + atomic_set(&ACCESS_PRIVATE(con, nbcon_state), new->atom); +} + +/** + * nbcon_state_read - Helper function to read the console state + * @con: Console to read + * @state: The state to store the result + */ +static inline void nbcon_state_read(struct console *con, struct nbcon_state *state) +{ + state->atom = atomic_read(&ACCESS_PRIVATE(con, nbcon_state)); +} + +/** + * nbcon_state_try_cmpxchg() - Helper function for atomic_try_cmpxchg() on console state + * @con: Console to update + * @cur: Old/expected state + * @new: New state + * + * Return: True on success. False on fail and @cur is updated. + */ +static inline bool nbcon_state_try_cmpxchg(struct console *con, struct nbcon_state *cur, + struct nbcon_state *new) +{ + return atomic_try_cmpxchg(&ACCESS_PRIVATE(con, nbcon_state), &cur->atom, new->atom); +} + +/** + * nbcon_init - Initialize the nbcon console specific data + * @con: Console to initialize + */ +void nbcon_init(struct console *con) +{ + struct nbcon_state state = { }; + + nbcon_state_set(con, &state); +} + +/** + * nbcon_cleanup - Cleanup the nbcon console specific data + * @con: Console to cleanup + */ +void nbcon_cleanup(struct console *con) +{ + struct nbcon_state state = { }; + + nbcon_state_set(con, &state); +} diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 7e0b4dd02398..5f372eaceb29 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -3326,9 +3326,10 @@ static void try_enable_default_console(struct console *newcon) newcon->flags |= CON_CONSDEV; } -#define con_printk(lvl, con, fmt, ...) \ - printk(lvl pr_fmt("%sconsole [%s%d] " fmt), \ - (con->flags & CON_BOOT) ? "boot" : "", \ +#define con_printk(lvl, con, fmt, ...) \ + printk(lvl pr_fmt("%s%sconsole [%s%d] " fmt), \ + (con->flags & CON_NBCON) ? "" : "legacy ", \ + (con->flags & CON_BOOT) ? "boot" : "", \ con->name, con->index, ##__VA_ARGS__) static void console_init_seq(struct console *newcon, bool bootcon_registered) @@ -3488,6 +3489,9 @@ void register_console(struct console *newcon) newcon->dropped = 0; console_init_seq(newcon, bootcon_registered); + if (newcon->flags & CON_NBCON) + nbcon_init(newcon); + /* * Put this console in the list - keep the * preferred driver at the head of the list. @@ -3579,6 +3583,9 @@ static int unregister_console_locked(struct console *console) */ synchronize_srcu(&console_srcu); + if (console->flags & CON_NBCON) + nbcon_cleanup(console); + console_sysfs_notify(); if (console->exit) From 3a5bb25162b880da749f7cdf281c78dbade4164b Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Sat, 16 Sep 2023 21:26:01 +0206 Subject: [PATCH 106/475] printk: nbcon: Add acquire/release logic Add per console acquire/release functionality. The state of the console is maintained in the "nbcon_state" atomic variable. The console is locked when: - The 'prio' field contains the priority of the context that owns the console. Only higher priority contexts are allowed to take over the lock. A value of 0 (NBCON_PRIO_NONE) means the console is not locked. - The 'cpu' field denotes on which CPU the console is locked. It is used to prevent busy waiting on the same CPU. Also it informs the lock owner that it has lost the lock in a more complex scenario when the lock was taken over by a higher priority context, released, and taken on another CPU with the same priority as the interrupted owner. The acquire mechanism uses a few more fields: - The 'req_prio' field is used by the handover approach to make the current owner aware that there is a context with a higher priority waiting for the friendly handover. - The 'unsafe' field allows to take over the console in a safe way in the middle of emitting a message. The field is set only when accessing some shared resources or when the console device is manipulated. It can be cleared, for example, after emitting one character when the console device is in a consistent state. - The 'unsafe_takeover' field is set when a hostile takeover took the console in an unsafe state. The console will stay in the unsafe state until re-initialized. The acquire mechanism uses three approaches: 1) Direct acquire when the console is not owned or is owned by a lower priority context and is in a safe state. 2) Friendly handover mechanism uses a request/grant handshake. It is used when the current owner has lower priority and the console is in an unsafe state. The requesting context: a) Sets its priority into the 'req_prio' field. b) Waits (with a timeout) for the owning context to unlock the console. c) Takes the lock and clears the 'req_prio' field. The owning context: a) Observes the 'req_prio' field set on exit from the unsafe console state. b) Gives up console ownership by clearing the 'prio' field. 3) Unsafe hostile takeover allows to take over the lock even when the console is an unsafe state. It is used only in panic() by the final attempt to flush consoles in a try and hope mode. Note that separate record buffers are used in panic(). As a result, the messages can be read and formatted without any risk even after using the hostile takeover in unsafe state. The release function simply clears the 'prio' field. All operations on @console::nbcon_state are atomic cmpxchg based to handle concurrency. The acquire/release functions implement only minimal policies: - Preference for higher priority contexts. - Protection of the panic CPU. All other policy decisions must be made at the call sites: - What is marked as an unsafe section. - Whether to spin-wait if there is already an owner and the console is in an unsafe state. - Whether to attempt an unsafe hostile takeover. The design allows to implement the well known: acquire() output_one_printk_record() release() The output of one printk record might be interrupted with a higher priority context. The new owner is supposed to reprint the entire interrupted record from scratch. Co-developed-by: John Ogness Signed-off-by: John Ogness Signed-off-by: Thomas Gleixner (Intel) Signed-off-by: Petr Mladek Link: https://lore.kernel.org/r/20230916192007.608398-3-john.ogness@linutronix.de --- include/linux/console.h | 56 +++++ kernel/printk/nbcon.c | 497 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 553 insertions(+) diff --git a/include/linux/console.h b/include/linux/console.h index a2d37a7a98a8..98210fd01f18 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -175,13 +175,29 @@ enum cons_flags { * struct nbcon_state - console state for nbcon consoles * @atom: Compound of the state fields for atomic operations * + * @req_prio: The priority of a handover request + * @prio: The priority of the current owner + * @unsafe: Console is busy in a non takeover region + * @unsafe_takeover: A hostile takeover in an unsafe state happened in the + * past. The console cannot be safe until re-initialized. + * @cpu: The CPU on which the owner runs + * * To be used for reading and preparing of the value stored in the nbcon * state variable @console::nbcon_state. + * + * The @prio and @req_prio fields are particularly important to allow + * spin-waiting to timeout and give up without the risk of a waiter being + * assigned the lock after giving up. */ struct nbcon_state { union { unsigned int atom; struct { + unsigned int prio : 2; + unsigned int req_prio : 2; + unsigned int unsafe : 1; + unsigned int unsafe_takeover : 1; + unsigned int cpu : 24; }; }; }; @@ -194,6 +210,46 @@ struct nbcon_state { */ static_assert(sizeof(struct nbcon_state) <= sizeof(int)); +/** + * nbcon_prio - console owner priority for nbcon consoles + * @NBCON_PRIO_NONE: Unused + * @NBCON_PRIO_NORMAL: Normal (non-emergency) usage + * @NBCON_PRIO_EMERGENCY: Emergency output (WARN/OOPS...) + * @NBCON_PRIO_PANIC: Panic output + * @NBCON_PRIO_MAX: The number of priority levels + * + * A higher priority context can takeover the console when it is + * in the safe state. The final attempt to flush consoles in panic() + * can be allowed to do so even in an unsafe state (Hope and pray). + */ +enum nbcon_prio { + NBCON_PRIO_NONE = 0, + NBCON_PRIO_NORMAL, + NBCON_PRIO_EMERGENCY, + NBCON_PRIO_PANIC, + NBCON_PRIO_MAX, +}; + +struct console; + +/** + * struct nbcon_context - Context for console acquire/release + * @console: The associated console + * @spinwait_max_us: Limit for spin-wait acquire + * @prio: Priority of the context + * @allow_unsafe_takeover: Allow performing takeover even if unsafe. Can + * be used only with NBCON_PRIO_PANIC @prio. It + * might cause a system freeze when the console + * is used later. + */ +struct nbcon_context { + /* members set by caller */ + struct console *console; + unsigned int spinwait_max_us; + enum nbcon_prio prio; + unsigned int allow_unsafe_takeover : 1; +}; + /** * struct console - The console descriptor structure * @name: The name of the console driver diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index 63d24ca62ac5..a2a354f859f9 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -4,10 +4,98 @@ #include #include +#include #include "internal.h" /* * Printk console printing implementation for consoles which does not depend * on the legacy style console_lock mechanism. + * + * The state of the console is maintained in the "nbcon_state" atomic + * variable. + * + * The console is locked when: + * + * - The 'prio' field contains the priority of the context that owns the + * console. Only higher priority contexts are allowed to take over the + * lock. A value of 0 (NBCON_PRIO_NONE) means the console is not locked. + * + * - The 'cpu' field denotes on which CPU the console is locked. It is used + * to prevent busy waiting on the same CPU. Also it informs the lock owner + * that it has lost the lock in a more complex scenario when the lock was + * taken over by a higher priority context, released, and taken on another + * CPU with the same priority as the interrupted owner. + * + * The acquire mechanism uses a few more fields: + * + * - The 'req_prio' field is used by the handover approach to make the + * current owner aware that there is a context with a higher priority + * waiting for the friendly handover. + * + * - The 'unsafe' field allows to take over the console in a safe way in the + * middle of emitting a message. The field is set only when accessing some + * shared resources or when the console device is manipulated. It can be + * cleared, for example, after emitting one character when the console + * device is in a consistent state. + * + * - The 'unsafe_takeover' field is set when a hostile takeover took the + * console in an unsafe state. The console will stay in the unsafe state + * until re-initialized. + * + * The acquire mechanism uses three approaches: + * + * 1) Direct acquire when the console is not owned or is owned by a lower + * priority context and is in a safe state. + * + * 2) Friendly handover mechanism uses a request/grant handshake. It is used + * when the current owner has lower priority and the console is in an + * unsafe state. + * + * The requesting context: + * + * a) Sets its priority into the 'req_prio' field. + * + * b) Waits (with a timeout) for the owning context to unlock the + * console. + * + * c) Takes the lock and clears the 'req_prio' field. + * + * The owning context: + * + * a) Observes the 'req_prio' field set on exit from the unsafe + * console state. + * + * b) Gives up console ownership by clearing the 'prio' field. + * + * 3) Unsafe hostile takeover allows to take over the lock even when the + * console is an unsafe state. It is used only in panic() by the final + * attempt to flush consoles in a try and hope mode. + * + * The release function simply clears the 'prio' field. + * + * All operations on @console::nbcon_state are atomic cmpxchg based to + * handle concurrency. + * + * The acquire/release functions implement only minimal policies: + * + * - Preference for higher priority contexts. + * - Protection of the panic CPU. + * + * All other policy decisions must be made at the call sites: + * + * - What is marked as an unsafe section. + * - Whether to spin-wait if there is already an owner and the console is + * in an unsafe state. + * - Whether to attempt an unsafe hostile takeover. + * + * The design allows to implement the well known: + * + * acquire() + * output_one_printk_record() + * release() + * + * The output of one printk record might be interrupted with a higher priority + * context. The new owner is supposed to reprint the entire interrupted record + * from scratch. */ /** @@ -47,6 +135,415 @@ static inline bool nbcon_state_try_cmpxchg(struct console *con, struct nbcon_sta return atomic_try_cmpxchg(&ACCESS_PRIVATE(con, nbcon_state), &cur->atom, new->atom); } +/** + * nbcon_context_try_acquire_direct - Try to acquire directly + * @ctxt: The context of the caller + * @cur: The current console state + * + * Acquire the console when it is released. Also acquire the console when + * the current owner has a lower priority and the console is in a safe state. + * + * Return: 0 on success. Otherwise, an error code on failure. Also @cur + * is updated to the latest state when failed to modify it. + * + * Errors: + * + * -EPERM: A panic is in progress and this is not the panic CPU. + * Or the current owner or waiter has the same or higher + * priority. No acquire method can be successful in + * this case. + * + * -EBUSY: The current owner has a lower priority but the console + * in an unsafe state. The caller should try using + * the handover acquire method. + */ +static int nbcon_context_try_acquire_direct(struct nbcon_context *ctxt, + struct nbcon_state *cur) +{ + unsigned int cpu = smp_processor_id(); + struct console *con = ctxt->console; + struct nbcon_state new; + + do { + if (other_cpu_in_panic()) + return -EPERM; + + if (ctxt->prio <= cur->prio || ctxt->prio <= cur->req_prio) + return -EPERM; + + if (cur->unsafe) + return -EBUSY; + + /* + * The console should never be safe for a direct acquire + * if an unsafe hostile takeover has ever happened. + */ + WARN_ON_ONCE(cur->unsafe_takeover); + + new.atom = cur->atom; + new.prio = ctxt->prio; + new.req_prio = NBCON_PRIO_NONE; + new.unsafe = cur->unsafe_takeover; + new.cpu = cpu; + + } while (!nbcon_state_try_cmpxchg(con, cur, &new)); + + return 0; +} + +static bool nbcon_waiter_matches(struct nbcon_state *cur, int expected_prio) +{ + /* + * The request context is well defined by the @req_prio because: + * + * - Only a context with a higher priority can take over the request. + * - There are only three priorities. + * - Only one CPU is allowed to request PANIC priority. + * - Lower priorities are ignored during panic() until reboot. + * + * As a result, the following scenario is *not* possible: + * + * 1. Another context with a higher priority directly takes ownership. + * 2. The higher priority context releases the ownership. + * 3. A lower priority context takes the ownership. + * 4. Another context with the same priority as this context + * creates a request and starts waiting. + */ + + return (cur->req_prio == expected_prio); +} + +/** + * nbcon_context_try_acquire_requested - Try to acquire after having + * requested a handover + * @ctxt: The context of the caller + * @cur: The current console state + * + * This is a helper function for nbcon_context_try_acquire_handover(). + * It is called when the console is in an unsafe state. The current + * owner will release the console on exit from the unsafe region. + * + * Return: 0 on success and @cur is updated to the new console state. + * Otherwise an error code on failure. + * + * Errors: + * + * -EPERM: A panic is in progress and this is not the panic CPU + * or this context is no longer the waiter. + * + * -EBUSY: The console is still locked. The caller should + * continue waiting. + * + * Note: The caller must still remove the request when an error has occurred + * except when this context is no longer the waiter. + */ +static int nbcon_context_try_acquire_requested(struct nbcon_context *ctxt, + struct nbcon_state *cur) +{ + unsigned int cpu = smp_processor_id(); + struct console *con = ctxt->console; + struct nbcon_state new; + + /* Note that the caller must still remove the request! */ + if (other_cpu_in_panic()) + return -EPERM; + + /* + * Note that the waiter will also change if there was an unsafe + * hostile takeover. + */ + if (!nbcon_waiter_matches(cur, ctxt->prio)) + return -EPERM; + + /* If still locked, caller should continue waiting. */ + if (cur->prio != NBCON_PRIO_NONE) + return -EBUSY; + + /* + * The previous owner should have never released ownership + * in an unsafe region. + */ + WARN_ON_ONCE(cur->unsafe); + + new.atom = cur->atom; + new.prio = ctxt->prio; + new.req_prio = NBCON_PRIO_NONE; + new.unsafe = cur->unsafe_takeover; + new.cpu = cpu; + + if (!nbcon_state_try_cmpxchg(con, cur, &new)) { + /* + * The acquire could fail only when it has been taken + * over by a higher priority context. + */ + WARN_ON_ONCE(nbcon_waiter_matches(cur, ctxt->prio)); + return -EPERM; + } + + /* Handover success. This context now owns the console. */ + return 0; +} + +/** + * nbcon_context_try_acquire_handover - Try to acquire via handover + * @ctxt: The context of the caller + * @cur: The current console state + * + * The function must be called only when the context has higher priority + * than the current owner and the console is in an unsafe state. + * It is the case when nbcon_context_try_acquire_direct() returns -EBUSY. + * + * The function sets "req_prio" field to make the current owner aware of + * the request. Then it waits until the current owner releases the console, + * or an even higher context takes over the request, or timeout expires. + * + * The current owner checks the "req_prio" field on exit from the unsafe + * region and releases the console. It does not touch the "req_prio" field + * so that the console stays reserved for the waiter. + * + * Return: 0 on success. Otherwise, an error code on failure. Also @cur + * is updated to the latest state when failed to modify it. + * + * Errors: + * + * -EPERM: A panic is in progress and this is not the panic CPU. + * Or a higher priority context has taken over the + * console or the handover request. + * + * -EBUSY: The current owner is on the same CPU so that the hand + * shake could not work. Or the current owner is not + * willing to wait (zero timeout). Or the console does + * not enter the safe state before timeout passed. The + * caller might still use the unsafe hostile takeover + * when allowed. + * + * -EAGAIN: @cur has changed when creating the handover request. + * The caller should retry with direct acquire. + */ +static int nbcon_context_try_acquire_handover(struct nbcon_context *ctxt, + struct nbcon_state *cur) +{ + unsigned int cpu = smp_processor_id(); + struct console *con = ctxt->console; + struct nbcon_state new; + int timeout; + int request_err = -EBUSY; + + /* + * Check that the handover is called when the direct acquire failed + * with -EBUSY. + */ + WARN_ON_ONCE(ctxt->prio <= cur->prio || ctxt->prio <= cur->req_prio); + WARN_ON_ONCE(!cur->unsafe); + + /* Handover is not possible on the same CPU. */ + if (cur->cpu == cpu) + return -EBUSY; + + /* + * Console stays unsafe after an unsafe takeover until re-initialized. + * Waiting is not going to help in this case. + */ + if (cur->unsafe_takeover) + return -EBUSY; + + /* Is the caller willing to wait? */ + if (ctxt->spinwait_max_us == 0) + return -EBUSY; + + /* + * Setup a request for the handover. The caller should try to acquire + * the console directly when the current state has been modified. + */ + new.atom = cur->atom; + new.req_prio = ctxt->prio; + if (!nbcon_state_try_cmpxchg(con, cur, &new)) + return -EAGAIN; + + cur->atom = new.atom; + + /* Wait until there is no owner and then acquire the console. */ + for (timeout = ctxt->spinwait_max_us; timeout >= 0; timeout--) { + /* On successful acquire, this request is cleared. */ + request_err = nbcon_context_try_acquire_requested(ctxt, cur); + if (!request_err) + return 0; + + /* + * If the acquire should be aborted, it must be ensured + * that the request is removed before returning to caller. + */ + if (request_err == -EPERM) + break; + + udelay(1); + + /* Re-read the state because some time has passed. */ + nbcon_state_read(con, cur); + } + + /* Timed out or aborted. Carefully remove handover request. */ + do { + /* + * No need to remove request if there is a new waiter. This + * can only happen if a higher priority context has taken over + * the console or the handover request. + */ + if (!nbcon_waiter_matches(cur, ctxt->prio)) + return -EPERM; + + /* Unset request for handover. */ + new.atom = cur->atom; + new.req_prio = NBCON_PRIO_NONE; + if (nbcon_state_try_cmpxchg(con, cur, &new)) { + /* + * Request successfully unset. Report failure of + * acquiring via handover. + */ + cur->atom = new.atom; + return request_err; + } + + /* + * Unable to remove request. Try to acquire in case + * the owner has released the lock. + */ + } while (nbcon_context_try_acquire_requested(ctxt, cur)); + + /* Lucky timing. The acquire succeeded while removing the request. */ + return 0; +} + +/** + * nbcon_context_try_acquire_hostile - Acquire via unsafe hostile takeover + * @ctxt: The context of the caller + * @cur: The current console state + * + * Acquire the console even in the unsafe state. + * + * It can be permitted by setting the 'allow_unsafe_takeover' field only + * by the final attempt to flush messages in panic(). + * + * Return: 0 on success. -EPERM when not allowed by the context. + */ +static int nbcon_context_try_acquire_hostile(struct nbcon_context *ctxt, + struct nbcon_state *cur) +{ + unsigned int cpu = smp_processor_id(); + struct console *con = ctxt->console; + struct nbcon_state new; + + if (!ctxt->allow_unsafe_takeover) + return -EPERM; + + /* Ensure caller is allowed to perform unsafe hostile takeovers. */ + if (WARN_ON_ONCE(ctxt->prio != NBCON_PRIO_PANIC)) + return -EPERM; + + /* + * Check that try_acquire_direct() and try_acquire_handover() returned + * -EBUSY in the right situation. + */ + WARN_ON_ONCE(ctxt->prio <= cur->prio || ctxt->prio <= cur->req_prio); + WARN_ON_ONCE(cur->unsafe != true); + + do { + new.atom = cur->atom; + new.cpu = cpu; + new.prio = ctxt->prio; + new.unsafe |= cur->unsafe_takeover; + new.unsafe_takeover |= cur->unsafe; + + } while (!nbcon_state_try_cmpxchg(con, cur, &new)); + + return 0; +} + +/** + * nbcon_context_try_acquire - Try to acquire nbcon console + * @ctxt: The context of the caller + * + * Return: True if the console was acquired. False otherwise. + * + * If the caller allowed an unsafe hostile takeover, on success the + * caller should check the current console state to see if it is + * in an unsafe state. Otherwise, on success the caller may assume + * the console is not in an unsafe state. + */ +__maybe_unused +static bool nbcon_context_try_acquire(struct nbcon_context *ctxt) +{ + struct console *con = ctxt->console; + struct nbcon_state cur; + int err; + + nbcon_state_read(con, &cur); +try_again: + err = nbcon_context_try_acquire_direct(ctxt, &cur); + if (err != -EBUSY) + goto out; + + err = nbcon_context_try_acquire_handover(ctxt, &cur); + if (err == -EAGAIN) + goto try_again; + if (err != -EBUSY) + goto out; + + err = nbcon_context_try_acquire_hostile(ctxt, &cur); +out: + return !err; +} + +static bool nbcon_owner_matches(struct nbcon_state *cur, int expected_cpu, + int expected_prio) +{ + /* + * Since consoles can only be acquired by higher priorities, + * owning contexts are uniquely identified by @prio. However, + * since contexts can unexpectedly lose ownership, it is + * possible that later another owner appears with the same + * priority. For this reason @cpu is also needed. + */ + + if (cur->prio != expected_prio) + return false; + + if (cur->cpu != expected_cpu) + return false; + + return true; +} + +/** + * nbcon_context_release - Release the console + * @ctxt: The nbcon context from nbcon_context_try_acquire() + */ +__maybe_unused +static void nbcon_context_release(struct nbcon_context *ctxt) +{ + unsigned int cpu = smp_processor_id(); + struct console *con = ctxt->console; + struct nbcon_state cur; + struct nbcon_state new; + + nbcon_state_read(con, &cur); + + do { + if (!nbcon_owner_matches(&cur, cpu, ctxt->prio)) + return; + + new.atom = cur.atom; + new.prio = NBCON_PRIO_NONE; + + /* + * If @unsafe_takeover is set, it is kept set so that + * the state remains permanently unsafe. + */ + new.unsafe |= cur.unsafe_takeover; + + } while (!nbcon_state_try_cmpxchg(con, &cur, &new)); +} + /** * nbcon_init - Initialize the nbcon console specific data * @con: Console to initialize From d818b56f77521ecc5e3eda71dc9b2beb3d6681e3 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Sat, 16 Sep 2023 21:26:02 +0206 Subject: [PATCH 107/475] printk: Make static printk buffers available to nbcon The nbcon boot consoles also need printk buffers that are available very early. Since the nbcon boot consoles will also be serialized by the console_lock, they can use the same static printk buffers that the legacy consoles are using. Make the legacy static printk buffers available outside of printk.c so they can be used by nbcon.c. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Signed-off-by: Petr Mladek Link: https://lore.kernel.org/r/20230916192007.608398-4-john.ogness@linutronix.de --- kernel/printk/internal.h | 2 ++ kernel/printk/printk.c | 13 +++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h index 2ca0ab78802c..7199d60bfc25 100644 --- a/kernel/printk/internal.h +++ b/kernel/printk/internal.h @@ -86,6 +86,8 @@ static inline void nbcon_cleanup(struct console *con) { } #endif /* CONFIG_PRINTK */ +extern struct printk_buffers printk_shared_pbufs; + /** * struct printk_buffers - Buffers to read/format/output printk messages. * @outbuf: After formatting, contains text to output. diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 5f372eaceb29..17def3791bc0 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -2846,6 +2846,13 @@ out: return true; } +/* + * Used as the printk buffers for non-panic, serialized console printing. + * This is for legacy (!CON_NBCON) as well as all boot (CON_BOOT) consoles. + * Its usage requires the console_lock held. + */ +struct printk_buffers printk_shared_pbufs; + /* * Print one record for the given console. The record printed is whatever * record is the next available record for the given console. @@ -2863,12 +2870,10 @@ out: */ static bool console_emit_next_record(struct console *con, bool *handover, int cookie) { - static struct printk_buffers pbufs; - bool is_extended = console_srcu_read_flags(con) & CON_EXTENDED; - char *outbuf = &pbufs.outbuf[0]; + char *outbuf = &printk_shared_pbufs.outbuf[0]; struct printk_message pmsg = { - .pbufs = &pbufs, + .pbufs = &printk_shared_pbufs, }; unsigned long flags; From 5634c90fd8553de7cbafecd048d0273690a2e84e Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Sat, 16 Sep 2023 21:26:03 +0206 Subject: [PATCH 108/475] printk: nbcon: Add buffer management In case of hostile takeovers it must be ensured that the previous owner cannot scribble over the output buffer of the emergency/panic context. This is achieved by: - Adding a global output buffer instance for the panic context. This is the only situation where hostile takeovers can occur and there is always at most 1 panic context. - Allocating an output buffer per non-boot console upon console registration. This buffer is used by the console owner when not in panic context. (For boot consoles, the existing shared global legacy output buffer is used instead. Boot console printing will be synchronized with legacy console printing.) - Choosing the appropriate buffer is handled in the acquire/release functions. Co-developed-by: John Ogness Signed-off-by: John Ogness Signed-off-by: Thomas Gleixner (Intel) Reviewed-by: Petr Mladek Signed-off-by: Petr Mladek Link: https://lore.kernel.org/r/20230916192007.608398-5-john.ogness@linutronix.de --- include/linux/console.h | 7 ++++ kernel/printk/internal.h | 12 +++++-- kernel/printk/nbcon.c | 73 +++++++++++++++++++++++++++++++++++++--- kernel/printk/printk.c | 22 +++++++----- 4 files changed, 99 insertions(+), 15 deletions(-) diff --git a/include/linux/console.h b/include/linux/console.h index 98210fd01f18..ca1ef8700e55 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -231,6 +231,7 @@ enum nbcon_prio { }; struct console; +struct printk_buffers; /** * struct nbcon_context - Context for console acquire/release @@ -241,6 +242,7 @@ struct console; * be used only with NBCON_PRIO_PANIC @prio. It * might cause a system freeze when the console * is used later. + * @pbufs: Pointer to the text buffer for this context */ struct nbcon_context { /* members set by caller */ @@ -248,6 +250,9 @@ struct nbcon_context { unsigned int spinwait_max_us; enum nbcon_prio prio; unsigned int allow_unsafe_takeover : 1; + + /* members set by acquire */ + struct printk_buffers *pbufs; }; /** @@ -271,6 +276,7 @@ struct nbcon_context { * @node: hlist node for the console list * * @nbcon_state: State for nbcon consoles + * @pbufs: Pointer to nbcon private buffer */ struct console { char name[16]; @@ -293,6 +299,7 @@ struct console { /* nbcon console specific members */ atomic_t __private nbcon_state; + struct printk_buffers *pbufs; }; #ifdef CONFIG_LOCKDEP diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h index 7199d60bfc25..f6161cd75d7d 100644 --- a/kernel/printk/internal.h +++ b/kernel/printk/internal.h @@ -13,6 +13,12 @@ int devkmsg_sysctl_set_loglvl(struct ctl_table *table, int write, #define printk_sysctl_init() do { } while (0) #endif +#define con_printk(lvl, con, fmt, ...) \ + printk(lvl pr_fmt("%s%sconsole [%s%d] " fmt), \ + (con->flags & CON_NBCON) ? "" : "legacy ", \ + (con->flags & CON_BOOT) ? "boot" : "", \ + con->name, con->index, ##__VA_ARGS__) + #ifdef CONFIG_PRINTK #ifdef CONFIG_PRINTK_CALLER @@ -63,8 +69,9 @@ void defer_console_output(void); u16 printk_parse_prefix(const char *text, int *level, enum printk_info_flags *flags); +bool nbcon_alloc(struct console *con); void nbcon_init(struct console *con); -void nbcon_cleanup(struct console *con); +void nbcon_free(struct console *con); #else @@ -81,8 +88,9 @@ void nbcon_cleanup(struct console *con); #define printk_safe_exit_irqrestore(flags) local_irq_restore(flags) static inline bool printk_percpu_data_ready(void) { return false; } +static inline bool nbcon_alloc(struct console *con) { return false; } static inline void nbcon_init(struct console *con) { } -static inline void nbcon_cleanup(struct console *con) { } +static inline void nbcon_free(struct console *con) { } #endif /* CONFIG_PRINTK */ diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index a2a354f859f9..ba1febf15db6 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -5,6 +5,7 @@ #include #include #include +#include #include "internal.h" /* * Printk console printing implementation for consoles which does not depend @@ -70,6 +71,10 @@ * console is an unsafe state. It is used only in panic() by the final * attempt to flush consoles in a try and hope mode. * + * Note that separate record buffers are used in panic(). As a result, + * the messages can be read and formatted without any risk even after + * using the hostile takeover in unsafe state. + * * The release function simply clears the 'prio' field. * * All operations on @console::nbcon_state are atomic cmpxchg based to @@ -459,6 +464,8 @@ static int nbcon_context_try_acquire_hostile(struct nbcon_context *ctxt, return 0; } +static struct printk_buffers panic_nbcon_pbufs; + /** * nbcon_context_try_acquire - Try to acquire nbcon console * @ctxt: The context of the caller @@ -473,6 +480,7 @@ static int nbcon_context_try_acquire_hostile(struct nbcon_context *ctxt, __maybe_unused static bool nbcon_context_try_acquire(struct nbcon_context *ctxt) { + unsigned int cpu = smp_processor_id(); struct console *con = ctxt->console; struct nbcon_state cur; int err; @@ -491,7 +499,18 @@ try_again: err = nbcon_context_try_acquire_hostile(ctxt, &cur); out: - return !err; + if (err) + return false; + + /* Acquire succeeded. */ + + /* Assign the appropriate buffer for this context. */ + if (atomic_read(&panic_cpu) == cpu) + ctxt->pbufs = &panic_nbcon_pbufs; + else + ctxt->pbufs = con->pbufs; + + return true; } static bool nbcon_owner_matches(struct nbcon_state *cur, int expected_cpu, @@ -530,7 +549,7 @@ static void nbcon_context_release(struct nbcon_context *ctxt) do { if (!nbcon_owner_matches(&cur, cpu, ctxt->prio)) - return; + break; new.atom = cur.atom; new.prio = NBCON_PRIO_NONE; @@ -542,26 +561,70 @@ static void nbcon_context_release(struct nbcon_context *ctxt) new.unsafe |= cur.unsafe_takeover; } while (!nbcon_state_try_cmpxchg(con, &cur, &new)); + + ctxt->pbufs = NULL; +} + +/** + * nbcon_alloc - Allocate buffers needed by the nbcon console + * @con: Console to allocate buffers for + * + * Return: True on success. False otherwise and the console cannot + * be used. + * + * This is not part of nbcon_init() because buffer allocation must + * be performed earlier in the console registration process. + */ +bool nbcon_alloc(struct console *con) +{ + if (con->flags & CON_BOOT) { + /* + * Boot console printing is synchronized with legacy console + * printing, so boot consoles can share the same global printk + * buffers. + */ + con->pbufs = &printk_shared_pbufs; + } else { + con->pbufs = kmalloc(sizeof(*con->pbufs), GFP_KERNEL); + if (!con->pbufs) { + con_printk(KERN_ERR, con, "failed to allocate printing buffer\n"); + return false; + } + } + + return true; } /** * nbcon_init - Initialize the nbcon console specific data * @con: Console to initialize + * + * nbcon_alloc() *must* be called and succeed before this function + * is called. */ void nbcon_init(struct console *con) { struct nbcon_state state = { }; + /* nbcon_alloc() must have been called and successful! */ + BUG_ON(!con->pbufs); + nbcon_state_set(con, &state); } /** - * nbcon_cleanup - Cleanup the nbcon console specific data - * @con: Console to cleanup + * nbcon_free - Free and cleanup the nbcon console specific data + * @con: Console to free/cleanup nbcon data */ -void nbcon_cleanup(struct console *con) +void nbcon_free(struct console *con) { struct nbcon_state state = { }; nbcon_state_set(con, &state); + + /* Boot consoles share global printk buffers. */ + if (!(con->flags & CON_BOOT)) + kfree(con->pbufs); + + con->pbufs = NULL; } diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 17def3791bc0..1c9720acd960 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -3331,12 +3331,6 @@ static void try_enable_default_console(struct console *newcon) newcon->flags |= CON_CONSDEV; } -#define con_printk(lvl, con, fmt, ...) \ - printk(lvl pr_fmt("%s%sconsole [%s%d] " fmt), \ - (con->flags & CON_NBCON) ? "" : "legacy ", \ - (con->flags & CON_BOOT) ? "boot" : "", \ - con->name, con->index, ##__VA_ARGS__) - static void console_init_seq(struct console *newcon, bool bootcon_registered) { struct console *con; @@ -3450,6 +3444,15 @@ void register_console(struct console *newcon) goto unlock; } + if (newcon->flags & CON_NBCON) { + /* + * Ensure the nbcon console buffers can be allocated + * before modifying any global data. + */ + if (!nbcon_alloc(newcon)) + goto unlock; + } + /* * See if we want to enable this console driver by default. * @@ -3477,8 +3480,11 @@ void register_console(struct console *newcon) err = try_enable_preferred_console(newcon, false); /* printk() messages are not printed to the Braille console. */ - if (err || newcon->flags & CON_BRL) + if (err || newcon->flags & CON_BRL) { + if (newcon->flags & CON_NBCON) + nbcon_free(newcon); goto unlock; + } /* * If we have a bootconsole, and are switching to a real console, @@ -3589,7 +3595,7 @@ static int unregister_console_locked(struct console *console) synchronize_srcu(&console_srcu); if (console->flags & CON_NBCON) - nbcon_cleanup(console); + nbcon_free(console); console_sysfs_notify(); From 4b08d9e24f50163a8bda0a702b45b06bf841d792 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Sat, 16 Sep 2023 21:26:04 +0206 Subject: [PATCH 109/475] printk: nbcon: Add ownership state functions Provide functions that are related to the safe handover mechanism and allow console drivers to dynamically specify unsafe regions: - nbcon_context_can_proceed() Invoked by a console owner to check whether a handover request is pending or whether the console has been taken over by another context. If a handover request is pending, this function will also perform the handover, thus cancelling its own ownership. - nbcon_context_enter_unsafe()/nbcon_context_exit_unsafe() Invoked by a console owner to denote that the driver is about to enter or leave a critical region where a take over is unsafe. The exit variant is the point where the current owner releases the lock for a higher priority context which asked for the friendly handover. The unsafe state is stored in the console state and allows a new context to make informed decisions whether to attempt a takeover of such a console. The unsafe state is also available to the driver so that it can make informed decisions about the required actions and possibly take a special emergency path. Co-developed-by: John Ogness Signed-off-by: John Ogness Signed-off-by: Thomas Gleixner (Intel) Reviewed-by: Petr Mladek Signed-off-by: Petr Mladek Link: https://lore.kernel.org/r/20230916192007.608398-6-john.ogness@linutronix.de --- kernel/printk/nbcon.c | 123 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 122 insertions(+), 1 deletion(-) diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index ba1febf15db6..98e4be5429f0 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -537,7 +537,6 @@ static bool nbcon_owner_matches(struct nbcon_state *cur, int expected_cpu, * nbcon_context_release - Release the console * @ctxt: The nbcon context from nbcon_context_try_acquire() */ -__maybe_unused static void nbcon_context_release(struct nbcon_context *ctxt) { unsigned int cpu = smp_processor_id(); @@ -565,6 +564,128 @@ static void nbcon_context_release(struct nbcon_context *ctxt) ctxt->pbufs = NULL; } +/** + * nbcon_context_can_proceed - Check whether ownership can proceed + * @ctxt: The nbcon context from nbcon_context_try_acquire() + * @cur: The current console state + * + * Return: True if this context still owns the console. False if + * ownership was handed over or taken. + * + * Must be invoked when entering the unsafe state to make sure that it still + * owns the lock. Also must be invoked when exiting the unsafe context + * to eventually free the lock for a higher priority context which asked + * for the friendly handover. + * + * It can be called inside an unsafe section when the console is just + * temporary in safe state instead of exiting and entering the unsafe + * state. + * + * Also it can be called in the safe context before doing an expensive + * safe operation. It does not make sense to do the operation when + * a higher priority context took the lock. + * + * When this function returns false then the calling context no longer owns + * the console and is no longer allowed to go forward. In this case it must + * back out immediately and carefully. The buffer content is also no longer + * trusted since it no longer belongs to the calling context. + */ +static bool nbcon_context_can_proceed(struct nbcon_context *ctxt, struct nbcon_state *cur) +{ + unsigned int cpu = smp_processor_id(); + + /* Make sure this context still owns the console. */ + if (!nbcon_owner_matches(cur, cpu, ctxt->prio)) + return false; + + /* The console owner can proceed if there is no waiter. */ + if (cur->req_prio == NBCON_PRIO_NONE) + return true; + + /* + * A console owner within an unsafe region is always allowed to + * proceed, even if there are waiters. It can perform a handover + * when exiting the unsafe region. Otherwise the waiter will + * need to perform an unsafe hostile takeover. + */ + if (cur->unsafe) + return true; + + /* Waiters always have higher priorities than owners. */ + WARN_ON_ONCE(cur->req_prio <= cur->prio); + + /* + * Having a safe point for take over and eventually a few + * duplicated characters or a full line is way better than a + * hostile takeover. Post processing can take care of the garbage. + * Release and hand over. + */ + nbcon_context_release(ctxt); + + /* + * It is not clear whether the waiter really took over ownership. The + * outermost callsite must make the final decision whether console + * ownership is needed for it to proceed. If yes, it must reacquire + * ownership (possibly hostile) before carefully proceeding. + * + * The calling context no longer owns the console so go back all the + * way instead of trying to implement reacquire heuristics in tons of + * places. + */ + return false; +} + +#define nbcon_context_enter_unsafe(c) __nbcon_context_update_unsafe(c, true) +#define nbcon_context_exit_unsafe(c) __nbcon_context_update_unsafe(c, false) + +/** + * __nbcon_context_update_unsafe - Update the unsafe bit in @con->nbcon_state + * @ctxt: The nbcon context from nbcon_context_try_acquire() + * @unsafe: The new value for the unsafe bit + * + * Return: True if the unsafe state was updated and this context still + * owns the console. Otherwise false if ownership was handed + * over or taken. + * + * This function allows console owners to modify the unsafe status of the + * console. + * + * When this function returns false then the calling context no longer owns + * the console and is no longer allowed to go forward. In this case it must + * back out immediately and carefully. The buffer content is also no longer + * trusted since it no longer belongs to the calling context. + * + * Internal helper to avoid duplicated code. + */ +__maybe_unused +static bool __nbcon_context_update_unsafe(struct nbcon_context *ctxt, bool unsafe) +{ + struct console *con = ctxt->console; + struct nbcon_state cur; + struct nbcon_state new; + + nbcon_state_read(con, &cur); + + do { + /* + * The unsafe bit must not be cleared if an + * unsafe hostile takeover has occurred. + */ + if (!unsafe && cur.unsafe_takeover) + goto out; + + if (!nbcon_context_can_proceed(ctxt, &cur)) + return false; + + new.atom = cur.atom; + new.unsafe = unsafe; + } while (!nbcon_state_try_cmpxchg(con, &cur, &new)); + + cur.atom = new.atom; +out: + return nbcon_context_can_proceed(ctxt, &cur); +} + /** * nbcon_alloc - Allocate buffers needed by the nbcon console * @con: Console to allocate buffers for From ad56ebd1d79b216dc147474fac89a11daf6b10df Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Sat, 16 Sep 2023 21:26:05 +0206 Subject: [PATCH 110/475] printk: nbcon: Add sequence handling Add an atomic_long_t field @nbcon_seq to the console struct to store the sequence number for nbcon consoles. For nbcon consoles this will be used instead of the non-atomic @seq field. The new field allows for safe atomic sequence number updates without requiring any locking. On 64bit systems the new field stores the full sequence number. On 32bit systems the new field stores the lower 32 bits of the sequence number, which are expanded to 64bit as needed by folding the values based on the sequence numbers available in the ringbuffer. For 32bit systems, having a 32bit representation in the console is sufficient. If a console ever gets more than 2^31 records behind the ringbuffer then this is the least of the problems. Co-developed-by: John Ogness Signed-off-by: John Ogness Signed-off-by: Thomas Gleixner (Intel) Signed-off-by: Petr Mladek Link: https://lore.kernel.org/r/20230916192007.608398-7-john.ogness@linutronix.de --- include/linux/console.h | 4 ++ kernel/printk/internal.h | 7 +++ kernel/printk/nbcon.c | 101 +++++++++++++++++++++++++++++++++++++++ kernel/printk/printk.c | 31 +++++++++--- 4 files changed, 136 insertions(+), 7 deletions(-) diff --git a/include/linux/console.h b/include/linux/console.h index ca1ef8700e55..20cd486b76ad 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -243,6 +243,7 @@ struct printk_buffers; * might cause a system freeze when the console * is used later. * @pbufs: Pointer to the text buffer for this context + * @seq: The sequence number to print for this context */ struct nbcon_context { /* members set by caller */ @@ -253,6 +254,7 @@ struct nbcon_context { /* members set by acquire */ struct printk_buffers *pbufs; + u64 seq; }; /** @@ -276,6 +278,7 @@ struct nbcon_context { * @node: hlist node for the console list * * @nbcon_state: State for nbcon consoles + * @nbcon_seq: Sequence number of the next record for nbcon to print * @pbufs: Pointer to nbcon private buffer */ struct console { @@ -299,6 +302,7 @@ struct console { /* nbcon console specific members */ atomic_t __private nbcon_state; + atomic_long_t __private nbcon_seq; struct printk_buffers *pbufs; }; diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h index f6161cd75d7d..6473f5ae4a18 100644 --- a/kernel/printk/internal.h +++ b/kernel/printk/internal.h @@ -4,6 +4,7 @@ */ #include #include +#include "printk_ringbuffer.h" #if defined(CONFIG_PRINTK) && defined(CONFIG_SYSCTL) void __init printk_sysctl_init(void); @@ -42,6 +43,8 @@ enum printk_info_flags { LOG_CONT = 8, /* text is a fragment of a continuation line */ }; +extern struct printk_ringbuffer *prb; + __printf(4, 0) int vprintk_store(int facility, int level, const struct dev_printk_info *dev_info, @@ -69,6 +72,8 @@ void defer_console_output(void); u16 printk_parse_prefix(const char *text, int *level, enum printk_info_flags *flags); +u64 nbcon_seq_read(struct console *con); +void nbcon_seq_force(struct console *con, u64 seq); bool nbcon_alloc(struct console *con); void nbcon_init(struct console *con); void nbcon_free(struct console *con); @@ -88,6 +93,8 @@ void nbcon_free(struct console *con); #define printk_safe_exit_irqrestore(flags) local_irq_restore(flags) static inline bool printk_percpu_data_ready(void) { return false; } +static inline u64 nbcon_seq_read(struct console *con) { return 0; } +static inline void nbcon_seq_force(struct console *con, u64 seq) { } static inline bool nbcon_alloc(struct console *con) { return false; } static inline void nbcon_init(struct console *con) { } static inline void nbcon_free(struct console *con) { } diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index 98e4be5429f0..e076096b31c0 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -140,6 +140,101 @@ static inline bool nbcon_state_try_cmpxchg(struct console *con, struct nbcon_sta return atomic_try_cmpxchg(&ACCESS_PRIVATE(con, nbcon_state), &cur->atom, new->atom); } +#ifdef CONFIG_64BIT + +#define __seq_to_nbcon_seq(seq) (seq) +#define __nbcon_seq_to_seq(seq) (seq) + +#else /* CONFIG_64BIT */ + +#define __seq_to_nbcon_seq(seq) ((u32)seq) + +static inline u64 __nbcon_seq_to_seq(u32 nbcon_seq) +{ + u64 seq; + u64 rb_next_seq; + + /* + * The provided sequence is only the lower 32 bits of the ringbuffer + * sequence. It needs to be expanded to 64bit. Get the next sequence + * number from the ringbuffer and fold it. + * + * Having a 32bit representation in the console is sufficient. + * If a console ever gets more than 2^31 records behind + * the ringbuffer then this is the least of the problems. + * + * Also the access to the ring buffer is always safe. + */ + rb_next_seq = prb_next_seq(prb); + seq = rb_next_seq - ((u32)rb_next_seq - nbcon_seq); + + return seq; +} + +#endif /* CONFIG_64BIT */ + +/** + * nbcon_seq_read - Read the current console sequence + * @con: Console to read the sequence of + * + * Return: Sequence number of the next record to print on @con. + */ +u64 nbcon_seq_read(struct console *con) +{ + unsigned long nbcon_seq = atomic_long_read(&ACCESS_PRIVATE(con, nbcon_seq)); + + return __nbcon_seq_to_seq(nbcon_seq); +} + +/** + * nbcon_seq_force - Force console sequence to a specific value + * @con: Console to work on + * @seq: Sequence number value to set + * + * Only to be used during init (before registration) or in extreme situations + * (such as panic with CONSOLE_REPLAY_ALL). + */ +void nbcon_seq_force(struct console *con, u64 seq) +{ + /* + * If the specified record no longer exists, the oldest available record + * is chosen. This is especially important on 32bit systems because only + * the lower 32 bits of the sequence number are stored. The upper 32 bits + * are derived from the sequence numbers available in the ringbuffer. + */ + u64 valid_seq = max_t(u64, seq, prb_first_valid_seq(prb)); + + atomic_long_set(&ACCESS_PRIVATE(con, nbcon_seq), __seq_to_nbcon_seq(valid_seq)); + + /* Clear con->seq since nbcon consoles use con->nbcon_seq instead. */ + con->seq = 0; +} + +/** + * nbcon_seq_try_update - Try to update the console sequence number + * @ctxt: Pointer to an acquire context that contains + * all information about the acquire mode + * @new_seq: The new sequence number to set + * + * @ctxt->seq is updated to the new value of @con::nbcon_seq (expanded to + * the 64bit value). This could be a different value than @new_seq if + * nbcon_seq_force() was used or the current context no longer owns the + * console. In the later case, it will stop printing anyway. + */ +__maybe_unused +static void nbcon_seq_try_update(struct nbcon_context *ctxt, u64 new_seq) +{ + unsigned long nbcon_seq = __seq_to_nbcon_seq(ctxt->seq); + struct console *con = ctxt->console; + + if (atomic_long_try_cmpxchg(&ACCESS_PRIVATE(con, nbcon_seq), &nbcon_seq, + __seq_to_nbcon_seq(new_seq))) { + ctxt->seq = new_seq; + } else { + ctxt->seq = nbcon_seq_read(con); + } +} + /** * nbcon_context_try_acquire_direct - Try to acquire directly * @ctxt: The context of the caller @@ -510,6 +605,9 @@ out: else ctxt->pbufs = con->pbufs; + /* Set the record sequence for this context to print. */ + ctxt->seq = nbcon_seq_read(ctxt->console); + return true; } @@ -722,6 +820,8 @@ bool nbcon_alloc(struct console *con) * * nbcon_alloc() *must* be called and succeed before this function * is called. + * + * This function expects that the legacy @con->seq has been set. */ void nbcon_init(struct console *con) { @@ -730,6 +830,7 @@ void nbcon_init(struct console *con) /* nbcon_alloc() must have been called and successful! */ BUG_ON(!con->pbufs); + nbcon_seq_force(con, con->seq); nbcon_state_set(con, &state); } diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 1c9720acd960..77857d2118ca 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -494,7 +494,7 @@ _DEFINE_PRINTKRB(printk_rb_static, CONFIG_LOG_BUF_SHIFT - PRB_AVGBITS, static struct printk_ringbuffer printk_rb_dynamic; -static struct printk_ringbuffer *prb = &printk_rb_static; +struct printk_ringbuffer *prb = &printk_rb_static; /* * We cannot access per-CPU data (e.g. per-CPU flush irq_work) before @@ -3168,6 +3168,7 @@ void console_flush_on_panic(enum con_flush_mode mode) if (mode == CONSOLE_REPLAY_ALL) { struct console *c; + short flags; int cookie; u64 seq; @@ -3175,11 +3176,17 @@ void console_flush_on_panic(enum con_flush_mode mode) cookie = console_srcu_read_lock(); for_each_console_srcu(c) { - /* - * This is an unsynchronized assignment, but the - * kernel is in "hope and pray" mode anyway. - */ - c->seq = seq; + flags = console_srcu_read_flags(c); + + if (flags & CON_NBCON) { + nbcon_seq_force(c, seq); + } else { + /* + * This is an unsynchronized assignment. On + * panic legacy consoles are only best effort. + */ + c->seq = seq; + } } console_srcu_read_unlock(cookie); } @@ -3750,6 +3757,7 @@ static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progre struct console *c; u64 last_diff = 0; u64 printk_seq; + short flags; int cookie; u64 diff; u64 seq; @@ -3771,6 +3779,9 @@ static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progre for_each_console_srcu(c) { if (con && con != c) continue; + + flags = console_srcu_read_flags(c); + /* * If consoles are not usable, it cannot be expected * that they make forward progress, so only increment @@ -3778,7 +3789,13 @@ static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progre */ if (!console_is_usable(c)) continue; - printk_seq = c->seq; + + if (flags & CON_NBCON) { + printk_seq = nbcon_seq_read(c); + } else { + printk_seq = c->seq; + } + if (printk_seq < seq) diff += seq - printk_seq; } From 06653d57ff283be627a2c769139d73ecc487810f Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Sat, 16 Sep 2023 21:26:06 +0206 Subject: [PATCH 111/475] printk: nbcon: Add emit function and callback function for atomic printing Implement an emit function for nbcon consoles to output printk messages. It utilizes the lockless printk_get_next_message() and console_prepend_dropped() functions to retrieve/build the output message. The emit function includes the required safety points to check for handover/takeover and calls a new write_atomic callback of the console driver to output the message. It also includes proper handling for updating the nbcon console sequence number. A new nbcon_write_context struct is introduced. This is provided to the write_atomic callback and includes only the information necessary for performing atomic writes. Co-developed-by: John Ogness Signed-off-by: John Ogness Signed-off-by: Thomas Gleixner (Intel) Reviewed-by: Petr Mladek Signed-off-by: Petr Mladek Link: https://lore.kernel.org/r/20230916192007.608398-8-john.ogness@linutronix.de --- include/linux/console.h | 21 ++++++++ kernel/printk/internal.h | 6 +++ kernel/printk/nbcon.c | 106 ++++++++++++++++++++++++++++++++++++++- kernel/printk/printk.c | 9 ++-- 4 files changed, 134 insertions(+), 8 deletions(-) diff --git a/include/linux/console.h b/include/linux/console.h index 20cd486b76ad..14563dcb34b1 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -242,6 +242,7 @@ struct printk_buffers; * be used only with NBCON_PRIO_PANIC @prio. It * might cause a system freeze when the console * is used later. + * @backlog: Ringbuffer has pending records * @pbufs: Pointer to the text buffer for this context * @seq: The sequence number to print for this context */ @@ -252,11 +253,28 @@ struct nbcon_context { enum nbcon_prio prio; unsigned int allow_unsafe_takeover : 1; + /* members set by emit */ + unsigned int backlog : 1; + /* members set by acquire */ struct printk_buffers *pbufs; u64 seq; }; +/** + * struct nbcon_write_context - Context handed to the nbcon write callbacks + * @ctxt: The core console context + * @outbuf: Pointer to the text buffer for output + * @len: Length to write + * @unsafe_takeover: If a hostile takeover in an unsafe state has occurred + */ +struct nbcon_write_context { + struct nbcon_context __private ctxt; + char *outbuf; + unsigned int len; + bool unsafe_takeover; +}; + /** * struct console - The console descriptor structure * @name: The name of the console driver @@ -277,6 +295,7 @@ struct nbcon_context { * @data: Driver private data * @node: hlist node for the console list * + * @write_atomic: Write callback for atomic context * @nbcon_state: State for nbcon consoles * @nbcon_seq: Sequence number of the next record for nbcon to print * @pbufs: Pointer to nbcon private buffer @@ -301,6 +320,8 @@ struct console { struct hlist_node node; /* nbcon console specific members */ + bool (*write_atomic)(struct console *con, + struct nbcon_write_context *wctxt); atomic_t __private nbcon_state; atomic_long_t __private nbcon_seq; struct printk_buffers *pbufs; diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h index 6473f5ae4a18..6c2afee5ef62 100644 --- a/kernel/printk/internal.h +++ b/kernel/printk/internal.h @@ -130,3 +130,9 @@ struct printk_message { }; bool other_cpu_in_panic(void); +bool printk_get_next_message(struct printk_message *pmsg, u64 seq, + bool is_extended, bool may_supress); + +#ifdef CONFIG_PRINTK +void console_prepend_dropped(struct printk_message *pmsg, unsigned long dropped); +#endif diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index e076096b31c0..6e05d263fd22 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -221,7 +221,6 @@ void nbcon_seq_force(struct console *con, u64 seq) * nbcon_seq_force() was used or the current context no longer owns the * console. In the later case, it will stop printing anyway. */ -__maybe_unused static void nbcon_seq_try_update(struct nbcon_context *ctxt, u64 new_seq) { unsigned long nbcon_seq = __seq_to_nbcon_seq(ctxt->seq); @@ -755,7 +754,6 @@ static bool nbcon_context_can_proceed(struct nbcon_context *ctxt, struct nbcon_s * * Internal helper to avoid duplicated code. */ -__maybe_unused static bool __nbcon_context_update_unsafe(struct nbcon_context *ctxt, bool unsafe) { struct console *con = ctxt->console; @@ -784,6 +782,110 @@ out: return nbcon_context_can_proceed(ctxt, &cur); } +/** + * nbcon_emit_next_record - Emit a record in the acquired context + * @wctxt: The write context that will be handed to the write function + * + * Return: True if this context still owns the console. False if + * ownership was handed over or taken. + * + * When this function returns false then the calling context no longer owns + * the console and is no longer allowed to go forward. In this case it must + * back out immediately and carefully. The buffer content is also no longer + * trusted since it no longer belongs to the calling context. If the caller + * wants to do more it must reacquire the console first. + * + * When true is returned, @wctxt->ctxt.backlog indicates whether there are + * still records pending in the ringbuffer, + */ +__maybe_unused +static bool nbcon_emit_next_record(struct nbcon_write_context *wctxt) +{ + struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt); + struct console *con = ctxt->console; + bool is_extended = console_srcu_read_flags(con) & CON_EXTENDED; + struct printk_message pmsg = { + .pbufs = ctxt->pbufs, + }; + unsigned long con_dropped; + struct nbcon_state cur; + unsigned long dropped; + bool done; + + /* + * The printk buffers are filled within an unsafe section. This + * prevents NBCON_PRIO_NORMAL and NBCON_PRIO_EMERGENCY from + * clobbering each other. + */ + + if (!nbcon_context_enter_unsafe(ctxt)) + return false; + + ctxt->backlog = printk_get_next_message(&pmsg, ctxt->seq, is_extended, true); + if (!ctxt->backlog) + return nbcon_context_exit_unsafe(ctxt); + + /* + * @con->dropped is not protected in case of an unsafe hostile + * takeover. In that situation the update can be racy so + * annotate it accordingly. + */ + con_dropped = data_race(READ_ONCE(con->dropped)); + + dropped = con_dropped + pmsg.dropped; + if (dropped && !is_extended) + console_prepend_dropped(&pmsg, dropped); + + if (!nbcon_context_exit_unsafe(ctxt)) + return false; + + /* For skipped records just update seq/dropped in @con. */ + if (pmsg.outbuf_len == 0) + goto update_con; + + /* Initialize the write context for driver callbacks. */ + wctxt->outbuf = &pmsg.pbufs->outbuf[0]; + wctxt->len = pmsg.outbuf_len; + nbcon_state_read(con, &cur); + wctxt->unsafe_takeover = cur.unsafe_takeover; + + if (con->write_atomic) { + done = con->write_atomic(con, wctxt); + } else { + nbcon_context_release(ctxt); + WARN_ON_ONCE(1); + done = false; + } + + /* If not done, the emit was aborted. */ + if (!done) + return false; + + /* + * Since any dropped message was successfully output, reset the + * dropped count for the console. + */ + dropped = 0; +update_con: + /* + * The dropped count and the sequence number are updated within an + * unsafe section. This limits update races to the panic context and + * allows the panic context to win. + */ + + if (!nbcon_context_enter_unsafe(ctxt)) + return false; + + if (dropped != con_dropped) { + /* Counterpart to the READ_ONCE() above. */ + WRITE_ONCE(con->dropped, dropped); + } + + nbcon_seq_try_update(ctxt, pmsg.seq + 1); + + return nbcon_context_exit_unsafe(ctxt); +} + /** * nbcon_alloc - Allocate buffers needed by the nbcon console * @con: Console to allocate buffers for diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 77857d2118ca..778359b21761 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -698,9 +698,6 @@ out: return len; } -static bool printk_get_next_message(struct printk_message *pmsg, u64 seq, - bool is_extended, bool may_supress); - /* /dev/kmsg - userspace message inject/listen interface */ struct devkmsg_user { atomic64_t seq; @@ -2733,7 +2730,7 @@ static void __console_unlock(void) * If @pmsg->pbufs->outbuf is modified, @pmsg->outbuf_len is updated. */ #ifdef CONFIG_PRINTK -static void console_prepend_dropped(struct printk_message *pmsg, unsigned long dropped) +void console_prepend_dropped(struct printk_message *pmsg, unsigned long dropped) { struct printk_buffers *pbufs = pmsg->pbufs; const size_t scratchbuf_sz = sizeof(pbufs->scratchbuf); @@ -2787,8 +2784,8 @@ static void console_prepend_dropped(struct printk_message *pmsg, unsigned long d * of @pmsg are valid. (See the documentation of struct printk_message * for information about the @pmsg fields.) */ -static bool printk_get_next_message(struct printk_message *pmsg, u64 seq, - bool is_extended, bool may_suppress) +bool printk_get_next_message(struct printk_message *pmsg, u64 seq, + bool is_extended, bool may_suppress) { static int panic_console_dropped; From 9757acd0a700ba4a0d16dde4ba820eb052aba1a7 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Sat, 16 Sep 2023 21:26:07 +0206 Subject: [PATCH 112/475] printk: nbcon: Allow drivers to mark unsafe regions and check state For the write_atomic callback, the console driver may have unsafe regions that need to be appropriately marked. Provide functions that accept the nbcon_write_context struct to allow for the driver to enter and exit unsafe regions. Also provide a function for drivers to check if they are still the owner of the console. Co-developed-by: John Ogness Signed-off-by: John Ogness Signed-off-by: Thomas Gleixner (Intel) Reviewed-by: Petr Mladek Signed-off-by: Petr Mladek Link: https://lore.kernel.org/r/20230916192007.608398-9-john.ogness@linutronix.de --- include/linux/console.h | 10 ++++++ kernel/printk/nbcon.c | 75 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) diff --git a/include/linux/console.h b/include/linux/console.h index 14563dcb34b1..e4fc6f7c1496 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -451,6 +451,16 @@ static inline bool console_is_registered(const struct console *con) lockdep_assert_console_list_lock_held(); \ hlist_for_each_entry(con, &console_list, node) +#ifdef CONFIG_PRINTK +extern bool nbcon_can_proceed(struct nbcon_write_context *wctxt); +extern bool nbcon_enter_unsafe(struct nbcon_write_context *wctxt); +extern bool nbcon_exit_unsafe(struct nbcon_write_context *wctxt); +#else +static inline bool nbcon_can_proceed(struct nbcon_write_context *wctxt) { return false; } +static inline bool nbcon_enter_unsafe(struct nbcon_write_context *wctxt) { return false; } +static inline bool nbcon_exit_unsafe(struct nbcon_write_context *wctxt) { return false; } +#endif + extern int console_set_on_cmdline; extern struct console *early_console; diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index 6e05d263fd22..b96077152f49 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -732,6 +732,41 @@ static bool nbcon_context_can_proceed(struct nbcon_context *ctxt, struct nbcon_s return false; } +/** + * nbcon_can_proceed - Check whether ownership can proceed + * @wctxt: The write context that was handed to the write function + * + * Return: True if this context still owns the console. False if + * ownership was handed over or taken. + * + * It is used in nbcon_enter_unsafe() to make sure that it still owns the + * lock. Also it is used in nbcon_exit_unsafe() to eventually free the lock + * for a higher priority context which asked for the friendly handover. + * + * It can be called inside an unsafe section when the console is just + * temporary in safe state instead of exiting and entering the unsafe state. + * + * Also it can be called in the safe context before doing an expensive safe + * operation. It does not make sense to do the operation when a higher + * priority context took the lock. + * + * When this function returns false then the calling context no longer owns + * the console and is no longer allowed to go forward. In this case it must + * back out immediately and carefully. The buffer content is also no longer + * trusted since it no longer belongs to the calling context. + */ +bool nbcon_can_proceed(struct nbcon_write_context *wctxt) +{ + struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt); + struct console *con = ctxt->console; + struct nbcon_state cur; + + nbcon_state_read(con, &cur); + + return nbcon_context_can_proceed(ctxt, &cur); +} +EXPORT_SYMBOL_GPL(nbcon_can_proceed); + #define nbcon_context_enter_unsafe(c) __nbcon_context_update_unsafe(c, true) #define nbcon_context_exit_unsafe(c) __nbcon_context_update_unsafe(c, false) @@ -782,6 +817,46 @@ out: return nbcon_context_can_proceed(ctxt, &cur); } +/** + * nbcon_enter_unsafe - Enter an unsafe region in the driver + * @wctxt: The write context that was handed to the write function + * + * Return: True if this context still owns the console. False if + * ownership was handed over or taken. + * + * When this function returns false then the calling context no longer owns + * the console and is no longer allowed to go forward. In this case it must + * back out immediately and carefully. The buffer content is also no longer + * trusted since it no longer belongs to the calling context. + */ +bool nbcon_enter_unsafe(struct nbcon_write_context *wctxt) +{ + struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt); + + return nbcon_context_enter_unsafe(ctxt); +} +EXPORT_SYMBOL_GPL(nbcon_enter_unsafe); + +/** + * nbcon_exit_unsafe - Exit an unsafe region in the driver + * @wctxt: The write context that was handed to the write function + * + * Return: True if this context still owns the console. False if + * ownership was handed over or taken. + * + * When this function returns false then the calling context no longer owns + * the console and is no longer allowed to go forward. In this case it must + * back out immediately and carefully. The buffer content is also no longer + * trusted since it no longer belongs to the calling context. + */ +bool nbcon_exit_unsafe(struct nbcon_write_context *wctxt) +{ + struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt); + + return nbcon_context_exit_unsafe(ctxt); +} +EXPORT_SYMBOL_GPL(nbcon_exit_unsafe); + /** * nbcon_emit_next_record - Emit a record in the acquired context * @wctxt: The write context that will be handed to the write function From 3ecf87b2d8ffabb88d1e3901d7a429fd1ecbfc6c Mon Sep 17 00:00:00 2001 From: Yang Li Date: Fri, 15 Sep 2023 14:38:32 +0800 Subject: [PATCH 113/475] perf kwork top: Simplify bool conversion ./tools/perf/util/bpf_kwork_top.c:120:53-58: WARNING: conversion to bool not needed here Signed-off-by: Yang Li Cc: bpf@vger.kernel.org Link: https://lore.kernel.org/r/20230915063832.120274-1-yang.lee@linux.alibaba.com Signed-off-by: Namhyung Kim --- tools/perf/util/bpf_kwork_top.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/perf/util/bpf_kwork_top.c b/tools/perf/util/bpf_kwork_top.c index 1a607b94f44d..035e02272790 100644 --- a/tools/perf/util/bpf_kwork_top.c +++ b/tools/perf/util/bpf_kwork_top.c @@ -117,7 +117,7 @@ kwork_class_bpf_supported_list[KWORK_CLASS_MAX] = { static bool valid_kwork_class_type(enum kwork_class_type type) { - return type >= 0 && type < KWORK_CLASS_MAX ? true : false; + return type >= 0 && type < KWORK_CLASS_MAX; } static int setup_filters(struct perf_kwork *kwork) From 727e43143737431fa973a0051e9cf644e3f110fe Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Thu, 14 Sep 2023 14:19:44 -0700 Subject: [PATCH 114/475] perf version: Add status of bpf skeletons Add status for BPF skeletons, to see if a build has them enabled: ``` $ perf version --build-options perf version 6.6.rc1.g0381ae36d1a6 dwarf: [ OFF ] # HAVE_DWARF_SUPPORT dwarf_getlocations: [ OFF ] # HAVE_DWARF_GETLOCATIONS_SUPPORT syscall_table: [ on ] # HAVE_SYSCALL_TABLE_SUPPORT libbfd: [ OFF ] # HAVE_LIBBFD_SUPPORT debuginfod: [ OFF ] # HAVE_DEBUGINFOD_SUPPORT libelf: [ OFF ] # HAVE_LIBELF_SUPPORT libnuma: [ OFF ] # HAVE_LIBNUMA_SUPPORT numa_num_possible_cpus: [ OFF ] # HAVE_LIBNUMA_SUPPORT libperl: [ on ] # HAVE_LIBPERL_SUPPORT libpython: [ on ] # HAVE_LIBPYTHON_SUPPORT libslang: [ on ] # HAVE_SLANG_SUPPORT libcrypto: [ on ] # HAVE_LIBCRYPTO_SUPPORT libunwind: [ OFF ] # HAVE_LIBUNWIND_SUPPORT libdw-dwarf-unwind: [ OFF ] # HAVE_DWARF_SUPPORT zlib: [ on ] # HAVE_ZLIB_SUPPORT lzma: [ on ] # HAVE_LZMA_SUPPORT get_cpuid: [ on ] # HAVE_AUXTRACE_SUPPORT bpf: [ OFF ] # HAVE_LIBBPF_SUPPORT aio: [ on ] # HAVE_AIO_SUPPORT zstd: [ on ] # HAVE_ZSTD_SUPPORT libpfm4: [ on ] # HAVE_LIBPFM libtraceevent: [ on ] # HAVE_LIBTRACEEVENT bpf_skeletons: [ OFF ] # HAVE_BPF_SKEL ``` Signed-off-by: Ian Rogers Tested-by: Namhyung Kim Cc: James Clark Cc: Nick Terrell Cc: Patrice Duroux Cc: Nick Desaulniers Cc: Andrii Nakryiko Cc: Nathan Chancellor Cc: Kajol Jain Cc: Athira Rajeev Cc: Tiezhu Yang Cc: Tom Rix Cc: llvm@lists.linux.dev Cc: bpf@vger.kernel.org Link: https://lore.kernel.org/r/20230914211948.814999-2-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/builtin-version.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/perf/builtin-version.c b/tools/perf/builtin-version.c index e5859c70e195..ac20c2b9bbc2 100644 --- a/tools/perf/builtin-version.c +++ b/tools/perf/builtin-version.c @@ -81,6 +81,7 @@ static void library_status(void) STATUS(HAVE_ZSTD_SUPPORT, zstd); STATUS(HAVE_LIBPFM, libpfm4); STATUS(HAVE_LIBTRACEEVENT, libtraceevent); + STATUS(HAVE_BPF_SKEL, bpf_skeletons); } int cmd_version(int argc, const char **argv) From 9925495d96efc14d885ba66c5696f664fe0e663c Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Thu, 14 Sep 2023 14:19:45 -0700 Subject: [PATCH 115/475] perf build: Default BUILD_BPF_SKEL, warn/disable for missing deps LIBBPF is dependent on zlib so move the NO_ZLIB and feature check early to avoid statically building when zlib is disabled. This avoids a linkage failure with perf and static libbpf when zlib isn't specified. Move BUILD_BPF_SKEL logic to one place and if not defined set BUILD_BPF_SKEL to 1. Detect dependencies of building with BPF skeletons and warn/disable if the dependencies aren't present. Change Makefile.perf to contain BPF skeleton logic dependent on the Makefile.config result and refresh the comment about BUILD_BPF_SKEL. Signed-off-by: Ian Rogers Tested-by: Namhyung Kim Cc: James Clark Cc: Nick Terrell Cc: Patrice Duroux Cc: Nick Desaulniers Cc: Andrii Nakryiko Cc: Nathan Chancellor Cc: Kajol Jain Cc: Athira Rajeev Cc: Tiezhu Yang Cc: Tom Rix Cc: llvm@lists.linux.dev Cc: bpf@vger.kernel.org Link: https://lore.kernel.org/r/20230914211948.814999-3-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/Makefile.config | 78 ++++++++++++++++++++++++-------------- tools/perf/Makefile.perf | 8 ++-- 2 files changed, 53 insertions(+), 33 deletions(-) diff --git a/tools/perf/Makefile.config b/tools/perf/Makefile.config index d66b52407e19..f5ccbfc1a444 100644 --- a/tools/perf/Makefile.config +++ b/tools/perf/Makefile.config @@ -216,12 +216,6 @@ ifeq ($(call get-executable,$(BISON)),) dummy := $(error Error: $(BISON) is missing on this system, please install it) endif -ifeq ($(BUILD_BPF_SKEL),1) - ifeq ($(call get-executable,$(CLANG)),) - dummy := $(error $(CLANG) is missing on this system, please install it to be able to build with BUILD_BPF_SKEL=1) - endif -endif - ifneq ($(OUTPUT),) ifeq ($(shell expr $(shell $(BISON) --version | grep bison | sed -e 's/.\+ \([0-9]\+\).\([0-9]\+\).\([0-9]\+\)/\1\2\3/g') \>\= 371), 1) BISON_FILE_PREFIX_MAP := --file-prefix-map=$(OUTPUT)= @@ -530,6 +524,16 @@ ifdef CORESIGHT endif endif +ifndef NO_ZLIB + ifeq ($(feature-zlib), 1) + CFLAGS += -DHAVE_ZLIB_SUPPORT + EXTLIBS += -lz + $(call detected,CONFIG_ZLIB) + else + NO_ZLIB := 1 + endif +endif + ifndef NO_LIBELF CFLAGS += -DHAVE_LIBELF_SUPPORT EXTLIBS += -lelf @@ -571,22 +575,28 @@ ifndef NO_LIBELF ifndef NO_LIBBPF ifeq ($(feature-bpf), 1) - CFLAGS += -DHAVE_LIBBPF_SUPPORT - $(call detected,CONFIG_LIBBPF) - # detecting libbpf without LIBBPF_DYNAMIC, so make VF=1 shows libbpf detection status $(call feature_check,libbpf) ifdef LIBBPF_DYNAMIC ifeq ($(feature-libbpf), 1) EXTLIBS += -lbpf + CFLAGS += -DHAVE_LIBBPF_SUPPORT + $(call detected,CONFIG_LIBBPF) $(call detected,CONFIG_LIBBPF_DYNAMIC) else dummy := $(error Error: No libbpf devel library found or older than v1.0, please install/update libbpf-devel); endif else - # Libbpf will be built as a static library from tools/lib/bpf. - LIBBPF_STATIC := 1 + ifeq ($(NO_ZLIB), 1) + dummy := $(warning Warning: Statically building libbpf not possible as zlib is missing) + NO_LIBBPF := 1 + else + # Libbpf will be built as a static library from tools/lib/bpf. + LIBBPF_STATIC := 1 + $(call detected,CONFIG_LIBBPF) + CFLAGS += -DHAVE_LIBBPF_SUPPORT + endif endif endif endif # NO_LIBBPF @@ -663,16 +673,36 @@ ifndef NO_LIBBPF endif endif -ifdef BUILD_BPF_SKEL - $(call feature_check,clang-bpf-co-re) - ifeq ($(feature-clang-bpf-co-re), 0) - dummy := $(error Error: clang too old/not installed. Please install recent clang to build with BUILD_BPF_SKEL) - endif +ifndef BUILD_BPF_SKEL + # BPF skeletons control a large number of perf features, by default + # they are enabled. + BUILD_BPF_SKEL := 1 +endif + +ifeq ($(BUILD_BPF_SKEL),1) ifeq ($(filter -DHAVE_LIBBPF_SUPPORT, $(CFLAGS)),) - dummy := $(error Error: BPF skeleton support requires libbpf) + dummy := $(warning Warning: Disabled BPF skeletons as libbpf is required) + BUILD_BPF_SKEL := 0 + else ifeq ($(filter -DHAVE_LIBELF_SUPPORT, $(CFLAGS)),) + dummy := $(warning Warning: Disabled BPF skeletons as libelf is required by bpftool) + BUILD_BPF_SKEL := 0 + else ifeq ($(filter -DHAVE_ZLIB_SUPPORT, $(CFLAGS)),) + dummy := $(warning Warning: Disabled BPF skeletons as zlib is required by bpftool) + BUILD_BPF_SKEL := 0 + else ifeq ($(call get-executable,$(CLANG)),) + dummy := $(warning Warning: Disabled BPF skeletons as clang ($(CLANG)) is missing) + BUILD_BPF_SKEL := 0 + else + $(call feature_check,clang-bpf-co-re) + ifeq ($(feature-clang-bpf-co-re), 0) + dummy := $(warning Warning: Disabled BPF skeletons as clang is too old) + BUILD_BPF_SKEL := 0 + endif + endif + ifeq ($(BUILD_BPF_SKEL),1) + $(call detected,CONFIG_PERF_BPF_SKEL) + CFLAGS += -DHAVE_BPF_SKEL endif - $(call detected,CONFIG_PERF_BPF_SKEL) - CFLAGS += -DHAVE_BPF_SKEL endif ifndef GEN_VMLINUX_H @@ -946,16 +976,6 @@ ifndef NO_DEMANGLE endif endif -ifndef NO_ZLIB - ifeq ($(feature-zlib), 1) - CFLAGS += -DHAVE_ZLIB_SUPPORT - EXTLIBS += -lz - $(call detected,CONFIG_ZLIB) - else - NO_ZLIB := 1 - endif -endif - ifndef NO_LZMA ifeq ($(feature-lzma), 1) CFLAGS += -DHAVE_LZMA_SUPPORT diff --git a/tools/perf/Makefile.perf b/tools/perf/Makefile.perf index 8d0f6d2bbc7a..98604e396ac3 100644 --- a/tools/perf/Makefile.perf +++ b/tools/perf/Makefile.perf @@ -120,7 +120,7 @@ include ../scripts/utilities.mak # # Define NO_LIBDEBUGINFOD if you do not want support debuginfod # -# Define BUILD_BPF_SKEL to enable BPF skeletons +# Set BUILD_BPF_SKEL to 0 to override BUILD_BPF_SKEL and not build BPF skeletons # # Define BUILD_NONDISTRO to enable building an linking against libbfd and # libiberty distribution license incompatible libraries. @@ -1042,7 +1042,7 @@ SKELETONS += $(SKEL_OUT)/augmented_raw_syscalls.skel.h $(SKEL_TMP_OUT) $(LIBAPI_OUTPUT) $(LIBBPF_OUTPUT) $(LIBPERF_OUTPUT) $(LIBSUBCMD_OUTPUT) $(LIBSYMBOL_OUTPUT): $(Q)$(MKDIR) -p $@ -ifdef BUILD_BPF_SKEL +ifeq ($(CONFIG_PERF_BPF_SKEL),y) BPFTOOL := $(SKEL_TMP_OUT)/bootstrap/bpftool # Get Clang's default includes on this system, as opposed to those seen by # '--target=bpf'. This fixes "missing" files on some architectures/distros, @@ -1120,11 +1120,11 @@ bpf-skel: $(SKELETONS) .PRECIOUS: $(SKEL_TMP_OUT)/%.bpf.o -else # BUILD_BPF_SKEL +else # CONFIG_PERF_BPF_SKEL bpf-skel: -endif # BUILD_BPF_SKEL +endif # CONFIG_PERF_BPF_SKEL bpf-skel-clean: $(call QUIET_CLEAN, bpf-skel) $(RM) -r $(SKEL_TMP_OUT) $(SKELETONS) From c67c631d52f017c2c4795e024ea6aaf3a092ce9e Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Thu, 14 Sep 2023 14:19:46 -0700 Subject: [PATCH 116/475] perf test: Update build test for changed BPF skeleton defaults Fix a target name and set BUILD_BPF_SKEL to 0 rather than 1. Signed-off-by: Ian Rogers Tested-by: Namhyung Kim Cc: James Clark Cc: Nick Terrell Cc: Patrice Duroux Cc: Nick Desaulniers Cc: Andrii Nakryiko Cc: Nathan Chancellor Cc: Kajol Jain Cc: Athira Rajeev Cc: Tiezhu Yang Cc: Tom Rix Cc: llvm@lists.linux.dev Cc: bpf@vger.kernel.org Link: https://lore.kernel.org/r/20230914211948.814999-4-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/tests/make | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/perf/tests/make b/tools/perf/tests/make index ea4c341f5af1..a3a0f2a8bba0 100644 --- a/tools/perf/tests/make +++ b/tools/perf/tests/make @@ -70,8 +70,8 @@ make_python_perf_so := $(python_perf_so) make_debug := DEBUG=1 make_nondistro := BUILD_NONDISTRO=1 make_extra_tests := EXTRA_TESTS=1 -make_bpf_skel := BUILD_BPF_SKEL=1 -make_gen_vmlinux_h := BUILD_BPF_SKEL=1 GEN_VMLINUX_H=1 +make_no_bpf_skel := BUILD_BPF_SKEL=0 +make_gen_vmlinux_h := GEN_VMLINUX_H=1 make_no_libperl := NO_LIBPERL=1 make_no_libpython := NO_LIBPYTHON=1 make_no_scripts := NO_LIBPYTHON=1 NO_LIBPERL=1 @@ -138,7 +138,7 @@ endif run += make_python_perf_so run += make_debug run += make_nondistro -run += make_build_bpf_skel +run += make_no_bpf_skel run += make_gen_vmlinux_h run += make_no_libperl run += make_no_libpython From c2ac838ef734cb0ff351820b77acc6fd3b6634fb Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Thu, 14 Sep 2023 14:19:47 -0700 Subject: [PATCH 117/475] perf test: Ensure EXTRA_TESTS is covered in build test Add to run variable. Signed-off-by: Ian Rogers Tested-by: Namhyung Kim Cc: James Clark Cc: Nick Terrell Cc: Patrice Duroux Cc: Nick Desaulniers Cc: Andrii Nakryiko Cc: Nathan Chancellor Cc: Kajol Jain Cc: Athira Rajeev Cc: Tiezhu Yang Cc: Tom Rix Cc: llvm@lists.linux.dev Cc: bpf@vger.kernel.org Link: https://lore.kernel.org/r/20230914211948.814999-5-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/tests/make | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/perf/tests/make b/tools/perf/tests/make index a3a0f2a8bba0..d9945ed25bc5 100644 --- a/tools/perf/tests/make +++ b/tools/perf/tests/make @@ -138,6 +138,7 @@ endif run += make_python_perf_so run += make_debug run += make_nondistro +run += make_extra_tests run += make_no_bpf_skel run += make_gen_vmlinux_h run += make_no_libperl From b4f48f34f9c195e8e82b31ecd8202445962c734b Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Thu, 14 Sep 2023 14:19:48 -0700 Subject: [PATCH 118/475] perf test: Detect off-cpu support from build options Use perf version to detect whether BPF skeletons were enabled in a build rather than a failing perf record. Signed-off-by: Ian Rogers Tested-by: Namhyung Kim Cc: James Clark Cc: Mark Rutland Cc: Nick Terrell Cc: Patrice Duroux Cc: Nick Desaulniers Cc: Andrii Nakryiko Cc: Nathan Chancellor Cc: Kajol Jain Cc: Athira Rajeev Cc: Tiezhu Yang Cc: Tom Rix Cc: llvm@lists.linux.dev Cc: bpf@vger.kernel.org Link: https://lore.kernel.org/r/20230914211948.814999-6-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/tests/shell/record_offcpu.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/perf/tests/shell/record_offcpu.sh b/tools/perf/tests/shell/record_offcpu.sh index a0d14cd0aa79..a1ef8f0d2b5c 100755 --- a/tools/perf/tests/shell/record_offcpu.sh +++ b/tools/perf/tests/shell/record_offcpu.sh @@ -28,7 +28,7 @@ test_offcpu_priv() { err=2 return fi - if perf record --off-cpu -o /dev/null --quiet true 2>&1 | grep BUILD_BPF_SKEL + if perf version --build-options 2>&1 | grep HAVE_BPF_SKEL | grep -q OFF then echo "off-cpu test [Skipped missing BPF support]" err=2 From ede72dca45b1893f3e9236b6ad6c4e790db232f6 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Thu, 14 Sep 2023 09:40:28 -0700 Subject: [PATCH 119/475] perf parse-events: Fix tracepoint name memory leak Fuzzing found that an invalid tracepoint name would create a memory leak with an address sanitizer build: ``` $ perf stat -e '*:o/' true event syntax error: '*:o/' \___ parser error Run 'perf list' for a list of valid events Usage: perf stat [] [] -e, --event event selector. use 'perf list' to list available events ================================================================= ==59380==ERROR: LeakSanitizer: detected memory leaks Direct leak of 4 byte(s) in 2 object(s) allocated from: #0 0x7f38ac07077b in __interceptor_strdup ../../../../src/libsanitizer/asan/asan_interceptors.cpp:439 #1 0x55f2f41be73b in str util/parse-events.l:49 #2 0x55f2f41d08e8 in parse_events_lex util/parse-events.l:338 #3 0x55f2f41dc3b1 in parse_events_parse util/parse-events-bison.c:1464 #4 0x55f2f410b8b3 in parse_events__scanner util/parse-events.c:1822 #5 0x55f2f410d1b9 in __parse_events util/parse-events.c:2094 #6 0x55f2f410e57f in parse_events_option util/parse-events.c:2279 #7 0x55f2f4427b56 in get_value tools/lib/subcmd/parse-options.c:251 #8 0x55f2f4428d98 in parse_short_opt tools/lib/subcmd/parse-options.c:351 #9 0x55f2f4429d80 in parse_options_step tools/lib/subcmd/parse-options.c:539 #10 0x55f2f442acb9 in parse_options_subcommand tools/lib/subcmd/parse-options.c:654 #11 0x55f2f3ec99fc in cmd_stat tools/perf/builtin-stat.c:2501 #12 0x55f2f4093289 in run_builtin tools/perf/perf.c:322 #13 0x55f2f40937f5 in handle_internal_command tools/perf/perf.c:375 #14 0x55f2f4093bbd in run_argv tools/perf/perf.c:419 #15 0x55f2f409412b in main tools/perf/perf.c:535 SUMMARY: AddressSanitizer: 4 byte(s) leaked in 2 allocation(s). ``` Fix by adding the missing destructor. Fixes: 865582c3f48e ("perf tools: Adds the tracepoint name parsing support") Signed-off-by: Ian Rogers Cc: He Kuang Link: https://lore.kernel.org/r/20230914164028.363220-1-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/util/parse-events.y | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/perf/util/parse-events.y b/tools/perf/util/parse-events.y index c4ce888a2550..4d9f3922a33d 100644 --- a/tools/perf/util/parse-events.y +++ b/tools/perf/util/parse-events.y @@ -103,6 +103,7 @@ static void free_list_evsel(struct list_head* list_evsel) %type groups %destructor { free_list_evsel ($$); } %type tracepoint_name +%destructor { free ($$.sys); free ($$.event); } %type PE_TERM_HW %destructor { free ($$.str); } From 6fcb13972bc2b41211e1dddb02f5e73199addc75 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Mon, 18 Sep 2023 12:14:44 +0200 Subject: [PATCH 120/475] powerpc: Replace GPL 2.0+ README.legal boilerplate with SPDX Upstream Linux never had a "README.legal" file, but it was present in early source releases of Linux/m68k. It contained a simple copyright notice and a link to a version of the "COPYING" file that predated the addition of the "only valid GPL version is v2" clause. Get rid of the references to non-existent files by replacing the boilerplate with SPDX license identifiers. Signed-off-by: Geert Uytterhoeven Acked-by: Michael Ellerman Signed-off-by: Michael Ellerman Link: https://msgid.link/d91725ff1ed5d4b6ba42474e2ebfeebe711cba23.1695031668.git.geert@linux-m68k.org --- arch/powerpc/kernel/ptrace/ptrace.c | 5 +---- arch/powerpc/kernel/signal.c | 5 +---- arch/powerpc/kernel/signal.h | 7 ++----- 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/arch/powerpc/kernel/ptrace/ptrace.c b/arch/powerpc/kernel/ptrace/ptrace.c index 5d7a72b41ae7..727ed4a14545 100644 --- a/arch/powerpc/kernel/ptrace/ptrace.c +++ b/arch/powerpc/kernel/ptrace/ptrace.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* * PowerPC version * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) @@ -9,10 +10,6 @@ * * Modified by Cort Dougan (cort@hq.fsmlabs.com) * and Paul Mackerras (paulus@samba.org). - * - * This file is subject to the terms and conditions of the GNU General - * Public License. See the file README.legal in the main directory of - * this archive for more details. */ #include diff --git a/arch/powerpc/kernel/signal.c b/arch/powerpc/kernel/signal.c index 68a91e553e14..aa17e62f3754 100644 --- a/arch/powerpc/kernel/signal.c +++ b/arch/powerpc/kernel/signal.c @@ -1,12 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* * Common signal handling code for both 32 and 64 bits * * Copyright (c) 2007 Benjamin Herrenschmidt, IBM Corporation * Extracted from signal_32.c and signal_64.c - * - * This file is subject to the terms and conditions of the GNU General - * Public License. See the file README.legal in the main directory of - * this archive for more details. */ #include diff --git a/arch/powerpc/kernel/signal.h b/arch/powerpc/kernel/signal.h index a429c57ed433..58ecea1cdc27 100644 --- a/arch/powerpc/kernel/signal.h +++ b/arch/powerpc/kernel/signal.h @@ -1,10 +1,7 @@ -/* +/* SPDX-License-Identifier: GPL-2.0-or-later + * * Copyright (c) 2007 Benjamin Herrenschmidt, IBM Corporation * Extracted from signal_32.c and signal_64.c - * - * This file is subject to the terms and conditions of the GNU General - * Public License. See the file README.legal in the main directory of - * this archive for more details. */ #ifndef _POWERPC_ARCH_SIGNAL_H From c945e6f453a361b0e9daddd2be9c099d1b80d6f8 Mon Sep 17 00:00:00 2001 From: Peter Lafreniere Date: Mon, 18 Sep 2023 17:56:43 +0000 Subject: [PATCH 121/475] powerpc/configs: Remove ReiserFS from defconfig ReiserFS has been deprecated for a year and a half, yet is still built as part of a defconfig kernel. According to commit eb103a51640e ("reiserfs: Deprecate reiserfs"), the filesystem is slated to be removed in 2025. Remove it from the defconfig profiles now, as part of its deprecation process. Signed-off-by: Peter Lafreniere Signed-off-by: Michael Ellerman Link: https://msgid.link/20230918175529.19011-3-peter@n8pjl.ca --- arch/powerpc/configs/44x/sam440ep_defconfig | 1 - arch/powerpc/configs/g5_defconfig | 4 ---- arch/powerpc/configs/ppc64e_defconfig | 4 ---- arch/powerpc/configs/ppc6xx_defconfig | 5 ----- 4 files changed, 14 deletions(-) diff --git a/arch/powerpc/configs/44x/sam440ep_defconfig b/arch/powerpc/configs/44x/sam440ep_defconfig index 51499ee6366b..2479ab62d12f 100644 --- a/arch/powerpc/configs/44x/sam440ep_defconfig +++ b/arch/powerpc/configs/44x/sam440ep_defconfig @@ -78,7 +78,6 @@ CONFIG_EXT2_FS_XATTR=y CONFIG_EXT2_FS_POSIX_ACL=y CONFIG_EXT4_FS=y CONFIG_EXT4_FS_POSIX_ACL=y -CONFIG_REISERFS_FS=y CONFIG_AUTOFS_FS=y CONFIG_ISO9660_FS=y CONFIG_JOLIET=y diff --git a/arch/powerpc/configs/g5_defconfig b/arch/powerpc/configs/g5_defconfig index 71d9d112c0b6..9215bed53291 100644 --- a/arch/powerpc/configs/g5_defconfig +++ b/arch/powerpc/configs/g5_defconfig @@ -202,10 +202,6 @@ CONFIG_EXT2_FS_SECURITY=y CONFIG_EXT4_FS=y CONFIG_EXT4_FS_POSIX_ACL=y CONFIG_EXT4_FS_SECURITY=y -CONFIG_REISERFS_FS=y -CONFIG_REISERFS_FS_XATTR=y -CONFIG_REISERFS_FS_POSIX_ACL=y -CONFIG_REISERFS_FS_SECURITY=y CONFIG_XFS_FS=m CONFIG_XFS_POSIX_ACL=y CONFIG_FS_DAX=y diff --git a/arch/powerpc/configs/ppc64e_defconfig b/arch/powerpc/configs/ppc64e_defconfig index 624c371ffcc3..4c05f4e4d505 100644 --- a/arch/powerpc/configs/ppc64e_defconfig +++ b/arch/powerpc/configs/ppc64e_defconfig @@ -175,10 +175,6 @@ CONFIG_EXT2_FS_SECURITY=y CONFIG_EXT4_FS=y CONFIG_EXT4_FS_POSIX_ACL=y CONFIG_EXT4_FS_SECURITY=y -CONFIG_REISERFS_FS=y -CONFIG_REISERFS_FS_XATTR=y -CONFIG_REISERFS_FS_POSIX_ACL=y -CONFIG_REISERFS_FS_SECURITY=y CONFIG_JFS_FS=y CONFIG_JFS_POSIX_ACL=y CONFIG_JFS_SECURITY=y diff --git a/arch/powerpc/configs/ppc6xx_defconfig b/arch/powerpc/configs/ppc6xx_defconfig index eaf3273372a9..f279703425d4 100644 --- a/arch/powerpc/configs/ppc6xx_defconfig +++ b/arch/powerpc/configs/ppc6xx_defconfig @@ -954,11 +954,6 @@ CONFIG_EXT4_FS=y CONFIG_EXT4_FS_POSIX_ACL=y CONFIG_EXT4_FS_SECURITY=y CONFIG_JBD2_DEBUG=y -CONFIG_REISERFS_FS=m -CONFIG_REISERFS_PROC_INFO=y -CONFIG_REISERFS_FS_XATTR=y -CONFIG_REISERFS_FS_POSIX_ACL=y -CONFIG_REISERFS_FS_SECURITY=y CONFIG_JFS_FS=m CONFIG_JFS_POSIX_ACL=y CONFIG_JFS_SECURITY=y From f3cfb875d0fd5f4af40cbb992f436ad396f69a71 Mon Sep 17 00:00:00 2001 From: Ingo Franzki Date: Thu, 31 Aug 2023 17:36:43 +0200 Subject: [PATCH 122/475] s390/zcrypt: update list of EP11 operation modes Add additional operation mode strings into the EP11 operation mode table. These strings are returned by sysfs entries /sys/devices/ap/cardxx/op_modes and /sys/devices/ap/cardxx/xx.yyyy/op_modes. Signed-off-by: Ingo Franzki Reviewed-by: Harald Freudenberger Signed-off-by: Vasily Gorbik --- drivers/s390/crypto/zcrypt_cex4.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/s390/crypto/zcrypt_cex4.c b/drivers/s390/crypto/zcrypt_cex4.c index 9cfce9ff2e65..5c4532ab0040 100644 --- a/drivers/s390/crypto/zcrypt_cex4.c +++ b/drivers/s390/crypto/zcrypt_cex4.c @@ -279,7 +279,11 @@ static const struct { { 1, "BSI2009" }, { 2, "FIPS2011" }, { 3, "BSI2011" }, + { 4, "SIGG-IMPORT" }, + { 5, "SIGG" }, { 6, "BSICC2017" }, + { 7, "FIPS2021" }, + { 8, "FIPS2024" }, { 0, NULL } }; From 4a1725281fc5b0009944b1c0e1d2c1dc311a09ec Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Tue, 5 Sep 2023 15:49:37 +0200 Subject: [PATCH 123/475] s390/smp,mcck: fix early IPI handling Both the external call as well as the emergency signal submask bits in control register 0 are set before any interrupt handler is registered. Change the order and first register the interrupt handler and only then enable the interrupts by setting the corresponding bits in control register 0. This prevents that the second part of the machine check handler for early machine check handling is not executed: the machine check handler sends an IPI to the CPU it runs on. If the corresponding interrupts are enabled, but no interrupt handler is present, the interrupt is ignored. Reviewed-by: Sven Schnelle Acked-by: Alexander Gordeev Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/kernel/early.c | 12 +++--------- arch/s390/kernel/smp.c | 4 ++-- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c index 442ce0489e1a..3a54733e4fc6 100644 --- a/arch/s390/kernel/early.c +++ b/arch/s390/kernel/early.c @@ -258,15 +258,9 @@ static inline void save_vector_registers(void) #endif } -static inline void setup_control_registers(void) +static inline void setup_low_address_protection(void) { - unsigned long reg; - - __ctl_store(reg, 0, 0); - reg |= CR0_LOW_ADDRESS_PROTECTION; - reg |= CR0_EMERGENCY_SIGNAL_SUBMASK; - reg |= CR0_EXTERNAL_CALL_SUBMASK; - __ctl_load(reg, 0, 0); + __ctl_set_bit(0, 28); } static inline void setup_access_registers(void) @@ -314,7 +308,7 @@ void __init startup_init(void) save_vector_registers(); setup_topology(); sclp_early_detect(); - setup_control_registers(); + setup_low_address_protection(); setup_access_registers(); lockdep_on(); } diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index a4edb7ea66ea..c63be2efd689 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -1013,12 +1013,12 @@ void __init smp_fill_possible_mask(void) void __init smp_prepare_cpus(unsigned int max_cpus) { - /* request the 0x1201 emergency signal external interrupt */ if (register_external_irq(EXT_IRQ_EMERGENCY_SIG, do_ext_call_interrupt)) panic("Couldn't request external interrupt 0x1201"); - /* request the 0x1202 external call external interrupt */ + ctl_set_bit(0, 14); if (register_external_irq(EXT_IRQ_EXTERNAL_CALL, do_ext_call_interrupt)) panic("Couldn't request external interrupt 0x1202"); + ctl_set_bit(0, 13); } void __init smp_prepare_boot_cpu(void) From 496bb034f4ff4866452ced7fdfaa4a0736230bdf Mon Sep 17 00:00:00 2001 From: Tobias Huschle Date: Wed, 16 Aug 2023 09:28:20 +0200 Subject: [PATCH 124/475] s390/smp: disallow CPU hotplug of CPU 0 On s390, CPU 0 has special properties in comparison to other CPUs, as it cannot be deconfigured for example. Therefore, allowing to hotplug CPU 0 introduces additional complexity when handling these properties. Disallowing to hotplug CPU 0 allows to remove such complexities. This follows x86 which also prevents offlining of CPU0 since commit e59e74dc48a3 ("x86/topology: Remove CPU0 hotplug option"). [hca@linux.ibm.com: changed commit message] Suggested-by: Heiko Carstens Signed-off-by: Tobias Huschle Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/kernel/smp.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index c63be2efd689..e8284cec36e7 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -1076,11 +1076,9 @@ static ssize_t cpu_configure_store(struct device *dev, cpus_read_lock(); mutex_lock(&smp_cpu_state_mutex); rc = -EBUSY; - /* disallow configuration changes of online cpus and cpu 0 */ + /* disallow configuration changes of online cpus */ cpu = dev->id; cpu = smp_get_base_cpu(cpu); - if (cpu == 0) - goto out; for (i = 0; i <= smp_cpu_mtid; i++) if (cpu_online(cpu + i)) goto out; @@ -1180,7 +1178,7 @@ static int smp_add_present_cpu(int cpu) return -ENOMEM; per_cpu(cpu_device, cpu) = c; s = &c->dev; - c->hotpluggable = 1; + c->hotpluggable = !!cpu; rc = register_cpu(c, cpu); if (rc) goto out; From 3570ee046c46b5dc3d077e3e60616b141a221027 Mon Sep 17 00:00:00 2001 From: Ilya Leoshkevich Date: Mon, 31 Jul 2023 17:07:11 +0200 Subject: [PATCH 125/475] s390/smp: keep the original lowcore for CPU 0 Now that CPU 0 is not hotpluggable, it is not necessary to support freeing its stacks. Delete all the code that migrates it to new stacks and a new lowcore. Suggested-by: Heiko Carstens Signed-off-by: Ilya Leoshkevich Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/smp.h | 1 - arch/s390/kernel/setup.c | 6 ---- arch/s390/kernel/smp.c | 57 ------------------------------------- 3 files changed, 64 deletions(-) diff --git a/arch/s390/include/asm/smp.h b/arch/s390/include/asm/smp.h index 73ed2781073b..6e5b1b4b19a9 100644 --- a/arch/s390/include/asm/smp.h +++ b/arch/s390/include/asm/smp.h @@ -59,7 +59,6 @@ static inline void smp_cpus_done(unsigned int max_cpus) { } -extern int smp_reinit_ipl_cpu(void); extern int smp_rescan_cpus(void); extern void __noreturn cpu_die(void); extern void __cpu_die(unsigned int cpu); diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index de6ad0fb2328..deb97723c890 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -381,12 +381,6 @@ void stack_free(unsigned long stack) #endif } -void __init __noreturn arch_call_rest_init(void) -{ - smp_reinit_ipl_cpu(); - rest_init(); -} - static unsigned long __init stack_alloc_early(void) { unsigned long stack; diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index e8284cec36e7..9c5e107da4b4 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -1256,60 +1256,3 @@ out: return rc; } subsys_initcall(s390_smp_init); - -static __always_inline void set_new_lowcore(struct lowcore *lc) -{ - union register_pair dst, src; - u32 pfx; - - src.even = (unsigned long) &S390_lowcore; - src.odd = sizeof(S390_lowcore); - dst.even = (unsigned long) lc; - dst.odd = sizeof(*lc); - pfx = __pa(lc); - - asm volatile( - " mvcl %[dst],%[src]\n" - " spx %[pfx]\n" - : [dst] "+&d" (dst.pair), [src] "+&d" (src.pair) - : [pfx] "Q" (pfx) - : "memory", "cc"); -} - -int __init smp_reinit_ipl_cpu(void) -{ - unsigned long async_stack, nodat_stack, mcck_stack; - struct lowcore *lc, *lc_ipl; - unsigned long flags, cr0; - u64 mcesad; - - lc_ipl = lowcore_ptr[0]; - lc = (struct lowcore *) __get_free_pages(GFP_KERNEL | GFP_DMA, LC_ORDER); - nodat_stack = __get_free_pages(GFP_KERNEL, THREAD_SIZE_ORDER); - async_stack = stack_alloc(); - mcck_stack = stack_alloc(); - if (!lc || !nodat_stack || !async_stack || !mcck_stack || nmi_alloc_mcesa(&mcesad)) - panic("Couldn't allocate memory"); - - local_irq_save(flags); - local_mcck_disable(); - set_new_lowcore(lc); - S390_lowcore.nodat_stack = nodat_stack + STACK_INIT_OFFSET; - S390_lowcore.async_stack = async_stack + STACK_INIT_OFFSET; - S390_lowcore.mcck_stack = mcck_stack + STACK_INIT_OFFSET; - __ctl_store(cr0, 0, 0); - __ctl_clear_bit(0, 28); /* disable lowcore protection */ - S390_lowcore.mcesad = mcesad; - __ctl_load(cr0, 0, 0); - if (abs_lowcore_map(0, lc, false)) - panic("Couldn't remap absolute lowcore"); - lowcore_ptr[0] = lc; - local_mcck_enable(); - local_irq_restore(flags); - - memblock_free_late(__pa(lc_ipl->mcck_stack - STACK_INIT_OFFSET), THREAD_SIZE); - memblock_free_late(__pa(lc_ipl->async_stack - STACK_INIT_OFFSET), THREAD_SIZE); - memblock_free_late(__pa(lc_ipl->nodat_stack - STACK_INIT_OFFSET), THREAD_SIZE); - memblock_free_late(__pa(lc_ipl), sizeof(*lc_ipl)); - return 0; -} From aa36d433b79657c1fed31ff5ae2bbd6f7bc123aa Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 13:03:00 +0200 Subject: [PATCH 126/475] s390/setup: use strlcat() instead of strcat() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use strlcat() instead of strcat() in order to get rid of this W=1 warning: In function ‘strlcat’, inlined from ‘strcat’ at ./include/linux/fortify-string.h:432:6, inlined from ‘setup_zfcpdump’ at arch/s390/kernel/setup.c:308:2, inlined from ‘setup_arch’ at arch/s390/kernel/setup.c:1002:2: ./include/linux/fortify-string.h:406:19: warning: writing 1 byte into a region of size 0 [-Wstringop-overflow=] 406 | p[actual] = '\0'; | ~~~~~~~~~~^~~~~~ As stated in fortify-string.h strcat() should not be used, since FORTIFY_SOURCE cannot figure out the size of the destination buffer. Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/kernel/setup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index deb97723c890..ef158bd0ea7b 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -305,7 +305,7 @@ static void __init setup_zfcpdump(void) return; if (oldmem_data.start) return; - strcat(boot_command_line, " cio_ignore=all,!ipldev,!condev"); + strlcat(boot_command_line, " cio_ignore=all,!ipldev,!condev", COMMAND_LINE_SIZE); console_loglevel = 2; } #else From 0c4d01f3952911b766e6394e0053146c24c98357 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 21:39:55 +0200 Subject: [PATCH 127/475] s390/ctlreg: move control register code to separate file Control register handling has nothing to do with low level SMP code. Move it to a separate file. Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/boot/vmem.c | 1 + arch/s390/include/asm/ctl_reg.h | 8 ++-- arch/s390/include/asm/fpu/internal.h | 1 - arch/s390/include/asm/uaccess.h | 1 - arch/s390/kernel/Makefile | 2 +- arch/s390/kernel/ctlreg.c | 62 ++++++++++++++++++++++++++++ arch/s390/kernel/smp.c | 53 ++---------------------- arch/s390/lib/uaccess.c | 1 + arch/s390/mm/vmem.c | 1 + drivers/s390/char/sclp_early_core.c | 1 + 10 files changed, 75 insertions(+), 56 deletions(-) create mode 100644 arch/s390/kernel/ctlreg.c diff --git a/arch/s390/boot/vmem.c b/arch/s390/boot/vmem.c index 01257ce3b89c..36b90864d69f 100644 --- a/arch/s390/boot/vmem.c +++ b/arch/s390/boot/vmem.c @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/s390/include/asm/ctl_reg.h b/arch/s390/include/asm/ctl_reg.h index adf7d8cdac7e..f5536fdb398d 100644 --- a/arch/s390/include/asm/ctl_reg.h +++ b/arch/s390/include/asm/ctl_reg.h @@ -74,16 +74,18 @@ static __always_inline void __ctl_clear_bit(unsigned int cr, unsigned int bit) __ctl_load(reg, cr, cr); } -void smp_ctl_set_clear_bit(int cr, int bit, bool set); +void ctlreg_lock(void); +void ctlreg_unlock(void); +void ctl_set_clear_bit(int cr, int bit, bool set); static inline void ctl_set_bit(int cr, int bit) { - smp_ctl_set_clear_bit(cr, bit, true); + ctl_set_clear_bit(cr, bit, true); } static inline void ctl_clear_bit(int cr, int bit) { - smp_ctl_set_clear_bit(cr, bit, false); + ctl_set_clear_bit(cr, bit, false); } union ctlreg0 { diff --git a/arch/s390/include/asm/fpu/internal.h b/arch/s390/include/asm/fpu/internal.h index bbdadb1c9efc..8634581b9011 100644 --- a/arch/s390/include/asm/fpu/internal.h +++ b/arch/s390/include/asm/fpu/internal.h @@ -10,7 +10,6 @@ #define _ASM_S390_FPU_INTERNAL_H #include -#include #include static inline void save_vx_regs(__vector128 *vxrs) diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h index 8a8c64a678c4..81ae8a98e7ec 100644 --- a/arch/s390/include/asm/uaccess.h +++ b/arch/s390/include/asm/uaccess.h @@ -15,7 +15,6 @@ */ #include #include -#include #include #include #include diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile index 0df2b88cc0da..353def93973b 100644 --- a/arch/s390/kernel/Makefile +++ b/arch/s390/kernel/Makefile @@ -37,7 +37,7 @@ CFLAGS_unwind_bc.o += -fno-optimize-sibling-calls obj-y := head64.o traps.o time.o process.o earlypgm.o early.o setup.o idle.o vtime.o obj-y += processor.o syscall.o ptrace.o signal.o cpcmd.o ebcdic.o nmi.o obj-y += debug.o irq.o ipl.o dis.o diag.o vdso.o cpufeature.o -obj-y += sysinfo.o lgr.o os_info.o +obj-y += sysinfo.o lgr.o os_info.o ctlreg.o obj-y += runtime_instr.o cache.o fpu.o dumpstack.o guarded_storage.o sthyi.o obj-y += entry.o reipl.o kdebugfs.o alternative.o obj-y += nospec-branch.o ipl_vmparm.o machine_kexec_reloc.o unwind_bc.o diff --git a/arch/s390/kernel/ctlreg.c b/arch/s390/kernel/ctlreg.c new file mode 100644 index 000000000000..176ada8b45c2 --- /dev/null +++ b/arch/s390/kernel/ctlreg.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright IBM Corp. 1999, 2023 + */ + +#include +#include +#include +#include + +/* + * ctl_lock guards access to global control register contents which + * are kept in the control register save area within absolute lowcore + * at physical address zero. + */ +static DEFINE_SPINLOCK(ctl_lock); + +void ctlreg_lock(void) + __acquires(&ctl_lock) +{ + spin_lock(&ctl_lock); +} + +void ctlreg_unlock(void) + __releases(&ctl_lock) +{ + spin_unlock(&ctl_lock); +} + +struct ctl_bit_parms { + unsigned long orval; + unsigned long andval; + int cr; +}; + +static void ctl_bit_callback(void *info) +{ + struct ctl_bit_parms *pp = info; + unsigned long regs[16]; + + __ctl_store(regs, 0, 15); + regs[pp->cr] &= pp->andval; + regs[pp->cr] |= pp->orval; + __ctl_load(regs, 0, 15); +} + +void ctl_set_clear_bit(int cr, int bit, bool set) +{ + struct ctl_bit_parms pp = { .cr = cr, }; + struct lowcore *abs_lc; + + pp.orval = set ? 1UL << bit : 0; + pp.andval = set ? -1UL : ~(1UL << bit); + ctlreg_lock(); + abs_lc = get_abs_lowcore(); + abs_lc->cregs_save_area[cr] &= pp.andval; + abs_lc->cregs_save_area[cr] |= pp.orval; + put_abs_lowcore(abs_lc); + on_each_cpu(ctl_bit_callback, &pp, 1); + ctlreg_unlock(); +} +EXPORT_SYMBOL(ctl_set_clear_bit); diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index 9c5e107da4b4..d12dbba9d03c 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -37,6 +37,7 @@ #include #include #include +#include #include #include #include @@ -567,54 +568,6 @@ void arch_irq_work_raise(void) } #endif -/* - * parameter area for the set/clear control bit callbacks - */ -struct ec_creg_mask_parms { - unsigned long orval; - unsigned long andval; - int cr; -}; - -/* - * callback for setting/clearing control bits - */ -static void smp_ctl_bit_callback(void *info) -{ - struct ec_creg_mask_parms *pp = info; - unsigned long cregs[16]; - - __ctl_store(cregs, 0, 15); - cregs[pp->cr] = (cregs[pp->cr] & pp->andval) | pp->orval; - __ctl_load(cregs, 0, 15); -} - -static DEFINE_SPINLOCK(ctl_lock); - -void smp_ctl_set_clear_bit(int cr, int bit, bool set) -{ - struct ec_creg_mask_parms parms = { .cr = cr, }; - struct lowcore *abs_lc; - u64 ctlreg; - - if (set) { - parms.orval = 1UL << bit; - parms.andval = -1UL; - } else { - parms.orval = 0; - parms.andval = ~(1UL << bit); - } - spin_lock(&ctl_lock); - abs_lc = get_abs_lowcore(); - ctlreg = abs_lc->cregs_save_area[cr]; - ctlreg = (ctlreg & parms.andval) | parms.orval; - abs_lc->cregs_save_area[cr] = ctlreg; - put_abs_lowcore(abs_lc); - on_each_cpu(smp_ctl_bit_callback, &parms, 1); - spin_unlock(&ctl_lock); -} -EXPORT_SYMBOL(smp_ctl_set_clear_bit); - #ifdef CONFIG_CRASH_DUMP int smp_store_status(int cpu) @@ -935,14 +888,14 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle) * Make sure global control register contents do not change * until new CPU has initialized control registers. */ - spin_lock(&ctl_lock); + ctlreg_lock(); pcpu_prepare_secondary(pcpu, cpu); pcpu_attach_task(pcpu, tidle); pcpu_start_fn(pcpu, smp_start_secondary, NULL); /* Wait until cpu puts itself in the online & active maps */ while (!cpu_online(cpu)) cpu_relax(); - spin_unlock(&ctl_lock); + ctlreg_unlock(); return 0; } diff --git a/arch/s390/lib/uaccess.c b/arch/s390/lib/uaccess.c index e4a13d7cab6e..80b9c2d039f6 100644 --- a/arch/s390/lib/uaccess.c +++ b/arch/s390/lib/uaccess.c @@ -12,6 +12,7 @@ #include #include #include +#include #ifdef CONFIG_DEBUG_ENTRY void debug_user_asce(int exit) diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c index 6957d2ed97bf..9d3f9fa6f498 100644 --- a/arch/s390/mm/vmem.c +++ b/arch/s390/mm/vmem.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/s390/char/sclp_early_core.c b/drivers/s390/char/sclp_early_core.c index dbd5c53d8edf..a191d69573fb 100644 --- a/drivers/s390/char/sclp_early_core.c +++ b/drivers/s390/char/sclp_early_core.c @@ -6,6 +6,7 @@ #include #include +#include #include #include #include From ebe1cd530fb2f6450656758bd904cfb5767f8d33 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 21:39:56 +0200 Subject: [PATCH 128/475] s390/ctlreg: rename ctl_reg.h to ctlreg.h Rename ctl_reg.h to ctlreg.h so it matches not only ctlreg.c but also other control register related function, union, and structure names, which all come with a ctlreg prefix. Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/boot/vmem.c | 2 +- arch/s390/include/asm/{ctl_reg.h => ctlreg.h} | 6 +++--- arch/s390/include/asm/mmu_context.h | 2 +- arch/s390/kernel/ctlreg.c | 2 +- arch/s390/kernel/nmi.c | 2 +- arch/s390/kernel/perf_pai_crypto.c | 3 +-- arch/s390/kernel/perf_pai_ext.c | 3 +-- arch/s390/kernel/smp.c | 2 +- arch/s390/lib/uaccess.c | 2 +- arch/s390/mm/init.c | 2 +- arch/s390/mm/maccess.c | 2 +- arch/s390/mm/vmem.c | 2 +- drivers/s390/char/diag_ftp.c | 2 +- drivers/s390/char/sclp_cmd.c | 2 +- drivers/s390/char/sclp_early.c | 2 +- drivers/s390/char/sclp_early_core.c | 2 +- drivers/s390/cio/crw.c | 2 +- 17 files changed, 19 insertions(+), 21 deletions(-) rename arch/s390/include/asm/{ctl_reg.h => ctlreg.h} (97%) diff --git a/arch/s390/boot/vmem.c b/arch/s390/boot/vmem.c index 36b90864d69f..9ac6d00537c1 100644 --- a/arch/s390/boot/vmem.c +++ b/arch/s390/boot/vmem.c @@ -2,10 +2,10 @@ #include #include #include -#include #include #include #include +#include #include #include #include diff --git a/arch/s390/include/asm/ctl_reg.h b/arch/s390/include/asm/ctlreg.h similarity index 97% rename from arch/s390/include/asm/ctl_reg.h rename to arch/s390/include/asm/ctlreg.h index f5536fdb398d..25b955db8998 100644 --- a/arch/s390/include/asm/ctl_reg.h +++ b/arch/s390/include/asm/ctlreg.h @@ -5,8 +5,8 @@ * Author(s): Martin Schwidefsky */ -#ifndef __ASM_CTL_REG_H -#define __ASM_CTL_REG_H +#ifndef __ASM_S390_CTLREG_H +#define __ASM_S390_CTLREG_H #include @@ -145,4 +145,4 @@ union ctlreg15 { }; #endif /* __ASSEMBLY__ */ -#endif /* __ASM_CTL_REG_H */ +#endif /* __ASM_S390_CTLREG_H */ diff --git a/arch/s390/include/asm/mmu_context.h b/arch/s390/include/asm/mmu_context.h index 2a38af5a00c2..50f247eede60 100644 --- a/arch/s390/include/asm/mmu_context.h +++ b/arch/s390/include/asm/mmu_context.h @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #define init_new_context init_new_context diff --git a/arch/s390/kernel/ctlreg.c b/arch/s390/kernel/ctlreg.c index 176ada8b45c2..a01b901d4866 100644 --- a/arch/s390/kernel/ctlreg.c +++ b/arch/s390/kernel/ctlreg.c @@ -6,7 +6,7 @@ #include #include #include -#include +#include /* * ctl_lock guards access to global control register contents which diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c index 38ec0487521c..b813d9dfa53c 100644 --- a/arch/s390/kernel/nmi.c +++ b/arch/s390/kernel/nmi.c @@ -22,13 +22,13 @@ #include #include #include +#include #include #include #include #include #include #include -#include #include #include #include diff --git a/arch/s390/kernel/perf_pai_crypto.c b/arch/s390/kernel/perf_pai_crypto.c index fe7d1774ded1..6280a0e588c5 100644 --- a/arch/s390/kernel/perf_pai_crypto.c +++ b/arch/s390/kernel/perf_pai_crypto.c @@ -16,8 +16,7 @@ #include #include #include - -#include +#include #include #include diff --git a/arch/s390/kernel/perf_pai_ext.c b/arch/s390/kernel/perf_pai_ext.c index c57c1a203256..b8877493dbb6 100644 --- a/arch/s390/kernel/perf_pai_ext.c +++ b/arch/s390/kernel/perf_pai_ext.c @@ -17,8 +17,7 @@ #include #include #include - -#include +#include #include #include diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index d12dbba9d03c..204c72428696 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -37,7 +37,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/s390/lib/uaccess.c b/arch/s390/lib/uaccess.c index 80b9c2d039f6..478ab4c7d197 100644 --- a/arch/s390/lib/uaccess.c +++ b/arch/s390/lib/uaccess.c @@ -12,7 +12,7 @@ #include #include #include -#include +#include #ifdef CONFIG_DEBUG_ENTRY void debug_user_asce(int exit) diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c index 8b94d2212d33..c6ab6f2707d7 100644 --- a/arch/s390/mm/init.c +++ b/arch/s390/mm/init.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -42,7 +43,6 @@ #include #include #include -#include #include #include #include diff --git a/arch/s390/mm/maccess.c b/arch/s390/mm/maccess.c index c805b3e2592b..632c3a55feed 100644 --- a/arch/s390/mm/maccess.c +++ b/arch/s390/mm/maccess.c @@ -15,10 +15,10 @@ #include #include #include -#include #include #include #include +#include unsigned long __bootdata_preserved(__memcpy_real_area); pte_t *__bootdata_preserved(memcpy_real_ptep); diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c index 9d3f9fa6f498..91c1f42f6a76 100644 --- a/arch/s390/mm/vmem.c +++ b/arch/s390/mm/vmem.c @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/s390/char/diag_ftp.c b/drivers/s390/char/diag_ftp.c index 65c7f2d565d8..9418a9270d03 100644 --- a/drivers/s390/char/diag_ftp.c +++ b/drivers/s390/char/diag_ftp.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include "hmcdrv_ftp.h" diff --git a/drivers/s390/char/sclp_cmd.c b/drivers/s390/char/sclp_cmd.c index 8b4575a0db9f..b73edf0cd725 100644 --- a/drivers/s390/char/sclp_cmd.c +++ b/drivers/s390/char/sclp_cmd.c @@ -19,7 +19,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/s390/char/sclp_early.c b/drivers/s390/char/sclp_early.c index fdc8668f3fba..60a247fdb2a7 100644 --- a/drivers/s390/char/sclp_early.c +++ b/drivers/s390/char/sclp_early.c @@ -10,7 +10,7 @@ #include #include -#include +#include #include #include #include diff --git a/drivers/s390/char/sclp_early_core.c b/drivers/s390/char/sclp_early_core.c index a191d69573fb..f7067a70c461 100644 --- a/drivers/s390/char/sclp_early_core.c +++ b/drivers/s390/char/sclp_early_core.c @@ -6,8 +6,8 @@ #include #include -#include #include +#include #include #include #include diff --git a/drivers/s390/cio/crw.c b/drivers/s390/cio/crw.c index 7b02a6349c4d..58f22270d9c4 100644 --- a/drivers/s390/cio/crw.c +++ b/drivers/s390/cio/crw.c @@ -12,8 +12,8 @@ #include #include #include +#include #include -#include #include "ioasm.h" static DEFINE_MUTEX(crw_handler_mutex); From a74e4fc168d284e1e4e7296f8cf4b884035ebd45 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 21:39:57 +0200 Subject: [PATCH 129/475] s390/ctlreg: cleanup inline assemblies Use symbolic names for operands, remove typedefs, and slightly refactor the code. Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/ctlreg.h | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/arch/s390/include/asm/ctlreg.h b/arch/s390/include/asm/ctlreg.h index 25b955db8998..0b55a1bb3113 100644 --- a/arch/s390/include/asm/ctlreg.h +++ b/arch/s390/include/asm/ctlreg.h @@ -36,24 +36,37 @@ #include #define __ctl_load(array, low, high) do { \ - typedef struct { char _[sizeof(array)]; } addrtype; \ + struct addrtype { \ + char _[sizeof(array)]; \ + }; \ + int _high = high; \ + int _low = low; \ + int _esize; \ \ - BUILD_BUG_ON(sizeof(addrtype) != (high - low + 1) * sizeof(long));\ + _esize = (_high - _low + 1) * sizeof(unsigned long); \ + BUILD_BUG_ON(sizeof(struct addrtype) != _esize); \ asm volatile( \ - " lctlg %1,%2,%0\n" \ + " lctlg %[_low],%[_high],%[_arr]\n" \ : \ - : "Q" (*(addrtype *)(&array)), "i" (low), "i" (high) \ + : [_arr] "Q" (*(struct addrtype *)(&array)), \ + [_low] "i" (low), [_high] "i" (high) \ : "memory"); \ } while (0) #define __ctl_store(array, low, high) do { \ - typedef struct { char _[sizeof(array)]; } addrtype; \ + struct addrtype { \ + char _[sizeof(array)]; \ + }; \ + int _high = high; \ + int _low = low; \ + int _esize; \ \ - BUILD_BUG_ON(sizeof(addrtype) != (high - low + 1) * sizeof(long));\ + _esize = (_high - _low + 1) * sizeof(unsigned long); \ + BUILD_BUG_ON(sizeof(struct addrtype) != _esize); \ asm volatile( \ - " stctg %1,%2,%0\n" \ - : "=Q" (*(addrtype *)(&array)) \ - : "i" (low), "i" (high)); \ + " stctg %[_low],%[_high],%[_arr]\n" \ + : [_arr] "=Q" (*(struct addrtype *)(&array)) \ + : [_low] "i" (low), [_high] "i" (high)); \ } while (0) static __always_inline void __ctl_set_bit(unsigned int cr, unsigned int bit) From 8d5e98f8d6b11dd0e61323ece3b7ccceea55c281 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 21:39:58 +0200 Subject: [PATCH 130/475] s390/ctlreg: add local and system prefix to some functions Add local and system prefix to some functions to clarify they change control register contents on either the local CPU or the on all CPUs. This results in the following API: Two defines which load and save multiple control registers. The defines correlate with the following C prototypes: void __local_ctl_load(unsigned long *, unsigned int cr_low, unsigned int cr_high); void __local_ctl_store(unsigned long *, unsigned int cr_low, unsigned int cr_high); Two functions which locally set or clear one bit for a specified control register: void local_ctl_set_bit(unsigned int cr, unsigned int bit); void local_ctl_clear_bit(unsigned int cr, unsigned int bit); Two functions which set or clear one bit for a specified control register on all CPUs: void system_ctl_set_bit(unsigned int cr, unsigned int bit); void system_ctl_clear_bit(unsigend int cr, unsigned int bit); Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/boot/startup.c | 2 +- arch/s390/boot/vmem.c | 6 +++--- arch/s390/include/asm/ctlreg.h | 30 ++++++++++++++--------------- arch/s390/include/asm/mmu_context.h | 6 +++--- arch/s390/kernel/ctlreg.c | 26 ++++++++++++------------- arch/s390/kernel/early.c | 12 ++++++------ arch/s390/kernel/guarded_storage.c | 6 +++--- arch/s390/kernel/ipl.c | 2 +- arch/s390/kernel/irq.c | 4 ++-- arch/s390/kernel/kprobes.c | 6 +++--- arch/s390/kernel/machine_kexec.c | 6 +++--- arch/s390/kernel/nmi.c | 18 ++++++++--------- arch/s390/kernel/perf_cpum_cf.c | 2 +- arch/s390/kernel/perf_pai_crypto.c | 4 ++-- arch/s390/kernel/perf_pai_ext.c | 4 ++-- arch/s390/kernel/ptrace.c | 14 +++++++------- arch/s390/kernel/setup.c | 14 +++++++------- arch/s390/kernel/smp.c | 12 ++++++------ arch/s390/kernel/time.c | 4 ++-- arch/s390/kvm/kvm-s390.c | 6 +++--- arch/s390/kvm/priv.c | 2 +- arch/s390/lib/uaccess.c | 4 ++-- arch/s390/mm/pgalloc.c | 2 +- arch/s390/mm/vmem.c | 2 +- arch/s390/pci/pci.c | 2 +- drivers/s390/char/sclp.c | 6 +++--- drivers/s390/char/sclp_early_core.c | 6 +++--- drivers/s390/cio/crw.c | 2 +- drivers/s390/cio/isc.c | 4 ++-- net/iucv/iucv.c | 4 ++-- 30 files changed, 109 insertions(+), 109 deletions(-) diff --git a/arch/s390/boot/startup.c b/arch/s390/boot/startup.c index d3e48bd9c394..cf622479647f 100644 --- a/arch/s390/boot/startup.c +++ b/arch/s390/boot/startup.c @@ -49,7 +49,7 @@ static void detect_facilities(void) { if (test_facility(8)) { machine.has_edat1 = 1; - __ctl_set_bit(0, 23); + local_ctl_set_bit(0, 23); } if (test_facility(78)) machine.has_edat2 = 1; diff --git a/arch/s390/boot/vmem.c b/arch/s390/boot/vmem.c index 9ac6d00537c1..666fc4e50c1e 100644 --- a/arch/s390/boot/vmem.c +++ b/arch/s390/boot/vmem.c @@ -446,9 +446,9 @@ void setup_vmem(unsigned long asce_limit) S390_lowcore.kernel_asce = swapper_pg_dir | asce_bits; S390_lowcore.user_asce = s390_invalid_asce; - __ctl_load(S390_lowcore.kernel_asce, 1, 1); - __ctl_load(S390_lowcore.user_asce, 7, 7); - __ctl_load(S390_lowcore.kernel_asce, 13, 13); + __local_ctl_load(S390_lowcore.kernel_asce, 1, 1); + __local_ctl_load(S390_lowcore.user_asce, 7, 7); + __local_ctl_load(S390_lowcore.kernel_asce, 13, 13); init_mm.context.asce = S390_lowcore.kernel_asce; } diff --git a/arch/s390/include/asm/ctlreg.h b/arch/s390/include/asm/ctlreg.h index 0b55a1bb3113..09d35ab3d1ce 100644 --- a/arch/s390/include/asm/ctlreg.h +++ b/arch/s390/include/asm/ctlreg.h @@ -35,7 +35,7 @@ #include -#define __ctl_load(array, low, high) do { \ +#define __local_ctl_load(array, low, high) do { \ struct addrtype { \ char _[sizeof(array)]; \ }; \ @@ -53,7 +53,7 @@ : "memory"); \ } while (0) -#define __ctl_store(array, low, high) do { \ +#define __local_ctl_store(array, low, high) do { \ struct addrtype { \ char _[sizeof(array)]; \ }; \ @@ -69,36 +69,36 @@ : [_low] "i" (low), [_high] "i" (high)); \ } while (0) -static __always_inline void __ctl_set_bit(unsigned int cr, unsigned int bit) +static __always_inline void local_ctl_set_bit(unsigned int cr, unsigned int bit) { unsigned long reg; - __ctl_store(reg, cr, cr); + __local_ctl_store(reg, cr, cr); reg |= 1UL << bit; - __ctl_load(reg, cr, cr); + __local_ctl_load(reg, cr, cr); } -static __always_inline void __ctl_clear_bit(unsigned int cr, unsigned int bit) +static __always_inline void local_ctl_clear_bit(unsigned int cr, unsigned int bit) { unsigned long reg; - __ctl_store(reg, cr, cr); + __local_ctl_store(reg, cr, cr); reg &= ~(1UL << bit); - __ctl_load(reg, cr, cr); + __local_ctl_load(reg, cr, cr); } -void ctlreg_lock(void); -void ctlreg_unlock(void); -void ctl_set_clear_bit(int cr, int bit, bool set); +void system_ctlreg_lock(void); +void system_ctlreg_unlock(void); +void system_ctl_set_clear_bit(unsigned int cr, unsigned int bit, bool set); -static inline void ctl_set_bit(int cr, int bit) +static inline void system_ctl_set_bit(unsigned int cr, unsigned int bit) { - ctl_set_clear_bit(cr, bit, true); + system_ctl_set_clear_bit(cr, bit, true); } -static inline void ctl_clear_bit(int cr, int bit) +static inline void system_ctl_clear_bit(unsigned int cr, unsigned int bit) { - ctl_set_clear_bit(cr, bit, false); + system_ctl_set_clear_bit(cr, bit, false); } union ctlreg0 { diff --git a/arch/s390/include/asm/mmu_context.h b/arch/s390/include/asm/mmu_context.h index 50f247eede60..a72865fce0ad 100644 --- a/arch/s390/include/asm/mmu_context.h +++ b/arch/s390/include/asm/mmu_context.h @@ -81,7 +81,7 @@ static inline void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct * S390_lowcore.user_asce = next->context.asce; cpumask_set_cpu(cpu, &next->context.cpu_attach_mask); /* Clear previous user-ASCE from CR7 */ - __ctl_load(s390_invalid_asce, 7, 7); + __local_ctl_load(s390_invalid_asce, 7, 7); if (prev != next) cpumask_clear_cpu(cpu, &prev->context.cpu_attach_mask); } @@ -111,7 +111,7 @@ static inline void finish_arch_post_lock_switch(void) __tlb_flush_mm_lazy(mm); preempt_enable(); } - __ctl_load(S390_lowcore.user_asce, 7, 7); + __local_ctl_load(S390_lowcore.user_asce, 7, 7); } #define activate_mm activate_mm @@ -120,7 +120,7 @@ static inline void activate_mm(struct mm_struct *prev, { switch_mm(prev, next, current); cpumask_set_cpu(smp_processor_id(), mm_cpumask(next)); - __ctl_load(S390_lowcore.user_asce, 7, 7); + __local_ctl_load(S390_lowcore.user_asce, 7, 7); } #include diff --git a/arch/s390/kernel/ctlreg.c b/arch/s390/kernel/ctlreg.c index a01b901d4866..795e7f4861fb 100644 --- a/arch/s390/kernel/ctlreg.c +++ b/arch/s390/kernel/ctlreg.c @@ -13,18 +13,18 @@ * are kept in the control register save area within absolute lowcore * at physical address zero. */ -static DEFINE_SPINLOCK(ctl_lock); +static DEFINE_SPINLOCK(system_ctl_lock); -void ctlreg_lock(void) - __acquires(&ctl_lock) +void system_ctlreg_lock(void) + __acquires(&system_ctl_lock) { - spin_lock(&ctl_lock); + spin_lock(&system_ctl_lock); } -void ctlreg_unlock(void) - __releases(&ctl_lock) +void system_ctlreg_unlock(void) + __releases(&system_ctl_lock) { - spin_unlock(&ctl_lock); + spin_unlock(&system_ctl_lock); } struct ctl_bit_parms { @@ -38,25 +38,25 @@ static void ctl_bit_callback(void *info) struct ctl_bit_parms *pp = info; unsigned long regs[16]; - __ctl_store(regs, 0, 15); + __local_ctl_store(regs, 0, 15); regs[pp->cr] &= pp->andval; regs[pp->cr] |= pp->orval; - __ctl_load(regs, 0, 15); + __local_ctl_load(regs, 0, 15); } -void ctl_set_clear_bit(int cr, int bit, bool set) +void system_ctl_set_clear_bit(unsigned int cr, unsigned int bit, bool set) { struct ctl_bit_parms pp = { .cr = cr, }; struct lowcore *abs_lc; pp.orval = set ? 1UL << bit : 0; pp.andval = set ? -1UL : ~(1UL << bit); - ctlreg_lock(); + system_ctlreg_lock(); abs_lc = get_abs_lowcore(); abs_lc->cregs_save_area[cr] &= pp.andval; abs_lc->cregs_save_area[cr] |= pp.orval; put_abs_lowcore(abs_lc); on_each_cpu(ctl_bit_callback, &pp, 1); - ctlreg_unlock(); + system_ctlreg_unlock(); } -EXPORT_SYMBOL(ctl_set_clear_bit); +EXPORT_SYMBOL(system_ctl_set_clear_bit); diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c index 3a54733e4fc6..1f359309cce6 100644 --- a/arch/s390/kernel/early.c +++ b/arch/s390/kernel/early.c @@ -216,7 +216,7 @@ static __init void detect_machine_facilities(void) { if (test_facility(8)) { S390_lowcore.machine_flags |= MACHINE_FLAG_EDAT1; - __ctl_set_bit(0, 23); + local_ctl_set_bit(0, 23); } if (test_facility(78)) S390_lowcore.machine_flags |= MACHINE_FLAG_EDAT2; @@ -224,13 +224,13 @@ static __init void detect_machine_facilities(void) S390_lowcore.machine_flags |= MACHINE_FLAG_IDTE; if (test_facility(50) && test_facility(73)) { S390_lowcore.machine_flags |= MACHINE_FLAG_TE; - __ctl_set_bit(0, 55); + local_ctl_set_bit(0, 55); } if (test_facility(51)) S390_lowcore.machine_flags |= MACHINE_FLAG_TLB_LC; if (test_facility(129)) { S390_lowcore.machine_flags |= MACHINE_FLAG_VX; - __ctl_set_bit(0, 17); + local_ctl_set_bit(0, 17); } if (test_facility(130)) S390_lowcore.machine_flags |= MACHINE_FLAG_NX; @@ -240,7 +240,7 @@ static __init void detect_machine_facilities(void) /* Enabled signed clock comparator comparisons */ S390_lowcore.machine_flags |= MACHINE_FLAG_SCC; clock_comparator_max = -1ULL >> 1; - __ctl_set_bit(0, 53); + local_ctl_set_bit(0, 53); } if (IS_ENABLED(CONFIG_PCI) && test_facility(153)) { S390_lowcore.machine_flags |= MACHINE_FLAG_PCI_MIO; @@ -260,7 +260,7 @@ static inline void save_vector_registers(void) static inline void setup_low_address_protection(void) { - __ctl_set_bit(0, 28); + local_ctl_set_bit(0, 28); } static inline void setup_access_registers(void) @@ -273,7 +273,7 @@ static inline void setup_access_registers(void) static int __init disable_vector_extension(char *str) { S390_lowcore.machine_flags &= ~MACHINE_FLAG_VX; - __ctl_clear_bit(0, 17); + local_ctl_clear_bit(0, 17); return 0; } early_param("novx", disable_vector_extension); diff --git a/arch/s390/kernel/guarded_storage.c b/arch/s390/kernel/guarded_storage.c index d14dd1c2e524..945acfc77ee4 100644 --- a/arch/s390/kernel/guarded_storage.c +++ b/arch/s390/kernel/guarded_storage.c @@ -28,7 +28,7 @@ static int gs_enable(void) return -ENOMEM; gs_cb->gsd = 25; preempt_disable(); - __ctl_set_bit(2, 4); + local_ctl_set_bit(2, 4); load_gs_cb(gs_cb); current->thread.gs_cb = gs_cb; preempt_enable(); @@ -42,7 +42,7 @@ static int gs_disable(void) preempt_disable(); kfree(current->thread.gs_cb); current->thread.gs_cb = NULL; - __ctl_clear_bit(2, 4); + local_ctl_clear_bit(2, 4); preempt_enable(); } return 0; @@ -84,7 +84,7 @@ void gs_load_bc_cb(struct pt_regs *regs) if (gs_cb) { kfree(current->thread.gs_cb); current->thread.gs_bc_cb = NULL; - __ctl_set_bit(2, 4); + local_ctl_set_bit(2, 4); load_gs_cb(gs_cb); current->thread.gs_cb = gs_cb; } diff --git a/arch/s390/kernel/ipl.c b/arch/s390/kernel/ipl.c index 05e51666db03..492dd8f371da 100644 --- a/arch/s390/kernel/ipl.c +++ b/arch/s390/kernel/ipl.c @@ -2381,7 +2381,7 @@ void s390_reset_system(void) set_prefix(0); /* Disable lowcore protection */ - __ctl_clear_bit(0, 28); + local_ctl_clear_bit(0, 28); diag_amode31_ops.diag308_reset(); } diff --git a/arch/s390/kernel/irq.c b/arch/s390/kernel/irq.c index b020ff17d206..6f71b0ce1068 100644 --- a/arch/s390/kernel/irq.c +++ b/arch/s390/kernel/irq.c @@ -385,7 +385,7 @@ void irq_subclass_register(enum irq_subclass subclass) { spin_lock(&irq_subclass_lock); if (!irq_subclass_refcount[subclass]) - ctl_set_bit(0, subclass); + system_ctl_set_bit(0, subclass); irq_subclass_refcount[subclass]++; spin_unlock(&irq_subclass_lock); } @@ -396,7 +396,7 @@ void irq_subclass_unregister(enum irq_subclass subclass) spin_lock(&irq_subclass_lock); irq_subclass_refcount[subclass]--; if (!irq_subclass_refcount[subclass]) - ctl_clear_bit(0, subclass); + system_ctl_clear_bit(0, subclass); spin_unlock(&irq_subclass_lock); } EXPORT_SYMBOL(irq_subclass_unregister); diff --git a/arch/s390/kernel/kprobes.c b/arch/s390/kernel/kprobes.c index d4b863ed0aa7..1b2e1903a54c 100644 --- a/arch/s390/kernel/kprobes.c +++ b/arch/s390/kernel/kprobes.c @@ -232,12 +232,12 @@ static void enable_singlestep(struct kprobe_ctlblk *kcb, per_kprobe.end = ip; /* Save control regs and psw mask */ - __ctl_store(kcb->kprobe_saved_ctl, 9, 11); + __local_ctl_store(kcb->kprobe_saved_ctl, 9, 11); kcb->kprobe_saved_imask = regs->psw.mask & (PSW_MASK_PER | PSW_MASK_IO | PSW_MASK_EXT); /* Set PER control regs, turns on single step for the given address */ - __ctl_load(per_kprobe, 9, 11); + __local_ctl_load(per_kprobe, 9, 11); regs->psw.mask |= PSW_MASK_PER; regs->psw.mask &= ~(PSW_MASK_IO | PSW_MASK_EXT); regs->psw.addr = ip; @@ -249,7 +249,7 @@ static void disable_singlestep(struct kprobe_ctlblk *kcb, unsigned long ip) { /* Restore control regs and psw mask, set new psw address */ - __ctl_load(kcb->kprobe_saved_ctl, 9, 11); + __local_ctl_load(kcb->kprobe_saved_ctl, 9, 11); regs->psw.mask &= ~PSW_MASK_PER; regs->psw.mask |= kcb->kprobe_saved_imask; regs->psw.addr = ip; diff --git a/arch/s390/kernel/machine_kexec.c b/arch/s390/kernel/machine_kexec.c index ce65fc01671f..07152e74afb1 100644 --- a/arch/s390/kernel/machine_kexec.c +++ b/arch/s390/kernel/machine_kexec.c @@ -94,12 +94,12 @@ static noinline void __machine_kdump(void *image) if (MACHINE_HAS_VX) save_vx_regs((__vector128 *) mcesa->vector_save_area); if (MACHINE_HAS_GS) { - __ctl_store(cr2_old.val, 2, 2); + __local_ctl_store(cr2_old.val, 2, 2); cr2_new = cr2_old; cr2_new.gse = 1; - __ctl_load(cr2_new.val, 2, 2); + __local_ctl_load(cr2_new.val, 2, 2); save_gs_cb((struct gs_cb *) mcesa->guarded_storage_save_area); - __ctl_load(cr2_old.val, 2, 2); + __local_ctl_load(cr2_old.val, 2, 2); } /* * To create a good backchain for this CPU in the dump store_status diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c index b813d9dfa53c..927bd83ac9c0 100644 --- a/arch/s390/kernel/nmi.c +++ b/arch/s390/kernel/nmi.c @@ -131,10 +131,10 @@ static notrace void s390_handle_damage(void) * Disable low address protection and make machine check new PSW a * disabled wait PSW. Any additional machine check cannot be handled. */ - __ctl_store(cr0.val, 0, 0); + __local_ctl_store(cr0.val, 0, 0); cr0_new = cr0; cr0_new.lap = 0; - __ctl_load(cr0_new.val, 0, 0); + __local_ctl_load(cr0_new.val, 0, 0); psw_save = S390_lowcore.mcck_new_psw; psw_bits(S390_lowcore.mcck_new_psw).io = 0; psw_bits(S390_lowcore.mcck_new_psw).ext = 0; @@ -146,7 +146,7 @@ static notrace void s390_handle_damage(void) * values. This makes possible system dump analysis easier. */ S390_lowcore.mcck_new_psw = psw_save; - __ctl_load(cr0.val, 0, 0); + __local_ctl_load(cr0.val, 0, 0); disabled_wait(); while (1); } @@ -185,7 +185,7 @@ void s390_handle_mcck(void) static int mchchk_wng_posted = 0; /* Use single cpu clear, as we cannot handle smp here. */ - __ctl_clear_bit(14, 24); /* Disable WARNING MCH */ + local_ctl_clear_bit(14, 24); /* Disable WARNING MCH */ if (xchg(&mchchk_wng_posted, 1) == 0) kill_cad_pid(SIGPWR, 1); } @@ -271,7 +271,7 @@ static int notrace s390_validate_registers(union mci mci) kill_task = 1; cr0.val = S390_lowcore.cregs_save_area[0]; cr0.afp = cr0.vx = 1; - __ctl_load(cr0.val, 0, 0); + __local_ctl_load(cr0.val, 0, 0); asm volatile( " la 1,%0\n" " VLM 0,15,0,1\n" @@ -279,7 +279,7 @@ static int notrace s390_validate_registers(union mci mci) : : "Q" (*(struct vx_array *)mcesa->vector_save_area) : "1"); - __ctl_load(S390_lowcore.cregs_save_area[0], 0, 0); + __local_ctl_load(S390_lowcore.cregs_save_area[0], 0, 0); } /* Validate access registers */ asm volatile( @@ -505,9 +505,9 @@ NOKPROBE_SYMBOL(s390_do_machine_check); static int __init machine_check_init(void) { - ctl_set_bit(14, 25); /* enable external damage MCH */ - ctl_set_bit(14, 27); /* enable system recovery MCH */ - ctl_set_bit(14, 24); /* enable warning MCH */ + system_ctl_set_bit(14, 25); /* enable external damage MCH */ + system_ctl_set_bit(14, 27); /* enable system recovery MCH */ + system_ctl_set_bit(14, 24); /* enable warning MCH */ return 0; } early_initcall(machine_check_init); diff --git a/arch/s390/kernel/perf_cpum_cf.c b/arch/s390/kernel/perf_cpum_cf.c index 850c11ea631a..4e05c0cb1648 100644 --- a/arch/s390/kernel/perf_cpum_cf.c +++ b/arch/s390/kernel/perf_cpum_cf.c @@ -1193,7 +1193,7 @@ static int __init cpumf_pmu_init(void) * Clear bit 15 of cr0 to unauthorize problem-state to * extract measurement counters */ - ctl_clear_bit(0, 48); + system_ctl_clear_bit(0, 48); /* register handler for measurement-alert interruptions */ rc = register_external_irq(EXT_IRQ_MEASURE_ALERT, diff --git a/arch/s390/kernel/perf_pai_crypto.c b/arch/s390/kernel/perf_pai_crypto.c index 6280a0e588c5..6612fd13e95b 100644 --- a/arch/s390/kernel/perf_pai_crypto.c +++ b/arch/s390/kernel/perf_pai_crypto.c @@ -265,7 +265,7 @@ static int paicrypt_add(struct perf_event *event, int flags) if (++cpump->active_events == 1) { ccd = virt_to_phys(cpump->page) | PAI_CRYPTO_KERNEL_OFFSET; WRITE_ONCE(S390_lowcore.ccd, ccd); - __ctl_set_bit(0, 50); + local_ctl_set_bit(0, 50); } cpump->event = event; if (flags & PERF_EF_START && !event->attr.sample_period) { @@ -294,7 +294,7 @@ static void paicrypt_del(struct perf_event *event, int flags) /* Only counting needs to read counter */ paicrypt_stop(event, PERF_EF_UPDATE); if (--cpump->active_events == 0) { - __ctl_clear_bit(0, 50); + local_ctl_clear_bit(0, 50); WRITE_ONCE(S390_lowcore.ccd, 0); } } diff --git a/arch/s390/kernel/perf_pai_ext.c b/arch/s390/kernel/perf_pai_ext.c index b8877493dbb6..9f724ae44fc7 100644 --- a/arch/s390/kernel/perf_pai_ext.c +++ b/arch/s390/kernel/perf_pai_ext.c @@ -339,7 +339,7 @@ static int paiext_add(struct perf_event *event, int flags) S390_lowcore.aicd = virt_to_phys(cpump->paiext_cb); pcb->acc = virt_to_phys(cpump->area) | 0x1; /* Enable CPU instruction lookup for PAIE1 control block */ - __ctl_set_bit(0, 49); + local_ctl_set_bit(0, 49); debug_sprintf_event(paiext_dbg, 4, "%s 1508 %llx acc %llx\n", __func__, S390_lowcore.aicd, pcb->acc); } @@ -375,7 +375,7 @@ static void paiext_del(struct perf_event *event, int flags) } if (--cpump->active_events == 0) { /* Disable CPU instruction lookup for PAIE1 control block */ - __ctl_clear_bit(0, 49); + local_ctl_clear_bit(0, 49); pcb->acc = 0; S390_lowcore.aicd = 0; debug_sprintf_event(paiext_dbg, 4, "%s 1508 %llx acc %llx\n", diff --git a/arch/s390/kernel/ptrace.c b/arch/s390/kernel/ptrace.c index ea244a73efad..1824c1208852 100644 --- a/arch/s390/kernel/ptrace.c +++ b/arch/s390/kernel/ptrace.c @@ -46,8 +46,8 @@ void update_cr_regs(struct task_struct *task) union ctlreg2 cr2_old, cr2_new; int cr0_changed, cr2_changed; - __ctl_store(cr0_old.val, 0, 0); - __ctl_store(cr2_old.val, 2, 2); + __local_ctl_store(cr0_old.val, 0, 0); + __local_ctl_store(cr2_old.val, 2, 2); cr0_new = cr0_old; cr2_new = cr2_old; /* Take care of the enable/disable of transactional execution. */ @@ -75,9 +75,9 @@ void update_cr_regs(struct task_struct *task) cr0_changed = cr0_new.val != cr0_old.val; cr2_changed = cr2_new.val != cr2_old.val; if (cr0_changed) - __ctl_load(cr0_new.val, 0, 0); + __local_ctl_load(cr0_new.val, 0, 0); if (cr2_changed) - __ctl_load(cr2_new.val, 2, 2); + __local_ctl_load(cr2_new.val, 2, 2); /* Copy user specified PER registers */ new.control = thread->per_user.control; new.start = thread->per_user.start; @@ -104,9 +104,9 @@ void update_cr_regs(struct task_struct *task) return; } regs->psw.mask |= PSW_MASK_PER; - __ctl_store(old, 9, 11); + __local_ctl_store(old, 9, 11); if (memcmp(&new, &old, sizeof(struct per_regs)) != 0) - __ctl_load(new, 9, 11); + __local_ctl_load(new, 9, 11); } void user_enable_single_step(struct task_struct *task) @@ -1107,7 +1107,7 @@ static int s390_gs_cb_set(struct task_struct *target, target->thread.gs_cb = data; *target->thread.gs_cb = gs_cb; if (target == current) { - __ctl_set_bit(2, 4); + local_ctl_set_bit(2, 4); restore_gs_cb(target->thread.gs_cb); } preempt_enable(); diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index ef158bd0ea7b..c516e2e909e6 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -449,7 +449,7 @@ static void __init setup_lowcore(void) lc->restart_fn = (unsigned long) do_restart; lc->restart_data = 0; lc->restart_source = -1U; - __ctl_store(lc->cregs_save_area, 0, 15); + __local_ctl_store(lc->cregs_save_area, 0, 15); lc->spinlock_lockval = arch_spin_lockval(0); lc->spinlock_index = 0; arch_spin_lock_setup(0); @@ -791,15 +791,15 @@ static void __init setup_cr(void) __ctl_duct[4] = (unsigned long)__ctl_duald; /* Update control registers CR2, CR5 and CR15 */ - __ctl_store(cr2.val, 2, 2); - __ctl_store(cr5.val, 5, 5); - __ctl_store(cr15.val, 15, 15); + __local_ctl_store(cr2.val, 2, 2); + __local_ctl_store(cr5.val, 5, 5); + __local_ctl_store(cr15.val, 15, 15); cr2.ducto = (unsigned long)__ctl_duct >> 6; cr5.pasteo = (unsigned long)__ctl_duct >> 6; cr15.lsea = (unsigned long)__ctl_linkage_stack >> 3; - __ctl_load(cr2.val, 2, 2); - __ctl_load(cr5.val, 5, 5); - __ctl_load(cr15.val, 15, 15); + __local_ctl_load(cr2.val, 2, 2); + __local_ctl_load(cr5.val, 5, 5); + __local_ctl_load(cr15.val, 15, 15); } /* diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index 204c72428696..1c65bbd411c7 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -888,14 +888,14 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle) * Make sure global control register contents do not change * until new CPU has initialized control registers. */ - ctlreg_lock(); + system_ctlreg_lock(); pcpu_prepare_secondary(pcpu, cpu); pcpu_attach_task(pcpu, tidle); pcpu_start_fn(pcpu, smp_start_secondary, NULL); /* Wait until cpu puts itself in the online & active maps */ while (!cpu_online(cpu)) cpu_relax(); - ctlreg_unlock(); + system_ctlreg_unlock(); return 0; } @@ -922,11 +922,11 @@ int __cpu_disable(void) /* Disable pseudo page faults on this cpu. */ pfault_fini(); /* Disable interrupt sources via control register. */ - __ctl_store(cregs, 0, 15); + __local_ctl_store(cregs, 0, 15); cregs[0] &= ~0x0000ee70UL; /* disable all external interrupts */ cregs[6] &= ~0xff000000UL; /* disable all I/O interrupts */ cregs[14] &= ~0x1f000000UL; /* disable most machine checks */ - __ctl_load(cregs, 0, 15); + __local_ctl_load(cregs, 0, 15); clear_cpu_flag(CIF_NOHZ_DELAY); return 0; } @@ -968,10 +968,10 @@ void __init smp_prepare_cpus(unsigned int max_cpus) { if (register_external_irq(EXT_IRQ_EMERGENCY_SIG, do_ext_call_interrupt)) panic("Couldn't request external interrupt 0x1201"); - ctl_set_bit(0, 14); + system_ctl_set_bit(0, 14); if (register_external_irq(EXT_IRQ_EXTERNAL_CALL, do_ext_call_interrupt)) panic("Couldn't request external interrupt 0x1202"); - ctl_set_bit(0, 13); + system_ctl_set_bit(0, 13); } void __init smp_prepare_boot_cpu(void) diff --git a/arch/s390/kernel/time.c b/arch/s390/kernel/time.c index d34d3548c046..34c4fbf4e555 100644 --- a/arch/s390/kernel/time.c +++ b/arch/s390/kernel/time.c @@ -173,10 +173,10 @@ void init_cpu_timer(void) clockevents_register_device(cd); /* Enable clock comparator timer interrupt. */ - __ctl_set_bit(0,11); + local_ctl_set_bit(0, 11); /* Always allow the timing alert external interrupt. */ - __ctl_set_bit(0, 4); + local_ctl_set_bit(0, 4); } static void clock_comparator_interrupt(struct ext_code ext_code, diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index b3f17e014cab..92bd15fe769a 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -4918,7 +4918,7 @@ static void sync_regs_fmt2(struct kvm_vcpu *vcpu) } if (MACHINE_HAS_GS) { preempt_disable(); - __ctl_set_bit(2, 4); + local_ctl_set_bit(2, 4); if (current->thread.gs_cb) { vcpu->arch.host_gscb = current->thread.gs_cb; save_gs_cb(vcpu->arch.host_gscb); @@ -4995,13 +4995,13 @@ static void store_regs_fmt2(struct kvm_vcpu *vcpu) kvm_run->s.regs.diag318 = vcpu->arch.diag318_info.val; if (MACHINE_HAS_GS) { preempt_disable(); - __ctl_set_bit(2, 4); + local_ctl_set_bit(2, 4); if (vcpu->arch.gs_enabled) save_gs_cb(current->thread.gs_cb); current->thread.gs_cb = vcpu->arch.host_gscb; restore_gs_cb(vcpu->arch.host_gscb); if (!vcpu->arch.host_gscb) - __ctl_clear_bit(2, 4); + local_ctl_clear_bit(2, 4); vcpu->arch.host_gscb = NULL; preempt_enable(); } diff --git a/arch/s390/kvm/priv.c b/arch/s390/kvm/priv.c index dc4cfa8795c0..c199e4b5e5ce 100644 --- a/arch/s390/kvm/priv.c +++ b/arch/s390/kvm/priv.c @@ -57,7 +57,7 @@ static int handle_gs(struct kvm_vcpu *vcpu) if (test_kvm_facility(vcpu->kvm, 133)) { VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (lazy)"); preempt_disable(); - __ctl_set_bit(2, 4); + local_ctl_set_bit(2, 4); current->thread.gs_cb = (struct gs_cb *)&vcpu->run->s.regs.gscb; restore_gs_cb(current->thread.gs_cb); preempt_enable(); diff --git a/arch/s390/lib/uaccess.c b/arch/s390/lib/uaccess.c index 478ab4c7d197..551de5394be6 100644 --- a/arch/s390/lib/uaccess.c +++ b/arch/s390/lib/uaccess.c @@ -19,8 +19,8 @@ void debug_user_asce(int exit) { unsigned long cr1, cr7; - __ctl_store(cr1, 1, 1); - __ctl_store(cr7, 7, 7); + __local_ctl_store(cr1, 1, 1); + __local_ctl_store(cr7, 7, 7); if (cr1 == S390_lowcore.kernel_asce && cr7 == S390_lowcore.user_asce) return; panic("incorrect ASCE on kernel %s\n" diff --git a/arch/s390/mm/pgalloc.c b/arch/s390/mm/pgalloc.c index 07fc660a24aa..43bef9cce9b2 100644 --- a/arch/s390/mm/pgalloc.c +++ b/arch/s390/mm/pgalloc.c @@ -63,7 +63,7 @@ static void __crst_table_upgrade(void *arg) /* change all active ASCEs to avoid the creation of new TLBs */ if (current->active_mm == mm) { S390_lowcore.user_asce = mm->context.asce; - __ctl_load(S390_lowcore.user_asce, 7, 7); + __local_ctl_load(S390_lowcore.user_asce, 7, 7); } __tlb_flush_local(); } diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c index 91c1f42f6a76..95f04a502e62 100644 --- a/arch/s390/mm/vmem.c +++ b/arch/s390/mm/vmem.c @@ -660,7 +660,7 @@ void __init vmem_map_init(void) __set_memory_4k(__va(0), RELOC_HIDE(__va(0), ident_map_size)); } if (MACHINE_HAS_NX) - ctl_set_bit(0, 20); + system_ctl_set_bit(0, 20); pr_info("Write protected kernel read-only data: %luk\n", (unsigned long)(__end_rodata - _stext) >> 10); } diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c index d34d5813d006..a0aecc94e284 100644 --- a/arch/s390/pci/pci.c +++ b/arch/s390/pci/pci.c @@ -1094,7 +1094,7 @@ static int __init pci_base_init(void) if (MACHINE_HAS_PCI_MIO) { static_branch_enable(&have_mio); - ctl_set_bit(2, 5); + system_ctl_set_bit(2, 5); } rc = zpci_debug_init(); diff --git a/drivers/s390/char/sclp.c b/drivers/s390/char/sclp.c index 8f74db689a0c..1ebc60d9cedc 100644 --- a/drivers/s390/char/sclp.c +++ b/drivers/s390/char/sclp.c @@ -732,10 +732,10 @@ sclp_sync_wait(void) /* Enable service-signal interruption, disable timer interrupts */ old_tick = local_tick_disable(); trace_hardirqs_on(); - __ctl_store(cr0, 0, 0); + __local_ctl_store(cr0, 0, 0); cr0_sync = cr0 & ~CR0_IRQ_SUBCLASS_MASK; cr0_sync |= 1UL << (63 - 54); - __ctl_load(cr0_sync, 0, 0); + __local_ctl_load(cr0_sync, 0, 0); __arch_local_irq_stosm(0x01); /* Loop until driver state indicates finished request */ while (sclp_running_state != sclp_running_state_idle) { @@ -745,7 +745,7 @@ sclp_sync_wait(void) cpu_relax(); } local_irq_disable(); - __ctl_load(cr0, 0, 0); + __local_ctl_load(cr0, 0, 0); if (!irq_context) _local_bh_enable(); local_tick_enable(old_tick); diff --git a/drivers/s390/char/sclp_early_core.c b/drivers/s390/char/sclp_early_core.c index f7067a70c461..fa5b7d63a254 100644 --- a/drivers/s390/char/sclp_early_core.c +++ b/drivers/s390/char/sclp_early_core.c @@ -32,11 +32,11 @@ void sclp_early_wait_irq(void) psw_t psw_ext_save, psw_wait; union ctlreg0 cr0, cr0_new; - __ctl_store(cr0.val, 0, 0); + __local_ctl_store(cr0.val, 0, 0); cr0_new.val = cr0.val & ~CR0_IRQ_SUBCLASS_MASK; cr0_new.lap = 0; cr0_new.sssm = 1; - __ctl_load(cr0_new.val, 0, 0); + __local_ctl_load(cr0_new.val, 0, 0); psw_ext_save = S390_lowcore.external_new_psw; psw_mask = __extract_psw(); @@ -59,7 +59,7 @@ void sclp_early_wait_irq(void) } while (S390_lowcore.ext_int_code != EXT_IRQ_SERVICE_SIG); S390_lowcore.external_new_psw = psw_ext_save; - __ctl_load(cr0.val, 0, 0); + __local_ctl_load(cr0.val, 0, 0); } int sclp_early_cmd(sclp_cmdw_t cmd, void *sccb) diff --git a/drivers/s390/cio/crw.c b/drivers/s390/cio/crw.c index 58f22270d9c4..872b1f2855cb 100644 --- a/drivers/s390/cio/crw.c +++ b/drivers/s390/cio/crw.c @@ -156,7 +156,7 @@ static int __init crw_machine_check_init(void) task = kthread_run(crw_collect_info, NULL, "kmcheck"); if (IS_ERR(task)) return PTR_ERR(task); - ctl_set_bit(14, 28); /* enable channel report MCH */ + system_ctl_set_bit(14, 28); /* enable channel report MCH */ return 0; } device_initcall(crw_machine_check_init); diff --git a/drivers/s390/cio/isc.c b/drivers/s390/cio/isc.c index 77fde9f5ea8b..dbc2ac7711e5 100644 --- a/drivers/s390/cio/isc.c +++ b/drivers/s390/cio/isc.c @@ -33,7 +33,7 @@ void isc_register(unsigned int isc) spin_lock(&isc_ref_lock); if (isc_refs[isc] == 0) - ctl_set_bit(6, 31 - isc); + system_ctl_set_bit(6, 31 - isc); isc_refs[isc]++; spin_unlock(&isc_ref_lock); } @@ -61,7 +61,7 @@ void isc_unregister(unsigned int isc) goto out_unlock; } if (isc_refs[isc] == 1) - ctl_clear_bit(6, 31 - isc); + system_ctl_clear_bit(6, 31 - isc); isc_refs[isc]--; out_unlock: spin_unlock(&isc_ref_lock); diff --git a/net/iucv/iucv.c b/net/iucv/iucv.c index fc3fddeb6f36..b5cee8dad926 100644 --- a/net/iucv/iucv.c +++ b/net/iucv/iucv.c @@ -1823,7 +1823,7 @@ static int __init iucv_init(void) rc = -EPROTONOSUPPORT; goto out; } - ctl_set_bit(0, 1); + system_ctl_set_bit(0, 1); rc = iucv_query_maxconn(); if (rc) goto out_ctl; @@ -1871,7 +1871,7 @@ out_dev: out_int: unregister_external_irq(EXT_IRQ_IUCV, iucv_external_interrupt); out_ctl: - ctl_clear_bit(0, 1); + system_ctl_clear_bit(0, 1); out: return rc; } From dfa33ce1245a4b88402947fa0a847e179044d2fc Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 21:39:59 +0200 Subject: [PATCH 131/475] s390/ctlreg: add local_ctl_load() and local_ctl_store() Add local_ctl_load() and local_ctl_store() which load and store contents for only a single control register. This allows for easier to read code, but also better type checking, since __local_ctl_load() and __local_ctl_store() do not come with any type checking at all (which will be changed with subsequent patches). Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/ctlreg.h | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/arch/s390/include/asm/ctlreg.h b/arch/s390/include/asm/ctlreg.h index 09d35ab3d1ce..e8e5aef08bfd 100644 --- a/arch/s390/include/asm/ctlreg.h +++ b/arch/s390/include/asm/ctlreg.h @@ -69,6 +69,23 @@ : [_low] "i" (low), [_high] "i" (high)); \ } while (0) +static __always_inline void local_ctl_load(unsigned int cr, unsigned long *reg) +{ + asm volatile( + " lctlg %[cr],%[cr],%[reg]\n" + : + : [reg] "Q" (*reg), [cr] "i" (cr) + : "memory"); +} + +static __always_inline void local_ctl_store(unsigned int cr, unsigned long *reg) +{ + asm volatile( + " stctg %[cr],%[cr],%[reg]\n" + : [reg] "=Q" (*reg) + : [cr] "i" (cr)); +} + static __always_inline void local_ctl_set_bit(unsigned int cr, unsigned int bit) { unsigned long reg; From 2372d391421350e318c98844d21ab9ad16e3eac0 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 21:40:00 +0200 Subject: [PATCH 132/475] s390/ctlreg: use local_ctl_load() and local_ctl_store() where possible Convert all single control register usages of __local_ctl_load() and __local_ctl_store() to local_ctl_load() and local_ctl_store(). This also requires to change the type of some struct lowcore members from __u64 to unsigned long. Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/boot/vmem.c | 6 +++--- arch/s390/include/asm/ctlreg.h | 8 ++++---- arch/s390/include/asm/lowcore.h | 6 +++--- arch/s390/include/asm/mmu_context.h | 6 +++--- arch/s390/kernel/machine_kexec.c | 6 +++--- arch/s390/kernel/nmi.c | 10 +++++----- arch/s390/kernel/ptrace.c | 8 ++++---- arch/s390/kernel/setup.c | 12 ++++++------ arch/s390/lib/uaccess.c | 6 +++--- arch/s390/mm/pgalloc.c | 2 +- drivers/s390/char/sclp.c | 6 +++--- drivers/s390/char/sclp_early_core.c | 6 +++--- 12 files changed, 41 insertions(+), 41 deletions(-) diff --git a/arch/s390/boot/vmem.c b/arch/s390/boot/vmem.c index 666fc4e50c1e..44278ae2512f 100644 --- a/arch/s390/boot/vmem.c +++ b/arch/s390/boot/vmem.c @@ -446,9 +446,9 @@ void setup_vmem(unsigned long asce_limit) S390_lowcore.kernel_asce = swapper_pg_dir | asce_bits; S390_lowcore.user_asce = s390_invalid_asce; - __local_ctl_load(S390_lowcore.kernel_asce, 1, 1); - __local_ctl_load(S390_lowcore.user_asce, 7, 7); - __local_ctl_load(S390_lowcore.kernel_asce, 13, 13); + local_ctl_load(1, &S390_lowcore.kernel_asce); + local_ctl_load(7, &S390_lowcore.user_asce); + local_ctl_load(13, &S390_lowcore.kernel_asce); init_mm.context.asce = S390_lowcore.kernel_asce; } diff --git a/arch/s390/include/asm/ctlreg.h b/arch/s390/include/asm/ctlreg.h index e8e5aef08bfd..86887b490bb5 100644 --- a/arch/s390/include/asm/ctlreg.h +++ b/arch/s390/include/asm/ctlreg.h @@ -90,18 +90,18 @@ static __always_inline void local_ctl_set_bit(unsigned int cr, unsigned int bit) { unsigned long reg; - __local_ctl_store(reg, cr, cr); + local_ctl_store(cr, ®); reg |= 1UL << bit; - __local_ctl_load(reg, cr, cr); + local_ctl_load(cr, ®); } static __always_inline void local_ctl_clear_bit(unsigned int cr, unsigned int bit) { unsigned long reg; - __local_ctl_store(reg, cr, cr); + local_ctl_store(cr, ®); reg &= ~(1UL << bit); - __local_ctl_load(reg, cr, cr); + local_ctl_load(cr, ®); } void system_ctlreg_lock(void); diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h index 69ccc464a430..2174f00e188b 100644 --- a/arch/s390/include/asm/lowcore.h +++ b/arch/s390/include/asm/lowcore.h @@ -139,8 +139,8 @@ struct lowcore { __u32 restart_flags; /* 0x0384 */ /* Address space pointer. */ - __u64 kernel_asce; /* 0x0388 */ - __u64 user_asce; /* 0x0390 */ + unsigned long kernel_asce; /* 0x0388 */ + unsigned long user_asce; /* 0x0390 */ /* * The lpp and current_pid fields form a @@ -199,7 +199,7 @@ struct lowcore { __u32 clock_comp_save_area[2]; /* 0x1330 */ __u64 last_break_save_area; /* 0x1338 */ __u32 access_regs_save_area[16]; /* 0x1340 */ - __u64 cregs_save_area[16]; /* 0x1380 */ + unsigned long cregs_save_area[16]; /* 0x1380 */ __u8 pad_0x1400[0x1500-0x1400]; /* 0x1400 */ /* Cryptography-counter designation */ __u64 ccd; /* 0x1500 */ diff --git a/arch/s390/include/asm/mmu_context.h b/arch/s390/include/asm/mmu_context.h index a72865fce0ad..0e93275f80f0 100644 --- a/arch/s390/include/asm/mmu_context.h +++ b/arch/s390/include/asm/mmu_context.h @@ -81,7 +81,7 @@ static inline void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct * S390_lowcore.user_asce = next->context.asce; cpumask_set_cpu(cpu, &next->context.cpu_attach_mask); /* Clear previous user-ASCE from CR7 */ - __local_ctl_load(s390_invalid_asce, 7, 7); + local_ctl_load(7, &s390_invalid_asce); if (prev != next) cpumask_clear_cpu(cpu, &prev->context.cpu_attach_mask); } @@ -111,7 +111,7 @@ static inline void finish_arch_post_lock_switch(void) __tlb_flush_mm_lazy(mm); preempt_enable(); } - __local_ctl_load(S390_lowcore.user_asce, 7, 7); + local_ctl_load(7, &S390_lowcore.user_asce); } #define activate_mm activate_mm @@ -120,7 +120,7 @@ static inline void activate_mm(struct mm_struct *prev, { switch_mm(prev, next, current); cpumask_set_cpu(smp_processor_id(), mm_cpumask(next)); - __local_ctl_load(S390_lowcore.user_asce, 7, 7); + local_ctl_load(7, &S390_lowcore.user_asce); } #include diff --git a/arch/s390/kernel/machine_kexec.c b/arch/s390/kernel/machine_kexec.c index 07152e74afb1..fe9d65060fa4 100644 --- a/arch/s390/kernel/machine_kexec.c +++ b/arch/s390/kernel/machine_kexec.c @@ -94,12 +94,12 @@ static noinline void __machine_kdump(void *image) if (MACHINE_HAS_VX) save_vx_regs((__vector128 *) mcesa->vector_save_area); if (MACHINE_HAS_GS) { - __local_ctl_store(cr2_old.val, 2, 2); + local_ctl_store(2, &cr2_old.val); cr2_new = cr2_old; cr2_new.gse = 1; - __local_ctl_load(cr2_new.val, 2, 2); + local_ctl_load(2, &cr2_new.val); save_gs_cb((struct gs_cb *) mcesa->guarded_storage_save_area); - __local_ctl_load(cr2_old.val, 2, 2); + local_ctl_load(2, &cr2_old.val); } /* * To create a good backchain for this CPU in the dump store_status diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c index 927bd83ac9c0..7880a42896a3 100644 --- a/arch/s390/kernel/nmi.c +++ b/arch/s390/kernel/nmi.c @@ -131,10 +131,10 @@ static notrace void s390_handle_damage(void) * Disable low address protection and make machine check new PSW a * disabled wait PSW. Any additional machine check cannot be handled. */ - __local_ctl_store(cr0.val, 0, 0); + local_ctl_store(0, &cr0.val); cr0_new = cr0; cr0_new.lap = 0; - __local_ctl_load(cr0_new.val, 0, 0); + local_ctl_load(0, &cr0_new.val); psw_save = S390_lowcore.mcck_new_psw; psw_bits(S390_lowcore.mcck_new_psw).io = 0; psw_bits(S390_lowcore.mcck_new_psw).ext = 0; @@ -146,7 +146,7 @@ static notrace void s390_handle_damage(void) * values. This makes possible system dump analysis easier. */ S390_lowcore.mcck_new_psw = psw_save; - __local_ctl_load(cr0.val, 0, 0); + local_ctl_load(0, &cr0.val); disabled_wait(); while (1); } @@ -271,7 +271,7 @@ static int notrace s390_validate_registers(union mci mci) kill_task = 1; cr0.val = S390_lowcore.cregs_save_area[0]; cr0.afp = cr0.vx = 1; - __local_ctl_load(cr0.val, 0, 0); + local_ctl_load(0, &cr0.val); asm volatile( " la 1,%0\n" " VLM 0,15,0,1\n" @@ -279,7 +279,7 @@ static int notrace s390_validate_registers(union mci mci) : : "Q" (*(struct vx_array *)mcesa->vector_save_area) : "1"); - __local_ctl_load(S390_lowcore.cregs_save_area[0], 0, 0); + local_ctl_load(0, &S390_lowcore.cregs_save_area[0]); } /* Validate access registers */ asm volatile( diff --git a/arch/s390/kernel/ptrace.c b/arch/s390/kernel/ptrace.c index 1824c1208852..a1fb4536784f 100644 --- a/arch/s390/kernel/ptrace.c +++ b/arch/s390/kernel/ptrace.c @@ -46,8 +46,8 @@ void update_cr_regs(struct task_struct *task) union ctlreg2 cr2_old, cr2_new; int cr0_changed, cr2_changed; - __local_ctl_store(cr0_old.val, 0, 0); - __local_ctl_store(cr2_old.val, 2, 2); + local_ctl_store(0, &cr0_old.val); + local_ctl_store(2, &cr2_old.val); cr0_new = cr0_old; cr2_new = cr2_old; /* Take care of the enable/disable of transactional execution. */ @@ -75,9 +75,9 @@ void update_cr_regs(struct task_struct *task) cr0_changed = cr0_new.val != cr0_old.val; cr2_changed = cr2_new.val != cr2_old.val; if (cr0_changed) - __local_ctl_load(cr0_new.val, 0, 0); + local_ctl_load(0, &cr0_new.val); if (cr2_changed) - __local_ctl_load(cr2_new.val, 2, 2); + local_ctl_load(2, &cr2_new.val); /* Copy user specified PER registers */ new.control = thread->per_user.control; new.start = thread->per_user.start; diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index c516e2e909e6..48a456a10206 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -791,15 +791,15 @@ static void __init setup_cr(void) __ctl_duct[4] = (unsigned long)__ctl_duald; /* Update control registers CR2, CR5 and CR15 */ - __local_ctl_store(cr2.val, 2, 2); - __local_ctl_store(cr5.val, 5, 5); - __local_ctl_store(cr15.val, 15, 15); + local_ctl_store(2, &cr2.val); + local_ctl_store(5, &cr5.val); + local_ctl_store(15, &cr15.val); cr2.ducto = (unsigned long)__ctl_duct >> 6; cr5.pasteo = (unsigned long)__ctl_duct >> 6; cr15.lsea = (unsigned long)__ctl_linkage_stack >> 3; - __local_ctl_load(cr2.val, 2, 2); - __local_ctl_load(cr5.val, 5, 5); - __local_ctl_load(cr15.val, 15, 15); + local_ctl_load(2, &cr2.val); + local_ctl_load(5, &cr5.val); + local_ctl_load(15, &cr15.val); } /* diff --git a/arch/s390/lib/uaccess.c b/arch/s390/lib/uaccess.c index 551de5394be6..56f7eb90b44d 100644 --- a/arch/s390/lib/uaccess.c +++ b/arch/s390/lib/uaccess.c @@ -19,13 +19,13 @@ void debug_user_asce(int exit) { unsigned long cr1, cr7; - __local_ctl_store(cr1, 1, 1); - __local_ctl_store(cr7, 7, 7); + local_ctl_store(1, &cr1); + local_ctl_store(7, &cr7); if (cr1 == S390_lowcore.kernel_asce && cr7 == S390_lowcore.user_asce) return; panic("incorrect ASCE on kernel %s\n" "cr1: %016lx cr7: %016lx\n" - "kernel: %016llx user: %016llx\n", + "kernel: %016lx user: %016lx\n", exit ? "exit" : "entry", cr1, cr7, S390_lowcore.kernel_asce, S390_lowcore.user_asce); } diff --git a/arch/s390/mm/pgalloc.c b/arch/s390/mm/pgalloc.c index 43bef9cce9b2..27720a687022 100644 --- a/arch/s390/mm/pgalloc.c +++ b/arch/s390/mm/pgalloc.c @@ -63,7 +63,7 @@ static void __crst_table_upgrade(void *arg) /* change all active ASCEs to avoid the creation of new TLBs */ if (current->active_mm == mm) { S390_lowcore.user_asce = mm->context.asce; - __local_ctl_load(S390_lowcore.user_asce, 7, 7); + local_ctl_load(7, &S390_lowcore.user_asce); } __tlb_flush_local(); } diff --git a/drivers/s390/char/sclp.c b/drivers/s390/char/sclp.c index 1ebc60d9cedc..30a7cd9748fe 100644 --- a/drivers/s390/char/sclp.c +++ b/drivers/s390/char/sclp.c @@ -732,10 +732,10 @@ sclp_sync_wait(void) /* Enable service-signal interruption, disable timer interrupts */ old_tick = local_tick_disable(); trace_hardirqs_on(); - __local_ctl_store(cr0, 0, 0); + local_ctl_store(0, &cr0); cr0_sync = cr0 & ~CR0_IRQ_SUBCLASS_MASK; cr0_sync |= 1UL << (63 - 54); - __local_ctl_load(cr0_sync, 0, 0); + local_ctl_load(0, &cr0_sync); __arch_local_irq_stosm(0x01); /* Loop until driver state indicates finished request */ while (sclp_running_state != sclp_running_state_idle) { @@ -745,7 +745,7 @@ sclp_sync_wait(void) cpu_relax(); } local_irq_disable(); - __local_ctl_load(cr0, 0, 0); + local_ctl_load(0, &cr0); if (!irq_context) _local_bh_enable(); local_tick_enable(old_tick); diff --git a/drivers/s390/char/sclp_early_core.c b/drivers/s390/char/sclp_early_core.c index fa5b7d63a254..8cb9fb7098e2 100644 --- a/drivers/s390/char/sclp_early_core.c +++ b/drivers/s390/char/sclp_early_core.c @@ -32,11 +32,11 @@ void sclp_early_wait_irq(void) psw_t psw_ext_save, psw_wait; union ctlreg0 cr0, cr0_new; - __local_ctl_store(cr0.val, 0, 0); + local_ctl_store(0, &cr0.val); cr0_new.val = cr0.val & ~CR0_IRQ_SUBCLASS_MASK; cr0_new.lap = 0; cr0_new.sssm = 1; - __local_ctl_load(cr0_new.val, 0, 0); + local_ctl_load(0, &cr0_new.val); psw_ext_save = S390_lowcore.external_new_psw; psw_mask = __extract_psw(); @@ -59,7 +59,7 @@ void sclp_early_wait_irq(void) } while (S390_lowcore.ext_int_code != EXT_IRQ_SERVICE_SIG); S390_lowcore.external_new_psw = psw_ext_save; - __local_ctl_load(cr0.val, 0, 0); + local_ctl_load(0, &cr0.val); } int sclp_early_cmd(sclp_cmdw_t cmd, void *sccb) From 80725978260ff823e1c442e75d66cf1b61684bba Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 21:40:01 +0200 Subject: [PATCH 133/475] s390/ctlreg: change parameters of __local_ctl_load() and __local_ctl_store() Change __local_ctl_load() and __local_ctl_store(), so that control register parameters come first. This way all control handling functions consistently have control register(s) parameter first. Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/ctlreg.h | 4 ++-- arch/s390/kernel/ctlreg.c | 4 ++-- arch/s390/kernel/kprobes.c | 6 +++--- arch/s390/kernel/ptrace.c | 4 ++-- arch/s390/kernel/setup.c | 2 +- arch/s390/kernel/smp.c | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/arch/s390/include/asm/ctlreg.h b/arch/s390/include/asm/ctlreg.h index 86887b490bb5..e60c672ffb9c 100644 --- a/arch/s390/include/asm/ctlreg.h +++ b/arch/s390/include/asm/ctlreg.h @@ -35,7 +35,7 @@ #include -#define __local_ctl_load(array, low, high) do { \ +#define __local_ctl_load(low, high, array) do { \ struct addrtype { \ char _[sizeof(array)]; \ }; \ @@ -53,7 +53,7 @@ : "memory"); \ } while (0) -#define __local_ctl_store(array, low, high) do { \ +#define __local_ctl_store(low, high, array) do { \ struct addrtype { \ char _[sizeof(array)]; \ }; \ diff --git a/arch/s390/kernel/ctlreg.c b/arch/s390/kernel/ctlreg.c index 795e7f4861fb..1c9fee1ca4a3 100644 --- a/arch/s390/kernel/ctlreg.c +++ b/arch/s390/kernel/ctlreg.c @@ -38,10 +38,10 @@ static void ctl_bit_callback(void *info) struct ctl_bit_parms *pp = info; unsigned long regs[16]; - __local_ctl_store(regs, 0, 15); + __local_ctl_store(0, 15, regs); regs[pp->cr] &= pp->andval; regs[pp->cr] |= pp->orval; - __local_ctl_load(regs, 0, 15); + __local_ctl_load(0, 15, regs); } void system_ctl_set_clear_bit(unsigned int cr, unsigned int bit, bool set) diff --git a/arch/s390/kernel/kprobes.c b/arch/s390/kernel/kprobes.c index 1b2e1903a54c..ef556b93596d 100644 --- a/arch/s390/kernel/kprobes.c +++ b/arch/s390/kernel/kprobes.c @@ -232,12 +232,12 @@ static void enable_singlestep(struct kprobe_ctlblk *kcb, per_kprobe.end = ip; /* Save control regs and psw mask */ - __local_ctl_store(kcb->kprobe_saved_ctl, 9, 11); + __local_ctl_store(9, 11, kcb->kprobe_saved_ctl); kcb->kprobe_saved_imask = regs->psw.mask & (PSW_MASK_PER | PSW_MASK_IO | PSW_MASK_EXT); /* Set PER control regs, turns on single step for the given address */ - __local_ctl_load(per_kprobe, 9, 11); + __local_ctl_load(9, 11, per_kprobe); regs->psw.mask |= PSW_MASK_PER; regs->psw.mask &= ~(PSW_MASK_IO | PSW_MASK_EXT); regs->psw.addr = ip; @@ -249,7 +249,7 @@ static void disable_singlestep(struct kprobe_ctlblk *kcb, unsigned long ip) { /* Restore control regs and psw mask, set new psw address */ - __local_ctl_load(kcb->kprobe_saved_ctl, 9, 11); + __local_ctl_load(9, 11, kcb->kprobe_saved_ctl); regs->psw.mask &= ~PSW_MASK_PER; regs->psw.mask |= kcb->kprobe_saved_imask; regs->psw.addr = ip; diff --git a/arch/s390/kernel/ptrace.c b/arch/s390/kernel/ptrace.c index a1fb4536784f..c03fda09f689 100644 --- a/arch/s390/kernel/ptrace.c +++ b/arch/s390/kernel/ptrace.c @@ -104,9 +104,9 @@ void update_cr_regs(struct task_struct *task) return; } regs->psw.mask |= PSW_MASK_PER; - __local_ctl_store(old, 9, 11); + __local_ctl_store(9, 11, old); if (memcmp(&new, &old, sizeof(struct per_regs)) != 0) - __local_ctl_load(new, 9, 11); + __local_ctl_load(9, 11, new); } void user_enable_single_step(struct task_struct *task) diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index 48a456a10206..2c4bfe41d284 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -449,7 +449,7 @@ static void __init setup_lowcore(void) lc->restart_fn = (unsigned long) do_restart; lc->restart_data = 0; lc->restart_source = -1U; - __local_ctl_store(lc->cregs_save_area, 0, 15); + __local_ctl_store(0, 15, lc->cregs_save_area); lc->spinlock_lockval = arch_spin_lockval(0); lc->spinlock_index = 0; arch_spin_lock_setup(0); diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index 1c65bbd411c7..90d3502379aa 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -922,11 +922,11 @@ int __cpu_disable(void) /* Disable pseudo page faults on this cpu. */ pfault_fini(); /* Disable interrupt sources via control register. */ - __local_ctl_store(cregs, 0, 15); + __local_ctl_store(0, 15, cregs); cregs[0] &= ~0x0000ee70UL; /* disable all external interrupts */ cregs[6] &= ~0xff000000UL; /* disable all I/O interrupts */ cregs[14] &= ~0x1f000000UL; /* disable most machine checks */ - __local_ctl_load(cregs, 0, 15); + __local_ctl_load(0, 15, cregs); clear_cpu_flag(CIF_NOHZ_DELAY); return 0; } From 4b440e01da51da0cc651c06a036b8914b7b61ccf Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 21:40:02 +0200 Subject: [PATCH 134/475] s390/kprobes,ptrace: open code struct per_reg Open code struct per_regs within kprobes and ptrace code, since at both locations a struct per_regs is passed to __local_ctl_load() and __local_ctl_store() which prevents to implement type checking for both functions. Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/kernel/kprobes.c | 11 +++++++++-- arch/s390/kernel/ptrace.c | 13 ++++++++++--- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/arch/s390/kernel/kprobes.c b/arch/s390/kernel/kprobes.c index ef556b93596d..d4c2ece4f839 100644 --- a/arch/s390/kernel/kprobes.c +++ b/arch/s390/kernel/kprobes.c @@ -224,7 +224,14 @@ static void enable_singlestep(struct kprobe_ctlblk *kcb, struct pt_regs *regs, unsigned long ip) { - struct per_regs per_kprobe; + union { + unsigned long regs[3]; + struct { + unsigned long control; + unsigned long start; + unsigned long end; + }; + } per_kprobe; /* Set up the PER control registers %cr9-%cr11 */ per_kprobe.control = PER_EVENT_IFETCH; @@ -237,7 +244,7 @@ static void enable_singlestep(struct kprobe_ctlblk *kcb, (PSW_MASK_PER | PSW_MASK_IO | PSW_MASK_EXT); /* Set PER control regs, turns on single step for the given address */ - __local_ctl_load(9, 11, per_kprobe); + __local_ctl_load(9, 11, per_kprobe.regs); regs->psw.mask |= PSW_MASK_PER; regs->psw.mask &= ~(PSW_MASK_IO | PSW_MASK_EXT); regs->psw.addr = ip; diff --git a/arch/s390/kernel/ptrace.c b/arch/s390/kernel/ptrace.c index c03fda09f689..1e1de907f24d 100644 --- a/arch/s390/kernel/ptrace.c +++ b/arch/s390/kernel/ptrace.c @@ -41,10 +41,17 @@ void update_cr_regs(struct task_struct *task) { struct pt_regs *regs = task_pt_regs(task); struct thread_struct *thread = &task->thread; - struct per_regs old, new; union ctlreg0 cr0_old, cr0_new; union ctlreg2 cr2_old, cr2_new; int cr0_changed, cr2_changed; + union { + unsigned long regs[3]; + struct { + unsigned long control; + unsigned long start; + unsigned long end; + }; + } old, new; local_ctl_store(0, &cr0_old.val); local_ctl_store(2, &cr2_old.val); @@ -104,9 +111,9 @@ void update_cr_regs(struct task_struct *task) return; } regs->psw.mask |= PSW_MASK_PER; - __local_ctl_store(9, 11, old); + __local_ctl_store(9, 11, old.regs); if (memcmp(&new, &old, sizeof(struct per_regs)) != 0) - __local_ctl_load(9, 11, new); + __local_ctl_load(9, 11, new.regs); } void user_enable_single_step(struct task_struct *task) From ecc53818f60447177e24ea11b7f136c405150976 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 21:40:03 +0200 Subject: [PATCH 135/475] s390/ctlreg: add type checking to __local_ctl_load() and __local_ctl_store() Add type checking to __local_ctl_load() and __local_ctl_store(). For both functions enforce to pass an array consisting of unsigned longs. Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/ctlreg.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/s390/include/asm/ctlreg.h b/arch/s390/include/asm/ctlreg.h index e60c672ffb9c..a49459adba9d 100644 --- a/arch/s390/include/asm/ctlreg.h +++ b/arch/s390/include/asm/ctlreg.h @@ -45,6 +45,7 @@ \ _esize = (_high - _low + 1) * sizeof(unsigned long); \ BUILD_BUG_ON(sizeof(struct addrtype) != _esize); \ + typecheck(unsigned long, array[0]); \ asm volatile( \ " lctlg %[_low],%[_high],%[_arr]\n" \ : \ @@ -63,6 +64,7 @@ \ _esize = (_high - _low + 1) * sizeof(unsigned long); \ BUILD_BUG_ON(sizeof(struct addrtype) != _esize); \ + typecheck(unsigned long, array[0]); \ asm volatile( \ " stctg %[_low],%[_high],%[_arr]\n" \ : [_arr] "=Q" (*(struct addrtype *)(&array)) \ From 527618abb92793b9d4dba548d55822dcebd95317 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 21:40:04 +0200 Subject: [PATCH 136/475] s390/ctlreg: add struct ctlreg Add struct ctlreg to enforce strict type checking / usage for control register functions. Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/boot/vmem.c | 8 +++--- arch/s390/include/asm/ctlreg.h | 28 +++++++++++++-------- arch/s390/include/asm/kprobes.h | 3 ++- arch/s390/include/asm/lowcore.h | 7 +++--- arch/s390/include/asm/mmu_context.h | 2 +- arch/s390/include/asm/pgtable.h | 3 ++- arch/s390/kernel/ctlreg.c | 10 ++++---- arch/s390/kernel/kprobes.c | 14 +++++------ arch/s390/kernel/machine_kexec.c | 6 ++--- arch/s390/kernel/nmi.c | 12 ++++----- arch/s390/kernel/ptrace.c | 38 ++++++++++++++--------------- arch/s390/kernel/setup.c | 12 ++++----- arch/s390/kernel/smp.c | 8 +++--- arch/s390/lib/uaccess.c | 8 +++--- arch/s390/mm/dump_pagetables.c | 2 +- arch/s390/mm/fault.c | 4 +-- arch/s390/mm/init.c | 2 +- arch/s390/mm/pageattr.c | 2 +- arch/s390/mm/pgalloc.c | 2 +- drivers/s390/char/sclp.c | 6 ++--- drivers/s390/char/sclp_early_core.c | 6 ++--- 21 files changed, 97 insertions(+), 86 deletions(-) diff --git a/arch/s390/boot/vmem.c b/arch/s390/boot/vmem.c index 44278ae2512f..bdbfee86d1ac 100644 --- a/arch/s390/boot/vmem.c +++ b/arch/s390/boot/vmem.c @@ -12,7 +12,7 @@ #include "decompressor.h" #include "boot.h" -unsigned long __bootdata_preserved(s390_invalid_asce); +struct ctlreg __bootdata_preserved(s390_invalid_asce); #ifdef CONFIG_PROC_FS atomic_long_t __bootdata_preserved(direct_pages_count[PG_DIRECT_MAP_MAX]); @@ -422,7 +422,7 @@ void setup_vmem(unsigned long asce_limit) asce_type = _REGION3_ENTRY_EMPTY; asce_bits = _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH; } - s390_invalid_asce = invalid_pg_dir | _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH; + s390_invalid_asce.val = invalid_pg_dir | _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH; crst_table_init((unsigned long *)swapper_pg_dir, asce_type); crst_table_init((unsigned long *)invalid_pg_dir, _REGION3_ENTRY_EMPTY); @@ -443,12 +443,12 @@ void setup_vmem(unsigned long asce_limit) kasan_populate_shadow(); - S390_lowcore.kernel_asce = swapper_pg_dir | asce_bits; + S390_lowcore.kernel_asce.val = swapper_pg_dir | asce_bits; S390_lowcore.user_asce = s390_invalid_asce; local_ctl_load(1, &S390_lowcore.kernel_asce); local_ctl_load(7, &S390_lowcore.user_asce); local_ctl_load(13, &S390_lowcore.kernel_asce); - init_mm.context.asce = S390_lowcore.kernel_asce; + init_mm.context.asce = S390_lowcore.kernel_asce.val; } diff --git a/arch/s390/include/asm/ctlreg.h b/arch/s390/include/asm/ctlreg.h index a49459adba9d..57cc610dd997 100644 --- a/arch/s390/include/asm/ctlreg.h +++ b/arch/s390/include/asm/ctlreg.h @@ -35,6 +35,10 @@ #include +struct ctlreg { + unsigned long val; +}; + #define __local_ctl_load(low, high, array) do { \ struct addrtype { \ char _[sizeof(array)]; \ @@ -43,9 +47,9 @@ int _low = low; \ int _esize; \ \ - _esize = (_high - _low + 1) * sizeof(unsigned long); \ + _esize = (_high - _low + 1) * sizeof(struct ctlreg); \ BUILD_BUG_ON(sizeof(struct addrtype) != _esize); \ - typecheck(unsigned long, array[0]); \ + typecheck(struct ctlreg, array[0]); \ asm volatile( \ " lctlg %[_low],%[_high],%[_arr]\n" \ : \ @@ -62,16 +66,16 @@ int _low = low; \ int _esize; \ \ - _esize = (_high - _low + 1) * sizeof(unsigned long); \ + _esize = (_high - _low + 1) * sizeof(struct ctlreg); \ BUILD_BUG_ON(sizeof(struct addrtype) != _esize); \ - typecheck(unsigned long, array[0]); \ + typecheck(struct ctlreg, array[0]); \ asm volatile( \ " stctg %[_low],%[_high],%[_arr]\n" \ : [_arr] "=Q" (*(struct addrtype *)(&array)) \ : [_low] "i" (low), [_high] "i" (high)); \ } while (0) -static __always_inline void local_ctl_load(unsigned int cr, unsigned long *reg) +static __always_inline void local_ctl_load(unsigned int cr, struct ctlreg *reg) { asm volatile( " lctlg %[cr],%[cr],%[reg]\n" @@ -80,7 +84,7 @@ static __always_inline void local_ctl_load(unsigned int cr, unsigned long *reg) : "memory"); } -static __always_inline void local_ctl_store(unsigned int cr, unsigned long *reg) +static __always_inline void local_ctl_store(unsigned int cr, struct ctlreg *reg) { asm volatile( " stctg %[cr],%[cr],%[reg]\n" @@ -90,19 +94,19 @@ static __always_inline void local_ctl_store(unsigned int cr, unsigned long *reg) static __always_inline void local_ctl_set_bit(unsigned int cr, unsigned int bit) { - unsigned long reg; + struct ctlreg reg; local_ctl_store(cr, ®); - reg |= 1UL << bit; + reg.val |= 1UL << bit; local_ctl_load(cr, ®); } static __always_inline void local_ctl_clear_bit(unsigned int cr, unsigned int bit) { - unsigned long reg; + struct ctlreg reg; local_ctl_store(cr, ®); - reg &= ~(1UL << bit); + reg.val &= ~(1UL << bit); local_ctl_load(cr, ®); } @@ -122,6 +126,7 @@ static inline void system_ctl_clear_bit(unsigned int cr, unsigned int bit) union ctlreg0 { unsigned long val; + struct ctlreg reg; struct { unsigned long : 8; unsigned long tcx : 1; /* Transactional-Execution control */ @@ -148,6 +153,7 @@ union ctlreg0 { union ctlreg2 { unsigned long val; + struct ctlreg reg; struct { unsigned long : 33; unsigned long ducto : 25; @@ -161,6 +167,7 @@ union ctlreg2 { union ctlreg5 { unsigned long val; + struct ctlreg reg; struct { unsigned long : 33; unsigned long pasteo: 25; @@ -170,6 +177,7 @@ union ctlreg5 { union ctlreg15 { unsigned long val; + struct ctlreg reg; struct { unsigned long lsea : 61; unsigned long : 3; diff --git a/arch/s390/include/asm/kprobes.h b/arch/s390/include/asm/kprobes.h index 83f732ca3af4..21b9e5290c04 100644 --- a/arch/s390/include/asm/kprobes.h +++ b/arch/s390/include/asm/kprobes.h @@ -15,6 +15,7 @@ * */ #include +#include #include #define BREAKPOINT_INSTRUCTION 0x0002 @@ -65,7 +66,7 @@ struct prev_kprobe { struct kprobe_ctlblk { unsigned long kprobe_status; unsigned long kprobe_saved_imask; - unsigned long kprobe_saved_ctl[3]; + struct ctlreg kprobe_saved_ctl[3]; struct prev_kprobe prev_kprobe; }; diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h index 2174f00e188b..3366431dcad5 100644 --- a/arch/s390/include/asm/lowcore.h +++ b/arch/s390/include/asm/lowcore.h @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -139,8 +140,8 @@ struct lowcore { __u32 restart_flags; /* 0x0384 */ /* Address space pointer. */ - unsigned long kernel_asce; /* 0x0388 */ - unsigned long user_asce; /* 0x0390 */ + struct ctlreg kernel_asce; /* 0x0388 */ + struct ctlreg user_asce; /* 0x0390 */ /* * The lpp and current_pid fields form a @@ -199,7 +200,7 @@ struct lowcore { __u32 clock_comp_save_area[2]; /* 0x1330 */ __u64 last_break_save_area; /* 0x1338 */ __u32 access_regs_save_area[16]; /* 0x1340 */ - unsigned long cregs_save_area[16]; /* 0x1380 */ + struct ctlreg cregs_save_area[16]; /* 0x1380 */ __u8 pad_0x1400[0x1500-0x1400]; /* 0x1400 */ /* Cryptography-counter designation */ __u64 ccd; /* 0x1500 */ diff --git a/arch/s390/include/asm/mmu_context.h b/arch/s390/include/asm/mmu_context.h index 0e93275f80f0..757fe6f0d802 100644 --- a/arch/s390/include/asm/mmu_context.h +++ b/arch/s390/include/asm/mmu_context.h @@ -78,7 +78,7 @@ static inline void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct * if (next == &init_mm) S390_lowcore.user_asce = s390_invalid_asce; else - S390_lowcore.user_asce = next->context.asce; + S390_lowcore.user_asce.val = next->context.asce; cpumask_set_cpu(cpu, &next->context.cpu_attach_mask); /* Clear previous user-ASCE from CR7 */ local_ctl_load(7, &s390_invalid_asce); diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index fb3ee7758b76..601e87fa8a9a 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -25,7 +26,7 @@ extern pgd_t swapper_pg_dir[]; extern pgd_t invalid_pg_dir[]; extern void paging_init(void); -extern unsigned long s390_invalid_asce; +extern struct ctlreg s390_invalid_asce; enum { PG_DIRECT_MAP_4K = 0, diff --git a/arch/s390/kernel/ctlreg.c b/arch/s390/kernel/ctlreg.c index 1c9fee1ca4a3..14f715bc5f73 100644 --- a/arch/s390/kernel/ctlreg.c +++ b/arch/s390/kernel/ctlreg.c @@ -36,11 +36,11 @@ struct ctl_bit_parms { static void ctl_bit_callback(void *info) { struct ctl_bit_parms *pp = info; - unsigned long regs[16]; + struct ctlreg regs[16]; __local_ctl_store(0, 15, regs); - regs[pp->cr] &= pp->andval; - regs[pp->cr] |= pp->orval; + regs[pp->cr].val &= pp->andval; + regs[pp->cr].val |= pp->orval; __local_ctl_load(0, 15, regs); } @@ -53,8 +53,8 @@ void system_ctl_set_clear_bit(unsigned int cr, unsigned int bit, bool set) pp.andval = set ? -1UL : ~(1UL << bit); system_ctlreg_lock(); abs_lc = get_abs_lowcore(); - abs_lc->cregs_save_area[cr] &= pp.andval; - abs_lc->cregs_save_area[cr] |= pp.orval; + abs_lc->cregs_save_area[cr].val &= pp.andval; + abs_lc->cregs_save_area[cr].val |= pp.orval; put_abs_lowcore(abs_lc); on_each_cpu(ctl_bit_callback, &pp, 1); system_ctlreg_unlock(); diff --git a/arch/s390/kernel/kprobes.c b/arch/s390/kernel/kprobes.c index d4c2ece4f839..f0cf20d4b3c5 100644 --- a/arch/s390/kernel/kprobes.c +++ b/arch/s390/kernel/kprobes.c @@ -225,18 +225,18 @@ static void enable_singlestep(struct kprobe_ctlblk *kcb, unsigned long ip) { union { - unsigned long regs[3]; + struct ctlreg regs[3]; struct { - unsigned long control; - unsigned long start; - unsigned long end; + struct ctlreg control; + struct ctlreg start; + struct ctlreg end; }; } per_kprobe; /* Set up the PER control registers %cr9-%cr11 */ - per_kprobe.control = PER_EVENT_IFETCH; - per_kprobe.start = ip; - per_kprobe.end = ip; + per_kprobe.control.val = PER_EVENT_IFETCH; + per_kprobe.start.val = ip; + per_kprobe.end.val = ip; /* Save control regs and psw mask */ __local_ctl_store(9, 11, kcb->kprobe_saved_ctl); diff --git a/arch/s390/kernel/machine_kexec.c b/arch/s390/kernel/machine_kexec.c index fe9d65060fa4..bb0d4d68fcbe 100644 --- a/arch/s390/kernel/machine_kexec.c +++ b/arch/s390/kernel/machine_kexec.c @@ -94,12 +94,12 @@ static noinline void __machine_kdump(void *image) if (MACHINE_HAS_VX) save_vx_regs((__vector128 *) mcesa->vector_save_area); if (MACHINE_HAS_GS) { - local_ctl_store(2, &cr2_old.val); + local_ctl_store(2, &cr2_old.reg); cr2_new = cr2_old; cr2_new.gse = 1; - local_ctl_load(2, &cr2_new.val); + local_ctl_load(2, &cr2_new.reg); save_gs_cb((struct gs_cb *) mcesa->guarded_storage_save_area); - local_ctl_load(2, &cr2_old.val); + local_ctl_load(2, &cr2_old.reg); } /* * To create a good backchain for this CPU in the dump store_status diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c index 7880a42896a3..579cebc58d8c 100644 --- a/arch/s390/kernel/nmi.c +++ b/arch/s390/kernel/nmi.c @@ -131,10 +131,10 @@ static notrace void s390_handle_damage(void) * Disable low address protection and make machine check new PSW a * disabled wait PSW. Any additional machine check cannot be handled. */ - local_ctl_store(0, &cr0.val); + local_ctl_store(0, &cr0.reg); cr0_new = cr0; cr0_new.lap = 0; - local_ctl_load(0, &cr0_new.val); + local_ctl_load(0, &cr0_new.reg); psw_save = S390_lowcore.mcck_new_psw; psw_bits(S390_lowcore.mcck_new_psw).io = 0; psw_bits(S390_lowcore.mcck_new_psw).ext = 0; @@ -146,7 +146,7 @@ static notrace void s390_handle_damage(void) * values. This makes possible system dump analysis easier. */ S390_lowcore.mcck_new_psw = psw_save; - local_ctl_load(0, &cr0.val); + local_ctl_load(0, &cr0.reg); disabled_wait(); while (1); } @@ -269,9 +269,9 @@ static int notrace s390_validate_registers(union mci mci) */ if (!mci.vr && !test_cpu_flag(CIF_MCCK_GUEST)) kill_task = 1; - cr0.val = S390_lowcore.cregs_save_area[0]; + cr0.reg = S390_lowcore.cregs_save_area[0]; cr0.afp = cr0.vx = 1; - local_ctl_load(0, &cr0.val); + local_ctl_load(0, &cr0.reg); asm volatile( " la 1,%0\n" " VLM 0,15,0,1\n" @@ -290,7 +290,7 @@ static int notrace s390_validate_registers(union mci mci) if (!mci.ar) kill_task = 1; /* Validate guarded storage registers */ - cr2.val = S390_lowcore.cregs_save_area[2]; + cr2.reg = S390_lowcore.cregs_save_area[2]; if (cr2.gse) { if (!mci.gs) { /* diff --git a/arch/s390/kernel/ptrace.c b/arch/s390/kernel/ptrace.c index 1e1de907f24d..6a825351ff41 100644 --- a/arch/s390/kernel/ptrace.c +++ b/arch/s390/kernel/ptrace.c @@ -45,16 +45,16 @@ void update_cr_regs(struct task_struct *task) union ctlreg2 cr2_old, cr2_new; int cr0_changed, cr2_changed; union { - unsigned long regs[3]; + struct ctlreg regs[3]; struct { - unsigned long control; - unsigned long start; - unsigned long end; + struct ctlreg control; + struct ctlreg start; + struct ctlreg end; }; } old, new; - local_ctl_store(0, &cr0_old.val); - local_ctl_store(2, &cr2_old.val); + local_ctl_store(0, &cr0_old.reg); + local_ctl_store(2, &cr2_old.reg); cr0_new = cr0_old; cr2_new = cr2_old; /* Take care of the enable/disable of transactional execution. */ @@ -82,31 +82,31 @@ void update_cr_regs(struct task_struct *task) cr0_changed = cr0_new.val != cr0_old.val; cr2_changed = cr2_new.val != cr2_old.val; if (cr0_changed) - local_ctl_load(0, &cr0_new.val); + local_ctl_load(0, &cr0_new.reg); if (cr2_changed) - local_ctl_load(2, &cr2_new.val); + local_ctl_load(2, &cr2_new.reg); /* Copy user specified PER registers */ - new.control = thread->per_user.control; - new.start = thread->per_user.start; - new.end = thread->per_user.end; + new.control.val = thread->per_user.control; + new.start.val = thread->per_user.start; + new.end.val = thread->per_user.end; /* merge TIF_SINGLE_STEP into user specified PER registers. */ if (test_tsk_thread_flag(task, TIF_SINGLE_STEP) || test_tsk_thread_flag(task, TIF_UPROBE_SINGLESTEP)) { if (test_tsk_thread_flag(task, TIF_BLOCK_STEP)) - new.control |= PER_EVENT_BRANCH; + new.control.val |= PER_EVENT_BRANCH; else - new.control |= PER_EVENT_IFETCH; - new.control |= PER_CONTROL_SUSPENSION; - new.control |= PER_EVENT_TRANSACTION_END; + new.control.val |= PER_EVENT_IFETCH; + new.control.val |= PER_CONTROL_SUSPENSION; + new.control.val |= PER_EVENT_TRANSACTION_END; if (test_tsk_thread_flag(task, TIF_UPROBE_SINGLESTEP)) - new.control |= PER_EVENT_IFETCH; - new.start = 0; - new.end = -1UL; + new.control.val |= PER_EVENT_IFETCH; + new.start.val = 0; + new.end.val = -1UL; } /* Take care of the PER enablement bit in the PSW. */ - if (!(new.control & PER_EVENT_MASK)) { + if (!(new.control.val & PER_EVENT_MASK)) { regs->psw.mask &= ~PSW_MASK_PER; return; } diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index 2c4bfe41d284..1c049a65c769 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -791,15 +791,15 @@ static void __init setup_cr(void) __ctl_duct[4] = (unsigned long)__ctl_duald; /* Update control registers CR2, CR5 and CR15 */ - local_ctl_store(2, &cr2.val); - local_ctl_store(5, &cr5.val); - local_ctl_store(15, &cr15.val); + local_ctl_store(2, &cr2.reg); + local_ctl_store(5, &cr5.reg); + local_ctl_store(15, &cr15.reg); cr2.ducto = (unsigned long)__ctl_duct >> 6; cr5.pasteo = (unsigned long)__ctl_duct >> 6; cr15.lsea = (unsigned long)__ctl_linkage_stack >> 3; - local_ctl_load(2, &cr2.val); - local_ctl_load(5, &cr5.val); - local_ctl_load(15, &cr15.val); + local_ctl_load(2, &cr2.reg); + local_ctl_load(5, &cr5.reg); + local_ctl_load(15, &cr15.reg); } /* diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index 90d3502379aa..fbd2801c7061 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -910,7 +910,7 @@ early_param("possible_cpus", _setup_possible_cpus); int __cpu_disable(void) { - unsigned long cregs[16]; + struct ctlreg cregs[16]; int cpu; /* Handle possible pending IPIs */ @@ -923,9 +923,9 @@ int __cpu_disable(void) pfault_fini(); /* Disable interrupt sources via control register. */ __local_ctl_store(0, 15, cregs); - cregs[0] &= ~0x0000ee70UL; /* disable all external interrupts */ - cregs[6] &= ~0xff000000UL; /* disable all I/O interrupts */ - cregs[14] &= ~0x1f000000UL; /* disable most machine checks */ + cregs[0].val &= ~0x0000ee70UL; /* disable all external interrupts */ + cregs[6].val &= ~0xff000000UL; /* disable all I/O interrupts */ + cregs[14].val &= ~0x1f000000UL; /* disable most machine checks */ __local_ctl_load(0, 15, cregs); clear_cpu_flag(CIF_NOHZ_DELAY); return 0; diff --git a/arch/s390/lib/uaccess.c b/arch/s390/lib/uaccess.c index 56f7eb90b44d..61d8dcd95bbc 100644 --- a/arch/s390/lib/uaccess.c +++ b/arch/s390/lib/uaccess.c @@ -17,17 +17,17 @@ #ifdef CONFIG_DEBUG_ENTRY void debug_user_asce(int exit) { - unsigned long cr1, cr7; + struct ctlreg cr1, cr7; local_ctl_store(1, &cr1); local_ctl_store(7, &cr7); - if (cr1 == S390_lowcore.kernel_asce && cr7 == S390_lowcore.user_asce) + if (cr1.val == S390_lowcore.kernel_asce.val && cr7.val == S390_lowcore.user_asce.val) return; panic("incorrect ASCE on kernel %s\n" "cr1: %016lx cr7: %016lx\n" "kernel: %016lx user: %016lx\n", - exit ? "exit" : "entry", cr1, cr7, - S390_lowcore.kernel_asce, S390_lowcore.user_asce); + exit ? "exit" : "entry", cr1.val, cr7.val, + S390_lowcore.kernel_asce.val, S390_lowcore.user_asce.val); } #endif /*CONFIG_DEBUG_ENTRY */ diff --git a/arch/s390/mm/dump_pagetables.c b/arch/s390/mm/dump_pagetables.c index b51666967aa1..d37a8f607b71 100644 --- a/arch/s390/mm/dump_pagetables.c +++ b/arch/s390/mm/dump_pagetables.c @@ -287,7 +287,7 @@ static int pt_dump_init(void) * kernel ASCE. We need this to keep the page table walker functions * from accessing non-existent entries. */ - max_addr = (S390_lowcore.kernel_asce & _REGION_ENTRY_TYPE_MASK) >> 2; + max_addr = (S390_lowcore.kernel_asce.val & _REGION_ENTRY_TYPE_MASK) >> 2; max_addr = 1UL << (max_addr * 11 + 31); address_markers[IDENTITY_AFTER_END_NR].start_address = ident_map_size; address_markers[AMODE31_START_NR].start_address = (unsigned long)__samode31; diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index b678295931c3..587b6b64185f 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -196,7 +196,7 @@ static void dump_fault_info(struct pt_regs *regs) pr_cont("mode while using "); switch (get_fault_type(regs)) { case USER_FAULT: - asce = S390_lowcore.user_asce; + asce = S390_lowcore.user_asce.val; pr_cont("user "); break; case GMAP_FAULT: @@ -204,7 +204,7 @@ static void dump_fault_info(struct pt_regs *regs) pr_cont("gmap "); break; case KERNEL_FAULT: - asce = S390_lowcore.kernel_asce; + asce = S390_lowcore.kernel_asce.val; pr_cont("kernel "); break; default: diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c index c6ab6f2707d7..7eca10c32caa 100644 --- a/arch/s390/mm/init.c +++ b/arch/s390/mm/init.c @@ -54,7 +54,7 @@ pgd_t swapper_pg_dir[PTRS_PER_PGD] __section(".bss..swapper_pg_dir"); pgd_t invalid_pg_dir[PTRS_PER_PGD] __section(".bss..invalid_pg_dir"); -unsigned long __bootdata_preserved(s390_invalid_asce); +struct ctlreg __bootdata_preserved(s390_invalid_asce); unsigned long empty_zero_page, zero_page_mask; EXPORT_SYMBOL(empty_zero_page); diff --git a/arch/s390/mm/pageattr.c b/arch/s390/mm/pageattr.c index b87e96c64b61..631e3a4ee2de 100644 --- a/arch/s390/mm/pageattr.c +++ b/arch/s390/mm/pageattr.c @@ -75,7 +75,7 @@ static void pgt_set(unsigned long *old, unsigned long new, unsigned long addr, break; } table = (unsigned long *)((unsigned long)old & mask); - crdte(*old, new, table, dtt, addr, S390_lowcore.kernel_asce); + crdte(*old, new, table, dtt, addr, S390_lowcore.kernel_asce.val); } else if (MACHINE_HAS_IDTE) { cspg(old, *old, new); } else { diff --git a/arch/s390/mm/pgalloc.c b/arch/s390/mm/pgalloc.c index 27720a687022..61fb157029c8 100644 --- a/arch/s390/mm/pgalloc.c +++ b/arch/s390/mm/pgalloc.c @@ -62,7 +62,7 @@ static void __crst_table_upgrade(void *arg) /* change all active ASCEs to avoid the creation of new TLBs */ if (current->active_mm == mm) { - S390_lowcore.user_asce = mm->context.asce; + S390_lowcore.user_asce.val = mm->context.asce; local_ctl_load(7, &S390_lowcore.user_asce); } __tlb_flush_local(); diff --git a/drivers/s390/char/sclp.c b/drivers/s390/char/sclp.c index 30a7cd9748fe..ba9b202c5dee 100644 --- a/drivers/s390/char/sclp.c +++ b/drivers/s390/char/sclp.c @@ -706,8 +706,8 @@ void sclp_sync_wait(void) { unsigned long long old_tick; + struct ctlreg cr0, cr0_sync; unsigned long flags; - unsigned long cr0, cr0_sync; static u64 sync_count; u64 timeout; int irq_context; @@ -733,8 +733,8 @@ sclp_sync_wait(void) old_tick = local_tick_disable(); trace_hardirqs_on(); local_ctl_store(0, &cr0); - cr0_sync = cr0 & ~CR0_IRQ_SUBCLASS_MASK; - cr0_sync |= 1UL << (63 - 54); + cr0_sync.val = cr0.val & ~CR0_IRQ_SUBCLASS_MASK; + cr0_sync.val |= 1UL << (63 - 54); local_ctl_load(0, &cr0_sync); __arch_local_irq_stosm(0x01); /* Loop until driver state indicates finished request */ diff --git a/drivers/s390/char/sclp_early_core.c b/drivers/s390/char/sclp_early_core.c index 8cb9fb7098e2..9f6165cafdc3 100644 --- a/drivers/s390/char/sclp_early_core.c +++ b/drivers/s390/char/sclp_early_core.c @@ -32,11 +32,11 @@ void sclp_early_wait_irq(void) psw_t psw_ext_save, psw_wait; union ctlreg0 cr0, cr0_new; - local_ctl_store(0, &cr0.val); + local_ctl_store(0, &cr0.reg); cr0_new.val = cr0.val & ~CR0_IRQ_SUBCLASS_MASK; cr0_new.lap = 0; cr0_new.sssm = 1; - local_ctl_load(0, &cr0_new.val); + local_ctl_load(0, &cr0_new.reg); psw_ext_save = S390_lowcore.external_new_psw; psw_mask = __extract_psw(); @@ -59,7 +59,7 @@ void sclp_early_wait_irq(void) } while (S390_lowcore.ext_int_code != EXT_IRQ_SERVICE_SIG); S390_lowcore.external_new_psw = psw_ext_save; - local_ctl_load(0, &cr0.val); + local_ctl_load(0, &cr0.reg); } int sclp_early_cmd(sclp_cmdw_t cmd, void *sccb) From d11d5c8c8426a4abdcb2252999f1b57e3aa79e5b Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 21:40:05 +0200 Subject: [PATCH 137/475] s390/ctltreg: make initialization of control register save area explicit Commit e1b9c2749af0 ("s390/smp: ensure global control register contents are in sync") made the control register save area contained within the lowcore at absolute address zero a resource which is used when initializing CPUs. However this is anything but obvious from the code. Add an ctlreg_init_save_area() function in order to make this explicit. Signed-off-by: Heiko Carstens Reviewed-by: Alexander Gordeev Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/ctlreg.h | 3 +++ arch/s390/kernel/ctlreg.c | 11 +++++++++++ arch/s390/kernel/setup.c | 3 +-- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/arch/s390/include/asm/ctlreg.h b/arch/s390/include/asm/ctlreg.h index 57cc610dd997..1f25601d75a0 100644 --- a/arch/s390/include/asm/ctlreg.h +++ b/arch/s390/include/asm/ctlreg.h @@ -110,9 +110,12 @@ static __always_inline void local_ctl_clear_bit(unsigned int cr, unsigned int bi local_ctl_load(cr, ®); } +struct lowcore; + void system_ctlreg_lock(void); void system_ctlreg_unlock(void); void system_ctl_set_clear_bit(unsigned int cr, unsigned int bit, bool set); +void system_ctlreg_init_save_area(struct lowcore *lc); static inline void system_ctl_set_bit(unsigned int cr, unsigned int bit) { diff --git a/arch/s390/kernel/ctlreg.c b/arch/s390/kernel/ctlreg.c index 14f715bc5f73..7acc40804874 100644 --- a/arch/s390/kernel/ctlreg.c +++ b/arch/s390/kernel/ctlreg.c @@ -4,6 +4,7 @@ */ #include +#include #include #include #include @@ -27,6 +28,16 @@ void system_ctlreg_unlock(void) spin_unlock(&system_ctl_lock); } +void __init system_ctlreg_init_save_area(struct lowcore *lc) +{ + struct lowcore *abs_lc; + + abs_lc = get_abs_lowcore(); + __local_ctl_store(0, 15, lc->cregs_save_area); + __local_ctl_store(0, 15, abs_lc->cregs_save_area); + put_abs_lowcore(abs_lc); +} + struct ctl_bit_parms { unsigned long orval; unsigned long andval; diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index 1c049a65c769..d399f18d0379 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -449,7 +449,6 @@ static void __init setup_lowcore(void) lc->restart_fn = (unsigned long) do_restart; lc->restart_data = 0; lc->restart_source = -1U; - __local_ctl_store(0, 15, lc->cregs_save_area); lc->spinlock_lockval = arch_spin_lockval(0); lc->spinlock_index = 0; arch_spin_lock_setup(0); @@ -459,6 +458,7 @@ static void __init setup_lowcore(void) lc->kernel_asce = S390_lowcore.kernel_asce; lc->user_asce = S390_lowcore.user_asce; + system_ctlreg_init_save_area(lc); abs_lc = get_abs_lowcore(); abs_lc->restart_stack = lc->restart_stack; abs_lc->restart_fn = lc->restart_fn; @@ -466,7 +466,6 @@ static void __init setup_lowcore(void) abs_lc->restart_source = lc->restart_source; abs_lc->restart_psw = lc->restart_psw; abs_lc->restart_flags = RESTART_FLAG_CTLREGS; - memcpy(abs_lc->cregs_save_area, lc->cregs_save_area, sizeof(abs_lc->cregs_save_area)); abs_lc->program_new_psw = lc->program_new_psw; abs_lc->mcesad = lc->mcesad; put_abs_lowcore(abs_lc); From cce2c8606f8ed3b0285842f4ad21e906d511dba1 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 21:40:06 +0200 Subject: [PATCH 138/475] s390/ctlreg: allow to call system_ctl_set/clear_bit() early Allow to call system_ctl_set_bit() and system_clt_clear_bit() early, so that users do not have to take care when the control register save area has been initialized. Users are supposed to use system_ctl_set_bit() and system:clt_clear_bit() for all control register changes which are supposed to be seen globally. Depending on the system state such calls will change: - local control register contents - save area and local control register contents - save area and global control register contents Signed-off-by: Heiko Carstens Reviewed-by: Alexander Gordeev Signed-off-by: Vasily Gorbik --- arch/s390/kernel/ctlreg.c | 41 ++++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/arch/s390/kernel/ctlreg.c b/arch/s390/kernel/ctlreg.c index 7acc40804874..27ba8db1d038 100644 --- a/arch/s390/kernel/ctlreg.c +++ b/arch/s390/kernel/ctlreg.c @@ -3,9 +3,12 @@ * Copyright IBM Corp. 1999, 2023 */ +#include #include +#include #include #include +#include #include #include @@ -28,6 +31,8 @@ void system_ctlreg_unlock(void) spin_unlock(&system_ctl_lock); } +static bool system_ctlreg_area_init __ro_after_init; + void __init system_ctlreg_init_save_area(struct lowcore *lc) { struct lowcore *abs_lc; @@ -36,6 +41,7 @@ void __init system_ctlreg_init_save_area(struct lowcore *lc) __local_ctl_store(0, 15, lc->cregs_save_area); __local_ctl_store(0, 15, abs_lc->cregs_save_area); put_abs_lowcore(abs_lc); + system_ctlreg_area_init = true; } struct ctl_bit_parms { @@ -55,6 +61,23 @@ static void ctl_bit_callback(void *info) __local_ctl_load(0, 15, regs); } +static void system_ctl_bit_update(void *info) +{ + unsigned long flags; + + if (system_state == SYSTEM_BOOTING) { + /* + * For very early calls do not call on_each_cpu() + * since not everything might be setup. + */ + local_irq_save(flags); + ctl_bit_callback(info); + local_irq_restore(flags); + } else { + on_each_cpu(ctl_bit_callback, info, 1); + } +} + void system_ctl_set_clear_bit(unsigned int cr, unsigned int bit, bool set) { struct ctl_bit_parms pp = { .cr = cr, }; @@ -62,12 +85,16 @@ void system_ctl_set_clear_bit(unsigned int cr, unsigned int bit, bool set) pp.orval = set ? 1UL << bit : 0; pp.andval = set ? -1UL : ~(1UL << bit); - system_ctlreg_lock(); - abs_lc = get_abs_lowcore(); - abs_lc->cregs_save_area[cr].val &= pp.andval; - abs_lc->cregs_save_area[cr].val |= pp.orval; - put_abs_lowcore(abs_lc); - on_each_cpu(ctl_bit_callback, &pp, 1); - system_ctlreg_unlock(); + if (system_ctlreg_area_init) { + system_ctlreg_lock(); + abs_lc = get_abs_lowcore(); + abs_lc->cregs_save_area[cr].val &= pp.andval; + abs_lc->cregs_save_area[cr].val |= pp.orval; + put_abs_lowcore(abs_lc); + system_ctl_bit_update(&pp); + system_ctlreg_unlock(); + } else { + system_ctl_bit_update(&pp); + } } EXPORT_SYMBOL(system_ctl_set_clear_bit); From 103dde702d698c2e17529550650407a386711cdf Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 21:40:07 +0200 Subject: [PATCH 139/475] s390/early: use system_ctl_set_bit() instead of local_ctl_set_bit() Use system_ctl_set_bit() instead of local_ctl_set_bit() to reflect that the control register changes are supposed to be global. This change is just for documentation purposes, since it still results only in local control register contents being changed. Signed-off-by: Heiko Carstens Reviewed-by: Alexander Gordeev Signed-off-by: Vasily Gorbik --- arch/s390/kernel/early.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c index 1f359309cce6..0beaa0ba58a8 100644 --- a/arch/s390/kernel/early.c +++ b/arch/s390/kernel/early.c @@ -216,7 +216,7 @@ static __init void detect_machine_facilities(void) { if (test_facility(8)) { S390_lowcore.machine_flags |= MACHINE_FLAG_EDAT1; - local_ctl_set_bit(0, 23); + system_ctl_set_bit(0, 23); } if (test_facility(78)) S390_lowcore.machine_flags |= MACHINE_FLAG_EDAT2; @@ -224,13 +224,13 @@ static __init void detect_machine_facilities(void) S390_lowcore.machine_flags |= MACHINE_FLAG_IDTE; if (test_facility(50) && test_facility(73)) { S390_lowcore.machine_flags |= MACHINE_FLAG_TE; - local_ctl_set_bit(0, 55); + system_ctl_set_bit(0, 55); } if (test_facility(51)) S390_lowcore.machine_flags |= MACHINE_FLAG_TLB_LC; if (test_facility(129)) { S390_lowcore.machine_flags |= MACHINE_FLAG_VX; - local_ctl_set_bit(0, 17); + system_ctl_set_bit(0, 17); } if (test_facility(130)) S390_lowcore.machine_flags |= MACHINE_FLAG_NX; @@ -240,7 +240,7 @@ static __init void detect_machine_facilities(void) /* Enabled signed clock comparator comparisons */ S390_lowcore.machine_flags |= MACHINE_FLAG_SCC; clock_comparator_max = -1ULL >> 1; - local_ctl_set_bit(0, 53); + system_ctl_set_bit(0, 53); } if (IS_ENABLED(CONFIG_PCI) && test_facility(153)) { S390_lowcore.machine_flags |= MACHINE_FLAG_PCI_MIO; @@ -260,7 +260,7 @@ static inline void save_vector_registers(void) static inline void setup_low_address_protection(void) { - local_ctl_set_bit(0, 28); + system_ctl_set_bit(0, 28); } static inline void setup_access_registers(void) @@ -273,7 +273,7 @@ static inline void setup_access_registers(void) static int __init disable_vector_extension(char *str) { S390_lowcore.machine_flags &= ~MACHINE_FLAG_VX; - local_ctl_clear_bit(0, 17); + system_ctl_clear_bit(0, 17); return 0; } early_param("novx", disable_vector_extension); From 305a5551af34e1848d80e095c23e94c985cc4ebc Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 21:40:08 +0200 Subject: [PATCH 140/475] s390/ctlreg: add system_ctl_load() Add system_ctl_load() which can be used to load a value to a control register system wide. Refactor system_ctl_set_clear_bit() so it can handle all different request types, and also rename it to system_ctlreg_modify() Signed-off-by: Heiko Carstens Reviewed-by: Alexander Gordeev Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/ctlreg.h | 17 ++++++++-- arch/s390/kernel/ctlreg.c | 57 +++++++++++++++++++++++----------- 2 files changed, 53 insertions(+), 21 deletions(-) diff --git a/arch/s390/include/asm/ctlreg.h b/arch/s390/include/asm/ctlreg.h index 1f25601d75a0..6582b1b9b97b 100644 --- a/arch/s390/include/asm/ctlreg.h +++ b/arch/s390/include/asm/ctlreg.h @@ -114,17 +114,28 @@ struct lowcore; void system_ctlreg_lock(void); void system_ctlreg_unlock(void); -void system_ctl_set_clear_bit(unsigned int cr, unsigned int bit, bool set); void system_ctlreg_init_save_area(struct lowcore *lc); +void system_ctlreg_modify(unsigned int cr, unsigned long data, int request); + +enum { + CTLREG_SET_BIT, + CTLREG_CLEAR_BIT, + CTLREG_LOAD, +}; static inline void system_ctl_set_bit(unsigned int cr, unsigned int bit) { - system_ctl_set_clear_bit(cr, bit, true); + system_ctlreg_modify(cr, bit, CTLREG_SET_BIT); } static inline void system_ctl_clear_bit(unsigned int cr, unsigned int bit) { - system_ctl_set_clear_bit(cr, bit, false); + system_ctlreg_modify(cr, bit, CTLREG_CLEAR_BIT); +} + +static inline void system_ctl_load(unsigned int cr, struct ctlreg *reg) +{ + system_ctlreg_modify(cr, reg->val, CTLREG_LOAD); } union ctlreg0 { diff --git a/arch/s390/kernel/ctlreg.c b/arch/s390/kernel/ctlreg.c index 27ba8db1d038..8cc26cf2c64a 100644 --- a/arch/s390/kernel/ctlreg.c +++ b/arch/s390/kernel/ctlreg.c @@ -44,24 +44,30 @@ void __init system_ctlreg_init_save_area(struct lowcore *lc) system_ctlreg_area_init = true; } -struct ctl_bit_parms { - unsigned long orval; +struct ctlreg_parms { unsigned long andval; + unsigned long orval; + unsigned long val; + int request; int cr; }; -static void ctl_bit_callback(void *info) +static void ctlreg_callback(void *info) { - struct ctl_bit_parms *pp = info; + struct ctlreg_parms *pp = info; struct ctlreg regs[16]; __local_ctl_store(0, 15, regs); - regs[pp->cr].val &= pp->andval; - regs[pp->cr].val |= pp->orval; + if (pp->request == CTLREG_LOAD) { + regs[pp->cr].val = pp->val; + } else { + regs[pp->cr].val &= pp->andval; + regs[pp->cr].val |= pp->orval; + } __local_ctl_load(0, 15, regs); } -static void system_ctl_bit_update(void *info) +static void system_ctlreg_update(void *info) { unsigned long flags; @@ -71,30 +77,45 @@ static void system_ctl_bit_update(void *info) * since not everything might be setup. */ local_irq_save(flags); - ctl_bit_callback(info); + ctlreg_callback(info); local_irq_restore(flags); } else { - on_each_cpu(ctl_bit_callback, info, 1); + on_each_cpu(ctlreg_callback, info, 1); } } -void system_ctl_set_clear_bit(unsigned int cr, unsigned int bit, bool set) +void system_ctlreg_modify(unsigned int cr, unsigned long data, int request) { - struct ctl_bit_parms pp = { .cr = cr, }; + struct ctlreg_parms pp = { .cr = cr, .request = request, }; struct lowcore *abs_lc; - pp.orval = set ? 1UL << bit : 0; - pp.andval = set ? -1UL : ~(1UL << bit); + switch (request) { + case CTLREG_SET_BIT: + pp.orval = 1UL << data; + pp.andval = -1UL; + break; + case CTLREG_CLEAR_BIT: + pp.orval = 0; + pp.andval = ~(1UL << data); + break; + case CTLREG_LOAD: + pp.val = data; + break; + } if (system_ctlreg_area_init) { system_ctlreg_lock(); abs_lc = get_abs_lowcore(); - abs_lc->cregs_save_area[cr].val &= pp.andval; - abs_lc->cregs_save_area[cr].val |= pp.orval; + if (request == CTLREG_LOAD) { + abs_lc->cregs_save_area[cr].val = pp.val; + } else { + abs_lc->cregs_save_area[cr].val &= pp.andval; + abs_lc->cregs_save_area[cr].val |= pp.orval; + } put_abs_lowcore(abs_lc); - system_ctl_bit_update(&pp); + system_ctlreg_update(&pp); system_ctlreg_unlock(); } else { - system_ctl_bit_update(&pp); + system_ctlreg_update(&pp); } } -EXPORT_SYMBOL(system_ctl_set_clear_bit); +EXPORT_SYMBOL(system_ctlreg_modify); From 0b6529e3dc1c60565f10ccf40533b127ab3262a1 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 21:40:09 +0200 Subject: [PATCH 141/475] s390/setup: make use of system_ctl_load() Use system_ctl_load() instead of local_ctl_load() to reflect that control register changes are supposed to be global. Even though setup_cr() was ok, note that the usage of local_ctl_load() would have been wrong, if it would have happened after the control register save area was initialized: only local control register contents would have been changed, but wouldn't be used for new CPUs. With using system_ctl_load() the caller doesn't need to take care. Signed-off-by: Heiko Carstens Reviewed-by: Alexander Gordeev Signed-off-by: Vasily Gorbik --- arch/s390/kernel/setup.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index d399f18d0379..558bcf18df52 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -796,9 +796,9 @@ static void __init setup_cr(void) cr2.ducto = (unsigned long)__ctl_duct >> 6; cr5.pasteo = (unsigned long)__ctl_duct >> 6; cr15.lsea = (unsigned long)__ctl_linkage_stack >> 3; - local_ctl_load(2, &cr2.reg); - local_ctl_load(5, &cr5.reg); - local_ctl_load(15, &cr15.reg); + system_ctl_load(2, &cr2.reg); + system_ctl_load(5, &cr5.reg); + system_ctl_load(15, &cr15.reg); } /* From 4f4cee9619edc877e8da913193968497e4e42cee Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 21:40:10 +0200 Subject: [PATCH 142/475] s390/ctlreg: add missing defines Add a couple of missing control register defines which otherwise would prevent to convert other open-coded usages. Acked-by: Thomas Richter Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/ctlreg.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/arch/s390/include/asm/ctlreg.h b/arch/s390/include/asm/ctlreg.h index 6582b1b9b97b..76ed70cca43c 100644 --- a/arch/s390/include/asm/ctlreg.h +++ b/arch/s390/include/asm/ctlreg.h @@ -10,10 +10,19 @@ #include +#define CR0_TRANSACTIONAL_EXECUTION BIT(63 - 8) #define CR0_CLOCK_COMPARATOR_SIGN BIT(63 - 10) +#define CR0_CRYPTOGRAPHY_COUNTER BIT(63 - 13) +#define CR0_PAI_EXTENSION BIT(63 - 14) +#define CR0_CPUMF_EXTRACTION_AUTH BIT(63 - 15) +#define CR0_WARNING_TRACK BIT(63 - 30) #define CR0_LOW_ADDRESS_PROTECTION BIT(63 - 35) #define CR0_FETCH_PROTECTION_OVERRIDE BIT(63 - 38) #define CR0_STORAGE_PROTECTION_OVERRIDE BIT(63 - 39) +#define CR0_EDAT BIT(63 - 40) +#define CR0_INSTRUCTION_EXEC_PROTECTION BIT(63 - 43) +#define CR0_VECTOR BIT(63 - 46) +#define CR0_MALFUNCTION_ALERT_SUBMASK BIT(63 - 48) #define CR0_EMERGENCY_SIGNAL_SUBMASK BIT(63 - 49) #define CR0_EXTERNAL_CALL_SUBMASK BIT(63 - 50) #define CR0_CLOCK_COMPARATOR_SUBMASK BIT(63 - 52) @@ -22,6 +31,11 @@ #define CR0_UNUSED_56 BIT(63 - 56) #define CR0_INTERRUPT_KEY_SUBMASK BIT(63 - 57) #define CR0_MEASUREMENT_ALERT_SUBMASK BIT(63 - 58) +#define CR0_ETR_SUBMASK BIT(63 - 59) +#define CR0_IUCV BIT(63 - 62) + +#define CR2_MIO_ADDRESSING BIT(63 - 58) +#define CR2_GUARDED_STORAGE BIT(63 - 59) #define CR14_UNUSED_32 BIT(63 - 32) #define CR14_UNUSED_33 BIT(63 - 33) From da290f4382eb98e152af59e416a08d43a5c31157 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 21:40:11 +0200 Subject: [PATCH 143/475] s390/irq: use CR0 defines to define CR0_IRQ_SUBCLASS_MASK Use existing CR0 defines to define CR0_IRQ_SUBCLASS_MASK instead of open-coding the defines again. Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/irq.h | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/arch/s390/include/asm/irq.h b/arch/s390/include/asm/irq.h index 89902f754740..54b42817f70a 100644 --- a/arch/s390/include/asm/irq.h +++ b/arch/s390/include/asm/irq.h @@ -31,6 +31,7 @@ #include #include #include +#include enum interruption_class { IRQEXT_CLK, @@ -101,17 +102,17 @@ enum irq_subclass { }; #define CR0_IRQ_SUBCLASS_MASK \ - ((1UL << (63 - 30)) /* Warning Track */ | \ - (1UL << (63 - 48)) /* Malfunction Alert */ | \ - (1UL << (63 - 49)) /* Emergency Signal */ | \ - (1UL << (63 - 50)) /* External Call */ | \ - (1UL << (63 - 52)) /* Clock Comparator */ | \ - (1UL << (63 - 53)) /* CPU Timer */ | \ - (1UL << (63 - 54)) /* Service Signal */ | \ - (1UL << (63 - 57)) /* Interrupt Key */ | \ - (1UL << (63 - 58)) /* Measurement Alert */ | \ - (1UL << (63 - 59)) /* Timing Alert */ | \ - (1UL << (63 - 62))) /* IUCV */ + (CR0_WARNING_TRACK | \ + CR0_MALFUNCTION_ALERT_SUBMASK | \ + CR0_EMERGENCY_SIGNAL_SUBMASK | \ + CR0_EXTERNAL_CALL_SUBMASK | \ + CR0_CLOCK_COMPARATOR_SUBMASK | \ + CR0_CPU_TIMER_SUBMASK | \ + CR0_SERVICE_SIGNAL_SUBMASK | \ + CR0_INTERRUPT_KEY_SUBMASK | \ + CR0_MEASUREMENT_ALERT_SUBMASK | \ + CR0_ETR_SUBMASK | \ + CR0_IUCV) void irq_subclass_register(enum irq_subclass subclass); void irq_subclass_unregister(enum irq_subclass subclass); From 9c66cc5609b13899ef7d143f51c59bdcfd63484d Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 21:40:12 +0200 Subject: [PATCH 144/475] s390/ctlreg: add control register bits Instead of having only masks for specific bit locations within control registers, also define the bit numbers and use them to define the masks. The bit defines can be used to convert plain numbers to defines. Acked-by: Alexander Gordeev Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/ctlreg.h | 99 +++++++++++++++++++++++----------- 1 file changed, 67 insertions(+), 32 deletions(-) diff --git a/arch/s390/include/asm/ctlreg.h b/arch/s390/include/asm/ctlreg.h index 76ed70cca43c..6d4b85f2b541 100644 --- a/arch/s390/include/asm/ctlreg.h +++ b/arch/s390/include/asm/ctlreg.h @@ -10,40 +10,75 @@ #include -#define CR0_TRANSACTIONAL_EXECUTION BIT(63 - 8) -#define CR0_CLOCK_COMPARATOR_SIGN BIT(63 - 10) -#define CR0_CRYPTOGRAPHY_COUNTER BIT(63 - 13) -#define CR0_PAI_EXTENSION BIT(63 - 14) -#define CR0_CPUMF_EXTRACTION_AUTH BIT(63 - 15) -#define CR0_WARNING_TRACK BIT(63 - 30) -#define CR0_LOW_ADDRESS_PROTECTION BIT(63 - 35) -#define CR0_FETCH_PROTECTION_OVERRIDE BIT(63 - 38) -#define CR0_STORAGE_PROTECTION_OVERRIDE BIT(63 - 39) -#define CR0_EDAT BIT(63 - 40) -#define CR0_INSTRUCTION_EXEC_PROTECTION BIT(63 - 43) -#define CR0_VECTOR BIT(63 - 46) -#define CR0_MALFUNCTION_ALERT_SUBMASK BIT(63 - 48) -#define CR0_EMERGENCY_SIGNAL_SUBMASK BIT(63 - 49) -#define CR0_EXTERNAL_CALL_SUBMASK BIT(63 - 50) -#define CR0_CLOCK_COMPARATOR_SUBMASK BIT(63 - 52) -#define CR0_CPU_TIMER_SUBMASK BIT(63 - 53) -#define CR0_SERVICE_SIGNAL_SUBMASK BIT(63 - 54) -#define CR0_UNUSED_56 BIT(63 - 56) -#define CR0_INTERRUPT_KEY_SUBMASK BIT(63 - 57) -#define CR0_MEASUREMENT_ALERT_SUBMASK BIT(63 - 58) -#define CR0_ETR_SUBMASK BIT(63 - 59) -#define CR0_IUCV BIT(63 - 62) +#define CR0_TRANSACTIONAL_EXECUTION_BIT (63 - 8) +#define CR0_CLOCK_COMPARATOR_SIGN_BIT (63 - 10) +#define CR0_CRYPTOGRAPHY_COUNTER_BIT (63 - 13) +#define CR0_PAI_EXTENSION_BIT (63 - 14) +#define CR0_CPUMF_EXTRACTION_AUTH_BIT (63 - 15) +#define CR0_WARNING_TRACK_BIT (63 - 30) +#define CR0_LOW_ADDRESS_PROTECTION_BIT (63 - 35) +#define CR0_FETCH_PROTECTION_OVERRIDE_BIT (63 - 38) +#define CR0_STORAGE_PROTECTION_OVERRIDE_BIT (63 - 39) +#define CR0_EDAT_BIT (63 - 40) +#define CR0_INSTRUCTION_EXEC_PROTECTION_BIT (63 - 43) +#define CR0_VECTOR_BIT (63 - 46) +#define CR0_MALFUNCTION_ALERT_SUBMASK_BIT (63 - 48) +#define CR0_EMERGENCY_SIGNAL_SUBMASK_BIT (63 - 49) +#define CR0_EXTERNAL_CALL_SUBMASK_BIT (63 - 50) +#define CR0_CLOCK_COMPARATOR_SUBMASK_BIT (63 - 52) +#define CR0_CPU_TIMER_SUBMASK_BIT (63 - 53) +#define CR0_SERVICE_SIGNAL_SUBMASK_BIT (63 - 54) +#define CR0_UNUSED_56_BIT (63 - 56) +#define CR0_INTERRUPT_KEY_SUBMASK_BIT (63 - 57) +#define CR0_MEASUREMENT_ALERT_SUBMASK_BIT (63 - 58) +#define CR0_ETR_SUBMASK_BIT (63 - 59) +#define CR0_IUCV_BIT (63 - 62) -#define CR2_MIO_ADDRESSING BIT(63 - 58) -#define CR2_GUARDED_STORAGE BIT(63 - 59) +#define CR0_TRANSACTIONAL_EXECUTION BIT(CR0_TRANSACTIONAL_EXECUTION_BIT) +#define CR0_CLOCK_COMPARATOR_SIGN BIT(CR0_CLOCK_COMPARATOR_SIGN_BIT) +#define CR0_CRYPTOGRAPHY_COUNTER BIT(CR0_CRYPTOGRAPHY_COUNTER_BIT) +#define CR0_PAI_EXTENSION BIT(CR0_PAI_EXTENSION_BIT) +#define CR0_CPUMF_EXTRACTION_AUTH BIT(CR0_CPUMF_EXTRACTION_AUTH_BIT) +#define CR0_WARNING_TRACK BIT(CR0_WARNING_TRACK_BIT) +#define CR0_LOW_ADDRESS_PROTECTION BIT(CR0_LOW_ADDRESS_PROTECTION_BIT) +#define CR0_FETCH_PROTECTION_OVERRIDE BIT(CR0_FETCH_PROTECTION_OVERRIDE_BIT) +#define CR0_STORAGE_PROTECTION_OVERRIDE BIT(CR0_STORAGE_PROTECTION_OVERRIDE_BIT) +#define CR0_EDAT BIT(CR0_EDAT_BIT) +#define CR0_INSTRUCTION_EXEC_PROTECTION BIT(CR0_INSTRUCTION_EXEC_PROTECTION_BIT) +#define CR0_VECTOR BIT(CR0_VECTOR_BIT) +#define CR0_MALFUNCTION_ALERT_SUBMASK BIT(CR0_MALFUNCTION_ALERT_SUBMASK_BIT) +#define CR0_EMERGENCY_SIGNAL_SUBMASK BIT(CR0_EMERGENCY_SIGNAL_SUBMASK_BIT) +#define CR0_EXTERNAL_CALL_SUBMASK BIT(CR0_EXTERNAL_CALL_SUBMASK_BIT) +#define CR0_CLOCK_COMPARATOR_SUBMASK BIT(CR0_CLOCK_COMPARATOR_SUBMASK_BIT) +#define CR0_CPU_TIMER_SUBMASK BIT(CR0_CPU_TIMER_SUBMASK_BIT) +#define CR0_SERVICE_SIGNAL_SUBMASK BIT(CR0_SERVICE_SIGNAL_SUBMASK_BIT) +#define CR0_UNUSED_56 BIT(CR0_UNUSED_56_BIT) +#define CR0_INTERRUPT_KEY_SUBMASK BIT(CR0_INTERRUPT_KEY_SUBMASK_BIT) +#define CR0_MEASUREMENT_ALERT_SUBMASK BIT(CR0_MEASUREMENT_ALERT_SUBMASK_BIT) +#define CR0_ETR_SUBMASK BIT(CR0_ETR_SUBMASK_BIT) +#define CR0_IUCV BIT(CR0_IUCV_BIT) -#define CR14_UNUSED_32 BIT(63 - 32) -#define CR14_UNUSED_33 BIT(63 - 33) -#define CR14_CHANNEL_REPORT_SUBMASK BIT(63 - 35) -#define CR14_RECOVERY_SUBMASK BIT(63 - 36) -#define CR14_DEGRADATION_SUBMASK BIT(63 - 37) -#define CR14_EXTERNAL_DAMAGE_SUBMASK BIT(63 - 38) -#define CR14_WARNING_SUBMASK BIT(63 - 39) +#define CR2_MIO_ADDRESSING_BIT (63 - 58) +#define CR2_GUARDED_STORAGE_BIT (63 - 59) + +#define CR2_MIO_ADDRESSING BIT(CR2_MIO_ADDRESSING_BIT) +#define CR2_GUARDED_STORAGE BIT(CR2_GUARDED_STORAGE_BIT) + +#define CR14_UNUSED_32_BIT (63 - 32) +#define CR14_UNUSED_33_BIT (63 - 33) +#define CR14_CHANNEL_REPORT_SUBMASK_BIT (63 - 35) +#define CR14_RECOVERY_SUBMASK_BIT (63 - 36) +#define CR14_DEGRADATION_SUBMASK_BIT (63 - 37) +#define CR14_EXTERNAL_DAMAGE_SUBMASK_BIT (63 - 38) +#define CR14_WARNING_SUBMASK_BIT (63 - 39) + +#define CR14_UNUSED_32 BIT(CR14_UNUSED_32_BIT) +#define CR14_UNUSED_33 BIT(CR14_UNUSED_33_BIT) +#define CR14_CHANNEL_REPORT_SUBMASK BIT(CR14_CHANNEL_REPORT_SUBMASK_BIT) +#define CR14_RECOVERY_SUBMASK BIT(CR14_RECOVERY_SUBMASK_BIT) +#define CR14_DEGRADATION_SUBMASK BIT(CR14_DEGRADATION_SUBMASK_BIT) +#define CR14_EXTERNAL_DAMAGE_SUBMASK BIT(CR14_EXTERNAL_DAMAGE_SUBMASK_BIT) +#define CR14_WARNING_SUBMASK BIT(CR14_WARNING_SUBMASK_BIT) #ifndef __ASSEMBLY__ From 99441a38c391b1115e405d1f47ede237fca37f1b Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 11 Sep 2023 21:40:13 +0200 Subject: [PATCH 145/475] s390: use control register bit defines Use control register bit defines instead of plain numbers where possible. Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/boot/startup.c | 2 +- arch/s390/kernel/early.c | 12 ++++++------ arch/s390/kernel/guarded_storage.c | 6 +++--- arch/s390/kernel/ipl.c | 2 +- arch/s390/kernel/nmi.c | 8 ++++---- arch/s390/kernel/perf_cpum_cf.c | 2 +- arch/s390/kernel/perf_pai_crypto.c | 4 ++-- arch/s390/kernel/perf_pai_ext.c | 4 ++-- arch/s390/kernel/ptrace.c | 2 +- arch/s390/kernel/time.c | 4 ++-- arch/s390/kvm/kvm-s390.c | 6 +++--- arch/s390/kvm/priv.c | 2 +- arch/s390/mm/vmem.c | 2 +- arch/s390/pci/pci.c | 2 +- drivers/s390/cio/crw.c | 2 +- net/iucv/iucv.c | 2 +- 16 files changed, 31 insertions(+), 31 deletions(-) diff --git a/arch/s390/boot/startup.c b/arch/s390/boot/startup.c index cf622479647f..8826c4f18645 100644 --- a/arch/s390/boot/startup.c +++ b/arch/s390/boot/startup.c @@ -49,7 +49,7 @@ static void detect_facilities(void) { if (test_facility(8)) { machine.has_edat1 = 1; - local_ctl_set_bit(0, 23); + local_ctl_set_bit(0, CR0_EDAT_BIT); } if (test_facility(78)) machine.has_edat2 = 1; diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c index 0beaa0ba58a8..ff1f02b54771 100644 --- a/arch/s390/kernel/early.c +++ b/arch/s390/kernel/early.c @@ -216,7 +216,7 @@ static __init void detect_machine_facilities(void) { if (test_facility(8)) { S390_lowcore.machine_flags |= MACHINE_FLAG_EDAT1; - system_ctl_set_bit(0, 23); + system_ctl_set_bit(0, CR0_EDAT_BIT); } if (test_facility(78)) S390_lowcore.machine_flags |= MACHINE_FLAG_EDAT2; @@ -224,13 +224,13 @@ static __init void detect_machine_facilities(void) S390_lowcore.machine_flags |= MACHINE_FLAG_IDTE; if (test_facility(50) && test_facility(73)) { S390_lowcore.machine_flags |= MACHINE_FLAG_TE; - system_ctl_set_bit(0, 55); + system_ctl_set_bit(0, CR0_TRANSACTIONAL_EXECUTION_BIT); } if (test_facility(51)) S390_lowcore.machine_flags |= MACHINE_FLAG_TLB_LC; if (test_facility(129)) { S390_lowcore.machine_flags |= MACHINE_FLAG_VX; - system_ctl_set_bit(0, 17); + system_ctl_set_bit(0, CR0_VECTOR_BIT); } if (test_facility(130)) S390_lowcore.machine_flags |= MACHINE_FLAG_NX; @@ -240,7 +240,7 @@ static __init void detect_machine_facilities(void) /* Enabled signed clock comparator comparisons */ S390_lowcore.machine_flags |= MACHINE_FLAG_SCC; clock_comparator_max = -1ULL >> 1; - system_ctl_set_bit(0, 53); + system_ctl_set_bit(0, CR0_CLOCK_COMPARATOR_SIGN_BIT); } if (IS_ENABLED(CONFIG_PCI) && test_facility(153)) { S390_lowcore.machine_flags |= MACHINE_FLAG_PCI_MIO; @@ -260,7 +260,7 @@ static inline void save_vector_registers(void) static inline void setup_low_address_protection(void) { - system_ctl_set_bit(0, 28); + system_ctl_set_bit(0, CR0_LOW_ADDRESS_PROTECTION_BIT); } static inline void setup_access_registers(void) @@ -273,7 +273,7 @@ static inline void setup_access_registers(void) static int __init disable_vector_extension(char *str) { S390_lowcore.machine_flags &= ~MACHINE_FLAG_VX; - system_ctl_clear_bit(0, 17); + system_ctl_clear_bit(0, CR0_VECTOR_BIT); return 0; } early_param("novx", disable_vector_extension); diff --git a/arch/s390/kernel/guarded_storage.c b/arch/s390/kernel/guarded_storage.c index 945acfc77ee4..0b68168d9566 100644 --- a/arch/s390/kernel/guarded_storage.c +++ b/arch/s390/kernel/guarded_storage.c @@ -28,7 +28,7 @@ static int gs_enable(void) return -ENOMEM; gs_cb->gsd = 25; preempt_disable(); - local_ctl_set_bit(2, 4); + local_ctl_set_bit(2, CR2_GUARDED_STORAGE_BIT); load_gs_cb(gs_cb); current->thread.gs_cb = gs_cb; preempt_enable(); @@ -42,7 +42,7 @@ static int gs_disable(void) preempt_disable(); kfree(current->thread.gs_cb); current->thread.gs_cb = NULL; - local_ctl_clear_bit(2, 4); + local_ctl_clear_bit(2, CR2_GUARDED_STORAGE_BIT); preempt_enable(); } return 0; @@ -84,7 +84,7 @@ void gs_load_bc_cb(struct pt_regs *regs) if (gs_cb) { kfree(current->thread.gs_cb); current->thread.gs_bc_cb = NULL; - local_ctl_set_bit(2, 4); + local_ctl_set_bit(2, CR2_GUARDED_STORAGE_BIT); load_gs_cb(gs_cb); current->thread.gs_cb = gs_cb; } diff --git a/arch/s390/kernel/ipl.c b/arch/s390/kernel/ipl.c index 492dd8f371da..cc364fce6aa9 100644 --- a/arch/s390/kernel/ipl.c +++ b/arch/s390/kernel/ipl.c @@ -2381,7 +2381,7 @@ void s390_reset_system(void) set_prefix(0); /* Disable lowcore protection */ - local_ctl_clear_bit(0, 28); + local_ctl_clear_bit(0, CR0_LOW_ADDRESS_PROTECTION_BIT); diag_amode31_ops.diag308_reset(); } diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c index 579cebc58d8c..0daf0f1cdfc9 100644 --- a/arch/s390/kernel/nmi.c +++ b/arch/s390/kernel/nmi.c @@ -185,7 +185,7 @@ void s390_handle_mcck(void) static int mchchk_wng_posted = 0; /* Use single cpu clear, as we cannot handle smp here. */ - local_ctl_clear_bit(14, 24); /* Disable WARNING MCH */ + local_ctl_clear_bit(14, CR14_WARNING_SUBMASK_BIT); if (xchg(&mchchk_wng_posted, 1) == 0) kill_cad_pid(SIGPWR, 1); } @@ -505,9 +505,9 @@ NOKPROBE_SYMBOL(s390_do_machine_check); static int __init machine_check_init(void) { - system_ctl_set_bit(14, 25); /* enable external damage MCH */ - system_ctl_set_bit(14, 27); /* enable system recovery MCH */ - system_ctl_set_bit(14, 24); /* enable warning MCH */ + system_ctl_set_bit(14, CR14_EXTERNAL_DAMAGE_SUBMASK_BIT); + system_ctl_set_bit(14, CR14_RECOVERY_SUBMASK_BIT); + system_ctl_set_bit(14, CR14_WARNING_SUBMASK_BIT); return 0; } early_initcall(machine_check_init); diff --git a/arch/s390/kernel/perf_cpum_cf.c b/arch/s390/kernel/perf_cpum_cf.c index 4e05c0cb1648..41ed6e0f0a2a 100644 --- a/arch/s390/kernel/perf_cpum_cf.c +++ b/arch/s390/kernel/perf_cpum_cf.c @@ -1193,7 +1193,7 @@ static int __init cpumf_pmu_init(void) * Clear bit 15 of cr0 to unauthorize problem-state to * extract measurement counters */ - system_ctl_clear_bit(0, 48); + system_ctl_clear_bit(0, CR0_CPUMF_EXTRACTION_AUTH_BIT); /* register handler for measurement-alert interruptions */ rc = register_external_irq(EXT_IRQ_MEASURE_ALERT, diff --git a/arch/s390/kernel/perf_pai_crypto.c b/arch/s390/kernel/perf_pai_crypto.c index 6612fd13e95b..db470243966f 100644 --- a/arch/s390/kernel/perf_pai_crypto.c +++ b/arch/s390/kernel/perf_pai_crypto.c @@ -265,7 +265,7 @@ static int paicrypt_add(struct perf_event *event, int flags) if (++cpump->active_events == 1) { ccd = virt_to_phys(cpump->page) | PAI_CRYPTO_KERNEL_OFFSET; WRITE_ONCE(S390_lowcore.ccd, ccd); - local_ctl_set_bit(0, 50); + local_ctl_set_bit(0, CR0_CRYPTOGRAPHY_COUNTER_BIT); } cpump->event = event; if (flags & PERF_EF_START && !event->attr.sample_period) { @@ -294,7 +294,7 @@ static void paicrypt_del(struct perf_event *event, int flags) /* Only counting needs to read counter */ paicrypt_stop(event, PERF_EF_UPDATE); if (--cpump->active_events == 0) { - local_ctl_clear_bit(0, 50); + local_ctl_clear_bit(0, CR0_CRYPTOGRAPHY_COUNTER_BIT); WRITE_ONCE(S390_lowcore.ccd, 0); } } diff --git a/arch/s390/kernel/perf_pai_ext.c b/arch/s390/kernel/perf_pai_ext.c index 9f724ae44fc7..e91c77155f08 100644 --- a/arch/s390/kernel/perf_pai_ext.c +++ b/arch/s390/kernel/perf_pai_ext.c @@ -339,7 +339,7 @@ static int paiext_add(struct perf_event *event, int flags) S390_lowcore.aicd = virt_to_phys(cpump->paiext_cb); pcb->acc = virt_to_phys(cpump->area) | 0x1; /* Enable CPU instruction lookup for PAIE1 control block */ - local_ctl_set_bit(0, 49); + local_ctl_set_bit(0, CR0_PAI_EXTENSION_BIT); debug_sprintf_event(paiext_dbg, 4, "%s 1508 %llx acc %llx\n", __func__, S390_lowcore.aicd, pcb->acc); } @@ -375,7 +375,7 @@ static void paiext_del(struct perf_event *event, int flags) } if (--cpump->active_events == 0) { /* Disable CPU instruction lookup for PAIE1 control block */ - local_ctl_clear_bit(0, 49); + local_ctl_clear_bit(0, CR0_PAI_EXTENSION_BIT); pcb->acc = 0; S390_lowcore.aicd = 0; debug_sprintf_event(paiext_dbg, 4, "%s 1508 %llx acc %llx\n", diff --git a/arch/s390/kernel/ptrace.c b/arch/s390/kernel/ptrace.c index 6a825351ff41..046403471c5d 100644 --- a/arch/s390/kernel/ptrace.c +++ b/arch/s390/kernel/ptrace.c @@ -1114,7 +1114,7 @@ static int s390_gs_cb_set(struct task_struct *target, target->thread.gs_cb = data; *target->thread.gs_cb = gs_cb; if (target == current) { - local_ctl_set_bit(2, 4); + local_ctl_set_bit(2, CR2_GUARDED_STORAGE_BIT); restore_gs_cb(target->thread.gs_cb); } preempt_enable(); diff --git a/arch/s390/kernel/time.c b/arch/s390/kernel/time.c index 34c4fbf4e555..14abad953c02 100644 --- a/arch/s390/kernel/time.c +++ b/arch/s390/kernel/time.c @@ -173,10 +173,10 @@ void init_cpu_timer(void) clockevents_register_device(cd); /* Enable clock comparator timer interrupt. */ - local_ctl_set_bit(0, 11); + local_ctl_set_bit(0, CR0_CLOCK_COMPARATOR_SUBMASK_BIT); /* Always allow the timing alert external interrupt. */ - local_ctl_set_bit(0, 4); + local_ctl_set_bit(0, CR0_ETR_SUBMASK_BIT); } static void clock_comparator_interrupt(struct ext_code ext_code, diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 92bd15fe769a..e44e83f004ce 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -4918,7 +4918,7 @@ static void sync_regs_fmt2(struct kvm_vcpu *vcpu) } if (MACHINE_HAS_GS) { preempt_disable(); - local_ctl_set_bit(2, 4); + local_ctl_set_bit(2, CR2_GUARDED_STORAGE_BIT); if (current->thread.gs_cb) { vcpu->arch.host_gscb = current->thread.gs_cb; save_gs_cb(vcpu->arch.host_gscb); @@ -4995,13 +4995,13 @@ static void store_regs_fmt2(struct kvm_vcpu *vcpu) kvm_run->s.regs.diag318 = vcpu->arch.diag318_info.val; if (MACHINE_HAS_GS) { preempt_disable(); - local_ctl_set_bit(2, 4); + local_ctl_set_bit(2, CR2_GUARDED_STORAGE_BIT); if (vcpu->arch.gs_enabled) save_gs_cb(current->thread.gs_cb); current->thread.gs_cb = vcpu->arch.host_gscb; restore_gs_cb(vcpu->arch.host_gscb); if (!vcpu->arch.host_gscb) - local_ctl_clear_bit(2, 4); + local_ctl_clear_bit(2, CR2_GUARDED_STORAGE_BIT); vcpu->arch.host_gscb = NULL; preempt_enable(); } diff --git a/arch/s390/kvm/priv.c b/arch/s390/kvm/priv.c index c199e4b5e5ce..621a17fd1a1b 100644 --- a/arch/s390/kvm/priv.c +++ b/arch/s390/kvm/priv.c @@ -57,7 +57,7 @@ static int handle_gs(struct kvm_vcpu *vcpu) if (test_kvm_facility(vcpu->kvm, 133)) { VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (lazy)"); preempt_disable(); - local_ctl_set_bit(2, 4); + local_ctl_set_bit(2, CR2_GUARDED_STORAGE_BIT); current->thread.gs_cb = (struct gs_cb *)&vcpu->run->s.regs.gscb; restore_gs_cb(current->thread.gs_cb); preempt_enable(); diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c index 95f04a502e62..5ed2e9254649 100644 --- a/arch/s390/mm/vmem.c +++ b/arch/s390/mm/vmem.c @@ -660,7 +660,7 @@ void __init vmem_map_init(void) __set_memory_4k(__va(0), RELOC_HIDE(__va(0), ident_map_size)); } if (MACHINE_HAS_NX) - system_ctl_set_bit(0, 20); + system_ctl_set_bit(0, CR0_INSTRUCTION_EXEC_PROTECTION_BIT); pr_info("Write protected kernel read-only data: %luk\n", (unsigned long)(__end_rodata - _stext) >> 10); } diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c index a0aecc94e284..6fab5c085565 100644 --- a/arch/s390/pci/pci.c +++ b/arch/s390/pci/pci.c @@ -1094,7 +1094,7 @@ static int __init pci_base_init(void) if (MACHINE_HAS_PCI_MIO) { static_branch_enable(&have_mio); - system_ctl_set_bit(2, 5); + system_ctl_set_bit(2, CR2_MIO_ADDRESSING_BIT); } rc = zpci_debug_init(); diff --git a/drivers/s390/cio/crw.c b/drivers/s390/cio/crw.c index 872b1f2855cb..4916dd0a7eb1 100644 --- a/drivers/s390/cio/crw.c +++ b/drivers/s390/cio/crw.c @@ -156,7 +156,7 @@ static int __init crw_machine_check_init(void) task = kthread_run(crw_collect_info, NULL, "kmcheck"); if (IS_ERR(task)) return PTR_ERR(task); - system_ctl_set_bit(14, 28); /* enable channel report MCH */ + system_ctl_set_bit(14, CR14_CHANNEL_REPORT_SUBMASK_BIT); return 0; } device_initcall(crw_machine_check_init); diff --git a/net/iucv/iucv.c b/net/iucv/iucv.c index b5cee8dad926..0ed6e34d6edd 100644 --- a/net/iucv/iucv.c +++ b/net/iucv/iucv.c @@ -1823,7 +1823,7 @@ static int __init iucv_init(void) rc = -EPROTONOSUPPORT; goto out; } - system_ctl_set_bit(0, 1); + system_ctl_set_bit(0, CR0_IUCV_BIT); rc = iucv_query_maxconn(); if (rc) goto out_ctl; From e49be27e18c59639be28cd0d766caf594fe2b77f Mon Sep 17 00:00:00 2001 From: Xu Yang Date: Tue, 19 Sep 2023 16:09:29 +0800 Subject: [PATCH 146/475] perf jevents: fix no member named 'entries' issue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The struct "pmu_events_table" has been changed after commit 2e255b4f9f41 (perf jevents: Group events by PMU, 2023-08-23). So there doesn't exist 'entries' in pmu_events_table anymore. This will align the members with that commit. Othewise, below errors will be printed when run jevent.py: pmu-events/pmu-events.c:5485:26: error: ‘struct pmu_metrics_table’ has no member named ‘entries’ 5485 | .entries = pmu_metrics__freescale_imx8dxl_sys, Signed-off-by: Xu Yang Reviewed-by: Ian Rogers Tested-by: Namhyung Kim Link: https://lore.kernel.org/r/20230919080929.3807123-1-xu.yang_2@nxp.com Signed-off-by: Namhyung Kim --- tools/perf/pmu-events/jevents.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/perf/pmu-events/jevents.py b/tools/perf/pmu-events/jevents.py index a7e88332276d..af15fa2cadbd 100755 --- a/tools/perf/pmu-events/jevents.py +++ b/tools/perf/pmu-events/jevents.py @@ -764,8 +764,8 @@ static const struct pmu_sys_events pmu_sys_event_tables[] = { continue _args.output_file.write(f"""\t{{ \t\t.metric_table = {{ -\t\t\t.entries = {tblname}, -\t\t\t.length = ARRAY_SIZE({tblname}) +\t\t\t.pmus = {tblname}, +\t\t\t.num_pmus = ARRAY_SIZE({tblname}) \t\t}}, \t\t.name = \"{tblname}\", \t}}, From 67e18e132f0fd738f8c8cac3aa1420312073f795 Mon Sep 17 00:00:00 2001 From: Zheng Yejian Date: Thu, 14 Sep 2023 15:26:44 +0800 Subject: [PATCH 147/475] livepatch: Fix missing newline character in klp_resolve_symbols() Without the newline character, the log may not be printed immediately after the error occurs. Fixes: ca376a937486 ("livepatch: Prevent module-specific KLP rela sections from referencing vmlinux symbols") Signed-off-by: Zheng Yejian Reviewed-by: Petr Mladek Signed-off-by: Petr Mladek Link: https://lore.kernel.org/r/20230914072644.4098857-1-zhengyejian1@huawei.com --- kernel/livepatch/core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/livepatch/core.c b/kernel/livepatch/core.c index 61328328c474..ecbc9b6aba3a 100644 --- a/kernel/livepatch/core.c +++ b/kernel/livepatch/core.c @@ -243,7 +243,7 @@ static int klp_resolve_symbols(Elf_Shdr *sechdrs, const char *strtab, * symbols are exported and normal relas can be used instead. */ if (!sec_vmlinux && sym_vmlinux) { - pr_err("invalid access to vmlinux symbol '%s' from module-specific livepatch relocation section", + pr_err("invalid access to vmlinux symbol '%s' from module-specific livepatch relocation section\n", sym_name); return -EINVAL; } From 98a0465531a5982dd897fd81222a5d3465999951 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Wed, 20 Sep 2023 17:58:38 +0206 Subject: [PATCH 148/475] printk: fix illegal pbufs access for !CONFIG_PRINTK When CONFIG_PRINTK is not set, PRINTK_MESSAGE_MAX is 0. This leads to a zero-sized array @outbuf in @printk_shared_pbufs. In console_flush_all() a pointer to the first element of the array is assigned with: char *outbuf = &printk_shared_pbufs.outbuf[0]; For !CONFIG_PRINTK this leads to a compiler warning: warning: array subscript 0 is outside array bounds of 'char[0]' [-Warray-bounds] This is not really dangerous because printk_get_next_message() always returns false for !CONFIG_PRINTK, which leads to @outbuf never being used. However, it makes no sense to even compile these functions for !CONFIG_PRINTK. Extend the existing '#ifdef CONFIG_PRINTK' block to contain the formatting and emitting functions since these have no purpose in !CONFIG_PRINTK. This also allows removing several more !CONFIG_PRINTK dummies as well as moving @suppress_panic_printk into a CONFIG_PRINTK block. Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202309201724.M9BMAQIh-lkp@intel.com/ Signed-off-by: John Ogness Reviewed-by: Sergey Senozhatsky Signed-off-by: Petr Mladek Link: https://lore.kernel.org/r/20230920155238.670439-1-john.ogness@linutronix.de --- kernel/printk/printk.c | 44 +++++++++++++++++------------------------- 1 file changed, 18 insertions(+), 26 deletions(-) diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 778359b21761..7f40d9122caa 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -102,12 +102,6 @@ DEFINE_STATIC_SRCU(console_srcu); */ int __read_mostly suppress_printk; -/* - * During panic, heavy printk by other CPUs can delay the - * panic and risk deadlock on console resources. - */ -static int __read_mostly suppress_panic_printk; - #ifdef CONFIG_LOCKDEP static struct lockdep_map console_lock_dep_map = { .name = "console_lock" @@ -445,6 +439,12 @@ static int console_msg_format = MSG_FORMAT_DEFAULT; static DEFINE_MUTEX(syslog_lock); #ifdef CONFIG_PRINTK +/* + * During panic, heavy printk by other CPUs can delay the + * panic and risk deadlock on console resources. + */ +static int __read_mostly suppress_panic_printk; + DECLARE_WAIT_QUEUE_HEAD(log_wait); /* All 3 protected by @syslog_lock. */ /* the next printk record to read by syslog(READ) or /proc/kmsg */ @@ -2346,22 +2346,6 @@ static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progre static u64 syslog_seq; -static size_t record_print_text(const struct printk_record *r, - bool syslog, bool time) -{ - return 0; -} -static ssize_t info_print_ext_header(char *buf, size_t size, - struct printk_info *info) -{ - return 0; -} -static ssize_t msg_print_ext_body(char *buf, size_t size, - char *text, size_t text_len, - struct dev_printk_info *dev_info) { return 0; } -static void console_lock_spinning_enable(void) { } -static int console_lock_spinning_disable_and_check(int cookie) { return 0; } -static bool suppress_message_printing(int level) { return false; } static bool pr_flush(int timeout_ms, bool reset_on_progress) { return true; } static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progress) { return true; } @@ -2715,6 +2699,8 @@ static void __console_unlock(void) up_console_sem(); } +#ifdef CONFIG_PRINTK + /* * Prepend the message in @pmsg->pbufs->outbuf with a "dropped message". This * is achieved by shifting the existing message over and inserting the dropped @@ -2729,7 +2715,6 @@ static void __console_unlock(void) * * If @pmsg->pbufs->outbuf is modified, @pmsg->outbuf_len is updated. */ -#ifdef CONFIG_PRINTK void console_prepend_dropped(struct printk_message *pmsg, unsigned long dropped) { struct printk_buffers *pbufs = pmsg->pbufs; @@ -2761,9 +2746,6 @@ void console_prepend_dropped(struct printk_message *pmsg, unsigned long dropped) memcpy(outbuf, scratchbuf, len); pmsg->outbuf_len += len; } -#else -#define console_prepend_dropped(pmsg, dropped) -#endif /* CONFIG_PRINTK */ /* * Read and format the specified record (or a later record if the specified @@ -2921,6 +2903,16 @@ skip: return true; } +#else + +static bool console_emit_next_record(struct console *con, bool *handover, int cookie) +{ + *handover = false; + return false; +} + +#endif /* CONFIG_PRINTK */ + /* * Print out all remaining records to all consoles. * From fa52d995d1d0b64e75f63571df2e57ba22ff0cb1 Mon Sep 17 00:00:00 2001 From: Veronika Molnarova Date: Tue, 19 Sep 2023 17:02:42 +0200 Subject: [PATCH 149/475] perf test stat+shadow_stat.sh: Add threshold for rounding errors The test was failing in specific scenarios due to imperfection of FP arithmetics. The `bc` command wasn't correctly rounding the result of division causing the failure. Replace the `bc` with `awk` which should work with more decimal places and add a threshold to catch any possible rounding errors. The acceptable rounding error is set to 0.01 when the test passes with a warning message. Signed-off-by: Veronika Molnarova Acked-by: Michael Petlan Link: https://lore.kernel.org/r/20230919150419.23193-1-vmolnaro@redhat.com Signed-off-by: Namhyung Kim --- tools/perf/tests/shell/stat+shadow_stat.sh | 30 +++++++++++++++++----- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/tools/perf/tests/shell/stat+shadow_stat.sh b/tools/perf/tests/shell/stat+shadow_stat.sh index a1918a15e36a..7d9b9d597a50 100755 --- a/tools/perf/tests/shell/stat+shadow_stat.sh +++ b/tools/perf/tests/shell/stat+shadow_stat.sh @@ -4,6 +4,8 @@ set -e +THRESHOLD=0.015 + # skip if system-wide mode is forbidden perf stat -a true > /dev/null 2>&1 || exit 2 @@ -33,10 +35,18 @@ test_global_aggr() fi # use printf for rounding and a leading zero - res=`printf "%.2f" "$(echo "scale=6; $num / $cyc" | bc -q)"` + res=`echo $num $cyc | awk '{printf "%.2f", $1 / $2}'` if [ "$ipc" != "$res" ]; then - echo "IPC is different: $res != $ipc ($num / $cyc)" - exit 1 + # check the difference from the real result for FP imperfections + diff=`echo $ipc $res $THRESHOLD | \ + awk '{x = ($1 - $2) < 0 ? ($2 - $1) : ($1 - $2); print (x > $3)}'` + + if [ $diff -eq 1 ]; then + echo "IPC is different: $res != $ipc ($num / $cyc)" + exit 1 + fi + + echo "Warning: Difference of IPC is under the threshold" fi done } @@ -67,10 +77,18 @@ test_no_aggr() fi # use printf for rounding and a leading zero - res=`printf "%.2f" "$(echo "scale=6; $num / $cyc" | bc -q)"` + res=`echo $num $cyc | awk '{printf "%.2f", $1 / $2}'` if [ "$ipc" != "$res" ]; then - echo "IPC is different for $cpu: $res != $ipc ($num / $cyc)" - exit 1 + # check difference from the real result for FP imperfections + diff=`echo $ipc $res $THRESHOLD | \ + awk '{x = ($1 - $2) < 0 ? ($2 - $1) : ($1 - $2); print (x > $3)}'` + + if [ $diff -eq 1 ]; then + echo "IPC is different: $res != $ipc ($num / $cyc)" + exit 1 + fi + + echo "Warning: Difference of IPC is under the threshold" fi done } From 29441ab3a30a1a5e2dc35ea519bbd5dd210b7666 Mon Sep 17 00:00:00 2001 From: Veronika Molnarova Date: Tue, 19 Sep 2023 17:02:43 +0200 Subject: [PATCH 150/475] perf test lock_contention.sh: Skip test if not enough CPUs Machines with less then 4 CPUs weren't consistently triggering lock events required for the test. Skip the test on those machines. The limit of 4 CPUs is set as it generates around 100 lock events for a test. Signed-off-by: Veronika Molnarova Acked-by: Michael Petlan Link: https://lore.kernel.org/r/20230919150419.23193-2-vmolnaro@redhat.com Signed-off-by: Namhyung Kim --- tools/perf/tests/shell/lock_contention.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/perf/tests/shell/lock_contention.sh b/tools/perf/tests/shell/lock_contention.sh index 966e67db75f3..d5a191d3d090 100755 --- a/tools/perf/tests/shell/lock_contention.sh +++ b/tools/perf/tests/shell/lock_contention.sh @@ -32,6 +32,12 @@ check() { err=2 exit fi + + if [ `nproc` -lt 4 ]; then + echo "[Skip] Low number of CPUs (`nproc`), lock event cannot be triggered certainly" + err=2 + exit + fi } test_record() From 59faeaf80d0239412056c4495cc49097e9cea4da Mon Sep 17 00:00:00 2001 From: Ilkka Koskinen Date: Tue, 19 Sep 2023 23:18:39 -0700 Subject: [PATCH 151/475] perf vendor events arm64: Fix for AmpereOne metrics This patch addresses review comments that were given for 705ed549148f ("perf vendor events arm64: Add AmpereOne metrics") but didn't make it to the original patch [1][2] Changes include: A fix for backend_memory formula, use of standard metrics when possible, using #slots, renaming metrics to avoid spaces in the names, and cleanup. [1] https://lore.kernel.org/linux-perf-users/e9bdacb-a231-36af-6a2e-6918ee7effa@os.amperecomputing.com/ [2] https://lore.kernel.org/linux-perf-users/20230826192352.3043220-1-ilkka@os.amperecomputing.com/ Fixes: 705ed549148f ("perf vendor events arm64: Add AmpereOne metrics") Signed-off-by: Ilkka Koskinen Reviewed-by: Ian Rogers Cc: James Clark Cc: Will Deacon Cc: Leo Yan Cc: Mike Leach Cc: Dave Kleikamp Cc: John Garry Cc: D Scott Phillips Cc: linux-arm-kernel@lists.infradead.org Link: https://lore.kernel.org/r/20230920061839.2437413-1-ilkka@os.amperecomputing.com Signed-off-by: Namhyung Kim --- .../arch/arm64/ampere/ampereone/metrics.json | 418 +++++++++--------- 1 file changed, 220 insertions(+), 198 deletions(-) diff --git a/tools/perf/pmu-events/arch/arm64/ampere/ampereone/metrics.json b/tools/perf/pmu-events/arch/arm64/ampere/ampereone/metrics.json index 1e7e8901a445..e2848a9d4848 100644 --- a/tools/perf/pmu-events/arch/arm64/ampere/ampereone/metrics.json +++ b/tools/perf/pmu-events/arch/arm64/ampere/ampereone/metrics.json @@ -1,362 +1,384 @@ [ { + "MetricName": "branch_miss_pred_rate", "MetricExpr": "BR_MIS_PRED / BR_PRED", "BriefDescription": "Branch predictor misprediction rate. May not count branches that are never resolved because they are in the misprediction shadow of an earlier branch", - "MetricGroup": "Branch Prediction", - "MetricName": "Misprediction" + "MetricGroup": "branch", + "ScaleUnit": "100%" }, { - "MetricExpr": "BR_MIS_PRED_RETIRED / BR_RETIRED", - "BriefDescription": "Branch predictor misprediction rate", - "MetricGroup": "Branch Prediction", - "MetricName": "Misprediction (retired)" - }, - { - "MetricExpr": "BUS_ACCESS / ( BUS_CYCLES * 1)", + "MetricName": "bus_utilization", + "MetricExpr": "((BUS_ACCESS / (BUS_CYCLES * 1)) * 100)", "BriefDescription": "Core-to-uncore bus utilization", "MetricGroup": "Bus", - "MetricName": "Bus utilization" + "ScaleUnit": "1percent of bus cycles" }, { - "MetricExpr": "L1D_CACHE_REFILL / L1D_CACHE", - "BriefDescription": "L1D cache miss rate", - "MetricGroup": "Cache", - "MetricName": "L1D cache miss" + "MetricName": "l1d_cache_miss_ratio", + "MetricExpr": "(L1D_CACHE_REFILL / L1D_CACHE)", + "BriefDescription": "This metric measures the ratio of level 1 data cache accesses missed to the total number of level 1 data cache accesses. This gives an indication of the effectiveness of the level 1 data cache.", + "MetricGroup": "Miss_Ratio;L1D_Cache_Effectiveness", + "ScaleUnit": "1per cache access" }, { + "MetricName": "l1i_cache_miss_ratio", + "MetricExpr": "(L1I_CACHE_REFILL / L1I_CACHE)", + "BriefDescription": "This metric measures the ratio of level 1 instruction cache accesses missed to the total number of level 1 instruction cache accesses. This gives an indication of the effectiveness of the level 1 instruction cache.", + "MetricGroup": "Miss_Ratio;L1I_Cache_Effectiveness", + "ScaleUnit": "1per cache access" + }, + { + "MetricName": "Miss_Ratio;l1d_cache_read_miss", "MetricExpr": "L1D_CACHE_LMISS_RD / L1D_CACHE_RD", "BriefDescription": "L1D cache read miss rate", "MetricGroup": "Cache", - "MetricName": "L1D cache read miss" + "ScaleUnit": "1per cache read access" }, { - "MetricExpr": "L1I_CACHE_REFILL / L1I_CACHE", - "BriefDescription": "L1I cache miss rate", - "MetricGroup": "Cache", - "MetricName": "L1I cache miss" - }, - { - "MetricExpr": "L2D_CACHE_REFILL / L2D_CACHE", - "BriefDescription": "L2 cache miss rate", - "MetricGroup": "Cache", - "MetricName": "L2 cache miss" + "MetricName": "l2_cache_miss_ratio", + "MetricExpr": "(L2D_CACHE_REFILL / L2D_CACHE)", + "BriefDescription": "This metric measures the ratio of level 2 cache accesses missed to the total number of level 2 cache accesses. This gives an indication of the effectiveness of the level 2 cache, which is a unified cache that stores both data and instruction. Note that cache accesses in this cache are either data memory access or instruction fetch as this is a unified cache.", + "MetricGroup": "Miss_Ratio;L2_Cache_Effectiveness", + "ScaleUnit": "1per cache access" }, { + "MetricName": "l1i_cache_read_miss_rate", "MetricExpr": "L1I_CACHE_LMISS / L1I_CACHE", "BriefDescription": "L1I cache read miss rate", "MetricGroup": "Cache", - "MetricName": "L1I cache read miss" + "ScaleUnit": "1per cache access" }, { + "MetricName": "l2d_cache_read_miss_rate", "MetricExpr": "L2D_CACHE_LMISS_RD / L2D_CACHE_RD", "BriefDescription": "L2 cache read miss rate", "MetricGroup": "Cache", - "MetricName": "L2 cache read miss" + "ScaleUnit": "1per cache read access" }, { - "MetricExpr": "(L1D_CACHE_LMISS_RD * 1000) / INST_RETIRED", + "MetricName": "l1d_cache_miss_mpki", + "MetricExpr": "(L1D_CACHE_LMISS_RD * 1e3) / INST_RETIRED", "BriefDescription": "Misses per thousand instructions (data)", "MetricGroup": "Cache", - "MetricName": "MPKI data" + "ScaleUnit": "1MPKI" }, { - "MetricExpr": "(L1I_CACHE_LMISS * 1000) / INST_RETIRED", + "MetricName": "l1i_cache_miss_mpki", + "MetricExpr": "(L1I_CACHE_LMISS * 1e3) / INST_RETIRED", "BriefDescription": "Misses per thousand instructions (instruction)", "MetricGroup": "Cache", - "MetricName": "MPKI instruction" + "ScaleUnit": "1MPKI" }, { - "MetricExpr": "ASE_SPEC / OP_SPEC", - "BriefDescription": "Proportion of advanced SIMD data processing operations (excluding DP_SPEC/LD_SPEC) operations", - "MetricGroup": "Instruction", - "MetricName": "ASE mix" + "MetricName": "simd_percentage", + "MetricExpr": "((ASE_SPEC / INST_SPEC) * 100)", + "BriefDescription": "This metric measures advanced SIMD operations as a percentage of total operations speculatively executed.", + "MetricGroup": "Operation_Mix", + "ScaleUnit": "1percent of operations" }, { - "MetricExpr": "CRYPTO_SPEC / OP_SPEC", - "BriefDescription": "Proportion of crypto data processing operations", - "MetricGroup": "Instruction", - "MetricName": "Crypto mix" + "MetricName": "crypto_percentage", + "MetricExpr": "((CRYPTO_SPEC / INST_SPEC) * 100)", + "BriefDescription": "This metric measures crypto operations as a percentage of operations speculatively executed.", + "MetricGroup": "Operation_Mix", + "ScaleUnit": "1percent of operations" }, { - "MetricExpr": "VFP_SPEC / (duration_time *1000000000)", + "MetricName": "gflops", + "MetricExpr": "VFP_SPEC / (duration_time * 1e9)", "BriefDescription": "Giga-floating point operations per second", - "MetricGroup": "Instruction", - "MetricName": "GFLOPS_ISSUED" + "MetricGroup": "InstructionMix" }, { - "MetricExpr": "DP_SPEC / OP_SPEC", - "BriefDescription": "Proportion of integer data processing operations", - "MetricGroup": "Instruction", - "MetricName": "Integer mix" + "MetricName": "integer_dp_percentage", + "MetricExpr": "((DP_SPEC / INST_SPEC) * 100)", + "BriefDescription": "This metric measures scalar integer operations as a percentage of operations speculatively executed.", + "MetricGroup": "Operation_Mix", + "ScaleUnit": "1percent of operations" }, { - "MetricExpr": "INST_RETIRED / CPU_CYCLES", - "BriefDescription": "Instructions per cycle", - "MetricGroup": "Instruction", - "MetricName": "IPC" + "MetricName": "ipc", + "MetricExpr": "(INST_RETIRED / CPU_CYCLES)", + "BriefDescription": "This metric measures the number of instructions retired per cycle.", + "MetricGroup": "General", + "ScaleUnit": "1per cycle" }, { - "MetricExpr": "LD_SPEC / OP_SPEC", - "BriefDescription": "Proportion of load operations", - "MetricGroup": "Instruction", - "MetricName": "Load mix" + "MetricName": "load_percentage", + "MetricExpr": "((LD_SPEC / INST_SPEC) * 100)", + "BriefDescription": "This metric measures load operations as a percentage of operations speculatively executed.", + "MetricGroup": "Operation_Mix", + "ScaleUnit": "1percent of operations" }, { - "MetricExpr": "LDST_SPEC/ OP_SPEC", - "BriefDescription": "Proportion of load & store operations", - "MetricGroup": "Instruction", - "MetricName": "Load-store mix" + "MetricName": "load_store_spec_rate", + "MetricExpr": "((LDST_SPEC / INST_SPEC) * 100)", + "BriefDescription": "The rate of load or store instructions speculatively executed to overall instructions speclatively executed", + "MetricGroup": "Operation_Mix", + "ScaleUnit": "1percent of operations" }, { - "MetricExpr": "INST_RETIRED / (duration_time * 1000000)", + "MetricName": "retired_mips", + "MetricExpr": "INST_RETIRED / (duration_time * 1e6)", "BriefDescription": "Millions of instructions per second", - "MetricGroup": "Instruction", - "MetricName": "MIPS_RETIRED" + "MetricGroup": "InstructionMix" }, { - "MetricExpr": "INST_SPEC / (duration_time * 1000000)", + "MetricName": "spec_utilization_mips", + "MetricExpr": "INST_SPEC / (duration_time * 1e6)", "BriefDescription": "Millions of instructions per second", - "MetricGroup": "Instruction", - "MetricName": "MIPS_UTILIZATION" + "MetricGroup": "PEutilization" }, { - "MetricExpr": "PC_WRITE_SPEC / OP_SPEC", - "BriefDescription": "Proportion of software change of PC operations", - "MetricGroup": "Instruction", - "MetricName": "PC write mix" + "MetricName": "pc_write_spec_rate", + "MetricExpr": "((PC_WRITE_SPEC / INST_SPEC) * 100)", + "BriefDescription": "The rate of software change of the PC speculatively executed to overall instructions speclatively executed", + "MetricGroup": "Operation_Mix", + "ScaleUnit": "1percent of operations" }, { - "MetricExpr": "ST_SPEC / OP_SPEC", - "BriefDescription": "Proportion of store operations", - "MetricGroup": "Instruction", - "MetricName": "Store mix" + "MetricName": "store_percentage", + "MetricExpr": "((ST_SPEC / INST_SPEC) * 100)", + "BriefDescription": "This metric measures store operations as a percentage of operations speculatively executed.", + "MetricGroup": "Operation_Mix", + "ScaleUnit": "1percent of operations" }, { - "MetricExpr": "VFP_SPEC / OP_SPEC", - "BriefDescription": "Proportion of FP operations", - "MetricGroup": "Instruction", - "MetricName": "VFP mix" + "MetricName": "scalar_fp_percentage", + "MetricExpr": "((VFP_SPEC / INST_SPEC) * 100)", + "BriefDescription": "This metric measures scalar floating point operations as a percentage of operations speculatively executed.", + "MetricGroup": "Operation_Mix", + "ScaleUnit": "1percent of operations" }, { - "MetricExpr": "1 - (OP_RETIRED/ (CPU_CYCLES * 4))", - "BriefDescription": "Proportion of slots lost", - "MetricGroup": "Speculation / TDA", - "MetricName": "CPU lost" + "MetricName": "retired_rate", + "MetricExpr": "OP_RETIRED / OP_SPEC", + "BriefDescription": "Of all the micro-operations issued, what percentage are retired(committed)", + "MetricGroup": "General", + "ScaleUnit": "100%" }, { - "MetricExpr": "OP_RETIRED/ (CPU_CYCLES * 4)", - "BriefDescription": "Proportion of slots retiring", - "MetricGroup": "Speculation / TDA", - "MetricName": "CPU utilization" + "MetricName": "wasted", + "MetricExpr": "1 - (OP_RETIRED / (CPU_CYCLES * #slots))", + "BriefDescription": "Of all the micro-operations issued, what proportion are lost", + "MetricGroup": "General", + "ScaleUnit": "100%" }, { - "MetricExpr": "OP_RETIRED - OP_SPEC", - "BriefDescription": "Operations lost due to misspeculation", - "MetricGroup": "Speculation / TDA", - "MetricName": "Operations lost" + "MetricName": "wasted_rate", + "MetricExpr": "1 - OP_RETIRED / OP_SPEC", + "BriefDescription": "Of all the micro-operations issued, what percentage are not retired(committed)", + "MetricGroup": "General", + "ScaleUnit": "100%" }, { - "MetricExpr": "1 - (OP_RETIRED / OP_SPEC)", - "BriefDescription": "Proportion of operations lost", - "MetricGroup": "Speculation / TDA", - "MetricName": "Operations lost (ratio)" - }, - { - "MetricExpr": "OP_RETIRED / OP_SPEC", - "BriefDescription": "Proportion of operations retired", - "MetricGroup": "Speculation / TDA", - "MetricName": "Operations retired" - }, - { - "MetricExpr": "STALL_BACKEND_CACHE / CPU_CYCLES", + "MetricName": "stall_backend_cache_rate", + "MetricExpr": "((STALL_BACKEND_CACHE / CPU_CYCLES) * 100)", "BriefDescription": "Proportion of cycles stalled and no operations issued to backend and cache miss", "MetricGroup": "Stall", - "MetricName": "Stall backend cache cycles" + "ScaleUnit": "1percent of cycles" }, { - "MetricExpr": "STALL_BACKEND_RESOURCE / CPU_CYCLES", + "MetricName": "stall_backend_resource_rate", + "MetricExpr": "((STALL_BACKEND_RESOURCE / CPU_CYCLES) * 100)", "BriefDescription": "Proportion of cycles stalled and no operations issued to backend and resource full", "MetricGroup": "Stall", - "MetricName": "Stall backend resource cycles" + "ScaleUnit": "1percent of cycles" }, { - "MetricExpr": "STALL_BACKEND_TLB / CPU_CYCLES", + "MetricName": "stall_backend_tlb_rate", + "MetricExpr": "((STALL_BACKEND_TLB / CPU_CYCLES) * 100)", "BriefDescription": "Proportion of cycles stalled and no operations issued to backend and TLB miss", "MetricGroup": "Stall", - "MetricName": "Stall backend tlb cycles" + "ScaleUnit": "1percent of cycles" }, { - "MetricExpr": "STALL_FRONTEND_CACHE / CPU_CYCLES", + "MetricName": "stall_frontend_cache_rate", + "MetricExpr": "((STALL_FRONTEND_CACHE / CPU_CYCLES) * 100)", "BriefDescription": "Proportion of cycles stalled and no ops delivered from frontend and cache miss", "MetricGroup": "Stall", - "MetricName": "Stall frontend cache cycles" + "ScaleUnit": "1percent of cycles" }, { - "MetricExpr": "STALL_FRONTEND_TLB / CPU_CYCLES", + "MetricName": "stall_frontend_tlb_rate", + "MetricExpr": "((STALL_FRONTEND_TLB / CPU_CYCLES) * 100)", "BriefDescription": "Proportion of cycles stalled and no ops delivered from frontend and TLB miss", "MetricGroup": "Stall", - "MetricName": "Stall frontend tlb cycles" + "ScaleUnit": "1percent of cycles" }, { - "MetricExpr": "DTLB_WALK / L1D_TLB", - "BriefDescription": "D-side walk per d-side translation request", - "MetricGroup": "TLB", - "MetricName": "DTLB walks" + "MetricName": "dtlb_walk_ratio", + "MetricExpr": "(DTLB_WALK / L1D_TLB)", + "BriefDescription": "This metric measures the ratio of data TLB Walks to the total number of data TLB accesses. This gives an indication of the effectiveness of the data TLB accesses.", + "MetricGroup": "Miss_Ratio;DTLB_Effectiveness", + "ScaleUnit": "1per TLB access" }, { - "MetricExpr": "ITLB_WALK / L1I_TLB", - "BriefDescription": "I-side walk per i-side translation request", - "MetricGroup": "TLB", - "MetricName": "ITLB walks" + "MetricName": "itlb_walk_ratio", + "MetricExpr": "(ITLB_WALK / L1I_TLB)", + "BriefDescription": "This metric measures the ratio of instruction TLB Walks to the total number of instruction TLB accesses. This gives an indication of the effectiveness of the instruction TLB accesses.", + "MetricGroup": "Miss_Ratio;ITLB_Effectiveness", + "ScaleUnit": "1per TLB access" }, { - "MetricExpr": "STALL_SLOT_BACKEND / (CPU_CYCLES * 4)", - "BriefDescription": "Fraction of slots backend bound", - "MetricGroup": "TopDownL1", - "MetricName": "backend" + "ArchStdEvent": "backend_bound" }, { - "MetricExpr": "1 - (retiring + lost + backend)", - "BriefDescription": "Fraction of slots frontend bound", - "MetricGroup": "TopDownL1", - "MetricName": "frontend" + "ArchStdEvent": "frontend_bound", + "MetricExpr": "100 - (retired_fraction + slots_lost_misspeculation_fraction + backend_bound)" }, { - "MetricExpr": "((OP_SPEC - OP_RETIRED) / (CPU_CYCLES * 4))", + "MetricName": "slots_lost_misspeculation_fraction", + "MetricExpr": "100 * ((OP_SPEC - OP_RETIRED) / (CPU_CYCLES * #slots))", "BriefDescription": "Fraction of slots lost due to misspeculation", - "MetricGroup": "TopDownL1", - "MetricName": "lost" + "MetricGroup": "Default;TopdownL1", + "ScaleUnit": "1percent of slots" }, { - "MetricExpr": "(OP_RETIRED / (CPU_CYCLES * 4))", + "MetricName": "retired_fraction", + "MetricExpr": "100 * (OP_RETIRED / (CPU_CYCLES * #slots))", "BriefDescription": "Fraction of slots retiring, useful work", - "MetricGroup": "TopDownL1", - "MetricName": "retiring" + "MetricGroup": "Default;TopdownL1", + "ScaleUnit": "1percent of slots" }, { - "MetricExpr": "backend - backend_memory", + "MetricName": "backend_core", + "MetricExpr": "(backend_bound / 100) - backend_memory", "BriefDescription": "Fraction of slots the CPU was stalled due to backend non-memory subsystem issues", - "MetricGroup": "TopDownL2", - "MetricName": "backend_core" + "MetricGroup": "TopdownL2", + "ScaleUnit": "100%" }, { - "MetricExpr": "(STALL_BACKEND_TLB + STALL_BACKEND_CACHE + STALL_BACKEND_MEM) / CPU_CYCLES ", + "MetricName": "backend_memory", + "MetricExpr": "(STALL_BACKEND_TLB + STALL_BACKEND_CACHE) / CPU_CYCLES", "BriefDescription": "Fraction of slots the CPU was stalled due to backend memory subsystem issues (cache/tlb miss)", - "MetricGroup": "TopDownL2", - "MetricName": "backend_memory" + "MetricGroup": "TopdownL2", + "ScaleUnit": "100%" }, { - "MetricExpr": " (BR_MIS_PRED_RETIRED / GPC_FLUSH) * lost", + "MetricName": "branch_mispredict", + "MetricExpr": "(BR_MIS_PRED_RETIRED / GPC_FLUSH) * slots_lost_misspeculation_fraction", "BriefDescription": "Fraction of slots lost due to branch misprediciton", - "MetricGroup": "TopDownL2", - "MetricName": "branch_mispredict" + "MetricGroup": "TopdownL2", + "ScaleUnit": "1percent of slots" }, { - "MetricExpr": "frontend - frontend_latency", + "MetricName": "frontend_bandwidth", + "MetricExpr": "frontend_bound - frontend_latency", "BriefDescription": "Fraction of slots the CPU did not dispatch at full bandwidth - able to dispatch partial slots only (1, 2, or 3 uops)", - "MetricGroup": "TopDownL2", - "MetricName": "frontend_bandwidth" + "MetricGroup": "TopdownL2", + "ScaleUnit": "1percent of slots" }, { - "MetricExpr": "(STALL_FRONTEND - ((STALL_SLOT_FRONTEND - (frontend * CPU_CYCLES * 4)) / 4)) / CPU_CYCLES", + "MetricName": "frontend_latency", + "MetricExpr": "((STALL_FRONTEND - ((STALL_SLOT_FRONTEND - ((frontend_bound / 100) * CPU_CYCLES * #slots)) / #slots)) / CPU_CYCLES) * 100", "BriefDescription": "Fraction of slots the CPU was stalled due to frontend latency issues (cache/tlb miss); nothing to dispatch", - "MetricGroup": "TopDownL2", - "MetricName": "frontend_latency" + "MetricGroup": "TopdownL2", + "ScaleUnit": "1percent of slots" }, { - "MetricExpr": "lost - branch_mispredict", + "MetricName": "other_miss_pred", + "MetricExpr": "slots_lost_misspeculation_fraction - branch_mispredict", "BriefDescription": "Fraction of slots lost due to other/non-branch misprediction misspeculation", - "MetricGroup": "TopDownL2", - "MetricName": "other_clears" + "MetricGroup": "TopdownL2", + "ScaleUnit": "1percent of slots" }, { - "MetricExpr": "(IXU_NUM_UOPS_ISSUED + FSU_ISSUED) / (CPU_CYCLES * 6)", + "MetricName": "pipe_utilization", + "MetricExpr": "100 * ((IXU_NUM_UOPS_ISSUED + FSU_ISSUED) / (CPU_CYCLES * 6))", "BriefDescription": "Fraction of execute slots utilized", - "MetricGroup": "TopDownL2", - "MetricName": "pipe_utilization" + "MetricGroup": "TopdownL2", + "ScaleUnit": "1percent of slots" }, { - "MetricExpr": "STALL_BACKEND_MEM / CPU_CYCLES", + "MetricName": "d_cache_l2_miss_rate", + "MetricExpr": "((STALL_BACKEND_MEM / CPU_CYCLES) * 100)", "BriefDescription": "Fraction of cycles the CPU was stalled due to data L2 cache miss", - "MetricGroup": "TopDownL3", - "MetricName": "d_cache_l2_miss" + "MetricGroup": "TopdownL3", + "ScaleUnit": "1percent of cycles" }, { - "MetricExpr": "STALL_BACKEND_CACHE / CPU_CYCLES", + "MetricName": "d_cache_miss_rate", + "MetricExpr": "((STALL_BACKEND_CACHE / CPU_CYCLES) * 100)", "BriefDescription": "Fraction of cycles the CPU was stalled due to data cache miss", - "MetricGroup": "TopDownL3", - "MetricName": "d_cache_miss" + "MetricGroup": "TopdownL3", + "ScaleUnit": "1percent of cycles" }, { - "MetricExpr": "STALL_BACKEND_TLB / CPU_CYCLES", + "MetricName": "d_tlb_miss_rate", + "MetricExpr": "((STALL_BACKEND_TLB / CPU_CYCLES) * 100)", "BriefDescription": "Fraction of cycles the CPU was stalled due to data TLB miss", - "MetricGroup": "TopDownL3", - "MetricName": "d_tlb_miss" + "MetricGroup": "TopdownL3", + "ScaleUnit": "1percent of cycles" }, { - "MetricExpr": "FSU_ISSUED / (CPU_CYCLES * 2)", + "MetricName": "fsu_pipe_utilization", + "MetricExpr": "((FSU_ISSUED / (CPU_CYCLES * 2)) * 100)", "BriefDescription": "Fraction of FSU execute slots utilized", - "MetricGroup": "TopDownL3", - "MetricName": "fsu_pipe_utilization" + "MetricGroup": "TopdownL3", + "ScaleUnit": "1percent of slots" }, { - "MetricExpr": "STALL_FRONTEND_CACHE / CPU_CYCLES", + "MetricName": "i_cache_miss_rate", + "MetricExpr": "((STALL_FRONTEND_CACHE / CPU_CYCLES) * 100)", "BriefDescription": "Fraction of cycles the CPU was stalled due to instruction cache miss", - "MetricGroup": "TopDownL3", - "MetricName": "i_cache_miss" + "MetricGroup": "TopdownL3", + "ScaleUnit": "1percent of slots" }, { - "MetricExpr": " STALL_FRONTEND_TLB / CPU_CYCLES ", + "MetricName": "i_tlb_miss_rate", + "MetricExpr": "((STALL_FRONTEND_TLB / CPU_CYCLES) * 100)", "BriefDescription": "Fraction of cycles the CPU was stalled due to instruction TLB miss", - "MetricGroup": "TopDownL3", - "MetricName": "i_tlb_miss" + "MetricGroup": "TopdownL3", + "ScaleUnit": "1percent of slots" }, { - "MetricExpr": "IXU_NUM_UOPS_ISSUED / (CPU_CYCLES / 4)", + "MetricName": "ixu_pipe_utilization", + "MetricExpr": "((IXU_NUM_UOPS_ISSUED / (CPU_CYCLES * #slots)) * 100)", "BriefDescription": "Fraction of IXU execute slots utilized", - "MetricGroup": "TopDownL3", - "MetricName": "ixu_pipe_utilization" + "MetricGroup": "TopdownL3", + "ScaleUnit": "1percent of slots" }, { - "MetricExpr": "IDR_STALL_FLUSH / CPU_CYCLES", + "MetricName": "stall_recovery_rate", + "MetricExpr": "((IDR_STALL_FLUSH / CPU_CYCLES) * 100)", "BriefDescription": "Fraction of cycles the CPU was stalled due to flush recovery", - "MetricGroup": "TopDownL3", - "MetricName": "recovery" + "MetricGroup": "TopdownL3", + "ScaleUnit": "1percent of slots" }, { - "MetricExpr": "STALL_BACKEND_RESOURCE / CPU_CYCLES", - "BriefDescription": "Fraction of cycles the CPU was stalled due to core resource shortage", - "MetricGroup": "TopDownL3", - "MetricName": "resource" - }, - { - "MetricExpr": "IDR_STALL_FSU_SCHED / CPU_CYCLES ", + "MetricName": "stall_fsu_sched_rate", + "MetricExpr": "((IDR_STALL_FSU_SCHED / CPU_CYCLES) * 100)", "BriefDescription": "Fraction of cycles the CPU was stalled and FSU was full", - "MetricGroup": "TopDownL4", - "MetricName": "stall_fsu_sched" + "MetricGroup": "TopdownL4", + "ScaleUnit": "1percent of cycles" }, { - "MetricExpr": "IDR_STALL_IXU_SCHED / CPU_CYCLES ", + "MetricName": "stall_ixu_sched_rate", + "MetricExpr": "((IDR_STALL_IXU_SCHED / CPU_CYCLES) * 100)", "BriefDescription": "Fraction of cycles the CPU was stalled and IXU was full", - "MetricGroup": "TopDownL4", - "MetricName": "stall_ixu_sched" + "MetricGroup": "TopdownL4", + "ScaleUnit": "1percent of cycles" }, { - "MetricExpr": "IDR_STALL_LOB_ID / CPU_CYCLES ", + "MetricName": "stall_lob_id_rate", + "MetricExpr": "((IDR_STALL_LOB_ID / CPU_CYCLES) * 100)", "BriefDescription": "Fraction of cycles the CPU was stalled and LOB was full", - "MetricGroup": "TopDownL4", - "MetricName": "stall_lob_id" + "MetricGroup": "TopdownL4", + "ScaleUnit": "1percent of cycles" }, { - "MetricExpr": "IDR_STALL_ROB_ID / CPU_CYCLES", + "MetricName": "stall_rob_id_rate", + "MetricExpr": "((IDR_STALL_ROB_ID / CPU_CYCLES) * 100)", "BriefDescription": "Fraction of cycles the CPU was stalled and ROB was full", - "MetricGroup": "TopDownL4", - "MetricName": "stall_rob_id" + "MetricGroup": "TopdownL4", + "ScaleUnit": "1percent of cycles" }, { - "MetricExpr": "IDR_STALL_SOB_ID / CPU_CYCLES ", + "MetricName": "stall_sob_id_rate", + "MetricExpr": "((IDR_STALL_SOB_ID / CPU_CYCLES) * 100)", "BriefDescription": "Fraction of cycles the CPU was stalled and SOB was full", - "MetricGroup": "TopDownL4", - "MetricName": "stall_sob_id" + "MetricGroup": "TopdownL4", + "ScaleUnit": "1percent of cycles" } ] From c87b8cc816da05d0a4d7e0696bb62909c8592809 Mon Sep 17 00:00:00 2001 From: Charles Han Date: Mon, 18 Sep 2023 11:36:23 +0800 Subject: [PATCH 152/475] perf vendors events: Remove repeated word in comments Remove the repeated word "of" in comments. Signed-off-by: Charles Han Reviewed-by: Ian Rogers Cc: james.clark@arm.com Cc: nick.forrington@arm.com Cc: leo.yan@linaro.org Cc: mike.leach@linaro.org Cc: john.g.garry@oracle.com Cc: ilkka@os.amperecomputing.com Link: https://lore.kernel.org/r/20230918033623.159213-1-hanchunchao@inspur.com Signed-off-by: Namhyung Kim --- tools/perf/pmu-events/arch/arm64/mapfile.csv | 2 +- tools/perf/pmu-events/arch/nds32/mapfile.csv | 2 +- tools/perf/pmu-events/arch/powerpc/mapfile.csv | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/perf/pmu-events/arch/arm64/mapfile.csv b/tools/perf/pmu-events/arch/arm64/mapfile.csv index 32674ddd2b63..5b58db5032c1 100644 --- a/tools/perf/pmu-events/arch/arm64/mapfile.csv +++ b/tools/perf/pmu-events/arch/arm64/mapfile.csv @@ -4,7 +4,7 @@ # where # MIDR Processor version # Variant[23:20] and Revision [3:0] should be zero. -# Version could be used to track version of of JSON file +# Version could be used to track version of JSON file # but currently unused. # JSON/file/pathname is the path to JSON file, relative # to tools/perf/pmu-events/arch/arm64/. diff --git a/tools/perf/pmu-events/arch/nds32/mapfile.csv b/tools/perf/pmu-events/arch/nds32/mapfile.csv index efb395f26883..c76e5fbdac23 100644 --- a/tools/perf/pmu-events/arch/nds32/mapfile.csv +++ b/tools/perf/pmu-events/arch/nds32/mapfile.csv @@ -4,7 +4,7 @@ # where # MIDR Processor version # Variant[23:20] and Revision [3:0] should be zero. -# Version could be used to track version of of JSON file +# Version could be used to track version of JSON file # but currently unused. # JSON/file/pathname is the path to JSON file, relative # to tools/perf/pmu-events/arch/arm64/. diff --git a/tools/perf/pmu-events/arch/powerpc/mapfile.csv b/tools/perf/pmu-events/arch/powerpc/mapfile.csv index 4abdfc3f9692..a534ff6db14b 100644 --- a/tools/perf/pmu-events/arch/powerpc/mapfile.csv +++ b/tools/perf/pmu-events/arch/powerpc/mapfile.csv @@ -3,7 +3,7 @@ # # where # PVR Processor version -# Version could be used to track version of of JSON file +# Version could be used to track version of JSON file # but currently unused. # JSON/file/pathname is the path to JSON file, relative # to tools/perf/pmu-events/arch/powerpc/. From f9cdeb58a9cf46c09b56f5f661ea8da24b6458c3 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Fri, 15 Sep 2023 20:56:40 -0700 Subject: [PATCH 153/475] perf evlist: Avoid frequency mode for the dummy event Dummy events are created with an attribute where the period and freq are zero. evsel__config will then see the uninitialized values and initialize them in evsel__default_freq_period. As fequency mode is used by default the dummy event would be set to use frequency mode. However, this has no effect on the dummy event but does cause unnecessary timers/interrupts. Avoid this overhead by setting the period to 1 for dummy events. evlist__add_aux_dummy calls evlist__add_dummy then sets freq=0 and period=1. This isn't necessary after this change and so the setting is removed. From Stephane: The dummy event is not counting anything. It is used to collect mmap records and avoid a race condition during the synthesize mmap phase of perf record. As such, it should not cause any overhead during active profiling. Yet, it did. Because of a bug the dummy event was programmed as a sampling event in frequency mode. Events in that mode incur more kernel overheads because on timer tick, the kernel has to look at the number of samples for each event and potentially adjust the sampling period to achieve the desired frequency. The dummy event was therefore adding a frequency event to task and ctx contexts we may otherwise not have any, e.g., perf record -a -e cpu/event=0x3c,period=10000000/. On each timer tick the perf_adjust_freq_unthr_context() is invoked and if ctx->nr_freq is non-zero, then the kernel will loop over ALL the events of the context looking for frequency mode ones. In doing, so it locks the context, and enable/disable the PMU of each hw event. If all the events of the context are in period mode, the kernel will have to traverse the list for nothing incurring overhead. The overhead is multiplied by a very large factor when this happens in a guest kernel. There is no need for the dummy event to be in frequency mode, it does not count anything and therefore should not cause extra overhead for no reason. Fixes: 5bae0250237f ("perf evlist: Introduce perf_evlist__new_dummy constructor") Reported-by: Stephane Eranian Signed-off-by: Ian Rogers Acked-by: Adrian Hunter Cc: Yang Jihong Cc: Kan Liang Link: https://lore.kernel.org/r/20230916035640.1074422-1-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/util/evlist.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/perf/util/evlist.c b/tools/perf/util/evlist.c index 25c3ebe2c2f5..e36da58522ef 100644 --- a/tools/perf/util/evlist.c +++ b/tools/perf/util/evlist.c @@ -251,6 +251,9 @@ static struct evsel *evlist__dummy_event(struct evlist *evlist) .type = PERF_TYPE_SOFTWARE, .config = PERF_COUNT_SW_DUMMY, .size = sizeof(attr), /* to capture ABI version */ + /* Avoid frequency mode for dummy events to avoid associated timers. */ + .freq = 0, + .sample_period = 1, }; return evsel__new_idx(&attr, evlist->core.nr_entries); @@ -277,8 +280,6 @@ struct evsel *evlist__add_aux_dummy(struct evlist *evlist, bool system_wide) evsel->core.attr.exclude_kernel = 1; evsel->core.attr.exclude_guest = 1; evsel->core.attr.exclude_hv = 1; - evsel->core.attr.freq = 0; - evsel->core.attr.sample_period = 1; evsel->core.system_wide = system_wide; evsel->no_aux_samples = true; evsel->name = strdup("dummy:u"); From cd0f9e3f7e046c253a23a728cb7953819cff1541 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Mon, 25 Sep 2023 06:50:37 +0100 Subject: [PATCH 154/475] perf kwork: Fix spelling mistake "Captuer" -> "Capture" There is a spelling mistake in a pr_debug message. Fix it. (I didn't see this one in the first spell check scan I ran). Signed-off-by: Colin Ian King Reviewed-by: Ian Rogers Cc: kernel-janitors@vger.kernel.org Link: https://lore.kernel.org/r/20230925055037.18089-1-colin.i.king@gmail.com Signed-off-by: Namhyung Kim --- tools/perf/builtin-kwork.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/perf/builtin-kwork.c b/tools/perf/builtin-kwork.c index 7e8dd35d764b..f007a9b27065 100644 --- a/tools/perf/builtin-kwork.c +++ b/tools/perf/builtin-kwork.c @@ -1866,7 +1866,7 @@ static void sig_handler(int sig) * Simply capture termination signal so that * the program can continue after pause returns */ - pr_debug("Captuer signal %d\n", sig); + pr_debug("Capture signal %d\n", sig); } static int perf_kwork__report_bpf(struct perf_kwork *kwork) From 35de80c709d730987c9a13911f836fbd92c5d1c5 Mon Sep 17 00:00:00 2001 From: Athira Rajeev Date: Thu, 7 Sep 2023 22:45:38 +0530 Subject: [PATCH 155/475] tests/shell: Fix shellcheck SC1090 to handle the location of sourced files Running shellcheck on some of the shell scripts throws below error: In tests/shell/coresight/unroll_loop_thread_10.sh line 8: . "$(dirname $0)"/../lib/coresight.sh ^-- SC1090: Can't follow non-constant source. Use a directive to specify location. This happens on shellcheck version "0.6.0". Fix shellcheck warning for SC1090 using "shellcheck source="i option to mention the location of sourced files. Signed-off-by: Athira Rajeev Tested-by: Ian Rogers Reviewed-by: Kajol Jain Cc: maddy@linux.ibm.com Cc: disgoel@linux.vnet.ibm.com Cc: linuxppc-dev@lists.ozlabs.org Link: https://lore.kernel.org/r/20230907171540.36736-2-atrajeev@linux.vnet.ibm.com Signed-off-by: Namhyung Kim --- tools/perf/tests/shell/coresight/asm_pure_loop.sh | 3 +++ tools/perf/tests/shell/coresight/memcpy_thread_16k_10.sh | 3 +++ tools/perf/tests/shell/coresight/thread_loop_check_tid_10.sh | 3 +++ tools/perf/tests/shell/coresight/thread_loop_check_tid_2.sh | 3 +++ tools/perf/tests/shell/coresight/unroll_loop_thread_10.sh | 3 +++ tools/perf/tests/shell/probe_vfs_getname.sh | 2 ++ tools/perf/tests/shell/record+probe_libc_inet_pton.sh | 2 ++ tools/perf/tests/shell/record+script_probe_vfs_getname.sh | 2 ++ tools/perf/tests/shell/record.sh | 1 + tools/perf/tests/shell/stat+csv_output.sh | 1 + tools/perf/tests/shell/stat+std_output.sh | 1 + tools/perf/tests/shell/test_intel_pt.sh | 1 + tools/perf/tests/shell/trace+probe_vfs_getname.sh | 1 + 13 files changed, 26 insertions(+) diff --git a/tools/perf/tests/shell/coresight/asm_pure_loop.sh b/tools/perf/tests/shell/coresight/asm_pure_loop.sh index 779bc8608e1e..04387061e9f3 100755 --- a/tools/perf/tests/shell/coresight/asm_pure_loop.sh +++ b/tools/perf/tests/shell/coresight/asm_pure_loop.sh @@ -5,7 +5,10 @@ # Carsten Haitzler , 2021 TEST="asm_pure_loop" + +# shellcheck source=../lib/coresight.sh . "$(dirname $0)"/../lib/coresight.sh + ARGS="" DATV="out" DATA="$DATD/perf-$TEST-$DATV.data" diff --git a/tools/perf/tests/shell/coresight/memcpy_thread_16k_10.sh b/tools/perf/tests/shell/coresight/memcpy_thread_16k_10.sh index 08a44e52ce9b..c17e442ac741 100755 --- a/tools/perf/tests/shell/coresight/memcpy_thread_16k_10.sh +++ b/tools/perf/tests/shell/coresight/memcpy_thread_16k_10.sh @@ -5,7 +5,10 @@ # Carsten Haitzler , 2021 TEST="memcpy_thread" + +# shellcheck source=../lib/coresight.sh . "$(dirname $0)"/../lib/coresight.sh + ARGS="16 10 1" DATV="16k_10" DATA="$DATD/perf-$TEST-$DATV.data" diff --git a/tools/perf/tests/shell/coresight/thread_loop_check_tid_10.sh b/tools/perf/tests/shell/coresight/thread_loop_check_tid_10.sh index c83a200dede4..e47c4e955d0e 100755 --- a/tools/perf/tests/shell/coresight/thread_loop_check_tid_10.sh +++ b/tools/perf/tests/shell/coresight/thread_loop_check_tid_10.sh @@ -5,7 +5,10 @@ # Carsten Haitzler , 2021 TEST="thread_loop" + +# shellcheck source=../lib/coresight.sh . "$(dirname $0)"/../lib/coresight.sh + ARGS="10 1" DATV="check-tid-10th" DATA="$DATD/perf-$TEST-$DATV.data" diff --git a/tools/perf/tests/shell/coresight/thread_loop_check_tid_2.sh b/tools/perf/tests/shell/coresight/thread_loop_check_tid_2.sh index 6346fd5e87c8..8bf94a02e384 100755 --- a/tools/perf/tests/shell/coresight/thread_loop_check_tid_2.sh +++ b/tools/perf/tests/shell/coresight/thread_loop_check_tid_2.sh @@ -5,7 +5,10 @@ # Carsten Haitzler , 2021 TEST="thread_loop" + +# shellcheck source=../lib/coresight.sh . "$(dirname $0)"/../lib/coresight.sh + ARGS="2 20" DATV="check-tid-2th" DATA="$DATD/perf-$TEST-$DATV.data" diff --git a/tools/perf/tests/shell/coresight/unroll_loop_thread_10.sh b/tools/perf/tests/shell/coresight/unroll_loop_thread_10.sh index 7304e3d3a6ff..0dc9ef424233 100755 --- a/tools/perf/tests/shell/coresight/unroll_loop_thread_10.sh +++ b/tools/perf/tests/shell/coresight/unroll_loop_thread_10.sh @@ -5,7 +5,10 @@ # Carsten Haitzler , 2021 TEST="unroll_loop_thread" + +# shellcheck source=../lib/coresight.sh . "$(dirname $0)"/../lib/coresight.sh + ARGS="10" DATV="10" DATA="$DATD/perf-$TEST-$DATV.data" diff --git a/tools/perf/tests/shell/probe_vfs_getname.sh b/tools/perf/tests/shell/probe_vfs_getname.sh index 871243d6d03a..554e12e83c55 100755 --- a/tools/perf/tests/shell/probe_vfs_getname.sh +++ b/tools/perf/tests/shell/probe_vfs_getname.sh @@ -4,10 +4,12 @@ # SPDX-License-Identifier: GPL-2.0 # Arnaldo Carvalho de Melo , 2017 +# shellcheck source=lib/probe.sh . "$(dirname $0)"/lib/probe.sh skip_if_no_perf_probe || exit 2 +# shellcheck source=lib/probe_vfs_getname.sh . "$(dirname $0)"/lib/probe_vfs_getname.sh add_probe_vfs_getname || skip_if_no_debuginfo diff --git a/tools/perf/tests/shell/record+probe_libc_inet_pton.sh b/tools/perf/tests/shell/record+probe_libc_inet_pton.sh index 89214a6d9951..eebeea6bdc76 100755 --- a/tools/perf/tests/shell/record+probe_libc_inet_pton.sh +++ b/tools/perf/tests/shell/record+probe_libc_inet_pton.sh @@ -10,7 +10,9 @@ # SPDX-License-Identifier: GPL-2.0 # Arnaldo Carvalho de Melo , 2017 +# shellcheck source=lib/probe.sh . "$(dirname "$0")/lib/probe.sh" +# shellcheck source=lib/probe_vfs_getname.sh . "$(dirname "$0")/lib/probe_vfs_getname.sh" libc=$(grep -w libc /proc/self/maps | head -1 | sed -r 's/.*[[:space:]](\/.*)/\1/g') diff --git a/tools/perf/tests/shell/record+script_probe_vfs_getname.sh b/tools/perf/tests/shell/record+script_probe_vfs_getname.sh index 7f664f1889d9..5eedbe29bba1 100755 --- a/tools/perf/tests/shell/record+script_probe_vfs_getname.sh +++ b/tools/perf/tests/shell/record+script_probe_vfs_getname.sh @@ -9,10 +9,12 @@ # SPDX-License-Identifier: GPL-2.0 # Arnaldo Carvalho de Melo , 2017 +# shellcheck source=lib/probe.sh . "$(dirname "$0")/lib/probe.sh" skip_if_no_perf_probe || exit 2 +# shellcheck source=lib/probe_vfs_getname.sh . "$(dirname "$0")/lib/probe_vfs_getname.sh" record_open_file() { diff --git a/tools/perf/tests/shell/record.sh b/tools/perf/tests/shell/record.sh index 4fbc74805d52..29443b8e8876 100755 --- a/tools/perf/tests/shell/record.sh +++ b/tools/perf/tests/shell/record.sh @@ -5,6 +5,7 @@ set -e shelldir=$(dirname "$0") +# shellcheck source=lib/waiting.sh . "${shelldir}"/lib/waiting.sh err=0 diff --git a/tools/perf/tests/shell/stat+csv_output.sh b/tools/perf/tests/shell/stat+csv_output.sh index d890eb26e914..f1818fa6d9ce 100755 --- a/tools/perf/tests/shell/stat+csv_output.sh +++ b/tools/perf/tests/shell/stat+csv_output.sh @@ -6,6 +6,7 @@ set -e +# shellcheck source=lib/stat_output.sh . "$(dirname $0)"/lib/stat_output.sh csv_sep=@ diff --git a/tools/perf/tests/shell/stat+std_output.sh b/tools/perf/tests/shell/stat+std_output.sh index fb2b10547a11..4fcdd1a9142c 100755 --- a/tools/perf/tests/shell/stat+std_output.sh +++ b/tools/perf/tests/shell/stat+std_output.sh @@ -6,6 +6,7 @@ set -e +# shellcheck source=lib/stat_output.sh . "$(dirname $0)"/lib/stat_output.sh stat_output=$(mktemp /tmp/__perf_test.stat_output.std.XXXXX) diff --git a/tools/perf/tests/shell/test_intel_pt.sh b/tools/perf/tests/shell/test_intel_pt.sh index 3a8b9bffa022..723ec501f99a 100755 --- a/tools/perf/tests/shell/test_intel_pt.sh +++ b/tools/perf/tests/shell/test_intel_pt.sh @@ -8,6 +8,7 @@ set -e perf list | grep -q 'intel_pt//' || exit 2 shelldir=$(dirname "$0") +# shellcheck source=lib/waiting.sh . "${shelldir}"/lib/waiting.sh skip_cnt=0 diff --git a/tools/perf/tests/shell/trace+probe_vfs_getname.sh b/tools/perf/tests/shell/trace+probe_vfs_getname.sh index 4014487cf4d9..3146a1eece07 100755 --- a/tools/perf/tests/shell/trace+probe_vfs_getname.sh +++ b/tools/perf/tests/shell/trace+probe_vfs_getname.sh @@ -10,6 +10,7 @@ # SPDX-License-Identifier: GPL-2.0 # Arnaldo Carvalho de Melo , 2017 +# shellcheck source=lib/probe.sh . "$(dirname $0)"/lib/probe.sh skip_if_no_perf_probe || exit 2 From 55312ca7527e018c5ceacc4e311ac201c8bca90d Mon Sep 17 00:00:00 2001 From: Athira Rajeev Date: Thu, 7 Sep 2023 22:45:39 +0530 Subject: [PATCH 156/475] tests/shell: Fix shellcheck issues in tests/shell/stat+shadow_stat.sh tetscase Running shellcheck on stat+shadow_stat.sh generates below warning In tests/shell/stat+csv_summary.sh line 26: while read _num _event _run _pct ^--^ SC2034: _num appears unused. Verify use (or export if used externally). ^----^ SC2034: _event appears unused. Verify use (or export if used externally). ^--^ SC2034: _run appears unused. Verify use (or export if used externally). ^--^ SC2034: _pct appears unused. Verify use (or export if used externally). This variable is intentionally unused since it is needed to parse through the output. commit used "_" as a prefix for this throw away variable. But this stil shows warning with shellcheck v0.6. Fix this by only using "_" instead of prefix and variable name. Signed-off-by: Athira Rajeev Tested-by: Ian Rogers Reviewed-by: Kajol Jain Cc: maddy@linux.ibm.com Cc: disgoel@linux.vnet.ibm.com Cc: linuxppc-dev@lists.ozlabs.org Link: https://lore.kernel.org/r/20230907171540.36736-3-atrajeev@linux.vnet.ibm.com Signed-off-by: Namhyung Kim --- tools/perf/tests/shell/stat+csv_summary.sh | 4 ++-- tools/perf/tests/shell/stat+shadow_stat.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/perf/tests/shell/stat+csv_summary.sh b/tools/perf/tests/shell/stat+csv_summary.sh index 8bae9c8a835e..323123ff4d19 100755 --- a/tools/perf/tests/shell/stat+csv_summary.sh +++ b/tools/perf/tests/shell/stat+csv_summary.sh @@ -10,7 +10,7 @@ set -e # perf stat -e cycles -x' ' -I1000 --interval-count 1 --summary 2>&1 | \ grep -e summary | \ -while read summary _num _event _run _pct +while read summary _ _ _ _ do if [ $summary != "summary" ]; then exit 1 @@ -23,7 +23,7 @@ done # perf stat -e cycles -x' ' -I1000 --interval-count 1 --summary --no-csv-summary 2>&1 | \ grep -e summary | \ -while read _num _event _run _pct +while read _ _ _ _ do exit 1 done diff --git a/tools/perf/tests/shell/stat+shadow_stat.sh b/tools/perf/tests/shell/stat+shadow_stat.sh index 7d9b9d597a50..0c7d79a230ea 100755 --- a/tools/perf/tests/shell/stat+shadow_stat.sh +++ b/tools/perf/tests/shell/stat+shadow_stat.sh @@ -16,7 +16,7 @@ test_global_aggr() { perf stat -a --no-big-num -e cycles,instructions sleep 1 2>&1 | \ grep -e cycles -e instructions | \ - while read num evt _hash ipc rest + while read num evt _ ipc rest do # skip not counted events if [ "$num" = "&1 | \ grep ^CPU | \ - while read cpu num evt _hash ipc rest + while read cpu num evt _ ipc rest do # skip not counted events if [ "$num" = " Date: Thu, 7 Sep 2023 22:45:40 +0530 Subject: [PATCH 157/475] tests/shell: Fix shellcheck warnings for SC2153 in multiple scripts Running shellcheck on some of the shell scripts, throws below warning on shellcheck v0.6. Example: In tests/shell/coresight/asm_pure_loop.sh line 14: DATA="$DATD/perf-$TEST-$DATV.data" ^---^ SC2153: Possible misspelling: DATD may not be assigned, but DATA is. Here, DATD is exported from "lib/coresight.sh" and this warning can be ignored. Use "shellcheck disable=" to ignore this check. Signed-off-by: Athira Rajeev Tested-by: Ian Rogers Reviewed-by: Kajol Jain Cc: maddy@linux.ibm.com Cc: disgoel@linux.vnet.ibm.com Cc: linuxppc-dev@lists.ozlabs.org Link: https://lore.kernel.org/r/20230907171540.36736-4-atrajeev@linux.vnet.ibm.com Signed-off-by: Namhyung Kim --- tools/perf/tests/shell/coresight/asm_pure_loop.sh | 1 + tools/perf/tests/shell/coresight/memcpy_thread_16k_10.sh | 1 + tools/perf/tests/shell/coresight/thread_loop_check_tid_10.sh | 1 + tools/perf/tests/shell/coresight/thread_loop_check_tid_2.sh | 1 + tools/perf/tests/shell/coresight/unroll_loop_thread_10.sh | 1 + 5 files changed, 5 insertions(+) diff --git a/tools/perf/tests/shell/coresight/asm_pure_loop.sh b/tools/perf/tests/shell/coresight/asm_pure_loop.sh index 04387061e9f3..2d65defb7e0f 100755 --- a/tools/perf/tests/shell/coresight/asm_pure_loop.sh +++ b/tools/perf/tests/shell/coresight/asm_pure_loop.sh @@ -11,6 +11,7 @@ TEST="asm_pure_loop" ARGS="" DATV="out" +# shellcheck disable=SC2153 DATA="$DATD/perf-$TEST-$DATV.data" perf record $PERFRECOPT -o "$DATA" "$BIN" $ARGS diff --git a/tools/perf/tests/shell/coresight/memcpy_thread_16k_10.sh b/tools/perf/tests/shell/coresight/memcpy_thread_16k_10.sh index c17e442ac741..ddcc9bb850f5 100755 --- a/tools/perf/tests/shell/coresight/memcpy_thread_16k_10.sh +++ b/tools/perf/tests/shell/coresight/memcpy_thread_16k_10.sh @@ -11,6 +11,7 @@ TEST="memcpy_thread" ARGS="16 10 1" DATV="16k_10" +# shellcheck disable=SC2153 DATA="$DATD/perf-$TEST-$DATV.data" perf record $PERFRECOPT -o "$DATA" "$BIN" $ARGS diff --git a/tools/perf/tests/shell/coresight/thread_loop_check_tid_10.sh b/tools/perf/tests/shell/coresight/thread_loop_check_tid_10.sh index e47c4e955d0e..2ce5e139b2fd 100755 --- a/tools/perf/tests/shell/coresight/thread_loop_check_tid_10.sh +++ b/tools/perf/tests/shell/coresight/thread_loop_check_tid_10.sh @@ -11,6 +11,7 @@ TEST="thread_loop" ARGS="10 1" DATV="check-tid-10th" +# shellcheck disable=SC2153 DATA="$DATD/perf-$TEST-$DATV.data" STDO="$DATD/perf-$TEST-$DATV.stdout" diff --git a/tools/perf/tests/shell/coresight/thread_loop_check_tid_2.sh b/tools/perf/tests/shell/coresight/thread_loop_check_tid_2.sh index 8bf94a02e384..3ad9498753d7 100755 --- a/tools/perf/tests/shell/coresight/thread_loop_check_tid_2.sh +++ b/tools/perf/tests/shell/coresight/thread_loop_check_tid_2.sh @@ -11,6 +11,7 @@ TEST="thread_loop" ARGS="2 20" DATV="check-tid-2th" +# shellcheck disable=SC2153 DATA="$DATD/perf-$TEST-$DATV.data" STDO="$DATD/perf-$TEST-$DATV.stdout" diff --git a/tools/perf/tests/shell/coresight/unroll_loop_thread_10.sh b/tools/perf/tests/shell/coresight/unroll_loop_thread_10.sh index 0dc9ef424233..4fbb4a29aad3 100755 --- a/tools/perf/tests/shell/coresight/unroll_loop_thread_10.sh +++ b/tools/perf/tests/shell/coresight/unroll_loop_thread_10.sh @@ -11,6 +11,7 @@ TEST="unroll_loop_thread" ARGS="10" DATV="10" +# shellcheck disable=SC2153 DATA="$DATD/perf-$TEST-$DATV.data" perf record $PERFRECOPT -o "$DATA" "$BIN" $ARGS From 8870261a70942ba7feee85c0f9a24c68c307af68 Mon Sep 17 00:00:00 2001 From: Yang Jihong Date: Sat, 23 Sep 2023 09:30:34 +0000 Subject: [PATCH 158/475] perf bench messaging: Fix coding style issues for sched-messaging Fixed several code style issues in sched-messaging: 1. Use one space around "-" and "+" operators. 2. When a long line is broken, the operator is at the end of the line. Signed-off-by: Yang Jihong Reviewed-by: Ian Rogers Link: https://lore.kernel.org/r/20230923093037.961232-2-yangjihong1@huawei.com Signed-off-by: Namhyung Kim --- tools/perf/bench/sched-messaging.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/perf/bench/sched-messaging.c b/tools/perf/bench/sched-messaging.c index fa1f8f998814..6a33118c8f9b 100644 --- a/tools/perf/bench/sched-messaging.c +++ b/tools/perf/bench/sched-messaging.c @@ -98,7 +98,7 @@ static void *sender(struct sender_context *ctx) again: ret = write(ctx->out_fds[j], data + done, - sizeof(data)-done); + sizeof(data) - done); if (ret < 0) err(EXIT_FAILURE, "SENDER: write"); done += ret; @@ -201,8 +201,8 @@ static unsigned int group(pthread_t *pth, int wakefd) { unsigned int i; - struct sender_context *snd_ctx = malloc(sizeof(struct sender_context) - + num_fds * sizeof(int)); + struct sender_context *snd_ctx = malloc(sizeof(struct sender_context) + + num_fds * sizeof(int)); if (!snd_ctx) err(EXIT_FAILURE, "malloc()"); @@ -239,7 +239,7 @@ static unsigned int group(pthread_t *pth, snd_ctx->wakefd = wakefd; snd_ctx->num_fds = num_fds; - pth[num_fds+i] = create_worker(snd_ctx, (void *)sender); + pth[num_fds + i] = create_worker(snd_ctx, (void *)sender); } /* Close the fds we have left */ @@ -288,7 +288,7 @@ int bench_sched_messaging(int argc, const char **argv) total_children = 0; for (i = 0; i < num_groups; i++) - total_children += group(pth_tab+total_children, num_fds, + total_children += group(pth_tab + total_children, num_fds, readyfds[1], wakefds[0]); /* Wait for everyone to be ready */ From 5d2050453d489595ffa9491e05964e85c1927f35 Mon Sep 17 00:00:00 2001 From: Yang Jihong Date: Sat, 23 Sep 2023 09:30:35 +0000 Subject: [PATCH 159/475] perf bench messaging: Factor out create_worker() Refactor the create_worker() helper: 1. Modify the return value and use pthread pointer as a parameter to facilitate value assignment in create_worker(). 2. The thread worker creation and process worker creation are abstracted into independent helpers. No functional change. Test result: # perf bench sched messaging # Running 'sched/messaging' benchmark: # 20 sender and receiver processes per group # 10 groups == 400 processes run Total time: 6.332 [sec] # perf bench sched messaging -t # Running 'sched/messaging' benchmark: # 20 sender and receiver threads per group # 10 groups == 400 threads run Total time: 5.545 [sec] Signed-off-by: Yang Jihong Reviewed-by: Ian Rogers Link: https://lore.kernel.org/r/20230923093037.961232-3-yangjihong1@huawei.com Signed-off-by: Namhyung Kim --- tools/perf/bench/sched-messaging.c | 50 ++++++++++++++++-------------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/tools/perf/bench/sched-messaging.c b/tools/perf/bench/sched-messaging.c index 6a33118c8f9b..ad8596bed77a 100644 --- a/tools/perf/bench/sched-messaging.c +++ b/tools/perf/bench/sched-messaging.c @@ -139,30 +139,12 @@ again: return NULL; } -static pthread_t create_worker(void *ctx, void *(*func)(void *)) +static void create_thread_worker(pthread_t *thread, + void *ctx, void *(*func)(void *)) { pthread_attr_t attr; - pthread_t childid; int ret; - if (!thread_mode) { - /* process mode */ - /* Fork the receiver. */ - switch (fork()) { - case -1: - err(EXIT_FAILURE, "fork()"); - break; - case 0: - (*func) (ctx); - exit(0); - break; - default: - break; - } - - return (pthread_t)0; - } - if (pthread_attr_init(&attr) != 0) err(EXIT_FAILURE, "pthread_attr_init:"); @@ -171,12 +153,32 @@ static pthread_t create_worker(void *ctx, void *(*func)(void *)) err(EXIT_FAILURE, "pthread_attr_setstacksize"); #endif - ret = pthread_create(&childid, &attr, func, ctx); + ret = pthread_create(thread, &attr, func, ctx); if (ret != 0) err(EXIT_FAILURE, "pthread_create failed"); pthread_attr_destroy(&attr); - return childid; +} + +static void create_process_worker(void *ctx, void *(*func)(void *)) +{ + /* Fork the receiver. */ + pid_t pid = fork(); + + if (pid == -1) { + err(EXIT_FAILURE, "fork()"); + } else if (pid == 0) { + (*func) (ctx); + exit(0); + } +} + +static void create_worker(pthread_t *thread, void *ctx, void *(*func)(void *)) +{ + if (!thread_mode) + return create_process_worker(ctx, func); + else + return create_thread_worker(thread, ctx, func); } static void reap_worker(pthread_t id) @@ -226,7 +228,7 @@ static unsigned int group(pthread_t *pth, ctx->ready_out = ready_out; ctx->wakefd = wakefd; - pth[i] = create_worker(ctx, (void *)receiver); + create_worker(pth + i, ctx, (void *)receiver); snd_ctx->out_fds[i] = fds[1]; if (!thread_mode) @@ -239,7 +241,7 @@ static unsigned int group(pthread_t *pth, snd_ctx->wakefd = wakefd; snd_ctx->num_fds = num_fds; - pth[num_fds + i] = create_worker(snd_ctx, (void *)sender); + create_worker(pth + num_fds + i, snd_ctx, (void *)sender); } /* Close the fds we have left */ From 07f3e6cf8581bcfe9f1bd4540768bc202983349f Mon Sep 17 00:00:00 2001 From: Yang Jihong Date: Sat, 23 Sep 2023 09:30:36 +0000 Subject: [PATCH 160/475] perf bench messaging: Store chlid process pid when creating worker for process mode To save pid of child processes when creating worker: 1. The messaging worker is changed to `union` type to store thread id and process pid. 2. Save child process pid in create_process_worker(). 3. Rename `pth_tab` as `work_tab`. Test result: # perf bench sched messaging # Running 'sched/messaging' benchmark: # 20 sender and receiver processes per group # 10 groups == 400 processes run Total time: 6.744 [sec] # perf bench sched messaging -t # Running 'sched/messaging' benchmark: # 20 sender and receiver threads per group # 10 groups == 400 threads run Total time: 5.788 [sec] Signed-off-by: Yang Jihong Reviewed-by: Ian Rogers Link: https://lore.kernel.org/r/20230923093037.961232-4-yangjihong1@huawei.com Signed-off-by: Namhyung Kim --- tools/perf/bench/sched-messaging.c | 47 +++++++++++++++++------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/tools/perf/bench/sched-messaging.c b/tools/perf/bench/sched-messaging.c index ad8596bed77a..04ffaabdd45b 100644 --- a/tools/perf/bench/sched-messaging.c +++ b/tools/perf/bench/sched-messaging.c @@ -55,6 +55,11 @@ struct receiver_context { int wakefd; }; +union messaging_worker { + pthread_t thread; + pid_t pid; +}; + static void fdpair(int fds[2]) { if (use_pipes) { @@ -139,7 +144,7 @@ again: return NULL; } -static void create_thread_worker(pthread_t *thread, +static void create_thread_worker(union messaging_worker *worker, void *ctx, void *(*func)(void *)) { pthread_attr_t attr; @@ -153,35 +158,37 @@ static void create_thread_worker(pthread_t *thread, err(EXIT_FAILURE, "pthread_attr_setstacksize"); #endif - ret = pthread_create(thread, &attr, func, ctx); + ret = pthread_create(&worker->thread, &attr, func, ctx); if (ret != 0) err(EXIT_FAILURE, "pthread_create failed"); pthread_attr_destroy(&attr); } -static void create_process_worker(void *ctx, void *(*func)(void *)) +static void create_process_worker(union messaging_worker *worker, + void *ctx, void *(*func)(void *)) { /* Fork the receiver. */ - pid_t pid = fork(); + worker->pid = fork(); - if (pid == -1) { + if (worker->pid == -1) { err(EXIT_FAILURE, "fork()"); - } else if (pid == 0) { + } else if (worker->pid == 0) { (*func) (ctx); exit(0); } } -static void create_worker(pthread_t *thread, void *ctx, void *(*func)(void *)) +static void create_worker(union messaging_worker *worker, + void *ctx, void *(*func)(void *)) { if (!thread_mode) - return create_process_worker(ctx, func); + return create_process_worker(worker, ctx, func); else - return create_thread_worker(thread, ctx, func); + return create_thread_worker(worker, ctx, func); } -static void reap_worker(pthread_t id) +static void reap_worker(union messaging_worker *worker) { int proc_status; void *thread_status; @@ -192,12 +199,12 @@ static void reap_worker(pthread_t id) if (!WIFEXITED(proc_status)) exit(1); } else { - pthread_join(id, &thread_status); + pthread_join(worker->thread, &thread_status); } } /* One group of senders and receivers */ -static unsigned int group(pthread_t *pth, +static unsigned int group(union messaging_worker *worker, unsigned int num_fds, int ready_out, int wakefd) @@ -228,7 +235,7 @@ static unsigned int group(pthread_t *pth, ctx->ready_out = ready_out; ctx->wakefd = wakefd; - create_worker(pth + i, ctx, (void *)receiver); + create_worker(worker + i, ctx, (void *)receiver); snd_ctx->out_fds[i] = fds[1]; if (!thread_mode) @@ -241,7 +248,7 @@ static unsigned int group(pthread_t *pth, snd_ctx->wakefd = wakefd; snd_ctx->num_fds = num_fds; - create_worker(pth + num_fds + i, snd_ctx, (void *)sender); + create_worker(worker + num_fds + i, snd_ctx, (void *)sender); } /* Close the fds we have left */ @@ -275,14 +282,14 @@ int bench_sched_messaging(int argc, const char **argv) unsigned int num_fds = 20; int readyfds[2], wakefds[2]; char dummy; - pthread_t *pth_tab; + union messaging_worker *worker_tab; struct sender_context *pos, *n; argc = parse_options(argc, argv, options, bench_sched_message_usage, 0); - pth_tab = malloc(num_fds * 2 * num_groups * sizeof(pthread_t)); - if (!pth_tab) + worker_tab = malloc(num_fds * 2 * num_groups * sizeof(union messaging_worker)); + if (!worker_tab) err(EXIT_FAILURE, "main:malloc()"); fdpair(readyfds); @@ -290,7 +297,7 @@ int bench_sched_messaging(int argc, const char **argv) total_children = 0; for (i = 0; i < num_groups; i++) - total_children += group(pth_tab + total_children, num_fds, + total_children += group(worker_tab + total_children, num_fds, readyfds[1], wakefds[0]); /* Wait for everyone to be ready */ @@ -306,7 +313,7 @@ int bench_sched_messaging(int argc, const char **argv) /* Reap them all */ for (i = 0; i < total_children; i++) - reap_worker(pth_tab[i]); + reap_worker(worker_tab + i); gettimeofday(&stop, NULL); @@ -334,7 +341,7 @@ int bench_sched_messaging(int argc, const char **argv) break; } - free(pth_tab); + free(worker_tab); list_for_each_entry_safe(pos, n, &sender_contexts, list) { list_del_init(&pos->list); free(pos); From bb2e04d4499c611382aeb75063cf7f185cf20197 Mon Sep 17 00:00:00 2001 From: Yang Jihong Date: Sat, 23 Sep 2023 09:30:37 +0000 Subject: [PATCH 161/475] perf bench messaging: Kill child processes when exit abnormally in process mode When exit abnormally in process mode, customize SIGINT and SIGTERM signal handler to kill the forked child processes. Before: # perf bench sched messaging -l 1000000 -g 1 & [1] 8519 # # Running 'sched/messaging' benchmark: # pgrep sched-messaging | wc -l 41 # kill -15 8519 [1]+ Terminated perf bench sched messaging -l 1000000 -g 1 # pgrep sched-messaging | wc -l 40 After: # perf bench sched messaging -l 1000000 -g 1 & [1] 8472 # # Running 'sched/messaging' benchmark: # pgrep sched-messaging | wc -l 41 # kill -15 8472 [1]+ Exit 1 perf bench sched messaging -l 1000000 -g 1 # pgrep sched-messaging | wc -l 0 Signed-off-by: Yang Jihong Reviewed-by: Ian Rogers Link: https://lore.kernel.org/r/20230923093037.961232-5-yangjihong1@huawei.com [ namhyung: fix a whitespace ] Signed-off-by: Namhyung Kim --- tools/perf/bench/sched-messaging.c | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/tools/perf/bench/sched-messaging.c b/tools/perf/bench/sched-messaging.c index 04ffaabdd45b..93dcd9dba3d0 100644 --- a/tools/perf/bench/sched-messaging.c +++ b/tools/perf/bench/sched-messaging.c @@ -36,6 +36,7 @@ static bool use_pipes = false; static unsigned int nr_loops = 100; static bool thread_mode = false; static unsigned int num_groups = 10; +static unsigned int total_children = 0; static struct list_head sender_contexts = LIST_HEAD_INIT(sender_contexts); static struct list_head receiver_contexts = LIST_HEAD_INIT(receiver_contexts); @@ -60,6 +61,8 @@ union messaging_worker { pid_t pid; }; +static union messaging_worker *worker_tab; + static void fdpair(int fds[2]) { if (use_pipes) { @@ -260,6 +263,17 @@ static unsigned int group(union messaging_worker *worker, return num_fds * 2; } +static void sig_handler(int sig __maybe_unused) +{ + unsigned int i; + + /* + * When exit abnormally, kill all forked child processes. + */ + for (i = 0; i < total_children; i++) + kill(worker_tab[i].pid, SIGKILL); +} + static const struct option options[] = { OPT_BOOLEAN('p', "pipe", &use_pipes, "Use pipe() instead of socketpair()"), @@ -277,12 +291,11 @@ static const char * const bench_sched_message_usage[] = { int bench_sched_messaging(int argc, const char **argv) { - unsigned int i, total_children; + unsigned int i; struct timeval start, stop, diff; unsigned int num_fds = 20; int readyfds[2], wakefds[2]; char dummy; - union messaging_worker *worker_tab; struct sender_context *pos, *n; argc = parse_options(argc, argv, options, @@ -295,7 +308,11 @@ int bench_sched_messaging(int argc, const char **argv) fdpair(readyfds); fdpair(wakefds); - total_children = 0; + if (!thread_mode) { + signal(SIGINT, sig_handler); + signal(SIGTERM, sig_handler); + } + for (i = 0; i < num_groups; i++) total_children += group(worker_tab + total_children, num_fds, readyfds[1], wakefds[0]); From 0e501a65d35bf72414379fed0e31a0b6b81ab57d Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Fri, 22 Sep 2023 16:44:44 -0700 Subject: [PATCH 162/475] perf record: Fix BTF type checks in the off-cpu profiling The BTF func proto for a tracepoint has one more argument than the actual tracepoint function since it has a context argument at the begining. So it should compare to 5 when the tracepoint has 4 arguments. typedef void (*btf_trace_sched_switch)(void *, bool, struct task_struct *, struct task_struct *, unsigned int); Also, recent change in the perf tool would use a hand-written minimal vmlinux.h to generate BTF in the skeleton. So it won't have the info of the tracepoint. Anyway it should use the kernel's vmlinux BTF to check the type in the kernel. Fixes: b36888f71c85 ("perf record: Handle argument change in sched_switch") Reviewed-by: Ian Rogers Acked-by: Song Liu Cc: Hao Luo CC: bpf@vger.kernel.org Link: https://lore.kernel.org/r/20230922234444.3115821-1-namhyung@kernel.org Signed-off-by: Namhyung Kim --- tools/perf/util/bpf_off_cpu.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/perf/util/bpf_off_cpu.c b/tools/perf/util/bpf_off_cpu.c index 01f70b8e705a..21f4d9ba023d 100644 --- a/tools/perf/util/bpf_off_cpu.c +++ b/tools/perf/util/bpf_off_cpu.c @@ -98,7 +98,7 @@ static void off_cpu_finish(void *arg __maybe_unused) /* v5.18 kernel added prev_state arg, so it needs to check the signature */ static void check_sched_switch_args(void) { - const struct btf *btf = bpf_object__btf(skel->obj); + const struct btf *btf = btf__load_vmlinux_btf(); const struct btf_type *t1, *t2, *t3; u32 type_id; @@ -116,7 +116,8 @@ static void check_sched_switch_args(void) return; t3 = btf__type_by_id(btf, t2->type); - if (t3 && btf_is_func_proto(t3) && btf_vlen(t3) == 4) { + /* btf_trace func proto has one more argument for the context */ + if (t3 && btf_is_func_proto(t3) && btf_vlen(t3) == 5) { /* new format: pass prev_state as 4th arg */ skel->rodata->has_prev_state = true; } From 2879ff36f5ed80deec5f9d82a7a4107f2347630e Mon Sep 17 00:00:00 2001 From: Jing Zhang Date: Wed, 27 Sep 2023 13:59:45 +0800 Subject: [PATCH 163/475] perf pmu: "Compat" supports regular expression matching identifiers The jevent "Compat" is used for uncore PMU alias or metric definitions. The same PMU driver has different PMU identifiers due to different hardware versions and types, but they may have some common PMU event. Since a Compat value can only match one identifier, when adding the same event alias to PMUs with different identifiers, each identifier needs to be defined once, which is not streamlined enough. So let "Compat" support using regular expression to match identifiers for uncore PMU alias. For example, if the "Compat" value is set to "43401|43c01", it would be able to match PMU identifiers such as "43401" or "43c01", which correspond to CMN600_r0p0 or CMN700_r0p0. Signed-off-by: Jing Zhang Reviewed-by: Ian Rogers Tested-by: Ian Rogers Cc: James Clark Cc: Will Deacon Cc: Leo Yan Cc: Mike Leach Cc: Shuai Xue Cc: Zhuo Song Cc: John Garry Cc: linux-arm-kernel@lists.infradead.org Cc: linux-doc@vger.kernel.org Link: https://lore.kernel.org/r/1695794391-34817-2-git-send-email-renyu.zj@linux.alibaba.com Signed-off-by: Namhyung Kim --- tools/perf/util/pmu.c | 27 +++++++++++++++++++++++++-- tools/perf/util/pmu.h | 1 + 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/tools/perf/util/pmu.c b/tools/perf/util/pmu.c index 0d81c059c91c..5b417509b8a9 100644 --- a/tools/perf/util/pmu.c +++ b/tools/perf/util/pmu.c @@ -28,6 +28,7 @@ #include "strbuf.h" #include "fncache.h" #include "util/evsel_config.h" +#include struct perf_pmu perf_pmu__fake = { .name = "fake", @@ -870,6 +871,28 @@ out: return res; } +bool pmu_uncore_identifier_match(const char *compat, const char *id) +{ + regex_t re; + regmatch_t pmatch[1]; + int match; + + if (regcomp(&re, compat, REG_EXTENDED) != 0) { + /* Warn unable to generate match particular string. */ + pr_info("Invalid regular expression %s\n", compat); + return false; + } + + match = !regexec(&re, id, 1, pmatch, 0); + if (match) { + /* Ensure a full match. */ + match = pmatch[0].rm_so == 0 && (size_t)pmatch[0].rm_eo == strlen(id); + } + regfree(&re); + + return match; +} + static int pmu_add_cpu_aliases_map_callback(const struct pmu_event *pe, const struct pmu_events_table *table __maybe_unused, void *vdata) @@ -910,8 +933,8 @@ static int pmu_add_sys_aliases_iter_fn(const struct pmu_event *pe, if (!pe->compat || !pe->pmu) return 0; - if (!strcmp(pmu->id, pe->compat) && - pmu_uncore_alias_match(pe->pmu, pmu->name)) { + if (pmu_uncore_alias_match(pe->pmu, pmu->name) && + pmu_uncore_identifier_match(pe->compat, pmu->id)) { perf_pmu__new_alias(pmu, pe->name, pe->desc, diff --git a/tools/perf/util/pmu.h b/tools/perf/util/pmu.h index 04b317b17d66..a682c6bc0fce 100644 --- a/tools/perf/util/pmu.h +++ b/tools/perf/util/pmu.h @@ -239,6 +239,7 @@ void pmu_add_cpu_aliases_table(struct perf_pmu *pmu, char *perf_pmu__getcpuid(struct perf_pmu *pmu); const struct pmu_metrics_table *pmu_metrics_table__find(void); +bool pmu_uncore_identifier_match(const char *compat, const char *id); int perf_pmu__convert_scale(const char *scale, char **end, double *sval); From 54409997d4b99ab63616bd431cf6244d58f8a597 Mon Sep 17 00:00:00 2001 From: Jing Zhang Date: Wed, 27 Sep 2023 13:59:46 +0800 Subject: [PATCH 164/475] perf metric: "Compat" supports regular expression matching identifiers The jevent "Compat" is used for uncore PMU alias or metric definitions. The same PMU driver has different PMU identifiers due to different hardware versions and types, but they may have some common PMU metric. Since a Compat value can only match one identifier, when adding the same metric to PMUs with different identifiers, each identifier needs to be defined once, which is not streamlined enough. So let "Compat" support using regular expression to match multiple identifiers for uncore PMU metric. Signed-off-by: Jing Zhang Reviewed-by: John Garry Reviewed-by: Ian Rogers Tested-by: Ian Rogers Cc: James Clark Cc: Will Deacon Cc: Leo Yan Cc: Mike Leach Cc: Shuai Xue Cc: Zhuo Song Cc: linux-arm-kernel@lists.infradead.org Cc: linux-doc@vger.kernel.org Link: https://lore.kernel.org/r/1695794391-34817-3-git-send-email-renyu.zj@linux.alibaba.com Signed-off-by: Namhyung Kim --- tools/perf/util/metricgroup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/perf/util/metricgroup.c b/tools/perf/util/metricgroup.c index 6231044a491e..0484736d9fe4 100644 --- a/tools/perf/util/metricgroup.c +++ b/tools/perf/util/metricgroup.c @@ -498,7 +498,7 @@ static int metricgroup__sys_event_iter(const struct pmu_metric *pm, while ((pmu = perf_pmus__scan(pmu))) { - if (!pmu->id || strcmp(pmu->id, pm->compat)) + if (!pmu->id || !pmu_uncore_identifier_match(pm->compat, pmu->id)) continue; return d->fn(pm, table, d->data); From e3e42e23c0c6e791a00eb8331dc948f316e6de1f Mon Sep 17 00:00:00 2001 From: Jing Zhang Date: Wed, 27 Sep 2023 13:59:47 +0800 Subject: [PATCH 165/475] perf jevents: Support EventidCode and NodeType The previous code assumes an event has either an "event=" or "config" field at the beginning. For CMN neither of these may be present, as an event is typically "type=xx,eventid=xxx". So add EventidCode and NodeType to support CMN event description. I compared pmu_event.c before and after compiling with JEVENT_ARCH=all, they are consistent. Signed-off-by: Jing Zhang Reviewed-by: Ian Rogers Tested-by: Ian Rogers Cc: James Clark Cc: Will Deacon Cc: Leo Yan Cc: Mike Leach Cc: Shuai Xue Cc: Zhuo Song Cc: John Garry Cc: linux-arm-kernel@lists.infradead.org Cc: linux-doc@vger.kernel.org Link: https://lore.kernel.org/r/1695794391-34817-4-git-send-email-renyu.zj@linux.alibaba.com Signed-off-by: Namhyung Kim --- tools/perf/pmu-events/jevents.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tools/perf/pmu-events/jevents.py b/tools/perf/pmu-events/jevents.py index af15fa2cadbd..f863aec2432c 100755 --- a/tools/perf/pmu-events/jevents.py +++ b/tools/perf/pmu-events/jevents.py @@ -298,6 +298,7 @@ class JsonEvent: if 'ExtSel' in jd: eventcode |= int(jd['ExtSel']) << 8 configcode = int(jd['ConfigCode'], 0) if 'ConfigCode' in jd else None + eventidcode = int(jd['EventidCode'], 0) if 'EventidCode' in jd else None self.name = jd['EventName'].lower() if 'EventName' in jd else None self.topic = '' self.compat = jd.get('Compat') @@ -335,7 +336,13 @@ class JsonEvent: if precise and self.desc and '(Precise Event)' not in self.desc: extra_desc += ' (Must be precise)' if precise == '2' else (' (Precise ' 'event)') - event = f'config={llx(configcode)}' if configcode is not None else f'event={llx(eventcode)}' + event = None + if configcode is not None: + event = f'config={llx(configcode)}' + elif eventidcode is not None: + event = f'eventid={llx(eventidcode)}' + else: + event = f'event={llx(eventcode)}' event_fields = [ ('AnyThread', 'any='), ('PortMask', 'ch_mask='), @@ -345,6 +352,7 @@ class JsonEvent: ('Invert', 'inv='), ('SampleAfterValue', 'period='), ('UMask', 'umask='), + ('NodeType', 'type='), ] for key, value in event_fields: if key in jd and jd[key] != '0': From 3bb59e759cbb357f8fb46cc5a48d2b0da09b37c4 Mon Sep 17 00:00:00 2001 From: Jing Zhang Date: Wed, 27 Sep 2023 13:59:48 +0800 Subject: [PATCH 166/475] perf test: Make matching_pmu effective The perf_pmu_test_event.matching_pmu didn't work. No matter what its value is, it does not affect the test results. So let matching_pmu be used for matching perf_pmu_test_pmu.pmu.name. Signed-off-by: Jing Zhang Reviewed-by: John Garry Reviewed-by: Ian Rogers Tested-by: Ian Rogers Cc: James Clark Cc: Will Deacon Cc: Leo Yan Cc: Mike Leach Cc: Shuai Xue Cc: Zhuo Song Cc: linux-arm-kernel@lists.infradead.org Cc: linux-doc@vger.kernel.org Link: https://lore.kernel.org/r/1695794391-34817-5-git-send-email-renyu.zj@linux.alibaba.com Signed-off-by: Namhyung Kim --- tools/perf/tests/pmu-events.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tools/perf/tests/pmu-events.c b/tools/perf/tests/pmu-events.c index f5321fbdee79..0cf572f7c1e7 100644 --- a/tools/perf/tests/pmu-events.c +++ b/tools/perf/tests/pmu-events.c @@ -245,7 +245,7 @@ static const struct perf_pmu_test_event sys_ddr_pmu_write_cycles = { }, .alias_str = "event=0x2b", .alias_long_desc = "ddr write-cycles event", - .matching_pmu = "uncore_sys_ddr_pmu", + .matching_pmu = "uncore_sys_ddr_pmu0", }; static const struct perf_pmu_test_event sys_ccn_pmu_read_cycles = { @@ -259,7 +259,7 @@ static const struct perf_pmu_test_event sys_ccn_pmu_read_cycles = { }, .alias_str = "config=0x2c", .alias_long_desc = "ccn read-cycles event", - .matching_pmu = "uncore_sys_ccn_pmu", + .matching_pmu = "uncore_sys_ccn_pmu4", }; static const struct perf_pmu_test_event *sys_events[] = { @@ -615,6 +615,12 @@ static int __test_uncore_pmu_event_aliases(struct perf_pmu_test_pmu *test_pmu) .count = &matched_count, }; + if (strcmp(pmu_name, test_event.matching_pmu)) { + pr_debug("testing aliases uncore PMU %s: mismatched matching_pmu, %s vs %s\n", + pmu_name, test_event.matching_pmu, pmu_name); + return -1; + } + err = perf_pmu__find_event(pmu, event->name, &args, test_core_pmu_event_aliases_cb); if (err) { From 7fded33c6971b6c8e87cbbf48e74536aacca2991 Mon Sep 17 00:00:00 2001 From: Jing Zhang Date: Wed, 27 Sep 2023 13:59:49 +0800 Subject: [PATCH 167/475] perf test: Add pmu-event test for "Compat" and new event_field. Add new event test for uncore system event which is used to verify the functionality of "Compat" matching multiple identifiers and the new event fields "EventidCode" and "NodeType". Signed-off-by: Jing Zhang Reviewed-by: Ian Rogers Tested-by: Ian Rogers Cc: James Clark Cc: Will Deacon Cc: Leo Yan Cc: Mike Leach Cc: Shuai Xue Cc: Zhuo Song Cc: John Garry Cc: linux-arm-kernel@lists.infradead.org Cc: linux-doc@vger.kernel.org Link: https://lore.kernel.org/r/1695794391-34817-6-git-send-email-renyu.zj@linux.alibaba.com Signed-off-by: Namhyung Kim --- .../arch/test/test_soc/sys/uncore.json | 8 +++ tools/perf/pmu-events/empty-pmu-events.c | 8 +++ tools/perf/tests/pmu-events.c | 55 +++++++++++++++++++ 3 files changed, 71 insertions(+) diff --git a/tools/perf/pmu-events/arch/test/test_soc/sys/uncore.json b/tools/perf/pmu-events/arch/test/test_soc/sys/uncore.json index c7e7528db315..4d423b149ad1 100644 --- a/tools/perf/pmu-events/arch/test/test_soc/sys/uncore.json +++ b/tools/perf/pmu-events/arch/test/test_soc/sys/uncore.json @@ -12,5 +12,13 @@ "EventName": "sys_ccn_pmu.read_cycles", "Unit": "sys_ccn_pmu", "Compat": "0x01" + }, + { + "BriefDescription": "Counts total cache misses in first lookup result (high priority)", + "EventidCode": "0x1", + "NodeType": "0x5", + "EventName": "sys_cmn_pmu.hnf_cache_miss", + "Unit": "sys_cmn_pmu", + "Compat": "(434|436|43c|43a).*" } ] diff --git a/tools/perf/pmu-events/empty-pmu-events.c b/tools/perf/pmu-events/empty-pmu-events.c index 12bd043a05e3..13727421d424 100644 --- a/tools/perf/pmu-events/empty-pmu-events.c +++ b/tools/perf/pmu-events/empty-pmu-events.c @@ -244,6 +244,14 @@ static const struct pmu_event pmu_events__test_soc_sys[] = { .topic = "uncore", .pmu = "uncore_sys_ccn_pmu", }, + { + .name = "sys_cmn_pmu.hnf_cache_miss", + .event = "eventid=0x1,type=0x5", + .desc = "Counts total cache misses in first lookup result (high priority). Unit: uncore_sys_cmn_pmu ", + .compat = "(434|436|43c|43a).*", + .topic = "uncore", + .pmu = "uncore_sys_cmn_pmu", + }, { .name = 0, .event = 0, diff --git a/tools/perf/tests/pmu-events.c b/tools/perf/tests/pmu-events.c index 0cf572f7c1e7..a56d32905743 100644 --- a/tools/perf/tests/pmu-events.c +++ b/tools/perf/tests/pmu-events.c @@ -262,9 +262,24 @@ static const struct perf_pmu_test_event sys_ccn_pmu_read_cycles = { .matching_pmu = "uncore_sys_ccn_pmu4", }; +static const struct perf_pmu_test_event sys_cmn_pmu_hnf_cache_miss = { + .event = { + .name = "sys_cmn_pmu.hnf_cache_miss", + .event = "eventid=0x1,type=0x5", + .desc = "Counts total cache misses in first lookup result (high priority)", + .topic = "uncore", + .pmu = "uncore_sys_cmn_pmu", + .compat = "(434|436|43c|43a).*", + }, + .alias_str = "eventid=0x1,type=0x5", + .alias_long_desc = "Counts total cache misses in first lookup result (high priority)", + .matching_pmu = "uncore_sys_cmn_pmu0", +}; + static const struct perf_pmu_test_event *sys_events[] = { &sys_ddr_pmu_write_cycles, &sys_ccn_pmu_read_cycles, + &sys_cmn_pmu_hnf_cache_miss, NULL }; @@ -707,6 +722,46 @@ static struct perf_pmu_test_pmu test_pmus[] = { &sys_ccn_pmu_read_cycles, }, }, + { + .pmu = { + .name = (char *)"uncore_sys_cmn_pmu0", + .is_uncore = 1, + .id = (char *)"43401", + }, + .aliases = { + &sys_cmn_pmu_hnf_cache_miss, + }, + }, + { + .pmu = { + .name = (char *)"uncore_sys_cmn_pmu0", + .is_uncore = 1, + .id = (char *)"43602", + }, + .aliases = { + &sys_cmn_pmu_hnf_cache_miss, + }, + }, + { + .pmu = { + .name = (char *)"uncore_sys_cmn_pmu0", + .is_uncore = 1, + .id = (char *)"43c03", + }, + .aliases = { + &sys_cmn_pmu_hnf_cache_miss, + }, + }, + { + .pmu = { + .name = (char *)"uncore_sys_cmn_pmu0", + .is_uncore = 1, + .id = (char *)"43a01", + }, + .aliases = { + &sys_cmn_pmu_hnf_cache_miss, + }, + } }; /* Test that aliases generated are as expected */ From 0b4de7bdf46c521518e38579d0ab5600a6949bec Mon Sep 17 00:00:00 2001 From: Jing Zhang Date: Wed, 27 Sep 2023 13:59:50 +0800 Subject: [PATCH 168/475] perf jevents: Add support for Arm CMN PMU aliasing Currently just add aliases for part of Arm CMN PMU events which are general and compatible for any SoC and CMN-ANY. "Compat" value "(434|436|43c|43a).*" means it is compatible with all CMN600/CMN650/CMN700/Ci700, which can be obtained from commit 7819e05a0dce ("perf/arm-cmn: Revamp model detection"). The arm-cmn PMU events got from: [0] https://developer.arm.com/documentation/100180/0302/?lang=en [1] https://developer.arm.com/documentation/101408/0100/?lang=en [2] https://developer.arm.com/documentation/102308/0302/?lang=en [3] https://developer.arm.com/documentation/101569/0300/?lang=en Signed-off-by: Jing Zhang Reviewed-by: John Garry Reviewed-by: Ian Rogers Tested-by: Ian Rogers Cc: James Clark Cc: Will Deacon Cc: Leo Yan Cc: Mike Leach Cc: Shuai Xue Cc: Zhuo Song Cc: linux-arm-kernel@lists.infradead.org Cc: linux-doc@vger.kernel.org Link: https://lore.kernel.org/r/1695794391-34817-7-git-send-email-renyu.zj@linux.alibaba.com Signed-off-by: Namhyung Kim --- .../arch/arm64/arm/cmn/sys/cmn.json | 266 ++++++++++++++++++ tools/perf/pmu-events/jevents.py | 1 + 2 files changed, 267 insertions(+) create mode 100644 tools/perf/pmu-events/arch/arm64/arm/cmn/sys/cmn.json diff --git a/tools/perf/pmu-events/arch/arm64/arm/cmn/sys/cmn.json b/tools/perf/pmu-events/arch/arm64/arm/cmn/sys/cmn.json new file mode 100644 index 000000000000..428605c37d10 --- /dev/null +++ b/tools/perf/pmu-events/arch/arm64/arm/cmn/sys/cmn.json @@ -0,0 +1,266 @@ +[ + { + "EventName": "hnf_cache_miss", + "EventidCode": "0x1", + "NodeType": "0x5", + "BriefDescription": "Counts total cache misses in first lookup result (high priority).", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + }, + { + "EventName": "hnf_slc_sf_cache_access", + "EventidCode": "0x2", + "NodeType": "0x5", + "BriefDescription": "Counts number of cache accesses in first access (high priority).", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + }, + { + "EventName": "hnf_cache_fill", + "EventidCode": "0x3", + "NodeType": "0x5", + "BriefDescription": "Counts total allocations in HN SLC (all cache line allocations to SLC).", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + }, + { + "EventName": "hnf_pocq_retry", + "EventidCode": "0x4", + "NodeType": "0x5", + "BriefDescription": "Counts number of retried requests.", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + }, + { + "EventName": "hnf_pocq_reqs_recvd", + "EventidCode": "0x5", + "NodeType": "0x5", + "BriefDescription": "Counts number of requests that HN receives.", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + }, + { + "EventName": "hnf_sf_hit", + "EventidCode": "0x6", + "NodeType": "0x5", + "BriefDescription": "Counts number of SF hits.", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + }, + { + "EventName": "hnf_sf_evictions", + "EventidCode": "0x7", + "NodeType": "0x5", + "BriefDescription": "Counts number of SF eviction cache invalidations initiated.", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + }, + { + "EventName": "hnf_dir_snoops_sent", + "EventidCode": "0x8", + "NodeType": "0x5", + "BriefDescription": "Counts number of directed snoops sent (not including SF back invalidation).", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + }, + { + "EventName": "hnf_brd_snoops_sent", + "EventidCode": "0x9", + "NodeType": "0x5", + "BriefDescription": "Counts number of multicast snoops sent (not including SF back invalidation).", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + }, + { + "EventName": "hnf_slc_eviction", + "EventidCode": "0xa", + "NodeType": "0x5", + "BriefDescription": "Counts number of SLC evictions (dirty only).", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + }, + { + "EventName": "hnf_slc_fill_invalid_way", + "EventidCode": "0xb", + "NodeType": "0x5", + "BriefDescription": "Counts number of SLC fills to an invalid way.", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + }, + { + "EventName": "hnf_mc_retries", + "EventidCode": "0xc", + "NodeType": "0x5", + "BriefDescription": "Counts number of retried transactions by the MC.", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + }, + { + "EventName": "hnf_mc_reqs", + "EventidCode": "0xd", + "NodeType": "0x5", + "BriefDescription": "Counts number of requests that are sent to MC.", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + }, + { + "EventName": "hnf_qos_hh_retry", + "EventidCode": "0xe", + "NodeType": "0x5", + "BriefDescription": "Counts number of times a HighHigh priority request is protocolretried at the HN‑F.", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + }, + { + "EventName": "rnid_s0_rdata_beats", + "EventidCode": "0x1", + "NodeType": "0xa", + "BriefDescription": "Number of RData beats (RVALID and RREADY) dispatched on port 0. This event measures the read bandwidth, including CMO responses.", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + }, + { + "EventName": "rnid_s1_rdata_beats", + "EventidCode": "0x2", + "NodeType": "0xa", + "BriefDescription": "Number of RData beats (RVALID and RREADY) dispatched on port 1. This event measures the read bandwidth, including CMO responses.", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + }, + { + "EventName": "rnid_s2_rdata_beats", + "EventidCode": "0x3", + "NodeType": "0xa", + "BriefDescription": "Number of RData beats (RVALID and RREADY) dispatched on port 2. This event measures the read bandwidth, including CMO responses.", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + }, + { + "EventName": "rnid_rxdat_flits", + "EventidCode": "0x4", + "NodeType": "0xa", + "BriefDescription": "Number of RXDAT flits received. This event measures the true read data bandwidth, excluding CMOs.", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + }, + { + "EventName": "rnid_txdat_flits", + "EventidCode": "0x5", + "NodeType": "0xa", + "BriefDescription": "Number of TXDAT flits dispatched. This event measures the write bandwidth.", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + }, + { + "EventName": "rnid_txreq_flits_total", + "EventidCode": "0x6", + "NodeType": "0xa", + "BriefDescription": "Number of TXREQ flits dispatched. This event measures the total request bandwidth.", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + }, + { + "EventName": "rnid_txreq_flits_retried", + "EventidCode": "0x7", + "NodeType": "0xa", + "BriefDescription": "Number of retried TXREQ flits dispatched. This event measures the retry rate.", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + }, + { + "EventName": "sbsx_txrsp_retryack", + "EventidCode": "0x4", + "NodeType": "0x7", + "BriefDescription": "Number of RXREQ flits dispatched. This event is a measure of the retry rate.", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + }, + { + "EventName": "sbsx_txdat_flitv", + "EventidCode": "0x5", + "NodeType": "0x7", + "BriefDescription": "Number of TXDAT flits dispatched from XP to SBSX. This event is a measure of the write bandwidth.", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + }, + { + "EventName": "sbsx_arvalid_no_arready", + "EventidCode": "0x21", + "NodeType": "0x7", + "BriefDescription": "Number of cycles the SBSX bridge is stalled because of backpressure on AR channel.", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + }, + { + "EventName": "sbsx_awvalid_no_awready", + "EventidCode": "0x22", + "NodeType": "0x7", + "BriefDescription": "Number of cycles the SBSX bridge is stalled because of backpressure on AW channel.", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + }, + { + "EventName": "sbsx_wvalid_no_wready", + "EventidCode": "0x23", + "NodeType": "0x7", + "BriefDescription": "Number of cycles the SBSX bridge is stalled because of backpressure on W channel.", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + }, + { + "EventName": "hni_txrsp_retryack", + "EventidCode": "0x2a", + "NodeType": "0x4", + "BriefDescription": "Number of RXREQ flits dispatched. This event is a measure of the retry rate.", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + }, + { + "EventName": "hni_arvalid_no_arready", + "EventidCode": "0x2b", + "NodeType": "0x4", + "BriefDescription": "Number of cycles the HN-I bridge is stalled because of backpressure on AR channel.", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + }, + { + "EventName": "hni_arready_no_arvalid", + "EventidCode": "0x2c", + "NodeType": "0x4", + "BriefDescription": "Number of cycles the AR channel is waiting for new requests from HN-I bridge.", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + }, + { + "EventName": "hni_awvalid_no_awready", + "EventidCode": "0x2d", + "NodeType": "0x4", + "BriefDescription": "Number of cycles the HN-I bridge is stalled because of backpressure on AW channel.", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + }, + { + "EventName": "hni_awready_no_awvalid", + "EventidCode": "0x2e", + "NodeType": "0x4", + "BriefDescription": "Number of cycles the AW channel is waiting for new requests from HN-I bridge.", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + }, + { + "EventName": "hni_wvalid_no_wready", + "EventidCode": "0x2f", + "NodeType": "0x4", + "BriefDescription": "Number of cycles the HN-I bridge is stalled because of backpressure on W channel.", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + }, + { + "EventName": "hni_txdat_stall", + "EventidCode": "0x30", + "NodeType": "0x4", + "BriefDescription": "TXDAT valid but no link credit available.", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + } +] diff --git a/tools/perf/pmu-events/jevents.py b/tools/perf/pmu-events/jevents.py index f863aec2432c..fd009752b427 100755 --- a/tools/perf/pmu-events/jevents.py +++ b/tools/perf/pmu-events/jevents.py @@ -289,6 +289,7 @@ class JsonEvent: 'cpu_core': 'cpu_core', 'cpu_atom': 'cpu_atom', 'ali_drw': 'ali_drw', + 'arm_cmn': 'arm_cmn', } return table[unit] if unit in table else f'uncore_{unit.lower()}' From 4f3ee7d1d5ced888e603c7fbe48e3468320745c1 Mon Sep 17 00:00:00 2001 From: Jing Zhang Date: Wed, 27 Sep 2023 13:59:51 +0800 Subject: [PATCH 169/475] perf vendor events: Add JSON metrics for Arm CMN Add JSON metrics for Arm CMN. Currently just add part of CMN PMU metrics which are general and compatible for any SoC with CMN-ANY. Signed-off-by: Jing Zhang Reviewed-by: John Garry Reviewed-by: Ian Rogers Tested-by: Ian Rogers Cc: James Clark Cc: Will Deacon Cc: Leo Yan Cc: Mike Leach Cc: Shuai Xue Cc: Zhuo Song Cc: linux-arm-kernel@lists.infradead.org Cc: linux-doc@vger.kernel.org Link: https://lore.kernel.org/r/1695794391-34817-8-git-send-email-renyu.zj@linux.alibaba.com Signed-off-by: Namhyung Kim --- .../arch/arm64/arm/cmn/sys/metric.json | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 tools/perf/pmu-events/arch/arm64/arm/cmn/sys/metric.json diff --git a/tools/perf/pmu-events/arch/arm64/arm/cmn/sys/metric.json b/tools/perf/pmu-events/arch/arm64/arm/cmn/sys/metric.json new file mode 100644 index 000000000000..f7823bd265db --- /dev/null +++ b/tools/perf/pmu-events/arch/arm64/arm/cmn/sys/metric.json @@ -0,0 +1,74 @@ +[ + { + "MetricName": "slc_miss_rate", + "BriefDescription": "The system level cache miss rate.", + "MetricGroup": "cmn", + "MetricExpr": "hnf_cache_miss / hnf_slc_sf_cache_access", + "ScaleUnit": "100%", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + }, + { + "MetricName": "hnf_message_retry_rate", + "BriefDescription": "HN-F message retry rate indicates whether a lack of credits is causing the bottlenecks.", + "MetricGroup": "cmn", + "MetricExpr": "hnf_pocq_retry / hnf_pocq_reqs_recvd", + "ScaleUnit": "100%", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + }, + { + "MetricName": "sf_hit_rate", + "BriefDescription": "Snoop filter hit rate can be used to measure the snoop filter efficiency.", + "MetricGroup": "cmn", + "MetricExpr": "hnf_sf_hit / hnf_slc_sf_cache_access", + "ScaleUnit": "100%", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + }, + { + "MetricName": "mc_message_retry_rate", + "BriefDescription": "The memory controller request retries rate indicates whether the memory controller is the bottleneck.", + "MetricGroup": "cmn", + "MetricExpr": "hnf_mc_retries / hnf_mc_reqs", + "ScaleUnit": "100%", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + }, + { + "MetricName": "rni_actual_read_bandwidth.all", + "BriefDescription": "This event measure the actual bandwidth that RN-I bridge sends to the interconnect.", + "MetricGroup": "cmn", + "MetricExpr": "rnid_rxdat_flits * 32 / 1e6 / duration_time", + "ScaleUnit": "1MB/s", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + }, + { + "MetricName": "rni_actual_write_bandwidth.all", + "BriefDescription": "This event measures the actual write bandwidth at RN-I bridges.", + "MetricGroup": "cmn", + "MetricExpr": "rnid_txdat_flits * 32 / 1e6 / duration_time", + "ScaleUnit": "1MB/s", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + }, + { + "MetricName": "rni_retry_rate", + "BriefDescription": "RN-I bridge retry rate indicates whether the memory controller is the bottleneck.", + "MetricGroup": "cmn", + "MetricExpr": "rnid_txreq_flits_retried / rnid_txreq_flits_total", + "ScaleUnit": "100%", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + }, + { + "MetricName": "sbsx_actual_write_bandwidth.all", + "BriefDescription": "sbsx actual write bandwidth.", + "MetricGroup": "cmn", + "MetricExpr": "sbsx_txdat_flitv * 32 / 1e6 / duration_time", + "ScaleUnit": "1MB/s", + "Unit": "arm_cmn", + "Compat": "(434|436|43c|43a).*" + } +] From ee33a0ef8468063b34eed4330b0023c1a8d62f8f Mon Sep 17 00:00:00 2001 From: Athira Rajeev Date: Wed, 27 Sep 2023 23:47:03 +0530 Subject: [PATCH 170/475] perf test: Fix parse-events tests to skip parametrized events Testcase "Parsing of all PMU events from sysfs" parse events for all PMUs, and not just cpu. In case of powerpc, the PowerVM environment supports events from hv_24x7 and hv_gpci PMU which is of example format like below: - hv_24x7/CPM_ADJUNCT_INST,domain=?,core=?/ - hv_gpci/event,partition_id=?/ The value for "?" needs to be filled in depending on system configuration. It is better to skip these parametrized events in this test as it is done in: 'commit b50d691e50e6 ("perf test: Fix "all PMU test" to skip parametrized events")' which handled a simialr instance with "all PMU test". Fix parse-events test to skip parametrized events since it needs proper setup of the parameters. Signed-off-by: Athira Rajeev Tested-by: Ian Rogers Tested-by: Sachin Sant Reviewed-by: Kajol Jain Cc: maddy@linux.ibm.com Cc: disgoel@linux.vnet.ibm.com Cc: linuxppc-dev@lists.ozlabs.org Link: https://lore.kernel.org/r/20230927181703.80936-1-atrajeev@linux.vnet.ibm.com Signed-off-by: Namhyung Kim --- tools/perf/tests/parse-events.c | 39 +++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tools/perf/tests/parse-events.c b/tools/perf/tests/parse-events.c index 93796e885cef..f78be21a5999 100644 --- a/tools/perf/tests/parse-events.c +++ b/tools/perf/tests/parse-events.c @@ -2514,9 +2514,14 @@ static int test__pmu_events(struct test_suite *test __maybe_unused, int subtest while ((pmu = perf_pmus__scan(pmu)) != NULL) { struct stat st; char path[PATH_MAX]; + char pmu_event[PATH_MAX]; + char *buf = NULL; + FILE *file; struct dirent *ent; + size_t len = 0; DIR *dir; int err; + int n; snprintf(path, PATH_MAX, "%s/bus/event_source/devices/%s/events/", sysfs__mountpoint(), pmu->name); @@ -2538,11 +2543,45 @@ static int test__pmu_events(struct test_suite *test __maybe_unused, int subtest struct evlist_test e = { .name = NULL, }; char name[2 * NAME_MAX + 1 + 12 + 3]; int test_ret; + bool is_event_parameterized = 0; /* Names containing . are special and cannot be used directly */ if (strchr(ent->d_name, '.')) continue; + /* exclude parametrized ones (name contains '?') */ + n = snprintf(pmu_event, sizeof(pmu_event), "%s%s", path, ent->d_name); + if (n >= PATH_MAX) { + pr_err("pmu event name crossed PATH_MAX(%d) size\n", PATH_MAX); + continue; + } + + file = fopen(pmu_event, "r"); + if (!file) { + pr_debug("can't open pmu event file for '%s'\n", ent->d_name); + ret = combine_test_results(ret, TEST_FAIL); + continue; + } + + if (getline(&buf, &len, file) < 0) { + pr_debug(" pmu event: %s is a null event\n", ent->d_name); + ret = combine_test_results(ret, TEST_FAIL); + fclose(file); + continue; + } + + if (strchr(buf, '?')) + is_event_parameterized = 1; + + free(buf); + buf = NULL; + fclose(file); + + if (is_event_parameterized == 1) { + pr_debug("skipping parametrized PMU event: %s which contains ?\n", pmu_event); + continue; + } + snprintf(name, sizeof(name), "%s/event=%s/u", pmu->name, ent->d_name); e.name = name; From b1f05622fef39dded385f9e360e859846c1ddaf1 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Sun, 24 Sep 2023 23:23:23 -0700 Subject: [PATCH 171/475] perf pmus: Make PMU alias name loading lazy PMU alias names were computed when the first perf_pmu is created, scanning all PMUs in event sources for a file called alias that generally doesn't exist. Switch to trying to load the file when all PMU related files are loaded in lookup. This would cause a PMU name lookup of an alias name to fail if no PMUs were loaded, so in that case all PMUs are loaded and the find repeated. The overhead is similar but in the (very) general case not all PMUs are scanned for the alias file. As the overhead occurs once per invocation it doesn't show in perf bench internals pmu-scan. On a tigerlake machine, the number of openat system calls for an event of cpu/cycles/ with perf stat reduces from 94 to 69 (ie 25 fewer openat calls). Signed-off-by: Ian Rogers Acked-by: Namhyung Kim Cc: Ravi Bangoria Cc: James Clark Cc: Leo Yan Cc: Kan Liang Link: https://lore.kernel.org/r/20230925062323.840799-1-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/arch/x86/util/pmu.c | 139 --------------------------------- tools/perf/util/pmu.c | 39 ++++----- tools/perf/util/pmu.h | 2 - tools/perf/util/pmus.c | 10 +++ 4 files changed, 31 insertions(+), 159 deletions(-) diff --git a/tools/perf/arch/x86/util/pmu.c b/tools/perf/arch/x86/util/pmu.c index f428cffb0378..8b53ca468a50 100644 --- a/tools/perf/arch/x86/util/pmu.c +++ b/tools/perf/arch/x86/util/pmu.c @@ -17,15 +17,6 @@ #include "../../../util/pmus.h" #include "env.h" -struct pmu_alias { - char *name; - char *alias; - struct list_head list; -}; - -static LIST_HEAD(pmu_alias_name_list); -static bool cached_list; - struct perf_event_attr *perf_pmu__get_default_config(struct perf_pmu *pmu __maybe_unused) { #ifdef HAVE_AUXTRACE_SUPPORT @@ -41,136 +32,6 @@ struct perf_event_attr *perf_pmu__get_default_config(struct perf_pmu *pmu __mayb return NULL; } -static void pmu_alias__delete(struct pmu_alias *pmu_alias) -{ - if (!pmu_alias) - return; - - zfree(&pmu_alias->name); - zfree(&pmu_alias->alias); - free(pmu_alias); -} - -static struct pmu_alias *pmu_alias__new(char *name, char *alias) -{ - struct pmu_alias *pmu_alias = zalloc(sizeof(*pmu_alias)); - - if (pmu_alias) { - pmu_alias->name = strdup(name); - if (!pmu_alias->name) - goto out_delete; - - pmu_alias->alias = strdup(alias); - if (!pmu_alias->alias) - goto out_delete; - } - return pmu_alias; - -out_delete: - pmu_alias__delete(pmu_alias); - return NULL; -} - -static int setup_pmu_alias_list(void) -{ - int fd, dirfd; - DIR *dir; - struct dirent *dent; - struct pmu_alias *pmu_alias; - char buf[MAX_PMU_NAME_LEN]; - FILE *file; - int ret = -ENOMEM; - - dirfd = perf_pmu__event_source_devices_fd(); - if (dirfd < 0) - return -1; - - dir = fdopendir(dirfd); - if (!dir) - return -errno; - - while ((dent = readdir(dir))) { - if (!strcmp(dent->d_name, ".") || - !strcmp(dent->d_name, "..")) - continue; - - fd = perf_pmu__pathname_fd(dirfd, dent->d_name, "alias", O_RDONLY); - if (fd < 0) - continue; - - file = fdopen(fd, "r"); - if (!file) - continue; - - if (!fgets(buf, sizeof(buf), file)) { - fclose(file); - continue; - } - - fclose(file); - - /* Remove the last '\n' */ - buf[strlen(buf) - 1] = 0; - - pmu_alias = pmu_alias__new(dent->d_name, buf); - if (!pmu_alias) - goto close_dir; - - list_add_tail(&pmu_alias->list, &pmu_alias_name_list); - } - - ret = 0; - -close_dir: - closedir(dir); - return ret; -} - -static const char *__pmu_find_real_name(const char *name) -{ - struct pmu_alias *pmu_alias; - - list_for_each_entry(pmu_alias, &pmu_alias_name_list, list) { - if (!strcmp(name, pmu_alias->alias)) - return pmu_alias->name; - } - - return name; -} - -const char *pmu_find_real_name(const char *name) -{ - if (cached_list) - return __pmu_find_real_name(name); - - setup_pmu_alias_list(); - cached_list = true; - - return __pmu_find_real_name(name); -} - -static const char *__pmu_find_alias_name(const char *name) -{ - struct pmu_alias *pmu_alias; - - list_for_each_entry(pmu_alias, &pmu_alias_name_list, list) { - if (!strcmp(name, pmu_alias->name)) - return pmu_alias->alias; - } - return NULL; -} - -const char *pmu_find_alias_name(const char *name) -{ - if (cached_list) - return __pmu_find_alias_name(name); - - setup_pmu_alias_list(); - cached_list = true; - - return __pmu_find_alias_name(name); -} - int perf_pmus__num_mem_pmus(void) { /* AMD uses IBS OP pmu and not a core PMU for perf mem/c2c */ diff --git a/tools/perf/util/pmu.c b/tools/perf/util/pmu.c index 5b417509b8a9..6b1b7f8f00fa 100644 --- a/tools/perf/util/pmu.c +++ b/tools/perf/util/pmu.c @@ -960,16 +960,27 @@ perf_pmu__get_default_config(struct perf_pmu *pmu __maybe_unused) return NULL; } -const char * __weak -pmu_find_real_name(const char *name) +static char *pmu_find_alias_name(struct perf_pmu *pmu, int dirfd) { - return name; -} + FILE *file = perf_pmu__open_file_at(pmu, dirfd, "alias"); + char *line = NULL; + size_t line_len = 0; + ssize_t ret; -const char * __weak -pmu_find_alias_name(const char *name __maybe_unused) -{ - return NULL; + if (!file) + return NULL; + + ret = getline(&line, &line_len, file); + if (ret < 0) { + fclose(file); + return NULL; + } + /* Remove trailing newline. */ + if (ret > 0 && line[ret - 1] == '\n') + line[--ret] = '\0'; + + fclose(file); + return line; } static int pmu_max_precise(int dirfd, struct perf_pmu *pmu) @@ -980,12 +991,10 @@ static int pmu_max_precise(int dirfd, struct perf_pmu *pmu) return max_precise; } -struct perf_pmu *perf_pmu__lookup(struct list_head *pmus, int dirfd, const char *lookup_name) +struct perf_pmu *perf_pmu__lookup(struct list_head *pmus, int dirfd, const char *name) { struct perf_pmu *pmu; __u32 type; - const char *name = pmu_find_real_name(lookup_name); - const char *alias_name; pmu = zalloc(sizeof(*pmu)); if (!pmu) @@ -1018,18 +1027,12 @@ struct perf_pmu *perf_pmu__lookup(struct list_head *pmus, int dirfd, const char pmu->is_core = is_pmu_core(name); pmu->cpus = pmu_cpumask(dirfd, name, pmu->is_core); - alias_name = pmu_find_alias_name(name); - if (alias_name) { - pmu->alias_name = strdup(alias_name); - if (!pmu->alias_name) - goto err; - } - pmu->type = type; pmu->is_uncore = pmu_is_uncore(dirfd, name); if (pmu->is_uncore) pmu->id = pmu_id(name); pmu->max_precise = pmu_max_precise(dirfd, pmu); + pmu->alias_name = pmu_find_alias_name(pmu, dirfd); pmu->events_table = perf_pmu__find_events_table(pmu); pmu_add_sys_aliases(pmu); list_add_tail(&pmu->list, pmus); diff --git a/tools/perf/util/pmu.h b/tools/perf/util/pmu.h index a682c6bc0fce..85190d058852 100644 --- a/tools/perf/util/pmu.h +++ b/tools/perf/util/pmu.h @@ -252,8 +252,6 @@ void perf_pmu__warn_invalid_formats(struct perf_pmu *pmu); int perf_pmu__match(const char *pattern, const char *name, const char *tok); -const char *pmu_find_real_name(const char *name); -const char *pmu_find_alias_name(const char *name); double perf_pmu__cpu_slots_per_cycle(void); int perf_pmu__event_source_devices_scnprintf(char *pathname, size_t size); int perf_pmu__pathname_scnprintf(char *buf, size_t size, diff --git a/tools/perf/util/pmus.c b/tools/perf/util/pmus.c index 64e798e68a2d..ce4931461741 100644 --- a/tools/perf/util/pmus.c +++ b/tools/perf/util/pmus.c @@ -37,6 +37,8 @@ static LIST_HEAD(other_pmus); static bool read_sysfs_core_pmus; static bool read_sysfs_all_pmus; +static void pmu_read_sysfs(bool core_only); + int pmu_name_len_no_suffix(const char *str, unsigned long *num) { int orig_len, len; @@ -124,6 +126,14 @@ struct perf_pmu *perf_pmus__find(const char *name) pmu = perf_pmu__lookup(core_pmu ? &core_pmus : &other_pmus, dirfd, name); close(dirfd); + if (!pmu) { + /* + * Looking up an inidividual PMU failed. This may mean name is + * an alias, so read the PMUs from sysfs and try to find again. + */ + pmu_read_sysfs(core_pmu); + pmu = pmu_find(name); + } return pmu; } From f2d87895cbc4af80649850dcf5da36de6b2ed3dd Mon Sep 17 00:00:00 2001 From: Adrian Hunter Date: Thu, 28 Sep 2023 10:29:53 +0300 Subject: [PATCH 172/475] perf intel-pt: Fix async branch flags Ensure PERF_IP_FLAG_ASYNC is set always for asynchronous branches (i.e. interrupts etc). Fixes: 90e457f7be08 ("perf tools: Add Intel PT support") Cc: stable@vger.kernel.org Signed-off-by: Adrian Hunter Acked-by: Namhyung Kim Link: https://lore.kernel.org/r/20230928072953.19369-1-adrian.hunter@intel.com Signed-off-by: Namhyung Kim --- tools/perf/util/intel-pt.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/perf/util/intel-pt.c b/tools/perf/util/intel-pt.c index dbf0bc71a63b..f38893e0b036 100644 --- a/tools/perf/util/intel-pt.c +++ b/tools/perf/util/intel-pt.c @@ -1512,9 +1512,11 @@ static void intel_pt_sample_flags(struct intel_pt_queue *ptq) } else if (ptq->state->flags & INTEL_PT_ASYNC) { if (!ptq->state->to_ip) ptq->flags = PERF_IP_FLAG_BRANCH | + PERF_IP_FLAG_ASYNC | PERF_IP_FLAG_TRACE_END; else if (ptq->state->from_nr && !ptq->state->to_nr) ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | + PERF_IP_FLAG_ASYNC | PERF_IP_FLAG_VMEXIT; else ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | From 0ebc7feae79ac07772a20382eebd8c3503313714 Mon Sep 17 00:00:00 2001 From: "Dr. David Alan Gilbert" Date: Fri, 25 Aug 2023 15:27:54 +0100 Subject: [PATCH 173/475] powerpc: Use shared font data PowerPC has a 'btext' font used for the console which is almost identical to the shared font_sun8x16, so use it rather than duplicating the data. They were actually identical until about a decade ago when commit bcfbeecea11c ("drivers: console: font_: Change a glyph from "broken bar" to "vertical line"") which changed the | in the shared font to be a solid bar rather than a broken bar. That's the only difference. This was originally spotted by the PMF source code analyser, which noticed that sparc does the same thing with the same data, and they also share a bunch of functions to manipulate the data. I've previously posted a near identical patch for sparc. Tested very lightly with a boot without FS in qemu. Signed-off-by: "Dr. David Alan Gilbert" Signed-off-by: Michael Ellerman Link: https://msgid.link/20230825142754.1487900-1-linux@treblig.org --- arch/powerpc/Kconfig.debug | 2 + arch/powerpc/kernel/btext.c | 360 +----------------------------------- lib/fonts/Kconfig | 2 +- 3 files changed, 9 insertions(+), 355 deletions(-) diff --git a/arch/powerpc/Kconfig.debug b/arch/powerpc/Kconfig.debug index 2a54fadbeaf5..50ff3359cde9 100644 --- a/arch/powerpc/Kconfig.debug +++ b/arch/powerpc/Kconfig.debug @@ -147,6 +147,8 @@ config BDI_SWITCH config BOOTX_TEXT bool "Support for early boot text console (BootX or OpenFirmware only)" depends on PPC_BOOK3S + select FONT_SUN8x16 + select FONT_SUPPORT help Say Y here to see progress messages from the boot firmware in text mode. Requires either BootX or Open Firmware. diff --git a/arch/powerpc/kernel/btext.c b/arch/powerpc/kernel/btext.c index 19e46fd623b0..7f63f1cdc6c3 100644 --- a/arch/powerpc/kernel/btext.c +++ b/arch/powerpc/kernel/btext.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -41,10 +42,6 @@ static unsigned char *logicalDisplayBase __force_data; unsigned long disp_BAT[2] __initdata = {0, 0}; -#define cmapsz (16*256) - -static unsigned char vga_font[cmapsz]; - static int boot_text_mapped __force_data; extern void rmci_on(void); @@ -407,7 +404,7 @@ static unsigned int expand_bits_16[4] = { }; -static void draw_byte_32(unsigned char *font, unsigned int *base, int rb) +static void draw_byte_32(const unsigned char *font, unsigned int *base, int rb) { int l, bits; int fg = 0xFFFFFFFFUL; @@ -428,7 +425,7 @@ static void draw_byte_32(unsigned char *font, unsigned int *base, int rb) } } -static inline void draw_byte_16(unsigned char *font, unsigned int *base, int rb) +static inline void draw_byte_16(const unsigned char *font, unsigned int *base, int rb) { int l, bits; int fg = 0xFFFFFFFFUL; @@ -446,7 +443,7 @@ static inline void draw_byte_16(unsigned char *font, unsigned int *base, int rb) } } -static inline void draw_byte_8(unsigned char *font, unsigned int *base, int rb) +static inline void draw_byte_8(const unsigned char *font, unsigned int *base, int rb) { int l, bits; int fg = 0x0F0F0F0FUL; @@ -465,7 +462,8 @@ static inline void draw_byte_8(unsigned char *font, unsigned int *base, int rb) static noinline void draw_byte(unsigned char c, long locX, long locY) { unsigned char *base = calc_base(locX << 3, locY << 4); - unsigned char *font = &vga_font[((unsigned int)c) * 16]; + unsigned int font_index = c * 16; + const unsigned char *font = font_sun_8x16.data + font_index; int rb = dispDeviceRowBytes; rmci_maybe_on(); @@ -583,349 +581,3 @@ void __init udbg_init_btext(void) */ udbg_putc = btext_drawchar; } - -static unsigned char vga_font[cmapsz] = { -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7e, 0x81, 0xa5, 0x81, 0x81, 0xbd, -0x99, 0x81, 0x81, 0x7e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7e, 0xff, -0xdb, 0xff, 0xff, 0xc3, 0xe7, 0xff, 0xff, 0x7e, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x6c, 0xfe, 0xfe, 0xfe, 0xfe, 0x7c, 0x38, 0x10, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x38, 0x7c, 0xfe, -0x7c, 0x38, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, -0x3c, 0x3c, 0xe7, 0xe7, 0xe7, 0x18, 0x18, 0x3c, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x18, 0x3c, 0x7e, 0xff, 0xff, 0x7e, 0x18, 0x18, 0x3c, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x3c, -0x3c, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, -0xff, 0xff, 0xe7, 0xc3, 0xc3, 0xe7, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, -0x00, 0x00, 0x00, 0x00, 0x00, 0x3c, 0x66, 0x42, 0x42, 0x66, 0x3c, 0x00, -0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xc3, 0x99, 0xbd, -0xbd, 0x99, 0xc3, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x1e, 0x0e, -0x1a, 0x32, 0x78, 0xcc, 0xcc, 0xcc, 0xcc, 0x78, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x3c, 0x66, 0x66, 0x66, 0x66, 0x3c, 0x18, 0x7e, 0x18, 0x18, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0x33, 0x3f, 0x30, 0x30, 0x30, -0x30, 0x70, 0xf0, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0x63, -0x7f, 0x63, 0x63, 0x63, 0x63, 0x67, 0xe7, 0xe6, 0xc0, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x18, 0x18, 0xdb, 0x3c, 0xe7, 0x3c, 0xdb, 0x18, 0x18, -0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0xc0, 0xe0, 0xf0, 0xf8, 0xfe, 0xf8, -0xf0, 0xe0, 0xc0, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x06, 0x0e, -0x1e, 0x3e, 0xfe, 0x3e, 0x1e, 0x0e, 0x06, 0x02, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x18, 0x3c, 0x7e, 0x18, 0x18, 0x18, 0x7e, 0x3c, 0x18, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, -0x66, 0x00, 0x66, 0x66, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0xdb, -0xdb, 0xdb, 0x7b, 0x1b, 0x1b, 0x1b, 0x1b, 0x1b, 0x00, 0x00, 0x00, 0x00, -0x00, 0x7c, 0xc6, 0x60, 0x38, 0x6c, 0xc6, 0xc6, 0x6c, 0x38, 0x0c, 0xc6, -0x7c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0xfe, 0xfe, 0xfe, 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x3c, -0x7e, 0x18, 0x18, 0x18, 0x7e, 0x3c, 0x18, 0x7e, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x18, 0x3c, 0x7e, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, -0x18, 0x7e, 0x3c, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x18, 0x0c, 0xfe, 0x0c, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x60, 0xfe, 0x60, 0x30, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0xc0, -0xc0, 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x24, 0x66, 0xff, 0x66, 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x10, 0x38, 0x38, 0x7c, 0x7c, 0xfe, 0xfe, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfe, 0xfe, 0x7c, 0x7c, -0x38, 0x38, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x18, 0x3c, 0x3c, 0x3c, 0x18, 0x18, 0x18, 0x00, 0x18, 0x18, -0x00, 0x00, 0x00, 0x00, 0x00, 0x66, 0x66, 0x66, 0x24, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6c, -0x6c, 0xfe, 0x6c, 0x6c, 0x6c, 0xfe, 0x6c, 0x6c, 0x00, 0x00, 0x00, 0x00, -0x18, 0x18, 0x7c, 0xc6, 0xc2, 0xc0, 0x7c, 0x06, 0x06, 0x86, 0xc6, 0x7c, -0x18, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc2, 0xc6, 0x0c, 0x18, -0x30, 0x60, 0xc6, 0x86, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x6c, -0x6c, 0x38, 0x76, 0xdc, 0xcc, 0xcc, 0xcc, 0x76, 0x00, 0x00, 0x00, 0x00, -0x00, 0x30, 0x30, 0x30, 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x18, 0x30, 0x30, 0x30, 0x30, -0x30, 0x30, 0x18, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x18, -0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x18, 0x30, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x66, 0x3c, 0xff, 0x3c, 0x66, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x18, 0x7e, -0x18, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x18, 0x18, 0x30, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7e, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x18, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x02, 0x06, 0x0c, 0x18, 0x30, 0x60, 0xc0, 0x80, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x7c, 0xc6, 0xc6, 0xce, 0xde, 0xf6, 0xe6, 0xc6, 0xc6, 0x7c, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x38, 0x78, 0x18, 0x18, 0x18, -0x18, 0x18, 0x18, 0x7e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, 0xc6, -0x06, 0x0c, 0x18, 0x30, 0x60, 0xc0, 0xc6, 0xfe, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x7c, 0xc6, 0x06, 0x06, 0x3c, 0x06, 0x06, 0x06, 0xc6, 0x7c, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x1c, 0x3c, 0x6c, 0xcc, 0xfe, -0x0c, 0x0c, 0x0c, 0x1e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfe, 0xc0, -0xc0, 0xc0, 0xfc, 0x06, 0x06, 0x06, 0xc6, 0x7c, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x38, 0x60, 0xc0, 0xc0, 0xfc, 0xc6, 0xc6, 0xc6, 0xc6, 0x7c, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfe, 0xc6, 0x06, 0x06, 0x0c, 0x18, -0x30, 0x30, 0x30, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, 0xc6, -0xc6, 0xc6, 0x7c, 0xc6, 0xc6, 0xc6, 0xc6, 0x7c, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x7c, 0xc6, 0xc6, 0xc6, 0x7e, 0x06, 0x06, 0x06, 0x0c, 0x78, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x18, 0x00, 0x00, -0x00, 0x18, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x18, 0x18, 0x00, 0x00, 0x00, 0x18, 0x18, 0x30, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x06, 0x0c, 0x18, 0x30, 0x60, 0x30, 0x18, 0x0c, 0x06, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7e, 0x00, 0x00, -0x7e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, -0x30, 0x18, 0x0c, 0x06, 0x0c, 0x18, 0x30, 0x60, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x7c, 0xc6, 0xc6, 0x0c, 0x18, 0x18, 0x18, 0x00, 0x18, 0x18, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, 0xc6, 0xc6, 0xc6, 0xde, 0xde, -0xde, 0xdc, 0xc0, 0x7c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x38, -0x6c, 0xc6, 0xc6, 0xfe, 0xc6, 0xc6, 0xc6, 0xc6, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0xfc, 0x66, 0x66, 0x66, 0x7c, 0x66, 0x66, 0x66, 0x66, 0xfc, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3c, 0x66, 0xc2, 0xc0, 0xc0, 0xc0, -0xc0, 0xc2, 0x66, 0x3c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x6c, -0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x6c, 0xf8, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0xfe, 0x66, 0x62, 0x68, 0x78, 0x68, 0x60, 0x62, 0x66, 0xfe, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfe, 0x66, 0x62, 0x68, 0x78, 0x68, -0x60, 0x60, 0x60, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3c, 0x66, -0xc2, 0xc0, 0xc0, 0xde, 0xc6, 0xc6, 0x66, 0x3a, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0xc6, 0xc6, 0xc6, 0xc6, 0xfe, 0xc6, 0xc6, 0xc6, 0xc6, 0xc6, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3c, 0x18, 0x18, 0x18, 0x18, 0x18, -0x18, 0x18, 0x18, 0x3c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1e, 0x0c, -0x0c, 0x0c, 0x0c, 0x0c, 0xcc, 0xcc, 0xcc, 0x78, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0xe6, 0x66, 0x66, 0x6c, 0x78, 0x78, 0x6c, 0x66, 0x66, 0xe6, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x60, 0x60, 0x60, 0x60, 0x60, -0x60, 0x62, 0x66, 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc3, 0xe7, -0xff, 0xff, 0xdb, 0xc3, 0xc3, 0xc3, 0xc3, 0xc3, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0xc6, 0xe6, 0xf6, 0xfe, 0xde, 0xce, 0xc6, 0xc6, 0xc6, 0xc6, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, 0xc6, 0xc6, 0xc6, 0xc6, 0xc6, -0xc6, 0xc6, 0xc6, 0x7c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfc, 0x66, -0x66, 0x66, 0x7c, 0x60, 0x60, 0x60, 0x60, 0xf0, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x7c, 0xc6, 0xc6, 0xc6, 0xc6, 0xc6, 0xc6, 0xd6, 0xde, 0x7c, -0x0c, 0x0e, 0x00, 0x00, 0x00, 0x00, 0xfc, 0x66, 0x66, 0x66, 0x7c, 0x6c, -0x66, 0x66, 0x66, 0xe6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, 0xc6, -0xc6, 0x60, 0x38, 0x0c, 0x06, 0xc6, 0xc6, 0x7c, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0xff, 0xdb, 0x99, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x3c, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc6, 0xc6, 0xc6, 0xc6, 0xc6, 0xc6, -0xc6, 0xc6, 0xc6, 0x7c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc3, 0xc3, -0xc3, 0xc3, 0xc3, 0xc3, 0xc3, 0x66, 0x3c, 0x18, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0xc3, 0xc3, 0xc3, 0xc3, 0xc3, 0xdb, 0xdb, 0xff, 0x66, 0x66, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc3, 0xc3, 0x66, 0x3c, 0x18, 0x18, -0x3c, 0x66, 0xc3, 0xc3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc3, 0xc3, -0xc3, 0x66, 0x3c, 0x18, 0x18, 0x18, 0x18, 0x3c, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0xff, 0xc3, 0x86, 0x0c, 0x18, 0x30, 0x60, 0xc1, 0xc3, 0xff, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3c, 0x30, 0x30, 0x30, 0x30, 0x30, -0x30, 0x30, 0x30, 0x3c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, -0xc0, 0xe0, 0x70, 0x38, 0x1c, 0x0e, 0x06, 0x02, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x3c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x3c, -0x00, 0x00, 0x00, 0x00, 0x10, 0x38, 0x6c, 0xc6, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0x00, 0x00, -0x30, 0x30, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x0c, 0x7c, -0xcc, 0xcc, 0xcc, 0x76, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x60, -0x60, 0x78, 0x6c, 0x66, 0x66, 0x66, 0x66, 0x7c, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, 0xc6, 0xc0, 0xc0, 0xc0, 0xc6, 0x7c, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1c, 0x0c, 0x0c, 0x3c, 0x6c, 0xcc, -0xcc, 0xcc, 0xcc, 0x76, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x7c, 0xc6, 0xfe, 0xc0, 0xc0, 0xc6, 0x7c, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x38, 0x6c, 0x64, 0x60, 0xf0, 0x60, 0x60, 0x60, 0x60, 0xf0, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x76, 0xcc, 0xcc, -0xcc, 0xcc, 0xcc, 0x7c, 0x0c, 0xcc, 0x78, 0x00, 0x00, 0x00, 0xe0, 0x60, -0x60, 0x6c, 0x76, 0x66, 0x66, 0x66, 0x66, 0xe6, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x18, 0x18, 0x00, 0x38, 0x18, 0x18, 0x18, 0x18, 0x18, 0x3c, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x06, 0x00, 0x0e, 0x06, 0x06, -0x06, 0x06, 0x06, 0x06, 0x66, 0x66, 0x3c, 0x00, 0x00, 0x00, 0xe0, 0x60, -0x60, 0x66, 0x6c, 0x78, 0x78, 0x6c, 0x66, 0xe6, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x38, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x3c, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe6, 0xff, 0xdb, -0xdb, 0xdb, 0xdb, 0xdb, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0xdc, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, 0xc6, 0xc6, 0xc6, 0xc6, 0xc6, 0x7c, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdc, 0x66, 0x66, -0x66, 0x66, 0x66, 0x7c, 0x60, 0x60, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x76, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0x7c, 0x0c, 0x0c, 0x1e, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0xdc, 0x76, 0x66, 0x60, 0x60, 0x60, 0xf0, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, 0xc6, 0x60, -0x38, 0x0c, 0xc6, 0x7c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x30, -0x30, 0xfc, 0x30, 0x30, 0x30, 0x30, 0x36, 0x1c, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0x76, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc3, 0xc3, 0xc3, -0xc3, 0x66, 0x3c, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0xc3, 0xc3, 0xc3, 0xdb, 0xdb, 0xff, 0x66, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0xc3, 0x66, 0x3c, 0x18, 0x3c, 0x66, 0xc3, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc6, 0xc6, 0xc6, -0xc6, 0xc6, 0xc6, 0x7e, 0x06, 0x0c, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0xfe, 0xcc, 0x18, 0x30, 0x60, 0xc6, 0xfe, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x0e, 0x18, 0x18, 0x18, 0x70, 0x18, 0x18, 0x18, 0x18, 0x0e, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x18, 0x18, 0x18, 0x00, 0x18, -0x18, 0x18, 0x18, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x18, -0x18, 0x18, 0x0e, 0x18, 0x18, 0x18, 0x18, 0x70, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x76, 0xdc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x38, 0x6c, 0xc6, -0xc6, 0xc6, 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3c, 0x66, -0xc2, 0xc0, 0xc0, 0xc0, 0xc2, 0x66, 0x3c, 0x0c, 0x06, 0x7c, 0x00, 0x00, -0x00, 0x00, 0xcc, 0x00, 0x00, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0x76, -0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x18, 0x30, 0x00, 0x7c, 0xc6, 0xfe, -0xc0, 0xc0, 0xc6, 0x7c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x38, 0x6c, -0x00, 0x78, 0x0c, 0x7c, 0xcc, 0xcc, 0xcc, 0x76, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0xcc, 0x00, 0x00, 0x78, 0x0c, 0x7c, 0xcc, 0xcc, 0xcc, 0x76, -0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x30, 0x18, 0x00, 0x78, 0x0c, 0x7c, -0xcc, 0xcc, 0xcc, 0x76, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x6c, 0x38, -0x00, 0x78, 0x0c, 0x7c, 0xcc, 0xcc, 0xcc, 0x76, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x3c, 0x66, 0x60, 0x60, 0x66, 0x3c, 0x0c, 0x06, -0x3c, 0x00, 0x00, 0x00, 0x00, 0x10, 0x38, 0x6c, 0x00, 0x7c, 0xc6, 0xfe, -0xc0, 0xc0, 0xc6, 0x7c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc6, 0x00, -0x00, 0x7c, 0xc6, 0xfe, 0xc0, 0xc0, 0xc6, 0x7c, 0x00, 0x00, 0x00, 0x00, -0x00, 0x60, 0x30, 0x18, 0x00, 0x7c, 0xc6, 0xfe, 0xc0, 0xc0, 0xc6, 0x7c, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x66, 0x00, 0x00, 0x38, 0x18, 0x18, -0x18, 0x18, 0x18, 0x3c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x3c, 0x66, -0x00, 0x38, 0x18, 0x18, 0x18, 0x18, 0x18, 0x3c, 0x00, 0x00, 0x00, 0x00, -0x00, 0x60, 0x30, 0x18, 0x00, 0x38, 0x18, 0x18, 0x18, 0x18, 0x18, 0x3c, -0x00, 0x00, 0x00, 0x00, 0x00, 0xc6, 0x00, 0x10, 0x38, 0x6c, 0xc6, 0xc6, -0xfe, 0xc6, 0xc6, 0xc6, 0x00, 0x00, 0x00, 0x00, 0x38, 0x6c, 0x38, 0x00, -0x38, 0x6c, 0xc6, 0xc6, 0xfe, 0xc6, 0xc6, 0xc6, 0x00, 0x00, 0x00, 0x00, -0x18, 0x30, 0x60, 0x00, 0xfe, 0x66, 0x60, 0x7c, 0x60, 0x60, 0x66, 0xfe, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6e, 0x3b, 0x1b, -0x7e, 0xd8, 0xdc, 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3e, 0x6c, -0xcc, 0xcc, 0xfe, 0xcc, 0xcc, 0xcc, 0xcc, 0xce, 0x00, 0x00, 0x00, 0x00, -0x00, 0x10, 0x38, 0x6c, 0x00, 0x7c, 0xc6, 0xc6, 0xc6, 0xc6, 0xc6, 0x7c, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc6, 0x00, 0x00, 0x7c, 0xc6, 0xc6, -0xc6, 0xc6, 0xc6, 0x7c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x30, 0x18, -0x00, 0x7c, 0xc6, 0xc6, 0xc6, 0xc6, 0xc6, 0x7c, 0x00, 0x00, 0x00, 0x00, -0x00, 0x30, 0x78, 0xcc, 0x00, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0x76, -0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x30, 0x18, 0x00, 0xcc, 0xcc, 0xcc, -0xcc, 0xcc, 0xcc, 0x76, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc6, 0x00, -0x00, 0xc6, 0xc6, 0xc6, 0xc6, 0xc6, 0xc6, 0x7e, 0x06, 0x0c, 0x78, 0x00, -0x00, 0xc6, 0x00, 0x7c, 0xc6, 0xc6, 0xc6, 0xc6, 0xc6, 0xc6, 0xc6, 0x7c, -0x00, 0x00, 0x00, 0x00, 0x00, 0xc6, 0x00, 0xc6, 0xc6, 0xc6, 0xc6, 0xc6, -0xc6, 0xc6, 0xc6, 0x7c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x18, 0x7e, -0xc3, 0xc0, 0xc0, 0xc0, 0xc3, 0x7e, 0x18, 0x18, 0x00, 0x00, 0x00, 0x00, -0x00, 0x38, 0x6c, 0x64, 0x60, 0xf0, 0x60, 0x60, 0x60, 0x60, 0xe6, 0xfc, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc3, 0x66, 0x3c, 0x18, 0xff, 0x18, -0xff, 0x18, 0x18, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfc, 0x66, 0x66, -0x7c, 0x62, 0x66, 0x6f, 0x66, 0x66, 0x66, 0xf3, 0x00, 0x00, 0x00, 0x00, -0x00, 0x0e, 0x1b, 0x18, 0x18, 0x18, 0x7e, 0x18, 0x18, 0x18, 0x18, 0x18, -0xd8, 0x70, 0x00, 0x00, 0x00, 0x18, 0x30, 0x60, 0x00, 0x78, 0x0c, 0x7c, -0xcc, 0xcc, 0xcc, 0x76, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x18, 0x30, -0x00, 0x38, 0x18, 0x18, 0x18, 0x18, 0x18, 0x3c, 0x00, 0x00, 0x00, 0x00, -0x00, 0x18, 0x30, 0x60, 0x00, 0x7c, 0xc6, 0xc6, 0xc6, 0xc6, 0xc6, 0x7c, -0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x30, 0x60, 0x00, 0xcc, 0xcc, 0xcc, -0xcc, 0xcc, 0xcc, 0x76, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x76, 0xdc, -0x00, 0xdc, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x00, 0x00, 0x00, 0x00, -0x76, 0xdc, 0x00, 0xc6, 0xe6, 0xf6, 0xfe, 0xde, 0xce, 0xc6, 0xc6, 0xc6, -0x00, 0x00, 0x00, 0x00, 0x00, 0x3c, 0x6c, 0x6c, 0x3e, 0x00, 0x7e, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x6c, 0x6c, -0x38, 0x00, 0x7c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x30, 0x30, 0x00, 0x30, 0x30, 0x60, 0xc0, 0xc6, 0xc6, 0x7c, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfe, 0xc0, -0xc0, 0xc0, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0xfe, 0x06, 0x06, 0x06, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0xc0, 0xc0, 0xc2, 0xc6, 0xcc, 0x18, 0x30, 0x60, 0xce, 0x9b, 0x06, -0x0c, 0x1f, 0x00, 0x00, 0x00, 0xc0, 0xc0, 0xc2, 0xc6, 0xcc, 0x18, 0x30, -0x66, 0xce, 0x96, 0x3e, 0x06, 0x06, 0x00, 0x00, 0x00, 0x00, 0x18, 0x18, -0x00, 0x18, 0x18, 0x18, 0x3c, 0x3c, 0x3c, 0x18, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x36, 0x6c, 0xd8, 0x6c, 0x36, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x6c, 0x36, -0x6c, 0xd8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x44, 0x11, 0x44, -0x11, 0x44, 0x11, 0x44, 0x11, 0x44, 0x11, 0x44, 0x11, 0x44, 0x11, 0x44, -0x55, 0xaa, 0x55, 0xaa, 0x55, 0xaa, 0x55, 0xaa, 0x55, 0xaa, 0x55, 0xaa, -0x55, 0xaa, 0x55, 0xaa, 0xdd, 0x77, 0xdd, 0x77, 0xdd, 0x77, 0xdd, 0x77, -0xdd, 0x77, 0xdd, 0x77, 0xdd, 0x77, 0xdd, 0x77, 0x18, 0x18, 0x18, 0x18, -0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, -0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0xf8, 0x18, 0x18, 0x18, 0x18, -0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0xf8, 0x18, 0xf8, -0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x36, 0x36, 0x36, 0x36, -0x36, 0x36, 0x36, 0xf6, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfe, 0x36, 0x36, 0x36, 0x36, -0x36, 0x36, 0x36, 0x36, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x18, 0xf8, -0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x36, 0x36, 0x36, 0x36, -0x36, 0xf6, 0x06, 0xf6, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, -0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, -0x36, 0x36, 0x36, 0x36, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfe, 0x06, 0xf6, -0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, -0x36, 0xf6, 0x06, 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0xfe, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x18, 0x18, 0x18, 0x18, 0x18, 0xf8, 0x18, 0xf8, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0xf8, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, -0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x1f, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0xff, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0xff, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, -0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x1f, 0x18, 0x18, 0x18, 0x18, -0x18, 0x18, 0x18, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x18, 0x18, 0x18, -0x18, 0x18, 0x18, 0xff, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, -0x18, 0x18, 0x18, 0x18, 0x18, 0x1f, 0x18, 0x1f, 0x18, 0x18, 0x18, 0x18, -0x18, 0x18, 0x18, 0x18, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x37, -0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, -0x36, 0x37, 0x30, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0x30, 0x37, 0x36, 0x36, 0x36, 0x36, -0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0xf7, 0x00, 0xff, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0xff, 0x00, 0xf7, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, -0x36, 0x36, 0x36, 0x36, 0x36, 0x37, 0x30, 0x37, 0x36, 0x36, 0x36, 0x36, -0x36, 0x36, 0x36, 0x36, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0x00, 0xff, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x36, 0x36, 0x36, 0x36, -0x36, 0xf7, 0x00, 0xf7, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, -0x18, 0x18, 0x18, 0x18, 0x18, 0xff, 0x00, 0xff, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0xff, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0xff, 0x00, 0xff, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0x36, 0x36, 0x36, 0x36, -0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x3f, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x18, 0x18, 0x18, -0x18, 0x1f, 0x18, 0x1f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x1f, 0x18, 0x1f, 0x18, 0x18, 0x18, 0x18, -0x18, 0x18, 0x18, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, -0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, -0x36, 0x36, 0x36, 0xff, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, -0x18, 0x18, 0x18, 0x18, 0x18, 0xff, 0x18, 0xff, 0x18, 0x18, 0x18, 0x18, -0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0xf8, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x1f, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, -0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, -0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, -0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf0, 0xf0, 0xf0, 0xf0, -0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, -0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, -0x0f, 0x0f, 0x0f, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x76, 0xdc, 0xd8, 0xd8, 0xd8, 0xdc, 0x76, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x78, 0xcc, 0xcc, 0xcc, 0xd8, 0xcc, 0xc6, 0xc6, 0xc6, 0xcc, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfe, 0xc6, 0xc6, 0xc0, 0xc0, 0xc0, -0xc0, 0xc0, 0xc0, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0xfe, 0x6c, 0x6c, 0x6c, 0x6c, 0x6c, 0x6c, 0x6c, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0xfe, 0xc6, 0x60, 0x30, 0x18, 0x30, 0x60, 0xc6, 0xfe, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7e, 0xd8, 0xd8, -0xd8, 0xd8, 0xd8, 0x70, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x66, 0x66, 0x66, 0x66, 0x66, 0x7c, 0x60, 0x60, 0xc0, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x76, 0xdc, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7e, 0x18, 0x3c, 0x66, 0x66, -0x66, 0x3c, 0x18, 0x7e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, -0x6c, 0xc6, 0xc6, 0xfe, 0xc6, 0xc6, 0x6c, 0x38, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x38, 0x6c, 0xc6, 0xc6, 0xc6, 0x6c, 0x6c, 0x6c, 0x6c, 0xee, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1e, 0x30, 0x18, 0x0c, 0x3e, 0x66, -0x66, 0x66, 0x66, 0x3c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x7e, 0xdb, 0xdb, 0xdb, 0x7e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x03, 0x06, 0x7e, 0xdb, 0xdb, 0xf3, 0x7e, 0x60, 0xc0, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1c, 0x30, 0x60, 0x60, 0x7c, 0x60, -0x60, 0x60, 0x30, 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, -0xc6, 0xc6, 0xc6, 0xc6, 0xc6, 0xc6, 0xc6, 0xc6, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0xfe, 0x00, 0x00, 0xfe, 0x00, 0x00, 0xfe, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x18, 0x7e, 0x18, -0x18, 0x00, 0x00, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, -0x18, 0x0c, 0x06, 0x0c, 0x18, 0x30, 0x00, 0x7e, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x0c, 0x18, 0x30, 0x60, 0x30, 0x18, 0x0c, 0x00, 0x7e, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x1b, 0x1b, 0x1b, 0x18, 0x18, -0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, -0x18, 0x18, 0x18, 0x18, 0xd8, 0xd8, 0xd8, 0x70, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x18, 0x18, 0x00, 0x7e, 0x00, 0x18, 0x18, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x76, 0xdc, 0x00, -0x76, 0xdc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x6c, 0x6c, -0x38, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x18, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x0c, 0x0c, -0x0c, 0x0c, 0x0c, 0xec, 0x6c, 0x6c, 0x3c, 0x1c, 0x00, 0x00, 0x00, 0x00, -0x00, 0xd8, 0x6c, 0x6c, 0x6c, 0x6c, 0x6c, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0xd8, 0x30, 0x60, 0xc8, 0xf8, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x7c, 0x7c, 0x7c, 0x7c, 0x7c, 0x7c, 0x7c, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, -}; - diff --git a/lib/fonts/Kconfig b/lib/fonts/Kconfig index c035fde66aeb..7ee468ef21ec 100644 --- a/lib/fonts/Kconfig +++ b/lib/fonts/Kconfig @@ -98,7 +98,7 @@ config FONT_10x18 config FONT_SUN8x16 bool "Sparc console 8x16 font" - depends on FRAMEBUFFER_CONSOLE && (!SPARC && FONTS || SPARC) + depends on (FRAMEBUFFER_CONSOLE && (FONTS || SPARC)) || BOOTX_TEXT help This is the high resolution console font for Sun machines. Say Y. From a1f157c7a3bb342b972c2830a0ad53f627000a04 Mon Sep 17 00:00:00 2001 From: Zheng Yejian Date: Wed, 6 Sep 2023 17:18:37 +0800 Subject: [PATCH 174/475] tracing: Expand all ring buffers individually The ring buffer of global_trace is set to the minimum size in order to save memory on boot up and then it will be expand when some trace feature enabled. However currently operations under an instance can also cause global_trace ring buffer being expanded, and the expanded memory would be wasted if global_trace then not being used. See following case, we enable 'sched_switch' event in instance 'A', then ring buffer of global_trace is unexpectedly expanded to be 1410KB, also the '(expanded: 1408)' from 'buffer_size_kb' of instance is confusing. # cd /sys/kernel/tracing # mkdir instances/A # cat buffer_size_kb 7 (expanded: 1408) # cat instances/A/buffer_size_kb 1410 (expanded: 1408) # echo sched:sched_switch > instances/A/set_event # cat buffer_size_kb 1410 # cat instances/A/buffer_size_kb 1410 To fix it, we can: - Make 'ring_buffer_expanded' as a member of 'struct trace_array'; - Make 'ring_buffer_expanded' of instance is defaultly true, global_trace is defaultly false; - In order not to expose 'global_trace' outside of file 'kernel/trace/trace.c', introduce trace_set_ring_buffer_expanded() to set 'ring_buffer_expanded' as 'true'; - Pass the expected trace_array to tracing_update_buffers(). Link: https://lore.kernel.org/linux-trace-kernel/20230906091837.3998020-1-zhengyejian1@huawei.com Signed-off-by: Zheng Yejian Signed-off-by: Steven Rostedt (Google) --- kernel/trace/trace.c | 47 ++++++++++++++++++++----------------- kernel/trace/trace.h | 9 +++++-- kernel/trace/trace_events.c | 22 +++++++++-------- 3 files changed, 45 insertions(+), 33 deletions(-) diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index abaaf516fcae..dd6395692ff9 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -54,12 +54,6 @@ #include "trace.h" #include "trace_output.h" -/* - * On boot up, the ring buffer is set to the minimum size, so that - * we do not waste memory on systems that are not using tracing. - */ -bool ring_buffer_expanded; - #ifdef CONFIG_FTRACE_STARTUP_TEST /* * We need to change this state when a selftest is running. @@ -202,7 +196,7 @@ static int __init set_cmdline_ftrace(char *str) strscpy(bootup_tracer_buf, str, MAX_TRACER_SIZE); default_bootup_tracer = bootup_tracer_buf; /* We are using ftrace early, expand it */ - ring_buffer_expanded = true; + trace_set_ring_buffer_expanded(NULL); return 1; } __setup("ftrace=", set_cmdline_ftrace); @@ -247,7 +241,7 @@ static int __init boot_alloc_snapshot(char *str) } else { allocate_snapshot = true; /* We also need the main ring buffer expanded */ - ring_buffer_expanded = true; + trace_set_ring_buffer_expanded(NULL); } return 1; } @@ -490,6 +484,13 @@ static struct trace_array global_trace = { .trace_flags = TRACE_DEFAULT_FLAGS, }; +void trace_set_ring_buffer_expanded(struct trace_array *tr) +{ + if (!tr) + tr = &global_trace; + tr->ring_buffer_expanded = true; +} + LIST_HEAD(ftrace_trace_arrays); int trace_array_get(struct trace_array *this_tr) @@ -2012,7 +2013,7 @@ static int run_tracer_selftest(struct tracer *type) #ifdef CONFIG_TRACER_MAX_TRACE if (type->use_max_tr) { /* If we expanded the buffers, make sure the max is expanded too */ - if (ring_buffer_expanded) + if (tr->ring_buffer_expanded) ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size, RING_BUFFER_ALL_CPUS); tr->allocated_snapshot = true; @@ -2038,7 +2039,7 @@ static int run_tracer_selftest(struct tracer *type) tr->allocated_snapshot = false; /* Shrink the max buffer again */ - if (ring_buffer_expanded) + if (tr->ring_buffer_expanded) ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS); } @@ -3403,7 +3404,7 @@ void trace_printk_init_buffers(void) pr_warn("**********************************************************\n"); /* Expand the buffers to set size */ - tracing_update_buffers(); + tracing_update_buffers(&global_trace); buffers_allocated = 1; @@ -6374,7 +6375,7 @@ static int __tracing_resize_ring_buffer(struct trace_array *tr, * we use the size that was given, and we can forget about * expanding it later. */ - ring_buffer_expanded = true; + trace_set_ring_buffer_expanded(tr); /* May be called before buffers are initialized */ if (!tr->array_buffer.buffer) @@ -6452,6 +6453,7 @@ out: /** * tracing_update_buffers - used by tracing facility to expand ring buffers + * @tr: The tracing instance * * To save on memory when the tracing is never used on a system with it * configured in. The ring buffers are set to a minimum size. But once @@ -6460,13 +6462,13 @@ out: * * This function is to be called when a tracer is about to be used. */ -int tracing_update_buffers(void) +int tracing_update_buffers(struct trace_array *tr) { int ret = 0; mutex_lock(&trace_types_lock); - if (!ring_buffer_expanded) - ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size, + if (!tr->ring_buffer_expanded) + ret = __tracing_resize_ring_buffer(tr, trace_buf_size, RING_BUFFER_ALL_CPUS); mutex_unlock(&trace_types_lock); @@ -6520,7 +6522,7 @@ int tracing_set_tracer(struct trace_array *tr, const char *buf) mutex_lock(&trace_types_lock); - if (!ring_buffer_expanded) { + if (!tr->ring_buffer_expanded) { ret = __tracing_resize_ring_buffer(tr, trace_buf_size, RING_BUFFER_ALL_CPUS); if (ret < 0) @@ -7192,7 +7194,7 @@ tracing_entries_read(struct file *filp, char __user *ubuf, } if (buf_size_same) { - if (!ring_buffer_expanded) + if (!tr->ring_buffer_expanded) r = sprintf(buf, "%lu (expanded: %lu)\n", size >> 10, trace_buf_size >> 10); @@ -7249,10 +7251,10 @@ tracing_total_entries_read(struct file *filp, char __user *ubuf, mutex_lock(&trace_types_lock); for_each_tracing_cpu(cpu) { size += per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10; - if (!ring_buffer_expanded) + if (!tr->ring_buffer_expanded) expanded_size += trace_buf_size >> 10; } - if (ring_buffer_expanded) + if (tr->ring_buffer_expanded) r = sprintf(buf, "%lu\n", size); else r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size); @@ -7646,7 +7648,7 @@ tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt, unsigned long val; int ret; - ret = tracing_update_buffers(); + ret = tracing_update_buffers(tr); if (ret < 0) return ret; @@ -9550,6 +9552,9 @@ static struct trace_array *trace_array_create(const char *name) if (allocate_trace_buffers(tr, trace_buf_size) < 0) goto out_free_tr; + /* The ring buffer is defaultly expanded */ + trace_set_ring_buffer_expanded(tr); + if (ftrace_allocate_ftrace_ops(tr) < 0) goto out_free_tr; @@ -10444,7 +10449,7 @@ __init static int tracer_alloc_buffers(void) trace_printk_init_buffers(); /* To save memory, keep the ring buffer size to its minimum */ - if (ring_buffer_expanded) + if (global_trace.ring_buffer_expanded) ring_buf_size = trace_buf_size; else ring_buf_size = 1; diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 77debe53f07c..e92cb9c1292f 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -410,6 +410,11 @@ struct trace_array { struct cond_snapshot *cond_snapshot; #endif struct trace_func_repeats __percpu *last_func_repeats; + /* + * On boot up, the ring buffer is set to the minimum size, so that + * we do not waste memory on systems that are not using tracing. + */ + bool ring_buffer_expanded; }; enum { @@ -761,7 +766,7 @@ extern int DYN_FTRACE_TEST_NAME(void); #define DYN_FTRACE_TEST_NAME2 trace_selftest_dynamic_test_func2 extern int DYN_FTRACE_TEST_NAME2(void); -extern bool ring_buffer_expanded; +extern void trace_set_ring_buffer_expanded(struct trace_array *tr); extern bool tracing_selftest_disabled; #ifdef CONFIG_FTRACE_STARTUP_TEST @@ -1305,7 +1310,7 @@ static inline void trace_branch_disable(void) #endif /* CONFIG_BRANCH_TRACER */ /* set ring buffers to default size if not already done so */ -int tracing_update_buffers(void); +int tracing_update_buffers(struct trace_array *tr); union trace_synth_field { u8 as_u8; diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index f49d6ddb6342..099b9b6bbdc4 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c @@ -1166,7 +1166,7 @@ ftrace_event_write(struct file *file, const char __user *ubuf, if (!cnt) return 0; - ret = tracing_update_buffers(); + ret = tracing_update_buffers(tr); if (ret < 0) return ret; @@ -1397,18 +1397,20 @@ event_enable_write(struct file *filp, const char __user *ubuf, size_t cnt, if (ret) return ret; - ret = tracing_update_buffers(); - if (ret < 0) - return ret; - switch (val) { case 0: case 1: ret = -ENODEV; mutex_lock(&event_mutex); file = event_file_data(filp); - if (likely(file)) + if (likely(file)) { + ret = tracing_update_buffers(file->tr); + if (ret < 0) { + mutex_unlock(&event_mutex); + return ret; + } ret = ftrace_event_enable_disable(file, val); + } mutex_unlock(&event_mutex); break; @@ -1482,7 +1484,7 @@ system_enable_write(struct file *filp, const char __user *ubuf, size_t cnt, if (ret) return ret; - ret = tracing_update_buffers(); + ret = tracing_update_buffers(dir->tr); if (ret < 0) return ret; @@ -1956,7 +1958,7 @@ event_pid_write(struct file *filp, const char __user *ubuf, if (!cnt) return 0; - ret = tracing_update_buffers(); + ret = tracing_update_buffers(tr); if (ret < 0) return ret; @@ -2824,7 +2826,7 @@ static __init int setup_trace_triggers(char *str) int i; strscpy(bootup_trigger_buf, str, COMMAND_LINE_SIZE); - ring_buffer_expanded = true; + trace_set_ring_buffer_expanded(NULL); disable_tracing_selftest("running event triggers"); buf = bootup_trigger_buf; @@ -3614,7 +3616,7 @@ static char bootup_event_buf[COMMAND_LINE_SIZE] __initdata; static __init int setup_trace_event(char *str) { strscpy(bootup_event_buf, str, COMMAND_LINE_SIZE); - ring_buffer_expanded = true; + trace_set_ring_buffer_expanded(NULL); disable_tracing_selftest("running event tracing"); return 1; From bdf4fb628093b4ab2f4eb312256233a2ae0594b7 Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Thu, 14 Sep 2023 18:34:02 +0200 Subject: [PATCH 175/475] ring_buffer: Use try_cmpxchg instead of cmpxchg in rb_insert_pages Use try_cmpxchg instead of cmpxchg (*ptr, old, new) == old in rb_insert_pages. x86 CMPXCHG instruction returns success in ZF flag, so this change saves a compare after cmpxchg (and related move instruction in front of cmpxchg). No functional change intended. Link: https://lore.kernel.org/linux-trace-kernel/20230914163420.12923-1-ubizjak@gmail.com Cc: Steven Rostedt Cc: Masami Hiramatsu Signed-off-by: Uros Bizjak Signed-off-by: Steven Rostedt (Google) --- kernel/trace/ring_buffer.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 515cafdb18d9..43cc47d7faaf 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -2056,7 +2056,7 @@ rb_insert_pages(struct ring_buffer_per_cpu *cpu_buffer) retries = 10; success = false; while (retries--) { - struct list_head *head_page, *prev_page, *r; + struct list_head *head_page, *prev_page; struct list_head *last_page, *first_page; struct list_head *head_page_with_bit; struct buffer_page *hpage = rb_set_head_page(cpu_buffer); @@ -2075,9 +2075,9 @@ rb_insert_pages(struct ring_buffer_per_cpu *cpu_buffer) last_page->next = head_page_with_bit; first_page->prev = prev_page; - r = cmpxchg(&prev_page->next, head_page_with_bit, first_page); - - if (r == head_page_with_bit) { + /* caution: head_page_with_bit gets updated on cmpxchg failure */ + if (try_cmpxchg(&prev_page->next, + &head_page_with_bit, first_page)) { /* * yay, we replaced the page pointer to our new list, * now, we just have to update to head page's prev From 5dbd04eddb2c0841d1b3930e0a9944a2343c9cac Mon Sep 17 00:00:00 2001 From: Beau Belgrave Date: Tue, 12 Sep 2023 18:07:02 +0000 Subject: [PATCH 176/475] tracing/user_events: Allow events to persist for perfmon_capable users There are several scenarios that have come up where having a user_event persist even if the process that registered it exits. The main one is having a daemon create events on bootup that shouldn't get deleted if the daemon has to exit or reload. Another is within OpenTelemetry exporters, they wish to potentially check if a user_event exists on the system to determine if exporting the data out should occur. The user_event in this case must exist even in the absence of the owning process running (such as the above daemon case). Expose the previously internal flag USER_EVENT_REG_PERSIST to user processes. Upon register or delete of events with this flag, ensure the user is perfmon_capable to prevent random user processes with access to tracefs from creating events that persist after exit. Link: https://lkml.kernel.org/r/20230912180704.1284-2-beaub@linux.microsoft.com Signed-off-by: Beau Belgrave Signed-off-by: Steven Rostedt (Google) --- include/uapi/linux/user_events.h | 11 +++++++++- kernel/trace/trace_events_user.c | 36 +++++++++++++++++++------------- 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/include/uapi/linux/user_events.h b/include/uapi/linux/user_events.h index 2984aae4a2b4..f74f3aedd49c 100644 --- a/include/uapi/linux/user_events.h +++ b/include/uapi/linux/user_events.h @@ -17,6 +17,15 @@ /* Create dynamic location entry within a 32-bit value */ #define DYN_LOC(offset, size) ((size) << 16 | (offset)) +/* List of supported registration flags */ +enum user_reg_flag { + /* Event will not delete upon last reference closing */ + USER_EVENT_REG_PERSIST = 1U << 0, + + /* This value or above is currently non-ABI */ + USER_EVENT_REG_MAX = 1U << 1, +}; + /* * Describes an event registration and stores the results of the registration. * This structure is passed to the DIAG_IOCSREG ioctl, callers at a minimum @@ -33,7 +42,7 @@ struct user_reg { /* Input: Enable size in bytes at address */ __u8 enable_size; - /* Input: Flags for future use, set to 0 */ + /* Input: Flags to use, if any */ __u16 flags; /* Input: Address to update when enabled */ diff --git a/kernel/trace/trace_events_user.c b/kernel/trace/trace_events_user.c index b87f41187c6a..9365ce407426 100644 --- a/kernel/trace/trace_events_user.c +++ b/kernel/trace/trace_events_user.c @@ -49,18 +49,6 @@ #define EVENT_STATUS_PERF BIT(1) #define EVENT_STATUS_OTHER BIT(7) -/* - * User register flags are not allowed yet, keep them here until we are - * ready to expose them out to the user ABI. - */ -enum user_reg_flag { - /* Event will not delete upon last reference closing */ - USER_EVENT_REG_PERSIST = 1U << 0, - - /* This value or above is currently non-ABI */ - USER_EVENT_REG_MAX = 1U << 1, -}; - /* * Stores the system name, tables, and locks for a group of events. This * allows isolation for events by various means. @@ -220,6 +208,17 @@ static u32 user_event_key(char *name) return jhash(name, strlen(name), 0); } +static bool user_event_capable(u16 reg_flags) +{ + /* Persistent events require CAP_PERFMON / CAP_SYS_ADMIN */ + if (reg_flags & USER_EVENT_REG_PERSIST) { + if (!perfmon_capable()) + return false; + } + + return true; +} + static struct user_event *user_event_get(struct user_event *user) { refcount_inc(&user->refcnt); @@ -1811,6 +1810,9 @@ static int user_event_free(struct dyn_event *ev) if (!user_event_last_ref(user)) return -EBUSY; + if (!user_event_capable(user->reg_flags)) + return -EPERM; + return destroy_user_event(user); } @@ -1926,10 +1928,13 @@ static int user_event_parse(struct user_event_group *group, char *name, int argc = 0; char **argv; - /* User register flags are not ready yet */ - if (reg_flags != 0 || flags != NULL) + /* Currently don't support any text based flags */ + if (flags != NULL) return -EINVAL; + if (!user_event_capable(reg_flags)) + return -EPERM; + /* Prevent dyn_event from racing */ mutex_lock(&event_mutex); user = find_user_event(group, name, &key); @@ -2062,6 +2067,9 @@ static int delete_user_event(struct user_event_group *group, char *name) if (!user_event_last_ref(user)) return -EBUSY; + if (!user_event_capable(user->reg_flags)) + return -EPERM; + return destroy_user_event(user); } From cf74c59c4fc10e45c6f415ee6624b07ed747f963 Mon Sep 17 00:00:00 2001 From: Beau Belgrave Date: Tue, 12 Sep 2023 18:07:03 +0000 Subject: [PATCH 177/475] selftests/user_events: Test persist flag cases Now that we have exposed USER_EVENT_REG_PERSIST events can persist both via the ABI and in the /sys/kernel/tracing/dynamic_events file. Ensure both the ABI and DYN cases work by calling both during the parse tests. Add new flags test that ensures only USER_EVENT_REG_PERSIST is honored and any other flag is invalid. Link: https://lkml.kernel.org/r/20230912180704.1284-3-beaub@linux.microsoft.com Signed-off-by: Beau Belgrave Signed-off-by: Steven Rostedt (Google) --- .../testing/selftests/user_events/abi_test.c | 55 ++++++++++++++++++- .../testing/selftests/user_events/dyn_test.c | 54 +++++++++++++++++- 2 files changed, 107 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/user_events/abi_test.c b/tools/testing/selftests/user_events/abi_test.c index 8202f1327c39..3d539e556dcd 100644 --- a/tools/testing/selftests/user_events/abi_test.c +++ b/tools/testing/selftests/user_events/abi_test.c @@ -24,6 +24,18 @@ const char *data_file = "/sys/kernel/tracing/user_events_data"; const char *enable_file = "/sys/kernel/tracing/events/user_events/__abi_event/enable"; +static bool event_exists(void) +{ + int fd = open(enable_file, O_RDWR); + + if (fd < 0) + return false; + + close(fd); + + return true; +} + static int change_event(bool enable) { int fd = open(enable_file, O_RDWR); @@ -47,7 +59,22 @@ static int change_event(bool enable) return ret; } -static int reg_enable(long *enable, int size, int bit) +static int event_delete(void) +{ + int fd = open(data_file, O_RDWR); + int ret; + + if (fd < 0) + return -1; + + ret = ioctl(fd, DIAG_IOCSDEL, "__abi_event"); + + close(fd); + + return ret; +} + +static int reg_enable_flags(long *enable, int size, int bit, int flags) { struct user_reg reg = {0}; int fd = open(data_file, O_RDWR); @@ -58,6 +85,7 @@ static int reg_enable(long *enable, int size, int bit) reg.size = sizeof(reg); reg.name_args = (__u64)"__abi_event"; + reg.flags = flags; reg.enable_bit = bit; reg.enable_addr = (__u64)enable; reg.enable_size = size; @@ -69,6 +97,11 @@ static int reg_enable(long *enable, int size, int bit) return ret; } +static int reg_enable(long *enable, int size, int bit) +{ + return reg_enable_flags(enable, size, bit, 0); +} + static int reg_disable(long *enable, int bit) { struct user_unreg reg = {0}; @@ -126,6 +159,26 @@ TEST_F(user, enablement) { ASSERT_EQ(0, change_event(false)); } +TEST_F(user, flags) { + /* USER_EVENT_REG_PERSIST is allowed */ + ASSERT_EQ(0, reg_enable_flags(&self->check, sizeof(int), 0, + USER_EVENT_REG_PERSIST)); + ASSERT_EQ(0, reg_disable(&self->check, 0)); + + /* Ensure it exists after close and disable */ + ASSERT_TRUE(event_exists()); + + /* Ensure we can delete it */ + ASSERT_EQ(0, event_delete()); + + /* USER_EVENT_REG_MAX or above is not allowed */ + ASSERT_EQ(-1, reg_enable_flags(&self->check, sizeof(int), 0, + USER_EVENT_REG_MAX)); + + /* Ensure it does not exist after invalid flags */ + ASSERT_FALSE(event_exists()); +} + TEST_F(user, bit_sizes) { /* Allow 0-31 bits for 32-bit */ ASSERT_EQ(0, reg_enable(&self->check, sizeof(int), 0)); diff --git a/tools/testing/selftests/user_events/dyn_test.c b/tools/testing/selftests/user_events/dyn_test.c index a85980190bea..bdf9ab127488 100644 --- a/tools/testing/selftests/user_events/dyn_test.c +++ b/tools/testing/selftests/user_events/dyn_test.c @@ -17,9 +17,25 @@ #include "../kselftest_harness.h" #include "user_events_selftests.h" +const char *dyn_file = "/sys/kernel/tracing/dynamic_events"; const char *abi_file = "/sys/kernel/tracing/user_events_data"; const char *enable_file = "/sys/kernel/tracing/events/user_events/__test_event/enable"; +static int event_delete(void) +{ + int fd = open(abi_file, O_RDWR); + int ret; + + if (fd < 0) + return -1; + + ret = ioctl(fd, DIAG_IOCSDEL, "__test_event"); + + close(fd); + + return ret; +} + static bool wait_for_delete(void) { int i; @@ -64,7 +80,31 @@ static int unreg_event(int fd, int *check, int bit) return ioctl(fd, DIAG_IOCSUNREG, &unreg); } -static int parse(int *check, const char *value) +static int parse_dyn(const char *value) +{ + int fd = open(dyn_file, O_RDWR | O_APPEND); + int len = strlen(value); + int ret; + + if (fd == -1) + return -1; + + ret = write(fd, value, len); + + if (ret == len) + ret = 0; + else + ret = -1; + + close(fd); + + if (ret == 0) + event_delete(); + + return ret; +} + +static int parse_abi(int *check, const char *value) { int fd = open(abi_file, O_RDWR); int ret; @@ -90,6 +130,18 @@ static int parse(int *check, const char *value) return ret; } +static int parse(int *check, const char *value) +{ + int abi_ret = parse_abi(check, value); + int dyn_ret = parse_dyn(value); + + /* Ensure both ABI and DYN parse the same way */ + if (dyn_ret != abi_ret) + return -1; + + return dyn_ret; +} + static int check_match(int *check, const char *first, const char *second, bool *match) { int fd = open(abi_file, O_RDWR); From 2c6d0950f65e537ddec96548e2f73c00e758f87e Mon Sep 17 00:00:00 2001 From: Beau Belgrave Date: Tue, 12 Sep 2023 18:07:04 +0000 Subject: [PATCH 178/475] tracing/user_events: Document persist event flags Users need to know how to make events persist now that we allow for that. We also now allow the dynamic_events file to create events by utilizing the persist flag during event register. Add back in to documentation how /sys/kernel/tracing/dynamic_events can be used to create persistent user_events. Add a section under registering for the currently supported flags (USER_EVENT_REG_PERSIST) and the required permissions. Add a note under deleting that deleting a persistent event also requires sufficient permission. Link: https://lkml.kernel.org/r/20230912180704.1284-4-beaub@linux.microsoft.com Signed-off-by: Beau Belgrave Signed-off-by: Steven Rostedt (Google) --- Documentation/trace/user_events.rst | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/Documentation/trace/user_events.rst b/Documentation/trace/user_events.rst index f9530d0ac5d3..d8f12442aaa6 100644 --- a/Documentation/trace/user_events.rst +++ b/Documentation/trace/user_events.rst @@ -14,6 +14,11 @@ Programs can view status of the events via /sys/kernel/tracing/user_events_status and can both register and write data out via /sys/kernel/tracing/user_events_data. +Programs can also use /sys/kernel/tracing/dynamic_events to register and +delete user based events via the u: prefix. The format of the command to +dynamic_events is the same as the ioctl with the u: prefix applied. This +requires CAP_PERFMON due to the event persisting, otherwise -EPERM is returned. + Typically programs will register a set of events that they wish to expose to tools that can read trace_events (such as ftrace and perf). The registration process tells the kernel which address and bit to reflect if any tool has @@ -45,7 +50,7 @@ This command takes a packed struct user_reg as an argument:: /* Input: Enable size in bytes at address */ __u8 enable_size; - /* Input: Flags for future use, set to 0 */ + /* Input: Flags to use, if any */ __u16 flags; /* Input: Address to update when enabled */ @@ -69,7 +74,7 @@ The struct user_reg requires all the above inputs to be set appropriately. This must be 4 (32-bit) or 8 (64-bit). 64-bit values are only allowed to be used on 64-bit kernels, however, 32-bit can be used on all kernels. -+ flags: The flags to use, if any. For the initial version this must be 0. ++ flags: The flags to use, if any. Callers should first attempt to use flags and retry without flags to ensure support for lower versions of the kernel. If a flag is not supported -EINVAL is returned. @@ -80,6 +85,13 @@ The struct user_reg requires all the above inputs to be set appropriately. + name_args: The name and arguments to describe the event, see command format for details. +The following flags are currently supported. + ++ USER_EVENT_REG_PERSIST: The event will not delete upon the last reference + closing. Callers may use this if an event should exist even after the + process closes or unregisters the event. Requires CAP_PERFMON otherwise + -EPERM is returned. + Upon successful registration the following is set. + write_index: The index to use for this file descriptor that represents this @@ -141,7 +153,10 @@ event (in both user and kernel space). User programs should use a separate file to request deletes than the one used for registration due to this. **NOTE:** By default events will auto-delete when there are no references left -to the event. Flags in the future may change this logic. +to the event. If programs do not want auto-delete, they must use the +USER_EVENT_REG_PERSIST flag when registering the event. Once that flag is used +the event exists until DIAG_IOCSDEL is invoked. Both register and delete of an +event that persists requires CAP_PERFMON, otherwise -EPERM is returned. Unregistering ------------- From 5790b1fb3d672d9a1fe3881a7181dfdbe741568f Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Wed, 4 Oct 2023 16:50:07 -0400 Subject: [PATCH 179/475] eventfs: Remove eventfs_file and just use eventfs_inode Instead of having a descriptor for every file represented in the eventfs directory, only have the directory itself represented. Change the API to send in a list of entries that represent all the files in the directory (but not other directories). The entry list contains a name and a callback function that will be used to create the files when they are accessed. struct eventfs_inode *eventfs_create_events_dir(const char *name, struct dentry *parent, const struct eventfs_entry *entries, int size, void *data); is used for the top level eventfs directory, and returns an eventfs_inode that will be used by: struct eventfs_inode *eventfs_create_dir(const char *name, struct eventfs_inode *parent, const struct eventfs_entry *entries, int size, void *data); where both of the above take an array of struct eventfs_entry entries for every file that is in the directory. The entries are defined by: typedef int (*eventfs_callback)(const char *name, umode_t *mode, void **data, const struct file_operations **fops); struct eventfs_entry { const char *name; eventfs_callback callback; }; Where the name is the name of the file and the callback gets called when the file is being created. The callback passes in the name (in case the same callback is used for multiple files), a pointer to the mode, data and fops. The data will be pointing to the data that was passed in eventfs_create_dir() or eventfs_create_events_dir() but may be overridden to point to something else, as it will be used to point to the inode->i_private that is created. The information passed back from the callback is used to create the dentry/inode. If the callback fills the data and the file should be created, it must return a positive number. On zero or negative, the file is ignored. This logic may also be used as a prototype to convert entire pseudo file systems into just-in-time allocation. The "show_events_dentry" file has been updated to show the directories, and any files they have. With just the eventfs_file allocations: Before after deltas for meminfo (in kB): MemFree: -14360 MemAvailable: -14260 Buffers: 40 Cached: 24 Active: 44 Inactive: 48 Inactive(anon): 28 Active(file): 44 Inactive(file): 20 Dirty: -4 AnonPages: 28 Mapped: 4 KReclaimable: 132 Slab: 1604 SReclaimable: 132 SUnreclaim: 1472 Committed_AS: 12 Before after deltas for slabinfo: : [ * = ] ext4_inode_cache 27 [* 1184 = 31968 ] extent_status 102 [* 40 = 4080 ] tracefs_inode_cache 144 [* 656 = 94464 ] buffer_head 39 [* 104 = 4056 ] shmem_inode_cache 49 [* 800 = 39200 ] filp -53 [* 256 = -13568 ] dentry 251 [* 192 = 48192 ] lsm_file_cache 277 [* 32 = 8864 ] vm_area_struct -14 [* 184 = -2576 ] trace_event_file 1748 [* 88 = 153824 ] kmalloc-1k 35 [* 1024 = 35840 ] kmalloc-256 49 [* 256 = 12544 ] kmalloc-192 -28 [* 192 = -5376 ] kmalloc-128 -30 [* 128 = -3840 ] kmalloc-96 10581 [* 96 = 1015776 ] kmalloc-64 3056 [* 64 = 195584 ] kmalloc-32 1291 [* 32 = 41312 ] kmalloc-16 2310 [* 16 = 36960 ] kmalloc-8 9216 [* 8 = 73728 ] Free memory dropped by 14,360 kB Available memory dropped by 14,260 kB Total slab additions in size: 1,771,032 bytes With this change: Before after deltas for meminfo (in kB): MemFree: -12084 MemAvailable: -11976 Buffers: 32 Cached: 32 Active: 72 Inactive: 168 Inactive(anon): 176 Active(file): 72 Inactive(file): -8 Dirty: 24 AnonPages: 196 Mapped: 8 KReclaimable: 148 Slab: 836 SReclaimable: 148 SUnreclaim: 688 Committed_AS: 324 Before after deltas for slabinfo: : [ * = ] tracefs_inode_cache 144 [* 656 = 94464 ] shmem_inode_cache -23 [* 800 = -18400 ] filp -92 [* 256 = -23552 ] dentry 179 [* 192 = 34368 ] lsm_file_cache -3 [* 32 = -96 ] vm_area_struct -13 [* 184 = -2392 ] trace_event_file 1748 [* 88 = 153824 ] kmalloc-1k -49 [* 1024 = -50176 ] kmalloc-256 -27 [* 256 = -6912 ] kmalloc-128 1864 [* 128 = 238592 ] kmalloc-64 4685 [* 64 = 299840 ] kmalloc-32 -72 [* 32 = -2304 ] kmalloc-16 256 [* 16 = 4096 ] total = 721352 Free memory dropped by 12,084 kB Available memory dropped by 11,976 kB Total slab additions in size: 721,352 bytes That's over 2 MB in savings per instance for free and available memory, and over 1 MB in savings per instance of slab memory. Link: https://lore.kernel.org/linux-trace-kernel/20231003184059.4924468e@gandalf.local.home Link: https://lore.kernel.org/linux-trace-kernel/20231004165007.43d79161@gandalf.local.home Cc: Masami Hiramatsu Cc: Mark Rutland Cc: Andrew Morton Cc: Ajay Kaher Signed-off-by: Steven Rostedt (Google) --- fs/tracefs/event_inode.c | 859 ++++++++++++++++++----------------- fs/tracefs/inode.c | 2 +- fs/tracefs/internal.h | 37 +- include/linux/trace_events.h | 2 +- include/linux/tracefs.h | 29 +- kernel/trace/trace.c | 7 +- kernel/trace/trace.h | 4 +- kernel/trace/trace_events.c | 313 +++++++++---- 8 files changed, 711 insertions(+), 542 deletions(-) diff --git a/fs/tracefs/event_inode.c b/fs/tracefs/event_inode.c index 8c8d64e76103..eab18b157ef5 100644 --- a/fs/tracefs/event_inode.c +++ b/fs/tracefs/event_inode.c @@ -2,8 +2,9 @@ /* * event_inode.c - part of tracefs, a pseudo file system for activating tracing * - * Copyright (C) 2020-23 VMware Inc, author: Steven Rostedt (VMware) + * Copyright (C) 2020-23 VMware Inc, author: Steven Rostedt * Copyright (C) 2020-23 VMware Inc, author: Ajay Kaher + * Copyright (C) 2023 Google, author: Steven Rostedt * * eventfs is used to dynamically create inodes and dentries based on the * meta data provided by the tracing system. @@ -23,46 +24,6 @@ #include #include "internal.h" -struct eventfs_inode { - struct list_head e_top_files; -}; - -/* - * struct eventfs_file - hold the properties of the eventfs files and - * directories. - * @name: the name of the file or directory to create - * @d_parent: holds parent's dentry - * @dentry: once accessed holds dentry - * @list: file or directory to be added to parent directory - * @ei: list of files and directories within directory - * @fop: file_operations for file or directory - * @iop: inode_operations for file or directory - * @data: something that the caller will want to get to later on - * @mode: the permission that the file or directory should have - */ -struct eventfs_file { - const char *name; - struct dentry *d_parent; - struct dentry *dentry; - struct list_head list; - struct eventfs_inode *ei; - const struct file_operations *fop; - const struct inode_operations *iop; - /* - * Union - used for deletion - * @del_list: list of eventfs_file to delete - * @rcu: eventfs_file to delete in RCU - * @is_freed: node is freed if one of the above is set - */ - union { - struct list_head del_list; - struct rcu_head rcu; - unsigned long is_freed; - }; - void *data; - umode_t mode; -}; - static DEFINE_MUTEX(eventfs_mutex); DEFINE_STATIC_SRCU(eventfs_srcu); @@ -93,16 +54,9 @@ static const struct file_operations eventfs_file_operations = { * @data: something that the caller will want to get to later on. * @fop: struct file_operations that should be used for this file. * - * This is the basic "create a file" function for tracefs. It allows for a - * wide range of flexibility in creating a file. - * - * This function will return a pointer to a dentry if it succeeds. This - * pointer must be passed to the tracefs_remove() function when the file is - * to be removed (no automatic cleanup happens if your module is unloaded, - * you are responsible here.) If an error occurs, %NULL will be returned. - * - * If tracefs is not enabled in the kernel, the value -%ENODEV will be - * returned. + * This function creates a dentry that represents a file in the eventsfs_inode + * directory. The inode.i_private pointer will point to @data in the open() + * call. */ static struct dentry *create_file(const char *name, umode_t mode, struct dentry *parent, void *data, @@ -118,6 +72,7 @@ static struct dentry *create_file(const char *name, umode_t mode, if (WARN_ON_ONCE(!S_ISREG(mode))) return NULL; + WARN_ON_ONCE(!parent); dentry = eventfs_start_creating(name, parent); if (IS_ERR(dentry)) @@ -142,20 +97,11 @@ static struct dentry *create_file(const char *name, umode_t mode, * create_dir - create a dir in the tracefs filesystem * @name: the name of the file to create. * @parent: parent dentry for this file. - * @data: something that the caller will want to get to later on. * - * This is the basic "create a dir" function for eventfs. It allows for a - * wide range of flexibility in creating a dir. - * - * This function will return a pointer to a dentry if it succeeds. This - * pointer must be passed to the tracefs_remove() function when the file is - * to be removed (no automatic cleanup happens if your module is unloaded, - * you are responsible here.) If an error occurs, %NULL will be returned. - * - * If tracefs is not enabled in the kernel, the value -%ENODEV will be - * returned. + * This function will create a dentry for a directory represented by + * a eventfs_inode. */ -static struct dentry *create_dir(const char *name, struct dentry *parent, void *data) +static struct dentry *create_dir(const char *name, struct dentry *parent) { struct tracefs_inode *ti; struct dentry *dentry; @@ -172,7 +118,6 @@ static struct dentry *create_dir(const char *name, struct dentry *parent, void * inode->i_mode = S_IFDIR | S_IRWXU | S_IRUGO | S_IXUGO; inode->i_op = &eventfs_root_dir_inode_operations; inode->i_fop = &eventfs_file_operations; - inode->i_private = data; ti = get_tracefs(inode); ti->flags |= TRACEFS_EVENT_INODE; @@ -185,18 +130,18 @@ static struct dentry *create_dir(const char *name, struct dentry *parent, void * } /** - * eventfs_set_ef_status_free - set the ef->status to free + * eventfs_set_ei_status_free - remove the dentry reference from an eventfs_inode * @ti: the tracefs_inode of the dentry - * @dentry: dentry who's status to be freed + * @dentry: dentry which has the reference to remove. * - * eventfs_set_ef_status_free will be called if no more - * references remain + * Remove the association between a dentry from an eventfs_inode. */ -void eventfs_set_ef_status_free(struct tracefs_inode *ti, struct dentry *dentry) +void eventfs_set_ei_status_free(struct tracefs_inode *ti, struct dentry *dentry) { struct tracefs_inode *ti_parent; + struct eventfs_inode *ei_child, *tmp; struct eventfs_inode *ei; - struct eventfs_file *ef, *tmp; + int i; /* The top level events directory may be freed by this */ if (unlikely(ti->flags & TRACEFS_EVENT_TOP_INODE)) { @@ -207,9 +152,9 @@ void eventfs_set_ef_status_free(struct tracefs_inode *ti, struct dentry *dentry) ei = ti->private; /* Record all the top level files */ - list_for_each_entry_srcu(ef, &ei->e_top_files, list, + list_for_each_entry_srcu(ei_child, &ei->children, list, lockdep_is_held(&eventfs_mutex)) { - list_add_tail(&ef->del_list, &ef_del_list); + list_add_tail(&ei_child->del_list, &ef_del_list); } /* Nothing should access this, but just in case! */ @@ -218,11 +163,13 @@ void eventfs_set_ef_status_free(struct tracefs_inode *ti, struct dentry *dentry) mutex_unlock(&eventfs_mutex); /* Now safely free the top level files and their children */ - list_for_each_entry_safe(ef, tmp, &ef_del_list, del_list) { - list_del(&ef->del_list); - eventfs_remove(ef); + list_for_each_entry_safe(ei_child, tmp, &ef_del_list, del_list) { + list_del(&ei_child->del_list); + eventfs_remove_dir(ei_child); } + kfree_const(ei->name); + kfree(ei->d_children); kfree(ei); return; } @@ -233,68 +180,162 @@ void eventfs_set_ef_status_free(struct tracefs_inode *ti, struct dentry *dentry) if (!ti_parent || !(ti_parent->flags & TRACEFS_EVENT_INODE)) goto out; - ef = dentry->d_fsdata; - if (!ef) + ei = dentry->d_fsdata; + if (!ei) goto out; /* - * If ef was freed, then the LSB bit is set for d_fsdata. + * If ei was freed, then the LSB bit is set for d_fsdata. * But this should not happen, as it should still have a * ref count that prevents it. Warn in case it does. */ - if (WARN_ON_ONCE((unsigned long)ef & 1)) + if (WARN_ON_ONCE((unsigned long)ei & 1)) goto out; + /* This could belong to one of the files of the ei */ + if (ei->dentry != dentry) { + for (i = 0; i < ei->nr_entries; i++) { + if (ei->d_children[i] == dentry) + break; + } + if (WARN_ON_ONCE(i == ei->nr_entries)) + goto out; + ei->d_children[i] = NULL; + } else { + ei->dentry = NULL; + } + dentry->d_fsdata = NULL; - ef->dentry = NULL; -out: + out: mutex_unlock(&eventfs_mutex); } +/** + * create_file_dentry - create a dentry for a file of an eventfs_inode + * @ei: the eventfs_inode that the file will be created under + * @e_dentry: a pointer to the d_children[] of the @ei + * @parent: The parent dentry of the created file. + * @name: The name of the file to create + * @mode: The mode of the file. + * @data: The data to use to set the inode of the file with on open() + * @fops: The fops of the file to be created. + * @lookup: If called by the lookup routine, in which case, dput() the created dentry. + * + * Create a dentry for a file of an eventfs_inode @ei and place it into the + * address located at @e_dentry. If the @e_dentry already has a dentry, then + * just do a dget() on it and return. Otherwise create the dentry and attach it. + */ +static struct dentry * +create_file_dentry(struct eventfs_inode *ei, struct dentry **e_dentry, + struct dentry *parent, const char *name, umode_t mode, void *data, + const struct file_operations *fops, bool lookup) +{ + struct dentry *dentry; + bool invalidate = false; + + mutex_lock(&eventfs_mutex); + /* If the e_dentry already has a dentry, use it */ + if (*e_dentry) { + /* lookup does not need to up the ref count */ + if (!lookup) + dget(*e_dentry); + mutex_unlock(&eventfs_mutex); + return *e_dentry; + } + mutex_unlock(&eventfs_mutex); + + /* The lookup already has the parent->d_inode locked */ + if (!lookup) + inode_lock(parent->d_inode); + + dentry = create_file(name, mode, parent, data, fops); + + if (!lookup) + inode_unlock(parent->d_inode); + + mutex_lock(&eventfs_mutex); + + if (IS_ERR_OR_NULL(dentry)) { + /* + * When the mutex was released, something else could have + * created the dentry for this e_dentry. In which case + * use that one. + * + * Note, with the mutex held, the e_dentry cannot have content + * and the ei->is_freed be true at the same time. + */ + WARN_ON_ONCE(ei->is_freed); + dentry = *e_dentry; + /* The lookup does not need to up the dentry refcount */ + if (dentry && !lookup) + dget(dentry); + mutex_unlock(&eventfs_mutex); + return dentry; + } + + if (!*e_dentry && !ei->is_freed) { + *e_dentry = dentry; + dentry->d_fsdata = ei; + } else { + /* + * Should never happen unless we get here due to being freed. + * Otherwise it means two dentries exist with the same name. + */ + WARN_ON_ONCE(!ei->is_freed); + invalidate = true; + } + mutex_unlock(&eventfs_mutex); + + if (invalidate) + d_invalidate(dentry); + + if (lookup || invalidate) + dput(dentry); + + return invalidate ? NULL : dentry; +} + /** * eventfs_post_create_dir - post create dir routine - * @ef: eventfs_file of recently created dir + * @ei: eventfs_inode of recently created dir * * Map the meta-data of files within an eventfs dir to their parent dentry */ -static void eventfs_post_create_dir(struct eventfs_file *ef) +static void eventfs_post_create_dir(struct eventfs_inode *ei) { - struct eventfs_file *ef_child; + struct eventfs_inode *ei_child; struct tracefs_inode *ti; /* srcu lock already held */ /* fill parent-child relation */ - list_for_each_entry_srcu(ef_child, &ef->ei->e_top_files, list, + list_for_each_entry_srcu(ei_child, &ei->children, list, srcu_read_lock_held(&eventfs_srcu)) { - ef_child->d_parent = ef->dentry; + ei_child->d_parent = ei->dentry; } - ti = get_tracefs(ef->dentry->d_inode); - ti->private = ef->ei; + ti = get_tracefs(ei->dentry->d_inode); + ti->private = ei; } /** - * create_dentry - helper function to create dentry - * @ef: eventfs_file of file or directory to create - * @parent: parent dentry - * @lookup: true if called from lookup routine + * create_dir_dentry - Create a directory dentry for the eventfs_inode + * @ei: The eventfs_inode to create the directory for + * @parent: The dentry of the parent of this directory + * @lookup: True if this is called by the lookup code * - * Used to create a dentry for file/dir, executes post dentry creation routine + * This creates and attaches a directory dentry to the eventfs_inode @ei. */ static struct dentry * -create_dentry(struct eventfs_file *ef, struct dentry *parent, bool lookup) +create_dir_dentry(struct eventfs_inode *ei, struct dentry *parent, bool lookup) { bool invalidate = false; - struct dentry *dentry; + struct dentry *dentry = NULL; mutex_lock(&eventfs_mutex); - if (ef->is_freed) { - mutex_unlock(&eventfs_mutex); - return NULL; - } - if (ef->dentry) { - dentry = ef->dentry; - /* On dir open, up the ref count */ + if (ei->dentry) { + /* If the dentry already has a dentry, use it */ + dentry = ei->dentry; + /* lookup does not need to up the ref count */ if (!lookup) dget(dentry); mutex_unlock(&eventfs_mutex); @@ -302,42 +343,44 @@ create_dentry(struct eventfs_file *ef, struct dentry *parent, bool lookup) } mutex_unlock(&eventfs_mutex); + /* The lookup already has the parent->d_inode locked */ if (!lookup) inode_lock(parent->d_inode); - if (ef->ei) - dentry = create_dir(ef->name, parent, ef->data); - else - dentry = create_file(ef->name, ef->mode, parent, - ef->data, ef->fop); + dentry = create_dir(ei->name, parent); if (!lookup) inode_unlock(parent->d_inode); mutex_lock(&eventfs_mutex); - if (IS_ERR_OR_NULL(dentry)) { - /* If the ef was already updated get it */ - dentry = ef->dentry; + + if (IS_ERR_OR_NULL(dentry) && !ei->is_freed) { + /* + * When the mutex was released, something else could have + * created the dentry for this e_dentry. In which case + * use that one. + * + * Note, with the mutex held, the e_dentry cannot have content + * and the ei->is_freed be true at the same time. + */ + dentry = ei->dentry; if (dentry && !lookup) dget(dentry); mutex_unlock(&eventfs_mutex); return dentry; } - if (!ef->dentry && !ef->is_freed) { - ef->dentry = dentry; - if (ef->ei) - eventfs_post_create_dir(ef); - dentry->d_fsdata = ef; + if (!ei->dentry && !ei->is_freed) { + ei->dentry = dentry; + eventfs_post_create_dir(ei); + dentry->d_fsdata = ei; } else { - /* A race here, should try again (unless freed) */ - invalidate = true; - /* * Should never happen unless we get here due to being freed. * Otherwise it means two dentries exist with the same name. */ - WARN_ON_ONCE(!ef->is_freed); + WARN_ON_ONCE(!ei->is_freed); + invalidate = true; } mutex_unlock(&eventfs_mutex); if (invalidate) @@ -349,50 +392,85 @@ create_dentry(struct eventfs_file *ef, struct dentry *parent, bool lookup) return invalidate ? NULL : dentry; } -static bool match_event_file(struct eventfs_file *ef, const char *name) -{ - bool ret; - - mutex_lock(&eventfs_mutex); - ret = !ef->is_freed && strcmp(ef->name, name) == 0; - mutex_unlock(&eventfs_mutex); - - return ret; -} - /** * eventfs_root_lookup - lookup routine to create file/dir * @dir: in which a lookup is being done * @dentry: file/dir dentry - * @flags: to pass as flags parameter to simple lookup + * @flags: Just passed to simple_lookup() * - * Used to create a dynamic file/dir within @dir. Use the eventfs_inode - * list of meta data to find the information needed to create the file/dir. + * Used to create dynamic file/dir with-in @dir, search with-in @ei + * list, if @dentry found go ahead and create the file/dir */ + static struct dentry *eventfs_root_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags) { + const struct file_operations *fops; + const struct eventfs_entry *entry; + struct eventfs_inode *ei_child; struct tracefs_inode *ti; struct eventfs_inode *ei; - struct eventfs_file *ef; + struct dentry *ei_dentry = NULL; struct dentry *ret = NULL; + const char *name = dentry->d_name.name; + bool created = false; + umode_t mode; + void *data; int idx; + int i; + int r; ti = get_tracefs(dir); if (!(ti->flags & TRACEFS_EVENT_INODE)) return NULL; - ei = ti->private; + /* Grab srcu to prevent the ei from going away */ idx = srcu_read_lock(&eventfs_srcu); - list_for_each_entry_srcu(ef, &ei->e_top_files, list, + + /* + * Grab the eventfs_mutex to consistent value from ti->private. + * This s + */ + mutex_lock(&eventfs_mutex); + ei = READ_ONCE(ti->private); + if (ei) + ei_dentry = READ_ONCE(ei->dentry); + mutex_unlock(&eventfs_mutex); + + if (!ei || !ei_dentry) + goto out; + + data = ei->data; + + list_for_each_entry_srcu(ei_child, &ei->children, list, srcu_read_lock_held(&eventfs_srcu)) { - if (!match_event_file(ef, dentry->d_name.name)) + if (strcmp(ei_child->name, name) != 0) continue; ret = simple_lookup(dir, dentry, flags); - create_dentry(ef, ef->d_parent, true); + create_dir_dentry(ei_child, ei_dentry, true); + created = true; break; } + + if (created) + goto out; + + for (i = 0; i < ei->nr_entries; i++) { + entry = &ei->entries[i]; + if (strcmp(name, entry->name) == 0) { + void *cdata = data; + r = entry->callback(name, &mode, &cdata, &fops); + if (r <= 0) + continue; + ret = simple_lookup(dir, dentry, flags); + create_file_dentry(ei, &ei->d_children[i], + ei_dentry, name, mode, cdata, + fops, true); + break; + } + } + out: srcu_read_unlock(&eventfs_srcu, idx); return ret; } @@ -432,29 +510,48 @@ static int eventfs_release(struct inode *inode, struct file *file) return dcache_dir_close(inode, file); } +static int add_dentries(struct dentry ***dentries, struct dentry *d, int cnt) +{ + struct dentry **tmp; + + tmp = krealloc(*dentries, sizeof(d) * (cnt + 2), GFP_KERNEL); + if (!tmp) + return -1; + tmp[cnt] = d; + tmp[cnt + 1] = NULL; + *dentries = tmp; + return 0; +} + /** * dcache_dir_open_wrapper - eventfs open wrapper * @inode: not used - * @file: dir to be opened (to create its child) + * @file: dir to be opened (to create it's children) * - * Used to dynamically create the file/dir within @file. @file is really a - * directory and all the files/dirs of the children within @file will be - * created. If any of the files/dirs have already been created, their - * reference count will be incremented. + * Used to dynamic create file/dir with-in @file, all the + * file/dir will be created. If already created then references + * will be increased */ static int dcache_dir_open_wrapper(struct inode *inode, struct file *file) { + const struct file_operations *fops; + const struct eventfs_entry *entry; + struct eventfs_inode *ei_child; struct tracefs_inode *ti; struct eventfs_inode *ei; - struct eventfs_file *ef; struct dentry_list *dlist; struct dentry **dentries = NULL; - struct dentry *dentry = file_dentry(file); + struct dentry *parent = file_dentry(file); struct dentry *d; struct inode *f_inode = file_inode(file); + const char *name = parent->d_name.name; + umode_t mode; + void *data; int cnt = 0; int idx; int ret; + int i; + int r; ti = get_tracefs(f_inode); if (!(ti->flags & TRACEFS_EVENT_INODE)) @@ -463,25 +560,51 @@ static int dcache_dir_open_wrapper(struct inode *inode, struct file *file) if (WARN_ON_ONCE(file->private_data)) return -EINVAL; - dlist = kmalloc(sizeof(*dlist), GFP_KERNEL); - if (!dlist) - return -ENOMEM; - - ei = ti->private; idx = srcu_read_lock(&eventfs_srcu); - list_for_each_entry_srcu(ef, &ei->e_top_files, list, - srcu_read_lock_held(&eventfs_srcu)) { - d = create_dentry(ef, dentry, false); - if (d) { - struct dentry **tmp; - tmp = krealloc(dentries, sizeof(d) * (cnt + 2), GFP_KERNEL); - if (!tmp) + mutex_lock(&eventfs_mutex); + ei = READ_ONCE(ti->private); + mutex_unlock(&eventfs_mutex); + + if (!ei) { + srcu_read_unlock(&eventfs_srcu, idx); + return -EINVAL; + } + + + data = ei->data; + + dlist = kmalloc(sizeof(*dlist), GFP_KERNEL); + if (!dlist) { + srcu_read_unlock(&eventfs_srcu, idx); + return -ENOMEM; + } + + list_for_each_entry_srcu(ei_child, &ei->children, list, + srcu_read_lock_held(&eventfs_srcu)) { + d = create_dir_dentry(ei_child, parent, false); + if (d) { + ret = add_dentries(&dentries, d, cnt); + if (ret < 0) + break; + cnt++; + } + } + + for (i = 0; i < ei->nr_entries; i++) { + void *cdata = data; + entry = &ei->entries[i]; + name = entry->name; + r = entry->callback(name, &mode, &cdata, &fops); + if (r <= 0) + continue; + d = create_file_dentry(ei, &ei->d_children[i], + parent, name, mode, cdata, fops, false); + if (d) { + ret = add_dentries(&dentries, d, cnt); + if (ret < 0) break; - tmp[cnt] = d; - tmp[cnt + 1] = NULL; cnt++; - dentries = tmp; } } srcu_read_unlock(&eventfs_srcu, idx); @@ -514,63 +637,90 @@ static int dcache_readdir_wrapper(struct file *file, struct dir_context *ctx) } /** - * eventfs_prepare_ef - helper function to prepare eventfs_file - * @name: the name of the file/directory to create. - * @mode: the permission that the file should have. - * @fop: struct file_operations that should be used for this file/directory. - * @iop: struct inode_operations that should be used for this file/directory. - * @data: something that the caller will want to get to later on. The - * inode.i_private pointer will point to this value on the open() call. + * eventfs_create_dir - Create the eventfs_inode for this directory + * @name: The name of the directory to create. + * @parent: The eventfs_inode of the parent directory. + * @entries: A list of entries that represent the files under this directory + * @size: The number of @entries + * @data: The default data to pass to the files (an entry may override it). * - * This function allocates and fills the eventfs_file structure. + * This function creates the descriptor to represent a directory in the + * eventfs. This descriptor is an eventfs_inode, and it is returned to be + * used to create other children underneath. + * + * The @entries is an array of eventfs_entry structures which has: + * const char *name + * eventfs_callback callback; + * + * The name is the name of the file, and the callback is a pointer to a function + * that will be called when the file is reference (either by lookup or by + * reading a directory). The callback is of the prototype: + * + * int callback(const char *name, umode_t *mode, void **data, + * const struct file_operations **fops); + * + * When a file needs to be created, this callback will be called with + * name = the name of the file being created (so that the same callback + * may be used for multiple files). + * mode = a place to set the file's mode + * data = A pointer to @data, and the callback may replace it, which will + * cause the file created to pass the new data to the open() call. + * fops = the fops to use for the created file. */ -static struct eventfs_file *eventfs_prepare_ef(const char *name, umode_t mode, - const struct file_operations *fop, - const struct inode_operations *iop, - void *data) +struct eventfs_inode *eventfs_create_dir(const char *name, struct eventfs_inode *parent, + const struct eventfs_entry *entries, + int size, void *data) { - struct eventfs_file *ef; + struct eventfs_inode *ei; - ef = kzalloc(sizeof(*ef), GFP_KERNEL); - if (!ef) + if (!parent) + return ERR_PTR(-EINVAL); + + ei = kzalloc(sizeof(*ei), GFP_KERNEL); + if (!ei) return ERR_PTR(-ENOMEM); - ef->name = kstrdup(name, GFP_KERNEL); - if (!ef->name) { - kfree(ef); + ei->name = kstrdup_const(name, GFP_KERNEL); + if (!ei->name) { + kfree(ei); return ERR_PTR(-ENOMEM); } - if (S_ISDIR(mode)) { - ef->ei = kzalloc(sizeof(*ef->ei), GFP_KERNEL); - if (!ef->ei) { - kfree(ef->name); - kfree(ef); + if (size) { + ei->d_children = kzalloc(sizeof(*ei->d_children) * size, GFP_KERNEL); + if (!ei->d_children) { + kfree_const(ei->name); + kfree(ei); return ERR_PTR(-ENOMEM); } - INIT_LIST_HEAD(&ef->ei->e_top_files); - } else { - ef->ei = NULL; } - ef->iop = iop; - ef->fop = fop; - ef->mode = mode; - ef->data = data; - return ef; + ei->entries = entries; + ei->nr_entries = size; + ei->data = data; + INIT_LIST_HEAD(&ei->children); + + mutex_lock(&eventfs_mutex); + list_add_tail(&ei->list, &parent->children); + ei->d_parent = parent->dentry; + mutex_unlock(&eventfs_mutex); + + return ei; } /** - * eventfs_create_events_dir - create the trace event structure - * @name: the name of the directory to create. - * @parent: parent dentry for this file. This should be a directory dentry - * if set. If this parameter is NULL, then the directory will be - * created in the root of the tracefs filesystem. + * eventfs_create_events_dir - create the top level events directory + * @name: The name of the top level directory to create. + * @parent: Parent dentry for this file in the tracefs directory. + * @entries: A list of entries that represent the files under this directory + * @size: The number of @entries + * @data: The default data to pass to the files (an entry may override it). * * This function creates the top of the trace event directory. */ -struct dentry *eventfs_create_events_dir(const char *name, - struct dentry *parent) +struct eventfs_inode *eventfs_create_events_dir(const char *name, struct dentry *parent, + const struct eventfs_entry *entries, + int size, void *data) { struct dentry *dentry = tracefs_start_creating(name, parent); struct eventfs_inode *ei; @@ -581,19 +731,32 @@ struct dentry *eventfs_create_events_dir(const char *name, return NULL; if (IS_ERR(dentry)) - return dentry; + return (struct eventfs_inode *)dentry; ei = kzalloc(sizeof(*ei), GFP_KERNEL); if (!ei) - return ERR_PTR(-ENOMEM); + goto fail; + inode = tracefs_get_inode(dentry->d_sb); - if (unlikely(!inode)) { - kfree(ei); - tracefs_failed_creating(dentry); - return ERR_PTR(-ENOMEM); + if (unlikely(!inode)) + goto fail; + + if (size) { + ei->d_children = kzalloc(sizeof(*ei->d_children) * size, GFP_KERNEL); + if (!ei->d_children) + goto fail; } - INIT_LIST_HEAD(&ei->e_top_files); + ei->dentry = dentry; + ei->entries = entries; + ei->nr_entries = size; + ei->data = data; + ei->name = kstrdup_const(name, GFP_KERNEL); + if (!ei->name) + goto fail; + + INIT_LIST_HEAD(&ei->children); + INIT_LIST_HEAD(&ei->list); ti = get_tracefs(inode); ti->flags |= TRACEFS_EVENT_INODE | TRACEFS_EVENT_TOP_INODE; @@ -608,193 +771,41 @@ struct dentry *eventfs_create_events_dir(const char *name, d_instantiate(dentry, inode); inc_nlink(dentry->d_parent->d_inode); fsnotify_mkdir(dentry->d_parent->d_inode, dentry); - return tracefs_end_creating(dentry); + tracefs_end_creating(dentry); + + /* Will call dput when the directory is removed */ + dget(dentry); + + return ei; + + fail: + kfree(ei->d_children); + kfree(ei); + tracefs_failed_creating(dentry); + return ERR_PTR(-ENOMEM); } -/** - * eventfs_add_subsystem_dir - add eventfs subsystem_dir to list to create later - * @name: the name of the file to create. - * @parent: parent dentry for this dir. - * - * This function adds eventfs subsystem dir to list. - * And all these dirs are created on the fly when they are looked up, - * and the dentry and inodes will be removed when they are done. - */ -struct eventfs_file *eventfs_add_subsystem_dir(const char *name, - struct dentry *parent) +static void free_ei(struct rcu_head *head) { - struct tracefs_inode *ti_parent; - struct eventfs_inode *ei_parent; - struct eventfs_file *ef; + struct eventfs_inode *ei = container_of(head, struct eventfs_inode, rcu); - if (security_locked_down(LOCKDOWN_TRACEFS)) - return NULL; - - if (!parent) - return ERR_PTR(-EINVAL); - - ti_parent = get_tracefs(parent->d_inode); - ei_parent = ti_parent->private; - - ef = eventfs_prepare_ef(name, S_IFDIR, NULL, NULL, NULL); - if (IS_ERR(ef)) - return ef; - - mutex_lock(&eventfs_mutex); - list_add_tail(&ef->list, &ei_parent->e_top_files); - ef->d_parent = parent; - mutex_unlock(&eventfs_mutex); - return ef; -} - -/** - * eventfs_add_dir - add eventfs dir to list to create later - * @name: the name of the file to create. - * @ef_parent: parent eventfs_file for this dir. - * - * This function adds eventfs dir to list. - * And all these dirs are created on the fly when they are looked up, - * and the dentry and inodes will be removed when they are done. - */ -struct eventfs_file *eventfs_add_dir(const char *name, - struct eventfs_file *ef_parent) -{ - struct eventfs_file *ef; - - if (security_locked_down(LOCKDOWN_TRACEFS)) - return NULL; - - if (!ef_parent) - return ERR_PTR(-EINVAL); - - ef = eventfs_prepare_ef(name, S_IFDIR, NULL, NULL, NULL); - if (IS_ERR(ef)) - return ef; - - mutex_lock(&eventfs_mutex); - list_add_tail(&ef->list, &ef_parent->ei->e_top_files); - ef->d_parent = ef_parent->dentry; - mutex_unlock(&eventfs_mutex); - return ef; -} - -/** - * eventfs_add_events_file - add the data needed to create a file for later reference - * @name: the name of the file to create. - * @mode: the permission that the file should have. - * @parent: parent dentry for this file. - * @data: something that the caller will want to get to later on. - * @fop: struct file_operations that should be used for this file. - * - * This function is used to add the information needed to create a - * dentry/inode within the top level events directory. The file created - * will have the @mode permissions. The @data will be used to fill the - * inode.i_private when the open() call is done. The dentry and inodes are - * all created when they are referenced, and removed when they are no - * longer referenced. - */ -int eventfs_add_events_file(const char *name, umode_t mode, - struct dentry *parent, void *data, - const struct file_operations *fop) -{ - struct tracefs_inode *ti; - struct eventfs_inode *ei; - struct eventfs_file *ef; - - if (security_locked_down(LOCKDOWN_TRACEFS)) - return -ENODEV; - - if (!parent) - return -EINVAL; - - if (!(mode & S_IFMT)) - mode |= S_IFREG; - - if (!parent->d_inode) - return -EINVAL; - - ti = get_tracefs(parent->d_inode); - if (!(ti->flags & TRACEFS_EVENT_INODE)) - return -EINVAL; - - ei = ti->private; - ef = eventfs_prepare_ef(name, mode, fop, NULL, data); - - if (IS_ERR(ef)) - return -ENOMEM; - - mutex_lock(&eventfs_mutex); - list_add_tail(&ef->list, &ei->e_top_files); - ef->d_parent = parent; - mutex_unlock(&eventfs_mutex); - return 0; -} - -/** - * eventfs_add_file - add eventfs file to list to create later - * @name: the name of the file to create. - * @mode: the permission that the file should have. - * @ef_parent: parent eventfs_file for this file. - * @data: something that the caller will want to get to later on. - * @fop: struct file_operations that should be used for this file. - * - * This function is used to add the information needed to create a - * file within a subdirectory of the events directory. The file created - * will have the @mode permissions. The @data will be used to fill the - * inode.i_private when the open() call is done. The dentry and inodes are - * all created when they are referenced, and removed when they are no - * longer referenced. - */ -int eventfs_add_file(const char *name, umode_t mode, - struct eventfs_file *ef_parent, - void *data, - const struct file_operations *fop) -{ - struct eventfs_file *ef; - - if (security_locked_down(LOCKDOWN_TRACEFS)) - return -ENODEV; - - if (!ef_parent) - return -EINVAL; - - if (!(mode & S_IFMT)) - mode |= S_IFREG; - - ef = eventfs_prepare_ef(name, mode, fop, NULL, data); - if (IS_ERR(ef)) - return -ENOMEM; - - mutex_lock(&eventfs_mutex); - list_add_tail(&ef->list, &ef_parent->ei->e_top_files); - ef->d_parent = ef_parent->dentry; - mutex_unlock(&eventfs_mutex); - return 0; -} - -static void free_ef(struct rcu_head *head) -{ - struct eventfs_file *ef = container_of(head, struct eventfs_file, rcu); - - kfree(ef->name); - kfree(ef->ei); - kfree(ef); + kfree_const(ei->name); + kfree(ei->d_children); + kfree(ei); } /** * eventfs_remove_rec - remove eventfs dir or file from list - * @ef: eventfs_file to be removed. - * @head: to create list of eventfs_file to be deleted - * @level: to check recursion depth + * @ei: eventfs_inode to be removed. * - * The helper function eventfs_remove_rec() is used to clean up and free the - * associated data from eventfs for both of the added functions. + * This function recursively remove eventfs_inode which + * contains info of file or dir. */ -static void eventfs_remove_rec(struct eventfs_file *ef, struct list_head *head, int level) +static void eventfs_remove_rec(struct eventfs_inode *ei, struct list_head *head, int level) { - struct eventfs_file *ef_child; + struct eventfs_inode *ei_child; - if (!ef) + if (!ei) return; /* * Check recursion depth. It should never be greater than 3: @@ -806,62 +817,68 @@ static void eventfs_remove_rec(struct eventfs_file *ef, struct list_head *head, if (WARN_ON_ONCE(level > 3)) return; - if (ef->ei) { - /* search for nested folders or files */ - list_for_each_entry_srcu(ef_child, &ef->ei->e_top_files, list, - lockdep_is_held(&eventfs_mutex)) { - eventfs_remove_rec(ef_child, head, level + 1); - } + /* search for nested folders or files */ + list_for_each_entry_srcu(ei_child, &ei->children, list, + lockdep_is_held(&eventfs_mutex)) { + eventfs_remove_rec(ei_child, head, level + 1); } - list_del_rcu(&ef->list); - list_add_tail(&ef->del_list, head); + list_del_rcu(&ei->list); + list_add_tail(&ei->del_list, head); } +static void unhook_dentry(struct dentry **dentry, struct dentry **list) +{ + if (*dentry) { + unsigned long ptr = (unsigned long)*list; + + /* Keep the dentry from being freed yet */ + dget(*dentry); + + /* + * Paranoid: The dget() above should prevent the dentry + * from being freed and calling eventfs_set_ei_status_free(). + * But just in case, set the link list LSB pointer to 1 + * and have eventfs_set_ei_status_free() check that to + * make sure that if it does happen, it will not think + * the d_fsdata is an eventfs_inode. + * + * For this to work, no eventfs_inode should be allocated + * on a odd space, as the ef should always be allocated + * to be at least word aligned. Check for that too. + */ + WARN_ON_ONCE(ptr & 1); + + (*dentry)->d_fsdata = (void *)(ptr | 1); + *list = *dentry; + *dentry = NULL; + } +} /** * eventfs_remove - remove eventfs dir or file from list - * @ef: eventfs_file to be removed. + * @ei: eventfs_inode to be removed. * * This function acquire the eventfs_mutex lock and call eventfs_remove_rec() */ -void eventfs_remove(struct eventfs_file *ef) +void eventfs_remove_dir(struct eventfs_inode *ei) { - struct eventfs_file *tmp; - LIST_HEAD(ef_del_list); + struct eventfs_inode *tmp; + LIST_HEAD(ei_del_list); struct dentry *dentry_list = NULL; struct dentry *dentry; + int i; - if (!ef) + if (!ei) return; mutex_lock(&eventfs_mutex); - eventfs_remove_rec(ef, &ef_del_list, 0); - list_for_each_entry_safe(ef, tmp, &ef_del_list, del_list) { - if (ef->dentry) { - unsigned long ptr = (unsigned long)dentry_list; + eventfs_remove_rec(ei, &ei_del_list, 0); - /* Keep the dentry from being freed yet */ - dget(ef->dentry); - - /* - * Paranoid: The dget() above should prevent the dentry - * from being freed and calling eventfs_set_ef_status_free(). - * But just in case, set the link list LSB pointer to 1 - * and have eventfs_set_ef_status_free() check that to - * make sure that if it does happen, it will not think - * the d_fsdata is an event_file. - * - * For this to work, no event_file should be allocated - * on a odd space, as the ef should always be allocated - * to be at least word aligned. Check for that too. - */ - WARN_ON_ONCE(ptr & 1); - - ef->dentry->d_fsdata = (void *)(ptr | 1); - dentry_list = ef->dentry; - ef->dentry = NULL; - } - call_srcu(&eventfs_srcu, &ef->rcu, free_ef); + list_for_each_entry_safe(ei, tmp, &ei_del_list, del_list) { + for (i = 0; i < ei->nr_entries; i++) + unhook_dentry(&ei->d_children[i], &dentry_list); + unhook_dentry(&ei->dentry, &dentry_list); + call_srcu(&eventfs_srcu, &ei->rcu, free_ei); } mutex_unlock(&eventfs_mutex); @@ -876,8 +893,8 @@ void eventfs_remove(struct eventfs_file *ef) mutex_lock(&eventfs_mutex); /* dentry should now have at least a single reference */ WARN_ONCE((int)d_count(dentry) < 1, - "dentry %p less than one reference (%d) after invalidate\n", - dentry, d_count(dentry)); + "dentry %px (%s) less than one reference (%d) after invalidate\n", + dentry, dentry->d_name.name, d_count(dentry)); mutex_unlock(&eventfs_mutex); dput(dentry); } diff --git a/fs/tracefs/inode.c b/fs/tracefs/inode.c index 891653ba9cf3..34ffb2f8114e 100644 --- a/fs/tracefs/inode.c +++ b/fs/tracefs/inode.c @@ -385,7 +385,7 @@ static void tracefs_dentry_iput(struct dentry *dentry, struct inode *inode) ti = get_tracefs(inode); if (ti && ti->flags & TRACEFS_EVENT_INODE) - eventfs_set_ef_status_free(ti, dentry); + eventfs_set_ei_status_free(ti, dentry); iput(inode); } diff --git a/fs/tracefs/internal.h b/fs/tracefs/internal.h index 4f2e49e2197b..298d3ecaf621 100644 --- a/fs/tracefs/internal.h +++ b/fs/tracefs/internal.h @@ -13,6 +13,41 @@ struct tracefs_inode { struct inode vfs_inode; }; +/* + * struct eventfs_inode - hold the properties of the eventfs directories. + * @list: link list into the parent directory + * @entries: the array of entries representing the files in the directory + * @name: the name of the directory to create + * @children: link list into the child eventfs_inode + * @dentry: the dentry of the directory + * @d_parent: pointer to the parent's dentry + * @d_children: The array of dentries to represent the files when created + * @data: The private data to pass to the callbacks + * @nr_entries: The number of items in @entries + */ +struct eventfs_inode { + struct list_head list; + const struct eventfs_entry *entries; + const char *name; + struct list_head children; + struct dentry *dentry; + struct dentry *d_parent; + struct dentry **d_children; + void *data; + /* + * Union - used for deletion + * @del_list: list of eventfs_inode to delete + * @rcu: eventfs_indoe to delete in RCU + * @is_freed: node is freed if one of the above is set + */ + union { + struct list_head del_list; + struct rcu_head rcu; + unsigned long is_freed; + }; + int nr_entries; +}; + static inline struct tracefs_inode *get_tracefs(const struct inode *inode) { return container_of(inode, struct tracefs_inode, vfs_inode); @@ -25,6 +60,6 @@ struct inode *tracefs_get_inode(struct super_block *sb); struct dentry *eventfs_start_creating(const char *name, struct dentry *parent); struct dentry *eventfs_failed_creating(struct dentry *dentry); struct dentry *eventfs_end_creating(struct dentry *dentry); -void eventfs_set_ef_status_free(struct tracefs_inode *ti, struct dentry *dentry); +void eventfs_set_ei_status_free(struct tracefs_inode *ti, struct dentry *dentry); #endif /* _TRACEFS_INTERNAL_H */ diff --git a/include/linux/trace_events.h b/include/linux/trace_events.h index 21ae37e49319..12207dc6722d 100644 --- a/include/linux/trace_events.h +++ b/include/linux/trace_events.h @@ -649,7 +649,7 @@ struct trace_event_file { struct list_head list; struct trace_event_call *event_call; struct event_filter __rcu *filter; - struct eventfs_file *ef; + struct eventfs_inode *ei; struct trace_array *tr; struct trace_subsystem_dir *system; struct list_head triggers; diff --git a/include/linux/tracefs.h b/include/linux/tracefs.h index 009072792fa3..0c39704455d9 100644 --- a/include/linux/tracefs.h +++ b/include/linux/tracefs.h @@ -23,26 +23,25 @@ struct file_operations; struct eventfs_file; -struct dentry *eventfs_create_events_dir(const char *name, - struct dentry *parent); +typedef int (*eventfs_callback)(const char *name, umode_t *mode, void **data, + const struct file_operations **fops); -struct eventfs_file *eventfs_add_subsystem_dir(const char *name, - struct dentry *parent); +struct eventfs_entry { + const char *name; + eventfs_callback callback; +}; -struct eventfs_file *eventfs_add_dir(const char *name, - struct eventfs_file *ef_parent); +struct eventfs_inode; -int eventfs_add_file(const char *name, umode_t mode, - struct eventfs_file *ef_parent, void *data, - const struct file_operations *fops); +struct eventfs_inode *eventfs_create_events_dir(const char *name, struct dentry *parent, + const struct eventfs_entry *entries, + int size, void *data); -int eventfs_add_events_file(const char *name, umode_t mode, - struct dentry *parent, void *data, - const struct file_operations *fops); +struct eventfs_inode *eventfs_create_dir(const char *name, struct eventfs_inode *parent, + const struct eventfs_entry *entries, + int size, void *data); -void eventfs_remove(struct eventfs_file *ef); - -void eventfs_remove_events_dir(struct dentry *dentry); +void eventfs_remove_dir(struct eventfs_inode *ei); struct dentry *tracefs_create_file(const char *name, umode_t mode, struct dentry *parent, void *data, diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index dd6395692ff9..4383be8fa1b0 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -9764,7 +9764,6 @@ static __init void create_trace_instances(struct dentry *d_tracer) static void init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer) { - struct trace_event_file *file; int cpu; trace_create_file("available_tracers", TRACE_MODE_READ, d_tracer, @@ -9797,11 +9796,7 @@ init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer) trace_create_file("trace_marker", 0220, d_tracer, tr, &tracing_mark_fops); - file = __find_event_file(tr, "ftrace", "print"); - if (file && file->ef) - eventfs_add_file("trigger", TRACE_MODE_WRITE, file->ef, - file, &event_trigger_fops); - tr->trace_marker_file = file; + tr->trace_marker_file = __find_event_file(tr, "ftrace", "print"); trace_create_file("trace_marker_raw", 0220, d_tracer, tr, &tracing_mark_raw_fops); diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index e92cb9c1292f..0e1405abf4f7 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -381,7 +381,7 @@ struct trace_array { struct dentry *dir; struct dentry *options; struct dentry *percpu_dir; - struct dentry *event_dir; + struct eventfs_inode *event_dir; struct trace_options *topts; struct list_head systems; struct list_head events; @@ -1349,7 +1349,7 @@ struct trace_subsystem_dir { struct list_head list; struct event_subsystem *subsystem; struct trace_array *tr; - struct eventfs_file *ef; + struct eventfs_inode *ei; int ref_count; int nr_events; }; diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index 099b9b6bbdc4..a3b9d9423824 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c @@ -984,7 +984,7 @@ static void remove_subsystem(struct trace_subsystem_dir *dir) return; if (!--dir->nr_events) { - eventfs_remove(dir->ef); + eventfs_remove_dir(dir->ei); list_del(&dir->list); __put_system_dir(dir); } @@ -992,7 +992,7 @@ static void remove_subsystem(struct trace_subsystem_dir *dir) static void remove_event_file_dir(struct trace_event_file *file) { - eventfs_remove(file->ef); + eventfs_remove_dir(file->ei); list_del(&file->list); remove_subsystem(file->system); free_event_filter(file->filter); @@ -2282,14 +2282,40 @@ create_new_subsystem(const char *name) return NULL; } -static struct eventfs_file * +int system_callback(const char *name, umode_t *mode, void **data, + const struct file_operations **fops) +{ + if (strcmp(name, "filter") == 0) + *fops = &ftrace_subsystem_filter_fops; + + else if (strcmp(name, "enable") == 0) + *fops = &ftrace_system_enable_fops; + + else + return 0; + + *mode = TRACE_MODE_WRITE; + return 1; +} + +static struct eventfs_inode * event_subsystem_dir(struct trace_array *tr, const char *name, - struct trace_event_file *file, struct dentry *parent) + struct trace_event_file *file, struct eventfs_inode *parent) { struct event_subsystem *system, *iter; struct trace_subsystem_dir *dir; - struct eventfs_file *ef; - int res; + struct eventfs_inode *ei; + int nr_entries; + static struct eventfs_entry system_entries[] = { + { + .name = "filter", + .callback = system_callback, + }, + { + .name = "enable", + .callback = system_callback, + } + }; /* First see if we did not already create this dir */ list_for_each_entry(dir, &tr->systems, list) { @@ -2297,7 +2323,7 @@ event_subsystem_dir(struct trace_array *tr, const char *name, if (strcmp(system->name, name) == 0) { dir->nr_events++; file->system = dir; - return dir->ef; + return dir->ei; } } @@ -2321,39 +2347,29 @@ event_subsystem_dir(struct trace_array *tr, const char *name, } else __get_system(system); - ef = eventfs_add_subsystem_dir(name, parent); - if (IS_ERR(ef)) { + /* ftrace only has directories no files */ + if (strcmp(name, "ftrace") == 0) + nr_entries = 0; + else + nr_entries = ARRAY_SIZE(system_entries); + + ei = eventfs_create_dir(name, parent, system_entries, nr_entries, dir); + if (!ei) { pr_warn("Failed to create system directory %s\n", name); __put_system(system); goto out_free; } - dir->ef = ef; + dir->ei = ei; dir->tr = tr; dir->ref_count = 1; dir->nr_events = 1; dir->subsystem = system; file->system = dir; - /* the ftrace system is special, do not create enable or filter files */ - if (strcmp(name, "ftrace") != 0) { - - res = eventfs_add_file("filter", TRACE_MODE_WRITE, - dir->ef, dir, - &ftrace_subsystem_filter_fops); - if (res) { - kfree(system->filter); - system->filter = NULL; - pr_warn("Could not create tracefs '%s/filter' entry\n", name); - } - - eventfs_add_file("enable", TRACE_MODE_WRITE, dir->ef, dir, - &ftrace_system_enable_fops); - } - list_add(&dir->list, &tr->systems); - return dir->ef; + return dir->ei; out_free: kfree(dir); @@ -2402,15 +2418,134 @@ event_define_fields(struct trace_event_call *call) return ret; } +static int event_callback(const char *name, umode_t *mode, void **data, + const struct file_operations **fops) +{ + struct trace_event_file *file = *data; + struct trace_event_call *call = file->event_call; + + if (strcmp(name, "format") == 0) { + *mode = TRACE_MODE_READ; + *fops = &ftrace_event_format_fops; + *data = call; + return 1; + } + + /* + * Only event directories that can be enabled should have + * triggers or filters, with the exception of the "print" + * event that can have a "trigger" file. + */ + if (!(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)) { + if (call->class->reg && strcmp(name, "enable") == 0) { + *mode = TRACE_MODE_WRITE; + *fops = &ftrace_enable_fops; + return 1; + } + + if (strcmp(name, "filter") == 0) { + *mode = TRACE_MODE_WRITE; + *fops = &ftrace_event_filter_fops; + return 1; + } + } + + if (!(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE) || + strcmp(trace_event_name(call), "print") == 0) { + if (strcmp(name, "trigger") == 0) { + *mode = TRACE_MODE_WRITE; + *fops = &event_trigger_fops; + return 1; + } + } + +#ifdef CONFIG_PERF_EVENTS + if (call->event.type && call->class->reg && + strcmp(name, "id") == 0) { + *mode = TRACE_MODE_READ; + *data = (void *)(long)call->event.type; + *fops = &ftrace_event_id_fops; + return 1; + } +#endif + +#ifdef CONFIG_HIST_TRIGGERS + if (strcmp(name, "hist") == 0) { + *mode = TRACE_MODE_READ; + *fops = &event_hist_fops; + return 1; + } +#endif +#ifdef CONFIG_HIST_TRIGGERS_DEBUG + if (strcmp(name, "hist_debug") == 0) { + *mode = TRACE_MODE_READ; + *fops = &event_hist_debug_fops; + return 1; + } +#endif +#ifdef CONFIG_TRACE_EVENT_INJECT + if (call->event.type && call->class->reg && + strcmp(name, "inject") == 0) { + *mode = 0200; + *fops = &event_inject_fops; + return 1; + } +#endif + return 0; +} + static int -event_create_dir(struct dentry *parent, struct trace_event_file *file) +event_create_dir(struct eventfs_inode *parent, struct trace_event_file *file) { struct trace_event_call *call = file->event_call; - struct eventfs_file *ef_subsystem = NULL; struct trace_array *tr = file->tr; - struct eventfs_file *ef; + struct eventfs_inode *e_events; + struct eventfs_inode *ei; const char *name; + int nr_entries; int ret; + static struct eventfs_entry event_entries[] = { + { + .name = "enable", + .callback = event_callback, + }, + { + .name = "filter", + .callback = event_callback, + }, + { + .name = "trigger", + .callback = event_callback, + }, + { + .name = "format", + .callback = event_callback, + }, +#ifdef CONFIG_PERF_EVENTS + { + .name = "id", + .callback = event_callback, + }, +#endif +#ifdef CONFIG_HIST_TRIGGERS + { + .name = "hist", + .callback = event_callback, + }, +#endif +#ifdef CONFIG_HIST_TRIGGERS_DEBUG + { + .name = "hist_debug", + .callback = event_callback, + }, +#endif +#ifdef CONFIG_TRACE_EVENT_INJECT + { + .name = "inject", + .callback = event_callback, + }, +#endif + }; /* * If the trace point header did not define TRACE_SYSTEM @@ -2420,29 +2555,20 @@ event_create_dir(struct dentry *parent, struct trace_event_file *file) if (WARN_ON_ONCE(strcmp(call->class->system, TRACE_SYSTEM) == 0)) return -ENODEV; - ef_subsystem = event_subsystem_dir(tr, call->class->system, file, parent); - if (!ef_subsystem) + e_events = event_subsystem_dir(tr, call->class->system, file, parent); + if (!e_events) return -ENOMEM; + nr_entries = ARRAY_SIZE(event_entries); + name = trace_event_name(call); - ef = eventfs_add_dir(name, ef_subsystem); - if (IS_ERR(ef)) { + ei = eventfs_create_dir(name, e_events, event_entries, nr_entries, file); + if (IS_ERR(ei)) { pr_warn("Could not create tracefs '%s' directory\n", name); return -1; } - file->ef = ef; - - if (call->class->reg && !(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)) - eventfs_add_file("enable", TRACE_MODE_WRITE, file->ef, file, - &ftrace_enable_fops); - -#ifdef CONFIG_PERF_EVENTS - if (call->event.type && call->class->reg) - eventfs_add_file("id", TRACE_MODE_READ, file->ef, - (void *)(long)call->event.type, - &ftrace_event_id_fops); -#endif + file->ei = ei; ret = event_define_fields(call); if (ret < 0) { @@ -2450,35 +2576,6 @@ event_create_dir(struct dentry *parent, struct trace_event_file *file) return ret; } - /* - * Only event directories that can be enabled should have - * triggers or filters. - */ - if (!(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)) { - eventfs_add_file("filter", TRACE_MODE_WRITE, file->ef, - file, &ftrace_event_filter_fops); - - eventfs_add_file("trigger", TRACE_MODE_WRITE, file->ef, - file, &event_trigger_fops); - } - -#ifdef CONFIG_HIST_TRIGGERS - eventfs_add_file("hist", TRACE_MODE_READ, file->ef, file, - &event_hist_fops); -#endif -#ifdef CONFIG_HIST_TRIGGERS_DEBUG - eventfs_add_file("hist_debug", TRACE_MODE_READ, file->ef, file, - &event_hist_debug_fops); -#endif - eventfs_add_file("format", TRACE_MODE_READ, file->ef, call, - &ftrace_event_format_fops); - -#ifdef CONFIG_TRACE_EVENT_INJECT - if (call->event.type && call->class->reg) - eventfs_add_file("inject", 0200, file->ef, file, - &event_inject_fops); -#endif - return 0; } @@ -3623,30 +3720,65 @@ static __init int setup_trace_event(char *str) } __setup("trace_event=", setup_trace_event); +static int events_callback(const char *name, umode_t *mode, void **data, + const struct file_operations **fops) +{ + if (strcmp(name, "enable") == 0) { + *mode = TRACE_MODE_WRITE; + *fops = &ftrace_tr_enable_fops; + return 1; + } + + if (strcmp(name, "header_page") == 0) + *data = ring_buffer_print_page_header; + + else if (strcmp(name, "header_event") == 0) + *data = ring_buffer_print_entry_header; + + else + return 0; + + *mode = TRACE_MODE_READ; + *fops = &ftrace_show_header_fops; + return 1; +} + /* Expects to have event_mutex held when called */ static int create_event_toplevel_files(struct dentry *parent, struct trace_array *tr) { - struct dentry *d_events; + struct eventfs_inode *e_events; struct dentry *entry; - int error = 0; + int nr_entries; + static struct eventfs_entry events_entries[] = { + { + .name = "enable", + .callback = events_callback, + }, + { + .name = "header_page", + .callback = events_callback, + }, + { + .name = "header_event", + .callback = events_callback, + }, + }; entry = trace_create_file("set_event", TRACE_MODE_WRITE, parent, tr, &ftrace_set_event_fops); if (!entry) return -ENOMEM; - d_events = eventfs_create_events_dir("events", parent); - if (IS_ERR(d_events)) { + nr_entries = ARRAY_SIZE(events_entries); + + e_events = eventfs_create_events_dir("events", parent, events_entries, + nr_entries, tr); + if (IS_ERR(e_events)) { pr_warn("Could not create tracefs 'events' directory\n"); return -ENOMEM; } - error = eventfs_add_events_file("enable", TRACE_MODE_WRITE, d_events, - tr, &ftrace_tr_enable_fops); - if (error) - return -ENOMEM; - /* There are not as crucial, just warn if they are not created */ trace_create_file("set_event_pid", TRACE_MODE_WRITE, parent, @@ -3656,16 +3788,7 @@ create_event_toplevel_files(struct dentry *parent, struct trace_array *tr) TRACE_MODE_WRITE, parent, tr, &ftrace_set_event_notrace_pid_fops); - /* ring buffer internal formats */ - eventfs_add_events_file("header_page", TRACE_MODE_READ, d_events, - ring_buffer_print_page_header, - &ftrace_show_header_fops); - - eventfs_add_events_file("header_event", TRACE_MODE_READ, d_events, - ring_buffer_print_entry_header, - &ftrace_show_header_fops); - - tr->event_dir = d_events; + tr->event_dir = e_events; return 0; } @@ -3749,7 +3872,7 @@ int event_trace_del_tracer(struct trace_array *tr) down_write(&trace_event_sem); __trace_remove_event_dirs(tr); - eventfs_remove_events_dir(tr->event_dir); + eventfs_remove_dir(tr->event_dir); up_write(&trace_event_sem); tr->event_dir = NULL; From f5d9e8e08f81c9e7c723de7abcce106808f0770c Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Thu, 14 Sep 2023 12:35:06 -0400 Subject: [PATCH 180/475] tracing/selftests: Update kprobe args char/string to match new functions The function that the kprobe_args_char and kprobes_arg_string attaches to for its test has changed its name once again. Now we need to check for eventfs_create_dir(), and if it exists, use that, otherwise check for eventfs_add_dir() and if that exists use that, otherwise use the original tracefs_create_dir()! Link: https://lore.kernel.org/linux-trace-kernel/20230914163535.487267410@goodmis.org Cc: Mark Rutland Cc: Andrew Morton Cc: Ajay Kaher Acked-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- .../selftests/ftrace/test.d/kprobe/kprobe_args_char.tc | 4 +++- .../selftests/ftrace/test.d/kprobe/kprobe_args_string.tc | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_char.tc b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_char.tc index ff7499eb98d6..c639c6c8ca03 100644 --- a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_char.tc +++ b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_char.tc @@ -34,7 +34,9 @@ mips*) esac : "Test get argument (1)" -if grep -q eventfs_add_dir available_filter_functions; then +if grep -q eventfs_create_dir available_filter_functions; then + DIR_NAME="eventfs_create_dir" +elif grep -q eventfs_add_dir available_filter_functions; then DIR_NAME="eventfs_add_dir" else DIR_NAME="tracefs_create_dir" diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_string.tc b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_string.tc index a202b2ea4baf..a5ab4d5c74ac 100644 --- a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_string.tc +++ b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_string.tc @@ -37,7 +37,9 @@ loongarch*) esac : "Test get argument (1)" -if grep -q eventfs_add_dir available_filter_functions; then +if grep -q eventfs_create_dir available_filter_functions; then + DIR_NAME="eventfs_create_dir" +elif grep -q eventfs_add_dir available_filter_functions; then DIR_NAME="eventfs_add_dir" else DIR_NAME="tracefs_create_dir" From be7a4caa7c45bd4b0a39cdb260905b52a87c8688 Mon Sep 17 00:00:00 2001 From: Kuan-Wei Chiu Date: Sat, 30 Sep 2023 15:27:19 +0800 Subject: [PATCH 181/475] perf hisi-ptt: Fix memory leak in lseek failure handling In the previous code, there was a memory leak issue where the previously allocated memory was not freed upon a failed lseek operation. This patch addresses the problem by releasing the old memory before returning -errno in case of a lseek failure. This ensures that memory is properly managed and avoids potential memory leaks. Signed-off-by: Kuan-Wei Chiu Acked-by: Namhyung Kim Cc: yangyicong@hisilicon.com Cc: jonathan.cameron@huawei.com Link: https://lore.kernel.org/r/20230930072719.1267784-1-visitorckw@gmail.com Signed-off-by: Namhyung Kim --- tools/perf/util/hisi-ptt.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/perf/util/hisi-ptt.c b/tools/perf/util/hisi-ptt.c index 45b614bb73bf..43bd1ca62d58 100644 --- a/tools/perf/util/hisi-ptt.c +++ b/tools/perf/util/hisi-ptt.c @@ -108,8 +108,10 @@ static int hisi_ptt_process_auxtrace_event(struct perf_session *session, data_offset = 0; } else { data_offset = lseek(fd, 0, SEEK_CUR); - if (data_offset == -1) + if (data_offset == -1) { + free(data); return -errno; + } } err = readn(fd, data, size); From 0ddce121b038c553d6ec103f3b9fc399f99945e9 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Fri, 29 Sep 2023 23:02:06 -0700 Subject: [PATCH 182/475] perf test: Avoid system wide when not privileged Switch the test program to sleep that makes more sense for system wide events. Only enable system wide when root or not paranoid. This avoids failures under some testing conditions like ARM cloud. Signed-off-by: Ian Rogers Acked-by: Namhyung Kim Link: https://lore.kernel.org/r/20230930060206.2353141-1-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/tests/shell/stat_all_metricgroups.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tools/perf/tests/shell/stat_all_metricgroups.sh b/tools/perf/tests/shell/stat_all_metricgroups.sh index cb35e488809a..f3e305649e2c 100755 --- a/tools/perf/tests/shell/stat_all_metricgroups.sh +++ b/tools/perf/tests/shell/stat_all_metricgroups.sh @@ -4,9 +4,21 @@ set -e -for m in $(perf list --raw-dump metricgroups); do +function ParanoidAndNotRoot() +{ + [ "$(id -u)" != 0 ] && [ "$(cat /proc/sys/kernel/perf_event_paranoid)" -gt $1 ] +} + +system_wide_flag="-a" +if ParanoidAndNotRoot 0 +then + system_wide_flag="" +fi + +for m in $(perf list --raw-dump metricgroups) +do echo "Testing $m" - perf stat -M "$m" -a true + perf stat -M "$m" $system_wide_flag sleep 0.01 done exit 0 From 26a5262d30e1452071bcfe86725f2fe4e164db04 Mon Sep 17 00:00:00 2001 From: Athira Rajeev Date: Thu, 28 Sep 2023 13:22:11 +0530 Subject: [PATCH 183/475] tools/perf: Add text_end to "struct dso" to save .text section size Update "struct dso" to include new member "text_end". This new field will represent the offset for end of text section for a dso. For elf, this value is derived as: sh_size (Size of section in byes) + sh_offset (Section file offst) of the elf header for text. For bfd, this value is derived as: 1. For PE file, section->size + ( section->vma - dso->text_offset) 2. Other cases: section->filepos (file position) + section->size (size of section) To resolve the address from a sample, perf looks at the DSO maps. In case of address from a kernel module, there were some address found to be not resolved. This was observed while running perf test for "Object code reading". Though the ip falls beteen the start address of the loaded module (perf map->start ) and end address ( perf map->end), it was unresolved. Example: Reading object code for memory address: 0xc008000007f0142c File is: /lib/modules/6.5.0-rc3+/kernel/fs/xfs/xfs.ko On file address is: 0x1114cc Objdump command is: objdump -z -d --start-address=0x11142c --stop-address=0x1114ac /lib/modules/6.5.0-rc3+/kernel/fs/xfs/xfs.ko objdump read too few bytes: 128 test child finished with -1 Here, module is loaded at: # cat /proc/modules | grep xfs xfs 2228224 3 - Live 0xc008000007d00000 From objdump for xfs module, text section is: text 0010f7bc 0000000000000000 0000000000000000 000000a0 2**4 Here the offset for 0xc008000007f0142c ie 0x112074 falls out .text section which is up to 0x10f7bc. In this case for module, the address 0xc008000007e11fd4 is pointing to stub instructions. This address range represents the module stubs which is allocated on module load and hence is not part of DSO offset. To identify such address, which falls out of text section and within module end, added the new field "text_end" to "struct dso". Reported-by: Disha Goel Signed-off-by: Athira Rajeev Reviewed-by: Adrian Hunter Reviewed-by: Kajol Jain Acked-by: Namhyung Kim Cc: maddy@linux.ibm.com Cc: disgoel@linux.vnet.ibm.com Cc: linuxppc-dev@lists.ozlabs.org Link: https://lore.kernel.org/r/20230928075213.84392-1-atrajeev@linux.vnet.ibm.com Signed-off-by: Namhyung Kim --- tools/perf/util/dso.h | 1 + tools/perf/util/symbol-elf.c | 4 +++- tools/perf/util/symbol.c | 2 ++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/perf/util/dso.h b/tools/perf/util/dso.h index b41c9782c754..70fe0fe69bef 100644 --- a/tools/perf/util/dso.h +++ b/tools/perf/util/dso.h @@ -181,6 +181,7 @@ struct dso { u8 rel; struct build_id bid; u64 text_offset; + u64 text_end; const char *short_name; const char *long_name; u16 long_name_len; diff --git a/tools/perf/util/symbol-elf.c b/tools/perf/util/symbol-elf.c index 95e99c332d7e..9e7eeaf616b8 100644 --- a/tools/perf/util/symbol-elf.c +++ b/tools/perf/util/symbol-elf.c @@ -1514,8 +1514,10 @@ dso__load_sym_internal(struct dso *dso, struct map *map, struct symsrc *syms_ss, } if (elf_section_by_name(runtime_ss->elf, &runtime_ss->ehdr, &tshdr, - ".text", NULL)) + ".text", NULL)) { dso->text_offset = tshdr.sh_addr - tshdr.sh_offset; + dso->text_end = tshdr.sh_offset + tshdr.sh_size; + } if (runtime_ss->opdsec) opddata = elf_rawdata(runtime_ss->opdsec, NULL); diff --git a/tools/perf/util/symbol.c b/tools/perf/util/symbol.c index 5b54d2639df4..2740d4457c13 100644 --- a/tools/perf/util/symbol.c +++ b/tools/perf/util/symbol.c @@ -1732,8 +1732,10 @@ int dso__load_bfd_symbols(struct dso *dso, const char *debugfile) /* PE symbols can only have 4 bytes, so use .text high bits */ dso->text_offset = section->vma - (u32)section->vma; dso->text_offset += (u32)bfd_asymbol_value(symbols[i]); + dso->text_end = (section->vma - dso->text_offset) + section->size; } else { dso->text_offset = section->vma - section->filepos; + dso->text_end = section->filepos + section->size; } } From 6be5d82862ee9b4e517dcfdfb064c2d671dbd45a Mon Sep 17 00:00:00 2001 From: Athira Rajeev Date: Thu, 28 Sep 2023 13:22:12 +0530 Subject: [PATCH 184/475] tools/perf: Add "is_kmod" to struct dso to check if it is kernel module Update "struct dso" to include new member "is_kmod". This new field will determine if the file is a kernel module or not. To resolve the address from a sample, perf looks at the DSO maps. In case of address from a kernel module, there were some address found to be not resolved. This was observed while running perf test for "Object code reading". Though the ip falls beteen the start address of the loaded module (perf map->start ) and end address ( perf map->end), it was unresolved. This was happening because in some cases for kernel modules, address from sample points to stub instructions. To identify if the DSO is a kernel module, the new field "is_kmod" is added to "struct dso". Reported-by: Disha Goel Signed-off-by: Athira Rajeev Acked-by: Namhyung Kim Cc: kjain@linux.ibm.com Cc: maddy@linux.ibm.com Cc: disgoel@linux.vnet.ibm.com Cc: linuxppc-dev@lists.ozlabs.org Link: https://lore.kernel.org/r/20230928075213.84392-2-atrajeev@linux.vnet.ibm.com Signed-off-by: Namhyung Kim --- tools/perf/util/dso.c | 2 ++ tools/perf/util/dso.h | 1 + 2 files changed, 3 insertions(+) diff --git a/tools/perf/util/dso.c b/tools/perf/util/dso.c index bdfead36b83a..1f629b6fb7cf 100644 --- a/tools/perf/util/dso.c +++ b/tools/perf/util/dso.c @@ -477,6 +477,7 @@ void dso__set_module_info(struct dso *dso, struct kmod_path *m, dso->comp = m->comp; } + dso->is_kmod = 1; dso__set_short_name(dso, strdup(m->name), true); } @@ -1338,6 +1339,7 @@ struct dso *dso__new_id(const char *name, struct dso_id *id) dso->has_srcline = 1; dso->a2l_fails = 1; dso->kernel = DSO_SPACE__USER; + dso->is_kmod = 0; dso->needs_swap = DSO_SWAP__UNSET; dso->comp = COMP_ID__NONE; RB_CLEAR_NODE(&dso->rb_node); diff --git a/tools/perf/util/dso.h b/tools/perf/util/dso.h index 70fe0fe69bef..3759de8c2267 100644 --- a/tools/perf/util/dso.h +++ b/tools/perf/util/dso.h @@ -162,6 +162,7 @@ struct dso { char *symsrc_filename; unsigned int a2l_fails; enum dso_space_type kernel; + bool is_kmod; enum dso_swap_type needs_swap; enum dso_binary_type symtab_type; enum dso_binary_type binary_type; From 8f5b62a100cb235ad7708af991b5b5899fb42778 Mon Sep 17 00:00:00 2001 From: Athira Rajeev Date: Thu, 28 Sep 2023 13:22:13 +0530 Subject: [PATCH 185/475] tools/perf/tests: Fix object code reading to skip address that falls out of text section MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The testcase "Object code reading" fails in somecases for "fs_something" sub test as below: Reading object code for memory address: 0xc008000007f0142c File is: /lib/modules/6.5.0-rc3+/kernel/fs/xfs/xfs.ko On file address is: 0x1114cc Objdump command is: objdump -z -d --start-address=0x11142c --stop-address=0x1114ac /lib/modules/6.5.0-rc3+/kernel/fs/xfs/xfs.ko objdump read too few bytes: 128 test child finished with -1 This can alo be reproduced when running perf record with workload that exercises fs_something() code. In the test setup, this is exercising xfs code since root is xfs. # perf record ./a.out # perf report -v |grep "xfs.ko" 0.76% a.out /lib/modules/6.5.0-rc3+/kernel/fs/xfs/xfs.ko 0xc008000007de5efc B [k] xlog_cil_commit 0.74% a.out /lib/modules/6.5.0-rc3+/kernel/fs/xfs/xfs.ko 0xc008000007d5ae18 B [k] xfs_btree_key_offset 0.74% a.out /lib/modules/6.5.0-rc3+/kernel/fs/xfs/xfs.ko 0xc008000007e11fd4 B [k] 0x0000000000112074 Here addr "0xc008000007e11fd4" is not resolved. since this is a kernel module, its offset is from the DSO. Xfs module is loaded at 0xc008000007d00000 # cat /proc/modules | grep xfs xfs 2228224 3 - Live 0xc008000007d00000 And size is 0x220000. So its loaded between  0xc008000007d00000 and 0xc008000007f20000. From objdump, text section is: text 0010f7bc 0000000000000000 0000000000000000 000000a0 2**4 Hence perf captured ip maps to 0x112074 which is: ( ip - start of module ) + a0 This offset 0x112074 falls out .text section which is up to 0x10f7bc In this case for module, the address 0xc008000007e11fd4 is pointing to stub instructions. This address range represents the module stubs which is allocated on module load and hence is not part of DSO offset. To address this issue in "object code reading", skip the sample if address falls out of text section and is within the module end. Use the "text_end" member of "struct dso" to do this check. To address this issue in "perf report", exploring an option of having stubs range as part of the /proc/kallsyms, so that perf report can resolve addresses in stubs range However this patch uses text_end to skip the stub range for Object code reading testcase. Reported-by: Disha Goel Signed-off-by: Athira Rajeev Tested-by: Disha Goel Reviewed-by: Adrian Hunter Reviewed-by: Kajol Jain Cc: maddy@linux.ibm.com Cc: disgoel@linux.vnet.ibm.com Cc: linuxppc-dev@lists.ozlabs.org Link: https://lore.kernel.org/r/20230928075213.84392-3-atrajeev@linux.vnet.ibm.com Signed-off-by: Namhyung Kim --- tools/perf/tests/code-reading.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tools/perf/tests/code-reading.c b/tools/perf/tests/code-reading.c index ed3815163d1b..3af81012014e 100644 --- a/tools/perf/tests/code-reading.c +++ b/tools/perf/tests/code-reading.c @@ -269,6 +269,16 @@ static int read_object_code(u64 addr, size_t len, u8 cpumode, if (addr + len > map__end(al.map)) len = map__end(al.map) - addr; + /* + * Some architectures (ex: powerpc) have stubs (trampolines) in kernel + * modules to manage long jumps. Check if the ip offset falls in stubs + * sections for kernel modules. And skip module address after text end + */ + if (dso->is_kmod && al.addr > dso->text_end) { + pr_debug("skipping the module address %#"PRIx64" after text end\n", al.addr); + goto out; + } + /* Read the object code using perf */ ret_len = dso__data_read_offset(dso, maps__machine(thread__maps(thread)), al.addr, buf1, len); From d7c9ae8d5d1be0c4156d1e20e4369a77b711a4cc Mon Sep 17 00:00:00 2001 From: Kajol Jain Date: Tue, 3 Oct 2023 14:51:13 +0530 Subject: [PATCH 186/475] tools/perf: Update call stack check in builtin-lock.c The perf test named "kernel lock contention analysis test" fails in powerpc system with below error: [command]# ./perf test 81 -vv 81: kernel lock contention analysis test : --- start --- test child forked, pid 2140 Testing perf lock record and perf lock contention Testing perf lock contention --use-bpf [Skip] No BPF support Testing perf lock record and perf lock contention at the same time Testing perf lock contention --threads Testing perf lock contention --lock-addr Testing perf lock contention --type-filter (w/ spinlock) Testing perf lock contention --lock-filter (w/ tasklist_lock) Testing perf lock contention --callstack-filter (w/ unix_stream) [Fail] Recorded result should have a lock from unix_stream: test child finished with -1 ---- end ---- kernel lock contention analysis test: FAILED! The test is failing because we get an address entry with 0 in perf lock samples for powerpc, and code for lock contention option "--callstack-filter" will not check further entries after address 0. Below are some of the samples from test generated perf.data file, which have 0 address in the 2nd entry of callstack: -------- sched-messaging 3409 [001] 7152.904029: lock:contention_begin: 0xc00000c80904ef00 (flags=SPIN) c0000000001e926c __traceiter_contention_begin+0x6c ([kernel.kallsyms]) 0 [unknown] ([unknown]) c000000000f8a178 native_queued_spin_lock_slowpath+0x1f8 ([kernel.kallsyms]) c000000000f89f44 _raw_spin_lock_irqsave+0x84 ([kernel.kallsyms]) c0000000001d9fd0 prepare_to_wait+0x50 ([kernel.kallsyms]) c000000000c80f50 sock_alloc_send_pskb+0x1b0 ([kernel.kallsyms]) c000000000e82298 unix_stream_sendmsg+0x2b8 ([kernel.kallsyms]) c000000000c78980 sock_sendmsg+0x80 ([kernel.kallsyms]) sched-messaging 3408 [005] 7152.904036: lock:contention_begin: 0xc00000c80904ef00 (flags=SPIN) c0000000001e926c __traceiter_contention_begin+0x6c ([kernel.kallsyms]) 0 [unknown] ([unknown]) c000000000f8a178 native_queued_spin_lock_slowpath+0x1f8 ([kernel.kallsyms]) c000000000f89f44 _raw_spin_lock_irqsave+0x84 ([kernel.kallsyms]) c0000000001d9fd0 prepare_to_wait+0x50 ([kernel.kallsyms]) c000000000c80f50 sock_alloc_send_pskb+0x1b0 ([kernel.kallsyms]) c000000000e82298 unix_stream_sendmsg+0x2b8 ([kernel.kallsyms]) c000000000c78980 sock_sendmsg+0x80 ([kernel.kallsyms]) -------- Based on commit 20002ded4d93 ("perf_counter: powerpc: Add callchain support"), incase of powerpc, the callchain saved by kernel always includes first three entries as the NIP (next instruction pointer), LR (link register), and the contents of LR save area in the second stack frame. In certain scenarios its possible to have invalid kernel instruction addresses in either of LR or the second stack frame's LR. In that case, kernel will store the address as zer0. Hence, its possible to have 2nd or 3rd callstack entry as 0. As per the current code in match_callstack_filter function, we skip the callstack check incase we get 0 address. And hence the test case is failing in powerpc. Fix this issue by updating the check in match_callstack_filter function, to not skip callstack check if the 2nd or 3rd entry have 0 address for powerpc. Result in powerpc after patch changes: [command]# ./perf test 81 -vv 81: kernel lock contention analysis test : --- start --- test child forked, pid 4570 Testing perf lock record and perf lock contention Testing perf lock contention --use-bpf [Skip] No BPF support Testing perf lock record and perf lock contention at the same time Testing perf lock contention --threads Testing perf lock contention --lock-addr Testing perf lock contention --type-filter (w/ spinlock) Testing perf lock contention --lock-filter (w/ tasklist_lock) [Skip] Could not find 'tasklist_lock' Testing perf lock contention --callstack-filter (w/ unix_stream) Testing perf lock contention --callstack-filter with task aggregation Testing perf lock contention CSV output [Skip] No BPF support test child finished with 0 ---- end ---- kernel lock contention analysis test: Ok Fixes: ebab291641be ("perf lock contention: Support filters for different aggregation") Reported-by: Disha Goel Tested-by: Disha Goel Signed-off-by: Kajol Jain Cc: maddy@linux.ibm.com Cc: atrajeev@linux.vnet.ibm.com Link: https://lore.kernel.org/r/20231003092113.252380-1-kjain@linux.ibm.com Signed-off-by: Namhyung Kim --- tools/perf/builtin-lock.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tools/perf/builtin-lock.c b/tools/perf/builtin-lock.c index d4b22313e5fc..fa7419978353 100644 --- a/tools/perf/builtin-lock.c +++ b/tools/perf/builtin-lock.c @@ -526,6 +526,7 @@ bool match_callstack_filter(struct machine *machine, u64 *callstack) struct map *kmap; struct symbol *sym; u64 ip; + const char *arch = perf_env__arch(machine->env); if (list_empty(&callstack_filters)) return true; @@ -533,7 +534,21 @@ bool match_callstack_filter(struct machine *machine, u64 *callstack) for (int i = 0; i < max_stack_depth; i++) { struct callstack_filter *filter; - if (!callstack || !callstack[i]) + /* + * In powerpc, the callchain saved by kernel always includes + * first three entries as the NIP (next instruction pointer), + * LR (link register), and the contents of LR save area in the + * second stack frame. In certain scenarios its possible to have + * invalid kernel instruction addresses in either LR or the second + * stack frame's LR. In that case, kernel will store that address as + * zero. + * + * The below check will continue to look into callstack, + * incase first or second callstack index entry has 0 + * address for powerpc. + */ + if (!callstack || (!callstack[i] && (strcmp(arch, "powerpc") || + (i != 1 && i != 2)))) break; ip = callstack[i]; From 2819f23ac12ce93ff79ca7a54597df9a4a1f6331 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Thu, 5 Oct 2023 09:13:48 -0400 Subject: [PATCH 187/475] eventfs: Use eventfs_remove_events_dir() The update to removing the eventfs_file changed the way the events top level directory was handled. Instead of returning a dentry, it now returns the eventfs_inode. In this changed, the removing of the events top level directory is not much different than removing any of the other directories. Because of this, the removal just called eventfs_remove_dir() instead of eventfs_remove_events_dir(). Although eventfs_remove_dir() does the clean up, it misses out on the dget() of the ei->dentry done in eventfs_create_events_dir(). It makes more sense to match eventfs_create_events_dir() with a specific function eventfs_remove_events_dir() and this specific function can then perform the dput() to the dentry that had the dget() when it was created. Fixes: 5790b1fb3d67 ("eventfs: Remove eventfs_file and just use eventfs_inode") Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202310051743.y9EobbUr-lkp@intel.com/ Signed-off-by: Steven Rostedt (Google) --- fs/tracefs/event_inode.c | 19 +++++++------------ include/linux/tracefs.h | 1 + kernel/trace/trace_events.c | 2 +- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/fs/tracefs/event_inode.c b/fs/tracefs/event_inode.c index eab18b157ef5..1ccd100bc565 100644 --- a/fs/tracefs/event_inode.c +++ b/fs/tracefs/event_inode.c @@ -901,22 +901,17 @@ void eventfs_remove_dir(struct eventfs_inode *ei) } /** - * eventfs_remove_events_dir - remove eventfs dir or file from list - * @dentry: events's dentry to be removed. + * eventfs_remove_events_dir - remove the top level eventfs directory + * @ei: the event_inode returned by eventfs_create_events_dir(). * - * This function remove events main directory + * This function removes the events main directory */ -void eventfs_remove_events_dir(struct dentry *dentry) +void eventfs_remove_events_dir(struct eventfs_inode *ei) { - struct tracefs_inode *ti; + struct dentry *dentry = ei->dentry; - if (!dentry || !dentry->d_inode) - return; + eventfs_remove_dir(ei); - ti = get_tracefs(dentry->d_inode); - if (!ti || !(ti->flags & TRACEFS_EVENT_INODE)) - return; - - d_invalidate(dentry); + /* Matches the dget() from eventfs_create_events_dir() */ dput(dentry); } diff --git a/include/linux/tracefs.h b/include/linux/tracefs.h index 0c39704455d9..13359b1a35d1 100644 --- a/include/linux/tracefs.h +++ b/include/linux/tracefs.h @@ -41,6 +41,7 @@ struct eventfs_inode *eventfs_create_dir(const char *name, struct eventfs_inode const struct eventfs_entry *entries, int size, void *data); +void eventfs_remove_events_dir(struct eventfs_inode *ei); void eventfs_remove_dir(struct eventfs_inode *ei); struct dentry *tracefs_create_file(const char *name, umode_t mode, diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index a3b9d9423824..0e3a1c70e410 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c @@ -3872,7 +3872,7 @@ int event_trace_del_tracer(struct trace_array *tr) down_write(&trace_event_sem); __trace_remove_event_dirs(tr); - eventfs_remove_dir(tr->event_dir); + eventfs_remove_events_dir(tr->event_dir); up_write(&trace_event_sem); tr->event_dir = NULL; From 5ddd8baa4857709b4e5d84b376d735152851955b Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Thu, 5 Oct 2023 10:47:45 -0400 Subject: [PATCH 188/475] tracing: Make system_callback() function static The system_callback() function in trace_events.c is only used within that file. The "static" annotation was missed. Fixes: 5790b1fb3d672 ("eventfs: Remove eventfs_file and just use eventfs_inode") Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202310051743.y9EobbUr-lkp@intel.com/ Signed-off-by: Steven Rostedt (Google) --- kernel/trace/trace_events.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index 0e3a1c70e410..db46d2116500 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c @@ -2282,7 +2282,7 @@ create_new_subsystem(const char *name) return NULL; } -int system_callback(const char *name, umode_t *mode, void **data, +static int system_callback(const char *name, umode_t *mode, void **data, const struct file_operations **fops) { if (strcmp(name, "filter") == 0) From c7e0d9bb9154c6e6b2ac8746faba27b53393f25e Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Fri, 22 Sep 2023 14:33:13 +0200 Subject: [PATCH 189/475] powerpc: Only define __parse_fpscr() when required Clang 17 reports: arch/powerpc/kernel/traps.c:1167:19: error: unused function '__parse_fpscr' [-Werror,-Wunused-function] __parse_fpscr() is called from two sites. First call is guarded by #ifdef CONFIG_PPC_FPU_REGS Second call is guarded by CONFIG_MATH_EMULATION which selects CONFIG_PPC_FPU_REGS. So only define __parse_fpscr() when CONFIG_PPC_FPU_REGS is defined. Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202309210327.WkqSd5Bq-lkp@intel.com/ Fixes: b6254ced4da6 ("powerpc/signal: Don't manage floating point regs when no FPU") Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://msgid.link/5de2998c57f3983563b27b39228ea9a7229d4110.1695385984.git.christophe.leroy@csgroup.eu --- arch/powerpc/kernel/traps.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c index eeff136b83d9..a52c786c2aec 100644 --- a/arch/powerpc/kernel/traps.c +++ b/arch/powerpc/kernel/traps.c @@ -1164,6 +1164,7 @@ void emulate_single_step(struct pt_regs *regs) __single_step_exception(regs); } +#ifdef CONFIG_PPC_FPU_REGS static inline int __parse_fpscr(unsigned long fpscr) { int ret = FPE_FLTUNK; @@ -1190,6 +1191,7 @@ static inline int __parse_fpscr(unsigned long fpscr) return ret; } +#endif static void parse_fpe(struct pt_regs *regs) { From 29fda1ad2a64a62e1c51d61207396e2de1c67362 Mon Sep 17 00:00:00 2001 From: Petr Mladek Date: Fri, 6 Oct 2023 10:21:51 +0200 Subject: [PATCH 190/475] printk: Reduce pr_flush() pooling time pr_flush() does not guarantee that all messages would really get flushed to the console. The best it could do is to wait with a given timeout.[*] The current interval 100ms for checking the progress might seem too long in some situations. For example, such delays are not appreciated during suspend and resume especially when the consoles have been flushed "long" time before the check. On the other hand, the sleeping wait might be useful in other situations. Especially, it would allow flushing the messages using printk kthreads on the same CPU[*]. Use msleep(1) as a compromise. Also measure the time using jiffies. msleep() does not guarantee precise wakeup after the given delay. It might be much longer, especially for times < 20s. See Documentation/timers/timers-howto.rst for more details. Note that msecs_to_jiffies() already translates a negative value into an infinite timeout. [*] console_unlock() does not guarantee flushing the consoles since the commit dbdda842fe96f893 ("printk: Add console owner and waiter logic to load balance console writes"). It would be possible to guarantee it another way. For example, the spinning might be enabled only when the console_lock has been taken via console_trylock(). But the load balancing is helpful. And more importantly, the flush with a timeout has been added as a preparation step for introducing printk kthreads. Signed-off-by: Petr Mladek Reviewed-by: John Ogness Link: https://lore.kernel.org/r/20231006082151.6969-3-pmladek@suse.com --- kernel/printk/printk.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 1919899b6897..80eb79177bab 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -3726,7 +3726,8 @@ late_initcall(printk_late_init); /* If @con is specified, only wait for that console. Otherwise wait for all. */ static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progress) { - int remaining = timeout_ms; + unsigned long timeout_jiffies = msecs_to_jiffies(timeout_ms); + unsigned long remaining_jiffies = timeout_jiffies; struct console *c; u64 last_diff = 0; u64 printk_seq; @@ -3743,6 +3744,9 @@ static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progre console_unlock(); for (;;) { + unsigned long begin_jiffies; + unsigned long slept_jiffies; + diff = 0; /* @@ -3771,24 +3775,20 @@ static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progre console_srcu_read_unlock(cookie); if (diff != last_diff && reset_on_progress) - remaining = timeout_ms; + remaining_jiffies = timeout_jiffies; console_unlock(); /* Note: @diff is 0 if there are no usable consoles. */ - if (diff == 0 || remaining == 0) + if (diff == 0 || remaining_jiffies == 0) break; - if (remaining < 0) { - /* no timeout limit */ - msleep(100); - } else if (remaining < 100) { - msleep(remaining); - remaining = 0; - } else { - msleep(100); - remaining -= 100; - } + /* msleep(1) might sleep much longer. Check time by jiffies. */ + begin_jiffies = jiffies; + msleep(1); + slept_jiffies = jiffies - begin_jiffies; + + remaining_jiffies -= min(slept_jiffies, remaining_jiffies); last_diff = diff; } From 03ff4c6b3e41b3530c6c88cc91a7af599322eb89 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Thu, 7 Sep 2023 14:05:33 -0700 Subject: [PATCH 191/475] perf parse-events: Avoid erange from hex numbers We specify that a "num_hex" comprises 1 or more digits, however, that allows strtoull to fail with ERANGE. Limit the number of hex digits to being between 1 and 16. Before: ``` $ perf stat -e 'cpu/rE7574c47490475745/' true perf: util/parse-events.c:215: fix_raw: Assertion `errno == 0' failed. Aborted (core dumped) ``` After: ``` $ perf stat -e 'cpu/rE7574c47490475745/' true event syntax error: 'cpu/rE7574c47490475745/' \___ Bad event or PMU Unable to find PMU or event on a PMU of 'cpu' Initial error: event syntax error: 'cpu/rE7574c47490475745/' \___ unknown term 'rE7574c47490475745' for pmu 'cpu' valid terms: event,pc,edge,offcore_rsp,ldlat,inv,umask,frontend,cmask,config,config1,config2,config3,name,period,percore,metric-id Run 'perf list' for a list of valid events Usage: perf stat [] [] -e, --event event selector. use 'perf list' to list available events ``` Issue found through fuzz testing. Signed-off-by: Ian Rogers Acked-by: Namhyung Kim Link: https://lore.kernel.org/r/20230907210533.3712979-1-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/util/parse-events.l | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/perf/util/parse-events.l b/tools/perf/util/parse-events.l index 7bdf0565a92c..e86c45675e1d 100644 --- a/tools/perf/util/parse-events.l +++ b/tools/perf/util/parse-events.l @@ -156,8 +156,8 @@ event_pmu [^,{}/]+[/][^/]*[/][^,{}/]* event [^,{}/]+ num_dec [0-9]+ -num_hex 0x[a-fA-F0-9]+ -num_raw_hex [a-fA-F0-9]+ +num_hex 0x[a-fA-F0-9]{1,16} +num_raw_hex [a-fA-F0-9]{1,16} name [a-zA-Z_*?\[\]][a-zA-Z0-9_*?.\[\]!\-]* name_tag [\'][a-zA-Z_*?\[\]][a-zA-Z0-9_*?\-,\.\[\]:=]*[\'] name_minus [a-zA-Z_*?][a-zA-Z0-9\-_*?.:]* From 29a2fd7c72b3bd8b67bcee363f8068275a4ef351 Mon Sep 17 00:00:00 2001 From: Arnaldo Carvalho de Melo Date: Thu, 5 Oct 2023 10:29:38 -0300 Subject: [PATCH 192/475] perf symbols: Add 'intel_idle_ibrs' to the list of idle symbols This is a longstanding to do list entry: we need a way to see that a sample took place while in idle state, as the current way to do it is to infer that by the name of the functions that in such state have more samples, IOW: a hack. Maybe we can do flip a bit in samples that take place inside the enter/exit idle section in do_idle()? But till then, add one more :-\ Signed-off-by: Arnaldo Carvalho de Melo Acked-by: Namhyung Kim Cc: Frederic Weisbecker Link: https://lore.kernel.org/r/ZR66Qgbcltt+zG7F@kernel.org Signed-off-by: Namhyung Kim --- tools/perf/util/symbol.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/perf/util/symbol.c b/tools/perf/util/symbol.c index 2740d4457c13..96587fd7a5a2 100644 --- a/tools/perf/util/symbol.c +++ b/tools/perf/util/symbol.c @@ -718,6 +718,7 @@ static bool symbol__is_idle(const char *name) "cpu_startup_entry", "idle_cpu", "intel_idle", + "intel_idle_ibrs", "default_idle", "native_safe_halt", "enter_idle", From c1783ddfb62420c44cdf4672dad2046f056c624b Mon Sep 17 00:00:00 2001 From: Arnaldo Carvalho de Melo Date: Thu, 5 Oct 2023 10:46:26 -0300 Subject: [PATCH 193/475] perf build: Add missing comment about NO_LIBTRACEEVENT=1 By default perf will fail the build if the development files for libtraceevent are not available. To build perf without libtraceevent support, disabling several features such as 'perf trace', one needs to add NO_LIBTRACEVENT=1 to the make command line. Add the missing comments about that to the tools/perf/Makefile.perf file, just like all the other such command line toggles. Fixes: 378ef0f5d9d7f465 ("perf build: Use libtraceevent from the system") Signed-off-by: Arnaldo Carvalho de Melo Reviewed-by: Ian Rogers Link: https://lore.kernel.org/r/ZR6+MhXtLnv6ow6E@kernel.org Signed-off-by: Namhyung Kim --- tools/perf/Makefile.perf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/perf/Makefile.perf b/tools/perf/Makefile.perf index 98604e396ac3..456872ac410d 100644 --- a/tools/perf/Makefile.perf +++ b/tools/perf/Makefile.perf @@ -69,6 +69,10 @@ include ../scripts/utilities.mak # Define NO_LIBDW_DWARF_UNWIND if you do not want libdw support # for dwarf backtrace post unwind. # +# Define NO_LIBTRACEEVENT=1 if you don't want libtraceevent to be linked, +# this will remove multiple features and tools, such as 'perf trace', +# that need it to read tracefs event format files, etc. +# # Define NO_PERF_READ_VDSO32 if you do not want to build perf-read-vdso32 # for reading the 32-bit compatibility VDSO in 64-bit mode # From b20576fd7fe39554b212095c3c0d7a3dff512515 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Wed, 27 Sep 2023 17:44:31 -0700 Subject: [PATCH 194/475] perf parse-events: Fix for term values that are raw events Raw events can be strings like 'r0xead' but the 0x is optional so they can also be 'read'. On IcelakeX uncore_imc_free_running has an event called 'read' which may be programmed as: ``` $ perf stat -e 'uncore_imc_free_running/event=read/' -a sleep 1 ``` However, the PE_RAW type isn't allowed on the right of a term, even though in this case we just want to interpret it as a string. This leads to the following error on IcelakeX: ``` $ perf stat -e 'uncore_imc_free_running/event=read/' -a sleep 1 event syntax error: '..nning/event=read/' \___ parser error Run 'perf list' for a list of valid events Usage: perf stat [] [] -e, --event event selector. use 'perf list' to list available events ``` Fix this by allowing raw types on the right of terms and treat them as strings, just as is already done for PE_LEGACY_CACHE. Make this consistent by just entirely removing name_or_legacy and always using name_or_raw that covers all three cases. Fixes: 6fd1e5191591 ("perf parse-events: Support PMUs for legacy cache events") Signed-off-by: Ian Rogers Cc: James Clark Cc: Kan Liang Link: https://lore.kernel.org/r/20230928004431.1926969-1-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/util/parse-events.y | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tools/perf/util/parse-events.y b/tools/perf/util/parse-events.y index 4d9f3922a33d..de098caf0c1c 100644 --- a/tools/perf/util/parse-events.y +++ b/tools/perf/util/parse-events.y @@ -78,7 +78,7 @@ static void free_list_evsel(struct list_head* list_evsel) %type PE_MODIFIER_BP %type PE_EVENT_NAME %type PE_DRV_CFG_TERM -%type name_or_raw name_or_legacy +%type name_or_raw %destructor { free ($$); } %type event_term %destructor { parse_events_term__delete ($$); } @@ -669,8 +669,6 @@ event_term name_or_raw: PE_RAW | PE_NAME | PE_LEGACY_CACHE -name_or_legacy: PE_NAME | PE_LEGACY_CACHE - event_term: PE_RAW { @@ -685,7 +683,7 @@ PE_RAW $$ = term; } | -name_or_raw '=' name_or_legacy +name_or_raw '=' name_or_raw { struct parse_events_term *term; int err = parse_events_term__str(&term, PARSE_EVENTS__TERM_TYPE_USER, $1, $3, &@1, &@3); @@ -765,7 +763,7 @@ PE_TERM_HW $$ = term; } | -PE_TERM '=' name_or_legacy +PE_TERM '=' name_or_raw { struct parse_events_term *term; int err = parse_events_term__str(&term, $1, /*config=*/NULL, $3, &@1, &@3); From 52c15e7e792857c42b4a926e45228e981c5a5f13 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Mon, 9 Oct 2023 11:39:02 -0700 Subject: [PATCH 195/475] gen_compile_commands: Allow the line prefix to still be cmd_ Builds in tools still use the cmd_ prefix in .cmd files, so don't require the saved part. Name the groups in the line pattern match so that changing the regular expression is more robust and works with the addition of a new match group. Signed-off-by: Ian Rogers Reviewed-by: Nick Desaulniers Cc: Ravi Bangoria Cc: Mark Rutland Cc: Yang Jihong Cc: Peter Zijlstra Cc: Adrian Hunter Cc: Arnaldo Carvalho de Melo Cc: Huacai Chen Cc: Jiri Olsa Cc: Nathan Chancellor Cc: Alexander Shishkin Cc: Kan Liang Cc: llvm@lists.linux.dev Cc: Ming Wang Cc: Ingo Molnar Cc: Tom Rix Cc: bpf@vger.kernel.org Cc: linux-kernel@vger.kernel.org Cc: linux-perf-users@vger.kernel.org Link: https://lore.kernel.org/r/20231009183920.200859-2-irogers@google.com Signed-off-by: Namhyung Kim --- scripts/clang-tools/gen_compile_commands.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/clang-tools/gen_compile_commands.py b/scripts/clang-tools/gen_compile_commands.py index a84cc5737c2c..b43f9149893c 100755 --- a/scripts/clang-tools/gen_compile_commands.py +++ b/scripts/clang-tools/gen_compile_commands.py @@ -19,7 +19,7 @@ _DEFAULT_OUTPUT = 'compile_commands.json' _DEFAULT_LOG_LEVEL = 'WARNING' _FILENAME_PATTERN = r'^\..*\.cmd$' -_LINE_PATTERN = r'^savedcmd_[^ ]*\.o := (.* )([^ ]*\.[cS]) *(;|$)' +_LINE_PATTERN = r'^(saved)?cmd_[^ ]*\.o := (?P.* )(?P[^ ]*\.[cS]) *(;|$)' _VALID_LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] # The tools/ directory adopts a different build system, and produces .cmd # files in a different format. Do not support it. @@ -213,8 +213,8 @@ def main(): result = line_matcher.match(f.readline()) if result: try: - entry = process_line(directory, result.group(1), - result.group(2)) + entry = process_line(directory, result.group('command_prefix'), + result.group('file_path')) compile_commands.append(entry) except ValueError as err: logging.info('Could not add line from %s: %s', From 9e56d3be4bfd2ec6433a7c44195bd1e687b8ed2e Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Mon, 9 Oct 2023 11:39:03 -0700 Subject: [PATCH 196/475] gen_compile_commands: Sort output compile commands by file name Make the output more stable and deterministic. Signed-off-by: Ian Rogers Reviewed-by: Nick Desaulniers Cc: Ravi Bangoria Cc: Mark Rutland Cc: Yang Jihong Cc: Peter Zijlstra Cc: Adrian Hunter Cc: Arnaldo Carvalho de Melo Cc: Huacai Chen Cc: Jiri Olsa Cc: Nathan Chancellor Cc: Alexander Shishkin Cc: Kan Liang Cc: llvm@lists.linux.dev Cc: Ming Wang Cc: Ingo Molnar Cc: Tom Rix Cc: bpf@vger.kernel.org Cc: linux-kernel@vger.kernel.org Cc: linux-perf-users@vger.kernel.org Link: https://lore.kernel.org/r/20231009183920.200859-3-irogers@google.com Signed-off-by: Namhyung Kim --- scripts/clang-tools/gen_compile_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/clang-tools/gen_compile_commands.py b/scripts/clang-tools/gen_compile_commands.py index b43f9149893c..180952fb91c1 100755 --- a/scripts/clang-tools/gen_compile_commands.py +++ b/scripts/clang-tools/gen_compile_commands.py @@ -221,7 +221,7 @@ def main(): cmdfile, err) with open(output, 'wt') as f: - json.dump(compile_commands, f, indent=2, sort_keys=True) + json.dump(sorted(compile_commands, key=lambda x: x["file"]), f, indent=2, sort_keys=True) if __name__ == '__main__': From b24520ffa9695c7249bdd181cf10bc0a3e365019 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Mon, 9 Oct 2023 11:39:04 -0700 Subject: [PATCH 197/475] run-clang-tools: Add pass through checks and and header-filter arguments Add a -checks argument to allow the checks passed to the clang-tool to be set on the command line. Add a pass through -header-filter option. Don't run analysis on non-C or CPP files. Signed-off-by: Ian Rogers Reviewed-by: Nick Desaulniers Cc: Ravi Bangoria Cc: Mark Rutland Cc: Yang Jihong Cc: Peter Zijlstra Cc: Adrian Hunter Cc: Arnaldo Carvalho de Melo Cc: Huacai Chen Cc: Jiri Olsa Cc: Nathan Chancellor Cc: Alexander Shishkin Cc: Kan Liang Cc: llvm@lists.linux.dev Cc: Ming Wang Cc: Ingo Molnar Cc: Tom Rix Cc: bpf@vger.kernel.org Cc: linux-kernel@vger.kernel.org Cc: linux-perf-users@vger.kernel.org Link: https://lore.kernel.org/r/20231009183920.200859-4-irogers@google.com Signed-off-by: Namhyung Kim --- scripts/clang-tools/run-clang-tools.py | 32 ++++++++++++++++++++------ 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/scripts/clang-tools/run-clang-tools.py b/scripts/clang-tools/run-clang-tools.py index 3266708a8658..f31ffd09e1ea 100755 --- a/scripts/clang-tools/run-clang-tools.py +++ b/scripts/clang-tools/run-clang-tools.py @@ -33,6 +33,11 @@ def parse_arguments(): path_help = "Path to the compilation database to parse" parser.add_argument("path", type=str, help=path_help) + checks_help = "Checks to pass to the analysis" + parser.add_argument("-checks", type=str, default=None, help=checks_help) + header_filter_help = "Pass the -header-filter value to the tool" + parser.add_argument("-header-filter", type=str, default=None, help=header_filter_help) + return parser.parse_args() @@ -45,14 +50,27 @@ def init(l, a): def run_analysis(entry): # Disable all checks, then re-enable the ones we want - checks = [] - checks.append("-checks=-*") - if args.type == "clang-tidy": - checks.append("linuxkernel-*") + global args + checks = None + if args.checks: + checks = args.checks.split(',') else: - checks.append("clang-analyzer-*") - checks.append("-clang-analyzer-security.insecureAPI.DeprecatedOrUnsafeBufferHandling") - p = subprocess.run(["clang-tidy", "-p", args.path, ",".join(checks), entry["file"]], + checks = ["-*"] + if args.type == "clang-tidy": + checks.append("linuxkernel-*") + else: + checks.append("clang-analyzer-*") + checks.append("-clang-analyzer-security.insecureAPI.DeprecatedOrUnsafeBufferHandling") + file = entry["file"] + if not file.endswith(".c") and not file.endswith(".cpp"): + with lock: + print(f"Skipping non-C file: '{file}'", file=sys.stderr) + return + pargs = ["clang-tidy", "-p", args.path, "-checks=" + ",".join(checks)] + if args.header_filter: + pargs.append("-header-filter=" + args.header_filter) + pargs.append(file) + p = subprocess.run(pargs, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=entry["directory"]) From da0c884b0756fc97a1c15f4cdb81e0a8490a6d7e Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Mon, 9 Oct 2023 11:39:06 -0700 Subject: [PATCH 198/475] perf bench uprobe: Fix potential use of memory after free Found by clang-tidy: ``` bench/uprobe.c:98:3: warning: Use of memory after it is freed [clang-analyzer-unix.Malloc] bench_uprobe_bpf__destroy(skel); ``` Signed-off-by: Ian Rogers Acked-by: Namhyung Kim Cc: Ravi Bangoria Cc: Nick Desaulniers Cc: Yang Jihong Cc: Huacai Chen Cc: Nathan Chancellor Cc: Kan Liang Cc: llvm@lists.linux.dev Cc: Ming Wang Cc: Tom Rix Cc: bpf@vger.kernel.org Link: https://lore.kernel.org/r/20231009183920.200859-6-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/bench/uprobe.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/perf/bench/uprobe.c b/tools/perf/bench/uprobe.c index 914c0817fe8a..5c71fdc419dd 100644 --- a/tools/perf/bench/uprobe.c +++ b/tools/perf/bench/uprobe.c @@ -89,6 +89,7 @@ static int bench_uprobe__setup_bpf_skel(enum bench_uprobe bench) return err; cleanup: bench_uprobe_bpf__destroy(skel); + skel = NULL; return err; } From 319d459898ce507dba58c28c17610314d16b7beb Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Mon, 9 Oct 2023 11:39:07 -0700 Subject: [PATCH 199/475] perf buildid-cache: Fix use of uninitialized value The buildid filename is first determined and then from this the buildid read. If getting the filename fails then the buildid will be used for a later memcmp uninitialized. Detected by clang-tidy. Signed-off-by: Ian Rogers Acked-by: Namhyung Kim Cc: Ravi Bangoria Cc: Nick Desaulniers Cc: Yang Jihong Cc: Huacai Chen Cc: Nathan Chancellor Cc: Kan Liang Cc: llvm@lists.linux.dev Cc: Ming Wang Cc: Tom Rix Cc: bpf@vger.kernel.org Link: https://lore.kernel.org/r/20231009183920.200859-7-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/builtin-buildid-cache.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/perf/builtin-buildid-cache.c b/tools/perf/builtin-buildid-cache.c index cd381693658b..e2a40f1d9225 100644 --- a/tools/perf/builtin-buildid-cache.c +++ b/tools/perf/builtin-buildid-cache.c @@ -277,8 +277,10 @@ static bool dso__missing_buildid_cache(struct dso *dso, int parm __maybe_unused) char filename[PATH_MAX]; struct build_id bid; - if (dso__build_id_filename(dso, filename, sizeof(filename), false) && - filename__read_build_id(filename, &bid) == -1) { + if (!dso__build_id_filename(dso, filename, sizeof(filename), false)) + return true; + + if (filename__read_build_id(filename, &bid) == -1) { if (errno == ENOENT) return false; From e2372136700d460276ca7ff07da523e8f61b69c7 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Mon, 9 Oct 2023 11:39:08 -0700 Subject: [PATCH 200/475] perf env: Remove unnecessary NULL tests clang-tidy was warning: ``` util/env.c:334:23: warning: Access to field 'nr_pmu_mappings' results in a dereference of a null pointer (loaded from variable 'env') [clang-analyzer-core.NullDereference] env->nr_pmu_mappings = pmu_num; ``` As functions are called potentially when !env was true. This condition could never be true as it would produce a segv, so remove the unnecessary NULL tests and silence clang-tidy. Signed-off-by: Ian Rogers Acked-by: Namhyung Kim Cc: Ravi Bangoria Cc: Nick Desaulniers Cc: Yang Jihong Cc: Huacai Chen Cc: Nathan Chancellor Cc: Kan Liang Cc: llvm@lists.linux.dev Cc: Ming Wang Cc: Tom Rix Cc: bpf@vger.kernel.org Link: https://lore.kernel.org/r/20231009183920.200859-8-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/util/env.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/perf/util/env.c b/tools/perf/util/env.c index a164164001fb..44140b7f596a 100644 --- a/tools/perf/util/env.c +++ b/tools/perf/util/env.c @@ -457,7 +457,7 @@ const char *perf_env__cpuid(struct perf_env *env) { int status; - if (!env || !env->cpuid) { /* Assume local operation */ + if (!env->cpuid) { /* Assume local operation */ status = perf_env__read_cpuid(env); if (status) return NULL; @@ -470,7 +470,7 @@ int perf_env__nr_pmu_mappings(struct perf_env *env) { int status; - if (!env || !env->nr_pmu_mappings) { /* Assume local operation */ + if (!env->nr_pmu_mappings) { /* Assume local operation */ status = perf_env__read_pmu_mappings(env); if (status) return 0; @@ -483,7 +483,7 @@ const char *perf_env__pmu_mappings(struct perf_env *env) { int status; - if (!env || !env->pmu_mappings) { /* Assume local operation */ + if (!env->pmu_mappings) { /* Assume local operation */ status = perf_env__read_pmu_mappings(env); if (status) return NULL; From b3aa09ee78defd3d2e5f7debb5279f8a92b69749 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Mon, 9 Oct 2023 11:39:09 -0700 Subject: [PATCH 201/475] perf jitdump: Avoid memory leak jit_repipe_unwinding_info is called in a loop by jit_process_dump, avoid leaking unwinding_data by free-ing before overwriting. Error detected by clang-tidy. Signed-off-by: Ian Rogers Acked-by: Namhyung Kim Cc: Ravi Bangoria Cc: Nick Desaulniers Cc: Yang Jihong Cc: Huacai Chen Cc: Nathan Chancellor Cc: Kan Liang Cc: llvm@lists.linux.dev Cc: Ming Wang Cc: Tom Rix Cc: bpf@vger.kernel.org Link: https://lore.kernel.org/r/20231009183920.200859-9-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/util/jitdump.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/perf/util/jitdump.c b/tools/perf/util/jitdump.c index 6b2b96c16ccd..1f657ef8975f 100644 --- a/tools/perf/util/jitdump.c +++ b/tools/perf/util/jitdump.c @@ -675,6 +675,7 @@ jit_repipe_unwinding_info(struct jit_buf_desc *jd, union jr_entry *jr) jd->eh_frame_hdr_size = jr->unwinding.eh_frame_hdr_size; jd->unwinding_size = jr->unwinding.unwinding_size; jd->unwinding_mapped_size = jr->unwinding.mapped_size; + free(jd->unwinding_data); jd->unwinding_data = unwinding_data; return 0; From 85f73c377b2ac9988a204b119aebb33ca5c60083 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Mon, 9 Oct 2023 11:39:10 -0700 Subject: [PATCH 202/475] perf mem-events: Avoid uninitialized read pmu should be initialized to NULL before perf_pmus__scan loop. Fix and shrink the scope of pmu at the same time. Issue detected by clang-tidy. Fixes: 5752c20f3787 ("perf mem: Scan all PMUs instead of just core ones") Signed-off-by: Ian Rogers Acked-by: Namhyung Kim Cc: Ravi Bangoria Cc: Nick Desaulniers Cc: Yang Jihong Cc: Huacai Chen Cc: Nathan Chancellor Cc: Kan Liang Cc: llvm@lists.linux.dev Cc: Ming Wang Cc: Tom Rix Cc: bpf@vger.kernel.org Link: https://lore.kernel.org/r/20231009183920.200859-10-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/util/mem-events.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/perf/util/mem-events.c b/tools/perf/util/mem-events.c index 39ffe8ceb380..954b235e12e5 100644 --- a/tools/perf/util/mem-events.c +++ b/tools/perf/util/mem-events.c @@ -185,7 +185,6 @@ int perf_mem_events__record_args(const char **rec_argv, int *argv_nr, { int i = *argv_nr, k = 0; struct perf_mem_event *e; - struct perf_pmu *pmu; for (int j = 0; j < PERF_MEM_EVENTS__MAX; j++) { e = perf_mem_events__ptr(j); @@ -202,6 +201,8 @@ int perf_mem_events__record_args(const char **rec_argv, int *argv_nr, rec_argv[i++] = "-e"; rec_argv[i++] = perf_mem_events__name(j, NULL); } else { + struct perf_pmu *pmu = NULL; + if (!e->supported) { perf_mem_events__print_unsupport_hybrid(e, j); return -1; From 52a5ad12f2147506899ee83e680ea2a1d763adeb Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Mon, 9 Oct 2023 11:39:11 -0700 Subject: [PATCH 203/475] perf dlfilter: Be defensive against potential NULL dereference In the unlikely case of having a symbol without a mapping, avoid a NULL dereference that clang-tidy warns about. Signed-off-by: Ian Rogers Acked-by: Namhyung Kim Cc: Ravi Bangoria Cc: Nick Desaulniers Cc: Yang Jihong Cc: Huacai Chen Cc: Nathan Chancellor Cc: Kan Liang Cc: llvm@lists.linux.dev Cc: Ming Wang Cc: Tom Rix Cc: bpf@vger.kernel.org Link: https://lore.kernel.org/r/20231009183920.200859-11-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/util/dlfilter.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/perf/util/dlfilter.c b/tools/perf/util/dlfilter.c index 1dbf27822ee2..5e54832137a9 100644 --- a/tools/perf/util/dlfilter.c +++ b/tools/perf/util/dlfilter.c @@ -52,8 +52,10 @@ static void al_to_d_al(struct addr_location *al, struct perf_dlfilter_al *d_al) d_al->sym_end = sym->end; if (al->addr < sym->end) d_al->symoff = al->addr - sym->start; - else + else if (al->map) d_al->symoff = al->addr - map__start(al->map) - sym->start; + else + d_al->symoff = 0; d_al->sym_binding = sym->binding; } else { d_al->sym = NULL; From 51e9ea99f9bfe6a9c283d041dbc2c1c0be9c3a0f Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Mon, 9 Oct 2023 11:39:12 -0700 Subject: [PATCH 204/475] perf hists browser: Reorder variables to reduce padding Address clang-tidy warning: ``` tools/perf/ui/browsers/hists.c:2416:8: warning: Excessive padding in 'struct popup_action' (8 padding bytes, where 0 is optimal). Optimal fields order: time, thread, evsel, fn, ms, socket, rstype, ``` Signed-off-by: Ian Rogers Acked-by: Namhyung Kim Cc: Ravi Bangoria Cc: Nick Desaulniers Cc: Yang Jihong Cc: Huacai Chen Cc: Nathan Chancellor Cc: Kan Liang Cc: llvm@lists.linux.dev Cc: Ming Wang Cc: Tom Rix Cc: bpf@vger.kernel.org Link: https://lore.kernel.org/r/20231009183920.200859-12-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/ui/browsers/hists.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/perf/ui/browsers/hists.c b/tools/perf/ui/browsers/hists.c index 70db5a717905..f02ee605bbce 100644 --- a/tools/perf/ui/browsers/hists.c +++ b/tools/perf/ui/browsers/hists.c @@ -2416,12 +2416,12 @@ close_file_and_continue: struct popup_action { unsigned long time; struct thread *thread; + struct evsel *evsel; + int (*fn)(struct hist_browser *browser, struct popup_action *act); struct map_symbol ms; int socket; - struct evsel *evsel; enum rstype rstype; - int (*fn)(struct hist_browser *browser, struct popup_action *act); }; static int From ef1aec6000a7364a6c1ef0a6f9cbf8b98b685255 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Mon, 9 Oct 2023 11:39:13 -0700 Subject: [PATCH 205/475] perf hists browser: Avoid potential NULL dereference On other code paths browser->he_selection is NULL checked, add a missing case reported by clang-tidy. Signed-off-by: Ian Rogers Acked-by: Namhyung Kim Cc: Ravi Bangoria Cc: Nick Desaulniers Cc: Yang Jihong Cc: Huacai Chen Cc: Nathan Chancellor Cc: Kan Liang Cc: llvm@lists.linux.dev Cc: Ming Wang Cc: Tom Rix Cc: bpf@vger.kernel.org Link: https://lore.kernel.org/r/20231009183920.200859-13-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/ui/browsers/hists.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/perf/ui/browsers/hists.c b/tools/perf/ui/browsers/hists.c index f02ee605bbce..f4812b226818 100644 --- a/tools/perf/ui/browsers/hists.c +++ b/tools/perf/ui/browsers/hists.c @@ -3302,7 +3302,7 @@ do_hotkey: // key came straight from options ui__popup_menu() &options[nr_options], &bi->to.ms, bi->to.al_addr); - } else { + } else if (browser->he_selection) { nr_options += add_annotate_opt(browser, &actions[nr_options], &options[nr_options], From 63d471979e49148e59eae0b33a57c12d535e20c6 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Mon, 9 Oct 2023 11:39:14 -0700 Subject: [PATCH 206/475] perf svghelper: Avoid memory leak On success path the sib_core and sib_thr values weren't being freed. Detected by clang-tidy. Signed-off-by: Ian Rogers Acked-by: Namhyung Kim Cc: Ravi Bangoria Cc: Nick Desaulniers Cc: Yang Jihong Cc: Huacai Chen Cc: Nathan Chancellor Cc: Kan Liang Cc: llvm@lists.linux.dev Cc: Ming Wang Cc: Tom Rix Cc: bpf@vger.kernel.org Link: https://lore.kernel.org/r/20231009183920.200859-14-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/util/svghelper.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/perf/util/svghelper.c b/tools/perf/util/svghelper.c index 0e4dc31c6c9c..1892e9b6aa7f 100644 --- a/tools/perf/util/svghelper.c +++ b/tools/perf/util/svghelper.c @@ -754,6 +754,7 @@ int svg_build_topology_map(struct perf_env *env) int i, nr_cpus; struct topology t; char *sib_core, *sib_thr; + int ret = -1; nr_cpus = min(env->nr_cpus_online, MAX_NR_CPUS); @@ -799,11 +800,11 @@ int svg_build_topology_map(struct perf_env *env) scan_core_topology(topology_map, &t, nr_cpus); - return 0; + ret = 0; exit: zfree(&t.sib_core); zfree(&t.sib_thr); - return -1; + return ret; } From 1370406d35b45457aae295a6cfce9fae2bbd785e Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Mon, 9 Oct 2023 11:39:15 -0700 Subject: [PATCH 207/475] perf lock: Fix a memory leak on an error path If a memory allocation fails then the strdup-ed string needs freeing. Detected by clang-tidy. Signed-off-by: Ian Rogers Acked-by: Namhyung Kim Cc: Ravi Bangoria Cc: Nick Desaulniers Cc: Yang Jihong Cc: Huacai Chen Cc: Nathan Chancellor Cc: Kan Liang Cc: llvm@lists.linux.dev Cc: Ming Wang Cc: Tom Rix Cc: bpf@vger.kernel.org Link: https://lore.kernel.org/r/20231009183920.200859-15-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/builtin-lock.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/perf/builtin-lock.c b/tools/perf/builtin-lock.c index fa7419978353..a3ff2f4edbaa 100644 --- a/tools/perf/builtin-lock.c +++ b/tools/perf/builtin-lock.c @@ -2478,6 +2478,7 @@ static int parse_call_stack(const struct option *opt __maybe_unused, const char entry = malloc(sizeof(*entry) + strlen(tok) + 1); if (entry == NULL) { pr_err("Memory allocation failure\n"); + free(s); return -1; } From 7875c72c8b0566590c888a2420d7e8fc12f67154 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Mon, 9 Oct 2023 11:39:16 -0700 Subject: [PATCH 208/475] perf parse-events: Fix unlikely memory leak when cloning terms Add missing free on an error path as detected by clang-tidy. Signed-off-by: Ian Rogers Acked-by: Namhyung Kim Cc: Ravi Bangoria Cc: Nick Desaulniers Cc: Yang Jihong Cc: Huacai Chen Cc: Nathan Chancellor Cc: Kan Liang Cc: llvm@lists.linux.dev Cc: Ming Wang Cc: Tom Rix Cc: bpf@vger.kernel.org Link: https://lore.kernel.org/r/20231009183920.200859-16-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/util/parse-events.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/perf/util/parse-events.c b/tools/perf/util/parse-events.c index c56e07bd7dd6..23c027cf20ae 100644 --- a/tools/perf/util/parse-events.c +++ b/tools/perf/util/parse-events.c @@ -2549,8 +2549,10 @@ int parse_events_term__clone(struct parse_events_term **new, return new_term(new, &temp, /*str=*/NULL, term->val.num); str = strdup(term->val.str); - if (!str) + if (!str) { + zfree(&temp.config); return -ENOMEM; + } return new_term(new, &temp, str, /*num=*/0); } From c4b5140c6eac2f757d9706c6c783b60554c48cb7 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Mon, 9 Oct 2023 11:39:17 -0700 Subject: [PATCH 209/475] tools api: Avoid potential double free io__getline will free the line on error but it doesn't clear the out argument. This may lead to the line being freed twice, like in tools/perf/util/srcline.c as detected by clang-tidy. Signed-off-by: Ian Rogers Acked-by: Namhyung Kim Cc: Ravi Bangoria Cc: Nick Desaulniers Cc: Yang Jihong Cc: Huacai Chen Cc: Nathan Chancellor Cc: Kan Liang Cc: llvm@lists.linux.dev Cc: Ming Wang Cc: Tom Rix Cc: bpf@vger.kernel.org Link: https://lore.kernel.org/r/20231009183920.200859-17-irogers@google.com Signed-off-by: Namhyung Kim --- tools/lib/api/io.h | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/lib/api/io.h b/tools/lib/api/io.h index 9fc429d2852d..a77b74c5fb65 100644 --- a/tools/lib/api/io.h +++ b/tools/lib/api/io.h @@ -180,6 +180,7 @@ static inline ssize_t io__getline(struct io *io, char **line_out, size_t *line_l return line_len; err_out: free(line); + *line_out = NULL; return -ENOMEM; } From 97fe038374bdf43fd025ac0e7aebf8bfbdd6d54f Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Mon, 9 Oct 2023 11:39:18 -0700 Subject: [PATCH 210/475] perf trace-event-info: Avoid passing NULL value to closedir If opendir failed then closedir was passed NULL which is erroneous. Caught by clang-tidy. Signed-off-by: Ian Rogers Acked-by: Namhyung Kim Cc: Ravi Bangoria Cc: Nick Desaulniers Cc: Yang Jihong Cc: Huacai Chen Cc: Nathan Chancellor Cc: Kan Liang Cc: llvm@lists.linux.dev Cc: Ming Wang Cc: Tom Rix Cc: bpf@vger.kernel.org Link: https://lore.kernel.org/r/20231009183920.200859-18-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/util/trace-event-info.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/perf/util/trace-event-info.c b/tools/perf/util/trace-event-info.c index 319ccf09a435..c8755679281e 100644 --- a/tools/perf/util/trace-event-info.c +++ b/tools/perf/util/trace-event-info.c @@ -313,7 +313,8 @@ static int record_event_files(struct tracepoint_path *tps) } err = 0; out: - closedir(dir); + if (dir) + closedir(dir); put_tracing_file(path); return err; From 105254501770c8952e50c71618fca6a8b63890f1 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Mon, 9 Oct 2023 11:39:19 -0700 Subject: [PATCH 211/475] perf header: Fix various error path memory leaks Memory leaks were detected by clang-tidy. Signed-off-by: Ian Rogers Acked-by: Namhyung Kim Cc: Ravi Bangoria Cc: Nick Desaulniers Cc: Yang Jihong Cc: Huacai Chen Cc: Nathan Chancellor Cc: Kan Liang Cc: llvm@lists.linux.dev Cc: Ming Wang Cc: Tom Rix Cc: bpf@vger.kernel.org Link: https://lore.kernel.org/r/20231009183920.200859-19-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/util/header.c | 60 +++++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/tools/perf/util/header.c b/tools/perf/util/header.c index d812e1e371a7..e86b9439ffee 100644 --- a/tools/perf/util/header.c +++ b/tools/perf/util/header.c @@ -2573,7 +2573,7 @@ error: static int process_cpu_topology(struct feat_fd *ff, void *data __maybe_unused) { u32 nr, i; - char *str; + char *str = NULL; struct strbuf sb; int cpu_nr = ff->ph->env.nr_cpus_avail; u64 size = 0; @@ -2601,7 +2601,7 @@ static int process_cpu_topology(struct feat_fd *ff, void *data __maybe_unused) if (strbuf_add(&sb, str, strlen(str) + 1) < 0) goto error; size += string_size(str); - free(str); + zfree(&str); } ph->env.sibling_cores = strbuf_detach(&sb, NULL); @@ -2620,7 +2620,7 @@ static int process_cpu_topology(struct feat_fd *ff, void *data __maybe_unused) if (strbuf_add(&sb, str, strlen(str) + 1) < 0) goto error; size += string_size(str); - free(str); + zfree(&str); } ph->env.sibling_threads = strbuf_detach(&sb, NULL); @@ -2684,7 +2684,7 @@ static int process_cpu_topology(struct feat_fd *ff, void *data __maybe_unused) if (strbuf_add(&sb, str, strlen(str) + 1) < 0) goto error; size += string_size(str); - free(str); + zfree(&str); } ph->env.sibling_dies = strbuf_detach(&sb, NULL); @@ -2699,6 +2699,7 @@ static int process_cpu_topology(struct feat_fd *ff, void *data __maybe_unused) error: strbuf_release(&sb); + zfree(&str); free_cpu: zfree(&ph->env.cpu); return -1; @@ -2736,10 +2737,9 @@ static int process_numa_topology(struct feat_fd *ff, void *data __maybe_unused) goto error; n->map = perf_cpu_map__new(str); + free(str); if (!n->map) goto error; - - free(str); } ff->ph->env.nr_numa_nodes = nr; ff->ph->env.numa_nodes = nodes; @@ -2913,10 +2913,10 @@ static int process_cache(struct feat_fd *ff, void *data __maybe_unused) return -1; for (i = 0; i < cnt; i++) { - struct cpu_cache_level c; + struct cpu_cache_level *c = &caches[i]; #define _R(v) \ - if (do_read_u32(ff, &c.v))\ + if (do_read_u32(ff, &c->v)) \ goto out_free_caches; \ _R(level) @@ -2926,22 +2926,25 @@ static int process_cache(struct feat_fd *ff, void *data __maybe_unused) #undef _R #define _R(v) \ - c.v = do_read_string(ff); \ - if (!c.v) \ - goto out_free_caches; + c->v = do_read_string(ff); \ + if (!c->v) \ + goto out_free_caches; \ _R(type) _R(size) _R(map) #undef _R - - caches[i] = c; } ff->ph->env.caches = caches; ff->ph->env.caches_cnt = cnt; return 0; out_free_caches: + for (i = 0; i < cnt; i++) { + free(caches[i].type); + free(caches[i].size); + free(caches[i].map); + } free(caches); return -1; } @@ -3585,18 +3588,16 @@ static int perf_header__adds_write(struct perf_header *header, struct feat_copier *fc) { int nr_sections; - struct feat_fd ff; + struct feat_fd ff = { + .fd = fd, + .ph = header, + }; struct perf_file_section *feat_sec, *p; int sec_size; u64 sec_start; int feat; int err; - ff = (struct feat_fd){ - .fd = fd, - .ph = header, - }; - nr_sections = bitmap_weight(header->adds_features, HEADER_FEAT_BITS); if (!nr_sections) return 0; @@ -3623,6 +3624,7 @@ static int perf_header__adds_write(struct perf_header *header, err = do_write(&ff, feat_sec, sec_size); if (err < 0) pr_debug("failed to write feature section\n"); + free(ff.buf); /* TODO: added to silence clang-tidy. */ free(feat_sec); return err; } @@ -3630,11 +3632,11 @@ static int perf_header__adds_write(struct perf_header *header, int perf_header__write_pipe(int fd) { struct perf_pipe_file_header f_header; - struct feat_fd ff; + struct feat_fd ff = { + .fd = fd, + }; int err; - ff = (struct feat_fd){ .fd = fd }; - f_header = (struct perf_pipe_file_header){ .magic = PERF_MAGIC, .size = sizeof(f_header), @@ -3645,7 +3647,7 @@ int perf_header__write_pipe(int fd) pr_debug("failed to write perf pipe header\n"); return err; } - + free(ff.buf); return 0; } @@ -3658,11 +3660,12 @@ static int perf_session__do_write_header(struct perf_session *session, struct perf_file_attr f_attr; struct perf_header *header = &session->header; struct evsel *evsel; - struct feat_fd ff; + struct feat_fd ff = { + .fd = fd, + }; u64 attr_offset; int err; - ff = (struct feat_fd){ .fd = fd}; lseek(fd, sizeof(f_header), SEEK_SET); evlist__for_each_entry(session->evlist, evsel) { @@ -3670,6 +3673,7 @@ static int perf_session__do_write_header(struct perf_session *session, err = do_write(&ff, evsel->core.id, evsel->core.ids * sizeof(u64)); if (err < 0) { pr_debug("failed to write perf header\n"); + free(ff.buf); return err; } } @@ -3695,6 +3699,7 @@ static int perf_session__do_write_header(struct perf_session *session, err = do_write(&ff, &f_attr, sizeof(f_attr)); if (err < 0) { pr_debug("failed to write perf header attribute\n"); + free(ff.buf); return err; } } @@ -3705,8 +3710,10 @@ static int perf_session__do_write_header(struct perf_session *session, if (at_exit) { err = perf_header__adds_write(header, evlist, fd, fc); - if (err < 0) + if (err < 0) { + free(ff.buf); return err; + } } f_header = (struct perf_file_header){ @@ -3728,6 +3735,7 @@ static int perf_session__do_write_header(struct perf_session *session, lseek(fd, 0, SEEK_SET); err = do_write(&ff, &f_header, sizeof(f_header)); + free(ff.buf); if (err < 0) { pr_debug("failed to write perf header\n"); return err; From b84b3f47921568a8172bec77d0370268e9fc62a2 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Mon, 9 Oct 2023 11:39:20 -0700 Subject: [PATCH 212/475] perf bpf_counter: Fix a few memory leaks Memory leaks were detected by clang-tidy. Signed-off-by: Ian Rogers Acked-by: Namhyung Kim Cc: Ravi Bangoria Cc: Nick Desaulniers Cc: Yang Jihong Cc: Huacai Chen Cc: Nathan Chancellor Cc: Kan Liang Cc: llvm@lists.linux.dev Cc: Ming Wang Cc: Tom Rix Cc: bpf@vger.kernel.org Link: https://lore.kernel.org/r/20231009183920.200859-20-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/util/bpf_counter.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/perf/util/bpf_counter.c b/tools/perf/util/bpf_counter.c index 6732cbbcf9b3..7f9b0e46e008 100644 --- a/tools/perf/util/bpf_counter.c +++ b/tools/perf/util/bpf_counter.c @@ -104,7 +104,7 @@ static int bpf_program_profiler_load_one(struct evsel *evsel, u32 prog_id) struct bpf_prog_profiler_bpf *skel; struct bpf_counter *counter; struct bpf_program *prog; - char *prog_name; + char *prog_name = NULL; int prog_fd; int err; @@ -155,10 +155,12 @@ static int bpf_program_profiler_load_one(struct evsel *evsel, u32 prog_id) assert(skel != NULL); counter->skel = skel; list_add(&counter->list, &evsel->bpf_counter_list); + free(prog_name); close(prog_fd); return 0; err_out: bpf_prog_profiler_bpf__destroy(skel); + free(prog_name); free(counter); close(prog_fd); return -1; @@ -180,6 +182,7 @@ static int bpf_program_profiler__load(struct evsel *evsel, struct target *target (*p != '\0' && *p != ',')) { pr_err("Failed to parse bpf prog ids %s\n", target->bpf_str); + free(bpf_str_); return -1; } From a16afcc58a8c5ebc65c852faf001f8f61f05e4ef Mon Sep 17 00:00:00 2001 From: Besar Wicaksono Date: Tue, 10 Oct 2023 18:48:03 -0500 Subject: [PATCH 213/475] perf cs-etm: Fix incorrect or missing decoder for raw trace The decoder creation for raw trace uses metadata from the first CPU. On per-cpu mode, this metadata is incorrectly used for every decoder. On per-process/per-thread traces, the first CPU is CPU0. If CPU0 trace is not enabled, its metadata will be marked unused and the decoder is not created. Perf report dump skips the decoding part because the decoder is missing. To fix this, use metadata of the CPU associated with sample object. Signed-off-by: Besar Wicaksono Reviewed-by: James Clark Cc: suzuki.poulose@arm.com Cc: mike.leach@linaro.org Cc: jonathanh@nvidia.com Cc: rwiley@nvidia.com Cc: treding@nvidia.com Cc: vsethi@nvidia.com Cc: ywan@nvidia.com Cc: linux-arm-kernel@lists.infradead.org Cc: coresight@lists.linaro.org Cc: linux-tegra@vger.kernel.org Link: https://lore.kernel.org/r/20231010234803.5419-1-bwicaksono@nvidia.com Signed-off-by: Namhyung Kim --- tools/perf/util/cs-etm.c | 114 +++++++++++++++++++++++---------------- 1 file changed, 69 insertions(+), 45 deletions(-) diff --git a/tools/perf/util/cs-etm.c b/tools/perf/util/cs-etm.c index 9729d006550d..a9873d14c632 100644 --- a/tools/perf/util/cs-etm.c +++ b/tools/perf/util/cs-etm.c @@ -282,23 +282,32 @@ static int cs_etm__metadata_set_trace_id(u8 trace_chan_id, u64 *cpu_metadata) return 0; } +/* + * Get a metadata index for a specific cpu from an array. + * + */ +static int get_cpu_data_idx(struct cs_etm_auxtrace *etm, int cpu) +{ + int i; + + for (i = 0; i < etm->num_cpu; i++) { + if (etm->metadata[i][CS_ETM_CPU] == (u64)cpu) { + return i; + } + } + + return -1; +} + /* * Get a metadata for a specific cpu from an array. * */ static u64 *get_cpu_data(struct cs_etm_auxtrace *etm, int cpu) { - int i; - u64 *metadata = NULL; + int idx = get_cpu_data_idx(etm, cpu); - for (i = 0; i < etm->num_cpu; i++) { - if (etm->metadata[i][CS_ETM_CPU] == (u64)cpu) { - metadata = etm->metadata[i]; - break; - } - } - - return metadata; + return (idx != -1) ? etm->metadata[idx] : NULL; } /* @@ -641,66 +650,80 @@ static void cs_etm__packet_dump(const char *pkt_string) } static void cs_etm__set_trace_param_etmv3(struct cs_etm_trace_params *t_params, - struct cs_etm_auxtrace *etm, int idx, - u32 etmidr) + struct cs_etm_auxtrace *etm, int t_idx, + int m_idx, u32 etmidr) { u64 **metadata = etm->metadata; - t_params[idx].protocol = cs_etm__get_v7_protocol_version(etmidr); - t_params[idx].etmv3.reg_ctrl = metadata[idx][CS_ETM_ETMCR]; - t_params[idx].etmv3.reg_trc_id = metadata[idx][CS_ETM_ETMTRACEIDR]; + t_params[t_idx].protocol = cs_etm__get_v7_protocol_version(etmidr); + t_params[t_idx].etmv3.reg_ctrl = metadata[m_idx][CS_ETM_ETMCR]; + t_params[t_idx].etmv3.reg_trc_id = metadata[m_idx][CS_ETM_ETMTRACEIDR]; } static void cs_etm__set_trace_param_etmv4(struct cs_etm_trace_params *t_params, - struct cs_etm_auxtrace *etm, int idx) + struct cs_etm_auxtrace *etm, int t_idx, + int m_idx) { u64 **metadata = etm->metadata; - t_params[idx].protocol = CS_ETM_PROTO_ETMV4i; - t_params[idx].etmv4.reg_idr0 = metadata[idx][CS_ETMV4_TRCIDR0]; - t_params[idx].etmv4.reg_idr1 = metadata[idx][CS_ETMV4_TRCIDR1]; - t_params[idx].etmv4.reg_idr2 = metadata[idx][CS_ETMV4_TRCIDR2]; - t_params[idx].etmv4.reg_idr8 = metadata[idx][CS_ETMV4_TRCIDR8]; - t_params[idx].etmv4.reg_configr = metadata[idx][CS_ETMV4_TRCCONFIGR]; - t_params[idx].etmv4.reg_traceidr = metadata[idx][CS_ETMV4_TRCTRACEIDR]; + t_params[t_idx].protocol = CS_ETM_PROTO_ETMV4i; + t_params[t_idx].etmv4.reg_idr0 = metadata[m_idx][CS_ETMV4_TRCIDR0]; + t_params[t_idx].etmv4.reg_idr1 = metadata[m_idx][CS_ETMV4_TRCIDR1]; + t_params[t_idx].etmv4.reg_idr2 = metadata[m_idx][CS_ETMV4_TRCIDR2]; + t_params[t_idx].etmv4.reg_idr8 = metadata[m_idx][CS_ETMV4_TRCIDR8]; + t_params[t_idx].etmv4.reg_configr = metadata[m_idx][CS_ETMV4_TRCCONFIGR]; + t_params[t_idx].etmv4.reg_traceidr = metadata[m_idx][CS_ETMV4_TRCTRACEIDR]; } static void cs_etm__set_trace_param_ete(struct cs_etm_trace_params *t_params, - struct cs_etm_auxtrace *etm, int idx) + struct cs_etm_auxtrace *etm, int t_idx, + int m_idx) { u64 **metadata = etm->metadata; - t_params[idx].protocol = CS_ETM_PROTO_ETE; - t_params[idx].ete.reg_idr0 = metadata[idx][CS_ETE_TRCIDR0]; - t_params[idx].ete.reg_idr1 = metadata[idx][CS_ETE_TRCIDR1]; - t_params[idx].ete.reg_idr2 = metadata[idx][CS_ETE_TRCIDR2]; - t_params[idx].ete.reg_idr8 = metadata[idx][CS_ETE_TRCIDR8]; - t_params[idx].ete.reg_configr = metadata[idx][CS_ETE_TRCCONFIGR]; - t_params[idx].ete.reg_traceidr = metadata[idx][CS_ETE_TRCTRACEIDR]; - t_params[idx].ete.reg_devarch = metadata[idx][CS_ETE_TRCDEVARCH]; + t_params[t_idx].protocol = CS_ETM_PROTO_ETE; + t_params[t_idx].ete.reg_idr0 = metadata[m_idx][CS_ETE_TRCIDR0]; + t_params[t_idx].ete.reg_idr1 = metadata[m_idx][CS_ETE_TRCIDR1]; + t_params[t_idx].ete.reg_idr2 = metadata[m_idx][CS_ETE_TRCIDR2]; + t_params[t_idx].ete.reg_idr8 = metadata[m_idx][CS_ETE_TRCIDR8]; + t_params[t_idx].ete.reg_configr = metadata[m_idx][CS_ETE_TRCCONFIGR]; + t_params[t_idx].ete.reg_traceidr = metadata[m_idx][CS_ETE_TRCTRACEIDR]; + t_params[t_idx].ete.reg_devarch = metadata[m_idx][CS_ETE_TRCDEVARCH]; } static int cs_etm__init_trace_params(struct cs_etm_trace_params *t_params, struct cs_etm_auxtrace *etm, + bool formatted, + int sample_cpu, int decoders) { - int i; + int t_idx, m_idx; u32 etmidr; u64 architecture; - for (i = 0; i < decoders; i++) { - architecture = etm->metadata[i][CS_ETM_MAGIC]; + for (t_idx = 0; t_idx < decoders; t_idx++) { + if (formatted) + m_idx = t_idx; + else { + m_idx = get_cpu_data_idx(etm, sample_cpu); + if (m_idx == -1) { + pr_warning("CS_ETM: unknown CPU, falling back to first metadata\n"); + m_idx = 0; + } + } + + architecture = etm->metadata[m_idx][CS_ETM_MAGIC]; switch (architecture) { case __perf_cs_etmv3_magic: - etmidr = etm->metadata[i][CS_ETM_ETMIDR]; - cs_etm__set_trace_param_etmv3(t_params, etm, i, etmidr); + etmidr = etm->metadata[m_idx][CS_ETM_ETMIDR]; + cs_etm__set_trace_param_etmv3(t_params, etm, t_idx, m_idx, etmidr); break; case __perf_cs_etmv4_magic: - cs_etm__set_trace_param_etmv4(t_params, etm, i); + cs_etm__set_trace_param_etmv4(t_params, etm, t_idx, m_idx); break; case __perf_cs_ete_magic: - cs_etm__set_trace_param_ete(t_params, etm, i); + cs_etm__set_trace_param_ete(t_params, etm, t_idx, m_idx); break; default: return -EINVAL; @@ -1016,7 +1039,7 @@ out: } static struct cs_etm_queue *cs_etm__alloc_queue(struct cs_etm_auxtrace *etm, - bool formatted) + bool formatted, int sample_cpu) { struct cs_etm_decoder_params d_params; struct cs_etm_trace_params *t_params = NULL; @@ -1041,7 +1064,7 @@ static struct cs_etm_queue *cs_etm__alloc_queue(struct cs_etm_auxtrace *etm, if (!t_params) goto out_free; - if (cs_etm__init_trace_params(t_params, etm, decoders)) + if (cs_etm__init_trace_params(t_params, etm, formatted, sample_cpu, decoders)) goto out_free; /* Set decoder parameters to decode trace packets */ @@ -1081,14 +1104,15 @@ out_free: static int cs_etm__setup_queue(struct cs_etm_auxtrace *etm, struct auxtrace_queue *queue, unsigned int queue_nr, - bool formatted) + bool formatted, + int sample_cpu) { struct cs_etm_queue *etmq = queue->priv; if (list_empty(&queue->head) || etmq) return 0; - etmq = cs_etm__alloc_queue(etm, formatted); + etmq = cs_etm__alloc_queue(etm, formatted, sample_cpu); if (!etmq) return -ENOMEM; @@ -2816,7 +2840,7 @@ static int cs_etm__process_auxtrace_event(struct perf_session *session, * formatted in piped mode (true). */ err = cs_etm__setup_queue(etm, &etm->queues.queue_array[idx], - idx, true); + idx, true, -1); if (err) return err; @@ -3022,7 +3046,7 @@ static int cs_etm__queue_aux_fragment(struct perf_session *session, off_t file_o idx = auxtrace_event->idx; formatted = !(aux_event->flags & PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW); return cs_etm__setup_queue(etm, &etm->queues.queue_array[idx], - idx, formatted); + idx, formatted, sample->cpu); } /* Wasn't inside this buffer, but there were no parse errors. 1 == 'not found' */ From 57f728d59f005dffdbb52a03531e480a71599bc5 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Mon, 31 Jul 2023 22:08:17 -0700 Subject: [PATCH 214/475] cpumask: kernel-doc cleanups and additions Clean up some punctutation and abbreviations. Add kernel-doc notation for one function and function return value for 39 functions. cpumask.h: Fix some punctuation (plural vs. possessive). Fix some abbreviations (ie. -> i.e., id -> ID). Fix 35 warnings like this: include/linux/cpumask.h:161: warning: No description found for return value of 'cpumask_first' cpumask.c: Add Return: value for 4 functions. Add kernel-doc for cpumask_any_distribute(). Signed-off-by: Randy Dunlap Reviewed-by: Andy Shevchenko Signed-off-by: Yury Norov --- include/linux/cpumask.h | 113 ++++++++++++++++++++++++---------------- lib/cpumask.c | 17 ++++-- 2 files changed, 81 insertions(+), 49 deletions(-) diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index f10fb87d49db..cfb545841a2c 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -4,7 +4,7 @@ /* * Cpumasks provide a bitmap suitable for representing the - * set of CPU's in a system, one bit position per CPU number. In general, + * set of CPUs in a system, one bit position per CPU number. In general, * only nr_cpu_ids (<= NR_CPUS) bits are valid. */ #include @@ -97,7 +97,7 @@ static inline void set_nr_cpu_ids(unsigned int nr) * * If !CONFIG_HOTPLUG_CPU, present == possible, and active == online. * - * The cpu_possible_mask is fixed at boot time, as the set of CPU id's + * The cpu_possible_mask is fixed at boot time, as the set of CPU IDs * that it is possible might ever be plugged in at anytime during the * life of that system boot. The cpu_present_mask is dynamic(*), * representing which CPUs are currently plugged in. And @@ -112,7 +112,7 @@ static inline void set_nr_cpu_ids(unsigned int nr) * hotplug, it's a copy of cpu_possible_mask, hence fixed at boot. * * Subtleties: - * 1) UP arch's (NR_CPUS == 1, CONFIG_SMP not defined) hardcode + * 1) UP ARCHes (NR_CPUS == 1, CONFIG_SMP not defined) hardcode * assumption that their single CPU is online. The UP * cpu_{online,possible,present}_masks are placebos. Changing them * will have no useful affect on the following num_*_cpus() @@ -155,7 +155,7 @@ static __always_inline unsigned int cpumask_check(unsigned int cpu) * cpumask_first - get the first cpu in a cpumask * @srcp: the cpumask pointer * - * Returns >= nr_cpu_ids if no cpus set. + * Return: >= nr_cpu_ids if no cpus set. */ static inline unsigned int cpumask_first(const struct cpumask *srcp) { @@ -166,7 +166,7 @@ static inline unsigned int cpumask_first(const struct cpumask *srcp) * cpumask_first_zero - get the first unset cpu in a cpumask * @srcp: the cpumask pointer * - * Returns >= nr_cpu_ids if all cpus are set. + * Return: >= nr_cpu_ids if all cpus are set. */ static inline unsigned int cpumask_first_zero(const struct cpumask *srcp) { @@ -178,7 +178,7 @@ static inline unsigned int cpumask_first_zero(const struct cpumask *srcp) * @srcp1: the first input * @srcp2: the second input * - * Returns >= nr_cpu_ids if no cpus set in both. See also cpumask_next_and(). + * Return: >= nr_cpu_ids if no cpus set in both. See also cpumask_next_and(). */ static inline unsigned int cpumask_first_and(const struct cpumask *srcp1, const struct cpumask *srcp2) @@ -190,7 +190,7 @@ unsigned int cpumask_first_and(const struct cpumask *srcp1, const struct cpumask * cpumask_last - get the last CPU in a cpumask * @srcp: - the cpumask pointer * - * Returns >= nr_cpumask_bits if no CPUs set. + * Return: >= nr_cpumask_bits if no CPUs set. */ static inline unsigned int cpumask_last(const struct cpumask *srcp) { @@ -199,10 +199,10 @@ static inline unsigned int cpumask_last(const struct cpumask *srcp) /** * cpumask_next - get the next cpu in a cpumask - * @n: the cpu prior to the place to search (ie. return will be > @n) + * @n: the cpu prior to the place to search (i.e. return will be > @n) * @srcp: the cpumask pointer * - * Returns >= nr_cpu_ids if no further cpus set. + * Return: >= nr_cpu_ids if no further cpus set. */ static inline unsigned int cpumask_next(int n, const struct cpumask *srcp) @@ -215,10 +215,10 @@ unsigned int cpumask_next(int n, const struct cpumask *srcp) /** * cpumask_next_zero - get the next unset cpu in a cpumask - * @n: the cpu prior to the place to search (ie. return will be > @n) + * @n: the cpu prior to the place to search (i.e. return will be > @n) * @srcp: the cpumask pointer * - * Returns >= nr_cpu_ids if no further cpus unset. + * Return: >= nr_cpu_ids if no further cpus unset. */ static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp) { @@ -254,11 +254,11 @@ unsigned int cpumask_any_distribute(const struct cpumask *srcp); /** * cpumask_next_and - get the next cpu in *src1p & *src2p - * @n: the cpu prior to the place to search (ie. return will be > @n) + * @n: the cpu prior to the place to search (i.e. return will be > @n) * @src1p: the first cpumask pointer * @src2p: the second cpumask pointer * - * Returns >= nr_cpu_ids if no further cpus set in both. + * Return: >= nr_cpu_ids if no further cpus set in both. */ static inline unsigned int cpumask_next_and(int n, const struct cpumask *src1p, @@ -373,7 +373,7 @@ unsigned int __pure cpumask_next_wrap(int n, const struct cpumask *mask, int sta * @cpu: the cpu to ignore. * * Often used to find any cpu but smp_processor_id() in a mask. - * Returns >= nr_cpu_ids if no cpus set. + * Return: >= nr_cpu_ids if no cpus set. */ static inline unsigned int cpumask_any_but(const struct cpumask *mask, unsigned int cpu) @@ -388,11 +388,11 @@ unsigned int cpumask_any_but(const struct cpumask *mask, unsigned int cpu) } /** - * cpumask_nth - get the first cpu in a cpumask + * cpumask_nth - get the Nth cpu in a cpumask * @srcp: the cpumask pointer - * @cpu: the N'th cpu to find, starting from 0 + * @cpu: the Nth cpu to find, starting from 0 * - * Returns >= nr_cpu_ids if such cpu doesn't exist. + * Return: >= nr_cpu_ids if such cpu doesn't exist. */ static inline unsigned int cpumask_nth(unsigned int cpu, const struct cpumask *srcp) { @@ -400,12 +400,12 @@ static inline unsigned int cpumask_nth(unsigned int cpu, const struct cpumask *s } /** - * cpumask_nth_and - get the first cpu in 2 cpumasks + * cpumask_nth_and - get the Nth cpu in 2 cpumasks * @srcp1: the cpumask pointer * @srcp2: the cpumask pointer - * @cpu: the N'th cpu to find, starting from 0 + * @cpu: the Nth cpu to find, starting from 0 * - * Returns >= nr_cpu_ids if such cpu doesn't exist. + * Return: >= nr_cpu_ids if such cpu doesn't exist. */ static inline unsigned int cpumask_nth_and(unsigned int cpu, const struct cpumask *srcp1, @@ -416,12 +416,12 @@ unsigned int cpumask_nth_and(unsigned int cpu, const struct cpumask *srcp1, } /** - * cpumask_nth_andnot - get the first cpu set in 1st cpumask, and clear in 2nd. + * cpumask_nth_andnot - get the Nth cpu set in 1st cpumask, and clear in 2nd. * @srcp1: the cpumask pointer * @srcp2: the cpumask pointer - * @cpu: the N'th cpu to find, starting from 0 + * @cpu: the Nth cpu to find, starting from 0 * - * Returns >= nr_cpu_ids if such cpu doesn't exist. + * Return: >= nr_cpu_ids if such cpu doesn't exist. */ static inline unsigned int cpumask_nth_andnot(unsigned int cpu, const struct cpumask *srcp1, @@ -436,9 +436,9 @@ unsigned int cpumask_nth_andnot(unsigned int cpu, const struct cpumask *srcp1, * @srcp1: the cpumask pointer * @srcp2: the cpumask pointer * @srcp3: the cpumask pointer - * @cpu: the N'th cpu to find, starting from 0 + * @cpu: the Nth cpu to find, starting from 0 * - * Returns >= nr_cpu_ids if such cpu doesn't exist. + * Return: >= nr_cpu_ids if such cpu doesn't exist. */ static __always_inline unsigned int cpumask_nth_and_andnot(unsigned int cpu, const struct cpumask *srcp1, @@ -497,7 +497,7 @@ static __always_inline void __cpumask_clear_cpu(int cpu, struct cpumask *dstp) * @cpu: cpu number (< nr_cpu_ids) * @cpumask: the cpumask pointer * - * Returns true if @cpu is set in @cpumask, else returns false + * Return: true if @cpu is set in @cpumask, else returns false */ static __always_inline bool cpumask_test_cpu(int cpu, const struct cpumask *cpumask) { @@ -509,9 +509,9 @@ static __always_inline bool cpumask_test_cpu(int cpu, const struct cpumask *cpum * @cpu: cpu number (< nr_cpu_ids) * @cpumask: the cpumask pointer * - * Returns true if @cpu is set in old bitmap of @cpumask, else returns false - * * test_and_set_bit wrapper for cpumasks. + * + * Return: true if @cpu is set in old bitmap of @cpumask, else returns false */ static __always_inline bool cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask) { @@ -523,9 +523,9 @@ static __always_inline bool cpumask_test_and_set_cpu(int cpu, struct cpumask *cp * @cpu: cpu number (< nr_cpu_ids) * @cpumask: the cpumask pointer * - * Returns true if @cpu is set in old bitmap of @cpumask, else returns false - * * test_and_clear_bit wrapper for cpumasks. + * + * Return: true if @cpu is set in old bitmap of @cpumask, else returns false */ static __always_inline bool cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask) { @@ -560,7 +560,7 @@ static inline void cpumask_clear(struct cpumask *dstp) * @src1p: the first input * @src2p: the second input * - * If *@dstp is empty, returns false, else returns true + * Return: false if *@dstp is empty, else returns true */ static inline bool cpumask_and(struct cpumask *dstp, const struct cpumask *src1p, @@ -603,7 +603,7 @@ static inline void cpumask_xor(struct cpumask *dstp, * @src1p: the first input * @src2p: the second input * - * If *@dstp is empty, returns false, else returns true + * Return: false if *@dstp is empty, else returns true */ static inline bool cpumask_andnot(struct cpumask *dstp, const struct cpumask *src1p, @@ -617,6 +617,8 @@ static inline bool cpumask_andnot(struct cpumask *dstp, * cpumask_equal - *src1p == *src2p * @src1p: the first input * @src2p: the second input + * + * Return: true if the cpumasks are equal, false if not */ static inline bool cpumask_equal(const struct cpumask *src1p, const struct cpumask *src2p) @@ -630,6 +632,9 @@ static inline bool cpumask_equal(const struct cpumask *src1p, * @src1p: the first input * @src2p: the second input * @src3p: the third input + * + * Return: true if first cpumask ORed with second cpumask == third cpumask, + * otherwise false */ static inline bool cpumask_or_equal(const struct cpumask *src1p, const struct cpumask *src2p, @@ -643,6 +648,9 @@ static inline bool cpumask_or_equal(const struct cpumask *src1p, * cpumask_intersects - (*src1p & *src2p) != 0 * @src1p: the first input * @src2p: the second input + * + * Return: true if first cpumask ANDed with second cpumask is non-empty, + * otherwise false */ static inline bool cpumask_intersects(const struct cpumask *src1p, const struct cpumask *src2p) @@ -656,7 +664,7 @@ static inline bool cpumask_intersects(const struct cpumask *src1p, * @src1p: the first input * @src2p: the second input * - * Returns true if *@src1p is a subset of *@src2p, else returns false + * Return: true if *@src1p is a subset of *@src2p, else returns false */ static inline bool cpumask_subset(const struct cpumask *src1p, const struct cpumask *src2p) @@ -668,6 +676,8 @@ static inline bool cpumask_subset(const struct cpumask *src1p, /** * cpumask_empty - *srcp == 0 * @srcp: the cpumask to that all cpus < nr_cpu_ids are clear. + * + * Return: true if srcp is empty (has no bits set), else false */ static inline bool cpumask_empty(const struct cpumask *srcp) { @@ -677,6 +687,8 @@ static inline bool cpumask_empty(const struct cpumask *srcp) /** * cpumask_full - *srcp == 0xFFFFFFFF... * @srcp: the cpumask to that all cpus < nr_cpu_ids are set. + * + * Return: true if srcp is full (has all bits set), else false */ static inline bool cpumask_full(const struct cpumask *srcp) { @@ -686,6 +698,8 @@ static inline bool cpumask_full(const struct cpumask *srcp) /** * cpumask_weight - Count of bits in *srcp * @srcp: the cpumask to count bits (< nr_cpu_ids) in. + * + * Return: count of bits set in *srcp */ static inline unsigned int cpumask_weight(const struct cpumask *srcp) { @@ -696,6 +710,8 @@ static inline unsigned int cpumask_weight(const struct cpumask *srcp) * cpumask_weight_and - Count of bits in (*srcp1 & *srcp2) * @srcp1: the cpumask to count bits (< nr_cpu_ids) in. * @srcp2: the cpumask to count bits (< nr_cpu_ids) in. + * + * Return: count of bits set in both *srcp1 and *srcp2 */ static inline unsigned int cpumask_weight_and(const struct cpumask *srcp1, const struct cpumask *srcp2) @@ -744,7 +760,7 @@ static inline void cpumask_copy(struct cpumask *dstp, * cpumask_any - pick a "random" cpu from *srcp * @srcp: the input cpumask * - * Returns >= nr_cpu_ids if no cpus set. + * Return: >= nr_cpu_ids if no cpus set. */ #define cpumask_any(srcp) cpumask_first(srcp) @@ -753,7 +769,7 @@ static inline void cpumask_copy(struct cpumask *dstp, * @mask1: the first input cpumask * @mask2: the second input cpumask * - * Returns >= nr_cpu_ids if no cpus set. + * Return: >= nr_cpu_ids if no cpus set. */ #define cpumask_any_and(mask1, mask2) cpumask_first_and((mask1), (mask2)) @@ -769,7 +785,7 @@ static inline void cpumask_copy(struct cpumask *dstp, * @len: the length of the buffer * @dstp: the cpumask to set. * - * Returns -errno, or 0 for success. + * Return: -errno, or 0 for success. */ static inline int cpumask_parse_user(const char __user *buf, int len, struct cpumask *dstp) @@ -783,7 +799,7 @@ static inline int cpumask_parse_user(const char __user *buf, int len, * @len: the length of the buffer * @dstp: the cpumask to set. * - * Returns -errno, or 0 for success. + * Return: -errno, or 0 for success. */ static inline int cpumask_parselist_user(const char __user *buf, int len, struct cpumask *dstp) @@ -797,7 +813,7 @@ static inline int cpumask_parselist_user(const char __user *buf, int len, * @buf: the buffer to extract from * @dstp: the cpumask to set. * - * Returns -errno, or 0 for success. + * Return: -errno, or 0 for success. */ static inline int cpumask_parse(const char *buf, struct cpumask *dstp) { @@ -809,7 +825,7 @@ static inline int cpumask_parse(const char *buf, struct cpumask *dstp) * @buf: the buffer to extract from * @dstp: the cpumask to set. * - * Returns -errno, or 0 for success. + * Return: -errno, or 0 for success. */ static inline int cpulist_parse(const char *buf, struct cpumask *dstp) { @@ -817,7 +833,9 @@ static inline int cpulist_parse(const char *buf, struct cpumask *dstp) } /** - * cpumask_size - size to allocate for a 'struct cpumask' in bytes + * cpumask_size - calculate size to allocate for a 'struct cpumask' in bytes + * + * Return: size to allocate for a &struct cpumask in bytes */ static inline unsigned int cpumask_size(void) { @@ -831,7 +849,7 @@ static inline unsigned int cpumask_size(void) * little more difficult, we typedef cpumask_var_t to an array or a * pointer: doing &mask on an array is a noop, so it still works. * - * ie. + * i.e. * cpumask_var_t tmpmask; * if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL)) * return -ENOMEM; @@ -887,6 +905,8 @@ bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node) * a nop returning a constant 1 (in ). * * See alloc_cpumask_var_node. + * + * Return: %true if allocation succeeded, %false if not */ static inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) @@ -1025,7 +1045,7 @@ set_cpu_dying(unsigned int cpu, bool dying) } /** - * to_cpumask - convert an NR_CPUS bitmap to a struct cpumask * + * to_cpumask - convert a NR_CPUS bitmap to a struct cpumask * * @bitmap: the bitmap * * There are a few places where cpumask_var_t isn't appropriate and @@ -1068,6 +1088,8 @@ static inline const struct cpumask *get_cpu_mask(unsigned int cpu) * interface gives only a momentary snapshot and is not protected against * concurrent CPU hotplug operations unless invoked from a cpuhp_lock held * region. + * + * Return: momentary snapshot of the number of online CPUs */ static __always_inline unsigned int num_online_cpus(void) { @@ -1160,7 +1182,7 @@ static inline bool cpu_dying(unsigned int cpu) * @mask: the cpumask to copy * @buf: the buffer to copy into * - * Returns the length of the (null-terminated) @buf string, zero if + * Return: the length of the (null-terminated) @buf string, zero if * nothing is copied. */ static inline ssize_t @@ -1183,7 +1205,7 @@ cpumap_print_to_pagebuf(bool list, char *buf, const struct cpumask *mask) * cpumask; Typically used by bin_attribute to export cpumask bitmask * ABI. * - * Returns the length of how many bytes have been copied, excluding + * Return: the length of how many bytes have been copied, excluding * terminating '\0'. */ static inline ssize_t @@ -1204,6 +1226,9 @@ cpumap_print_bitmask_to_buf(char *buf, const struct cpumask *mask, * * Everything is same with the above cpumap_print_bitmask_to_buf() * except the print format. + * + * Return: the length of how many bytes have been copied, excluding + * terminating '\0'. */ static inline ssize_t cpumap_print_list_to_buf(char *buf, const struct cpumask *mask, diff --git a/lib/cpumask.c b/lib/cpumask.c index a7fd02b5ae26..19277c6d551f 100644 --- a/lib/cpumask.c +++ b/lib/cpumask.c @@ -14,7 +14,7 @@ * @start: the start point of the iteration * @wrap: assume @n crossing @start terminates the iteration * - * Returns >= nr_cpu_ids on completion + * Return: >= nr_cpu_ids on completion * * Note: the @wrap argument is required for the start condition when * we cannot assume @start is set in @mask. @@ -48,8 +48,9 @@ EXPORT_SYMBOL(cpumask_next_wrap); * @node: memory node from which to allocate or %NUMA_NO_NODE * * Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is - * a nop returning a constant 1 (in ) - * Returns TRUE if memory allocation succeeded, FALSE otherwise. + * a nop returning a constant 1 (in ). + * + * Return: TRUE if memory allocation succeeded, FALSE otherwise. * * In addition, mask will be NULL if this fails. Note that gcc is * usually smart enough to know that mask can never be NULL if @@ -115,7 +116,7 @@ void __init free_bootmem_cpumask_var(cpumask_var_t mask) * @i: index number * @node: local numa_node * - * Returns online CPU according to a numa aware policy; local cpus are returned + * Return: online CPU according to a numa aware policy; local cpus are returned * first, followed by non-local ones, then it wraps around. * * For those who wants to enumerate all CPUs based on their NUMA distances, @@ -165,7 +166,7 @@ static DEFINE_PER_CPU(int, distribute_cpu_mask_prev); * Iterated calls using the same srcp1 and srcp2 will be distributed within * their intersection. * - * Returns >= nr_cpu_ids if the intersection is empty. + * Return: >= nr_cpu_ids if the intersection is empty. */ unsigned int cpumask_any_and_distribute(const struct cpumask *src1p, const struct cpumask *src2p) @@ -184,6 +185,12 @@ unsigned int cpumask_any_and_distribute(const struct cpumask *src1p, } EXPORT_SYMBOL(cpumask_any_and_distribute); +/** + * cpumask_any_distribute - Return an arbitrary cpu from srcp + * @srcp: &cpumask for selection + * + * Return: >= nr_cpu_ids if the intersection is empty. + */ unsigned int cpumask_any_distribute(const struct cpumask *srcp) { unsigned int next, prev; From 8ed13a762ca0661e4df16ce5eba929e73f4bd8f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20Neusch=C3=A4fer?= Date: Mon, 14 Aug 2023 19:37:08 +0200 Subject: [PATCH 215/475] bitmap: Fix a typo ("identify map") MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A map in which each element is mapped to itself is called an "identity map". Signed-off-by: Jonathan Neuschäfer Reviewed-by: Andy Shevchenko Signed-off-by: Yury Norov --- lib/bitmap.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/bitmap.c b/lib/bitmap.c index ddb31015e38a..24284caadbcc 100644 --- a/lib/bitmap.c +++ b/lib/bitmap.c @@ -988,7 +988,7 @@ static int bitmap_pos_to_ord(const unsigned long *buf, unsigned int pos, unsigne * to @dst. * * The positions of unset bits in @old are mapped to themselves - * (the identify map). + * (the identity map). * * Apply the above specified mapping to @src, placing the result in * @dst, clearing any bits previously set in @dst. @@ -1037,7 +1037,7 @@ EXPORT_SYMBOL(bitmap_remap); * the position of the m-th set bit in @new, where m == n % w. * * The positions of unset bits in @old are mapped to themselves - * (the identify map). + * (the identity map). * * Apply the above specified mapping to bit position @oldbit, returning * the new bit position. From 7733aa893847f021c674d0d30b723d892109369d Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Thu, 17 Aug 2023 19:20:54 +0300 Subject: [PATCH 216/475] bitmap: Remove dead code, i.e. bitmap_copy_le() Besides the fact it's not used anywhere it should be implemented differently, i.e. via helpers from linux/byteorder/generic.h. Yet the helpers themselves need to be introduced first. Also note, the function lacks of the test cases, they must be provided. Hence, drop the current dead code for good. Signed-off-by: Andy Shevchenko Signed-off-by: Yury Norov --- include/linux/bitmap.h | 5 ----- lib/bitmap.c | 23 ----------------------- 2 files changed, 28 deletions(-) diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h index 03644237e1ef..1516ff979315 100644 --- a/include/linux/bitmap.h +++ b/include/linux/bitmap.h @@ -220,11 +220,6 @@ int bitmap_find_free_region(unsigned long *bitmap, unsigned int bits, int order) void bitmap_release_region(unsigned long *bitmap, unsigned int pos, int order); int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos, int order); -#ifdef __BIG_ENDIAN -void bitmap_copy_le(unsigned long *dst, const unsigned long *src, unsigned int nbits); -#else -#define bitmap_copy_le bitmap_copy -#endif int bitmap_print_to_pagebuf(bool list, char *buf, const unsigned long *maskp, int nmaskbits); diff --git a/lib/bitmap.c b/lib/bitmap.c index 24284caadbcc..935e0f96e785 100644 --- a/lib/bitmap.c +++ b/lib/bitmap.c @@ -1360,29 +1360,6 @@ int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos, int order) } EXPORT_SYMBOL(bitmap_allocate_region); -/** - * bitmap_copy_le - copy a bitmap, putting the bits into little-endian order. - * @dst: destination buffer - * @src: bitmap to copy - * @nbits: number of bits in the bitmap - * - * Require nbits % BITS_PER_LONG == 0. - */ -#ifdef __BIG_ENDIAN -void bitmap_copy_le(unsigned long *dst, const unsigned long *src, unsigned int nbits) -{ - unsigned int i; - - for (i = 0; i < nbits/BITS_PER_LONG; i++) { - if (BITS_PER_LONG == 64) - dst[i] = cpu_to_le64(src[i]); - else - dst[i] = cpu_to_le32(src[i]); - } -} -EXPORT_SYMBOL(bitmap_copy_le); -#endif - unsigned long *bitmap_alloc(unsigned int nbits, gfp_t flags) { return kmalloc_array(BITS_TO_LONGS(nbits), sizeof(unsigned long), From aae06fc1b5a2e4b52f8504a1f12f9b8b98e80641 Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Sat, 7 Oct 2023 16:35:10 -0700 Subject: [PATCH 217/475] lib/bitmap: split-out string-related operations to a separate files lib/bitmap.c and corresponding include/linux/bitmap.h are intended to hold functions related to operations on bitmaps, like bitmap_shift or bitmap_set. Historically, some string-related operations like bitmap_parse are also reside in lib/bitmap.c. Now that the subsystem evolves, string-related bitmap operations became a significant part of the file. Because they are quite different from the other bitmap functions by nature, it's worth to split them to a separate source/header files. CC: Andrew Morton CC: Andy Shevchenko CC: Rasmus Villemoes Signed-off-by: Yury Norov --- MAINTAINERS | 2 + include/linux/bitmap-str.h | 16 ++ include/linux/bitmap.h | 18 +- lib/Makefile | 2 +- lib/bitmap-str.c | 510 ++++++++++++++++++++++++++++++++++++ lib/bitmap.c | 512 ------------------------------------- 6 files changed, 530 insertions(+), 530 deletions(-) create mode 100644 include/linux/bitmap-str.h create mode 100644 lib/bitmap-str.c diff --git a/MAINTAINERS b/MAINTAINERS index 6c4cce45a09d..561def8239f9 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -3506,12 +3506,14 @@ R: Andy Shevchenko R: Rasmus Villemoes S: Maintained F: include/linux/bitfield.h +F: include/linux/bitmap-str.h F: include/linux/bitmap.h F: include/linux/bits.h F: include/linux/cpumask.h F: include/linux/find.h F: include/linux/nodemask.h F: include/vdso/bits.h +F: lib/bitmap-str.c F: lib/bitmap.c F: lib/cpumask.c F: lib/cpumask_kunit.c diff --git a/include/linux/bitmap-str.h b/include/linux/bitmap-str.h new file mode 100644 index 000000000000..17caeca94cab --- /dev/null +++ b/include/linux/bitmap-str.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __LINUX_BITMAP_STR_H +#define __LINUX_BITMAP_STR_H + +int bitmap_parse_user(const char __user *ubuf, unsigned int ulen, unsigned long *dst, int nbits); +int bitmap_print_to_pagebuf(bool list, char *buf, const unsigned long *maskp, int nmaskbits); +extern int bitmap_print_bitmask_to_buf(char *buf, const unsigned long *maskp, + int nmaskbits, loff_t off, size_t count); +extern int bitmap_print_list_to_buf(char *buf, const unsigned long *maskp, + int nmaskbits, loff_t off, size_t count); +int bitmap_parse(const char *buf, unsigned int buflen, unsigned long *dst, int nbits); +int bitmap_parselist(const char *buf, unsigned long *maskp, int nmaskbits); +int bitmap_parselist_user(const char __user *ubuf, unsigned int ulen, + unsigned long *dst, int nbits); + +#endif /* __LINUX_BITMAP_STR_H */ diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h index 1516ff979315..1cca950a54ae 100644 --- a/include/linux/bitmap.h +++ b/include/linux/bitmap.h @@ -10,6 +10,7 @@ #include #include #include +#include struct device; @@ -200,14 +201,6 @@ bitmap_find_next_zero_area(unsigned long *map, align_mask, 0); } -int bitmap_parse(const char *buf, unsigned int buflen, - unsigned long *dst, int nbits); -int bitmap_parse_user(const char __user *ubuf, unsigned int ulen, - unsigned long *dst, int nbits); -int bitmap_parselist(const char *buf, unsigned long *maskp, - int nmaskbits); -int bitmap_parselist_user(const char __user *ubuf, unsigned int ulen, - unsigned long *dst, int nbits); void bitmap_remap(unsigned long *dst, const unsigned long *src, const unsigned long *old, const unsigned long *new, unsigned int nbits); int bitmap_bitremap(int oldbit, @@ -220,15 +213,6 @@ int bitmap_find_free_region(unsigned long *bitmap, unsigned int bits, int order) void bitmap_release_region(unsigned long *bitmap, unsigned int pos, int order); int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos, int order); -int bitmap_print_to_pagebuf(bool list, char *buf, - const unsigned long *maskp, int nmaskbits); - -extern int bitmap_print_bitmask_to_buf(char *buf, const unsigned long *maskp, - int nmaskbits, loff_t off, size_t count); - -extern int bitmap_print_list_to_buf(char *buf, const unsigned long *maskp, - int nmaskbits, loff_t off, size_t count); - #define BITMAP_FIRST_WORD_MASK(start) (~0UL << ((start) & (BITS_PER_LONG - 1))) #define BITMAP_LAST_WORD_MASK(nbits) (~0UL >> (-(nbits) & (BITS_PER_LONG - 1))) diff --git a/lib/Makefile b/lib/Makefile index 740109b6e2c8..9e8f9f6dd3b2 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -48,7 +48,7 @@ obj-y += bcd.o sort.o parser.o debug_locks.o random32.o \ bsearch.o find_bit.o llist.o memweight.o kfifo.o \ percpu-refcount.o rhashtable.o base64.o \ once.o refcount.o rcuref.o usercopy.o errseq.o bucket_locks.o \ - generic-radix-tree.o + generic-radix-tree.o bitmap-str.o obj-$(CONFIG_STRING_SELFTEST) += test_string.o obj-y += string_helpers.o obj-$(CONFIG_TEST_STRING_HELPERS) += test-string_helpers.o diff --git a/lib/bitmap-str.c b/lib/bitmap-str.c new file mode 100644 index 000000000000..be745209507a --- /dev/null +++ b/lib/bitmap-str.c @@ -0,0 +1,510 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "kstrtox.h" + +/** + * bitmap_parse_user - convert an ASCII hex string in a user buffer into a bitmap + * + * @ubuf: pointer to user buffer containing string. + * @ulen: buffer size in bytes. If string is smaller than this + * then it must be terminated with a \0. + * @maskp: pointer to bitmap array that will contain result. + * @nmaskbits: size of bitmap, in bits. + */ +int bitmap_parse_user(const char __user *ubuf, + unsigned int ulen, unsigned long *maskp, + int nmaskbits) +{ + char *buf; + int ret; + + buf = memdup_user_nul(ubuf, ulen); + if (IS_ERR(buf)) + return PTR_ERR(buf); + + ret = bitmap_parse(buf, UINT_MAX, maskp, nmaskbits); + + kfree(buf); + return ret; +} +EXPORT_SYMBOL(bitmap_parse_user); + +/** + * bitmap_print_to_pagebuf - convert bitmap to list or hex format ASCII string + * @list: indicates whether the bitmap must be list + * @buf: page aligned buffer into which string is placed + * @maskp: pointer to bitmap to convert + * @nmaskbits: size of bitmap, in bits + * + * Output format is a comma-separated list of decimal numbers and + * ranges if list is specified or hex digits grouped into comma-separated + * sets of 8 digits/set. Returns the number of characters written to buf. + * + * It is assumed that @buf is a pointer into a PAGE_SIZE, page-aligned + * area and that sufficient storage remains at @buf to accommodate the + * bitmap_print_to_pagebuf() output. Returns the number of characters + * actually printed to @buf, excluding terminating '\0'. + */ +int bitmap_print_to_pagebuf(bool list, char *buf, const unsigned long *maskp, + int nmaskbits) +{ + ptrdiff_t len = PAGE_SIZE - offset_in_page(buf); + + return list ? scnprintf(buf, len, "%*pbl\n", nmaskbits, maskp) : + scnprintf(buf, len, "%*pb\n", nmaskbits, maskp); +} +EXPORT_SYMBOL(bitmap_print_to_pagebuf); + +/** + * bitmap_print_to_buf - convert bitmap to list or hex format ASCII string + * @list: indicates whether the bitmap must be list + * true: print in decimal list format + * false: print in hexadecimal bitmask format + * @buf: buffer into which string is placed + * @maskp: pointer to bitmap to convert + * @nmaskbits: size of bitmap, in bits + * @off: in the string from which we are copying, We copy to @buf + * @count: the maximum number of bytes to print + */ +static int bitmap_print_to_buf(bool list, char *buf, const unsigned long *maskp, + int nmaskbits, loff_t off, size_t count) +{ + const char *fmt = list ? "%*pbl\n" : "%*pb\n"; + ssize_t size; + void *data; + + data = kasprintf(GFP_KERNEL, fmt, nmaskbits, maskp); + if (!data) + return -ENOMEM; + + size = memory_read_from_buffer(buf, count, &off, data, strlen(data) + 1); + kfree(data); + + return size; +} + +/** + * bitmap_print_bitmask_to_buf - convert bitmap to hex bitmask format ASCII string + * @buf: buffer into which string is placed + * @maskp: pointer to bitmap to convert + * @nmaskbits: size of bitmap, in bits + * @off: in the string from which we are copying, We copy to @buf + * @count: the maximum number of bytes to print + * + * The bitmap_print_to_pagebuf() is used indirectly via its cpumap wrapper + * cpumap_print_to_pagebuf() or directly by drivers to export hexadecimal + * bitmask and decimal list to userspace by sysfs ABI. + * Drivers might be using a normal attribute for this kind of ABIs. A + * normal attribute typically has show entry as below:: + * + * static ssize_t example_attribute_show(struct device *dev, + * struct device_attribute *attr, char *buf) + * { + * ... + * return bitmap_print_to_pagebuf(true, buf, &mask, nr_trig_max); + * } + * + * show entry of attribute has no offset and count parameters and this + * means the file is limited to one page only. + * bitmap_print_to_pagebuf() API works terribly well for this kind of + * normal attribute with buf parameter and without offset, count:: + * + * bitmap_print_to_pagebuf(bool list, char *buf, const unsigned long *maskp, + * int nmaskbits) + * { + * } + * + * The problem is once we have a large bitmap, we have a chance to get a + * bitmask or list more than one page. Especially for list, it could be + * as complex as 0,3,5,7,9,... We have no simple way to know it exact size. + * It turns out bin_attribute is a way to break this limit. bin_attribute + * has show entry as below:: + * + * static ssize_t + * example_bin_attribute_show(struct file *filp, struct kobject *kobj, + * struct bin_attribute *attr, char *buf, + * loff_t offset, size_t count) + * { + * ... + * } + * + * With the new offset and count parameters, this makes sysfs ABI be able + * to support file size more than one page. For example, offset could be + * >= 4096. + * bitmap_print_bitmask_to_buf(), bitmap_print_list_to_buf() wit their + * cpumap wrapper cpumap_print_bitmask_to_buf(), cpumap_print_list_to_buf() + * make those drivers be able to support large bitmask and list after they + * move to use bin_attribute. In result, we have to pass the corresponding + * parameters such as off, count from bin_attribute show entry to this API. + * + * The role of cpumap_print_bitmask_to_buf() and cpumap_print_list_to_buf() + * is similar with cpumap_print_to_pagebuf(), the difference is that + * bitmap_print_to_pagebuf() mainly serves sysfs attribute with the assumption + * the destination buffer is exactly one page and won't be more than one page. + * cpumap_print_bitmask_to_buf() and cpumap_print_list_to_buf(), on the other + * hand, mainly serves bin_attribute which doesn't work with exact one page, + * and it can break the size limit of converted decimal list and hexadecimal + * bitmask. + * + * WARNING! + * + * This function is not a replacement for sprintf() or bitmap_print_to_pagebuf(). + * It is intended to workaround sysfs limitations discussed above and should be + * used carefully in general case for the following reasons: + * + * - Time complexity is O(nbits^2/count), comparing to O(nbits) for snprintf(). + * - Memory complexity is O(nbits), comparing to O(1) for snprintf(). + * - @off and @count are NOT offset and number of bits to print. + * - If printing part of bitmap as list, the resulting string is not a correct + * list representation of bitmap. Particularly, some bits within or out of + * related interval may be erroneously set or unset. The format of the string + * may be broken, so bitmap_parselist-like parser may fail parsing it. + * - If printing the whole bitmap as list by parts, user must ensure the order + * of calls of the function such that the offset is incremented linearly. + * - If printing the whole bitmap as list by parts, user must keep bitmap + * unchanged between the very first and very last call. Otherwise concatenated + * result may be incorrect, and format may be broken. + * + * Returns the number of characters actually printed to @buf + */ +int bitmap_print_bitmask_to_buf(char *buf, const unsigned long *maskp, + int nmaskbits, loff_t off, size_t count) +{ + return bitmap_print_to_buf(false, buf, maskp, nmaskbits, off, count); +} +EXPORT_SYMBOL(bitmap_print_bitmask_to_buf); + +/** + * bitmap_print_list_to_buf - convert bitmap to decimal list format ASCII string + * @buf: buffer into which string is placed + * @maskp: pointer to bitmap to convert + * @nmaskbits: size of bitmap, in bits + * @off: in the string from which we are copying, We copy to @buf + * @count: the maximum number of bytes to print + * + * Everything is same with the above bitmap_print_bitmask_to_buf() except + * the print format. + */ +int bitmap_print_list_to_buf(char *buf, const unsigned long *maskp, + int nmaskbits, loff_t off, size_t count) +{ + return bitmap_print_to_buf(true, buf, maskp, nmaskbits, off, count); +} +EXPORT_SYMBOL(bitmap_print_list_to_buf); + +/* + * Region 9-38:4/10 describes the following bitmap structure: + * 0 9 12 18 38 N + * .........****......****......****.................. + * ^ ^ ^ ^ ^ + * start off group_len end nbits + */ +struct region { + unsigned int start; + unsigned int off; + unsigned int group_len; + unsigned int end; + unsigned int nbits; +}; + +static void bitmap_set_region(const struct region *r, unsigned long *bitmap) +{ + unsigned int start; + + for (start = r->start; start <= r->end; start += r->group_len) + bitmap_set(bitmap, start, min(r->end - start + 1, r->off)); +} + +static int bitmap_check_region(const struct region *r) +{ + if (r->start > r->end || r->group_len == 0 || r->off > r->group_len) + return -EINVAL; + + if (r->end >= r->nbits) + return -ERANGE; + + return 0; +} + +static const char *bitmap_getnum(const char *str, unsigned int *num, + unsigned int lastbit) +{ + unsigned long long n; + unsigned int len; + + if (str[0] == 'N') { + *num = lastbit; + return str + 1; + } + + len = _parse_integer(str, 10, &n); + if (!len) + return ERR_PTR(-EINVAL); + if (len & KSTRTOX_OVERFLOW || n != (unsigned int)n) + return ERR_PTR(-EOVERFLOW); + + *num = n; + return str + len; +} + +static inline bool end_of_str(char c) +{ + return c == '\0' || c == '\n'; +} + +static inline bool __end_of_region(char c) +{ + return isspace(c) || c == ','; +} + +static inline bool end_of_region(char c) +{ + return __end_of_region(c) || end_of_str(c); +} + +/* + * The format allows commas and whitespaces at the beginning + * of the region. + */ +static const char *bitmap_find_region(const char *str) +{ + while (__end_of_region(*str)) + str++; + + return end_of_str(*str) ? NULL : str; +} + +static const char *bitmap_find_region_reverse(const char *start, const char *end) +{ + while (start <= end && __end_of_region(*end)) + end--; + + return end; +} + +static const char *bitmap_parse_region(const char *str, struct region *r) +{ + unsigned int lastbit = r->nbits - 1; + + if (!strncasecmp(str, "all", 3)) { + r->start = 0; + r->end = lastbit; + str += 3; + + goto check_pattern; + } + + str = bitmap_getnum(str, &r->start, lastbit); + if (IS_ERR(str)) + return str; + + if (end_of_region(*str)) + goto no_end; + + if (*str != '-') + return ERR_PTR(-EINVAL); + + str = bitmap_getnum(str + 1, &r->end, lastbit); + if (IS_ERR(str)) + return str; + +check_pattern: + if (end_of_region(*str)) + goto no_pattern; + + if (*str != ':') + return ERR_PTR(-EINVAL); + + str = bitmap_getnum(str + 1, &r->off, lastbit); + if (IS_ERR(str)) + return str; + + if (*str != '/') + return ERR_PTR(-EINVAL); + + return bitmap_getnum(str + 1, &r->group_len, lastbit); + +no_end: + r->end = r->start; +no_pattern: + r->off = r->end + 1; + r->group_len = r->end + 1; + + return end_of_str(*str) ? NULL : str; +} + +/** + * bitmap_parselist - convert list format ASCII string to bitmap + * @buf: read user string from this buffer; must be terminated + * with a \0 or \n. + * @maskp: write resulting mask here + * @nmaskbits: number of bits in mask to be written + * + * Input format is a comma-separated list of decimal numbers and + * ranges. Consecutively set bits are shown as two hyphen-separated + * decimal numbers, the smallest and largest bit numbers set in + * the range. + * Optionally each range can be postfixed to denote that only parts of it + * should be set. The range will divided to groups of specific size. + * From each group will be used only defined amount of bits. + * Syntax: range:used_size/group_size + * Example: 0-1023:2/256 ==> 0,1,256,257,512,513,768,769 + * The value 'N' can be used as a dynamically substituted token for the + * maximum allowed value; i.e (nmaskbits - 1). Keep in mind that it is + * dynamic, so if system changes cause the bitmap width to change, such + * as more cores in a CPU list, then any ranges using N will also change. + * + * Returns: 0 on success, -errno on invalid input strings. Error values: + * + * - ``-EINVAL``: wrong region format + * - ``-EINVAL``: invalid character in string + * - ``-ERANGE``: bit number specified too large for mask + * - ``-EOVERFLOW``: integer overflow in the input parameters + */ +int bitmap_parselist(const char *buf, unsigned long *maskp, int nmaskbits) +{ + struct region r; + long ret; + + r.nbits = nmaskbits; + bitmap_zero(maskp, r.nbits); + + while (buf) { + buf = bitmap_find_region(buf); + if (buf == NULL) + return 0; + + buf = bitmap_parse_region(buf, &r); + if (IS_ERR(buf)) + return PTR_ERR(buf); + + ret = bitmap_check_region(&r); + if (ret) + return ret; + + bitmap_set_region(&r, maskp); + } + + return 0; +} +EXPORT_SYMBOL(bitmap_parselist); + + +/** + * bitmap_parselist_user() - convert user buffer's list format ASCII + * string to bitmap + * + * @ubuf: pointer to user buffer containing string. + * @ulen: buffer size in bytes. If string is smaller than this + * then it must be terminated with a \0. + * @maskp: pointer to bitmap array that will contain result. + * @nmaskbits: size of bitmap, in bits. + * + * Wrapper for bitmap_parselist(), providing it with user buffer. + */ +int bitmap_parselist_user(const char __user *ubuf, + unsigned int ulen, unsigned long *maskp, + int nmaskbits) +{ + char *buf; + int ret; + + buf = memdup_user_nul(ubuf, ulen); + if (IS_ERR(buf)) + return PTR_ERR(buf); + + ret = bitmap_parselist(buf, maskp, nmaskbits); + + kfree(buf); + return ret; +} +EXPORT_SYMBOL(bitmap_parselist_user); + +static const char *bitmap_get_x32_reverse(const char *start, + const char *end, u32 *num) +{ + u32 ret = 0; + int c, i; + + for (i = 0; i < 32; i += 4) { + c = hex_to_bin(*end--); + if (c < 0) + return ERR_PTR(-EINVAL); + + ret |= c << i; + + if (start > end || __end_of_region(*end)) + goto out; + } + + if (hex_to_bin(*end--) >= 0) + return ERR_PTR(-EOVERFLOW); +out: + *num = ret; + return end; +} + +/** + * bitmap_parse - convert an ASCII hex string into a bitmap. + * @start: pointer to buffer containing string. + * @buflen: buffer size in bytes. If string is smaller than this + * then it must be terminated with a \0 or \n. In that case, + * UINT_MAX may be provided instead of string length. + * @maskp: pointer to bitmap array that will contain result. + * @nmaskbits: size of bitmap, in bits. + * + * Commas group hex digits into chunks. Each chunk defines exactly 32 + * bits of the resultant bitmask. No chunk may specify a value larger + * than 32 bits (%-EOVERFLOW), and if a chunk specifies a smaller value + * then leading 0-bits are prepended. %-EINVAL is returned for illegal + * characters. Grouping such as "1,,5", ",44", "," or "" is allowed. + * Leading, embedded and trailing whitespace accepted. + */ +int bitmap_parse(const char *start, unsigned int buflen, + unsigned long *maskp, int nmaskbits) +{ + const char *end = strnchrnul(start, buflen, '\n') - 1; + int chunks = BITS_TO_U32(nmaskbits); + u32 *bitmap = (u32 *)maskp; + int unset_bit; + int chunk; + + for (chunk = 0; ; chunk++) { + end = bitmap_find_region_reverse(start, end); + if (start > end) + break; + + if (!chunks--) + return -EOVERFLOW; + +#if defined(CONFIG_64BIT) && defined(__BIG_ENDIAN) + end = bitmap_get_x32_reverse(start, end, &bitmap[chunk ^ 1]); +#else + end = bitmap_get_x32_reverse(start, end, &bitmap[chunk]); +#endif + if (IS_ERR(end)) + return PTR_ERR(end); + } + + unset_bit = (BITS_TO_U32(nmaskbits) - chunks) * 32; + if (unset_bit < nmaskbits) { + bitmap_clear(maskp, unset_bit, nmaskbits - unset_bit); + return 0; + } + + if (find_next_bit(maskp, unset_bit, nmaskbits) != unset_bit) + return -EOVERFLOW; + + return 0; +} +EXPORT_SYMBOL(bitmap_parse); diff --git a/lib/bitmap.c b/lib/bitmap.c index 935e0f96e785..abc5579768e9 100644 --- a/lib/bitmap.c +++ b/lib/bitmap.c @@ -6,21 +6,11 @@ #include #include -#include #include #include #include #include -#include -#include #include -#include -#include -#include - -#include - -#include "kstrtox.h" /** * DOC: bitmap introduction @@ -440,508 +430,6 @@ again: } EXPORT_SYMBOL(bitmap_find_next_zero_area_off); -/* - * Bitmap printing & parsing functions: first version by Nadia Yvette Chambers, - * second version by Paul Jackson, third by Joe Korty. - */ - -/** - * bitmap_parse_user - convert an ASCII hex string in a user buffer into a bitmap - * - * @ubuf: pointer to user buffer containing string. - * @ulen: buffer size in bytes. If string is smaller than this - * then it must be terminated with a \0. - * @maskp: pointer to bitmap array that will contain result. - * @nmaskbits: size of bitmap, in bits. - */ -int bitmap_parse_user(const char __user *ubuf, - unsigned int ulen, unsigned long *maskp, - int nmaskbits) -{ - char *buf; - int ret; - - buf = memdup_user_nul(ubuf, ulen); - if (IS_ERR(buf)) - return PTR_ERR(buf); - - ret = bitmap_parse(buf, UINT_MAX, maskp, nmaskbits); - - kfree(buf); - return ret; -} -EXPORT_SYMBOL(bitmap_parse_user); - -/** - * bitmap_print_to_pagebuf - convert bitmap to list or hex format ASCII string - * @list: indicates whether the bitmap must be list - * @buf: page aligned buffer into which string is placed - * @maskp: pointer to bitmap to convert - * @nmaskbits: size of bitmap, in bits - * - * Output format is a comma-separated list of decimal numbers and - * ranges if list is specified or hex digits grouped into comma-separated - * sets of 8 digits/set. Returns the number of characters written to buf. - * - * It is assumed that @buf is a pointer into a PAGE_SIZE, page-aligned - * area and that sufficient storage remains at @buf to accommodate the - * bitmap_print_to_pagebuf() output. Returns the number of characters - * actually printed to @buf, excluding terminating '\0'. - */ -int bitmap_print_to_pagebuf(bool list, char *buf, const unsigned long *maskp, - int nmaskbits) -{ - ptrdiff_t len = PAGE_SIZE - offset_in_page(buf); - - return list ? scnprintf(buf, len, "%*pbl\n", nmaskbits, maskp) : - scnprintf(buf, len, "%*pb\n", nmaskbits, maskp); -} -EXPORT_SYMBOL(bitmap_print_to_pagebuf); - -/** - * bitmap_print_to_buf - convert bitmap to list or hex format ASCII string - * @list: indicates whether the bitmap must be list - * true: print in decimal list format - * false: print in hexadecimal bitmask format - * @buf: buffer into which string is placed - * @maskp: pointer to bitmap to convert - * @nmaskbits: size of bitmap, in bits - * @off: in the string from which we are copying, We copy to @buf - * @count: the maximum number of bytes to print - */ -static int bitmap_print_to_buf(bool list, char *buf, const unsigned long *maskp, - int nmaskbits, loff_t off, size_t count) -{ - const char *fmt = list ? "%*pbl\n" : "%*pb\n"; - ssize_t size; - void *data; - - data = kasprintf(GFP_KERNEL, fmt, nmaskbits, maskp); - if (!data) - return -ENOMEM; - - size = memory_read_from_buffer(buf, count, &off, data, strlen(data) + 1); - kfree(data); - - return size; -} - -/** - * bitmap_print_bitmask_to_buf - convert bitmap to hex bitmask format ASCII string - * @buf: buffer into which string is placed - * @maskp: pointer to bitmap to convert - * @nmaskbits: size of bitmap, in bits - * @off: in the string from which we are copying, We copy to @buf - * @count: the maximum number of bytes to print - * - * The bitmap_print_to_pagebuf() is used indirectly via its cpumap wrapper - * cpumap_print_to_pagebuf() or directly by drivers to export hexadecimal - * bitmask and decimal list to userspace by sysfs ABI. - * Drivers might be using a normal attribute for this kind of ABIs. A - * normal attribute typically has show entry as below:: - * - * static ssize_t example_attribute_show(struct device *dev, - * struct device_attribute *attr, char *buf) - * { - * ... - * return bitmap_print_to_pagebuf(true, buf, &mask, nr_trig_max); - * } - * - * show entry of attribute has no offset and count parameters and this - * means the file is limited to one page only. - * bitmap_print_to_pagebuf() API works terribly well for this kind of - * normal attribute with buf parameter and without offset, count:: - * - * bitmap_print_to_pagebuf(bool list, char *buf, const unsigned long *maskp, - * int nmaskbits) - * { - * } - * - * The problem is once we have a large bitmap, we have a chance to get a - * bitmask or list more than one page. Especially for list, it could be - * as complex as 0,3,5,7,9,... We have no simple way to know it exact size. - * It turns out bin_attribute is a way to break this limit. bin_attribute - * has show entry as below:: - * - * static ssize_t - * example_bin_attribute_show(struct file *filp, struct kobject *kobj, - * struct bin_attribute *attr, char *buf, - * loff_t offset, size_t count) - * { - * ... - * } - * - * With the new offset and count parameters, this makes sysfs ABI be able - * to support file size more than one page. For example, offset could be - * >= 4096. - * bitmap_print_bitmask_to_buf(), bitmap_print_list_to_buf() wit their - * cpumap wrapper cpumap_print_bitmask_to_buf(), cpumap_print_list_to_buf() - * make those drivers be able to support large bitmask and list after they - * move to use bin_attribute. In result, we have to pass the corresponding - * parameters such as off, count from bin_attribute show entry to this API. - * - * The role of cpumap_print_bitmask_to_buf() and cpumap_print_list_to_buf() - * is similar with cpumap_print_to_pagebuf(), the difference is that - * bitmap_print_to_pagebuf() mainly serves sysfs attribute with the assumption - * the destination buffer is exactly one page and won't be more than one page. - * cpumap_print_bitmask_to_buf() and cpumap_print_list_to_buf(), on the other - * hand, mainly serves bin_attribute which doesn't work with exact one page, - * and it can break the size limit of converted decimal list and hexadecimal - * bitmask. - * - * WARNING! - * - * This function is not a replacement for sprintf() or bitmap_print_to_pagebuf(). - * It is intended to workaround sysfs limitations discussed above and should be - * used carefully in general case for the following reasons: - * - * - Time complexity is O(nbits^2/count), comparing to O(nbits) for snprintf(). - * - Memory complexity is O(nbits), comparing to O(1) for snprintf(). - * - @off and @count are NOT offset and number of bits to print. - * - If printing part of bitmap as list, the resulting string is not a correct - * list representation of bitmap. Particularly, some bits within or out of - * related interval may be erroneously set or unset. The format of the string - * may be broken, so bitmap_parselist-like parser may fail parsing it. - * - If printing the whole bitmap as list by parts, user must ensure the order - * of calls of the function such that the offset is incremented linearly. - * - If printing the whole bitmap as list by parts, user must keep bitmap - * unchanged between the very first and very last call. Otherwise concatenated - * result may be incorrect, and format may be broken. - * - * Returns the number of characters actually printed to @buf - */ -int bitmap_print_bitmask_to_buf(char *buf, const unsigned long *maskp, - int nmaskbits, loff_t off, size_t count) -{ - return bitmap_print_to_buf(false, buf, maskp, nmaskbits, off, count); -} -EXPORT_SYMBOL(bitmap_print_bitmask_to_buf); - -/** - * bitmap_print_list_to_buf - convert bitmap to decimal list format ASCII string - * @buf: buffer into which string is placed - * @maskp: pointer to bitmap to convert - * @nmaskbits: size of bitmap, in bits - * @off: in the string from which we are copying, We copy to @buf - * @count: the maximum number of bytes to print - * - * Everything is same with the above bitmap_print_bitmask_to_buf() except - * the print format. - */ -int bitmap_print_list_to_buf(char *buf, const unsigned long *maskp, - int nmaskbits, loff_t off, size_t count) -{ - return bitmap_print_to_buf(true, buf, maskp, nmaskbits, off, count); -} -EXPORT_SYMBOL(bitmap_print_list_to_buf); - -/* - * Region 9-38:4/10 describes the following bitmap structure: - * 0 9 12 18 38 N - * .........****......****......****.................. - * ^ ^ ^ ^ ^ - * start off group_len end nbits - */ -struct region { - unsigned int start; - unsigned int off; - unsigned int group_len; - unsigned int end; - unsigned int nbits; -}; - -static void bitmap_set_region(const struct region *r, unsigned long *bitmap) -{ - unsigned int start; - - for (start = r->start; start <= r->end; start += r->group_len) - bitmap_set(bitmap, start, min(r->end - start + 1, r->off)); -} - -static int bitmap_check_region(const struct region *r) -{ - if (r->start > r->end || r->group_len == 0 || r->off > r->group_len) - return -EINVAL; - - if (r->end >= r->nbits) - return -ERANGE; - - return 0; -} - -static const char *bitmap_getnum(const char *str, unsigned int *num, - unsigned int lastbit) -{ - unsigned long long n; - unsigned int len; - - if (str[0] == 'N') { - *num = lastbit; - return str + 1; - } - - len = _parse_integer(str, 10, &n); - if (!len) - return ERR_PTR(-EINVAL); - if (len & KSTRTOX_OVERFLOW || n != (unsigned int)n) - return ERR_PTR(-EOVERFLOW); - - *num = n; - return str + len; -} - -static inline bool end_of_str(char c) -{ - return c == '\0' || c == '\n'; -} - -static inline bool __end_of_region(char c) -{ - return isspace(c) || c == ','; -} - -static inline bool end_of_region(char c) -{ - return __end_of_region(c) || end_of_str(c); -} - -/* - * The format allows commas and whitespaces at the beginning - * of the region. - */ -static const char *bitmap_find_region(const char *str) -{ - while (__end_of_region(*str)) - str++; - - return end_of_str(*str) ? NULL : str; -} - -static const char *bitmap_find_region_reverse(const char *start, const char *end) -{ - while (start <= end && __end_of_region(*end)) - end--; - - return end; -} - -static const char *bitmap_parse_region(const char *str, struct region *r) -{ - unsigned int lastbit = r->nbits - 1; - - if (!strncasecmp(str, "all", 3)) { - r->start = 0; - r->end = lastbit; - str += 3; - - goto check_pattern; - } - - str = bitmap_getnum(str, &r->start, lastbit); - if (IS_ERR(str)) - return str; - - if (end_of_region(*str)) - goto no_end; - - if (*str != '-') - return ERR_PTR(-EINVAL); - - str = bitmap_getnum(str + 1, &r->end, lastbit); - if (IS_ERR(str)) - return str; - -check_pattern: - if (end_of_region(*str)) - goto no_pattern; - - if (*str != ':') - return ERR_PTR(-EINVAL); - - str = bitmap_getnum(str + 1, &r->off, lastbit); - if (IS_ERR(str)) - return str; - - if (*str != '/') - return ERR_PTR(-EINVAL); - - return bitmap_getnum(str + 1, &r->group_len, lastbit); - -no_end: - r->end = r->start; -no_pattern: - r->off = r->end + 1; - r->group_len = r->end + 1; - - return end_of_str(*str) ? NULL : str; -} - -/** - * bitmap_parselist - convert list format ASCII string to bitmap - * @buf: read user string from this buffer; must be terminated - * with a \0 or \n. - * @maskp: write resulting mask here - * @nmaskbits: number of bits in mask to be written - * - * Input format is a comma-separated list of decimal numbers and - * ranges. Consecutively set bits are shown as two hyphen-separated - * decimal numbers, the smallest and largest bit numbers set in - * the range. - * Optionally each range can be postfixed to denote that only parts of it - * should be set. The range will divided to groups of specific size. - * From each group will be used only defined amount of bits. - * Syntax: range:used_size/group_size - * Example: 0-1023:2/256 ==> 0,1,256,257,512,513,768,769 - * The value 'N' can be used as a dynamically substituted token for the - * maximum allowed value; i.e (nmaskbits - 1). Keep in mind that it is - * dynamic, so if system changes cause the bitmap width to change, such - * as more cores in a CPU list, then any ranges using N will also change. - * - * Returns: 0 on success, -errno on invalid input strings. Error values: - * - * - ``-EINVAL``: wrong region format - * - ``-EINVAL``: invalid character in string - * - ``-ERANGE``: bit number specified too large for mask - * - ``-EOVERFLOW``: integer overflow in the input parameters - */ -int bitmap_parselist(const char *buf, unsigned long *maskp, int nmaskbits) -{ - struct region r; - long ret; - - r.nbits = nmaskbits; - bitmap_zero(maskp, r.nbits); - - while (buf) { - buf = bitmap_find_region(buf); - if (buf == NULL) - return 0; - - buf = bitmap_parse_region(buf, &r); - if (IS_ERR(buf)) - return PTR_ERR(buf); - - ret = bitmap_check_region(&r); - if (ret) - return ret; - - bitmap_set_region(&r, maskp); - } - - return 0; -} -EXPORT_SYMBOL(bitmap_parselist); - - -/** - * bitmap_parselist_user() - convert user buffer's list format ASCII - * string to bitmap - * - * @ubuf: pointer to user buffer containing string. - * @ulen: buffer size in bytes. If string is smaller than this - * then it must be terminated with a \0. - * @maskp: pointer to bitmap array that will contain result. - * @nmaskbits: size of bitmap, in bits. - * - * Wrapper for bitmap_parselist(), providing it with user buffer. - */ -int bitmap_parselist_user(const char __user *ubuf, - unsigned int ulen, unsigned long *maskp, - int nmaskbits) -{ - char *buf; - int ret; - - buf = memdup_user_nul(ubuf, ulen); - if (IS_ERR(buf)) - return PTR_ERR(buf); - - ret = bitmap_parselist(buf, maskp, nmaskbits); - - kfree(buf); - return ret; -} -EXPORT_SYMBOL(bitmap_parselist_user); - -static const char *bitmap_get_x32_reverse(const char *start, - const char *end, u32 *num) -{ - u32 ret = 0; - int c, i; - - for (i = 0; i < 32; i += 4) { - c = hex_to_bin(*end--); - if (c < 0) - return ERR_PTR(-EINVAL); - - ret |= c << i; - - if (start > end || __end_of_region(*end)) - goto out; - } - - if (hex_to_bin(*end--) >= 0) - return ERR_PTR(-EOVERFLOW); -out: - *num = ret; - return end; -} - -/** - * bitmap_parse - convert an ASCII hex string into a bitmap. - * @start: pointer to buffer containing string. - * @buflen: buffer size in bytes. If string is smaller than this - * then it must be terminated with a \0 or \n. In that case, - * UINT_MAX may be provided instead of string length. - * @maskp: pointer to bitmap array that will contain result. - * @nmaskbits: size of bitmap, in bits. - * - * Commas group hex digits into chunks. Each chunk defines exactly 32 - * bits of the resultant bitmask. No chunk may specify a value larger - * than 32 bits (%-EOVERFLOW), and if a chunk specifies a smaller value - * then leading 0-bits are prepended. %-EINVAL is returned for illegal - * characters. Grouping such as "1,,5", ",44", "," or "" is allowed. - * Leading, embedded and trailing whitespace accepted. - */ -int bitmap_parse(const char *start, unsigned int buflen, - unsigned long *maskp, int nmaskbits) -{ - const char *end = strnchrnul(start, buflen, '\n') - 1; - int chunks = BITS_TO_U32(nmaskbits); - u32 *bitmap = (u32 *)maskp; - int unset_bit; - int chunk; - - for (chunk = 0; ; chunk++) { - end = bitmap_find_region_reverse(start, end); - if (start > end) - break; - - if (!chunks--) - return -EOVERFLOW; - -#if defined(CONFIG_64BIT) && defined(__BIG_ENDIAN) - end = bitmap_get_x32_reverse(start, end, &bitmap[chunk ^ 1]); -#else - end = bitmap_get_x32_reverse(start, end, &bitmap[chunk]); -#endif - if (IS_ERR(end)) - return PTR_ERR(end); - } - - unset_bit = (BITS_TO_U32(nmaskbits) - chunks) * 32; - if (unset_bit < nmaskbits) { - bitmap_clear(maskp, unset_bit, nmaskbits - unset_bit); - return 0; - } - - if (find_next_bit(maskp, unset_bit, nmaskbits) != unset_bit) - return -EOVERFLOW; - - return 0; -} -EXPORT_SYMBOL(bitmap_parse); - /** * bitmap_pos_to_ord - find ordinal of set bit at given position in bitmap * @buf: pointer to a bitmap From 82bf9bdfbce9cfa73ccd86642542159119e0419f Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Sun, 24 Sep 2023 19:38:10 -0700 Subject: [PATCH 218/475] bitmap: align __reg_op() wrappers with modern coding style Fix comments so that scripts/kernel-doc doesn't warn, and fix for-loop stype in bitmap_find_free_region(). CC: Rasmus Villemoes Suggested-by: Andy Shevchenko Reviewed-by: Andy Shevchenko Signed-off-by: Yury Norov --- lib/bitmap.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/lib/bitmap.c b/lib/bitmap.c index abc5579768e9..f77243ba5e1c 100644 --- a/lib/bitmap.c +++ b/lib/bitmap.c @@ -722,8 +722,8 @@ void bitmap_fold(unsigned long *dst, const unsigned long *orig, * some size '1 << order' (a power of two), aligned to that same * '1 << order' power of two. * - * Returns 1 if REG_OP_ISFREE succeeds (region is all zero bits). - * Returns 0 in all other cases and reg_ops. + * Return: 1 if REG_OP_ISFREE succeeds (region is all zero bits). + * 0 in all other cases and reg_ops. */ enum { @@ -795,14 +795,14 @@ done: * a power (@order) of two, aligned to that power of two, which * makes the search algorithm much faster. * - * Return the bit offset in bitmap of the allocated region, + * Return: the bit offset in bitmap of the allocated region, * or -errno on failure. */ int bitmap_find_free_region(unsigned long *bitmap, unsigned int bits, int order) { unsigned int pos, end; /* scans bitmap by regions of size order */ - for (pos = 0 ; (end = pos + (1U << order)) <= bits; pos = end) { + for (pos = 0; (end = pos + BIT(order)) <= bits; pos = end) { if (!__reg_op(bitmap, pos, order, REG_OP_ISFREE)) continue; __reg_op(bitmap, pos, order, REG_OP_ALLOC); @@ -820,8 +820,6 @@ EXPORT_SYMBOL(bitmap_find_free_region); * * This is the complement to __bitmap_find_free_region() and releases * the found region (by clearing it in the bitmap). - * - * No return value. */ void bitmap_release_region(unsigned long *bitmap, unsigned int pos, int order) { @@ -837,7 +835,7 @@ EXPORT_SYMBOL(bitmap_release_region); * * Allocate (set bits in) a specified region of a bitmap. * - * Return 0 on success, or %-EBUSY if specified region wasn't + * Return: 0 on success, or %-EBUSY if specified region wasn't * free (not all bits were zero). */ int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos, int order) From 6d5d3a0c33e0b3cab91e2d229f94b51d80d91fe3 Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Sun, 24 Sep 2023 19:38:11 -0700 Subject: [PATCH 219/475] bitmap: add test for bitmap_*_region() functions Test basic functionality of bitmap_{allocate,release,find_free}_region() functions. CC: Rasmus Villemoes Reviewed-by: Andy Shevchenko Signed-off-by: Yury Norov --- lib/test_bitmap.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/lib/test_bitmap.c b/lib/test_bitmap.c index f2ea9f30c7c5..65f22c2578b0 100644 --- a/lib/test_bitmap.c +++ b/lib/test_bitmap.c @@ -330,6 +330,29 @@ static void __init test_copy(void) expect_eq_pbl("0-108,128-1023", bmap2, 1024); } +static void __init test_bitmap_region(void) +{ + int pos, order; + + DECLARE_BITMAP(bmap, 1000); + + bitmap_zero(bmap, 1000); + + for (order = 0; order < 10; order++) { + pos = bitmap_find_free_region(bmap, 1000, order); + if (order == 0) + expect_eq_uint(pos, 0); + else + expect_eq_uint(pos, order < 9 ? BIT(order) : -ENOMEM); + } + + bitmap_release_region(bmap, 0, 0); + for (order = 1; order < 9; order++) + bitmap_release_region(bmap, BIT(order), order); + + expect_eq_uint(bitmap_weight(bmap, 1000), 0); +} + #define EXP2_IN_BITS (sizeof(exp2) * 8) static void __init test_replace(void) @@ -1227,6 +1250,7 @@ static void __init selftest(void) test_zero_clear(); test_fill_set(); test_copy(); + test_bitmap_region(); test_replace(); test_bitmap_arr32(); test_bitmap_arr64(); From b085f969ed3df1916b9d9029225f2472df33cc17 Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Sun, 24 Sep 2023 19:38:12 -0700 Subject: [PATCH 220/475] bitmap: fix opencoded bitmap_allocate_region() bitmap_find_region() opencodes bitmap_allocate_region(). CC: Andy Shevchenko CC: Rasmus Villemoes Signed-off-by: Yury Norov --- lib/bitmap.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/bitmap.c b/lib/bitmap.c index f77243ba5e1c..a0b02974d898 100644 --- a/lib/bitmap.c +++ b/lib/bitmap.c @@ -803,10 +803,8 @@ int bitmap_find_free_region(unsigned long *bitmap, unsigned int bits, int order) unsigned int pos, end; /* scans bitmap by regions of size order */ for (pos = 0; (end = pos + BIT(order)) <= bits; pos = end) { - if (!__reg_op(bitmap, pos, order, REG_OP_ISFREE)) - continue; - __reg_op(bitmap, pos, order, REG_OP_ALLOC); - return pos; + if (!bitmap_allocate_region(bitmap, pos, order)) + return pos; } return -ENOMEM; } From eae5acbd7572f2874cd2f04dd540870dca256826 Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Sun, 24 Sep 2023 19:38:13 -0700 Subject: [PATCH 221/475] bitmap: replace _reg_op(REG_OP_ALLOC) with bitmap_set() _reg_op(REG_OP_ALLOC) duplicates bitmap_set(). CC: Andy Shevchenko CC: Rasmus Villemoes Signed-off-by: Yury Norov --- lib/bitmap.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/bitmap.c b/lib/bitmap.c index a0b02974d898..8ee7d4857179 100644 --- a/lib/bitmap.c +++ b/lib/bitmap.c @@ -838,9 +838,12 @@ EXPORT_SYMBOL(bitmap_release_region); */ int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos, int order) { + unsigned int len = BIT(order); + if (!__reg_op(bitmap, pos, order, REG_OP_ISFREE)) return -EBUSY; - return __reg_op(bitmap, pos, order, REG_OP_ALLOC); + bitmap_set(bitmap, pos, len); + return 0; } EXPORT_SYMBOL(bitmap_allocate_region); From add00c76ee4dafbc35b170bea144358fd62daebb Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Sun, 24 Sep 2023 19:38:14 -0700 Subject: [PATCH 222/475] bitmap: replace _reg_op(REG_OP_RELEASE) with bitmap_clear() _reg_op(REG_OP_RELEASE) duplicates bitmap_clear(). CC: Andy Shevchenko CC: Rasmus Villemoes Signed-off-by: Yury Norov --- lib/bitmap.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bitmap.c b/lib/bitmap.c index 8ee7d4857179..babed4dcb467 100644 --- a/lib/bitmap.c +++ b/lib/bitmap.c @@ -821,7 +821,7 @@ EXPORT_SYMBOL(bitmap_find_free_region); */ void bitmap_release_region(unsigned long *bitmap, unsigned int pos, int order) { - __reg_op(bitmap, pos, order, REG_OP_RELEASE); + bitmap_clear(bitmap, pos, BIT(order)); } EXPORT_SYMBOL(bitmap_release_region); From 9276819a68b52cf2577f77985041faf527cf477c Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Sun, 24 Sep 2023 19:38:15 -0700 Subject: [PATCH 223/475] bitmap: replace _reg_op(REG_OP_ISFREE) with find_next_bit() _reg_op(REG_OP_ISFREE) can be trivially replaced with find_next_bit(). Doing that opens room for potential small_const_nbits() optimization. CC: Andy Shevchenko CC: Rasmus Villemoes Signed-off-by: Yury Norov --- lib/bitmap.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bitmap.c b/lib/bitmap.c index babed4dcb467..e4d761a69fe8 100644 --- a/lib/bitmap.c +++ b/lib/bitmap.c @@ -840,7 +840,7 @@ int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos, int order) { unsigned int len = BIT(order); - if (!__reg_op(bitmap, pos, order, REG_OP_ISFREE)) + if (find_next_bit(bitmap, pos + len, pos) < pos + len) return -EBUSY; bitmap_set(bitmap, pos, len); return 0; From 1d4836527d4168d648010909637fb6cfe45d3fac Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Sun, 24 Sep 2023 19:38:16 -0700 Subject: [PATCH 224/475] bitmap: drop _reg_op() function Now that all _reg_op() users are switched to alternative functions, _reg_op() machinery is not needed anymore. CC: Andy Shevchenko CC: Rasmus Villemoes Signed-off-by: Yury Norov --- lib/bitmap.c | 76 ---------------------------------------------------- 1 file changed, 76 deletions(-) diff --git a/lib/bitmap.c b/lib/bitmap.c index e4d761a69fe8..66da8c33fb1b 100644 --- a/lib/bitmap.c +++ b/lib/bitmap.c @@ -708,82 +708,6 @@ void bitmap_fold(unsigned long *dst, const unsigned long *orig, } #endif /* CONFIG_NUMA */ -/* - * Common code for bitmap_*_region() routines. - * bitmap: array of unsigned longs corresponding to the bitmap - * pos: the beginning of the region - * order: region size (log base 2 of number of bits) - * reg_op: operation(s) to perform on that region of bitmap - * - * Can set, verify and/or release a region of bits in a bitmap, - * depending on which combination of REG_OP_* flag bits is set. - * - * A region of a bitmap is a sequence of bits in the bitmap, of - * some size '1 << order' (a power of two), aligned to that same - * '1 << order' power of two. - * - * Return: 1 if REG_OP_ISFREE succeeds (region is all zero bits). - * 0 in all other cases and reg_ops. - */ - -enum { - REG_OP_ISFREE, /* true if region is all zero bits */ - REG_OP_ALLOC, /* set all bits in region */ - REG_OP_RELEASE, /* clear all bits in region */ -}; - -static int __reg_op(unsigned long *bitmap, unsigned int pos, int order, int reg_op) -{ - int nbits_reg; /* number of bits in region */ - int index; /* index first long of region in bitmap */ - int offset; /* bit offset region in bitmap[index] */ - int nlongs_reg; /* num longs spanned by region in bitmap */ - int nbitsinlong; /* num bits of region in each spanned long */ - unsigned long mask; /* bitmask for one long of region */ - int i; /* scans bitmap by longs */ - int ret = 0; /* return value */ - - /* - * Either nlongs_reg == 1 (for small orders that fit in one long) - * or (offset == 0 && mask == ~0UL) (for larger multiword orders.) - */ - nbits_reg = 1 << order; - index = pos / BITS_PER_LONG; - offset = pos - (index * BITS_PER_LONG); - nlongs_reg = BITS_TO_LONGS(nbits_reg); - nbitsinlong = min(nbits_reg, BITS_PER_LONG); - - /* - * Can't do "mask = (1UL << nbitsinlong) - 1", as that - * overflows if nbitsinlong == BITS_PER_LONG. - */ - mask = (1UL << (nbitsinlong - 1)); - mask += mask - 1; - mask <<= offset; - - switch (reg_op) { - case REG_OP_ISFREE: - for (i = 0; i < nlongs_reg; i++) { - if (bitmap[index + i] & mask) - goto done; - } - ret = 1; /* all bits in region free (zero) */ - break; - - case REG_OP_ALLOC: - for (i = 0; i < nlongs_reg; i++) - bitmap[index + i] |= mask; - break; - - case REG_OP_RELEASE: - for (i = 0; i < nlongs_reg; i++) - bitmap[index + i] &= ~mask; - break; - } -done: - return ret; -} - /** * bitmap_find_free_region - find a contiguous aligned mem region * @bitmap: array of unsigned longs corresponding to the bitmap From 5ebb39eb90376fbf098df04be8b58f235a14ce5a Mon Sep 17 00:00:00 2001 From: Xiu Jianfeng Date: Wed, 9 Aug 2023 10:02:44 +0000 Subject: [PATCH 225/475] apparmor: remove unneeded #ifdef in decompress_zstd() The whole function is guarded by CONFIG_SECURITY_APPARMOR_EXPORT_BINARY, so the #ifdef here is redundant, remove it. Signed-off-by: Xiu Jianfeng Signed-off-by: John Johansen --- security/apparmor/apparmorfs.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/security/apparmor/apparmorfs.c b/security/apparmor/apparmorfs.c index 0e8d690c911b..37234e385fd1 100644 --- a/security/apparmor/apparmorfs.c +++ b/security/apparmor/apparmorfs.c @@ -1314,7 +1314,6 @@ SEQ_RAWDATA_FOPS(compressed_size); static int decompress_zstd(char *src, size_t slen, char *dst, size_t dlen) { -#ifdef CONFIG_SECURITY_APPARMOR_EXPORT_BINARY if (slen < dlen) { const size_t wksp_len = zstd_dctx_workspace_bound(); zstd_dctx *ctx; @@ -1341,7 +1340,6 @@ cleanup: kvfree(wksp); return ret; } -#endif if (dlen < slen) return -EINVAL; From fee5304a9c820bd12bd40b1f2591f8e2be0ccbcf Mon Sep 17 00:00:00 2001 From: Xiu Jianfeng Date: Thu, 10 Aug 2023 20:10:56 +0000 Subject: [PATCH 226/475] apparmor: remove unused functions in policy_ns.c/.h These functions are not used now, remove them. Signed-off-by: Xiu Jianfeng Signed-off-by: John Johansen --- security/apparmor/include/policy_ns.h | 14 ---------- security/apparmor/policy_ns.c | 37 --------------------------- 2 files changed, 51 deletions(-) diff --git a/security/apparmor/include/policy_ns.h b/security/apparmor/include/policy_ns.h index 33d665516fc1..d646070fd966 100644 --- a/security/apparmor/include/policy_ns.h +++ b/security/apparmor/include/policy_ns.h @@ -86,10 +86,7 @@ const char *aa_ns_name(struct aa_ns *parent, struct aa_ns *child, bool subns); void aa_free_ns(struct aa_ns *ns); int aa_alloc_root_ns(void); void aa_free_root_ns(void); -void aa_free_ns_kref(struct kref *kref); -struct aa_ns *aa_find_ns(struct aa_ns *root, const char *name); -struct aa_ns *aa_findn_ns(struct aa_ns *root, const char *name, size_t n); struct aa_ns *__aa_lookupn_ns(struct aa_ns *view, const char *hname, size_t n); struct aa_ns *aa_lookupn_ns(struct aa_ns *view, const char *name, size_t n); struct aa_ns *__aa_find_or_create_ns(struct aa_ns *parent, const char *name, @@ -151,15 +148,4 @@ static inline struct aa_ns *__aa_find_ns(struct list_head *head, return __aa_findn_ns(head, name, strlen(name)); } -static inline struct aa_ns *__aa_lookup_ns(struct aa_ns *base, - const char *hname) -{ - return __aa_lookupn_ns(base, hname, strlen(hname)); -} - -static inline struct aa_ns *aa_lookup_ns(struct aa_ns *view, const char *name) -{ - return aa_lookupn_ns(view, name, strlen(name)); -} - #endif /* AA_NAMESPACE_H */ diff --git a/security/apparmor/policy_ns.c b/security/apparmor/policy_ns.c index fd5b7afbcb48..1f02cfe1d974 100644 --- a/security/apparmor/policy_ns.c +++ b/security/apparmor/policy_ns.c @@ -159,43 +159,6 @@ void aa_free_ns(struct aa_ns *ns) kfree_sensitive(ns); } -/** - * aa_findn_ns - look up a profile namespace on the namespace list - * @root: namespace to search in (NOT NULL) - * @name: name of namespace to find (NOT NULL) - * @n: length of @name - * - * Returns: a refcounted namespace on the list, or NULL if no namespace - * called @name exists. - * - * refcount released by caller - */ -struct aa_ns *aa_findn_ns(struct aa_ns *root, const char *name, size_t n) -{ - struct aa_ns *ns = NULL; - - rcu_read_lock(); - ns = aa_get_ns(__aa_findn_ns(&root->sub_ns, name, n)); - rcu_read_unlock(); - - return ns; -} - -/** - * aa_find_ns - look up a profile namespace on the namespace list - * @root: namespace to search in (NOT NULL) - * @name: name of namespace to find (NOT NULL) - * - * Returns: a refcounted namespace on the list, or NULL if no namespace - * called @name exists. - * - * refcount released by caller - */ -struct aa_ns *aa_find_ns(struct aa_ns *root, const char *name) -{ - return aa_findn_ns(root, name, strlen(name)); -} - /** * __aa_lookupn_ns - lookup the namespace matching @hname * @view: namespace to search in (NOT NULL) From 2516fde1fa00cf92e0932182c4b315df3de41806 Mon Sep 17 00:00:00 2001 From: Vinicius Costa Gomes Date: Thu, 31 Aug 2023 16:22:24 -0700 Subject: [PATCH 227/475] apparmor: Optimize retrieving current task secid When running will-it-scale[1] open2_process testcase, in a system with a large number of cores, a bottleneck in retrieving the current task secid was detected: 27.73% ima_file_check;do_open (inlined);path_openat;do_filp_open;do_sys_openat2;__x64_sys_openat;do_syscall_x64 (inlined);do_syscall_64;entry_SYSCALL_64_after_hwframe (inlined);__libc_open64 (inlined) 27.72% 0.01% [kernel.vmlinux] [k] security_current_getsecid_subj - - 27.71% security_current_getsecid_subj;ima_file_check;do_open (inlined);path_openat;do_filp_open;do_sys_openat2;__x64_sys_openat;do_syscall_x64 (inlined);do_syscall_64;entry_SYSCALL_64_after_hwframe (inlined);__libc_open64 (inlined) 27.71% 27.68% [kernel.vmlinux] [k] apparmor_current_getsecid_subj - - 19.94% __refcount_add (inlined);__refcount_inc (inlined);refcount_inc (inlined);kref_get (inlined);aa_get_label (inlined);aa_get_label (inlined);aa_get_current_label (inlined);apparmor_current_getsecid_subj;security_current_getsecid_subj;ima_file_check;do_open (inlined);path_openat;do_filp_open;do_sys_openat2;__x64_sys_openat;do_syscall_x64 (inlined);do_syscall_64;entry_SYSCALL_64_after_hwframe (inlined);__libc_open64 (inlined) 7.72% __refcount_sub_and_test (inlined);__refcount_dec_and_test (inlined);refcount_dec_and_test (inlined);kref_put (inlined);aa_put_label (inlined);aa_put_label (inlined);apparmor_current_getsecid_subj;security_current_getsecid_subj;ima_file_check;do_open (inlined);path_openat;do_filp_open;do_sys_openat2;__x64_sys_openat;do_syscall_x64 (inlined);do_syscall_64;entry_SYSCALL_64_after_hwframe (inlined);__libc_open64 (inlined) A large amount of time was spent in the refcount. The most common case is that the current task label is available, and no need to take references for that one. That is exactly what the critical section helpers do, make use of them. New perf output: 39.12% vfs_open;path_openat;do_filp_open;do_sys_openat2;__x64_sys_openat;do_syscall_64;entry_SYSCALL_64_after_hwframe;__libc_open64 (inlined) 39.07% 0.13% [kernel.vmlinux] [k] do_dentry_open - - 39.05% do_dentry_open;vfs_open;path_openat;do_filp_open;do_sys_openat2;__x64_sys_openat;do_syscall_64;entry_SYSCALL_64_after_hwframe;__libc_open64 (inlined) 38.71% 0.01% [kernel.vmlinux] [k] security_file_open - - 38.70% security_file_open;do_dentry_open;vfs_open;path_openat;do_filp_open;do_sys_openat2;__x64_sys_openat;do_syscall_64;entry_SYSCALL_64_after_hwframe;__libc_open64 (inlined) 38.65% 38.60% [kernel.vmlinux] [k] apparmor_file_open - - 38.65% apparmor_file_open;security_file_open;do_dentry_open;vfs_open;path_openat;do_filp_open;do_sys_openat2;__x64_sys_openat;do_syscall_64;entry_SYSCALL_64_after_hwframe;__libc_open64 (inlined) The result is a throughput improvement of around 20% across the board on the open2 testcase. On more realistic workloads the impact should be much less. [1] https://github.com/antonblanchard/will-it-scale Signed-off-by: Vinicius Costa Gomes Signed-off-by: John Johansen --- security/apparmor/lsm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/security/apparmor/lsm.c b/security/apparmor/lsm.c index c9463bd0307d..5289dfe60bc1 100644 --- a/security/apparmor/lsm.c +++ b/security/apparmor/lsm.c @@ -766,9 +766,9 @@ static void apparmor_bprm_committed_creds(struct linux_binprm *bprm) static void apparmor_current_getsecid_subj(u32 *secid) { - struct aa_label *label = aa_get_current_label(); + struct aa_label *label = __begin_current_label_crit_section(); *secid = label->secid; - aa_put_label(label); + __end_current_label_crit_section(label); } static void apparmor_task_getsecid_obj(struct task_struct *p, u32 *secid) From 32d1d9204f8db3360be55e65bd182a1a68f93308 Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Tue, 12 Sep 2023 09:54:25 +0200 Subject: [PATCH 228/475] s390/ap: re-init AP queues on config on On a state toggle from config off to config on and on the state toggle from checkstop to not checkstop the queue's internal states was set but the state machine was not nudged. This did not care as on the first enqueue of a request the state machine kick ran. However, within an Secure Execution guest a queue is only chosen by the scheduler when it has been bound. But to bind a queue, it needs to run through the initial states (reset, enable interrupts, ...). So this is like a chicken-and-egg problem and the result was in fact that a queue was unusable after a config off/on toggle. With some slight rework of the handling of these states now the new function _ap_queue_init_state() is called which is the core of the ap_queue_init_state() function but without locking handling. This has the benefit that it can be called on all the places where a (re-)init of the AP queue's state machine is needed. Fixes: 2d72eaf036d2 ("s390/ap: implement SE AP bind, unbind and associate") Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Signed-off-by: Vasily Gorbik --- drivers/s390/crypto/ap_bus.c | 21 ++++++++++----------- drivers/s390/crypto/ap_bus.h | 1 + drivers/s390/crypto/ap_queue.c | 9 +++++++-- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c index 339812efe822..d09e08b71cfb 100644 --- a/drivers/s390/crypto/ap_bus.c +++ b/drivers/s390/crypto/ap_bus.c @@ -1865,15 +1865,18 @@ static inline void ap_scan_domains(struct ap_card *ac) } /* get it and thus adjust reference counter */ get_device(dev); - if (decfg) + if (decfg) { AP_DBF_INFO("%s(%d,%d) new (decfg) queue dev created\n", __func__, ac->id, dom); - else if (chkstop) + } else if (chkstop) { AP_DBF_INFO("%s(%d,%d) new (chkstop) queue dev created\n", __func__, ac->id, dom); - else + } else { + /* nudge the queue's state machine */ + ap_queue_init_state(aq); AP_DBF_INFO("%s(%d,%d) new queue dev created\n", __func__, ac->id, dom); + } goto put_dev_and_continue; } /* handle state changes on already existing queue device */ @@ -1895,10 +1898,8 @@ static inline void ap_scan_domains(struct ap_card *ac) } else if (!chkstop && aq->chkstop) { /* checkstop off */ aq->chkstop = false; - if (aq->dev_state > AP_DEV_STATE_UNINITIATED) { - aq->dev_state = AP_DEV_STATE_OPERATING; - aq->sm_state = AP_SM_STATE_RESET_START; - } + if (aq->dev_state > AP_DEV_STATE_UNINITIATED) + _ap_queue_init_state(aq); spin_unlock_bh(&aq->lock); AP_DBF_DBG("%s(%d,%d) queue dev checkstop off\n", __func__, ac->id, dom); @@ -1922,10 +1923,8 @@ static inline void ap_scan_domains(struct ap_card *ac) } else if (!decfg && !aq->config) { /* config on this queue device */ aq->config = true; - if (aq->dev_state > AP_DEV_STATE_UNINITIATED) { - aq->dev_state = AP_DEV_STATE_OPERATING; - aq->sm_state = AP_SM_STATE_RESET_START; - } + if (aq->dev_state > AP_DEV_STATE_UNINITIATED) + _ap_queue_init_state(aq); spin_unlock_bh(&aq->lock); AP_DBF_DBG("%s(%d,%d) queue dev config on\n", __func__, ac->id, dom); diff --git a/drivers/s390/crypto/ap_bus.h b/drivers/s390/crypto/ap_bus.h index be54b070c031..3e34912a6050 100644 --- a/drivers/s390/crypto/ap_bus.h +++ b/drivers/s390/crypto/ap_bus.h @@ -287,6 +287,7 @@ struct ap_queue *ap_queue_create(ap_qid_t qid, int device_type); void ap_queue_prepare_remove(struct ap_queue *aq); void ap_queue_remove(struct ap_queue *aq); void ap_queue_init_state(struct ap_queue *aq); +void _ap_queue_init_state(struct ap_queue *aq); struct ap_card *ap_card_create(int id, int queue_depth, int raw_type, int comp_type, unsigned int functions, int ml); diff --git a/drivers/s390/crypto/ap_queue.c b/drivers/s390/crypto/ap_queue.c index 1336e632adc4..2943b2529d3a 100644 --- a/drivers/s390/crypto/ap_queue.c +++ b/drivers/s390/crypto/ap_queue.c @@ -1160,14 +1160,19 @@ void ap_queue_remove(struct ap_queue *aq) spin_unlock_bh(&aq->lock); } -void ap_queue_init_state(struct ap_queue *aq) +void _ap_queue_init_state(struct ap_queue *aq) { - spin_lock_bh(&aq->lock); aq->dev_state = AP_DEV_STATE_OPERATING; aq->sm_state = AP_SM_STATE_RESET_START; aq->last_err_rc = 0; aq->assoc_idx = ASSOC_IDX_INVALID; ap_wait(ap_sm_event(aq, AP_SM_EVENT_POLL)); +} + +void ap_queue_init_state(struct ap_queue *aq) +{ + spin_lock_bh(&aq->lock); + _ap_queue_init_state(aq); spin_unlock_bh(&aq->lock); } EXPORT_SYMBOL(ap_queue_init_state); From a19a161482b1739c2207861d086403389dc1bd47 Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Tue, 12 Sep 2023 10:08:51 +0200 Subject: [PATCH 229/475] s390/zcrypt: introduce new internal AP queue se_bound attribute This patch introduces a new AP queue internal attribute se_bound which reflects the bound state of an APQN within a Secure Execution environment. With introduction of Secure Execution guests now an AP firmware queue needs to be bound to the guest before usage. This patch introduces a new internal attribute reflecting this bound state and some glue code to handle this new field during lifetime of an AP queue device. Together with that now the zcrypt scheduler considers the state of the AP queues when a message is about to be distributed among the existing queues. There is a new function ap_queue_usable() which returns true only when all conditions for using this AP queue device are fulfilled. In details this means: the AP queue needs to be configured, not checkstopped and within an SE environment it needs to be bound. So the new function gives and indication if the AP queue device is ready to serve requests or not. Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Signed-off-by: Vasily Gorbik --- drivers/s390/crypto/ap_bus.h | 2 ++ drivers/s390/crypto/ap_queue.c | 49 +++++++++++++++++++++++++++++++- drivers/s390/crypto/zcrypt_api.c | 10 +++---- 3 files changed, 55 insertions(+), 6 deletions(-) diff --git a/drivers/s390/crypto/ap_bus.h b/drivers/s390/crypto/ap_bus.h index 3e34912a6050..359a35f894d5 100644 --- a/drivers/s390/crypto/ap_bus.h +++ b/drivers/s390/crypto/ap_bus.h @@ -207,6 +207,7 @@ struct ap_queue { bool chkstop; /* checkstop state */ ap_qid_t qid; /* AP queue id. */ bool interrupt; /* indicate if interrupts are enabled */ + bool se_bound; /* SE bound state */ unsigned int assoc_idx; /* SE association index */ int queue_count; /* # messages currently on AP queue. */ int pendingq_count; /* # requests on pendingq list. */ @@ -271,6 +272,7 @@ enum ap_sm_wait ap_sm_event_loop(struct ap_queue *aq, enum ap_sm_event event); int ap_queue_message(struct ap_queue *aq, struct ap_message *ap_msg); void ap_cancel_message(struct ap_queue *aq, struct ap_message *ap_msg); void ap_flush_queue(struct ap_queue *aq); +bool ap_queue_usable(struct ap_queue *aq); void *ap_airq_ptr(void); int ap_sb_available(void); diff --git a/drivers/s390/crypto/ap_queue.c b/drivers/s390/crypto/ap_queue.c index 2943b2529d3a..993240370ecf 100644 --- a/drivers/s390/crypto/ap_queue.c +++ b/drivers/s390/crypto/ap_queue.c @@ -33,6 +33,11 @@ static inline bool ap_q_supports_assoc(struct ap_queue *aq) return ap_test_bit(&aq->card->functions, AP_FUNC_EP11); } +static inline bool ap_q_needs_bind(struct ap_queue *aq) +{ + return ap_q_supports_bind(aq) && ap_sb_available(); +} + /** * ap_queue_enable_irq(): Enable interrupt support on this AP queue. * @aq: The AP queue @@ -304,6 +309,7 @@ static enum ap_sm_wait ap_sm_reset(struct ap_queue *aq) aq->sm_state = AP_SM_STATE_RESET_WAIT; aq->interrupt = false; aq->rapq_fbit = 0; + aq->se_bound = false; return AP_SM_WAIT_LOW_TIMEOUT; default: aq->dev_state = AP_DEV_STATE_ERROR; @@ -868,7 +874,12 @@ static ssize_t se_bind_store(struct device *dev, } status = ap_bapq(aq->qid); spin_unlock_bh(&aq->lock); - if (status.response_code) { + if (!status.response_code) { + aq->se_bound = true; + AP_DBF_INFO("%s bapq(0x%02x.%04x) success\n", __func__, + AP_QID_CARD(aq->qid), + AP_QID_QUEUE(aq->qid)); + } else { AP_DBF_WARN("%s RC 0x%02x on bapq(0x%02x.%04x)\n", __func__, status.response_code, AP_QID_CARD(aq->qid), @@ -1073,6 +1084,42 @@ int ap_queue_message(struct ap_queue *aq, struct ap_message *ap_msg) } EXPORT_SYMBOL(ap_queue_message); +/** + * ap_queue_usable(): Check if queue is usable just now. + * @aq: The AP queue device to test for usability. + * This function is intended for the scheduler to query if it makes + * sense to enqueue a message into this AP queue device by calling + * ap_queue_message(). The perspective is very short-term as the + * state machine and device state(s) may change at any time. + */ +bool ap_queue_usable(struct ap_queue *aq) +{ + bool rc = true; + + spin_lock_bh(&aq->lock); + + /* check for not configured or checkstopped */ + if (!aq->config || aq->chkstop) { + rc = false; + goto unlock_and_out; + } + + /* device state needs to be ok */ + if (aq->dev_state != AP_DEV_STATE_OPERATING) { + rc = false; + goto unlock_and_out; + } + + /* SE guest's queues additionally need to be bound */ + if (ap_q_needs_bind(aq) && !aq->se_bound) + rc = false; + +unlock_and_out: + spin_unlock_bh(&aq->lock); + return rc; +} +EXPORT_SYMBOL(ap_queue_usable); + /** * ap_cancel_message(): Cancel a crypto request. * @aq: The AP device that has the message queued diff --git a/drivers/s390/crypto/zcrypt_api.c b/drivers/s390/crypto/zcrypt_api.c index ce04caa7913f..dcd6c7299fa9 100644 --- a/drivers/s390/crypto/zcrypt_api.c +++ b/drivers/s390/crypto/zcrypt_api.c @@ -693,7 +693,7 @@ static long zcrypt_rsa_modexpo(struct ap_perms *perms, for_each_zcrypt_queue(zq, zc) { /* check if device is usable and eligible */ if (!zq->online || !zq->ops->rsa_modexpo || - !zq->queue->config || zq->queue->chkstop) + !ap_queue_usable(zq->queue)) continue; /* check if device node has admission for this queue */ if (!zcrypt_check_queue(perms, @@ -798,7 +798,7 @@ static long zcrypt_rsa_crt(struct ap_perms *perms, for_each_zcrypt_queue(zq, zc) { /* check if device is usable and eligible */ if (!zq->online || !zq->ops->rsa_modexpo_crt || - !zq->queue->config || zq->queue->chkstop) + !ap_queue_usable(zq->queue)) continue; /* check if device node has admission for this queue */ if (!zcrypt_check_queue(perms, @@ -916,7 +916,7 @@ static long _zcrypt_send_cprb(bool userspace, struct ap_perms *perms, for_each_zcrypt_queue(zq, zc) { /* check for device usable and eligible */ if (!zq->online || !zq->ops->send_cprb || - !zq->queue->config || zq->queue->chkstop || + !ap_queue_usable(zq->queue) || (tdom != AUTOSEL_DOM && tdom != AP_QID_QUEUE(zq->queue->qid))) continue; @@ -1087,7 +1087,7 @@ static long _zcrypt_send_ep11_cprb(bool userspace, struct ap_perms *perms, for_each_zcrypt_queue(zq, zc) { /* check if device is usable and eligible */ if (!zq->online || !zq->ops->send_ep11_cprb || - !zq->queue->config || zq->queue->chkstop || + !ap_queue_usable(zq->queue) || (targets && !is_desired_ep11_queue(zq->queue->qid, target_num, targets))) @@ -1186,7 +1186,7 @@ static long zcrypt_rng(char *buffer) for_each_zcrypt_queue(zq, zc) { /* check if device is usable and eligible */ if (!zq->online || !zq->ops->rng || - !zq->queue->config || zq->queue->chkstop) + !ap_queue_usable(zq->queue)) continue; if (!zcrypt_queue_compare(zq, pref_zq, wgt, pref_wgt)) continue; From 8d6be876bbd7c09190b025ccc478260f5a40c216 Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Tue, 12 Sep 2023 09:02:07 +0200 Subject: [PATCH 230/475] s390/ap: show APFS value on error reply 0x8B With Secure Execution the error reply RX 0x8B now carries an APFS value indicating why the request has been filtered by a lower layer of the firmware. So display this value as a warning line in the s390 debug feature trace. Signed-off-by: Harald Freudenberger Reviewed-by: Holger Dengler Signed-off-by: Vasily Gorbik --- drivers/s390/crypto/zcrypt_error.h | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/drivers/s390/crypto/zcrypt_error.h b/drivers/s390/crypto/zcrypt_error.h index d36177e65a3d..a44fcfcec938 100644 --- a/drivers/s390/crypto/zcrypt_error.h +++ b/drivers/s390/crypto/zcrypt_error.h @@ -98,8 +98,22 @@ static inline int convert_error(struct zcrypt_queue *zq, case REP88_ERROR_MESSAGE_MALFORMD: /* 0x22 */ case REP88_ERROR_KEY_TYPE: /* 0x34 */ /* RY indicates malformed request */ - ZCRYPT_DBF_WARN("%s dev=%02x.%04x RY=0x%02x => rc=EINVAL\n", - __func__, card, queue, ehdr->reply_code); + if (ehdr->reply_code == REP82_ERROR_FILTERED_BY_HYPERVISOR && + ehdr->type == TYPE86_RSP_CODE) { + struct { + struct type86_hdr hdr; + struct type86_fmt2_ext fmt2; + } __packed * head = reply->msg; + unsigned int apfs = *((u32 *)head->fmt2.apfs); + + ZCRYPT_DBF_WARN("%s dev=%02x.%04x RY=0x%02x apfs=0x%x => rc=EINVAL\n", + __func__, card, queue, + ehdr->reply_code, apfs); + } else { + ZCRYPT_DBF_WARN("%s dev=%02x.%04x RY=0x%02x => rc=EINVAL\n", + __func__, card, queue, + ehdr->reply_code); + } return -EINVAL; case REP82_ERROR_MACHINE_FAILURE: /* 0x10 */ case REP82_ERROR_MESSAGE_TYPE: /* 0x20 */ From 4ff1261354d33caf7b2aa1f47db00deef27863ad Mon Sep 17 00:00:00 2001 From: Alexander Gordeev Date: Sun, 1 Oct 2023 10:09:34 +0200 Subject: [PATCH 231/475] s390/crash: remove unused parameter Funciton loads_init() does not use loads_offset parameter. Reviewed-by: Heiko Carstens Signed-off-by: Alexander Gordeev Signed-off-by: Vasily Gorbik --- arch/s390/kernel/crash_dump.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/s390/kernel/crash_dump.c b/arch/s390/kernel/crash_dump.c index 7af69948b290..237c5a9199d9 100644 --- a/arch/s390/kernel/crash_dump.c +++ b/arch/s390/kernel/crash_dump.c @@ -498,7 +498,7 @@ static int get_mem_chunk_cnt(void) /* * Initialize ELF loads (new kernel) */ -static void loads_init(Elf64_Phdr *phdr, u64 loads_offset) +static void loads_init(Elf64_Phdr *phdr) { phys_addr_t start, end; u64 idx; @@ -612,7 +612,7 @@ int elfcorehdr_alloc(unsigned long long *addr, unsigned long long *size) ptr = notes_init(phdr_notes, ptr, ((unsigned long) hdr) + hdr_off); /* Init loads */ hdr_off = PTR_DIFF(ptr, hdr); - loads_init(phdr_loads, hdr_off); + loads_init(phdr_loads); *addr = (unsigned long long) hdr; *size = (unsigned long long) hdr_off; BUG_ON(elfcorehdr_size > alloc_size); From ecfe7f53286748bca0ef1646a45f428b915b0916 Mon Sep 17 00:00:00 2001 From: Alexander Gordeev Date: Sun, 1 Oct 2023 10:01:31 +0200 Subject: [PATCH 232/475] s390/crash: fix virtual vs physical address confusion Fix virtual vs physical address confusion (which currently are the same). Reviewed-by: Heiko Carstens Signed-off-by: Alexander Gordeev Signed-off-by: Vasily Gorbik --- arch/s390/kernel/crash_dump.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/kernel/crash_dump.c b/arch/s390/kernel/crash_dump.c index 237c5a9199d9..514feadd4c58 100644 --- a/arch/s390/kernel/crash_dump.c +++ b/arch/s390/kernel/crash_dump.c @@ -507,7 +507,7 @@ static void loads_init(Elf64_Phdr *phdr) phdr->p_filesz = end - start; phdr->p_type = PT_LOAD; phdr->p_offset = start; - phdr->p_vaddr = start; + phdr->p_vaddr = (unsigned long)__va(start); phdr->p_paddr = start; phdr->p_memsz = end - start; phdr->p_flags = PF_R | PF_W | PF_X; From 88b2c332fcecbcf71a5d69a214a8805cf916fbf8 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Fri, 6 Oct 2023 15:42:40 +0200 Subject: [PATCH 233/475] s390/extable: reduce number of extable macros Get rid of __EX_TABLE() macro, rename __EX_TABLE_UA() to __EX_TABLE() and convert users of old __EX_TABLE() macro so they pass more parameters to the changed __EX_TABLE() semantics. Reviewed-by: Sven Schnelle Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/asm-extable.h | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/arch/s390/include/asm/asm-extable.h b/arch/s390/include/asm/asm-extable.h index e6532477f126..d1ecc2b6e5d3 100644 --- a/arch/s390/include/asm/asm-extable.h +++ b/arch/s390/include/asm/asm-extable.h @@ -23,16 +23,7 @@ #define EX_DATA_LEN_SHIFT 8 #define EX_DATA_LEN GENMASK(11, 8) -#define __EX_TABLE(_section, _fault, _target, _type) \ - stringify_in_c(.section _section,"a";) \ - stringify_in_c(.balign 4;) \ - stringify_in_c(.long (_fault) - .;) \ - stringify_in_c(.long (_target) - .;) \ - stringify_in_c(.short (_type);) \ - stringify_in_c(.short 0;) \ - stringify_in_c(.previous) - -#define __EX_TABLE_UA(_section, _fault, _target, _type, _regerr, _regaddr, _len)\ +#define __EX_TABLE(_section, _fault, _target, _type, _regerr, _regaddr, _len) \ stringify_in_c(.section _section,"a";) \ stringify_in_c(.balign 4;) \ stringify_in_c(.long (_fault) - .;) \ @@ -72,21 +63,21 @@ stringify_in_c(.previous) #define EX_TABLE(_fault, _target) \ - __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_FIXUP) + __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_FIXUP, __stringify(%%r0), __stringify(%%r0), 0) #define EX_TABLE_AMODE31(_fault, _target) \ - __EX_TABLE(.amode31.ex_table, _fault, _target, EX_TYPE_FIXUP) + __EX_TABLE(.amode31.ex_table, _fault, _target, EX_TYPE_FIXUP, __stringify(%%r0), __stringify(%%r0), 0) #define EX_TABLE_UA_STORE(_fault, _target, _regerr) \ - __EX_TABLE_UA(__ex_table, _fault, _target, EX_TYPE_UA_STORE, _regerr, _regerr, 0) + __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_STORE, _regerr, _regerr, 0) #define EX_TABLE_UA_LOAD_MEM(_fault, _target, _regerr, _regmem, _len) \ - __EX_TABLE_UA(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_MEM, _regerr, _regmem, _len) + __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_MEM, _regerr, _regmem, _len) #define EX_TABLE_UA_LOAD_REG(_fault, _target, _regerr, _regzero) \ - __EX_TABLE_UA(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_REG, _regerr, _regzero, 0) + __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_REG, _regerr, _regzero, 0) #define EX_TABLE_UA_LOAD_REGPAIR(_fault, _target, _regerr, _regzero) \ - __EX_TABLE_UA(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_REGPAIR, _regerr, _regzero, 0) + __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_REGPAIR, _regerr, _regzero, 0) #endif /* __ASM_EXTABLE_H */ From 76292d7243f809486ec26dd6cd3436dbcb9e5319 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Fri, 6 Oct 2023 15:42:41 +0200 Subject: [PATCH 234/475] s390: provide word-at-a-time implementation Provide an s390 specific word-at-a-time implementation. Compared to the generic variant the generated code for has_zero() is slightly better. However find_zero() is much simpler since it reuses the result of __fls() aka flogr() and now comes without any conditional branches, while the generic variant has three of them. Reviewed-by: Sven Schnelle Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/word-at-a-time.h | 42 ++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 arch/s390/include/asm/word-at-a-time.h diff --git a/arch/s390/include/asm/word-at-a-time.h b/arch/s390/include/asm/word-at-a-time.h new file mode 100644 index 000000000000..09b37541a175 --- /dev/null +++ b/arch/s390/include/asm/word-at-a-time.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_WORD_AT_A_TIME_H +#define _ASM_WORD_AT_A_TIME_H + +#include +#include + +struct word_at_a_time { + const unsigned long bits; +}; + +#define WORD_AT_A_TIME_CONSTANTS { REPEAT_BYTE(0x7f) } + +static inline unsigned long prep_zero_mask(unsigned long val, unsigned long data, const struct word_at_a_time *c) +{ + return data; +} + +static inline unsigned long create_zero_mask(unsigned long data) +{ + return __fls(data); +} + +static inline unsigned long find_zero(unsigned long data) +{ + return (data ^ (BITS_PER_LONG - 1)) >> 3; +} + +static inline unsigned long has_zero(unsigned long val, unsigned long *data, const struct word_at_a_time *c) +{ + unsigned long mask = (val & c->bits) + c->bits; + + *data = ~(mask | val | c->bits); + return *data; +} + +static inline unsigned long zero_bytemask(unsigned long data) +{ + return ~1UL << data; +} + +#endif /* _ASM_WORD_AT_A_TIME_H */ From 802ba53eefc592a6a82231f74e19bafe3256f172 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Fri, 6 Oct 2023 15:42:42 +0200 Subject: [PATCH 235/475] s390: add support for DCACHE_WORD_ACCESS Implement load_unaligned_zeropad() and enable DCACHE_WORD_ACCESS to speed up string operations in fs/dcache.c and fs/namei.c. Reviewed-by: Sven Schnelle Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/Kconfig | 1 + arch/s390/include/asm/asm-extable.h | 4 ++++ arch/s390/include/asm/word-at-a-time.h | 22 ++++++++++++++++++++++ arch/s390/mm/extable.c | 18 ++++++++++++++++++ 4 files changed, 45 insertions(+) diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index ae29e4392664..b0d67ac8695f 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -131,6 +131,7 @@ config S390 select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP select BUILDTIME_TABLE_SORT select CLONE_BACKWARDS2 + select DCACHE_WORD_ACCESS if !KMSAN select DMA_OPS if PCI select DYNAMIC_FTRACE if FUNCTION_TRACER select FUNCTION_ALIGNMENT_8B if CC_IS_GCC diff --git a/arch/s390/include/asm/asm-extable.h b/arch/s390/include/asm/asm-extable.h index d1ecc2b6e5d3..4a6b0a8b6412 100644 --- a/arch/s390/include/asm/asm-extable.h +++ b/arch/s390/include/asm/asm-extable.h @@ -13,6 +13,7 @@ #define EX_TYPE_UA_LOAD_MEM 4 #define EX_TYPE_UA_LOAD_REG 5 #define EX_TYPE_UA_LOAD_REGPAIR 6 +#define EX_TYPE_ZEROPAD 7 #define EX_DATA_REG_ERR_SHIFT 0 #define EX_DATA_REG_ERR GENMASK(3, 0) @@ -80,4 +81,7 @@ #define EX_TABLE_UA_LOAD_REGPAIR(_fault, _target, _regerr, _regzero) \ __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_REGPAIR, _regerr, _regzero, 0) +#define EX_TABLE_ZEROPAD(_fault, _target, _regdata, _regaddr) \ + __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_ZEROPAD, _regdata, _regaddr, 0) + #endif /* __ASM_EXTABLE_H */ diff --git a/arch/s390/include/asm/word-at-a-time.h b/arch/s390/include/asm/word-at-a-time.h index 09b37541a175..2579f1694b82 100644 --- a/arch/s390/include/asm/word-at-a-time.h +++ b/arch/s390/include/asm/word-at-a-time.h @@ -3,6 +3,7 @@ #define _ASM_WORD_AT_A_TIME_H #include +#include #include struct word_at_a_time { @@ -39,4 +40,25 @@ static inline unsigned long zero_bytemask(unsigned long data) return ~1UL << data; } +/* + * Load an unaligned word from kernel space. + * + * In the (very unlikely) case of the word being a page-crosser + * and the next page not being mapped, take the exception and + * return zeroes in the non-existing part. + */ +static inline unsigned long load_unaligned_zeropad(const void *addr) +{ + unsigned long data; + + asm volatile( + "0: lg %[data],0(%[addr])\n" + "1: nopr %%r7\n" + EX_TABLE_ZEROPAD(0b, 1b, %[data], %[addr]) + EX_TABLE_ZEROPAD(1b, 1b, %[data], %[addr]) + : [data] "=d" (data) + : [addr] "a" (addr), "m" (*(unsigned long *)addr)); + return data; +} + #endif /* _ASM_WORD_AT_A_TIME_H */ diff --git a/arch/s390/mm/extable.c b/arch/s390/mm/extable.c index fe87291df95d..0a0738a473af 100644 --- a/arch/s390/mm/extable.c +++ b/arch/s390/mm/extable.c @@ -61,6 +61,22 @@ static bool ex_handler_ua_load_reg(const struct exception_table_entry *ex, return true; } +static bool ex_handler_zeropad(const struct exception_table_entry *ex, struct pt_regs *regs) +{ + unsigned int reg_addr = FIELD_GET(EX_DATA_REG_ADDR, ex->data); + unsigned int reg_data = FIELD_GET(EX_DATA_REG_ERR, ex->data); + unsigned long data, addr, offset; + + addr = regs->gprs[reg_addr]; + offset = addr & (sizeof(unsigned long) - 1); + addr &= ~(sizeof(unsigned long) - 1); + data = *(unsigned long *)addr; + data <<= BITS_PER_BYTE * offset; + regs->gprs[reg_data] = data; + regs->psw.addr = extable_fixup(ex); + return true; +} + bool fixup_exception(struct pt_regs *regs) { const struct exception_table_entry *ex; @@ -81,6 +97,8 @@ bool fixup_exception(struct pt_regs *regs) return ex_handler_ua_load_reg(ex, false, regs); case EX_TYPE_UA_LOAD_REGPAIR: return ex_handler_ua_load_reg(ex, true, regs); + case EX_TYPE_ZEROPAD: + return ex_handler_zeropad(ex, regs); } panic("invalid exception table entry"); } From 19ba9ead8a88ab02f3b00d5a6cab7853f03445e7 Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Thu, 12 Oct 2023 11:39:29 +0200 Subject: [PATCH 236/475] s390/vmem: remove unused variable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix the follow warning reported by sparse: arch/s390/boot/vmem.c:170:15: warning: unused variable ‘entry’ [-Wunused-variable] 170 | pte_t entry; | ^~~~~ Reviewed-by: Alexander Gordeev Signed-off-by: Vasily Gorbik --- arch/s390/boot/vmem.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/s390/boot/vmem.c b/arch/s390/boot/vmem.c index bdbfee86d1ac..007e87239bbf 100644 --- a/arch/s390/boot/vmem.c +++ b/arch/s390/boot/vmem.c @@ -162,8 +162,6 @@ static bool kasan_pmd_populate_zero_shadow(pmd_t *pmd, unsigned long addr, static bool kasan_pte_populate_zero_shadow(pte_t *pte, enum populate_mode mode) { - pte_t entry; - if (mode == POPULATE_KASAN_ZERO_SHADOW) { set_pte(pte, pte_z); return true; From 6cb42f91aa6dfd10fd847c469caebe63b35141ff Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Sun, 24 Sep 2023 19:38:17 -0700 Subject: [PATCH 237/475] bitmap: move bitmap_*_region() functions to bitmap.h Now that bitmap_*_region() functions are implemented as thin wrappers around others, it's worth to move them to the header, as it opens room for compile-time optimizations. CC: Andy Shevchenko CC: Rasmus Villemoes CC: Greg Kroah-Hartman Signed-off-by: Yury Norov --- include/linux/bitmap.h | 64 ++++++++++++++++++++++++++++++++++++++++-- lib/bitmap.c | 64 ------------------------------------------ 2 files changed, 61 insertions(+), 67 deletions(-) diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h index 1cca950a54ae..99451431e4d6 100644 --- a/include/linux/bitmap.h +++ b/include/linux/bitmap.h @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -209,9 +210,6 @@ void bitmap_onto(unsigned long *dst, const unsigned long *orig, const unsigned long *relmap, unsigned int bits); void bitmap_fold(unsigned long *dst, const unsigned long *orig, unsigned int sz, unsigned int nbits); -int bitmap_find_free_region(unsigned long *bitmap, unsigned int bits, int order); -void bitmap_release_region(unsigned long *bitmap, unsigned int pos, int order); -int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos, int order); #define BITMAP_FIRST_WORD_MASK(start) (~0UL << ((start) & (BITS_PER_LONG - 1))) #define BITMAP_LAST_WORD_MASK(nbits) (~0UL >> (-(nbits) & (BITS_PER_LONG - 1))) @@ -497,6 +495,66 @@ static inline void bitmap_next_set_region(unsigned long *bitmap, *re = find_next_zero_bit(bitmap, end, *rs + 1); } +/** + * bitmap_release_region - release allocated bitmap region + * @bitmap: array of unsigned longs corresponding to the bitmap + * @pos: beginning of bit region to release + * @order: region size (log base 2 of number of bits) to release + * + * This is the complement to __bitmap_find_free_region() and releases + * the found region (by clearing it in the bitmap). + */ +static inline void bitmap_release_region(unsigned long *bitmap, unsigned int pos, int order) +{ + bitmap_clear(bitmap, pos, BIT(order)); +} + +/** + * bitmap_allocate_region - allocate bitmap region + * @bitmap: array of unsigned longs corresponding to the bitmap + * @pos: beginning of bit region to allocate + * @order: region size (log base 2 of number of bits) to allocate + * + * Allocate (set bits in) a specified region of a bitmap. + * + * Returns: 0 on success, or %-EBUSY if specified region wasn't + * free (not all bits were zero). + */ +static inline int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos, int order) +{ + unsigned int len = BIT(order); + + if (find_next_bit(bitmap, pos + len, pos) < pos + len) + return -EBUSY; + bitmap_set(bitmap, pos, len); + return 0; +} + +/** + * bitmap_find_free_region - find a contiguous aligned mem region + * @bitmap: array of unsigned longs corresponding to the bitmap + * @bits: number of bits in the bitmap + * @order: region size (log base 2 of number of bits) to find + * + * Find a region of free (zero) bits in a @bitmap of @bits bits and + * allocate them (set them to one). Only consider regions of length + * a power (@order) of two, aligned to that power of two, which + * makes the search algorithm much faster. + * + * Returns: the bit offset in bitmap of the allocated region, + * or -errno on failure. + */ +static inline int bitmap_find_free_region(unsigned long *bitmap, unsigned int bits, int order) +{ + unsigned int pos, end; /* scans bitmap by regions of size order */ + + for (pos = 0; (end = pos + BIT(order)) <= bits; pos = end) { + if (!bitmap_allocate_region(bitmap, pos, order)) + return pos; + } + return -ENOMEM; +} + /** * BITMAP_FROM_U64() - Represent u64 value in the format suitable for bitmap. * @n: u64 value diff --git a/lib/bitmap.c b/lib/bitmap.c index 66da8c33fb1b..09522af227f1 100644 --- a/lib/bitmap.c +++ b/lib/bitmap.c @@ -8,7 +8,6 @@ #include #include #include -#include #include #include @@ -708,69 +707,6 @@ void bitmap_fold(unsigned long *dst, const unsigned long *orig, } #endif /* CONFIG_NUMA */ -/** - * bitmap_find_free_region - find a contiguous aligned mem region - * @bitmap: array of unsigned longs corresponding to the bitmap - * @bits: number of bits in the bitmap - * @order: region size (log base 2 of number of bits) to find - * - * Find a region of free (zero) bits in a @bitmap of @bits bits and - * allocate them (set them to one). Only consider regions of length - * a power (@order) of two, aligned to that power of two, which - * makes the search algorithm much faster. - * - * Return: the bit offset in bitmap of the allocated region, - * or -errno on failure. - */ -int bitmap_find_free_region(unsigned long *bitmap, unsigned int bits, int order) -{ - unsigned int pos, end; /* scans bitmap by regions of size order */ - - for (pos = 0; (end = pos + BIT(order)) <= bits; pos = end) { - if (!bitmap_allocate_region(bitmap, pos, order)) - return pos; - } - return -ENOMEM; -} -EXPORT_SYMBOL(bitmap_find_free_region); - -/** - * bitmap_release_region - release allocated bitmap region - * @bitmap: array of unsigned longs corresponding to the bitmap - * @pos: beginning of bit region to release - * @order: region size (log base 2 of number of bits) to release - * - * This is the complement to __bitmap_find_free_region() and releases - * the found region (by clearing it in the bitmap). - */ -void bitmap_release_region(unsigned long *bitmap, unsigned int pos, int order) -{ - bitmap_clear(bitmap, pos, BIT(order)); -} -EXPORT_SYMBOL(bitmap_release_region); - -/** - * bitmap_allocate_region - allocate bitmap region - * @bitmap: array of unsigned longs corresponding to the bitmap - * @pos: beginning of bit region to allocate - * @order: region size (log base 2 of number of bits) to allocate - * - * Allocate (set bits in) a specified region of a bitmap. - * - * Return: 0 on success, or %-EBUSY if specified region wasn't - * free (not all bits were zero). - */ -int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos, int order) -{ - unsigned int len = BIT(order); - - if (find_next_bit(bitmap, pos + len, pos) < pos + len) - return -EBUSY; - bitmap_set(bitmap, pos, len); - return 0; -} -EXPORT_SYMBOL(bitmap_allocate_region); - unsigned long *bitmap_alloc(unsigned int nbits, gfp_t flags) { return kmalloc_array(BITS_TO_LONGS(nbits), sizeof(unsigned long), From a91c9872546514438796fa53a7591b934ccbbcaa Mon Sep 17 00:00:00 2001 From: Adrian Hunter Date: Thu, 5 Oct 2023 22:04:47 +0300 Subject: [PATCH 238/475] perf tools: Add get_unaligned_leNN() Add get_unaligned_le16(), get_unaligned_le32 and get_unaligned_le64, same as include/asm-generic/unaligned.h. And add include/asm-generic/unaligned.h to check-headers.sh bringing tools/include/asm-generic/unaligned.h up to date so that the kernel and tools versions match. Use diagnostic pragmas to ignore -Wpacked used by perf build. Signed-off-by: Adrian Hunter Reviewed-by: Ian Rogers Link: https://lore.kernel.org/r/20231005190451.175568-2-adrian.hunter@intel.com Link: https://lore.kernel.org/r/20231010142234.20061-1-adrian.hunter@intel.com [ squashed check-header.sh addition ] Signed-off-by: Namhyung Kim --- tools/include/asm-generic/unaligned.h | 147 ++++++++++++++++++++++++-- tools/perf/check-headers.sh | 1 + 2 files changed, 141 insertions(+), 7 deletions(-) diff --git a/tools/include/asm-generic/unaligned.h b/tools/include/asm-generic/unaligned.h index 47387c607035..156743d399ae 100644 --- a/tools/include/asm-generic/unaligned.h +++ b/tools/include/asm-generic/unaligned.h @@ -1,10 +1,13 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Copied from the kernel sources to tools/perf/: - */ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_GENERIC_UNALIGNED_H +#define __ASM_GENERIC_UNALIGNED_H -#ifndef __TOOLS_LINUX_ASM_GENERIC_UNALIGNED_H -#define __TOOLS_LINUX_ASM_GENERIC_UNALIGNED_H +/* + * This is the most generic implementation of unaligned accesses + * and should work almost anywhere. + */ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpacked" #define __get_unaligned_t(type, ptr) ({ \ const struct { type x; } __packed *__pptr = (typeof(__pptr))(ptr); \ @@ -19,5 +22,135 @@ #define get_unaligned(ptr) __get_unaligned_t(typeof(*(ptr)), (ptr)) #define put_unaligned(val, ptr) __put_unaligned_t(typeof(*(ptr)), (val), (ptr)) -#endif /* __TOOLS_LINUX_ASM_GENERIC_UNALIGNED_H */ +static inline u16 get_unaligned_le16(const void *p) +{ + return le16_to_cpu(__get_unaligned_t(__le16, p)); +} +static inline u32 get_unaligned_le32(const void *p) +{ + return le32_to_cpu(__get_unaligned_t(__le32, p)); +} + +static inline u64 get_unaligned_le64(const void *p) +{ + return le64_to_cpu(__get_unaligned_t(__le64, p)); +} + +static inline void put_unaligned_le16(u16 val, void *p) +{ + __put_unaligned_t(__le16, cpu_to_le16(val), p); +} + +static inline void put_unaligned_le32(u32 val, void *p) +{ + __put_unaligned_t(__le32, cpu_to_le32(val), p); +} + +static inline void put_unaligned_le64(u64 val, void *p) +{ + __put_unaligned_t(__le64, cpu_to_le64(val), p); +} + +static inline u16 get_unaligned_be16(const void *p) +{ + return be16_to_cpu(__get_unaligned_t(__be16, p)); +} + +static inline u32 get_unaligned_be32(const void *p) +{ + return be32_to_cpu(__get_unaligned_t(__be32, p)); +} + +static inline u64 get_unaligned_be64(const void *p) +{ + return be64_to_cpu(__get_unaligned_t(__be64, p)); +} + +static inline void put_unaligned_be16(u16 val, void *p) +{ + __put_unaligned_t(__be16, cpu_to_be16(val), p); +} + +static inline void put_unaligned_be32(u32 val, void *p) +{ + __put_unaligned_t(__be32, cpu_to_be32(val), p); +} + +static inline void put_unaligned_be64(u64 val, void *p) +{ + __put_unaligned_t(__be64, cpu_to_be64(val), p); +} + +static inline u32 __get_unaligned_be24(const u8 *p) +{ + return p[0] << 16 | p[1] << 8 | p[2]; +} + +static inline u32 get_unaligned_be24(const void *p) +{ + return __get_unaligned_be24(p); +} + +static inline u32 __get_unaligned_le24(const u8 *p) +{ + return p[0] | p[1] << 8 | p[2] << 16; +} + +static inline u32 get_unaligned_le24(const void *p) +{ + return __get_unaligned_le24(p); +} + +static inline void __put_unaligned_be24(const u32 val, u8 *p) +{ + *p++ = val >> 16; + *p++ = val >> 8; + *p++ = val; +} + +static inline void put_unaligned_be24(const u32 val, void *p) +{ + __put_unaligned_be24(val, p); +} + +static inline void __put_unaligned_le24(const u32 val, u8 *p) +{ + *p++ = val; + *p++ = val >> 8; + *p++ = val >> 16; +} + +static inline void put_unaligned_le24(const u32 val, void *p) +{ + __put_unaligned_le24(val, p); +} + +static inline void __put_unaligned_be48(const u64 val, u8 *p) +{ + *p++ = val >> 40; + *p++ = val >> 32; + *p++ = val >> 24; + *p++ = val >> 16; + *p++ = val >> 8; + *p++ = val; +} + +static inline void put_unaligned_be48(const u64 val, void *p) +{ + __put_unaligned_be48(val, p); +} + +static inline u64 __get_unaligned_be48(const u8 *p) +{ + return (u64)p[0] << 40 | (u64)p[1] << 32 | (u64)p[2] << 24 | + p[3] << 16 | p[4] << 8 | p[5]; +} + +static inline u64 get_unaligned_be48(const void *p) +{ + return __get_unaligned_be48(p); +} +#pragma GCC diagnostic pop + +#endif /* __ASM_GENERIC_UNALIGNED_H */ diff --git a/tools/perf/check-headers.sh b/tools/perf/check-headers.sh index e21caadda7c1..66ba33dbcef2 100755 --- a/tools/perf/check-headers.sh +++ b/tools/perf/check-headers.sh @@ -162,6 +162,7 @@ check arch/x86/lib/memcpy_64.S '-I "^EXPORT_SYMBOL" -I "^#include " -I"^SYM_FUNC_START\(_LOCAL\)*(memset_\(erms\|orig\))"' check arch/x86/include/asm/amd-ibs.h '-I "^#include [<\"]\(asm/\)*msr-index.h"' check arch/arm64/include/asm/cputype.h '-I "^#include [<\"]\(asm/\)*sysreg.h"' +check include/asm-generic/unaligned.h '-I "^#include " -I "^#include " -I "^#pragma GCC diagnostic"' check include/uapi/asm-generic/mman.h '-I "^#include <\(uapi/\)*asm-generic/mman-common\(-tools\)*.h>"' check include/uapi/linux/mman.h '-I "^#include <\(uapi/\)*asm/mman.h>"' check include/linux/build_bug.h '-I "^#\(ifndef\|endif\)\( \/\/\)* static_assert$"' From 1d2dbce9bb9243ca18cacff5ada9dc00fc1e74b7 Mon Sep 17 00:00:00 2001 From: Adrian Hunter Date: Thu, 5 Oct 2023 22:04:48 +0300 Subject: [PATCH 239/475] perf intel-pt: Simplify intel_pt_get_vmcs() Simplify and remove unnecessary constant expressions. Signed-off-by: Adrian Hunter Acked-by: Ian Rogers Link: https://lore.kernel.org/r/20231005190451.175568-3-adrian.hunter@intel.com Signed-off-by: Namhyung Kim --- .../util/intel-pt-decoder/intel-pt-pkt-decoder.c | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/tools/perf/util/intel-pt-decoder/intel-pt-pkt-decoder.c b/tools/perf/util/intel-pt-decoder/intel-pt-pkt-decoder.c index af9710622a1f..249610cc9284 100644 --- a/tools/perf/util/intel-pt-decoder/intel-pt-pkt-decoder.c +++ b/tools/perf/util/intel-pt-decoder/intel-pt-pkt-decoder.c @@ -131,19 +131,14 @@ static int intel_pt_get_cbr(const unsigned char *buf, size_t len, static int intel_pt_get_vmcs(const unsigned char *buf, size_t len, struct intel_pt_pkt *packet) { - unsigned int count = (52 - 5) >> 3; - - if (count < 1 || count > 7) - return INTEL_PT_BAD_PACKET; - - if (len < count + 2) + if (len < 7) return INTEL_PT_NEED_MORE_BYTES; packet->type = INTEL_PT_VMCS; - packet->count = count; - memcpy_le64(&packet->payload, buf + 2, count); + packet->count = 5; + memcpy_le64(&packet->payload, buf + 2, 5); - return count + 2; + return 7; } static int intel_pt_get_ovf(struct intel_pt_pkt *packet) From f058fa5b07556abed26e5ae9d8e8ad13f79e1fa2 Mon Sep 17 00:00:00 2001 From: Adrian Hunter Date: Thu, 5 Oct 2023 22:04:49 +0300 Subject: [PATCH 240/475] perf intel-pt: Use existing definitions of le16_to_cpu() etc Use definitions from tools/include/linux/kernel.h Signed-off-by: Adrian Hunter Acked-by: Ian Rogers Link: https://lore.kernel.org/r/20231005190451.175568-4-adrian.hunter@intel.com Signed-off-by: Namhyung Kim --- tools/perf/util/intel-pt-decoder/intel-pt-pkt-decoder.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tools/perf/util/intel-pt-decoder/intel-pt-pkt-decoder.c b/tools/perf/util/intel-pt-decoder/intel-pt-pkt-decoder.c index 249610cc9284..ffd0d647473c 100644 --- a/tools/perf/util/intel-pt-decoder/intel-pt-pkt-decoder.c +++ b/tools/perf/util/intel-pt-decoder/intel-pt-pkt-decoder.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include "intel-pt-pkt-decoder.h" @@ -17,17 +18,11 @@ #define BIT63 ((uint64_t)1 << 63) #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -#define le16_to_cpu bswap_16 -#define le32_to_cpu bswap_32 -#define le64_to_cpu bswap_64 #define memcpy_le64(d, s, n) do { \ memcpy((d), (s), (n)); \ *(d) = le64_to_cpu(*(d)); \ } while (0) #else -#define le16_to_cpu -#define le32_to_cpu -#define le64_to_cpu #define memcpy_le64 memcpy #endif From 3b4fa67fc666eb88ae4c0ffdfbe338bec989fbee Mon Sep 17 00:00:00 2001 From: Adrian Hunter Date: Thu, 5 Oct 2023 22:04:50 +0300 Subject: [PATCH 241/475] perf intel-pt: Use get_unaligned_le16() etc Avoid unaligned access by using get_unaligned_le16(), get_unaligned_le32() and get_unaligned_le64(). Signed-off-by: Adrian Hunter Acked-by: Ian Rogers Link: https://lore.kernel.org/r/20231005190451.175568-5-adrian.hunter@intel.com Signed-off-by: Namhyung Kim --- .../intel-pt-decoder/intel-pt-pkt-decoder.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tools/perf/util/intel-pt-decoder/intel-pt-pkt-decoder.c b/tools/perf/util/intel-pt-decoder/intel-pt-pkt-decoder.c index ffd0d647473c..7a90218aecb1 100644 --- a/tools/perf/util/intel-pt-decoder/intel-pt-pkt-decoder.c +++ b/tools/perf/util/intel-pt-decoder/intel-pt-pkt-decoder.c @@ -10,6 +10,7 @@ #include #include #include +#include #include "intel-pt-pkt-decoder.h" @@ -78,7 +79,7 @@ static int intel_pt_get_long_tnt(const unsigned char *buf, size_t len, if (len < 8) return INTEL_PT_NEED_MORE_BYTES; - payload = le64_to_cpu(*(uint64_t *)buf); + payload = get_unaligned_le64(buf); for (count = 47; count; count--) { if (payload & BIT63) @@ -119,7 +120,7 @@ static int intel_pt_get_cbr(const unsigned char *buf, size_t len, if (len < 4) return INTEL_PT_NEED_MORE_BYTES; packet->type = INTEL_PT_CBR; - packet->payload = le16_to_cpu(*(uint16_t *)(buf + 2)); + packet->payload = get_unaligned_le16(buf + 2); return 4; } @@ -218,12 +219,12 @@ static int intel_pt_get_ptwrite(const unsigned char *buf, size_t len, case 0: if (len < 6) return INTEL_PT_NEED_MORE_BYTES; - packet->payload = le32_to_cpu(*(uint32_t *)(buf + 2)); + packet->payload = get_unaligned_le32(buf + 2); return 6; case 1: if (len < 10) return INTEL_PT_NEED_MORE_BYTES; - packet->payload = le64_to_cpu(*(uint64_t *)(buf + 2)); + packet->payload = get_unaligned_le64(buf + 2); return 10; default: return INTEL_PT_BAD_PACKET; @@ -248,7 +249,7 @@ static int intel_pt_get_mwait(const unsigned char *buf, size_t len, if (len < 10) return INTEL_PT_NEED_MORE_BYTES; packet->type = INTEL_PT_MWAIT; - packet->payload = le64_to_cpu(*(uint64_t *)(buf + 2)); + packet->payload = get_unaligned_le64(buf + 2); return 10; } @@ -455,13 +456,13 @@ static int intel_pt_get_ip(enum intel_pt_pkt_type type, unsigned int byte, if (len < 3) return INTEL_PT_NEED_MORE_BYTES; ip_len = 2; - packet->payload = le16_to_cpu(*(uint16_t *)(buf + 1)); + packet->payload = get_unaligned_le16(buf + 1); break; case 2: if (len < 5) return INTEL_PT_NEED_MORE_BYTES; ip_len = 4; - packet->payload = le32_to_cpu(*(uint32_t *)(buf + 1)); + packet->payload = get_unaligned_le32(buf + 1); break; case 3: case 4: @@ -474,7 +475,7 @@ static int intel_pt_get_ip(enum intel_pt_pkt_type type, unsigned int byte, if (len < 9) return INTEL_PT_NEED_MORE_BYTES; ip_len = 8; - packet->payload = le64_to_cpu(*(uint64_t *)(buf + 1)); + packet->payload = get_unaligned_le64(buf + 1); break; default: return INTEL_PT_BAD_PACKET; From 661ce78105d739f3297de8a2384f4e48a2a16043 Mon Sep 17 00:00:00 2001 From: Adrian Hunter Date: Thu, 5 Oct 2023 22:04:51 +0300 Subject: [PATCH 242/475] perf intel-pt: Prefer get_unaligned_le64 to memcpy_le64 Use get_unaligned_le64() instead of memcpy_le64(..., 8) because it produces simpler code. Signed-off-by: Adrian Hunter Acked-by: Ian Rogers Link: https://lore.kernel.org/r/20231005190451.175568-6-adrian.hunter@intel.com Signed-off-by: Namhyung Kim --- tools/perf/util/intel-pt-decoder/intel-pt-pkt-decoder.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/perf/util/intel-pt-decoder/intel-pt-pkt-decoder.c b/tools/perf/util/intel-pt-decoder/intel-pt-pkt-decoder.c index 7a90218aecb1..bccb988a7a44 100644 --- a/tools/perf/util/intel-pt-decoder/intel-pt-pkt-decoder.c +++ b/tools/perf/util/intel-pt-decoder/intel-pt-pkt-decoder.c @@ -190,7 +190,7 @@ static int intel_pt_get_mnt(const unsigned char *buf, size_t len, if (len < 11) return INTEL_PT_NEED_MORE_BYTES; packet->type = INTEL_PT_MNT; - memcpy_le64(&packet->payload, buf + 3, 8); + packet->payload = get_unaligned_le64(buf + 3); return 11; } @@ -302,7 +302,7 @@ static int intel_pt_get_bip_8(const unsigned char *buf, size_t len, return INTEL_PT_NEED_MORE_BYTES; packet->type = INTEL_PT_BIP; packet->count = buf[0] >> 3; - memcpy_le64(&packet->payload, buf + 1, 8); + packet->payload = get_unaligned_le64(buf + 1); return 9; } @@ -341,7 +341,7 @@ static int intel_pt_get_evd(const unsigned char *buf, size_t len, packet->type = INTEL_PT_EVD; packet->count = buf[2] & 0x3f; packet->payload = buf[3]; - memcpy_le64(&packet->payload, buf + 3, 8); + packet->payload = get_unaligned_le64(buf + 3); return 11; } From aa61360155ac69003dc5c177f03a7f177a435286 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Thu, 12 Oct 2023 10:56:39 -0700 Subject: [PATCH 243/475] perf pmu: Rename perf_pmu__get_default_config to perf_pmu__arch_init Assign default_config as part of the init. perf_pmu__get_default_config was doing more than just getting the default config and so this is intended to better align with the code. Signed-off-by: Ian Rogers Cc: Ravi Bangoria Cc: James Clark Cc: Suzuki K Poulose Cc: Yang Jihong Cc: Will Deacon Cc: Leo Yan Cc: Mike Leach Cc: Jing Zhang Cc: Kajol Jain Cc: Thomas Richter Cc: Alexander Shishkin Cc: Kan Liang Cc: John Garry Cc: linux-arm-kernel@lists.infradead.org Cc: coresight@lists.linaro.org Link: https://lore.kernel.org/r/20231012175645.1849503-2-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/arch/arm/util/pmu.c | 8 +++----- tools/perf/arch/s390/util/pmu.c | 3 +-- tools/perf/arch/x86/util/pmu.c | 5 ++--- tools/perf/util/pmu.c | 13 ++++++------- tools/perf/util/pmu.h | 2 +- 5 files changed, 13 insertions(+), 18 deletions(-) diff --git a/tools/perf/arch/arm/util/pmu.c b/tools/perf/arch/arm/util/pmu.c index a9623b128ece..d55d2b15f2e6 100644 --- a/tools/perf/arch/arm/util/pmu.c +++ b/tools/perf/arch/arm/util/pmu.c @@ -14,22 +14,20 @@ #include "../../../util/pmu.h" #include "../../../util/cs-etm.h" -struct perf_event_attr -*perf_pmu__get_default_config(struct perf_pmu *pmu __maybe_unused) +void perf_pmu__arch_init(struct perf_pmu *pmu __maybe_unused) { #ifdef HAVE_AUXTRACE_SUPPORT if (!strcmp(pmu->name, CORESIGHT_ETM_PMU_NAME)) { /* add ETM default config here */ pmu->selectable = true; - return cs_etm_get_default_config(pmu); + pmu->default_config = cs_etm_get_default_config(pmu); #if defined(__aarch64__) } else if (strstarts(pmu->name, ARM_SPE_PMU_NAME)) { - return arm_spe_pmu_default_config(pmu); + pmu->default_config = arm_spe_pmu_default_config(pmu); } else if (strstarts(pmu->name, HISI_PTT_PMU_NAME)) { pmu->selectable = true; #endif } #endif - return NULL; } diff --git a/tools/perf/arch/s390/util/pmu.c b/tools/perf/arch/s390/util/pmu.c index 11f03f32e3fd..886c30e001fa 100644 --- a/tools/perf/arch/s390/util/pmu.c +++ b/tools/perf/arch/s390/util/pmu.c @@ -13,11 +13,10 @@ #define S390_PMUPAI_EXT "pai_ext" #define S390_PMUCPUM_CF "cpum_cf" -struct perf_event_attr *perf_pmu__get_default_config(struct perf_pmu *pmu) +void perf_pmu__arch_init(struct perf_pmu *pmu) { if (!strcmp(pmu->name, S390_PMUPAI_CRYPTO) || !strcmp(pmu->name, S390_PMUPAI_EXT) || !strcmp(pmu->name, S390_PMUCPUM_CF)) pmu->selectable = true; - return NULL; } diff --git a/tools/perf/arch/x86/util/pmu.c b/tools/perf/arch/x86/util/pmu.c index 8b53ca468a50..811e2377d2d5 100644 --- a/tools/perf/arch/x86/util/pmu.c +++ b/tools/perf/arch/x86/util/pmu.c @@ -17,19 +17,18 @@ #include "../../../util/pmus.h" #include "env.h" -struct perf_event_attr *perf_pmu__get_default_config(struct perf_pmu *pmu __maybe_unused) +void perf_pmu__arch_init(struct perf_pmu *pmu __maybe_unused) { #ifdef HAVE_AUXTRACE_SUPPORT if (!strcmp(pmu->name, INTEL_PT_PMU_NAME)) { pmu->auxtrace = true; - return intel_pt_pmu_default_config(pmu); + pmu->default_config = intel_pt_pmu_default_config(pmu); } if (!strcmp(pmu->name, INTEL_BTS_PMU_NAME)) { pmu->auxtrace = true; pmu->selectable = true; } #endif - return NULL; } int perf_pmus__num_mem_pmus(void) diff --git a/tools/perf/util/pmu.c b/tools/perf/util/pmu.c index 6428e2648289..d075da0eecc0 100644 --- a/tools/perf/util/pmu.c +++ b/tools/perf/util/pmu.c @@ -954,12 +954,6 @@ void pmu_add_sys_aliases(struct perf_pmu *pmu) pmu_for_each_sys_event(pmu_add_sys_aliases_iter_fn, pmu); } -struct perf_event_attr * __weak -perf_pmu__get_default_config(struct perf_pmu *pmu __maybe_unused) -{ - return NULL; -} - static char *pmu_find_alias_name(struct perf_pmu *pmu, int dirfd) { FILE *file = perf_pmu__open_file_at(pmu, dirfd, "alias"); @@ -991,6 +985,11 @@ static int pmu_max_precise(int dirfd, struct perf_pmu *pmu) return max_precise; } +void __weak +perf_pmu__arch_init(struct perf_pmu *pmu __maybe_unused) +{ +} + struct perf_pmu *perf_pmu__lookup(struct list_head *pmus, int dirfd, const char *name) { struct perf_pmu *pmu; @@ -1037,7 +1036,7 @@ struct perf_pmu *perf_pmu__lookup(struct list_head *pmus, int dirfd, const char pmu_add_sys_aliases(pmu); list_add_tail(&pmu->list, pmus); - pmu->default_config = perf_pmu__get_default_config(pmu); + perf_pmu__arch_init(pmu); return pmu; err: diff --git a/tools/perf/util/pmu.h b/tools/perf/util/pmu.h index 85190d058852..588c64e38d6b 100644 --- a/tools/perf/util/pmu.h +++ b/tools/perf/util/pmu.h @@ -233,7 +233,7 @@ bool perf_pmu__file_exists(struct perf_pmu *pmu, const char *name); int perf_pmu__test(void); -struct perf_event_attr *perf_pmu__get_default_config(struct perf_pmu *pmu); +void perf_pmu__arch_init(struct perf_pmu *pmu); void pmu_add_cpu_aliases_table(struct perf_pmu *pmu, const struct pmu_events_table *table); From 461e3e636a26518d3dd9c0ae3aff32894ec00b26 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Thu, 12 Oct 2023 10:56:40 -0700 Subject: [PATCH 244/475] perf intel-pt: Move PMU initialization from default config code Avoid setting PMU values in intel_pt_pmu_default_config, move to perf_pmu__arch_init. Signed-off-by: Ian Rogers Reviewed-by: Adrian Hunter Cc: Ravi Bangoria Cc: James Clark Cc: Suzuki K Poulose Cc: Yang Jihong Cc: Will Deacon Cc: Leo Yan Cc: Mike Leach Cc: Jing Zhang Cc: Kajol Jain Cc: Thomas Richter Cc: Kan Liang Cc: John Garry Cc: linux-arm-kernel@lists.infradead.org Cc: coresight@lists.linaro.org Link: https://lore.kernel.org/r/20231012175645.1849503-3-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/arch/x86/util/intel-pt.c | 2 -- tools/perf/arch/x86/util/pmu.c | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/perf/arch/x86/util/intel-pt.c b/tools/perf/arch/x86/util/intel-pt.c index b923bca939d9..6d6cd8f9133c 100644 --- a/tools/perf/arch/x86/util/intel-pt.c +++ b/tools/perf/arch/x86/util/intel-pt.c @@ -267,8 +267,6 @@ intel_pt_pmu_default_config(struct perf_pmu *intel_pt_pmu) attr->config = intel_pt_default_config(intel_pt_pmu); - intel_pt_pmu->selectable = true; - return attr; } diff --git a/tools/perf/arch/x86/util/pmu.c b/tools/perf/arch/x86/util/pmu.c index 811e2377d2d5..949b3e2c67bd 100644 --- a/tools/perf/arch/x86/util/pmu.c +++ b/tools/perf/arch/x86/util/pmu.c @@ -22,6 +22,7 @@ void perf_pmu__arch_init(struct perf_pmu *pmu __maybe_unused) #ifdef HAVE_AUXTRACE_SUPPORT if (!strcmp(pmu->name, INTEL_PT_PMU_NAME)) { pmu->auxtrace = true; + pmu->selectable = true; pmu->default_config = intel_pt_pmu_default_config(pmu); } if (!strcmp(pmu->name, INTEL_BTS_PMU_NAME)) { From 672bd21390d38dfccbe52b9537555985d65446aa Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Thu, 12 Oct 2023 10:56:41 -0700 Subject: [PATCH 245/475] perf arm-spe: Move PMU initialization from default config code Avoid setting PMU values in arm_spe_pmu_default_config, move to perf_pmu__arch_init. Signed-off-by: Ian Rogers Reviewed-by: Adrian Hunter Tested-by: Leo Yan Cc: Ravi Bangoria Cc: James Clark Cc: Suzuki K Poulose Cc: Yang Jihong Cc: Will Deacon Cc: Mike Leach Cc: Jing Zhang Cc: Kajol Jain Cc: Thomas Richter Cc: Alexander Shishkin Cc: Kan Liang Cc: John Garry Cc: linux-arm-kernel@lists.infradead.org Cc: coresight@lists.linaro.org Link: https://lore.kernel.org/r/20231012175645.1849503-4-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/arch/arm/util/pmu.c | 2 ++ tools/perf/arch/arm64/util/arm-spe.c | 3 --- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tools/perf/arch/arm/util/pmu.c b/tools/perf/arch/arm/util/pmu.c index d55d2b15f2e6..f25f68f84a94 100644 --- a/tools/perf/arch/arm/util/pmu.c +++ b/tools/perf/arch/arm/util/pmu.c @@ -23,6 +23,8 @@ void perf_pmu__arch_init(struct perf_pmu *pmu __maybe_unused) pmu->default_config = cs_etm_get_default_config(pmu); #if defined(__aarch64__) } else if (strstarts(pmu->name, ARM_SPE_PMU_NAME)) { + pmu->selectable = true; + pmu->is_uncore = false; pmu->default_config = arm_spe_pmu_default_config(pmu); } else if (strstarts(pmu->name, HISI_PTT_PMU_NAME)) { pmu->selectable = true; diff --git a/tools/perf/arch/arm64/util/arm-spe.c b/tools/perf/arch/arm64/util/arm-spe.c index 9cc3d6dcb849..08a76734ccd2 100644 --- a/tools/perf/arch/arm64/util/arm-spe.c +++ b/tools/perf/arch/arm64/util/arm-spe.c @@ -516,8 +516,5 @@ struct perf_event_attr attr->sample_period = 4096; } - arm_spe_pmu->selectable = true; - arm_spe_pmu->is_uncore = false; - return attr; } From 3a42f4c796ce45a6b1d14975401deb2c457ae79d Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Thu, 12 Oct 2023 10:56:42 -0700 Subject: [PATCH 246/475] perf pmu: Const-ify file APIs File APIs don't alter the struct pmu so allow const ones to be passed. Signed-off-by: Ian Rogers Reviewed-by: Adrian Hunter Cc: Ravi Bangoria Cc: James Clark Cc: Suzuki K Poulose Cc: Yang Jihong Cc: Will Deacon Cc: Leo Yan Cc: Mike Leach Cc: Jing Zhang Cc: Kajol Jain Cc: Thomas Richter Cc: Kan Liang Cc: John Garry Cc: linux-arm-kernel@lists.infradead.org Cc: coresight@lists.linaro.org Link: https://lore.kernel.org/r/20231012175645.1849503-5-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/util/pmu.c | 12 ++++++------ tools/perf/util/pmu.h | 11 ++++++----- tools/perf/util/python.c | 2 +- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/tools/perf/util/pmu.c b/tools/perf/util/pmu.c index d075da0eecc0..861e485e0a70 100644 --- a/tools/perf/util/pmu.c +++ b/tools/perf/util/pmu.c @@ -575,7 +575,7 @@ static int perf_pmu__new_alias(struct perf_pmu *pmu, const char *name, return 0; } -static inline bool pmu_alias_info_file(char *name) +static inline bool pmu_alias_info_file(const char *name) { size_t len; @@ -1770,7 +1770,7 @@ bool perf_pmu__is_software(const struct perf_pmu *pmu) return !strcmp(pmu->name, "kprobe") || !strcmp(pmu->name, "uprobe"); } -FILE *perf_pmu__open_file(struct perf_pmu *pmu, const char *name) +FILE *perf_pmu__open_file(const struct perf_pmu *pmu, const char *name) { char path[PATH_MAX]; @@ -1781,7 +1781,7 @@ FILE *perf_pmu__open_file(struct perf_pmu *pmu, const char *name) return fopen(path, "r"); } -FILE *perf_pmu__open_file_at(struct perf_pmu *pmu, int dirfd, const char *name) +FILE *perf_pmu__open_file_at(const struct perf_pmu *pmu, int dirfd, const char *name) { int fd; @@ -1792,7 +1792,7 @@ FILE *perf_pmu__open_file_at(struct perf_pmu *pmu, int dirfd, const char *name) return fdopen(fd, "r"); } -int perf_pmu__scan_file(struct perf_pmu *pmu, const char *name, const char *fmt, +int perf_pmu__scan_file(const struct perf_pmu *pmu, const char *name, const char *fmt, ...) { va_list args; @@ -1809,7 +1809,7 @@ int perf_pmu__scan_file(struct perf_pmu *pmu, const char *name, const char *fmt, return ret; } -int perf_pmu__scan_file_at(struct perf_pmu *pmu, int dirfd, const char *name, +int perf_pmu__scan_file_at(const struct perf_pmu *pmu, int dirfd, const char *name, const char *fmt, ...) { va_list args; @@ -1826,7 +1826,7 @@ int perf_pmu__scan_file_at(struct perf_pmu *pmu, int dirfd, const char *name, return ret; } -bool perf_pmu__file_exists(struct perf_pmu *pmu, const char *name) +bool perf_pmu__file_exists(const struct perf_pmu *pmu, const char *name) { char path[PATH_MAX]; diff --git a/tools/perf/util/pmu.h b/tools/perf/util/pmu.h index 588c64e38d6b..24af7297b522 100644 --- a/tools/perf/util/pmu.h +++ b/tools/perf/util/pmu.h @@ -222,14 +222,15 @@ bool pmu__name_match(const struct perf_pmu *pmu, const char *pmu_name); */ bool perf_pmu__is_software(const struct perf_pmu *pmu); -FILE *perf_pmu__open_file(struct perf_pmu *pmu, const char *name); -FILE *perf_pmu__open_file_at(struct perf_pmu *pmu, int dirfd, const char *name); +FILE *perf_pmu__open_file(const struct perf_pmu *pmu, const char *name); +FILE *perf_pmu__open_file_at(const struct perf_pmu *pmu, int dirfd, const char *name); -int perf_pmu__scan_file(struct perf_pmu *pmu, const char *name, const char *fmt, ...) __scanf(3, 4); -int perf_pmu__scan_file_at(struct perf_pmu *pmu, int dirfd, const char *name, +int perf_pmu__scan_file(const struct perf_pmu *pmu, const char *name, const char *fmt, ...) + __scanf(3, 4); +int perf_pmu__scan_file_at(const struct perf_pmu *pmu, int dirfd, const char *name, const char *fmt, ...) __scanf(4, 5); -bool perf_pmu__file_exists(struct perf_pmu *pmu, const char *name); +bool perf_pmu__file_exists(const struct perf_pmu *pmu, const char *name); int perf_pmu__test(void); diff --git a/tools/perf/util/python.c b/tools/perf/util/python.c index c29f5f0bb552..8761f51b5c7c 100644 --- a/tools/perf/util/python.c +++ b/tools/perf/util/python.c @@ -98,7 +98,7 @@ struct perf_pmu *evsel__find_pmu(const struct evsel *evsel __maybe_unused) return NULL; } -int perf_pmu__scan_file(struct perf_pmu *pmu, const char *name, const char *fmt, ...) +int perf_pmu__scan_file(const struct perf_pmu *pmu, const char *name, const char *fmt, ...) { return EOF; } From 63883cb063846e9d0574038bb702a3a62a1ec046 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Thu, 12 Oct 2023 10:56:43 -0700 Subject: [PATCH 247/475] perf pmu: Const-ify perf_pmu__config_terms Add const to related APIs, this is so they can be used to default initialize a perf_event_attr from a const pmu. Signed-off-by: Ian Rogers Reviewed-by: Adrian Hunter Cc: Ravi Bangoria Cc: James Clark Cc: Suzuki K Poulose Cc: Yang Jihong Cc: Will Deacon Cc: Leo Yan Cc: Mike Leach Cc: Jing Zhang Cc: Kajol Jain Cc: Thomas Richter Cc: Kan Liang Cc: John Garry Cc: linux-arm-kernel@lists.infradead.org Cc: coresight@lists.linaro.org Link: https://lore.kernel.org/r/20231012175645.1849503-6-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/util/pmu.c | 10 +++++----- tools/perf/util/pmu.h | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/perf/util/pmu.c b/tools/perf/util/pmu.c index 861e485e0a70..8ef675ea7bdd 100644 --- a/tools/perf/util/pmu.c +++ b/tools/perf/util/pmu.c @@ -156,7 +156,7 @@ static void __perf_pmu_format__load(struct perf_pmu_format *format, FILE *file) format->loaded = true; } -static void perf_pmu_format__load(struct perf_pmu *pmu, struct perf_pmu_format *format) +static void perf_pmu_format__load(const struct perf_pmu *pmu, struct perf_pmu_format *format) { char path[PATH_MAX]; FILE *file = NULL; @@ -1131,7 +1131,7 @@ void evsel__set_config_if_unset(struct perf_pmu *pmu, struct evsel *evsel, } static struct perf_pmu_format * -pmu_find_format(struct list_head *formats, const char *name) +pmu_find_format(const struct list_head *formats, const char *name) { struct perf_pmu_format *format; @@ -1229,7 +1229,7 @@ static int pmu_resolve_param_term(struct parse_events_term *term, return -1; } -static char *pmu_formats_string(struct list_head *formats) +static char *pmu_formats_string(const struct list_head *formats) { struct perf_pmu_format *format; char *str = NULL; @@ -1255,7 +1255,7 @@ error: * Setup one of config[12] attr members based on the * user input data - term parameter. */ -static int pmu_config_term(struct perf_pmu *pmu, +static int pmu_config_term(const struct perf_pmu *pmu, struct perf_event_attr *attr, struct parse_events_term *term, struct parse_events_terms *head_terms, @@ -1378,7 +1378,7 @@ static int pmu_config_term(struct perf_pmu *pmu, return 0; } -int perf_pmu__config_terms(struct perf_pmu *pmu, +int perf_pmu__config_terms(const struct perf_pmu *pmu, struct perf_event_attr *attr, struct parse_events_terms *terms, bool zero, struct parse_events_error *err) diff --git a/tools/perf/util/pmu.h b/tools/perf/util/pmu.h index 24af7297b522..5a05131aa4ce 100644 --- a/tools/perf/util/pmu.h +++ b/tools/perf/util/pmu.h @@ -193,7 +193,7 @@ void pmu_add_sys_aliases(struct perf_pmu *pmu); int perf_pmu__config(struct perf_pmu *pmu, struct perf_event_attr *attr, struct parse_events_terms *head_terms, struct parse_events_error *error); -int perf_pmu__config_terms(struct perf_pmu *pmu, +int perf_pmu__config_terms(const struct perf_pmu *pmu, struct perf_event_attr *attr, struct parse_events_terms *terms, bool zero, struct parse_events_error *error); From f20c15d13f017d4b46523990a0f3ba42025391a6 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Thu, 12 Oct 2023 10:56:44 -0700 Subject: [PATCH 248/475] perf pmu-events: Remember the perf_events_map for a PMU strcmp_cpuid_str performs regular expression comparisons and so per CPUID linear searches over the perf_events_map are expensive. Add a helper function called map_for_pmu that does the search but also caches the map specific to a PMU. As the PMU may differ, also cache the CPUID string so that PMUs with the same CPUID string don't require the linear search and regular expression comparisons. This speeds loading PMUs as the search is done once per PMU to find the appropriate tables. Signed-off-by: Ian Rogers Tested-by: Yang Jihong Cc: Ravi Bangoria Cc: James Clark Cc: Suzuki K Poulose Cc: Will Deacon Cc: Leo Yan Cc: Mike Leach Cc: Jing Zhang Cc: Kajol Jain Cc: Thomas Richter Cc: Kan Liang Cc: John Garry Cc: linux-arm-kernel@lists.infradead.org Cc: coresight@lists.linaro.org Link: https://lore.kernel.org/r/20231012175645.1849503-7-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/pmu-events/jevents.py | 109 ++++++++++++++++++++----------- 1 file changed, 70 insertions(+), 39 deletions(-) diff --git a/tools/perf/pmu-events/jevents.py b/tools/perf/pmu-events/jevents.py index 96dc74c90b20..3c091ab75305 100755 --- a/tools/perf/pmu-events/jevents.py +++ b/tools/perf/pmu-events/jevents.py @@ -976,68 +976,99 @@ int pmu_metrics_table__for_each_metric(const struct pmu_metrics_table *table, return 0; } -const struct pmu_events_table *perf_pmu__find_events_table(struct perf_pmu *pmu) +static const struct pmu_events_map *map_for_pmu(struct perf_pmu *pmu) { - const struct pmu_events_table *table = NULL; - char *cpuid = perf_pmu__getcpuid(pmu); + static struct { + const struct pmu_events_map *map; + struct perf_pmu *pmu; + } last_result; + static struct { + const struct pmu_events_map *map; + char *cpuid; + } last_map_search; + static bool has_last_result, has_last_map_search; + const struct pmu_events_map *map = NULL; + char *cpuid = NULL; size_t i; - /* on some platforms which uses cpus map, cpuid can be NULL for + if (has_last_result && last_result.pmu == pmu) + return last_result.map; + + cpuid = perf_pmu__getcpuid(pmu); + + /* + * On some platforms which uses cpus map, cpuid can be NULL for * PMUs other than CORE PMUs. */ if (!cpuid) + goto out_update_last_result; + + if (has_last_map_search && !strcmp(last_map_search.cpuid, cpuid)) { + map = last_map_search.map; + free(cpuid); + } else { + i = 0; + for (;;) { + map = &pmu_events_map[i++]; + + if (!map->arch) { + map = NULL; + break; + } + + if (!strcmp_cpuid_str(map->cpuid, cpuid)) + break; + } + free(last_map_search.cpuid); + last_map_search.cpuid = cpuid; + last_map_search.map = map; + has_last_map_search = true; + } +out_update_last_result: + last_result.pmu = pmu; + last_result.map = map; + has_last_result = true; + return map; +} + +const struct pmu_events_table *perf_pmu__find_events_table(struct perf_pmu *pmu) +{ + const struct pmu_events_map *map = map_for_pmu(pmu); + + if (!map) return NULL; - i = 0; - for (;;) { - const struct pmu_events_map *map = &pmu_events_map[i++]; - if (!map->arch) - break; + if (!pmu) + return &map->event_table; - if (!strcmp_cpuid_str(map->cpuid, cpuid)) { - table = &map->event_table; - break; - } - } - free(cpuid); - if (!pmu || !table) - return table; - - for (i = 0; i < table->num_pmus; i++) { - const struct pmu_table_entry *table_pmu = &table->pmus[i]; + for (size_t i = 0; i < map->event_table.num_pmus; i++) { + const struct pmu_table_entry *table_pmu = &map->event_table.pmus[i]; const char *pmu_name = &big_c_string[table_pmu->pmu_name.offset]; if (pmu__name_match(pmu, pmu_name)) - return table; + return &map->event_table; } return NULL; } const struct pmu_metrics_table *perf_pmu__find_metrics_table(struct perf_pmu *pmu) { - const struct pmu_metrics_table *table = NULL; - char *cpuid = perf_pmu__getcpuid(pmu); - int i; + const struct pmu_events_map *map = map_for_pmu(pmu); - /* on some platforms which uses cpus map, cpuid can be NULL for - * PMUs other than CORE PMUs. - */ - if (!cpuid) + if (!map) return NULL; - i = 0; - for (;;) { - const struct pmu_events_map *map = &pmu_events_map[i++]; - if (!map->arch) - break; + if (!pmu) + return &map->metric_table; - if (!strcmp_cpuid_str(map->cpuid, cpuid)) { - table = &map->metric_table; - break; - } + for (size_t i = 0; i < map->metric_table.num_pmus; i++) { + const struct pmu_table_entry *table_pmu = &map->metric_table.pmus[i]; + const char *pmu_name = &big_c_string[table_pmu->pmu_name.offset]; + + if (pmu__name_match(pmu, pmu_name)) + return &map->metric_table; } - free(cpuid); - return table; + return NULL; } const struct pmu_events_table *find_core_events_table(const char *arch, const char *cpuid) From 0197da7affab502cd6e25da616ad038b169a7a77 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Thu, 12 Oct 2023 10:56:45 -0700 Subject: [PATCH 249/475] perf pmu: Lazily compute default config The default config is computed during creation of the PMU and may do things like scanning sysfs, when the PMU may just be used as part of scanning. Change default_config to perf_event_attr_init_default, a callback that is used when a default config needs initializing. This avoids holding onto the memory for a perf_event_attr and copying. On a tigerlake laptop running the pmu-scan benchmark: Before: Running 'internals/pmu-scan' benchmark: Computing performance of sysfs PMU event scan for 100 times Average core PMU scanning took: 28.780 usec (+- 0.503 usec) Average PMU scanning took: 283.480 usec (+- 18.471 usec) Number of openat syscalls: 30,227 After: Running 'internals/pmu-scan' benchmark: Computing performance of sysfs PMU event scan for 100 times Average core PMU scanning took: 27.880 usec (+- 0.169 usec) Average PMU scanning took: 245.260 usec (+- 15.758 usec) Number of openat syscalls: 28,914 Over 3 runs it is a nearly 12% reduction in execution time and a 4.3% of openat calls. Signed-off-by: Ian Rogers Reviewed-by: Adrian Hunter Cc: Ravi Bangoria Cc: James Clark Cc: Suzuki K Poulose Cc: Yang Jihong Cc: Will Deacon Cc: Leo Yan Cc: Mike Leach Cc: Jing Zhang Cc: Kajol Jain Cc: Thomas Richter Cc: Kan Liang Cc: John Garry Cc: linux-arm-kernel@lists.infradead.org Cc: coresight@lists.linaro.org Link: https://lore.kernel.org/r/20231012175645.1849503-8-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/arch/arm/util/cs-etm.c | 13 ++------ tools/perf/arch/arm/util/pmu.c | 4 +-- tools/perf/arch/arm64/util/arm-spe.c | 45 ++++++++++++++-------------- tools/perf/arch/x86/util/intel-pt.c | 25 ++++++++-------- tools/perf/arch/x86/util/pmu.c | 2 +- tools/perf/util/arm-spe.h | 4 ++- tools/perf/util/cs-etm.h | 2 +- tools/perf/util/intel-pt.h | 3 +- tools/perf/util/parse-events.c | 12 ++++---- tools/perf/util/pmu.c | 3 +- tools/perf/util/pmu.h | 7 +++-- 11 files changed, 58 insertions(+), 62 deletions(-) diff --git a/tools/perf/arch/arm/util/cs-etm.c b/tools/perf/arch/arm/util/cs-etm.c index b8d6a953fd74..16bba74f048b 100644 --- a/tools/perf/arch/arm/util/cs-etm.c +++ b/tools/perf/arch/arm/util/cs-etm.c @@ -917,16 +917,9 @@ out: * (CFG_CHG and evsel__set_config_if_unset()). If no default is set then user * changes aren't tracked. */ -struct perf_event_attr * -cs_etm_get_default_config(struct perf_pmu *pmu __maybe_unused) +void +cs_etm_get_default_config(const struct perf_pmu *pmu __maybe_unused, + struct perf_event_attr *attr) { - struct perf_event_attr *attr; - - attr = zalloc(sizeof(struct perf_event_attr)); - if (!attr) - return NULL; - attr->sample_period = 1; - - return attr; } diff --git a/tools/perf/arch/arm/util/pmu.c b/tools/perf/arch/arm/util/pmu.c index f25f68f84a94..7f3af3b97f3b 100644 --- a/tools/perf/arch/arm/util/pmu.c +++ b/tools/perf/arch/arm/util/pmu.c @@ -20,12 +20,12 @@ void perf_pmu__arch_init(struct perf_pmu *pmu __maybe_unused) if (!strcmp(pmu->name, CORESIGHT_ETM_PMU_NAME)) { /* add ETM default config here */ pmu->selectable = true; - pmu->default_config = cs_etm_get_default_config(pmu); + pmu->perf_event_attr_init_default = cs_etm_get_default_config; #if defined(__aarch64__) } else if (strstarts(pmu->name, ARM_SPE_PMU_NAME)) { pmu->selectable = true; pmu->is_uncore = false; - pmu->default_config = arm_spe_pmu_default_config(pmu); + pmu->perf_event_attr_init_default = arm_spe_pmu_default_config; } else if (strstarts(pmu->name, HISI_PTT_PMU_NAME)) { pmu->selectable = true; #endif diff --git a/tools/perf/arch/arm64/util/arm-spe.c b/tools/perf/arch/arm64/util/arm-spe.c index 08a76734ccd2..e3acc739bd00 100644 --- a/tools/perf/arch/arm64/util/arm-spe.c +++ b/tools/perf/arch/arm64/util/arm-spe.c @@ -113,6 +113,25 @@ arm_spe_snapshot_resolve_auxtrace_defaults(struct record_opts *opts, } } +static __u64 arm_spe_pmu__sample_period(const struct perf_pmu *arm_spe_pmu) +{ + static __u64 sample_period; + + if (sample_period) + return sample_period; + + /* + * If kernel driver doesn't advertise a minimum, + * use max allowable by PMSIDR_EL1.INTERVAL + */ + if (perf_pmu__scan_file(arm_spe_pmu, "caps/min_interval", "%llu", + &sample_period) != 1) { + pr_debug("arm_spe driver doesn't advertise a min. interval. Using 4096\n"); + sample_period = 4096; + } + return sample_period; +} + static int arm_spe_recording_options(struct auxtrace_record *itr, struct evlist *evlist, struct record_opts *opts) @@ -136,7 +155,7 @@ static int arm_spe_recording_options(struct auxtrace_record *itr, return -EINVAL; } evsel->core.attr.freq = 0; - evsel->core.attr.sample_period = arm_spe_pmu->default_config->sample_period; + evsel->core.attr.sample_period = arm_spe_pmu__sample_period(arm_spe_pmu); evsel->needs_auxtrace_mmap = true; arm_spe_evsel = evsel; opts->full_auxtrace = true; @@ -495,26 +514,8 @@ struct auxtrace_record *arm_spe_recording_init(int *err, return &sper->itr; } -struct perf_event_attr -*arm_spe_pmu_default_config(struct perf_pmu *arm_spe_pmu) +void +arm_spe_pmu_default_config(const struct perf_pmu *arm_spe_pmu, struct perf_event_attr *attr) { - struct perf_event_attr *attr; - - attr = zalloc(sizeof(struct perf_event_attr)); - if (!attr) { - pr_err("arm_spe default config cannot allocate a perf_event_attr\n"); - return NULL; - } - - /* - * If kernel driver doesn't advertise a minimum, - * use max allowable by PMSIDR_EL1.INTERVAL - */ - if (perf_pmu__scan_file(arm_spe_pmu, "caps/min_interval", "%llu", - &attr->sample_period) != 1) { - pr_debug("arm_spe driver doesn't advertise a min. interval. Using 4096\n"); - attr->sample_period = 4096; - } - - return attr; + attr->sample_period = arm_spe_pmu__sample_period(arm_spe_pmu); } diff --git a/tools/perf/arch/x86/util/intel-pt.c b/tools/perf/arch/x86/util/intel-pt.c index 6d6cd8f9133c..fa0c718b9e72 100644 --- a/tools/perf/arch/x86/util/intel-pt.c +++ b/tools/perf/arch/x86/util/intel-pt.c @@ -60,7 +60,7 @@ struct intel_pt_recording { size_t priv_size; }; -static int intel_pt_parse_terms_with_default(struct perf_pmu *pmu, +static int intel_pt_parse_terms_with_default(const struct perf_pmu *pmu, const char *str, u64 *config) { @@ -84,7 +84,7 @@ out_free: return err; } -static int intel_pt_parse_terms(struct perf_pmu *pmu, const char *str, u64 *config) +static int intel_pt_parse_terms(const struct perf_pmu *pmu, const char *str, u64 *config) { *config = 0; return intel_pt_parse_terms_with_default(pmu, str, config); @@ -177,7 +177,7 @@ static int intel_pt_pick_bit(int bits, int target) return pick; } -static u64 intel_pt_default_config(struct perf_pmu *intel_pt_pmu) +static u64 intel_pt_default_config(const struct perf_pmu *intel_pt_pmu) { char buf[256]; int mtc, mtc_periods = 0, mtc_period; @@ -256,18 +256,17 @@ static int intel_pt_parse_snapshot_options(struct auxtrace_record *itr, return 0; } -struct perf_event_attr * -intel_pt_pmu_default_config(struct perf_pmu *intel_pt_pmu) +void intel_pt_pmu_default_config(const struct perf_pmu *intel_pt_pmu, + struct perf_event_attr *attr) { - struct perf_event_attr *attr; + static u64 config; + static bool initialized; - attr = zalloc(sizeof(struct perf_event_attr)); - if (!attr) - return NULL; - - attr->config = intel_pt_default_config(intel_pt_pmu); - - return attr; + if (!initialized) { + config = intel_pt_default_config(intel_pt_pmu); + initialized = true; + } + attr->config = config; } static const char *intel_pt_find_filter(struct evlist *evlist, diff --git a/tools/perf/arch/x86/util/pmu.c b/tools/perf/arch/x86/util/pmu.c index 949b3e2c67bd..469555ae9b3c 100644 --- a/tools/perf/arch/x86/util/pmu.c +++ b/tools/perf/arch/x86/util/pmu.c @@ -23,7 +23,7 @@ void perf_pmu__arch_init(struct perf_pmu *pmu __maybe_unused) if (!strcmp(pmu->name, INTEL_PT_PMU_NAME)) { pmu->auxtrace = true; pmu->selectable = true; - pmu->default_config = intel_pt_pmu_default_config(pmu); + pmu->perf_event_attr_init_default = intel_pt_pmu_default_config; } if (!strcmp(pmu->name, INTEL_BTS_PMU_NAME)) { pmu->auxtrace = true; diff --git a/tools/perf/util/arm-spe.h b/tools/perf/util/arm-spe.h index 98d3235781c3..4f4900c18f3e 100644 --- a/tools/perf/util/arm-spe.h +++ b/tools/perf/util/arm-spe.h @@ -27,5 +27,7 @@ struct auxtrace_record *arm_spe_recording_init(int *err, int arm_spe_process_auxtrace_info(union perf_event *event, struct perf_session *session); -struct perf_event_attr *arm_spe_pmu_default_config(struct perf_pmu *arm_spe_pmu); +void arm_spe_pmu_default_config(const struct perf_pmu *arm_spe_pmu, + struct perf_event_attr *attr); + #endif diff --git a/tools/perf/util/cs-etm.h b/tools/perf/util/cs-etm.h index 7cca37887917..4696267a32f0 100644 --- a/tools/perf/util/cs-etm.h +++ b/tools/perf/util/cs-etm.h @@ -242,7 +242,7 @@ struct cs_etm_packet_queue { int cs_etm__process_auxtrace_info(union perf_event *event, struct perf_session *session); -struct perf_event_attr *cs_etm_get_default_config(struct perf_pmu *pmu); +void cs_etm_get_default_config(const struct perf_pmu *pmu, struct perf_event_attr *attr); enum cs_etm_pid_fmt { CS_ETM_PIDFMT_NONE, diff --git a/tools/perf/util/intel-pt.h b/tools/perf/util/intel-pt.h index c7d6068e3a6b..18fd0be52e6c 100644 --- a/tools/perf/util/intel-pt.h +++ b/tools/perf/util/intel-pt.h @@ -42,6 +42,7 @@ struct auxtrace_record *intel_pt_recording_init(int *err); int intel_pt_process_auxtrace_info(union perf_event *event, struct perf_session *session); -struct perf_event_attr *intel_pt_pmu_default_config(struct perf_pmu *pmu); +void intel_pt_pmu_default_config(const struct perf_pmu *intel_pt_pmu, + struct perf_event_attr *attr); #endif diff --git a/tools/perf/util/parse-events.c b/tools/perf/util/parse-events.c index 23c027cf20ae..aa2f5c6fc7fc 100644 --- a/tools/perf/util/parse-events.c +++ b/tools/perf/util/parse-events.c @@ -1418,11 +1418,10 @@ int parse_events_add_pmu(struct parse_events_state *parse_state, } fix_raw(&parsed_terms, pmu); - if (pmu->default_config) { - memcpy(&attr, pmu->default_config, sizeof(struct perf_event_attr)); - } else { - memset(&attr, 0, sizeof(attr)); - } + memset(&attr, 0, sizeof(attr)); + if (pmu->perf_event_attr_init_default) + pmu->perf_event_attr_init_default(pmu, &attr); + attr.type = pmu->type; if (list_empty(&parsed_terms.terms)) { @@ -1466,7 +1465,8 @@ int parse_events_add_pmu(struct parse_events_state *parse_state, * When using default config, record which bits of attr->config were * changed by the user. */ - if (pmu->default_config && get_config_chgs(pmu, &parsed_terms, &config_terms)) { + if (pmu->perf_event_attr_init_default && + get_config_chgs(pmu, &parsed_terms, &config_terms)) { parse_events_terms__exit(&parsed_terms); return -ENOMEM; } diff --git a/tools/perf/util/pmu.c b/tools/perf/util/pmu.c index 8ef675ea7bdd..a967d25e899b 100644 --- a/tools/perf/util/pmu.c +++ b/tools/perf/util/pmu.c @@ -1402,7 +1402,7 @@ int perf_pmu__config(struct perf_pmu *pmu, struct perf_event_attr *attr, struct parse_events_terms *head_terms, struct parse_events_error *err) { - bool zero = !!pmu->default_config; + bool zero = !!pmu->perf_event_attr_init_default; return perf_pmu__config_terms(pmu, attr, head_terms, zero, err); } @@ -2064,7 +2064,6 @@ void perf_pmu__delete(struct perf_pmu *pmu) perf_cpu_map__put(pmu->cpus); - zfree(&pmu->default_config); zfree(&pmu->name); zfree(&pmu->alias_name); zfree(&pmu->id); diff --git a/tools/perf/util/pmu.h b/tools/perf/util/pmu.h index 5a05131aa4ce..d2895d415f08 100644 --- a/tools/perf/util/pmu.h +++ b/tools/perf/util/pmu.h @@ -92,10 +92,11 @@ struct perf_pmu { */ int max_precise; /** - * @default_config: Optional default perf_event_attr determined in - * architecture specific code. + * @perf_event_attr_init_default: Optional function to default + * initialize PMU specific parts of the perf_event_attr. */ - struct perf_event_attr *default_config; + void (*perf_event_attr_init_default)(const struct perf_pmu *pmu, + struct perf_event_attr *attr); /** * @cpus: Empty or the contents of either of: * /bus/event_source/devices//cpumask. From f8ccc2d5cc6516b019bcf8e361ae2a380cb36019 Mon Sep 17 00:00:00 2001 From: Leo Yan Date: Sat, 14 Oct 2023 15:41:58 +0800 Subject: [PATCH 250/475] perf cs-etm: Validate timestamp tracing in per-thread mode So far, it's impossible to validate timestamp trace in Arm CoreSight when the perf is in the per-thread mode. E.g. for the command: perf record -e cs_etm/timestamp/ --per-thread -- ls The command enables config 'timestamp' for 'cs_etm' event in the per-thread mode. In this case, the function cs_etm_validate_config() directly bails out and skips validation. Given profiled process can be scheduled on any CPUs in the per-thread mode, this patch validates timestamp tracing for all CPUs when detect the CPU map is empty. Signed-off-by: Leo Yan Reviewed-by: James Clark Acked-by: Suzuki K Poulose Cc: Will Deacon Cc: Mike Leach Cc: John Garry Cc: linux-arm-kernel@lists.infradead.org Cc: coresight@lists.linaro.org Link: https://lore.kernel.org/r/20231014074159.1667880-2-leo.yan@linaro.org Signed-off-by: Namhyung Kim --- tools/perf/arch/arm/util/cs-etm.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tools/perf/arch/arm/util/cs-etm.c b/tools/perf/arch/arm/util/cs-etm.c index 16bba74f048b..39ec6cae91ed 100644 --- a/tools/perf/arch/arm/util/cs-etm.c +++ b/tools/perf/arch/arm/util/cs-etm.c @@ -205,8 +205,17 @@ static int cs_etm_validate_config(struct auxtrace_record *itr, for (i = 0; i < cpu__max_cpu().cpu; i++) { struct perf_cpu cpu = { .cpu = i, }; - if (!perf_cpu_map__has(event_cpus, cpu) || - !perf_cpu_map__has(online_cpus, cpu)) + /* + * In per-cpu case, do the validation for CPUs to work with. + * In per-thread case, the CPU map is empty. Since the traced + * program can run on any CPUs in this case, thus don't skip + * validation. + */ + if (!perf_cpu_map__empty(event_cpus) && + !perf_cpu_map__has(event_cpus, cpu)) + continue; + + if (!perf_cpu_map__has(online_cpus, cpu)) continue; err = cs_etm_validate_context_id(itr, evsel, i); From 78efa7b411450a534e9d326cdf5578f681c73988 Mon Sep 17 00:00:00 2001 From: Leo Yan Date: Sat, 14 Oct 2023 15:41:59 +0800 Subject: [PATCH 251/475] perf cs-etm: Respect timestamp option When users pass the option '--timestamp' or '-T' in the record command, all events will set the PERF_SAMPLE_TIME bit in the attribution. In this case, the AUX event will record the kernel timestamp, but it doesn't mean Arm CoreSight enables timestamp packets in its hardware tracing. If the option '--timestamp' or '-T' is set, this patch always enables Arm CoreSight timestamp, as a result, the bit 28 in event's config is to be set. Before: # perf record -e cs_etm// --per-thread --timestamp -- ls # perf script --header-only ... # event : name = cs_etm//, , id = { 69 }, type = 12, size = 136, config = 0, { sample_period, sample_freq } = 1, sample_type = IP|TID|TIME|CPU|IDENTIFIER, read_format = ID|LOST, disabled = 1, enable_on_exec = 1, sample_id_all = 1, exclude_guest = 1 ... After: # perf record -e cs_etm// --per-thread --timestamp -- ls # perf script --header-only ... # event : name = cs_etm//, , id = { 49 }, type = 12, size = 136, config = 0x10000000, { sample_period, sample_freq } = 1, sample_type = IP|TID|TIME|CPU|IDENTIFIER, read_format = ID|LOST, disabled = 1, enable_on_exec = 1, sample_id_all = 1, exclude_guest = 1 ... Signed-off-by: Leo Yan Reviewed-by: James Clark Acked-by: Suzuki K Poulose Cc: Will Deacon Cc: Mike Leach Cc: Alexander Shishkin Cc: John Garry Cc: linux-arm-kernel@lists.infradead.org Cc: coresight@lists.linaro.org Link: https://lore.kernel.org/r/20231014074159.1667880-3-leo.yan@linaro.org Signed-off-by: Namhyung Kim --- tools/perf/arch/arm/util/cs-etm.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tools/perf/arch/arm/util/cs-etm.c b/tools/perf/arch/arm/util/cs-etm.c index 39ec6cae91ed..2cf873d71dff 100644 --- a/tools/perf/arch/arm/util/cs-etm.c +++ b/tools/perf/arch/arm/util/cs-etm.c @@ -442,6 +442,15 @@ static int cs_etm_recording_options(struct auxtrace_record *itr, "contextid", 1); } + /* + * When the option '--timestamp' or '-T' is enabled, the PERF_SAMPLE_TIME + * bit is set for all events. In this case, always enable Arm CoreSight + * timestamp tracing. + */ + if (opts->sample_time_set) + evsel__set_config_if_unset(cs_etm_pmu, cs_etm_evsel, + "timestamp", 1); + /* Add dummy event to keep tracking */ err = parse_event(evlist, "dummy:u"); if (err) From f6a66ff98ac1cfbfb83ec2445f5652ae065a1bf0 Mon Sep 17 00:00:00 2001 From: Athira Rajeev Date: Mon, 9 Oct 2023 10:30:52 +0530 Subject: [PATCH 252/475] tools/perf/arch/powerpc: Fix the CPU ID const char* value by adding 0x prefix Simple expression parser test fails in powerpc as below: 4: Simple expression parser test child forked, pid 170385 Using CPUID 004e2102 division by zero syntax error syntax error FAILED tests/expr.c:65 parse test failed test child finished with -1 Simple expression parser: FAILED! This is observed after commit: 'commit 9d5da30e4ae9 ("perf jevents: Add a new expression builtin strcmp_cpuid_str()")' With this commit, a new expression builtin strcmp_cpuid_str got added. This function takes an 'ID' type value, which is a string. So expression parse for strcmp_cpuid_str expects const char * as cpuid value type. In case of powerpc, CPU IDs are numbers. Hence it doesn't get interpreted correctly by bison parser. Example in case of power9, cpuid string returns as: 004e2102 cpuid of string type is expected in two cases: 1. char *get_cpuid_str(struct perf_pmu *pmu __maybe_unused); Testcase "tests/expr.c" uses "perf_pmu__getcpuid" which calls get_cpuid_str to get the cpuid string. 2. cpuid field in :struct pmu_events_map struct pmu_events_map { const char *arch; const char *cpuid; Here cpuid field is used in "perf_pmu__find_events_table" function as "strcmp_cpuid_str(map->cpuid, cpuid)". The value for cpuid field is picked from mapfile.csv. Fix the mapfile.csv and get_cpuid_str function to prefix cpuid with 0x so that it gets correctly interpreted by the bison parser Signed-off-by: Athira Rajeev Tested-by: Disha Goel Cc: kjain@linux.ibm.com Cc: maddy@linux.ibm.com Cc: disgoel@linux.vnet.ibm.com Cc: linuxppc-dev@lists.ozlabs.org Link: https://lore.kernel.org/r/20231009050052.64935-1-atrajeev@linux.vnet.ibm.com Signed-off-by: Namhyung Kim --- tools/perf/arch/powerpc/util/header.c | 2 +- tools/perf/pmu-events/arch/powerpc/mapfile.csv | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/perf/arch/powerpc/util/header.c b/tools/perf/arch/powerpc/util/header.c index c8d0dc775e5d..6b00efd53638 100644 --- a/tools/perf/arch/powerpc/util/header.c +++ b/tools/perf/arch/powerpc/util/header.c @@ -34,7 +34,7 @@ get_cpuid_str(struct perf_pmu *pmu __maybe_unused) { char *bufp; - if (asprintf(&bufp, "%.8lx", mfspr(SPRN_PVR)) < 0) + if (asprintf(&bufp, "0x%.8lx", mfspr(SPRN_PVR)) < 0) bufp = NULL; return bufp; diff --git a/tools/perf/pmu-events/arch/powerpc/mapfile.csv b/tools/perf/pmu-events/arch/powerpc/mapfile.csv index a534ff6db14b..f4908af7ad66 100644 --- a/tools/perf/pmu-events/arch/powerpc/mapfile.csv +++ b/tools/perf/pmu-events/arch/powerpc/mapfile.csv @@ -13,6 +13,6 @@ # # Power8 entries -004[bcd][[:xdigit:]]{4},1,power8,core -004e[[:xdigit:]]{4},1,power9,core -0080[[:xdigit:]]{4},1,power10,core +0x004[bcd][[:xdigit:]]{4},1,power8,core +0x004e[[:xdigit:]]{4},1,power9,core +0x0080[[:xdigit:]]{4},1,power10,core From 47f5693c4ce9b2bf2364303a531423e43278d3b6 Mon Sep 17 00:00:00 2001 From: Athira Rajeev Date: Fri, 13 Oct 2023 13:00:19 +0530 Subject: [PATCH 253/475] perf tests: Ignore shellcheck warning in lock_contention Running shellcheck on lock_contention.sh generates below warning In tests/shell/lock_contention.sh line 36: if [ `nproc` -lt 4 ]; then ^-----^ SC2046: Quote this to prevent word splitting. Here since nproc will generate a single word output and there is no possibility of word splitting, this warning can be ignored. Use exception for this with "disable" option in shellcheck. This warning is observed after commit: "commit 29441ab3a30a ("perf test lock_contention.sh: Skip test if not enough CPUs")" Fixes: 29441ab3a30a ("perf test lock_contention.sh: Skip test if not enough CPUs") Signed-off-by: Athira Rajeev Reviewed-by: Kajol Jain Acked-by: Namhyung Kim Cc: maddy@linux.ibm.com Cc: disgoel@linux.vnet.ibm.com Cc: linuxppc-dev@lists.ozlabs.org Link: https://lore.kernel.org/r/20231013073021.99794-2-atrajeev@linux.vnet.ibm.com Signed-off-by: Namhyung Kim --- tools/perf/tests/shell/lock_contention.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/perf/tests/shell/lock_contention.sh b/tools/perf/tests/shell/lock_contention.sh index d5a191d3d090..c1ec5762215b 100755 --- a/tools/perf/tests/shell/lock_contention.sh +++ b/tools/perf/tests/shell/lock_contention.sh @@ -33,6 +33,7 @@ check() { exit fi + # shellcheck disable=SC2046 if [ `nproc` -lt 4 ]; then echo "[Skip] Low number of CPUs (`nproc`), lock event cannot be triggered certainly" err=2 From eff65ee26ed73f3ca635bac13c386a4538a608d8 Mon Sep 17 00:00:00 2001 From: Athira Rajeev Date: Fri, 13 Oct 2023 13:00:20 +0530 Subject: [PATCH 254/475] perf tests: Fix shellcheck warning in record_sideband.sh Running shellcheck on record_sideband.sh throws below warning: In tests/shell/record_sideband.sh line 25: if ! perf record -o ${perfdata} -BN --no-bpf-event -C $1 true 2>&1 >/dev/null ^--^ SC2069: To redirect stdout+stderr, 2>&1 must be last (or use '{ cmd > file; } 2>&1' to clarify). This shows shellcheck warning SC2069 where the redirection order needs to be fixed. Use "cmd > /dev/null 2>&1" to fix the redirection of perf record output Fixes: 23b97c7ee963 ("perf test: Add test case for record sideband events") Signed-off-by: Athira Rajeev Reviewed-by: Kajol Jain Acked-by: Namhyung Kim Cc: disgoel@linux.vnet.ibm.com Link: https://lore.kernel.org/r/20231013073021.99794-3-atrajeev@linux.vnet.ibm.com Signed-off-by: Namhyung Kim --- tools/perf/tests/shell/record_sideband.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/perf/tests/shell/record_sideband.sh b/tools/perf/tests/shell/record_sideband.sh index 5024a7ce0c51..ac70ac27d590 100755 --- a/tools/perf/tests/shell/record_sideband.sh +++ b/tools/perf/tests/shell/record_sideband.sh @@ -22,7 +22,7 @@ trap trap_cleanup EXIT TERM INT can_cpu_wide() { - if ! perf record -o ${perfdata} -BN --no-bpf-event -C $1 true 2>&1 >/dev/null + if ! perf record -o ${perfdata} -BN --no-bpf-event -C $1 true > /dev/null 2>&1 then echo "record sideband test [Skipped cannot record cpu$1]" err=2 From a20fca2c5db19a09260827e526f37397ba698ce0 Mon Sep 17 00:00:00 2001 From: Athira Rajeev Date: Fri, 13 Oct 2023 13:00:21 +0530 Subject: [PATCH 255/475] perf tests: Fix shellcheck warning in stat_all_metricgroups Running shellcheck on stat_all_metricgroups.sh reports below warning: In ./tests/shell/stat_all_metricgroups.sh line 7: function ParanoidAndNotRoot() ^-- SC2112: 'function' keyword is non-standard. Delete it. As per the format, "function" is a non-standard keyword that can be used to declare functions. Fix this by removing the "function" keyword from ParanoidAndNotRoot function Signed-off-by: Athira Rajeev Acked-by: Namhyung Kim Cc: kjain@linux.ibm.com Cc: maddy@linux.ibm.com Cc: disgoel@linux.vnet.ibm.com Cc: linuxppc-dev@lists.ozlabs.org Link: https://lore.kernel.org/r/20231013073021.99794-4-atrajeev@linux.vnet.ibm.com Signed-off-by: Namhyung Kim --- tools/perf/tests/shell/stat_all_metricgroups.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/perf/tests/shell/stat_all_metricgroups.sh b/tools/perf/tests/shell/stat_all_metricgroups.sh index f3e305649e2c..55ef9c9ded2d 100755 --- a/tools/perf/tests/shell/stat_all_metricgroups.sh +++ b/tools/perf/tests/shell/stat_all_metricgroups.sh @@ -4,7 +4,7 @@ set -e -function ParanoidAndNotRoot() +ParanoidAndNotRoot() { [ "$(id -u)" != 0 ] && [ "$(cat /proc/sys/kernel/perf_event_paranoid)" -gt $1 ] } From 9a13ee457a6e7a850ac1d145b0731b1d729b8f42 Mon Sep 17 00:00:00 2001 From: Changbin Du Date: Tue, 17 Oct 2023 09:55:24 +0800 Subject: [PATCH 256/475] perf: script: fix missing ',' for fields option A comma is missed at the end of line. Signed-off-by: Changbin Du Link: https://lore.kernel.org/r/20231017015524.797065-1-changbin.du@huawei.com Signed-off-by: Namhyung Kim --- tools/perf/builtin-script.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/perf/builtin-script.c b/tools/perf/builtin-script.c index 517bf25750c8..b1f57401ff23 100644 --- a/tools/perf/builtin-script.c +++ b/tools/perf/builtin-script.c @@ -3899,7 +3899,7 @@ int cmd_script(int argc, const char **argv) "comma separated output fields prepend with 'type:'. " "+field to add and -field to remove." "Valid types: hw,sw,trace,raw,synth. " - "Fields: comm,tid,pid,time,cpu,event,trace,ip,sym,dso,dsoff" + "Fields: comm,tid,pid,time,cpu,event,trace,ip,sym,dso,dsoff," "addr,symoff,srcline,period,iregs,uregs,brstack," "brstacksym,flags,data_src,weight,bpf-output,brstackinsn," "brstackinsnlen,brstackoff,callindent,insn,insnlen,synth," From 4b47b0fa4b15e0de916e7dd93cd787fdab208ff2 Mon Sep 17 00:00:00 2001 From: Muhammad Muzammil Date: Fri, 13 Oct 2023 10:31:18 +0500 Subject: [PATCH 257/475] powerpc/bpf: Fixed 'instead' typo in bpf_jit_build_body() Fixed 'instead' typo. Signed-off-by: Muhammad Muzammil Signed-off-by: Michael Ellerman Link: https://msgid.link/20231013053118.11221-1-m.muzzammilashraf@gmail.com --- arch/powerpc/net/bpf_jit_comp32.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/powerpc/net/bpf_jit_comp32.c b/arch/powerpc/net/bpf_jit_comp32.c index 7f91ea064c08..bc7f92ec7f2d 100644 --- a/arch/powerpc/net/bpf_jit_comp32.c +++ b/arch/powerpc/net/bpf_jit_comp32.c @@ -940,7 +940,7 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context * * !fp->aux->verifier_zext. Emit NOP otherwise. * * Note that "li reg_h,0" is emitted for BPF_B/H/W case, - * if necessary. So, jump there insted of emitting an + * if necessary. So, jump there instead of emitting an * additional "li reg_h,0" instruction. */ if (size == BPF_DW && !fp->aux->verifier_zext) From b8a555dc31e5aa18d976de0bc228006e398a2e7d Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Wed, 18 Oct 2023 11:10:31 -0700 Subject: [PATCH 258/475] eventfs: Use ERR_CAST() in eventfs_create_events_dir() When building with clang and CONFIG_RANDSTRUCT_FULL=y, there is an error due to a cast in eventfs_create_events_dir(): fs/tracefs/event_inode.c:734:10: error: casting from randomized structure pointer type 'struct dentry *' to 'struct eventfs_inode *' 734 | return (struct eventfs_inode *)dentry; | ^ 1 error generated. Use the ERR_CAST() function to resolve the error, as it was designed for this exact situation (casting an error pointer to another type). Link: https://lore.kernel.org/linux-trace-kernel/20231018-ftrace-fix-clang-randstruct-v1-1-338cb214abfb@kernel.org Closes: https://github.com/ClangBuiltLinux/linux/issues/1947 Fixes: 5790b1fb3d67 ("eventfs: Remove eventfs_file and just use eventfs_inode") Reviewed-by: Kees Cook Signed-off-by: Nathan Chancellor Signed-off-by: Steven Rostedt (Google) --- fs/tracefs/event_inode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/tracefs/event_inode.c b/fs/tracefs/event_inode.c index 1ccd100bc565..9f19b6608954 100644 --- a/fs/tracefs/event_inode.c +++ b/fs/tracefs/event_inode.c @@ -731,7 +731,7 @@ struct eventfs_inode *eventfs_create_events_dir(const char *name, struct dentry return NULL; if (IS_ERR(dentry)) - return (struct eventfs_inode *)dentry; + return ERR_CAST(dentry); ei = kzalloc(sizeof(*ei), GFP_KERNEL); if (!ei) From d9997f7ffb137447aa2f820c26cb1e6f5890d978 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Sun, 8 Oct 2023 23:22:50 +0200 Subject: [PATCH 259/475] tools/build: Fix -s detection code in tools/build/Makefile.build As Dmitry described in [1] changelog the current way of detecting -s option is broken for new make. Changing the tools/build -s option detection the same way as it was fixed for root Makefile in [1]. [1] 4bf73588165b ("kbuild: Port silent mode detection to future gnu make.") Cc: Dmitry Goncharov Signed-off-by: Jiri Olsa Tested-by: Arnaldo Carvalho de Melo Acked-by: Namhyung Kim Cc: Peter Zijlstra Cc: KP Singh Cc: Martin KaFai Lau Cc: Song Liu Cc: Yonghong Song Cc: John Fastabend Cc: Hao Luo Cc: Ian Rogers Cc: Stanislav Fomichev Cc: Daniel Borkmann Cc: Quentin Monnet Cc: Arnaldo Carvalho de Melo Cc: Andrii Nakryiko Cc: Alexei Starovoitov Cc: Ingo Molnar Cc: Alexander Shishkin Cc: bpf@vger.kernel.org Cc: linux-perf-users@vger.kernel.org Link: https://lore.kernel.org/r/20231008212251.236023-2-jolsa@kernel.org Signed-off-by: Namhyung Kim --- tools/build/Makefile.build | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tools/build/Makefile.build b/tools/build/Makefile.build index fac42486a8cf..5fb3fb3d97e0 100644 --- a/tools/build/Makefile.build +++ b/tools/build/Makefile.build @@ -20,7 +20,15 @@ else Q=@ endif -ifneq ($(findstring s,$(filter-out --%,$(MAKEFLAGS))),) +# If the user is running make -s (silent mode), suppress echoing of commands +# make-4.0 (and later) keep single letter options in the 1st word of MAKEFLAGS. +ifeq ($(filter 3.%,$(MAKE_VERSION)),) +short-opts := $(firstword -$(MAKEFLAGS)) +else +short-opts := $(filter-out --%,$(MAKEFLAGS)) +endif + +ifneq ($(findstring s,$(short-opts)),) quiet=silent_ endif From b5c532e90478e134b66b067c2b0487526ac4161e Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Sun, 8 Oct 2023 23:22:51 +0200 Subject: [PATCH 260/475] tools/build: Fix -s detection code in tools/scripts/Makefile.include As Dmitry described in [1] changelog the current way of detecting -s option is broken for new make. Changing the tools/build -s option detection the same way as it was fixed for root Makefile in [1]. [1] 4bf73588165b ("kbuild: Port silent mode detection to future gnu make.") Cc: Dmitry Goncharov Signed-off-by: Jiri Olsa Tested-by: Arnaldo Carvalho de Melo Acked-by: Namhyung Kim Cc: Peter Zijlstra Cc: KP Singh Cc: Martin KaFai Lau Cc: Song Liu Cc: Yonghong Song Cc: John Fastabend Cc: Hao Luo Cc: Ian Rogers Cc: Stanislav Fomichev Cc: Daniel Borkmann Cc: Quentin Monnet Cc: Arnaldo Carvalho de Melo Cc: Andrii Nakryiko Cc: Alexei Starovoitov Cc: Ingo Molnar Cc: Alexander Shishkin Cc: bpf@vger.kernel.org Cc: linux-perf-users@vger.kernel.org Link: https://lore.kernel.org/r/20231008212251.236023-3-jolsa@kernel.org Signed-off-by: Namhyung Kim --- tools/scripts/Makefile.include | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tools/scripts/Makefile.include b/tools/scripts/Makefile.include index ff527ac065cf..6fba29f3222d 100644 --- a/tools/scripts/Makefile.include +++ b/tools/scripts/Makefile.include @@ -136,7 +136,15 @@ else NO_SUBDIR = : endif -ifneq ($(findstring s,$(filter-out --%,$(MAKEFLAGS))),) +# If the user is running make -s (silent mode), suppress echoing of commands +# make-4.0 (and later) keep single letter options in the 1st word of MAKEFLAGS. +ifeq ($(filter 3.%,$(MAKE_VERSION)),) +short-opts := $(firstword -$(MAKEFLAGS)) +else +short-opts := $(filter-out --%,$(MAKEFLAGS)) +endif + +ifneq ($(findstring s,$(short-opts)),) silent=1 endif From 79ddd4a7c5fa8883f99a88409e5ad9812e484094 Mon Sep 17 00:00:00 2001 From: John Johansen Date: Mon, 6 Mar 2023 07:27:55 -0800 Subject: [PATCH 261/475] apparmor: rename SK_CTX() to aa_sock and make it an inline fn In preparation for LSM stacking rework the macro to an inline fn Reviewed-by: Georgia Garcia Signed-off-by: John Johansen --- security/apparmor/include/net.h | 5 +++++ security/apparmor/lsm.c | 22 +++++++++++----------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/security/apparmor/include/net.h b/security/apparmor/include/net.h index 64a0112bf62d..e29f011be3d7 100644 --- a/security/apparmor/include/net.h +++ b/security/apparmor/include/net.h @@ -52,6 +52,11 @@ struct aa_sk_ctx { }; #define SK_CTX(X) ((X)->sk_security) +static inline struct aa_sk_ctx *aa_sock(const struct sock *sk) +{ + return sk->sk_security; +} + #define DEFINE_AUDIT_NET(NAME, OP, SK, F, T, P) \ struct lsm_network_audit NAME ## _net = { .sk = (SK), \ .family = (F)}; \ diff --git a/security/apparmor/lsm.c b/security/apparmor/lsm.c index 5289dfe60bc1..63c6f21fb5ea 100644 --- a/security/apparmor/lsm.c +++ b/security/apparmor/lsm.c @@ -829,7 +829,7 @@ static int apparmor_sk_alloc_security(struct sock *sk, int family, gfp_t flags) if (!ctx) return -ENOMEM; - SK_CTX(sk) = ctx; + sk->sk_security = ctx; return 0; } @@ -839,9 +839,9 @@ static int apparmor_sk_alloc_security(struct sock *sk, int family, gfp_t flags) */ static void apparmor_sk_free_security(struct sock *sk) { - struct aa_sk_ctx *ctx = SK_CTX(sk); + struct aa_sk_ctx *ctx = aa_sock(sk); - SK_CTX(sk) = NULL; + sk->sk_security = NULL; aa_put_label(ctx->label); aa_put_label(ctx->peer); kfree(ctx); @@ -853,8 +853,8 @@ static void apparmor_sk_free_security(struct sock *sk) static void apparmor_sk_clone_security(const struct sock *sk, struct sock *newsk) { - struct aa_sk_ctx *ctx = SK_CTX(sk); - struct aa_sk_ctx *new = SK_CTX(newsk); + struct aa_sk_ctx *ctx = aa_sock(sk); + struct aa_sk_ctx *new = aa_sock(newsk); if (new->label) aa_put_label(new->label); @@ -907,7 +907,7 @@ static int apparmor_socket_post_create(struct socket *sock, int family, label = aa_get_current_label(); if (sock->sk) { - struct aa_sk_ctx *ctx = SK_CTX(sock->sk); + struct aa_sk_ctx *ctx = aa_sock(sock->sk); aa_put_label(ctx->label); ctx->label = aa_get_label(label); @@ -1092,7 +1092,7 @@ static int apparmor_socket_shutdown(struct socket *sock, int how) */ static int apparmor_socket_sock_rcv_skb(struct sock *sk, struct sk_buff *skb) { - struct aa_sk_ctx *ctx = SK_CTX(sk); + struct aa_sk_ctx *ctx = aa_sock(sk); if (!skb->secmark) return 0; @@ -1105,7 +1105,7 @@ static int apparmor_socket_sock_rcv_skb(struct sock *sk, struct sk_buff *skb) static struct aa_label *sk_peer_label(struct sock *sk) { - struct aa_sk_ctx *ctx = SK_CTX(sk); + struct aa_sk_ctx *ctx = aa_sock(sk); if (ctx->peer) return ctx->peer; @@ -1186,7 +1186,7 @@ static int apparmor_socket_getpeersec_dgram(struct socket *sock, */ static void apparmor_sock_graft(struct sock *sk, struct socket *parent) { - struct aa_sk_ctx *ctx = SK_CTX(sk); + struct aa_sk_ctx *ctx = aa_sock(sk); if (!ctx->label) ctx->label = aa_get_current_label(); @@ -1196,7 +1196,7 @@ static void apparmor_sock_graft(struct sock *sk, struct socket *parent) static int apparmor_inet_conn_request(const struct sock *sk, struct sk_buff *skb, struct request_sock *req) { - struct aa_sk_ctx *ctx = SK_CTX(sk); + struct aa_sk_ctx *ctx = aa_sock(sk); if (!skb->secmark) return 0; @@ -1809,7 +1809,7 @@ static unsigned int apparmor_ip_postroute(void *priv, if (sk == NULL) return NF_ACCEPT; - ctx = SK_CTX(sk); + ctx = aa_sock(sk); if (!apparmor_secmark_check(ctx->label, OP_SENDMSG, AA_MAY_SEND, skb->secmark, sk)) return NF_ACCEPT; From bd7bd201ca46c211c3ab251ca9854787d1331a2f Mon Sep 17 00:00:00 2001 From: John Johansen Date: Wed, 14 Sep 2022 00:20:12 -0700 Subject: [PATCH 262/475] apparmor: combine common_audit_data and apparmor_audit_data Everywhere where common_audit_data is used apparmor audit_data is also used. We can simplify the code and drop the use of the aad macro everywhere by combining the two structures. Reviewed-by: Georgia Garcia Signed-off-by: John Johansen --- security/apparmor/audit.c | 70 +++++++++++++++---------------- security/apparmor/capability.c | 24 +++++------ security/apparmor/file.c | 64 ++++++++++++++-------------- security/apparmor/include/audit.h | 32 +++++++------- security/apparmor/include/net.h | 13 +++--- security/apparmor/include/perms.h | 4 +- security/apparmor/ipc.c | 39 ++++++++--------- security/apparmor/lib.c | 47 +++++++++++---------- security/apparmor/lsm.c | 12 +++--- security/apparmor/mount.c | 41 +++++++++--------- security/apparmor/net.c | 44 +++++++++---------- security/apparmor/policy.c | 19 +++++---- security/apparmor/policy_unpack.c | 29 ++++++------- security/apparmor/resource.c | 23 +++++----- security/apparmor/task.c | 35 ++++++++-------- 15 files changed, 254 insertions(+), 242 deletions(-) diff --git a/security/apparmor/audit.c b/security/apparmor/audit.c index a3db0f8bd4f8..06ad6a8fcce1 100644 --- a/security/apparmor/audit.c +++ b/security/apparmor/audit.c @@ -85,37 +85,36 @@ static const char *const aa_class_names[] = { /** * audit_pre() - core AppArmor function. * @ab: audit buffer to fill (NOT NULL) - * @ca: audit structure containing data to audit (NOT NULL) + * @va: audit structure containing data to audit (NOT NULL) * - * Record common AppArmor audit data from @sa + * Record common AppArmor audit data from @va */ -static void audit_pre(struct audit_buffer *ab, void *ca) +static void audit_pre(struct audit_buffer *ab, void *va) { - struct common_audit_data *sa = ca; + struct apparmor_audit_data *ad = aad_of_va(va); if (aa_g_audit_header) { audit_log_format(ab, "apparmor=\"%s\"", - aa_audit_type[aad(sa)->type]); + aa_audit_type[ad->type]); } - if (aad(sa)->op) { - audit_log_format(ab, " operation=\"%s\"", aad(sa)->op); - } + if (ad->op) + audit_log_format(ab, " operation=\"%s\"", ad->op); - if (aad(sa)->class) + if (ad->class) audit_log_format(ab, " class=\"%s\"", - aad(sa)->class <= AA_CLASS_LAST ? - aa_class_names[aad(sa)->class] : + ad->class <= AA_CLASS_LAST ? + aa_class_names[ad->class] : "unknown"); - if (aad(sa)->info) { - audit_log_format(ab, " info=\"%s\"", aad(sa)->info); - if (aad(sa)->error) - audit_log_format(ab, " error=%d", aad(sa)->error); + if (ad->info) { + audit_log_format(ab, " info=\"%s\"", ad->info); + if (ad->error) + audit_log_format(ab, " error=%d", ad->error); } - if (aad(sa)->label) { - struct aa_label *label = aad(sa)->label; + if (ad->label) { + struct aa_label *label = ad->label; if (label_isprofile(label)) { struct aa_profile *profile = labels_profile(label); @@ -134,43 +133,44 @@ static void audit_pre(struct audit_buffer *ab, void *ca) } } - if (aad(sa)->name) { + if (ad->name) { audit_log_format(ab, " name="); - audit_log_untrustedstring(ab, aad(sa)->name); + audit_log_untrustedstring(ab, ad->name); } } /** * aa_audit_msg - Log a message to the audit subsystem * @type: audit type for the message - * @sa: audit event structure (NOT NULL) + * @ad: audit event structure (NOT NULL) * @cb: optional callback fn for type specific fields (MAYBE NULL) */ -void aa_audit_msg(int type, struct common_audit_data *sa, +void aa_audit_msg(int type, struct apparmor_audit_data *ad, void (*cb) (struct audit_buffer *, void *)) { - aad(sa)->type = type; - common_lsm_audit(sa, audit_pre, cb); + ad->type = type; + common_lsm_audit(&ad->common, audit_pre, cb); } /** * aa_audit - Log a profile based audit event to the audit subsystem * @type: audit type for the message * @profile: profile to check against (NOT NULL) - * @sa: audit event (NOT NULL) + * @ad: audit event (NOT NULL) * @cb: optional callback fn for type specific fields (MAYBE NULL) * * Handle default message switching based off of audit mode flags * * Returns: error on failure */ -int aa_audit(int type, struct aa_profile *profile, struct common_audit_data *sa, +int aa_audit(int type, struct aa_profile *profile, + struct apparmor_audit_data *ad, void (*cb) (struct audit_buffer *, void *)) { AA_BUG(!profile); if (type == AUDIT_APPARMOR_AUTO) { - if (likely(!aad(sa)->error)) { + if (likely(!ad->error)) { if (AUDIT_MODE(profile) != AUDIT_ALL) return 0; type = AUDIT_APPARMOR_AUDIT; @@ -182,24 +182,24 @@ int aa_audit(int type, struct aa_profile *profile, struct common_audit_data *sa, if (AUDIT_MODE(profile) == AUDIT_QUIET || (type == AUDIT_APPARMOR_DENIED && AUDIT_MODE(profile) == AUDIT_QUIET_DENIED)) - return aad(sa)->error; + return ad->error; if (KILL_MODE(profile) && type == AUDIT_APPARMOR_DENIED) type = AUDIT_APPARMOR_KILL; - aad(sa)->label = &profile->label; + ad->label = &profile->label; - aa_audit_msg(type, sa, cb); + aa_audit_msg(type, ad, cb); - if (aad(sa)->type == AUDIT_APPARMOR_KILL) + if (ad->type == AUDIT_APPARMOR_KILL) (void)send_sig_info(SIGKILL, NULL, - sa->type == LSM_AUDIT_DATA_TASK && sa->u.tsk ? - sa->u.tsk : current); + ad->common.type == LSM_AUDIT_DATA_TASK && + ad->common.u.tsk ? ad->common.u.tsk : current); - if (aad(sa)->type == AUDIT_APPARMOR_ALLOWED) - return complain_error(aad(sa)->error); + if (ad->type == AUDIT_APPARMOR_ALLOWED) + return complain_error(ad->error); - return aad(sa)->error; + return ad->error; } struct aa_audit_rule { diff --git a/security/apparmor/capability.c b/security/apparmor/capability.c index 8ebf0fcb796c..35979a832994 100644 --- a/security/apparmor/capability.c +++ b/security/apparmor/capability.c @@ -51,7 +51,7 @@ static void audit_cb(struct audit_buffer *ab, void *va) /** * audit_caps - audit a capability - * @sa: audit data + * @as: audit data * @profile: profile being tested for confinement (NOT NULL) * @cap: capability tested * @error: error code returned by test @@ -59,9 +59,9 @@ static void audit_cb(struct audit_buffer *ab, void *va) * Do auditing of capability and handle, audit/complain/kill modes switching * and duplicate message elimination. * - * Returns: 0 or sa->error on success, error code on failure + * Returns: 0 or ad->error on success, error code on failure */ -static int audit_caps(struct common_audit_data *sa, struct aa_profile *profile, +static int audit_caps(struct apparmor_audit_data *ad, struct aa_profile *profile, int cap, int error) { struct aa_ruleset *rules = list_first_entry(&profile->rules, @@ -69,7 +69,7 @@ static int audit_caps(struct common_audit_data *sa, struct aa_profile *profile, struct audit_cache *ent; int type = AUDIT_APPARMOR_AUTO; - aad(sa)->error = error; + ad->error = error; if (likely(!error)) { /* test if auditing is being forced */ @@ -101,7 +101,7 @@ static int audit_caps(struct common_audit_data *sa, struct aa_profile *profile, } put_cpu_var(audit_cache); - return aa_audit(type, profile, sa, audit_cb); + return aa_audit(type, profile, ad, audit_cb); } /** @@ -109,12 +109,12 @@ static int audit_caps(struct common_audit_data *sa, struct aa_profile *profile, * @profile: profile being enforced (NOT NULL, NOT unconfined) * @cap: capability to test if allowed * @opts: CAP_OPT_NOAUDIT bit determines whether audit record is generated - * @sa: audit data (MAY BE NULL indicating no auditing) + * @ad: audit data (MAY BE NULL indicating no auditing) * * Returns: 0 if allowed else -EPERM */ static int profile_capable(struct aa_profile *profile, int cap, - unsigned int opts, struct common_audit_data *sa) + unsigned int opts, struct apparmor_audit_data *ad) { struct aa_ruleset *rules = list_first_entry(&profile->rules, typeof(*rules), list); @@ -132,10 +132,10 @@ static int profile_capable(struct aa_profile *profile, int cap, /* audit the cap request in complain mode but note that it * should be optional. */ - aad(sa)->info = "optional: no audit"; + ad->info = "optional: no audit"; } - return audit_caps(sa, profile, cap, error); + return audit_caps(ad, profile, cap, error); } /** @@ -152,11 +152,11 @@ int aa_capable(struct aa_label *label, int cap, unsigned int opts) { struct aa_profile *profile; int error = 0; - DEFINE_AUDIT_DATA(sa, LSM_AUDIT_DATA_CAP, AA_CLASS_CAP, OP_CAPABLE); + DEFINE_AUDIT_DATA(ad, LSM_AUDIT_DATA_CAP, AA_CLASS_CAP, OP_CAPABLE); - sa.u.cap = cap; + ad.common.u.cap = cap; error = fn_for_each_confined(label, profile, - profile_capable(profile, cap, opts, &sa)); + profile_capable(profile, cap, opts, &ad)); return error; } diff --git a/security/apparmor/file.c b/security/apparmor/file.c index c0fc5e77e889..01ff612c060e 100644 --- a/security/apparmor/file.c +++ b/security/apparmor/file.c @@ -44,33 +44,34 @@ static u32 map_mask_to_chr_mask(u32 mask) static void file_audit_cb(struct audit_buffer *ab, void *va) { struct common_audit_data *sa = va; + struct apparmor_audit_data *ad = aad(sa); kuid_t fsuid = current_fsuid(); char str[10]; - if (aad(sa)->request & AA_AUDIT_FILE_MASK) { + if (ad->request & AA_AUDIT_FILE_MASK) { aa_perm_mask_to_str(str, sizeof(str), aa_file_perm_chrs, - map_mask_to_chr_mask(aad(sa)->request)); + map_mask_to_chr_mask(ad->request)); audit_log_format(ab, " requested_mask=\"%s\"", str); } - if (aad(sa)->denied & AA_AUDIT_FILE_MASK) { + if (ad->denied & AA_AUDIT_FILE_MASK) { aa_perm_mask_to_str(str, sizeof(str), aa_file_perm_chrs, - map_mask_to_chr_mask(aad(sa)->denied)); + map_mask_to_chr_mask(ad->denied)); audit_log_format(ab, " denied_mask=\"%s\"", str); } - if (aad(sa)->request & AA_AUDIT_FILE_MASK) { + if (ad->request & AA_AUDIT_FILE_MASK) { audit_log_format(ab, " fsuid=%d", from_kuid(&init_user_ns, fsuid)); audit_log_format(ab, " ouid=%d", - from_kuid(&init_user_ns, aad(sa)->fs.ouid)); + from_kuid(&init_user_ns, ad->fs.ouid)); } - if (aad(sa)->peer) { + if (ad->peer) { audit_log_format(ab, " target="); - aa_label_xaudit(ab, labels_ns(aad(sa)->label), aad(sa)->peer, + aa_label_xaudit(ab, labels_ns(ad->label), ad->peer, FLAG_VIEW_SUBNS, GFP_KERNEL); - } else if (aad(sa)->fs.target) { + } else if (ad->fs.target) { audit_log_format(ab, " target="); - audit_log_untrustedstring(ab, aad(sa)->fs.target); + audit_log_untrustedstring(ab, ad->fs.target); } } @@ -95,50 +96,49 @@ int aa_audit_file(struct aa_profile *profile, struct aa_perms *perms, kuid_t ouid, const char *info, int error) { int type = AUDIT_APPARMOR_AUTO; - DEFINE_AUDIT_DATA(sa, LSM_AUDIT_DATA_TASK, AA_CLASS_FILE, op); + DEFINE_AUDIT_DATA(ad, LSM_AUDIT_DATA_TASK, AA_CLASS_FILE, op); - sa.u.tsk = NULL; - aad(&sa)->request = request; - aad(&sa)->name = name; - aad(&sa)->fs.target = target; - aad(&sa)->peer = tlabel; - aad(&sa)->fs.ouid = ouid; - aad(&sa)->info = info; - aad(&sa)->error = error; - sa.u.tsk = NULL; + ad.request = request; + ad.name = name; + ad.fs.target = target; + ad.peer = tlabel; + ad.fs.ouid = ouid; + ad.info = info; + ad.error = error; + ad.common.u.tsk = NULL; - if (likely(!aad(&sa)->error)) { + if (likely(!ad.error)) { u32 mask = perms->audit; if (unlikely(AUDIT_MODE(profile) == AUDIT_ALL)) mask = 0xffff; /* mask off perms that are not being force audited */ - aad(&sa)->request &= mask; + ad.request &= mask; - if (likely(!aad(&sa)->request)) + if (likely(!ad.request)) return 0; type = AUDIT_APPARMOR_AUDIT; } else { /* only report permissions that were denied */ - aad(&sa)->request = aad(&sa)->request & ~perms->allow; - AA_BUG(!aad(&sa)->request); + ad.request = ad.request & ~perms->allow; + AA_BUG(!ad.request); - if (aad(&sa)->request & perms->kill) + if (ad.request & perms->kill) type = AUDIT_APPARMOR_KILL; /* quiet known rejects, assumes quiet and kill do not overlap */ - if ((aad(&sa)->request & perms->quiet) && + if ((ad.request & perms->quiet) && AUDIT_MODE(profile) != AUDIT_NOQUIET && AUDIT_MODE(profile) != AUDIT_ALL) - aad(&sa)->request &= ~perms->quiet; + ad.request &= ~perms->quiet; - if (!aad(&sa)->request) - return aad(&sa)->error; + if (!ad.request) + return ad.error; } - aad(&sa)->denied = aad(&sa)->request & ~perms->allow; - return aa_audit(type, profile, &sa, file_audit_cb); + ad.denied = ad.request & ~perms->allow; + return aa_audit(type, profile, &ad, file_audit_cb); } static int path_name(const char *op, struct aa_label *label, diff --git a/security/apparmor/include/audit.h b/security/apparmor/include/audit.h index c328f07f11cd..85931ec94e91 100644 --- a/security/apparmor/include/audit.h +++ b/security/apparmor/include/audit.h @@ -152,33 +152,35 @@ struct apparmor_audit_data { unsigned long flags; } mnt; }; + + struct common_audit_data common; }; /* macros for dealing with apparmor_audit_data structure */ -#define aad(SA) ((SA)->apparmor_audit_data) +#define aad(SA) (container_of(SA, struct apparmor_audit_data, common)) +#define aad_of_va(VA) aad((struct common_audit_data *)(VA)) + #define DEFINE_AUDIT_DATA(NAME, T, C, X) \ /* TODO: cleanup audit init so we don't need _aad = {0,} */ \ - struct apparmor_audit_data NAME ## _aad = { \ + struct apparmor_audit_data NAME = { \ .class = (C), \ .op = (X), \ - }; \ - struct common_audit_data NAME = \ - { \ - .type = (T), \ - .u.tsk = NULL, \ - }; \ - NAME.apparmor_audit_data = &(NAME ## _aad) + .common.type = (T), \ + .common.u.tsk = NULL, \ + .common.apparmor_audit_data = &NAME, \ + }; -void aa_audit_msg(int type, struct common_audit_data *sa, +void aa_audit_msg(int type, struct apparmor_audit_data *ad, void (*cb) (struct audit_buffer *, void *)); -int aa_audit(int type, struct aa_profile *profile, struct common_audit_data *sa, +int aa_audit(int type, struct aa_profile *profile, + struct apparmor_audit_data *ad, void (*cb) (struct audit_buffer *, void *)); -#define aa_audit_error(ERROR, SA, CB) \ +#define aa_audit_error(ERROR, AD, CB) \ ({ \ - aad((SA))->error = (ERROR); \ - aa_audit_msg(AUDIT_APPARMOR_ERROR, (SA), (CB)); \ - aad((SA))->error; \ + (AD)->error = (ERROR); \ + aa_audit_msg(AUDIT_APPARMOR_ERROR, (AD), (CB)); \ + (AD)->error; \ }) diff --git a/security/apparmor/include/net.h b/security/apparmor/include/net.h index e29f011be3d7..e0068a3a4efd 100644 --- a/security/apparmor/include/net.h +++ b/security/apparmor/include/net.h @@ -65,9 +65,9 @@ static inline struct aa_sk_ctx *aa_sock(const struct sock *sk) LSM_AUDIT_DATA_NONE, \ AA_CLASS_NET, \ OP); \ - NAME.u.net = &(NAME ## _net); \ - aad(&NAME)->net.type = (T); \ - aad(&NAME)->net.protocol = (P) + NAME.common.u.net = &(NAME ## _net); \ + NAME.net.type = (T); \ + NAME.net.protocol = (P) #define DEFINE_AUDIT_SK(NAME, OP, SK) \ DEFINE_AUDIT_NET(NAME, OP, SK, (SK)->sk_family, (SK)->sk_type, \ @@ -94,16 +94,17 @@ struct aa_secmark { extern struct aa_sfs_entry aa_sfs_entry_network[]; void audit_net_cb(struct audit_buffer *ab, void *va); -int aa_profile_af_perm(struct aa_profile *profile, struct common_audit_data *sa, +int aa_profile_af_perm(struct aa_profile *profile, + struct apparmor_audit_data *ad, u32 request, u16 family, int type); int aa_af_perm(struct aa_label *label, const char *op, u32 request, u16 family, int type, int protocol); static inline int aa_profile_af_sk_perm(struct aa_profile *profile, - struct common_audit_data *sa, + struct apparmor_audit_data *ad, u32 request, struct sock *sk) { - return aa_profile_af_perm(profile, sa, request, sk->sk_family, + return aa_profile_af_perm(profile, ad, request, sk->sk_family, sk->sk_type); } int aa_sk_perm(const char *op, u32 request, struct sock *sk); diff --git a/security/apparmor/include/perms.h b/security/apparmor/include/perms.h index 797a7a00644d..83534df8939f 100644 --- a/security/apparmor/include/perms.h +++ b/security/apparmor/include/perms.h @@ -212,8 +212,8 @@ void aa_profile_match_label(struct aa_profile *profile, int type, u32 request, struct aa_perms *perms); int aa_profile_label_perm(struct aa_profile *profile, struct aa_profile *target, u32 request, int type, u32 *deny, - struct common_audit_data *sa); + struct apparmor_audit_data *ad); int aa_check_perms(struct aa_profile *profile, struct aa_perms *perms, - u32 request, struct common_audit_data *sa, + u32 request, struct apparmor_audit_data *ad, void (*cb)(struct audit_buffer *, void *)); #endif /* __AA_PERM_H */ diff --git a/security/apparmor/ipc.c b/security/apparmor/ipc.c index 5acde746775f..f198b8d620a4 100644 --- a/security/apparmor/ipc.c +++ b/security/apparmor/ipc.c @@ -52,31 +52,32 @@ static const char *audit_signal_mask(u32 mask) static void audit_signal_cb(struct audit_buffer *ab, void *va) { struct common_audit_data *sa = va; + struct apparmor_audit_data *ad = aad(sa); - if (aad(sa)->request & AA_SIGNAL_PERM_MASK) { + if (ad->request & AA_SIGNAL_PERM_MASK) { audit_log_format(ab, " requested_mask=\"%s\"", - audit_signal_mask(aad(sa)->request)); - if (aad(sa)->denied & AA_SIGNAL_PERM_MASK) { + audit_signal_mask(ad->request)); + if (ad->denied & AA_SIGNAL_PERM_MASK) { audit_log_format(ab, " denied_mask=\"%s\"", - audit_signal_mask(aad(sa)->denied)); + audit_signal_mask(ad->denied)); } } - if (aad(sa)->signal == SIGUNKNOWN) + if (ad->signal == SIGUNKNOWN) audit_log_format(ab, "signal=unknown(%d)", - aad(sa)->unmappedsig); - else if (aad(sa)->signal < MAXMAPPED_SIGNAME) - audit_log_format(ab, " signal=%s", sig_names[aad(sa)->signal]); + ad->unmappedsig); + else if (ad->signal < MAXMAPPED_SIGNAME) + audit_log_format(ab, " signal=%s", sig_names[ad->signal]); else audit_log_format(ab, " signal=rtmin+%d", - aad(sa)->signal - SIGRT_BASE); + ad->signal - SIGRT_BASE); audit_log_format(ab, " peer="); - aa_label_xaudit(ab, labels_ns(aad(sa)->label), aad(sa)->peer, + aa_label_xaudit(ab, labels_ns(ad->label), ad->peer, FLAGS_NONE, GFP_ATOMIC); } static int profile_signal_perm(struct aa_profile *profile, struct aa_label *peer, u32 request, - struct common_audit_data *sa) + struct apparmor_audit_data *ad) { struct aa_ruleset *rules = list_first_entry(&profile->rules, typeof(*rules), list); @@ -87,24 +88,24 @@ static int profile_signal_perm(struct aa_profile *profile, !ANY_RULE_MEDIATES(&profile->rules, AA_CLASS_SIGNAL)) return 0; - aad(sa)->peer = peer; + ad->peer = peer; /* TODO: secondary cache check */ state = aa_dfa_next(rules->policy.dfa, rules->policy.start[AA_CLASS_SIGNAL], - aad(sa)->signal); + ad->signal); aa_label_match(profile, rules, peer, state, false, request, &perms); aa_apply_modes_to_perms(profile, &perms); - return aa_check_perms(profile, &perms, request, sa, audit_signal_cb); + return aa_check_perms(profile, &perms, request, ad, audit_signal_cb); } int aa_may_signal(struct aa_label *sender, struct aa_label *target, int sig) { struct aa_profile *profile; - DEFINE_AUDIT_DATA(sa, LSM_AUDIT_DATA_NONE, AA_CLASS_SIGNAL, OP_SIGNAL); + DEFINE_AUDIT_DATA(ad, LSM_AUDIT_DATA_NONE, AA_CLASS_SIGNAL, OP_SIGNAL); - aad(&sa)->signal = map_signal_num(sig); - aad(&sa)->unmappedsig = sig; + ad.signal = map_signal_num(sig); + ad.unmappedsig = sig; return xcheck_labels(sender, target, profile, - profile_signal_perm(profile, target, MAY_WRITE, &sa), - profile_signal_perm(profile, sender, MAY_READ, &sa)); + profile_signal_perm(profile, target, MAY_WRITE, &ad), + profile_signal_perm(profile, sender, MAY_READ, &ad)); } diff --git a/security/apparmor/lib.c b/security/apparmor/lib.c index 8e1073477c09..d6b2750fd72e 100644 --- a/security/apparmor/lib.c +++ b/security/apparmor/lib.c @@ -144,10 +144,10 @@ const char *aa_splitn_fqname(const char *fqname, size_t n, const char **ns_name, void aa_info_message(const char *str) { if (audit_enabled) { - DEFINE_AUDIT_DATA(sa, LSM_AUDIT_DATA_NONE, AA_CLASS_NONE, NULL); + DEFINE_AUDIT_DATA(ad, LSM_AUDIT_DATA_NONE, AA_CLASS_NONE, NULL); - aad(&sa)->info = str; - aa_audit_msg(AUDIT_APPARMOR_STATUS, &sa, NULL); + ad.info = str; + aa_audit_msg(AUDIT_APPARMOR_STATUS, &ad, NULL); } printk(KERN_INFO "AppArmor: %s\n", str); } @@ -282,21 +282,22 @@ void aa_audit_perm_mask(struct audit_buffer *ab, u32 mask, const char *chrs, static void aa_audit_perms_cb(struct audit_buffer *ab, void *va) { struct common_audit_data *sa = va; + struct apparmor_audit_data *ad = aad(sa); - if (aad(sa)->request) { + if (ad->request) { audit_log_format(ab, " requested_mask="); - aa_audit_perm_mask(ab, aad(sa)->request, aa_file_perm_chrs, + aa_audit_perm_mask(ab, ad->request, aa_file_perm_chrs, PERMS_CHRS_MASK, aa_file_perm_names, PERMS_NAMES_MASK); } - if (aad(sa)->denied) { + if (ad->denied) { audit_log_format(ab, "denied_mask="); - aa_audit_perm_mask(ab, aad(sa)->denied, aa_file_perm_chrs, + aa_audit_perm_mask(ab, ad->denied, aa_file_perm_chrs, PERMS_CHRS_MASK, aa_file_perm_names, PERMS_NAMES_MASK); } audit_log_format(ab, " peer="); - aa_label_xaudit(ab, labels_ns(aad(sa)->label), aad(sa)->peer, + aa_label_xaudit(ab, labels_ns(ad->label), ad->peer, FLAGS_NONE, GFP_ATOMIC); } @@ -350,21 +351,21 @@ void aa_profile_match_label(struct aa_profile *profile, /* currently unused */ int aa_profile_label_perm(struct aa_profile *profile, struct aa_profile *target, u32 request, int type, u32 *deny, - struct common_audit_data *sa) + struct apparmor_audit_data *ad) { struct aa_ruleset *rules = list_first_entry(&profile->rules, typeof(*rules), list); struct aa_perms perms; - aad(sa)->label = &profile->label; - aad(sa)->peer = &target->label; - aad(sa)->request = request; + ad->label = &profile->label; + ad->peer = &target->label; + ad->request = request; aa_profile_match_label(profile, rules, &target->label, type, request, &perms); aa_apply_modes_to_perms(profile, &perms); *deny |= request & perms.deny; - return aa_check_perms(profile, &perms, request, sa, aa_audit_perms_cb); + return aa_check_perms(profile, &perms, request, ad, aa_audit_perms_cb); } /** @@ -372,7 +373,7 @@ int aa_profile_label_perm(struct aa_profile *profile, struct aa_profile *target, * @profile: profile being checked * @perms: perms computed for the request * @request: requested perms - * @sa: initialized audit structure (MAY BE NULL if not auditing) + * @ad: initialized audit structure (MAY BE NULL if not auditing) * @cb: callback fn for type specific fields (MAY BE NULL) * * Returns: 0 if permission else error code @@ -385,7 +386,7 @@ int aa_profile_label_perm(struct aa_profile *profile, struct aa_profile *target, * with a positive value. */ int aa_check_perms(struct aa_profile *profile, struct aa_perms *perms, - u32 request, struct common_audit_data *sa, + u32 request, struct apparmor_audit_data *ad, void (*cb)(struct audit_buffer *, void *)) { int type, error; @@ -394,7 +395,7 @@ int aa_check_perms(struct aa_profile *profile, struct aa_perms *perms, if (likely(!denied)) { /* mask off perms that are not being force audited */ request &= perms->audit; - if (!request || !sa) + if (!request || !ad) return 0; type = AUDIT_APPARMOR_AUDIT; @@ -413,16 +414,16 @@ int aa_check_perms(struct aa_profile *profile, struct aa_perms *perms, error = -ENOENT; denied &= ~perms->quiet; - if (!sa || !denied) + if (!ad || !denied) return error; } - if (sa) { - aad(sa)->label = &profile->label; - aad(sa)->request = request; - aad(sa)->denied = denied; - aad(sa)->error = error; - aa_audit_msg(type, sa, cb); + if (ad) { + ad->label = &profile->label; + ad->request = request; + ad->denied = denied; + ad->error = error; + aa_audit_msg(type, ad, cb); } if (type == AUDIT_APPARMOR_ALLOWED) diff --git a/security/apparmor/lsm.c b/security/apparmor/lsm.c index 63c6f21fb5ea..a8c8b179a16c 100644 --- a/security/apparmor/lsm.c +++ b/security/apparmor/lsm.c @@ -662,7 +662,7 @@ static int apparmor_setprocattr(const char *name, void *value, char *command, *largs = NULL, *args = value; size_t arg_size; int error; - DEFINE_AUDIT_DATA(sa, LSM_AUDIT_DATA_NONE, AA_CLASS_NONE, + DEFINE_AUDIT_DATA(ad, LSM_AUDIT_DATA_NONE, AA_CLASS_NONE, OP_SETPROCATTR); if (size == 0) @@ -722,11 +722,11 @@ out: return error; fail: - aad(&sa)->label = begin_current_label_crit_section(); - aad(&sa)->info = name; - aad(&sa)->error = error = -EINVAL; - aa_audit_msg(AUDIT_APPARMOR_DENIED, &sa, NULL); - end_current_label_crit_section(aad(&sa)->label); + ad.label = begin_current_label_crit_section(); + ad.info = name; + ad.error = error = -EINVAL; + aa_audit_msg(AUDIT_APPARMOR_DENIED, &ad, NULL); + end_current_label_crit_section(ad.label); goto out; } diff --git a/security/apparmor/mount.c b/security/apparmor/mount.c index cdfa430ae216..3830bceff9c8 100644 --- a/security/apparmor/mount.c +++ b/security/apparmor/mount.c @@ -86,27 +86,28 @@ static void audit_mnt_flags(struct audit_buffer *ab, unsigned long flags) static void audit_cb(struct audit_buffer *ab, void *va) { struct common_audit_data *sa = va; + struct apparmor_audit_data *ad = aad(sa); - if (aad(sa)->mnt.type) { + if (ad->mnt.type) { audit_log_format(ab, " fstype="); - audit_log_untrustedstring(ab, aad(sa)->mnt.type); + audit_log_untrustedstring(ab, ad->mnt.type); } - if (aad(sa)->mnt.src_name) { + if (ad->mnt.src_name) { audit_log_format(ab, " srcname="); - audit_log_untrustedstring(ab, aad(sa)->mnt.src_name); + audit_log_untrustedstring(ab, ad->mnt.src_name); } - if (aad(sa)->mnt.trans) { + if (ad->mnt.trans) { audit_log_format(ab, " trans="); - audit_log_untrustedstring(ab, aad(sa)->mnt.trans); + audit_log_untrustedstring(ab, ad->mnt.trans); } - if (aad(sa)->mnt.flags) { + if (ad->mnt.flags) { audit_log_format(ab, " flags=\""); - audit_mnt_flags(ab, aad(sa)->mnt.flags); + audit_mnt_flags(ab, ad->mnt.flags); audit_log_format(ab, "\""); } - if (aad(sa)->mnt.data) { + if (ad->mnt.data) { audit_log_format(ab, " options="); - audit_log_untrustedstring(ab, aad(sa)->mnt.data); + audit_log_untrustedstring(ab, ad->mnt.data); } } @@ -134,7 +135,7 @@ static int audit_mount(struct aa_profile *profile, const char *op, struct aa_perms *perms, const char *info, int error) { int audit_type = AUDIT_APPARMOR_AUTO; - DEFINE_AUDIT_DATA(sa, LSM_AUDIT_DATA_NONE, AA_CLASS_MOUNT, op); + DEFINE_AUDIT_DATA(ad, LSM_AUDIT_DATA_NONE, AA_CLASS_MOUNT, op); if (likely(!error)) { u32 mask = perms->audit; @@ -165,17 +166,17 @@ static int audit_mount(struct aa_profile *profile, const char *op, return error; } - aad(&sa)->name = name; - aad(&sa)->mnt.src_name = src_name; - aad(&sa)->mnt.type = type; - aad(&sa)->mnt.trans = trans; - aad(&sa)->mnt.flags = flags; + ad.name = name; + ad.mnt.src_name = src_name; + ad.mnt.type = type; + ad.mnt.trans = trans; + ad.mnt.flags = flags; if (data && (perms->audit & AA_AUDIT_DATA)) - aad(&sa)->mnt.data = data; - aad(&sa)->info = info; - aad(&sa)->error = error; + ad.mnt.data = data; + ad.info = info; + ad.error = error; - return aa_audit(audit_type, profile, &sa, audit_cb); + return aa_audit(audit_type, profile, &ad, audit_cb); } /** diff --git a/security/apparmor/net.c b/security/apparmor/net.c index 788be1609a86..0c7304cd479c 100644 --- a/security/apparmor/net.c +++ b/security/apparmor/net.c @@ -71,6 +71,7 @@ static const char * const net_mask_names[] = { void audit_net_cb(struct audit_buffer *ab, void *va) { struct common_audit_data *sa = va; + struct apparmor_audit_data *ad = aad(sa); if (address_family_names[sa->u.net->family]) audit_log_format(ab, " family=\"%s\"", @@ -78,35 +79,36 @@ void audit_net_cb(struct audit_buffer *ab, void *va) else audit_log_format(ab, " family=\"unknown(%d)\"", sa->u.net->family); - if (sock_type_names[aad(sa)->net.type]) + if (sock_type_names[ad->net.type]) audit_log_format(ab, " sock_type=\"%s\"", - sock_type_names[aad(sa)->net.type]); + sock_type_names[ad->net.type]); else audit_log_format(ab, " sock_type=\"unknown(%d)\"", - aad(sa)->net.type); - audit_log_format(ab, " protocol=%d", aad(sa)->net.protocol); + ad->net.type); + audit_log_format(ab, " protocol=%d", ad->net.protocol); - if (aad(sa)->request & NET_PERMS_MASK) { + if (ad->request & NET_PERMS_MASK) { audit_log_format(ab, " requested_mask="); - aa_audit_perm_mask(ab, aad(sa)->request, NULL, 0, + aa_audit_perm_mask(ab, ad->request, NULL, 0, net_mask_names, NET_PERMS_MASK); - if (aad(sa)->denied & NET_PERMS_MASK) { + if (ad->denied & NET_PERMS_MASK) { audit_log_format(ab, " denied_mask="); - aa_audit_perm_mask(ab, aad(sa)->denied, NULL, 0, + aa_audit_perm_mask(ab, ad->denied, NULL, 0, net_mask_names, NET_PERMS_MASK); } } - if (aad(sa)->peer) { + if (ad->peer) { audit_log_format(ab, " peer="); - aa_label_xaudit(ab, labels_ns(aad(sa)->label), aad(sa)->peer, + aa_label_xaudit(ab, labels_ns(ad->label), ad->peer, FLAGS_NONE, GFP_ATOMIC); } } /* Generic af perm */ -int aa_profile_af_perm(struct aa_profile *profile, struct common_audit_data *sa, - u32 request, u16 family, int type) +int aa_profile_af_perm(struct aa_profile *profile, + struct apparmor_audit_data *ad, u32 request, u16 family, + int type) { struct aa_ruleset *rules = list_first_entry(&profile->rules, typeof(*rules), list); @@ -130,17 +132,17 @@ int aa_profile_af_perm(struct aa_profile *profile, struct common_audit_data *sa, perms = *aa_lookup_perms(&rules->policy, state); aa_apply_modes_to_perms(profile, &perms); - return aa_check_perms(profile, &perms, request, sa, audit_net_cb); + return aa_check_perms(profile, &perms, request, ad, audit_net_cb); } int aa_af_perm(struct aa_label *label, const char *op, u32 request, u16 family, int type, int protocol) { struct aa_profile *profile; - DEFINE_AUDIT_NET(sa, op, NULL, family, type, protocol); + DEFINE_AUDIT_NET(ad, op, NULL, family, type, protocol); return fn_for_each_confined(label, profile, - aa_profile_af_perm(profile, &sa, request, family, + aa_profile_af_perm(profile, &ad, request, family, type)); } @@ -155,10 +157,10 @@ static int aa_label_sk_perm(struct aa_label *label, const char *op, u32 request, if (ctx->label != kernel_t && !unconfined(label)) { struct aa_profile *profile; - DEFINE_AUDIT_SK(sa, op, sk); + DEFINE_AUDIT_SK(ad, op, sk); error = fn_for_each_confined(label, profile, - aa_profile_af_sk_perm(profile, &sa, request, sk)); + aa_profile_af_sk_perm(profile, &ad, request, sk)); } return error; @@ -214,7 +216,7 @@ static int apparmor_secmark_init(struct aa_secmark *secmark) } static int aa_secmark_perm(struct aa_profile *profile, u32 request, u32 secid, - struct common_audit_data *sa) + struct apparmor_audit_data *ad) { int i, ret; struct aa_perms perms = { }; @@ -245,17 +247,17 @@ static int aa_secmark_perm(struct aa_profile *profile, u32 request, u32 secid, aa_apply_modes_to_perms(profile, &perms); - return aa_check_perms(profile, &perms, request, sa, audit_net_cb); + return aa_check_perms(profile, &perms, request, ad, audit_net_cb); } int apparmor_secmark_check(struct aa_label *label, char *op, u32 request, u32 secid, const struct sock *sk) { struct aa_profile *profile; - DEFINE_AUDIT_SK(sa, op, sk); + DEFINE_AUDIT_SK(ad, op, sk); return fn_for_each_confined(label, profile, aa_secmark_perm(profile, request, secid, - &sa)); + &ad)); } #endif diff --git a/security/apparmor/policy.c b/security/apparmor/policy.c index d7525f228c43..4e26c97f1396 100644 --- a/security/apparmor/policy.c +++ b/security/apparmor/policy.c @@ -723,10 +723,11 @@ static int replacement_allowed(struct aa_profile *profile, int noreplace, static void audit_cb(struct audit_buffer *ab, void *va) { struct common_audit_data *sa = va; + struct apparmor_audit_data *ad = aad(sa); - if (aad(sa)->iface.ns) { + if (ad->iface.ns) { audit_log_format(ab, " ns="); - audit_log_untrustedstring(ab, aad(sa)->iface.ns); + audit_log_untrustedstring(ab, ad->iface.ns); } } @@ -745,15 +746,15 @@ static int audit_policy(struct aa_label *label, const char *op, const char *ns_name, const char *name, const char *info, int error) { - DEFINE_AUDIT_DATA(sa, LSM_AUDIT_DATA_NONE, AA_CLASS_NONE, op); + DEFINE_AUDIT_DATA(ad, LSM_AUDIT_DATA_NONE, AA_CLASS_NONE, op); - aad(&sa)->iface.ns = ns_name; - aad(&sa)->name = name; - aad(&sa)->info = info; - aad(&sa)->error = error; - aad(&sa)->label = label; + ad.iface.ns = ns_name; + ad.name = name; + ad.info = info; + ad.error = error; + ad.label = label; - aa_audit_msg(AUDIT_APPARMOR_STATUS, &sa, audit_cb); + aa_audit_msg(AUDIT_APPARMOR_STATUS, &ad, audit_cb); return error; } diff --git a/security/apparmor/policy_unpack.c b/security/apparmor/policy_unpack.c index 8b2ad42c80ba..cb8b5c497812 100644 --- a/security/apparmor/policy_unpack.c +++ b/security/apparmor/policy_unpack.c @@ -34,17 +34,18 @@ static void audit_cb(struct audit_buffer *ab, void *va) { struct common_audit_data *sa = va; + struct apparmor_audit_data *ad = aad(sa); - if (aad(sa)->iface.ns) { + if (ad->iface.ns) { audit_log_format(ab, " ns="); - audit_log_untrustedstring(ab, aad(sa)->iface.ns); + audit_log_untrustedstring(ab, ad->iface.ns); } - if (aad(sa)->name) { + if (ad->name) { audit_log_format(ab, " name="); - audit_log_untrustedstring(ab, aad(sa)->name); + audit_log_untrustedstring(ab, ad->name); } - if (aad(sa)->iface.pos) - audit_log_format(ab, " offset=%ld", aad(sa)->iface.pos); + if (ad->iface.pos) + audit_log_format(ab, " offset=%ld", ad->iface.pos); } /** @@ -63,18 +64,18 @@ static int audit_iface(struct aa_profile *new, const char *ns_name, int error) { struct aa_profile *profile = labels_profile(aa_current_raw_label()); - DEFINE_AUDIT_DATA(sa, LSM_AUDIT_DATA_NONE, AA_CLASS_NONE, NULL); + DEFINE_AUDIT_DATA(ad, LSM_AUDIT_DATA_NONE, AA_CLASS_NONE, NULL); if (e) - aad(&sa)->iface.pos = e->pos - e->start; - aad(&sa)->iface.ns = ns_name; + ad.iface.pos = e->pos - e->start; + ad.iface.ns = ns_name; if (new) - aad(&sa)->name = new->base.hname; + ad.name = new->base.hname; else - aad(&sa)->name = name; - aad(&sa)->info = info; - aad(&sa)->error = error; + ad.name = name; + ad.info = info; + ad.error = error; - return aa_audit(AUDIT_APPARMOR_STATUS, profile, &sa, audit_cb); + return aa_audit(AUDIT_APPARMOR_STATUS, profile, &ad, audit_cb); } void __aa_loaddata_update(struct aa_loaddata *data, long revision) diff --git a/security/apparmor/resource.c b/security/apparmor/resource.c index 2bebc5d9e741..b6b5e1bfe9a2 100644 --- a/security/apparmor/resource.c +++ b/security/apparmor/resource.c @@ -30,12 +30,13 @@ struct aa_sfs_entry aa_sfs_entry_rlimit[] = { static void audit_cb(struct audit_buffer *ab, void *va) { struct common_audit_data *sa = va; + struct apparmor_audit_data *ad = aad(sa); audit_log_format(ab, " rlimit=%s value=%lu", - rlim_names[aad(sa)->rlim.rlim], aad(sa)->rlim.max); - if (aad(sa)->peer) { + rlim_names[ad->rlim.rlim], ad->rlim.max); + if (ad->peer) { audit_log_format(ab, " peer="); - aa_label_xaudit(ab, labels_ns(aad(sa)->label), aad(sa)->peer, + aa_label_xaudit(ab, labels_ns(ad->label), ad->peer, FLAGS_NONE, GFP_ATOMIC); } } @@ -49,22 +50,22 @@ static void audit_cb(struct audit_buffer *ab, void *va) * @info: info being auditing * @error: error value * - * Returns: 0 or sa->error else other error code on failure + * Returns: 0 or ad->error else other error code on failure */ static int audit_resource(struct aa_profile *profile, unsigned int resource, unsigned long value, struct aa_label *peer, const char *info, int error) { - DEFINE_AUDIT_DATA(sa, LSM_AUDIT_DATA_NONE, AA_CLASS_RLIMITS, + DEFINE_AUDIT_DATA(ad, LSM_AUDIT_DATA_NONE, AA_CLASS_RLIMITS, OP_SETRLIMIT); - aad(&sa)->rlim.rlim = resource; - aad(&sa)->rlim.max = value; - aad(&sa)->peer = peer; - aad(&sa)->info = info; - aad(&sa)->error = error; + ad.rlim.rlim = resource; + ad.rlim.max = value; + ad.peer = peer; + ad.info = info; + ad.error = error; - return aa_audit(AUDIT_APPARMOR_AUTO, profile, &sa, audit_cb); + return aa_audit(AUDIT_APPARMOR_AUTO, profile, &ad, audit_cb); } /** diff --git a/security/apparmor/task.c b/security/apparmor/task.c index 5671a716fcd2..b2a777ed14d8 100644 --- a/security/apparmor/task.c +++ b/security/apparmor/task.c @@ -205,18 +205,19 @@ static const char *audit_ptrace_mask(u32 mask) static void audit_ptrace_cb(struct audit_buffer *ab, void *va) { struct common_audit_data *sa = va; + struct apparmor_audit_data *ad = aad(sa); - if (aad(sa)->request & AA_PTRACE_PERM_MASK) { + if (ad->request & AA_PTRACE_PERM_MASK) { audit_log_format(ab, " requested_mask=\"%s\"", - audit_ptrace_mask(aad(sa)->request)); + audit_ptrace_mask(ad->request)); - if (aad(sa)->denied & AA_PTRACE_PERM_MASK) { + if (ad->denied & AA_PTRACE_PERM_MASK) { audit_log_format(ab, " denied_mask=\"%s\"", - audit_ptrace_mask(aad(sa)->denied)); + audit_ptrace_mask(ad->denied)); } } audit_log_format(ab, " peer="); - aa_label_xaudit(ab, labels_ns(aad(sa)->label), aad(sa)->peer, + aa_label_xaudit(ab, labels_ns(ad->label), ad->peer, FLAGS_NONE, GFP_ATOMIC); } @@ -224,51 +225,51 @@ static void audit_ptrace_cb(struct audit_buffer *ab, void *va) /* TODO: conditionals */ static int profile_ptrace_perm(struct aa_profile *profile, struct aa_label *peer, u32 request, - struct common_audit_data *sa) + struct apparmor_audit_data *ad) { struct aa_ruleset *rules = list_first_entry(&profile->rules, typeof(*rules), list); struct aa_perms perms = { }; - aad(sa)->peer = peer; + ad->peer = peer; aa_profile_match_label(profile, rules, peer, AA_CLASS_PTRACE, request, &perms); aa_apply_modes_to_perms(profile, &perms); - return aa_check_perms(profile, &perms, request, sa, audit_ptrace_cb); + return aa_check_perms(profile, &perms, request, ad, audit_ptrace_cb); } static int profile_tracee_perm(struct aa_profile *tracee, struct aa_label *tracer, u32 request, - struct common_audit_data *sa) + struct apparmor_audit_data *ad) { if (profile_unconfined(tracee) || unconfined(tracer) || !ANY_RULE_MEDIATES(&tracee->rules, AA_CLASS_PTRACE)) return 0; - return profile_ptrace_perm(tracee, tracer, request, sa); + return profile_ptrace_perm(tracee, tracer, request, ad); } static int profile_tracer_perm(struct aa_profile *tracer, struct aa_label *tracee, u32 request, - struct common_audit_data *sa) + struct apparmor_audit_data *ad) { if (profile_unconfined(tracer)) return 0; if (ANY_RULE_MEDIATES(&tracer->rules, AA_CLASS_PTRACE)) - return profile_ptrace_perm(tracer, tracee, request, sa); + return profile_ptrace_perm(tracer, tracee, request, ad); /* profile uses the old style capability check for ptrace */ if (&tracer->label == tracee) return 0; - aad(sa)->label = &tracer->label; - aad(sa)->peer = tracee; - aad(sa)->request = 0; - aad(sa)->error = aa_capable(&tracer->label, CAP_SYS_PTRACE, + ad->label = &tracer->label; + ad->peer = tracee; + ad->request = 0; + ad->error = aa_capable(&tracer->label, CAP_SYS_PTRACE, CAP_OPT_NONE); - return aa_audit(AUDIT_APPARMOR_AUTO, tracer, sa, audit_ptrace_cb); + return aa_audit(AUDIT_APPARMOR_AUTO, tracer, ad, audit_ptrace_cb); } /** From d20f5a1a6e792d22199c9989ec7ab9e95c48d60c Mon Sep 17 00:00:00 2001 From: John Johansen Date: Mon, 19 Sep 2022 00:46:09 -0700 Subject: [PATCH 263/475] apparmor: rename audit_data->label to audit_data->subj_label rename audit_data's label field to subj_label to better reflect its use. Also at the same time drop unneeded assignments to ->subj_label as the later call to aa_check_perms will do the assignment if needed. Reviewed-by: Georgia Garcia Signed-off-by: John Johansen --- security/apparmor/audit.c | 6 +++--- security/apparmor/file.c | 2 +- security/apparmor/include/audit.h | 2 +- security/apparmor/ipc.c | 2 +- security/apparmor/lib.c | 5 ++--- security/apparmor/lsm.c | 4 ++-- security/apparmor/net.c | 2 +- security/apparmor/policy.c | 6 +++--- security/apparmor/resource.c | 2 +- security/apparmor/task.c | 4 ++-- 10 files changed, 17 insertions(+), 18 deletions(-) diff --git a/security/apparmor/audit.c b/security/apparmor/audit.c index 06ad6a8fcce1..6933cb2f679b 100644 --- a/security/apparmor/audit.c +++ b/security/apparmor/audit.c @@ -113,8 +113,8 @@ static void audit_pre(struct audit_buffer *ab, void *va) audit_log_format(ab, " error=%d", ad->error); } - if (ad->label) { - struct aa_label *label = ad->label; + if (ad->subj_label) { + struct aa_label *label = ad->subj_label; if (label_isprofile(label)) { struct aa_profile *profile = labels_profile(label); @@ -187,7 +187,7 @@ int aa_audit(int type, struct aa_profile *profile, if (KILL_MODE(profile) && type == AUDIT_APPARMOR_DENIED) type = AUDIT_APPARMOR_KILL; - ad->label = &profile->label; + ad->subj_label = &profile->label; aa_audit_msg(type, ad, cb); diff --git a/security/apparmor/file.c b/security/apparmor/file.c index 01ff612c060e..26d45effdbc7 100644 --- a/security/apparmor/file.c +++ b/security/apparmor/file.c @@ -67,7 +67,7 @@ static void file_audit_cb(struct audit_buffer *ab, void *va) if (ad->peer) { audit_log_format(ab, " target="); - aa_label_xaudit(ab, labels_ns(ad->label), ad->peer, + aa_label_xaudit(ab, labels_ns(ad->subj_label), ad->peer, FLAG_VIEW_SUBNS, GFP_KERNEL); } else if (ad->fs.target) { audit_log_format(ab, " target="); diff --git a/security/apparmor/include/audit.h b/security/apparmor/include/audit.h index 85931ec94e91..096f0a04af87 100644 --- a/security/apparmor/include/audit.h +++ b/security/apparmor/include/audit.h @@ -109,7 +109,7 @@ struct apparmor_audit_data { int type; u16 class; const char *op; - struct aa_label *label; + struct aa_label *subj_label; const char *name; const char *info; u32 request; diff --git a/security/apparmor/ipc.c b/security/apparmor/ipc.c index f198b8d620a4..fd8306399b82 100644 --- a/security/apparmor/ipc.c +++ b/security/apparmor/ipc.c @@ -71,7 +71,7 @@ static void audit_signal_cb(struct audit_buffer *ab, void *va) audit_log_format(ab, " signal=rtmin+%d", ad->signal - SIGRT_BASE); audit_log_format(ab, " peer="); - aa_label_xaudit(ab, labels_ns(ad->label), ad->peer, + aa_label_xaudit(ab, labels_ns(ad->subj_label), ad->peer, FLAGS_NONE, GFP_ATOMIC); } diff --git a/security/apparmor/lib.c b/security/apparmor/lib.c index d6b2750fd72e..c87bccafff44 100644 --- a/security/apparmor/lib.c +++ b/security/apparmor/lib.c @@ -297,7 +297,7 @@ static void aa_audit_perms_cb(struct audit_buffer *ab, void *va) PERMS_NAMES_MASK); } audit_log_format(ab, " peer="); - aa_label_xaudit(ab, labels_ns(ad->label), ad->peer, + aa_label_xaudit(ab, labels_ns(ad->subj_label), ad->peer, FLAGS_NONE, GFP_ATOMIC); } @@ -357,7 +357,6 @@ int aa_profile_label_perm(struct aa_profile *profile, struct aa_profile *target, typeof(*rules), list); struct aa_perms perms; - ad->label = &profile->label; ad->peer = &target->label; ad->request = request; @@ -419,7 +418,7 @@ int aa_check_perms(struct aa_profile *profile, struct aa_perms *perms, } if (ad) { - ad->label = &profile->label; + ad->subj_label = &profile->label; ad->request = request; ad->denied = denied; ad->error = error; diff --git a/security/apparmor/lsm.c b/security/apparmor/lsm.c index a8c8b179a16c..3a9a9902d117 100644 --- a/security/apparmor/lsm.c +++ b/security/apparmor/lsm.c @@ -722,11 +722,11 @@ out: return error; fail: - ad.label = begin_current_label_crit_section(); + ad.subj_label = begin_current_label_crit_section(); ad.info = name; ad.error = error = -EINVAL; aa_audit_msg(AUDIT_APPARMOR_DENIED, &ad, NULL); - end_current_label_crit_section(ad.label); + end_current_label_crit_section(ad.subj_label); goto out; } diff --git a/security/apparmor/net.c b/security/apparmor/net.c index 0c7304cd479c..5e50f80e35db 100644 --- a/security/apparmor/net.c +++ b/security/apparmor/net.c @@ -100,7 +100,7 @@ void audit_net_cb(struct audit_buffer *ab, void *va) } if (ad->peer) { audit_log_format(ab, " peer="); - aa_label_xaudit(ab, labels_ns(ad->label), ad->peer, + aa_label_xaudit(ab, labels_ns(ad->subj_label), ad->peer, FLAGS_NONE, GFP_ATOMIC); } } diff --git a/security/apparmor/policy.c b/security/apparmor/policy.c index 4e26c97f1396..d78970d4a851 100644 --- a/security/apparmor/policy.c +++ b/security/apparmor/policy.c @@ -733,7 +733,7 @@ static void audit_cb(struct audit_buffer *ab, void *va) /** * audit_policy - Do auditing of policy changes - * @label: label to check if it can manage policy + * @subj_label: label to check if it can manage policy * @op: policy operation being performed * @ns_name: name of namespace being manipulated * @name: name of profile being manipulated (NOT NULL) @@ -742,7 +742,7 @@ static void audit_cb(struct audit_buffer *ab, void *va) * * Returns: the error to be returned after audit is done */ -static int audit_policy(struct aa_label *label, const char *op, +static int audit_policy(struct aa_label *subj_label, const char *op, const char *ns_name, const char *name, const char *info, int error) { @@ -752,7 +752,7 @@ static int audit_policy(struct aa_label *label, const char *op, ad.name = name; ad.info = info; ad.error = error; - ad.label = label; + ad.subj_label = subj_label; aa_audit_msg(AUDIT_APPARMOR_STATUS, &ad, audit_cb); diff --git a/security/apparmor/resource.c b/security/apparmor/resource.c index b6b5e1bfe9a2..73ba26c646a5 100644 --- a/security/apparmor/resource.c +++ b/security/apparmor/resource.c @@ -36,7 +36,7 @@ static void audit_cb(struct audit_buffer *ab, void *va) rlim_names[ad->rlim.rlim], ad->rlim.max); if (ad->peer) { audit_log_format(ab, " peer="); - aa_label_xaudit(ab, labels_ns(ad->label), ad->peer, + aa_label_xaudit(ab, labels_ns(ad->subj_label), ad->peer, FLAGS_NONE, GFP_ATOMIC); } } diff --git a/security/apparmor/task.c b/security/apparmor/task.c index b2a777ed14d8..552442844ff8 100644 --- a/security/apparmor/task.c +++ b/security/apparmor/task.c @@ -217,7 +217,7 @@ static void audit_ptrace_cb(struct audit_buffer *ab, void *va) } } audit_log_format(ab, " peer="); - aa_label_xaudit(ab, labels_ns(ad->label), ad->peer, + aa_label_xaudit(ab, labels_ns(ad->subj_label), ad->peer, FLAGS_NONE, GFP_ATOMIC); } @@ -263,7 +263,7 @@ static int profile_tracer_perm(struct aa_profile *tracer, if (&tracer->label == tracee) return 0; - ad->label = &tracer->label; + ad->subj_label = &tracer->label; ad->peer = tracee; ad->request = 0; ad->error = aa_capable(&tracer->label, CAP_SYS_PTRACE, From 90c436a64a6e20482a9a613c47eb4af2e8a5328e Mon Sep 17 00:00:00 2001 From: John Johansen Date: Mon, 19 Sep 2022 20:48:48 -0700 Subject: [PATCH 264/475] apparmor: pass cred through to audit info. The cred is needed to properly audit some messages, and will be needed in the future for uid conditional mediation. So pass it through to where the apparmor_audit_data struct gets defined. Reviewed-by: Georgia Garcia Signed-off-by: John Johansen --- security/apparmor/apparmorfs.c | 11 ++- security/apparmor/capability.c | 5 +- security/apparmor/domain.c | 97 +++++++++++------- security/apparmor/file.c | 131 +++++++++++++++++-------- security/apparmor/include/audit.h | 1 + security/apparmor/include/capability.h | 3 +- security/apparmor/include/file.h | 17 ++-- security/apparmor/include/ipc.h | 4 +- security/apparmor/include/mount.h | 21 ++-- security/apparmor/include/net.h | 6 +- security/apparmor/include/policy.h | 9 +- security/apparmor/include/resource.h | 3 +- security/apparmor/include/task.h | 3 +- security/apparmor/ipc.c | 14 ++- security/apparmor/lsm.c | 85 +++++++++------- security/apparmor/mount.c | 85 ++++++++++------ security/apparmor/net.c | 17 ++-- security/apparmor/policy.c | 33 ++++--- security/apparmor/resource.c | 23 +++-- security/apparmor/task.c | 31 +++--- 20 files changed, 388 insertions(+), 211 deletions(-) diff --git a/security/apparmor/apparmorfs.c b/security/apparmor/apparmorfs.c index 37234e385fd1..f20d923944e4 100644 --- a/security/apparmor/apparmorfs.c +++ b/security/apparmor/apparmorfs.c @@ -423,7 +423,7 @@ static ssize_t policy_update(u32 mask, const char __user *buf, size_t size, /* high level check about policy management - fine grained in * below after unpack */ - error = aa_may_manage_policy(label, ns, mask); + error = aa_may_manage_policy(current_cred(), label, ns, mask); if (error) goto end_section; @@ -486,7 +486,8 @@ static ssize_t profile_remove(struct file *f, const char __user *buf, /* high level check about policy management - fine grained in * below after unpack */ - error = aa_may_manage_policy(label, ns, AA_MAY_REMOVE_POLICY); + error = aa_may_manage_policy(current_cred(), label, ns, + AA_MAY_REMOVE_POLICY); if (error) goto out; @@ -1800,7 +1801,8 @@ static int ns_mkdir_op(struct mnt_idmap *idmap, struct inode *dir, int error; label = begin_current_label_crit_section(); - error = aa_may_manage_policy(label, NULL, AA_MAY_LOAD_POLICY); + error = aa_may_manage_policy(current_cred(), label, NULL, + AA_MAY_LOAD_POLICY); end_current_label_crit_section(label); if (error) return error; @@ -1849,7 +1851,8 @@ static int ns_rmdir_op(struct inode *dir, struct dentry *dentry) int error; label = begin_current_label_crit_section(); - error = aa_may_manage_policy(label, NULL, AA_MAY_LOAD_POLICY); + error = aa_may_manage_policy(current_cred(), label, NULL, + AA_MAY_LOAD_POLICY); end_current_label_crit_section(label); if (error) return error; diff --git a/security/apparmor/capability.c b/security/apparmor/capability.c index 35979a832994..0b7d2b1086c9 100644 --- a/security/apparmor/capability.c +++ b/security/apparmor/capability.c @@ -140,6 +140,7 @@ static int profile_capable(struct aa_profile *profile, int cap, /** * aa_capable - test permission to use capability + * @subj_cread: cred we are testing capability against * @label: label being tested for capability (NOT NULL) * @cap: capability to be tested * @opts: CAP_OPT_NOAUDIT bit determines whether audit record is generated @@ -148,12 +149,14 @@ static int profile_capable(struct aa_profile *profile, int cap, * * Returns: 0 on success, or else an error code. */ -int aa_capable(struct aa_label *label, int cap, unsigned int opts) +int aa_capable(const struct cred *subj_cred, struct aa_label *label, + int cap, unsigned int opts) { struct aa_profile *profile; int error = 0; DEFINE_AUDIT_DATA(ad, LSM_AUDIT_DATA_CAP, AA_CLASS_CAP, OP_CAPABLE); + ad.subj_cred = subj_cred; ad.common.u.cap = cap; error = fn_for_each_confined(label, profile, profile_capable(profile, cap, opts, &ad)); diff --git a/security/apparmor/domain.c b/security/apparmor/domain.c index 7f81502019da..d95292a23bcf 100644 --- a/security/apparmor/domain.c +++ b/security/apparmor/domain.c @@ -31,6 +31,7 @@ /** * may_change_ptraced_domain - check if can change profile on ptraced task + * @cred: cred of task changing domain * @to_label: profile to change to (NOT NULL) * @info: message if there is an error * @@ -39,28 +40,34 @@ * * Returns: %0 or error if change not allowed */ -static int may_change_ptraced_domain(struct aa_label *to_label, +static int may_change_ptraced_domain(const struct cred *to_cred, + struct aa_label *to_label, const char **info) { struct task_struct *tracer; struct aa_label *tracerl = NULL; + const struct cred *tracer_cred = NULL; + int error = 0; rcu_read_lock(); tracer = ptrace_parent(current); - if (tracer) + if (tracer) { /* released below */ tracerl = aa_get_task_label(tracer); - + tracer_cred = get_task_cred(tracer); + } /* not ptraced */ if (!tracer || unconfined(tracerl)) goto out; - error = aa_may_ptrace(tracerl, to_label, PTRACE_MODE_ATTACH); + error = aa_may_ptrace(tracer_cred, tracerl, to_cred, to_label, + PTRACE_MODE_ATTACH); out: rcu_read_unlock(); aa_put_label(tracerl); + put_cred(tracer_cred); if (error) *info = "ptrace prevents transition"; @@ -621,7 +628,8 @@ static struct aa_label *x_to_label(struct aa_profile *profile, return new; } -static struct aa_label *profile_transition(struct aa_profile *profile, +static struct aa_label *profile_transition(const struct cred *subj_cred, + struct aa_profile *profile, const struct linux_binprm *bprm, char *buffer, struct path_cond *cond, bool *secure_exec) @@ -711,7 +719,8 @@ static struct aa_label *profile_transition(struct aa_profile *profile, } audit: - aa_audit_file(profile, &perms, OP_EXEC, MAY_EXEC, name, target, new, + aa_audit_file(subj_cred, profile, &perms, OP_EXEC, MAY_EXEC, name, + target, new, cond->uid, info, error); if (!new || nonewprivs) { aa_put_label(new); @@ -721,7 +730,8 @@ audit: return new; } -static int profile_onexec(struct aa_profile *profile, struct aa_label *onexec, +static int profile_onexec(const struct cred *subj_cred, + struct aa_profile *profile, struct aa_label *onexec, bool stack, const struct linux_binprm *bprm, char *buffer, struct path_cond *cond, bool *secure_exec) @@ -789,13 +799,15 @@ static int profile_onexec(struct aa_profile *profile, struct aa_label *onexec, } audit: - return aa_audit_file(profile, &perms, OP_EXEC, AA_MAY_ONEXEC, xname, + return aa_audit_file(subj_cred, profile, &perms, OP_EXEC, + AA_MAY_ONEXEC, xname, NULL, onexec, cond->uid, info, error); } /* ensure none ns domain transitions are correctly applied with onexec */ -static struct aa_label *handle_onexec(struct aa_label *label, +static struct aa_label *handle_onexec(const struct cred *subj_cred, + struct aa_label *label, struct aa_label *onexec, bool stack, const struct linux_binprm *bprm, char *buffer, struct path_cond *cond, @@ -812,26 +824,28 @@ static struct aa_label *handle_onexec(struct aa_label *label, if (!stack) { error = fn_for_each_in_ns(label, profile, - profile_onexec(profile, onexec, stack, + profile_onexec(subj_cred, profile, onexec, stack, bprm, buffer, cond, unsafe)); if (error) return ERR_PTR(error); new = fn_label_build_in_ns(label, profile, GFP_KERNEL, aa_get_newest_label(onexec), - profile_transition(profile, bprm, buffer, + profile_transition(subj_cred, profile, bprm, + buffer, cond, unsafe)); } else { /* TODO: determine how much we want to loosen this */ error = fn_for_each_in_ns(label, profile, - profile_onexec(profile, onexec, stack, bprm, + profile_onexec(subj_cred, profile, onexec, stack, bprm, buffer, cond, unsafe)); if (error) return ERR_PTR(error); new = fn_label_build_in_ns(label, profile, GFP_KERNEL, aa_label_merge(&profile->label, onexec, GFP_KERNEL), - profile_transition(profile, bprm, buffer, + profile_transition(subj_cred, profile, bprm, + buffer, cond, unsafe)); } @@ -840,7 +854,8 @@ static struct aa_label *handle_onexec(struct aa_label *label, /* TODO: get rid of GLOBAL_ROOT_UID */ error = fn_for_each_in_ns(label, profile, - aa_audit_file(profile, &nullperms, OP_CHANGE_ONEXEC, + aa_audit_file(subj_cred, profile, &nullperms, + OP_CHANGE_ONEXEC, AA_MAY_ONEXEC, bprm->filename, NULL, onexec, GLOBAL_ROOT_UID, "failed to build target label", -ENOMEM)); @@ -859,6 +874,7 @@ int apparmor_bprm_creds_for_exec(struct linux_binprm *bprm) { struct aa_task_ctx *ctx; struct aa_label *label, *new = NULL; + const struct cred *subj_cred; struct aa_profile *profile; char *buffer = NULL; const char *info = NULL; @@ -871,6 +887,7 @@ int apparmor_bprm_creds_for_exec(struct linux_binprm *bprm) file_inode(bprm->file)->i_mode }; + subj_cred = current_cred(); ctx = task_ctx(current); AA_BUG(!cred_label(bprm->cred)); AA_BUG(!ctx); @@ -897,11 +914,12 @@ int apparmor_bprm_creds_for_exec(struct linux_binprm *bprm) /* Test for onexec first as onexec override other x transitions. */ if (ctx->onexec) - new = handle_onexec(label, ctx->onexec, ctx->token, + new = handle_onexec(subj_cred, label, ctx->onexec, ctx->token, bprm, buffer, &cond, &unsafe); else new = fn_label_build(label, profile, GFP_KERNEL, - profile_transition(profile, bprm, buffer, + profile_transition(subj_cred, profile, bprm, + buffer, &cond, &unsafe)); AA_BUG(!new); @@ -936,7 +954,7 @@ int apparmor_bprm_creds_for_exec(struct linux_binprm *bprm) if (bprm->unsafe & (LSM_UNSAFE_PTRACE)) { /* TODO: test needs to be profile of label to new */ - error = may_change_ptraced_domain(new, &info); + error = may_change_ptraced_domain(bprm->cred, new, &info); if (error) goto audit; } @@ -973,7 +991,8 @@ done: audit: error = fn_for_each(label, profile, - aa_audit_file(profile, &nullperms, OP_EXEC, MAY_EXEC, + aa_audit_file(current_cred(), profile, &nullperms, + OP_EXEC, MAY_EXEC, bprm->filename, NULL, new, vfsuid_into_kuid(vfsuid), info, error)); aa_put_label(new); @@ -989,7 +1008,8 @@ audit: * * Returns: label for hat transition OR ERR_PTR. Does NOT return NULL */ -static struct aa_label *build_change_hat(struct aa_profile *profile, +static struct aa_label *build_change_hat(const struct cred *subj_cred, + struct aa_profile *profile, const char *name, bool sibling) { struct aa_profile *root, *hat = NULL; @@ -1021,7 +1041,8 @@ static struct aa_label *build_change_hat(struct aa_profile *profile, aa_put_profile(root); audit: - aa_audit_file(profile, &nullperms, OP_CHANGE_HAT, AA_MAY_CHANGEHAT, + aa_audit_file(subj_cred, profile, &nullperms, OP_CHANGE_HAT, + AA_MAY_CHANGEHAT, name, hat ? hat->base.hname : NULL, hat ? &hat->label : NULL, GLOBAL_ROOT_UID, info, error); @@ -1037,7 +1058,8 @@ audit: * * Returns: label for hat transition or ERR_PTR. Does not return NULL */ -static struct aa_label *change_hat(struct aa_label *label, const char *hats[], +static struct aa_label *change_hat(const struct cred *subj_cred, + struct aa_label *label, const char *hats[], int count, int flags) { struct aa_profile *profile, *root, *hat = NULL; @@ -1113,7 +1135,8 @@ fail: */ /* TODO: get rid of GLOBAL_ROOT_UID */ if (count > 1 || COMPLAIN_MODE(profile)) { - aa_audit_file(profile, &nullperms, OP_CHANGE_HAT, + aa_audit_file(subj_cred, profile, &nullperms, + OP_CHANGE_HAT, AA_MAY_CHANGEHAT, name, NULL, NULL, GLOBAL_ROOT_UID, info, error); } @@ -1122,7 +1145,8 @@ fail: build: new = fn_label_build_in_ns(label, profile, GFP_KERNEL, - build_change_hat(profile, name, sibling), + build_change_hat(subj_cred, profile, name, + sibling), aa_get_label(&profile->label)); if (!new) { info = "label build failed"; @@ -1152,7 +1176,7 @@ build: */ int aa_change_hat(const char *hats[], int count, u64 token, int flags) { - const struct cred *cred; + const struct cred *subj_cred; struct aa_task_ctx *ctx = task_ctx(current); struct aa_label *label, *previous, *new = NULL, *target = NULL; struct aa_profile *profile; @@ -1161,8 +1185,8 @@ int aa_change_hat(const char *hats[], int count, u64 token, int flags) int error = 0; /* released below */ - cred = get_current_cred(); - label = aa_get_newest_cred_label(cred); + subj_cred = get_current_cred(); + label = aa_get_newest_cred_label(subj_cred); previous = aa_get_newest_label(ctx->previous); /* @@ -1182,7 +1206,7 @@ int aa_change_hat(const char *hats[], int count, u64 token, int flags) } if (count) { - new = change_hat(label, hats, count, flags); + new = change_hat(subj_cred, label, hats, count, flags); AA_BUG(!new); if (IS_ERR(new)) { error = PTR_ERR(new); @@ -1191,7 +1215,8 @@ int aa_change_hat(const char *hats[], int count, u64 token, int flags) goto out; } - error = may_change_ptraced_domain(new, &info); + /* target cred is the same as current except new label */ + error = may_change_ptraced_domain(subj_cred, new, &info); if (error) goto fail; @@ -1244,7 +1269,7 @@ out: aa_put_label(new); aa_put_label(previous); aa_put_label(label); - put_cred(cred); + put_cred(subj_cred); return error; @@ -1254,7 +1279,7 @@ kill: fail: fn_for_each_in_ns(label, profile, - aa_audit_file(profile, &perms, OP_CHANGE_HAT, + aa_audit_file(subj_cred, profile, &perms, OP_CHANGE_HAT, AA_MAY_CHANGEHAT, NULL, NULL, target, GLOBAL_ROOT_UID, info, error)); @@ -1263,6 +1288,7 @@ fail: static int change_profile_perms_wrapper(const char *op, const char *name, + const struct cred *subj_cred, struct aa_profile *profile, struct aa_label *target, bool stack, u32 request, struct aa_perms *perms) @@ -1277,7 +1303,8 @@ static int change_profile_perms_wrapper(const char *op, const char *name, rules->file.start[AA_CLASS_FILE], perms); if (error) - error = aa_audit_file(profile, perms, op, request, name, + error = aa_audit_file(subj_cred, profile, perms, op, request, + name, NULL, target, GLOBAL_ROOT_UID, info, error); @@ -1306,6 +1333,7 @@ int aa_change_profile(const char *fqname, int flags) const char *auditname = fqname; /* retain leading & if stack */ bool stack = flags & AA_CHANGE_STACK; struct aa_task_ctx *ctx = task_ctx(current); + const struct cred *subj_cred = get_current_cred(); int error = 0; char *op; u32 request; @@ -1383,6 +1411,7 @@ int aa_change_profile(const char *fqname, int flags) */ error = fn_for_each_in_ns(label, profile, change_profile_perms_wrapper(op, auditname, + subj_cred, profile, target, stack, request, &perms)); if (error) @@ -1393,7 +1422,7 @@ int aa_change_profile(const char *fqname, int flags) check: /* check if tracing task is allowed to trace target domain */ - error = may_change_ptraced_domain(target, &info); + error = may_change_ptraced_domain(subj_cred, target, &info); if (error && !fn_for_each_in_ns(label, profile, COMPLAIN_MODE(profile))) goto audit; @@ -1453,7 +1482,8 @@ check: audit: error = fn_for_each_in_ns(label, profile, - aa_audit_file(profile, &perms, op, request, auditname, + aa_audit_file(subj_cred, + profile, &perms, op, request, auditname, NULL, new ? new : target, GLOBAL_ROOT_UID, info, error)); @@ -1461,6 +1491,7 @@ out: aa_put_label(new); aa_put_label(target); aa_put_label(label); + put_cred(subj_cred); return error; } diff --git a/security/apparmor/file.c b/security/apparmor/file.c index 26d45effdbc7..48afcef45694 100644 --- a/security/apparmor/file.c +++ b/security/apparmor/file.c @@ -45,7 +45,7 @@ static void file_audit_cb(struct audit_buffer *ab, void *va) { struct common_audit_data *sa = va; struct apparmor_audit_data *ad = aad(sa); - kuid_t fsuid = current_fsuid(); + kuid_t fsuid = ad->subj_cred ? ad->subj_cred->fsuid : current_fsuid(); char str[10]; if (ad->request & AA_AUDIT_FILE_MASK) { @@ -77,6 +77,7 @@ static void file_audit_cb(struct audit_buffer *ab, void *va) /** * aa_audit_file - handle the auditing of file operations + * @subj_cred: cred of the subject * @profile: the profile being enforced (NOT NULL) * @perms: the permissions computed for the request (NOT NULL) * @op: operation being mediated @@ -90,7 +91,8 @@ static void file_audit_cb(struct audit_buffer *ab, void *va) * * Returns: %0 or error on failure */ -int aa_audit_file(struct aa_profile *profile, struct aa_perms *perms, +int aa_audit_file(const struct cred *subj_cred, + struct aa_profile *profile, struct aa_perms *perms, const char *op, u32 request, const char *name, const char *target, struct aa_label *tlabel, kuid_t ouid, const char *info, int error) @@ -98,6 +100,7 @@ int aa_audit_file(struct aa_profile *profile, struct aa_perms *perms, int type = AUDIT_APPARMOR_AUTO; DEFINE_AUDIT_DATA(ad, LSM_AUDIT_DATA_TASK, AA_CLASS_FILE, op); + ad.subj_cred = subj_cred; ad.request = request; ad.name = name; ad.fs.target = target; @@ -141,7 +144,21 @@ int aa_audit_file(struct aa_profile *profile, struct aa_perms *perms, return aa_audit(type, profile, &ad, file_audit_cb); } -static int path_name(const char *op, struct aa_label *label, +/** + * is_deleted - test if a file has been completely unlinked + * @dentry: dentry of file to test for deletion (NOT NULL) + * + * Returns: true if deleted else false + */ +static inline bool is_deleted(struct dentry *dentry) +{ + if (d_unlinked(dentry) && d_backing_inode(dentry)->i_nlink == 0) + return true; + return false; +} + +static int path_name(const char *op, const struct cred *subj_cred, + struct aa_label *label, const struct path *path, int flags, char *buffer, const char **name, struct path_cond *cond, u32 request) { @@ -153,7 +170,8 @@ static int path_name(const char *op, struct aa_label *label, labels_profile(label)->disconnected); if (error) { fn_for_each_confined(label, profile, - aa_audit_file(profile, &nullperms, op, request, *name, + aa_audit_file(subj_cred, + profile, &nullperms, op, request, *name, NULL, NULL, cond->uid, info, error)); return error; } @@ -207,9 +225,9 @@ aa_state_t aa_str_perms(struct aa_policydb *file_rules, aa_state_t start, return state; } -static int __aa_path_perm(const char *op, struct aa_profile *profile, - const char *name, u32 request, - struct path_cond *cond, int flags, +static int __aa_path_perm(const char *op, const struct cred *subj_cred, + struct aa_profile *profile, const char *name, + u32 request, struct path_cond *cond, int flags, struct aa_perms *perms) { struct aa_ruleset *rules = list_first_entry(&profile->rules, @@ -222,12 +240,14 @@ static int __aa_path_perm(const char *op, struct aa_profile *profile, name, cond, perms); if (request & ~perms->allow) e = -EACCES; - return aa_audit_file(profile, perms, op, request, name, NULL, NULL, + return aa_audit_file(subj_cred, + profile, perms, op, request, name, NULL, NULL, cond->uid, NULL, e); } -static int profile_path_perm(const char *op, struct aa_profile *profile, +static int profile_path_perm(const char *op, const struct cred *subj_cred, + struct aa_profile *profile, const struct path *path, char *buffer, u32 request, struct path_cond *cond, int flags, struct aa_perms *perms) @@ -238,18 +258,19 @@ static int profile_path_perm(const char *op, struct aa_profile *profile, if (profile_unconfined(profile)) return 0; - error = path_name(op, &profile->label, path, + error = path_name(op, subj_cred, &profile->label, path, flags | profile->path_flags, buffer, &name, cond, request); if (error) return error; - return __aa_path_perm(op, profile, name, request, cond, flags, - perms); + return __aa_path_perm(op, subj_cred, profile, name, request, cond, + flags, perms); } /** * aa_path_perm - do permissions check & audit for @path * @op: operation being checked + * @subj_cred: subject cred * @label: profile being enforced (NOT NULL) * @path: path to check permissions of (NOT NULL) * @flags: any additional path flags beyond what the profile specifies @@ -258,7 +279,8 @@ static int profile_path_perm(const char *op, struct aa_profile *profile, * * Returns: %0 else error if access denied or other error */ -int aa_path_perm(const char *op, struct aa_label *label, +int aa_path_perm(const char *op, const struct cred *subj_cred, + struct aa_label *label, const struct path *path, int flags, u32 request, struct path_cond *cond) { @@ -273,8 +295,8 @@ int aa_path_perm(const char *op, struct aa_label *label, if (!buffer) return -ENOMEM; error = fn_for_each_confined(label, profile, - profile_path_perm(op, profile, path, buffer, request, - cond, flags, &perms)); + profile_path_perm(op, subj_cred, profile, path, buffer, + request, cond, flags, &perms)); aa_put_buffer(buffer); @@ -301,7 +323,8 @@ static inline bool xindex_is_subset(u32 link, u32 target) return true; } -static int profile_path_link(struct aa_profile *profile, +static int profile_path_link(const struct cred *subj_cred, + struct aa_profile *profile, const struct path *link, char *buffer, const struct path *target, char *buffer2, struct path_cond *cond) @@ -315,13 +338,15 @@ static int profile_path_link(struct aa_profile *profile, aa_state_t state; int error; - error = path_name(OP_LINK, &profile->label, link, profile->path_flags, + error = path_name(OP_LINK, subj_cred, &profile->label, link, + profile->path_flags, buffer, &lname, cond, AA_MAY_LINK); if (error) goto audit; /* buffer2 freed below, tname is pointer in buffer2 */ - error = path_name(OP_LINK, &profile->label, target, profile->path_flags, + error = path_name(OP_LINK, subj_cred, &profile->label, target, + profile->path_flags, buffer2, &tname, cond, AA_MAY_LINK); if (error) goto audit; @@ -381,12 +406,14 @@ done_tests: error = 0; audit: - return aa_audit_file(profile, &lperms, OP_LINK, request, lname, tname, + return aa_audit_file(subj_cred, + profile, &lperms, OP_LINK, request, lname, tname, NULL, cond->uid, info, error); } /** * aa_path_link - Handle hard link permission check + * @subj_cred: subject cred * @label: the label being enforced (NOT NULL) * @old_dentry: the target dentry (NOT NULL) * @new_dir: directory the new link will be created in (NOT NULL) @@ -403,7 +430,8 @@ audit: * * Returns: %0 if allowed else error */ -int aa_path_link(struct aa_label *label, struct dentry *old_dentry, +int aa_path_link(const struct cred *subj_cred, + struct aa_label *label, struct dentry *old_dentry, const struct path *new_dir, struct dentry *new_dentry) { struct path link = { .mnt = new_dir->mnt, .dentry = new_dentry }; @@ -424,8 +452,8 @@ int aa_path_link(struct aa_label *label, struct dentry *old_dentry, goto out; error = fn_for_each_confined(label, profile, - profile_path_link(profile, &link, buffer, &target, - buffer2, &cond)); + profile_path_link(subj_cred, profile, &link, buffer, + &target, buffer2, &cond)); out: aa_put_buffer(buffer); aa_put_buffer(buffer2); @@ -453,7 +481,8 @@ static void update_file_ctx(struct aa_file_ctx *fctx, struct aa_label *label, spin_unlock(&fctx->lock); } -static int __file_path_perm(const char *op, struct aa_label *label, +static int __file_path_perm(const char *op, const struct cred *subj_cred, + struct aa_label *label, struct aa_label *flabel, struct file *file, u32 request, u32 denied, bool in_atomic) { @@ -480,7 +509,8 @@ static int __file_path_perm(const char *op, struct aa_label *label, /* check every profile in task label not in current cache */ error = fn_for_each_not_in_set(flabel, label, profile, - profile_path_perm(op, profile, &file->f_path, buffer, + profile_path_perm(op, subj_cred, profile, + &file->f_path, buffer, request, &cond, flags, &perms)); if (denied && !error) { /* @@ -493,12 +523,14 @@ static int __file_path_perm(const char *op, struct aa_label *label, */ if (label == flabel) error = fn_for_each(label, profile, - profile_path_perm(op, profile, &file->f_path, + profile_path_perm(op, subj_cred, + profile, &file->f_path, buffer, request, &cond, flags, &perms)); else error = fn_for_each_not_in_set(label, flabel, profile, - profile_path_perm(op, profile, &file->f_path, + profile_path_perm(op, subj_cred, + profile, &file->f_path, buffer, request, &cond, flags, &perms)); } @@ -510,7 +542,8 @@ static int __file_path_perm(const char *op, struct aa_label *label, return error; } -static int __file_sock_perm(const char *op, struct aa_label *label, +static int __file_sock_perm(const char *op, const struct cred *subj_cred, + struct aa_label *label, struct aa_label *flabel, struct file *file, u32 request, u32 denied) { @@ -524,11 +557,12 @@ static int __file_sock_perm(const char *op, struct aa_label *label, return 0; /* TODO: improve to skip profiles cached in flabel */ - error = aa_sock_file_perm(label, op, request, sock); + error = aa_sock_file_perm(subj_cred, label, op, request, sock); if (denied) { /* TODO: improve to skip profiles checked above */ /* check every profile in file label to is cached */ - last_error(error, aa_sock_file_perm(flabel, op, request, sock)); + last_error(error, aa_sock_file_perm(subj_cred, flabel, op, + request, sock)); } if (!error) update_file_ctx(file_ctx(file), label, request); @@ -539,6 +573,7 @@ static int __file_sock_perm(const char *op, struct aa_label *label, /** * aa_file_perm - do permission revalidation check & audit for @file * @op: operation being checked + * @subj_cred: subject cred * @label: label being enforced (NOT NULL) * @file: file to revalidate access permissions on (NOT NULL) * @request: requested permissions @@ -546,7 +581,8 @@ static int __file_sock_perm(const char *op, struct aa_label *label, * * Returns: %0 if access allowed else error */ -int aa_file_perm(const char *op, struct aa_label *label, struct file *file, +int aa_file_perm(const char *op, const struct cred *subj_cred, + struct aa_label *label, struct file *file, u32 request, bool in_atomic) { struct aa_file_ctx *fctx; @@ -582,19 +618,19 @@ int aa_file_perm(const char *op, struct aa_label *label, struct file *file, /* TODO: label cross check */ if (file->f_path.mnt && path_mediated_fs(file->f_path.dentry)) - error = __file_path_perm(op, label, flabel, file, request, - denied, in_atomic); + error = __file_path_perm(op, subj_cred, label, flabel, file, + request, denied, in_atomic); else if (S_ISSOCK(file_inode(file)->i_mode)) - error = __file_sock_perm(op, label, flabel, file, request, - denied); + error = __file_sock_perm(op, subj_cred, label, flabel, file, + request, denied); aa_put_label(flabel); done: return error; } -static void revalidate_tty(struct aa_label *label) +static void revalidate_tty(const struct cred *subj_cred, struct aa_label *label) { struct tty_struct *tty; int drop_tty = 0; @@ -612,8 +648,8 @@ static void revalidate_tty(struct aa_label *label) struct tty_file_private, list); file = file_priv->file; - if (aa_file_perm(OP_INHERIT, label, file, MAY_READ | MAY_WRITE, - IN_ATOMIC)) + if (aa_file_perm(OP_INHERIT, subj_cred, label, file, + MAY_READ | MAY_WRITE, IN_ATOMIC)) drop_tty = 1; } spin_unlock(&tty->files_lock); @@ -623,12 +659,17 @@ static void revalidate_tty(struct aa_label *label) no_tty(); } +struct cred_label { + const struct cred *cred; + struct aa_label *label; +}; + static int match_file(const void *p, struct file *file, unsigned int fd) { - struct aa_label *label = (struct aa_label *)p; + struct cred_label *cl = (struct cred_label *)p; - if (aa_file_perm(OP_INHERIT, label, file, aa_map_file_to_perms(file), - IN_ATOMIC)) + if (aa_file_perm(OP_INHERIT, cl->cred, cl->label, file, + aa_map_file_to_perms(file), IN_ATOMIC)) return fd + 1; return 0; } @@ -638,13 +679,17 @@ static int match_file(const void *p, struct file *file, unsigned int fd) void aa_inherit_files(const struct cred *cred, struct files_struct *files) { struct aa_label *label = aa_get_newest_cred_label(cred); + struct cred_label cl = { + .cred = cred, + .label = label, + }; struct file *devnull = NULL; unsigned int n; - revalidate_tty(label); + revalidate_tty(cred, label); /* Revalidate access to inherited open files. */ - n = iterate_fd(files, 0, match_file, label); + n = iterate_fd(files, 0, match_file, &cl); if (!n) /* none found? */ goto out; @@ -654,7 +699,7 @@ void aa_inherit_files(const struct cred *cred, struct files_struct *files) /* replace all the matching ones with this */ do { replace_fd(n - 1, devnull, 0); - } while ((n = iterate_fd(files, n, match_file, label)) != 0); + } while ((n = iterate_fd(files, n, match_file, &cl)) != 0); if (devnull) fput(devnull); out: diff --git a/security/apparmor/include/audit.h b/security/apparmor/include/audit.h index 096f0a04af87..42d701fec5a6 100644 --- a/security/apparmor/include/audit.h +++ b/security/apparmor/include/audit.h @@ -109,6 +109,7 @@ struct apparmor_audit_data { int type; u16 class; const char *op; + const struct cred *subj_cred; struct aa_label *subj_label; const char *name; const char *info; diff --git a/security/apparmor/include/capability.h b/security/apparmor/include/capability.h index d420e2d10b31..d6dcc604ec0c 100644 --- a/security/apparmor/include/capability.h +++ b/security/apparmor/include/capability.h @@ -36,7 +36,8 @@ struct aa_caps { extern struct aa_sfs_entry aa_sfs_entry_caps[]; -int aa_capable(struct aa_label *label, int cap, unsigned int opts); +int aa_capable(const struct cred *subj_cred, struct aa_label *label, + int cap, unsigned int opts); static inline void aa_free_cap_rules(struct aa_caps *caps) { diff --git a/security/apparmor/include/file.h b/security/apparmor/include/file.h index 23cb6f9dbe6e..6e8f2aa66cd6 100644 --- a/security/apparmor/include/file.h +++ b/security/apparmor/include/file.h @@ -71,7 +71,8 @@ struct path_cond { #define COMBINED_PERM_MASK(X) ((X).allow | (X).audit | (X).quiet | (X).kill) -int aa_audit_file(struct aa_profile *profile, struct aa_perms *perms, +int aa_audit_file(const struct cred *cred, + struct aa_profile *profile, struct aa_perms *perms, const char *op, u32 request, const char *name, const char *target, struct aa_label *tlabel, kuid_t ouid, const char *info, int error); @@ -82,14 +83,16 @@ aa_state_t aa_str_perms(struct aa_policydb *file_rules, aa_state_t start, const char *name, struct path_cond *cond, struct aa_perms *perms); -int aa_path_perm(const char *op, struct aa_label *label, - const struct path *path, int flags, u32 request, - struct path_cond *cond); +int aa_path_perm(const char *op, const struct cred *subj_cred, + struct aa_label *label, const struct path *path, + int flags, u32 request, struct path_cond *cond); -int aa_path_link(struct aa_label *label, struct dentry *old_dentry, - const struct path *new_dir, struct dentry *new_dentry); +int aa_path_link(const struct cred *subj_cred, struct aa_label *label, + struct dentry *old_dentry, const struct path *new_dir, + struct dentry *new_dentry); -int aa_file_perm(const char *op, struct aa_label *label, struct file *file, +int aa_file_perm(const char *op, const struct cred *subj_cred, + struct aa_label *label, struct file *file, u32 request, bool in_atomic); void aa_inherit_files(const struct cred *cred, struct files_struct *files); diff --git a/security/apparmor/include/ipc.h b/security/apparmor/include/ipc.h index a1ac6ffb95e9..74d17052f76b 100644 --- a/security/apparmor/include/ipc.h +++ b/security/apparmor/include/ipc.h @@ -13,6 +13,8 @@ #include -int aa_may_signal(struct aa_label *sender, struct aa_label *target, int sig); +int aa_may_signal(const struct cred *subj_cred, struct aa_label *sender, + const struct cred *target_cred, struct aa_label *target, + int sig); #endif /* __AA_IPC_H */ diff --git a/security/apparmor/include/mount.h b/security/apparmor/include/mount.h index a710683b2496..10c76f906a65 100644 --- a/security/apparmor/include/mount.h +++ b/security/apparmor/include/mount.h @@ -25,26 +25,33 @@ #define AA_MS_IGNORE_MASK (MS_KERNMOUNT | MS_NOSEC | MS_ACTIVE | MS_BORN) -int aa_remount(struct aa_label *label, const struct path *path, +int aa_remount(const struct cred *subj_cred, + struct aa_label *label, const struct path *path, unsigned long flags, void *data); -int aa_bind_mount(struct aa_label *label, const struct path *path, +int aa_bind_mount(const struct cred *subj_cred, + struct aa_label *label, const struct path *path, const char *old_name, unsigned long flags); -int aa_mount_change_type(struct aa_label *label, const struct path *path, +int aa_mount_change_type(const struct cred *subj_cred, + struct aa_label *label, const struct path *path, unsigned long flags); -int aa_move_mount(struct aa_label *label, const struct path *path, +int aa_move_mount(const struct cred *subj_cred, + struct aa_label *label, const struct path *path, const char *old_name); -int aa_new_mount(struct aa_label *label, const char *dev_name, +int aa_new_mount(const struct cred *subj_cred, + struct aa_label *label, const char *dev_name, const struct path *path, const char *type, unsigned long flags, void *data); -int aa_umount(struct aa_label *label, struct vfsmount *mnt, int flags); +int aa_umount(const struct cred *subj_cred, + struct aa_label *label, struct vfsmount *mnt, int flags); -int aa_pivotroot(struct aa_label *label, const struct path *old_path, +int aa_pivotroot(const struct cred *subj_cred, + struct aa_label *label, const struct path *old_path, const struct path *new_path); #endif /* __AA_MOUNT_H */ diff --git a/security/apparmor/include/net.h b/security/apparmor/include/net.h index e0068a3a4efd..67bf888c3bd6 100644 --- a/security/apparmor/include/net.h +++ b/security/apparmor/include/net.h @@ -97,7 +97,8 @@ void audit_net_cb(struct audit_buffer *ab, void *va); int aa_profile_af_perm(struct aa_profile *profile, struct apparmor_audit_data *ad, u32 request, u16 family, int type); -int aa_af_perm(struct aa_label *label, const char *op, u32 request, u16 family, +int aa_af_perm(const struct cred *subj_cred, struct aa_label *label, + const char *op, u32 request, u16 family, int type, int protocol); static inline int aa_profile_af_sk_perm(struct aa_profile *profile, struct apparmor_audit_data *ad, @@ -109,7 +110,8 @@ static inline int aa_profile_af_sk_perm(struct aa_profile *profile, } int aa_sk_perm(const char *op, u32 request, struct sock *sk); -int aa_sock_file_perm(struct aa_label *label, const char *op, u32 request, +int aa_sock_file_perm(const struct cred *subj_cred, struct aa_label *label, + const char *op, u32 request, struct socket *sock); int apparmor_secmark_check(struct aa_label *label, char *op, u32 request, diff --git a/security/apparmor/include/policy.h b/security/apparmor/include/policy.h index 062a4cd3c367..5572447d7c37 100644 --- a/security/apparmor/include/policy.h +++ b/security/apparmor/include/policy.h @@ -361,9 +361,12 @@ static inline int AUDIT_MODE(struct aa_profile *profile) return profile->audit; } -bool aa_policy_view_capable(struct aa_label *label, struct aa_ns *ns); -bool aa_policy_admin_capable(struct aa_label *label, struct aa_ns *ns); -int aa_may_manage_policy(struct aa_label *label, struct aa_ns *ns, +bool aa_policy_view_capable(const struct cred *subj_cred, + struct aa_label *label, struct aa_ns *ns); +bool aa_policy_admin_capable(const struct cred *subj_cred, + struct aa_label *label, struct aa_ns *ns); +int aa_may_manage_policy(const struct cred *subj_cred, + struct aa_label *label, struct aa_ns *ns, u32 mask); bool aa_current_policy_view_capable(struct aa_ns *ns); bool aa_current_policy_admin_capable(struct aa_ns *ns); diff --git a/security/apparmor/include/resource.h b/security/apparmor/include/resource.h index 961d85d328ea..ad2c0da8e64f 100644 --- a/security/apparmor/include/resource.h +++ b/security/apparmor/include/resource.h @@ -33,7 +33,8 @@ struct aa_rlimit { extern struct aa_sfs_entry aa_sfs_entry_rlimit[]; int aa_map_resource(int resource); -int aa_task_setrlimit(struct aa_label *label, struct task_struct *task, +int aa_task_setrlimit(const struct cred *subj_cred, struct aa_label *label, + struct task_struct *task, unsigned int resource, struct rlimit *new_rlim); void __aa_transition_rlimits(struct aa_label *old, struct aa_label *new); diff --git a/security/apparmor/include/task.h b/security/apparmor/include/task.h index 01717fe432c3..13945e2495f0 100644 --- a/security/apparmor/include/task.h +++ b/security/apparmor/include/task.h @@ -91,7 +91,8 @@ static inline void aa_clear_task_ctx_trans(struct aa_task_ctx *ctx) "segv usr2 pipe alrm term stkflt chld cont stop stp ttin ttou urg " \ "xcpu xfsz vtalrm prof winch io pwr sys emt lost" -int aa_may_ptrace(struct aa_label *tracer, struct aa_label *tracee, +int aa_may_ptrace(const struct cred *tracer_cred, struct aa_label *tracer, + const struct cred *tracee_cred, struct aa_label *tracee, u32 request); diff --git a/security/apparmor/ipc.c b/security/apparmor/ipc.c index fd8306399b82..c0d0dbd7b4c4 100644 --- a/security/apparmor/ipc.c +++ b/security/apparmor/ipc.c @@ -75,7 +75,8 @@ static void audit_signal_cb(struct audit_buffer *ab, void *va) FLAGS_NONE, GFP_ATOMIC); } -static int profile_signal_perm(struct aa_profile *profile, +static int profile_signal_perm(const struct cred *cred, + struct aa_profile *profile, struct aa_label *peer, u32 request, struct apparmor_audit_data *ad) { @@ -88,6 +89,7 @@ static int profile_signal_perm(struct aa_profile *profile, !ANY_RULE_MEDIATES(&profile->rules, AA_CLASS_SIGNAL)) return 0; + ad->subj_cred = cred; ad->peer = peer; /* TODO: secondary cache check */ state = aa_dfa_next(rules->policy.dfa, @@ -98,7 +100,9 @@ static int profile_signal_perm(struct aa_profile *profile, return aa_check_perms(profile, &perms, request, ad, audit_signal_cb); } -int aa_may_signal(struct aa_label *sender, struct aa_label *target, int sig) +int aa_may_signal(const struct cred *subj_cred, struct aa_label *sender, + const struct cred *target_cred, struct aa_label *target, + int sig) { struct aa_profile *profile; DEFINE_AUDIT_DATA(ad, LSM_AUDIT_DATA_NONE, AA_CLASS_SIGNAL, OP_SIGNAL); @@ -106,6 +110,8 @@ int aa_may_signal(struct aa_label *sender, struct aa_label *target, int sig) ad.signal = map_signal_num(sig); ad.unmappedsig = sig; return xcheck_labels(sender, target, profile, - profile_signal_perm(profile, target, MAY_WRITE, &ad), - profile_signal_perm(profile, sender, MAY_READ, &ad)); + profile_signal_perm(subj_cred, profile, target, + MAY_WRITE, &ad), + profile_signal_perm(target_cred, profile, sender, + MAY_READ, &ad)); } diff --git a/security/apparmor/lsm.c b/security/apparmor/lsm.c index 3a9a9902d117..9e31a932a644 100644 --- a/security/apparmor/lsm.c +++ b/security/apparmor/lsm.c @@ -116,15 +116,17 @@ static int apparmor_ptrace_access_check(struct task_struct *child, unsigned int mode) { struct aa_label *tracer, *tracee; + const struct cred *cred; int error; + cred = get_task_cred(child); + tracee = cred_label(cred); /* ref count on cred */ tracer = __begin_current_label_crit_section(); - tracee = aa_get_task_label(child); - error = aa_may_ptrace(tracer, tracee, + error = aa_may_ptrace(current_cred(), tracer, cred, tracee, (mode & PTRACE_MODE_READ) ? AA_PTRACE_READ : AA_PTRACE_TRACE); - aa_put_label(tracee); __end_current_label_crit_section(tracer); + put_cred(cred); return error; } @@ -132,12 +134,15 @@ static int apparmor_ptrace_access_check(struct task_struct *child, static int apparmor_ptrace_traceme(struct task_struct *parent) { struct aa_label *tracer, *tracee; + const struct cred *cred; int error; tracee = __begin_current_label_crit_section(); - tracer = aa_get_task_label(parent); - error = aa_may_ptrace(tracer, tracee, AA_PTRACE_TRACE); - aa_put_label(tracer); + cred = get_task_cred(parent); + tracer = cred_label(cred); /* ref count on cred */ + error = aa_may_ptrace(cred, tracer, current_cred(), tracee, + AA_PTRACE_TRACE); + put_cred(cred); __end_current_label_crit_section(tracee); return error; @@ -188,7 +193,7 @@ static int apparmor_capable(const struct cred *cred, struct user_namespace *ns, label = aa_get_newest_cred_label(cred); if (!unconfined(label)) - error = aa_capable(label, cap, opts); + error = aa_capable(cred, label, cap, opts); aa_put_label(label); return error; @@ -211,7 +216,8 @@ static int common_perm(const char *op, const struct path *path, u32 mask, label = __begin_current_label_crit_section(); if (!unconfined(label)) - error = aa_path_perm(op, label, path, 0, mask, cond); + error = aa_path_perm(op, current_cred(), label, path, 0, mask, + cond); __end_current_label_crit_section(label); return error; @@ -357,7 +363,8 @@ static int apparmor_path_link(struct dentry *old_dentry, const struct path *new_ label = begin_current_label_crit_section(); if (!unconfined(label)) - error = aa_path_link(label, old_dentry, new_dir, new_dentry); + error = aa_path_link(current_cred(), label, old_dentry, new_dir, + new_dentry); end_current_label_crit_section(label); return error; @@ -396,23 +403,27 @@ static int apparmor_path_rename(const struct path *old_dir, struct dentry *old_d vfsuid = i_uid_into_vfsuid(idmap, d_backing_inode(old_dentry)); cond_exchange.uid = vfsuid_into_kuid(vfsuid); - error = aa_path_perm(OP_RENAME_SRC, label, &new_path, 0, + error = aa_path_perm(OP_RENAME_SRC, current_cred(), + label, &new_path, 0, MAY_READ | AA_MAY_GETATTR | MAY_WRITE | AA_MAY_SETATTR | AA_MAY_DELETE, &cond_exchange); if (!error) - error = aa_path_perm(OP_RENAME_DEST, label, &old_path, + error = aa_path_perm(OP_RENAME_DEST, current_cred(), + label, &old_path, 0, MAY_WRITE | AA_MAY_SETATTR | AA_MAY_CREATE, &cond_exchange); } if (!error) - error = aa_path_perm(OP_RENAME_SRC, label, &old_path, 0, + error = aa_path_perm(OP_RENAME_SRC, current_cred(), + label, &old_path, 0, MAY_READ | AA_MAY_GETATTR | MAY_WRITE | AA_MAY_SETATTR | AA_MAY_DELETE, &cond); if (!error) - error = aa_path_perm(OP_RENAME_DEST, label, &new_path, + error = aa_path_perm(OP_RENAME_DEST, current_cred(), + label, &new_path, 0, MAY_WRITE | AA_MAY_SETATTR | AA_MAY_CREATE, &cond); @@ -467,7 +478,8 @@ static int apparmor_file_open(struct file *file) vfsuid = i_uid_into_vfsuid(idmap, inode); cond.uid = vfsuid_into_kuid(vfsuid); - error = aa_path_perm(OP_OPEN, label, &file->f_path, 0, + error = aa_path_perm(OP_OPEN, file->f_cred, + label, &file->f_path, 0, aa_map_file_to_perms(file), &cond); /* todo cache full allowed permissions set and state */ fctx->allow = aa_map_file_to_perms(file); @@ -507,7 +519,7 @@ static int common_file_perm(const char *op, struct file *file, u32 mask, return -EACCES; label = __begin_current_label_crit_section(); - error = aa_file_perm(op, label, file, mask, in_atomic); + error = aa_file_perm(op, current_cred(), label, file, mask, in_atomic); __end_current_label_crit_section(label); return error; @@ -585,17 +597,21 @@ static int apparmor_sb_mount(const char *dev_name, const struct path *path, label = __begin_current_label_crit_section(); if (!unconfined(label)) { if (flags & MS_REMOUNT) - error = aa_remount(label, path, flags, data); + error = aa_remount(current_cred(), label, path, flags, + data); else if (flags & MS_BIND) - error = aa_bind_mount(label, path, dev_name, flags); + error = aa_bind_mount(current_cred(), label, path, + dev_name, flags); else if (flags & (MS_SHARED | MS_PRIVATE | MS_SLAVE | MS_UNBINDABLE)) - error = aa_mount_change_type(label, path, flags); + error = aa_mount_change_type(current_cred(), label, + path, flags); else if (flags & MS_MOVE) - error = aa_move_mount(label, path, dev_name); + error = aa_move_mount(current_cred(), label, path, + dev_name); else - error = aa_new_mount(label, dev_name, path, type, - flags, data); + error = aa_new_mount(current_cred(), label, dev_name, + path, type, flags, data); } __end_current_label_crit_section(label); @@ -609,7 +625,7 @@ static int apparmor_sb_umount(struct vfsmount *mnt, int flags) label = __begin_current_label_crit_section(); if (!unconfined(label)) - error = aa_umount(label, mnt, flags); + error = aa_umount(current_cred(), label, mnt, flags); __end_current_label_crit_section(label); return error; @@ -623,7 +639,7 @@ static int apparmor_sb_pivotroot(const struct path *old_path, label = aa_get_current_label(); if (!unconfined(label)) - error = aa_pivotroot(label, old_path, new_path); + error = aa_pivotroot(current_cred(), label, old_path, new_path); aa_put_label(label); return error; @@ -785,7 +801,8 @@ static int apparmor_task_setrlimit(struct task_struct *task, int error = 0; if (!unconfined(label)) - error = aa_task_setrlimit(label, task, resource, new_rlim); + error = aa_task_setrlimit(current_cred(), label, task, + resource, new_rlim); __end_current_label_crit_section(label); return error; @@ -794,26 +811,27 @@ static int apparmor_task_setrlimit(struct task_struct *task, static int apparmor_task_kill(struct task_struct *target, struct kernel_siginfo *info, int sig, const struct cred *cred) { + const struct cred *tc; struct aa_label *cl, *tl; int error; + tc = get_task_cred(target); + tl = aa_get_newest_cred_label(tc); if (cred) { /* * Dealing with USB IO specific behavior */ cl = aa_get_newest_cred_label(cred); - tl = aa_get_task_label(target); - error = aa_may_signal(cl, tl, sig); + error = aa_may_signal(cred, cl, tc, tl, sig); aa_put_label(cl); - aa_put_label(tl); return error; + } else { + cl = __begin_current_label_crit_section(); + error = aa_may_signal(current_cred(), cl, tc, tl, sig); + __end_current_label_crit_section(cl); } - - cl = __begin_current_label_crit_section(); - tl = aa_get_task_label(target); - error = aa_may_signal(cl, tl, sig); aa_put_label(tl); - __end_current_label_crit_section(cl); + put_cred(tc); return error; } @@ -879,7 +897,8 @@ static int apparmor_socket_create(int family, int type, int protocol, int kern) if (!(kern || unconfined(label))) error = af_select(family, create_perm(label, family, type, protocol), - aa_af_perm(label, OP_CREATE, AA_MAY_CREATE, + aa_af_perm(current_cred(), label, + OP_CREATE, AA_MAY_CREATE, family, type, protocol)); end_current_label_crit_section(label); diff --git a/security/apparmor/mount.c b/security/apparmor/mount.c index 3830bceff9c8..2bb77aacc49a 100644 --- a/security/apparmor/mount.c +++ b/security/apparmor/mount.c @@ -113,6 +113,7 @@ static void audit_cb(struct audit_buffer *ab, void *va) /** * audit_mount - handle the auditing of mount operations + * @subj_cred: cred of the subject * @profile: the profile being enforced (NOT NULL) * @op: operation being mediated (NOT NULL) * @name: name of object being mediated (MAYBE NULL) @@ -128,7 +129,8 @@ static void audit_cb(struct audit_buffer *ab, void *va) * * Returns: %0 or error on failure */ -static int audit_mount(struct aa_profile *profile, const char *op, +static int audit_mount(const struct cred *subj_cred, + struct aa_profile *profile, const char *op, const char *name, const char *src_name, const char *type, const char *trans, unsigned long flags, const void *data, u32 request, @@ -166,6 +168,7 @@ static int audit_mount(struct aa_profile *profile, const char *op, return error; } + ad.subj_cred = subj_cred; ad.name = name; ad.mnt.src_name = src_name; ad.mnt.type = type; @@ -284,6 +287,7 @@ static int path_flags(struct aa_profile *profile, const struct path *path) /** * match_mnt_path_str - handle path matching for mount + * @subj_cred: cred of confined subject * @profile: the confining profile * @mntpath: for the mntpnt (NOT NULL) * @buffer: buffer to be used to lookup mntpath @@ -296,7 +300,8 @@ static int path_flags(struct aa_profile *profile, const struct path *path) * * Returns: 0 on success else error */ -static int match_mnt_path_str(struct aa_profile *profile, +static int match_mnt_path_str(const struct cred *subj_cred, + struct aa_profile *profile, const struct path *mntpath, char *buffer, const char *devname, const char *type, unsigned long flags, void *data, bool binary, @@ -337,12 +342,14 @@ static int match_mnt_path_str(struct aa_profile *profile, error = 0; audit: - return audit_mount(profile, OP_MOUNT, mntpnt, devname, type, NULL, + return audit_mount(subj_cred, profile, OP_MOUNT, mntpnt, devname, + type, NULL, flags, data, AA_MAY_MOUNT, &perms, info, error); } /** * match_mnt - handle path matching for mount + * @subj_cred: cred of the subject * @profile: the confining profile * @path: for the mntpnt (NOT NULL) * @buffer: buffer to be used to lookup mntpath @@ -355,7 +362,8 @@ audit: * * Returns: 0 on success else error */ -static int match_mnt(struct aa_profile *profile, const struct path *path, +static int match_mnt(const struct cred *subj_cred, + struct aa_profile *profile, const struct path *path, char *buffer, const struct path *devpath, char *devbuffer, const char *type, unsigned long flags, void *data, bool binary) @@ -379,11 +387,12 @@ static int match_mnt(struct aa_profile *profile, const struct path *path, devname = ERR_PTR(error); } - return match_mnt_path_str(profile, path, buffer, devname, type, flags, - data, binary, info); + return match_mnt_path_str(subj_cred, profile, path, buffer, devname, + type, flags, data, binary, info); } -int aa_remount(struct aa_label *label, const struct path *path, +int aa_remount(const struct cred *subj_cred, + struct aa_label *label, const struct path *path, unsigned long flags, void *data) { struct aa_profile *profile; @@ -400,14 +409,16 @@ int aa_remount(struct aa_label *label, const struct path *path, if (!buffer) return -ENOMEM; error = fn_for_each_confined(label, profile, - match_mnt(profile, path, buffer, NULL, NULL, NULL, + match_mnt(subj_cred, profile, path, buffer, NULL, + NULL, NULL, flags, data, binary)); aa_put_buffer(buffer); return error; } -int aa_bind_mount(struct aa_label *label, const struct path *path, +int aa_bind_mount(const struct cred *subj_cred, + struct aa_label *label, const struct path *path, const char *dev_name, unsigned long flags) { struct aa_profile *profile; @@ -434,8 +445,8 @@ int aa_bind_mount(struct aa_label *label, const struct path *path, goto out; error = fn_for_each_confined(label, profile, - match_mnt(profile, path, buffer, &old_path, old_buffer, - NULL, flags, NULL, false)); + match_mnt(subj_cred, profile, path, buffer, &old_path, + old_buffer, NULL, flags, NULL, false)); out: aa_put_buffer(buffer); aa_put_buffer(old_buffer); @@ -444,7 +455,8 @@ out: return error; } -int aa_mount_change_type(struct aa_label *label, const struct path *path, +int aa_mount_change_type(const struct cred *subj_cred, + struct aa_label *label, const struct path *path, unsigned long flags) { struct aa_profile *profile; @@ -462,14 +474,16 @@ int aa_mount_change_type(struct aa_label *label, const struct path *path, if (!buffer) return -ENOMEM; error = fn_for_each_confined(label, profile, - match_mnt(profile, path, buffer, NULL, NULL, NULL, + match_mnt(subj_cred, profile, path, buffer, NULL, + NULL, NULL, flags, NULL, false)); aa_put_buffer(buffer); return error; } -int aa_move_mount(struct aa_label *label, const struct path *path, +int aa_move_mount(const struct cred *subj_cred, + struct aa_label *label, const struct path *path, const char *orig_name) { struct aa_profile *profile; @@ -493,7 +507,8 @@ int aa_move_mount(struct aa_label *label, const struct path *path, if (!buffer || !old_buffer) goto out; error = fn_for_each_confined(label, profile, - match_mnt(profile, path, buffer, &old_path, old_buffer, + match_mnt(subj_cred, profile, path, buffer, &old_path, + old_buffer, NULL, MS_MOVE, NULL, false)); out: aa_put_buffer(buffer); @@ -503,9 +518,9 @@ out: return error; } -int aa_new_mount(struct aa_label *label, const char *dev_name, - const struct path *path, const char *type, unsigned long flags, - void *data) +int aa_new_mount(const struct cred *subj_cred, struct aa_label *label, + const char *dev_name, const struct path *path, + const char *type, unsigned long flags, void *data) { struct aa_profile *profile; char *buffer = NULL, *dev_buffer = NULL; @@ -550,12 +565,14 @@ int aa_new_mount(struct aa_label *label, const char *dev_name, goto out; } error = fn_for_each_confined(label, profile, - match_mnt(profile, path, buffer, dev_path, dev_buffer, + match_mnt(subj_cred, profile, path, buffer, + dev_path, dev_buffer, type, flags, data, binary)); } else { error = fn_for_each_confined(label, profile, - match_mnt_path_str(profile, path, buffer, dev_name, - type, flags, data, binary, NULL)); + match_mnt_path_str(subj_cred, profile, path, + buffer, dev_name, + type, flags, data, binary, NULL)); } out: @@ -567,7 +584,8 @@ out: return error; } -static int profile_umount(struct aa_profile *profile, const struct path *path, +static int profile_umount(const struct cred *subj_cred, + struct aa_profile *profile, const struct path *path, char *buffer) { struct aa_ruleset *rules = list_first_entry(&profile->rules, @@ -596,11 +614,13 @@ static int profile_umount(struct aa_profile *profile, const struct path *path, error = -EACCES; audit: - return audit_mount(profile, OP_UMOUNT, name, NULL, NULL, NULL, 0, NULL, + return audit_mount(subj_cred, profile, OP_UMOUNT, name, NULL, NULL, + NULL, 0, NULL, AA_MAY_UMOUNT, &perms, info, error); } -int aa_umount(struct aa_label *label, struct vfsmount *mnt, int flags) +int aa_umount(const struct cred *subj_cred, struct aa_label *label, + struct vfsmount *mnt, int flags) { struct aa_profile *profile; char *buffer = NULL; @@ -615,7 +635,7 @@ int aa_umount(struct aa_label *label, struct vfsmount *mnt, int flags) return -ENOMEM; error = fn_for_each_confined(label, profile, - profile_umount(profile, &path, buffer)); + profile_umount(subj_cred, profile, &path, buffer)); aa_put_buffer(buffer); return error; @@ -625,7 +645,8 @@ int aa_umount(struct aa_label *label, struct vfsmount *mnt, int flags) * * Returns: label for transition or ERR_PTR. Does not return NULL */ -static struct aa_label *build_pivotroot(struct aa_profile *profile, +static struct aa_label *build_pivotroot(const struct cred *subj_cred, + struct aa_profile *profile, const struct path *new_path, char *new_buffer, const struct path *old_path, @@ -670,7 +691,8 @@ static struct aa_label *build_pivotroot(struct aa_profile *profile, error = 0; audit: - error = audit_mount(profile, OP_PIVOTROOT, new_name, old_name, + error = audit_mount(subj_cred, profile, OP_PIVOTROOT, new_name, + old_name, NULL, trans_name, 0, NULL, AA_MAY_PIVOTROOT, &perms, info, error); if (error) @@ -679,7 +701,8 @@ audit: return aa_get_newest_label(&profile->label); } -int aa_pivotroot(struct aa_label *label, const struct path *old_path, +int aa_pivotroot(const struct cred *subj_cred, struct aa_label *label, + const struct path *old_path, const struct path *new_path) { struct aa_profile *profile; @@ -697,7 +720,8 @@ int aa_pivotroot(struct aa_label *label, const struct path *old_path, if (!old_buffer || !new_buffer) goto out; target = fn_label_build(label, profile, GFP_KERNEL, - build_pivotroot(profile, new_path, new_buffer, + build_pivotroot(subj_cred, profile, new_path, + new_buffer, old_path, old_buffer)); if (!target) { info = "label build failed"; @@ -723,7 +747,8 @@ out: fail: /* TODO: add back in auditing of new_name and old_name */ error = fn_for_each(label, profile, - audit_mount(profile, OP_PIVOTROOT, NULL /*new_name */, + audit_mount(subj_cred, profile, OP_PIVOTROOT, + NULL /*new_name */, NULL /* old_name */, NULL, NULL, 0, NULL, AA_MAY_PIVOTROOT, &nullperms, info, diff --git a/security/apparmor/net.c b/security/apparmor/net.c index 5e50f80e35db..704c171232ab 100644 --- a/security/apparmor/net.c +++ b/security/apparmor/net.c @@ -135,8 +135,8 @@ int aa_profile_af_perm(struct aa_profile *profile, return aa_check_perms(profile, &perms, request, ad, audit_net_cb); } -int aa_af_perm(struct aa_label *label, const char *op, u32 request, u16 family, - int type, int protocol) +int aa_af_perm(const struct cred *subj_cred, struct aa_label *label, + const char *op, u32 request, u16 family, int type, int protocol) { struct aa_profile *profile; DEFINE_AUDIT_NET(ad, op, NULL, family, type, protocol); @@ -146,7 +146,9 @@ int aa_af_perm(struct aa_label *label, const char *op, u32 request, u16 family, type)); } -static int aa_label_sk_perm(struct aa_label *label, const char *op, u32 request, +static int aa_label_sk_perm(const struct cred *subj_cred, + struct aa_label *label, + const char *op, u32 request, struct sock *sk) { struct aa_sk_ctx *ctx = SK_CTX(sk); @@ -159,6 +161,7 @@ static int aa_label_sk_perm(struct aa_label *label, const char *op, u32 request, struct aa_profile *profile; DEFINE_AUDIT_SK(ad, op, sk); + ad.subj_cred = subj_cred; error = fn_for_each_confined(label, profile, aa_profile_af_sk_perm(profile, &ad, request, sk)); } @@ -176,21 +179,21 @@ int aa_sk_perm(const char *op, u32 request, struct sock *sk) /* TODO: switch to begin_current_label ???? */ label = begin_current_label_crit_section(); - error = aa_label_sk_perm(label, op, request, sk); + error = aa_label_sk_perm(current_cred(), label, op, request, sk); end_current_label_crit_section(label); return error; } -int aa_sock_file_perm(struct aa_label *label, const char *op, u32 request, - struct socket *sock) +int aa_sock_file_perm(const struct cred *subj_cred, struct aa_label *label, + const char *op, u32 request, struct socket *sock) { AA_BUG(!label); AA_BUG(!sock); AA_BUG(!sock->sk); - return aa_label_sk_perm(label, op, request, sock->sk); + return aa_label_sk_perm(subj_cred, label, op, request, sock->sk); } #ifdef CONFIG_NETWORK_SECMARK diff --git a/security/apparmor/policy.c b/security/apparmor/policy.c index d78970d4a851..6f80e1207d24 100644 --- a/security/apparmor/policy.c +++ b/security/apparmor/policy.c @@ -762,21 +762,23 @@ static int audit_policy(struct aa_label *subj_label, const char *op, /* don't call out to other LSMs in the stack for apparmor policy admin * permissions */ -static int policy_ns_capable(struct aa_label *label, +static int policy_ns_capable(const struct cred *subj_cred, + struct aa_label *label, struct user_namespace *userns, int cap) { int err; /* check for MAC_ADMIN cap in cred */ - err = cap_capable(current_cred(), userns, cap, CAP_OPT_NONE); + err = cap_capable(subj_cred, userns, cap, CAP_OPT_NONE); if (!err) - err = aa_capable(label, cap, CAP_OPT_NONE); + err = aa_capable(subj_cred, label, cap, CAP_OPT_NONE); return err; } /** * aa_policy_view_capable - check if viewing policy in at @ns is allowed + * @subj_cred: cred of subject * @label: label that is trying to view policy in ns * @ns: namespace being viewed by @label (may be NULL if @label's ns) * @@ -785,9 +787,10 @@ static int policy_ns_capable(struct aa_label *label, * If @ns is NULL then the namespace being viewed is assumed to be the * tasks current namespace. */ -bool aa_policy_view_capable(struct aa_label *label, struct aa_ns *ns) +bool aa_policy_view_capable(const struct cred *subj_cred, + struct aa_label *label, struct aa_ns *ns) { - struct user_namespace *user_ns = current_user_ns(); + struct user_namespace *user_ns = subj_cred->user_ns; struct aa_ns *view_ns = labels_view(label); bool root_in_user_ns = uid_eq(current_euid(), make_kuid(user_ns, 0)) || in_egroup_p(make_kgid(user_ns, 0)); @@ -804,15 +807,17 @@ bool aa_policy_view_capable(struct aa_label *label, struct aa_ns *ns) return response; } -bool aa_policy_admin_capable(struct aa_label *label, struct aa_ns *ns) +bool aa_policy_admin_capable(const struct cred *subj_cred, + struct aa_label *label, struct aa_ns *ns) { - struct user_namespace *user_ns = current_user_ns(); - bool capable = policy_ns_capable(label, user_ns, CAP_MAC_ADMIN) == 0; + struct user_namespace *user_ns = subj_cred->user_ns; + bool capable = policy_ns_capable(subj_cred, label, user_ns, + CAP_MAC_ADMIN) == 0; AA_DEBUG("cap_mac_admin? %d\n", capable); AA_DEBUG("policy locked? %d\n", aa_g_lock_policy); - return aa_policy_view_capable(label, ns) && capable && + return aa_policy_view_capable(subj_cred, label, ns) && capable && !aa_g_lock_policy; } @@ -822,7 +827,7 @@ bool aa_current_policy_view_capable(struct aa_ns *ns) bool res; label = __begin_current_label_crit_section(); - res = aa_policy_view_capable(label, ns); + res = aa_policy_view_capable(current_cred(), label, ns); __end_current_label_crit_section(label); return res; @@ -834,7 +839,7 @@ bool aa_current_policy_admin_capable(struct aa_ns *ns) bool res; label = __begin_current_label_crit_section(); - res = aa_policy_admin_capable(label, ns); + res = aa_policy_admin_capable(current_cred(), label, ns); __end_current_label_crit_section(label); return res; @@ -842,13 +847,15 @@ bool aa_current_policy_admin_capable(struct aa_ns *ns) /** * aa_may_manage_policy - can the current task manage policy + * @subj_cred; subjects cred * @label: label to check if it can manage policy * @ns: namespace being managed by @label (may be NULL if @label's ns) * @mask: contains the policy manipulation operation being done * * Returns: 0 if the task is allowed to manipulate policy else error */ -int aa_may_manage_policy(struct aa_label *label, struct aa_ns *ns, u32 mask) +int aa_may_manage_policy(const struct cred *subj_cred, struct aa_label *label, + struct aa_ns *ns, u32 mask) { const char *op; @@ -864,7 +871,7 @@ int aa_may_manage_policy(struct aa_label *label, struct aa_ns *ns, u32 mask) return audit_policy(label, op, NULL, NULL, "policy_locked", -EACCES); - if (!aa_policy_admin_capable(label, ns)) + if (!aa_policy_admin_capable(subj_cred, label, ns)) return audit_policy(label, op, NULL, NULL, "not policy admin", -EACCES); diff --git a/security/apparmor/resource.c b/security/apparmor/resource.c index 73ba26c646a5..dcc94c3153d5 100644 --- a/security/apparmor/resource.c +++ b/security/apparmor/resource.c @@ -43,6 +43,7 @@ static void audit_cb(struct audit_buffer *ab, void *va) /** * audit_resource - audit setting resource limit + * @subj_cred: cred setting the resource * @profile: profile being enforced (NOT NULL) * @resource: rlimit being auditing * @value: value being set @@ -52,13 +53,15 @@ static void audit_cb(struct audit_buffer *ab, void *va) * * Returns: 0 or ad->error else other error code on failure */ -static int audit_resource(struct aa_profile *profile, unsigned int resource, +static int audit_resource(const struct cred *subj_cred, + struct aa_profile *profile, unsigned int resource, unsigned long value, struct aa_label *peer, const char *info, int error) { DEFINE_AUDIT_DATA(ad, LSM_AUDIT_DATA_NONE, AA_CLASS_RLIMITS, OP_SETRLIMIT); + ad.subj_cred = subj_cred; ad.rlim.rlim = resource; ad.rlim.max = value; ad.peer = peer; @@ -82,7 +85,8 @@ int aa_map_resource(int resource) return rlim_map[resource]; } -static int profile_setrlimit(struct aa_profile *profile, unsigned int resource, +static int profile_setrlimit(const struct cred *subj_cred, + struct aa_profile *profile, unsigned int resource, struct rlimit *new_rlim) { struct aa_ruleset *rules = list_first_entry(&profile->rules, @@ -92,12 +96,13 @@ static int profile_setrlimit(struct aa_profile *profile, unsigned int resource, if (rules->rlimits.mask & (1 << resource) && new_rlim->rlim_max > rules->rlimits.limits[resource].rlim_max) e = -EACCES; - return audit_resource(profile, resource, new_rlim->rlim_max, NULL, NULL, - e); + return audit_resource(subj_cred, profile, resource, new_rlim->rlim_max, + NULL, NULL, e); } /** * aa_task_setrlimit - test permission to set an rlimit + * @subj_cred: cred setting the limit * @label: label confining the task (NOT NULL) * @task: task the resource is being set on * @resource: the resource being set @@ -107,7 +112,8 @@ static int profile_setrlimit(struct aa_profile *profile, unsigned int resource, * * Returns: 0 or error code if setting resource failed */ -int aa_task_setrlimit(struct aa_label *label, struct task_struct *task, +int aa_task_setrlimit(const struct cred *subj_cred, struct aa_label *label, + struct task_struct *task, unsigned int resource, struct rlimit *new_rlim) { struct aa_profile *profile; @@ -126,14 +132,15 @@ int aa_task_setrlimit(struct aa_label *label, struct task_struct *task, */ if (label != peer && - aa_capable(label, CAP_SYS_RESOURCE, CAP_OPT_NOAUDIT) != 0) + aa_capable(subj_cred, label, CAP_SYS_RESOURCE, CAP_OPT_NOAUDIT) != 0) error = fn_for_each(label, profile, - audit_resource(profile, resource, + audit_resource(subj_cred, profile, resource, new_rlim->rlim_max, peer, "cap_sys_resource", -EACCES)); else error = fn_for_each_confined(label, profile, - profile_setrlimit(profile, resource, new_rlim)); + profile_setrlimit(subj_cred, profile, resource, + new_rlim)); aa_put_label(peer); return error; diff --git a/security/apparmor/task.c b/security/apparmor/task.c index 552442844ff8..1a7c9d02e31d 100644 --- a/security/apparmor/task.c +++ b/security/apparmor/task.c @@ -223,14 +223,16 @@ static void audit_ptrace_cb(struct audit_buffer *ab, void *va) /* assumes check for RULE_MEDIATES is already done */ /* TODO: conditionals */ -static int profile_ptrace_perm(struct aa_profile *profile, - struct aa_label *peer, u32 request, - struct apparmor_audit_data *ad) +static int profile_ptrace_perm(const struct cred *cred, + struct aa_profile *profile, + struct aa_label *peer, u32 request, + struct apparmor_audit_data *ad) { struct aa_ruleset *rules = list_first_entry(&profile->rules, typeof(*rules), list); struct aa_perms perms = { }; + ad->subj_cred = cred; ad->peer = peer; aa_profile_match_label(profile, rules, peer, AA_CLASS_PTRACE, request, &perms); @@ -238,7 +240,8 @@ static int profile_ptrace_perm(struct aa_profile *profile, return aa_check_perms(profile, &perms, request, ad, audit_ptrace_cb); } -static int profile_tracee_perm(struct aa_profile *tracee, +static int profile_tracee_perm(const struct cred *cred, + struct aa_profile *tracee, struct aa_label *tracer, u32 request, struct apparmor_audit_data *ad) { @@ -246,10 +249,11 @@ static int profile_tracee_perm(struct aa_profile *tracee, !ANY_RULE_MEDIATES(&tracee->rules, AA_CLASS_PTRACE)) return 0; - return profile_ptrace_perm(tracee, tracer, request, ad); + return profile_ptrace_perm(cred, tracee, tracer, request, ad); } -static int profile_tracer_perm(struct aa_profile *tracer, +static int profile_tracer_perm(const struct cred *cred, + struct aa_profile *tracer, struct aa_label *tracee, u32 request, struct apparmor_audit_data *ad) { @@ -257,7 +261,7 @@ static int profile_tracer_perm(struct aa_profile *tracer, return 0; if (ANY_RULE_MEDIATES(&tracer->rules, AA_CLASS_PTRACE)) - return profile_ptrace_perm(tracer, tracee, request, ad); + return profile_ptrace_perm(cred, tracer, tracee, request, ad); /* profile uses the old style capability check for ptrace */ if (&tracer->label == tracee) @@ -266,8 +270,8 @@ static int profile_tracer_perm(struct aa_profile *tracer, ad->subj_label = &tracer->label; ad->peer = tracee; ad->request = 0; - ad->error = aa_capable(&tracer->label, CAP_SYS_PTRACE, - CAP_OPT_NONE); + ad->error = aa_capable(cred, &tracer->label, CAP_SYS_PTRACE, + CAP_OPT_NONE); return aa_audit(AUDIT_APPARMOR_AUTO, tracer, ad, audit_ptrace_cb); } @@ -280,7 +284,8 @@ static int profile_tracer_perm(struct aa_profile *tracer, * * Returns: %0 else error code if permission denied or error */ -int aa_may_ptrace(struct aa_label *tracer, struct aa_label *tracee, +int aa_may_ptrace(const struct cred *tracer_cred, struct aa_label *tracer, + const struct cred *tracee_cred, struct aa_label *tracee, u32 request) { struct aa_profile *profile; @@ -288,6 +293,8 @@ int aa_may_ptrace(struct aa_label *tracer, struct aa_label *tracee, DEFINE_AUDIT_DATA(sa, LSM_AUDIT_DATA_NONE, AA_CLASS_PTRACE, OP_PTRACE); return xcheck_labels(tracer, tracee, profile, - profile_tracer_perm(profile, tracee, request, &sa), - profile_tracee_perm(profile, tracer, xrequest, &sa)); + profile_tracer_perm(tracer_cred, profile, tracee, + request, &sa), + profile_tracee_perm(tracee_cred, profile, tracer, + xrequest, &sa)); } From 75c77e9e0713fddbe99a21a036aa6482402f9e34 Mon Sep 17 00:00:00 2001 From: John Johansen Date: Mon, 17 Apr 2023 03:27:36 -0700 Subject: [PATCH 265/475] apparmor: provide separate audit messages for file and policy checks Improve policy load failure messages by identifying which dfa the verification check failed in. Reviewed-by: Georgia Garcia Signed-off-by: John Johansen --- security/apparmor/policy_unpack.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/security/apparmor/policy_unpack.c b/security/apparmor/policy_unpack.c index cb8b5c497812..1eb98d6994e8 100644 --- a/security/apparmor/policy_unpack.c +++ b/security/apparmor/policy_unpack.c @@ -1240,12 +1240,18 @@ static int verify_profile(struct aa_profile *profile) if (!rules) return 0; - if ((rules->file.dfa && !verify_dfa_accept_index(rules->file.dfa, - rules->file.size)) || - (rules->policy.dfa && - !verify_dfa_accept_index(rules->policy.dfa, rules->policy.size))) { + if (rules->file.dfa && !verify_dfa_accept_index(rules->file.dfa, + rules->file.size)) { audit_iface(profile, NULL, NULL, - "Unpack: Invalid named transition", NULL, -EPROTO); + "Unpack: file Invalid named transition", NULL, + -EPROTO); + return -EPROTO; + } + if (rules->policy.dfa && + !verify_dfa_accept_index(rules->policy.dfa, rules->policy.size)) { + audit_iface(profile, NULL, NULL, + "Unpack: policy Invalid named transition", NULL, + -EPROTO); return -EPROTO; } From 98b824ff8984fd523fc264fbb13208098ab09da3 Mon Sep 17 00:00:00 2001 From: John Johansen Date: Fri, 28 Apr 2023 05:32:52 -0700 Subject: [PATCH 266/475] apparmor: refcount the pdb With the move to permission tables the dfa is no longer a stand alone entity when used, needing a minimum of a permission table. However it still could be shared among different pdbs each using a different permission table. Instead of duping the permission table when sharing a pdb, add a refcount to the pdb so it can be easily shared. Reviewed-by: Georgia Garcia Signed-off-by: John Johansen --- security/apparmor/apparmorfs.c | 18 ++--- security/apparmor/domain.c | 60 ++++++++-------- security/apparmor/file.c | 12 ++-- security/apparmor/include/lib.h | 2 + security/apparmor/include/match.h | 6 -- security/apparmor/include/policy.h | 51 ++++++++++---- security/apparmor/ipc.c | 4 +- security/apparmor/label.c | 18 ++--- security/apparmor/lib.c | 4 +- security/apparmor/lsm.c | 63 +++++++++++++++++ security/apparmor/match.c | 44 ------------ security/apparmor/mount.c | 20 +++--- security/apparmor/net.c | 4 +- security/apparmor/policy.c | 58 +++++++++++----- security/apparmor/policy_unpack.c | 108 +++++++++++++---------------- 15 files changed, 261 insertions(+), 211 deletions(-) diff --git a/security/apparmor/apparmorfs.c b/security/apparmor/apparmorfs.c index f20d923944e4..0bba46ccdb71 100644 --- a/security/apparmor/apparmorfs.c +++ b/security/apparmor/apparmorfs.c @@ -619,23 +619,23 @@ static void profile_query_cb(struct aa_profile *profile, struct aa_perms *perms, if (profile_unconfined(profile)) return; - if (rules->file.dfa && *match_str == AA_CLASS_FILE) { - state = aa_dfa_match_len(rules->file.dfa, - rules->file.start[AA_CLASS_FILE], + if (rules->file->dfa && *match_str == AA_CLASS_FILE) { + state = aa_dfa_match_len(rules->file->dfa, + rules->file->start[AA_CLASS_FILE], match_str + 1, match_len - 1); if (state) { struct path_cond cond = { }; - tmp = *(aa_lookup_fperms(&(rules->file), state, &cond)); + tmp = *(aa_lookup_fperms(rules->file, state, &cond)); } - } else if (rules->policy.dfa) { + } else if (rules->policy->dfa) { if (!RULE_MEDIATES(rules, *match_str)) return; /* no change to current perms */ - state = aa_dfa_match_len(rules->policy.dfa, - rules->policy.start[0], + state = aa_dfa_match_len(rules->policy->dfa, + rules->policy->start[0], match_str, match_len); if (state) - tmp = *aa_lookup_perms(&rules->policy, state); + tmp = *aa_lookup_perms(rules->policy, state); } aa_apply_modes_to_perms(profile, &tmp); aa_perms_accum_raw(perms, &tmp); @@ -1096,7 +1096,7 @@ static int seq_profile_attach_show(struct seq_file *seq, void *v) struct aa_profile *profile = labels_profile(label); if (profile->attach.xmatch_str) seq_printf(seq, "%s\n", profile->attach.xmatch_str); - else if (profile->attach.xmatch.dfa) + else if (profile->attach.xmatch->dfa) seq_puts(seq, "\n"); else seq_printf(seq, "%s\n", profile->base.name); diff --git a/security/apparmor/domain.c b/security/apparmor/domain.c index d95292a23bcf..87dfa0e40398 100644 --- a/security/apparmor/domain.c +++ b/security/apparmor/domain.c @@ -77,7 +77,7 @@ out: /**** TODO: dedup to aa_label_match - needs perm and dfa, merging * specifically this is an exact copy of aa_label_match except * aa_compute_perms is replaced with aa_compute_fperms - * and policy.dfa with file.dfa + * and policy->dfa with file->dfa ****/ /* match a profile and its associated ns component if needed * Assumes visibility test has already been done. @@ -93,16 +93,16 @@ static inline aa_state_t match_component(struct aa_profile *profile, const char *ns_name; if (stack) - state = aa_dfa_match(rules->file.dfa, state, "&"); + state = aa_dfa_match(rules->file->dfa, state, "&"); if (profile->ns == tp->ns) - return aa_dfa_match(rules->file.dfa, state, tp->base.hname); + return aa_dfa_match(rules->file->dfa, state, tp->base.hname); /* try matching with namespace name and then profile */ ns_name = aa_ns_name(profile->ns, tp->ns, true); - state = aa_dfa_match_len(rules->file.dfa, state, ":", 1); - state = aa_dfa_match(rules->file.dfa, state, ns_name); - state = aa_dfa_match_len(rules->file.dfa, state, ":", 1); - return aa_dfa_match(rules->file.dfa, state, tp->base.hname); + state = aa_dfa_match_len(rules->file->dfa, state, ":", 1); + state = aa_dfa_match(rules->file->dfa, state, ns_name); + state = aa_dfa_match_len(rules->file->dfa, state, ":", 1); + return aa_dfa_match(rules->file->dfa, state, tp->base.hname); } /** @@ -150,12 +150,12 @@ next: label_for_each_cont(i, label, tp) { if (!aa_ns_visible(profile->ns, tp->ns, subns)) continue; - state = aa_dfa_match(rules->file.dfa, state, "//&"); + state = aa_dfa_match(rules->file->dfa, state, "//&"); state = match_component(profile, tp, false, state); if (!state) goto fail; } - *perms = *(aa_lookup_fperms(&(rules->file), state, &cond)); + *perms = *(aa_lookup_fperms(rules->file, state, &cond)); aa_apply_modes_to_perms(profile, perms); if ((perms->allow & request) != request) return -EACCES; @@ -210,7 +210,7 @@ static int label_components_match(struct aa_profile *profile, return 0; next: - tmp = *(aa_lookup_fperms(&(rules->file), state, &cond)); + tmp = *(aa_lookup_fperms(rules->file, state, &cond)); aa_apply_modes_to_perms(profile, &tmp); aa_perms_accum(perms, &tmp); label_for_each_cont(i, label, tp) { @@ -219,7 +219,7 @@ next: state = match_component(profile, tp, stack, start); if (!state) goto fail; - tmp = *(aa_lookup_fperms(&(rules->file), state, &cond)); + tmp = *(aa_lookup_fperms(rules->file, state, &cond)); aa_apply_modes_to_perms(profile, &tmp); aa_perms_accum(perms, &tmp); } @@ -317,7 +317,7 @@ static int aa_xattrs_match(const struct linux_binprm *bprm, might_sleep(); /* transition from exec match to xattr set */ - state = aa_dfa_outofband_transition(attach->xmatch.dfa, state); + state = aa_dfa_outofband_transition(attach->xmatch->dfa, state); d = bprm->file->f_path.dentry; for (i = 0; i < attach->xattr_count; i++) { @@ -331,20 +331,20 @@ static int aa_xattrs_match(const struct linux_binprm *bprm, * that not present xattr can be distinguished from a 0 * length value or rule that matches any value */ - state = aa_dfa_null_transition(attach->xmatch.dfa, + state = aa_dfa_null_transition(attach->xmatch->dfa, state); /* Check xattr value */ - state = aa_dfa_match_len(attach->xmatch.dfa, state, + state = aa_dfa_match_len(attach->xmatch->dfa, state, value, size); - index = ACCEPT_TABLE(attach->xmatch.dfa)[state]; - perm = attach->xmatch.perms[index].allow; + index = ACCEPT_TABLE(attach->xmatch->dfa)[state]; + perm = attach->xmatch->perms[index].allow; if (!(perm & MAY_EXEC)) { ret = -EINVAL; goto out; } } /* transition to next element */ - state = aa_dfa_outofband_transition(attach->xmatch.dfa, state); + state = aa_dfa_outofband_transition(attach->xmatch->dfa, state); if (size < 0) { /* * No xattr match, so verify if transition to @@ -413,16 +413,16 @@ restart: * as another profile, signal a conflict and refuse to * match. */ - if (attach->xmatch.dfa) { + if (attach->xmatch->dfa) { unsigned int count; aa_state_t state; u32 index, perm; - state = aa_dfa_leftmatch(attach->xmatch.dfa, - attach->xmatch.start[AA_CLASS_XMATCH], + state = aa_dfa_leftmatch(attach->xmatch->dfa, + attach->xmatch->start[AA_CLASS_XMATCH], name, &count); - index = ACCEPT_TABLE(attach->xmatch.dfa)[state]; - perm = attach->xmatch.perms[index].allow; + index = ACCEPT_TABLE(attach->xmatch->dfa)[state]; + perm = attach->xmatch->perms[index].allow; /* any accepting state means a valid match. */ if (perm & MAY_EXEC) { int ret = 0; @@ -525,7 +525,7 @@ struct aa_label *x_table_lookup(struct aa_profile *profile, u32 xindex, /* TODO: move lookup parsing to unpack time so this is a straight * index into the resultant label */ - for (*name = rules->file.trans.table[index]; !label && *name; + for (*name = rules->file->trans.table[index]; !label && *name; *name = next_name(xtype, *name)) { if (xindex & AA_X_CHILD) { struct aa_profile *new_profile; @@ -579,7 +579,7 @@ static struct aa_label *x_to_label(struct aa_profile *profile, break; case AA_X_TABLE: /* TODO: fix when perm mapping done at unload */ - stack = rules->file.trans.table[xindex & AA_X_INDEX_MASK]; + stack = rules->file->trans.table[xindex & AA_X_INDEX_MASK]; if (*stack != '&') { /* released by caller */ new = x_table_lookup(profile, xindex, lookupname); @@ -638,7 +638,7 @@ static struct aa_label *profile_transition(const struct cred *subj_cred, typeof(*rules), list); struct aa_label *new = NULL; const char *info = NULL, *name = NULL, *target = NULL; - aa_state_t state = rules->file.start[AA_CLASS_FILE]; + aa_state_t state = rules->file->start[AA_CLASS_FILE]; struct aa_perms perms = {}; bool nonewprivs = false; int error = 0; @@ -672,7 +672,7 @@ static struct aa_label *profile_transition(const struct cred *subj_cred, } /* find exec permissions for name */ - state = aa_str_perms(&(rules->file), state, name, cond, &perms); + state = aa_str_perms(rules->file, state, name, cond, &perms); if (perms.allow & MAY_EXEC) { /* exec permission determine how to transition */ new = x_to_label(profile, bprm, name, perms.xindex, &target, @@ -738,7 +738,7 @@ static int profile_onexec(const struct cred *subj_cred, { struct aa_ruleset *rules = list_first_entry(&profile->rules, typeof(*rules), list); - aa_state_t state = rules->file.start[AA_CLASS_FILE]; + aa_state_t state = rules->file->start[AA_CLASS_FILE]; struct aa_perms perms = {}; const char *xname = NULL, *info = "change_profile onexec"; int error = -EACCES; @@ -771,7 +771,7 @@ static int profile_onexec(const struct cred *subj_cred, } /* find exec permissions for name */ - state = aa_str_perms(&(rules->file), state, xname, cond, &perms); + state = aa_str_perms(rules->file, state, xname, cond, &perms); if (!(perms.allow & AA_MAY_ONEXEC)) { info = "no change_onexec valid for executable"; goto audit; @@ -780,7 +780,7 @@ static int profile_onexec(const struct cred *subj_cred, * onexec permission is linked to exec with a standard pairing * exec\0change_profile */ - state = aa_dfa_null_transition(rules->file.dfa, state); + state = aa_dfa_null_transition(rules->file->dfa, state); error = change_profile_perms(profile, onexec, stack, AA_MAY_ONEXEC, state, &perms); if (error) { @@ -1300,7 +1300,7 @@ static int change_profile_perms_wrapper(const char *op, const char *name, if (!error) error = change_profile_perms(profile, target, stack, request, - rules->file.start[AA_CLASS_FILE], + rules->file->start[AA_CLASS_FILE], perms); if (error) error = aa_audit_file(subj_cred, profile, perms, op, request, diff --git a/security/apparmor/file.c b/security/apparmor/file.c index 48afcef45694..c03eb7c19f16 100644 --- a/security/apparmor/file.c +++ b/security/apparmor/file.c @@ -236,7 +236,7 @@ static int __aa_path_perm(const char *op, const struct cred *subj_cred, if (profile_unconfined(profile)) return 0; - aa_str_perms(&(rules->file), rules->file.start[AA_CLASS_FILE], + aa_str_perms(rules->file, rules->file->start[AA_CLASS_FILE], name, cond, perms); if (request & ~perms->allow) e = -EACCES; @@ -353,16 +353,16 @@ static int profile_path_link(const struct cred *subj_cred, error = -EACCES; /* aa_str_perms - handles the case of the dfa being NULL */ - state = aa_str_perms(&(rules->file), - rules->file.start[AA_CLASS_FILE], lname, + state = aa_str_perms(rules->file, + rules->file->start[AA_CLASS_FILE], lname, cond, &lperms); if (!(lperms.allow & AA_MAY_LINK)) goto audit; /* test to see if target can be paired with link */ - state = aa_dfa_null_transition(rules->file.dfa, state); - aa_str_perms(&(rules->file), state, tname, cond, &perms); + state = aa_dfa_null_transition(rules->file->dfa, state); + aa_str_perms(rules->file, state, tname, cond, &perms); /* force audit/quiet masks for link are stored in the second entry * in the link pair. @@ -384,7 +384,7 @@ static int profile_path_link(const struct cred *subj_cred, /* Do link perm subset test requiring allowed permission on link are * a subset of the allowed permissions on target. */ - aa_str_perms(&(rules->file), rules->file.start[AA_CLASS_FILE], + aa_str_perms(rules->file, rules->file->start[AA_CLASS_FILE], tname, cond, &perms); /* AA_MAY_LINK is not considered in the subset test */ diff --git a/security/apparmor/include/lib.h b/security/apparmor/include/lib.h index 73c8a32c6861..d7a894b1031f 100644 --- a/security/apparmor/include/lib.h +++ b/security/apparmor/include/lib.h @@ -16,6 +16,8 @@ #include "match.h" +extern struct aa_dfa *stacksplitdfa; + /* * DEBUG remains global (no per profile flag) since it is mostly used in sysctl * which is not related to profile accesses. diff --git a/security/apparmor/include/match.h b/security/apparmor/include/match.h index 58fbf67139b9..4bb0405c9190 100644 --- a/security/apparmor/include/match.h +++ b/security/apparmor/include/match.h @@ -102,9 +102,6 @@ struct aa_dfa { struct table_header *tables[YYTD_ID_TSIZE]; }; -extern struct aa_dfa *nulldfa; -extern struct aa_dfa *stacksplitdfa; - #define byte_to_byte(X) (X) #define UNPACK_ARRAY(TABLE, BLOB, LEN, TTYPE, BTYPE, NTOHX) \ @@ -122,9 +119,6 @@ static inline size_t table_size(size_t len, size_t el_size) return ALIGN(sizeof(struct table_header) + len * el_size, 8); } -int aa_setup_dfa_engine(void); -void aa_teardown_dfa_engine(void); - #define aa_state_t unsigned int struct aa_dfa *aa_dfa_unpack(void *blob, size_t size, int flags); diff --git a/security/apparmor/include/policy.h b/security/apparmor/include/policy.h index 5572447d7c37..e69c91619a7c 100644 --- a/security/apparmor/include/policy.h +++ b/security/apparmor/include/policy.h @@ -74,12 +74,14 @@ enum profile_mode { /* struct aa_policydb - match engine for a policy + * count: refcount for the pdb * dfa: dfa pattern match * perms: table of permissions * strs: table of strings, index by x * start: set of start states for the different classes of data */ struct aa_policydb { + struct kref count; struct aa_dfa *dfa; struct { struct aa_perms *perms; @@ -89,13 +91,36 @@ struct aa_policydb { aa_state_t start[AA_CLASS_LAST + 1]; }; -static inline void aa_destroy_policydb(struct aa_policydb *policy) -{ - aa_put_dfa(policy->dfa); - if (policy->perms) - kvfree(policy->perms); - aa_free_str_table(&policy->trans); +extern struct aa_policydb *nullpdb; +struct aa_policydb *aa_alloc_pdb(gfp_t gfp); +void aa_pdb_free_kref(struct kref *kref); + +/** + * aa_get_pdb - increment refcount on @pdb + * @pdb: policydb (MAYBE NULL) + * + * Returns: pointer to @pdb if @pdb is NULL will return NULL + * Requires: @pdb must be held with valid refcount when called + */ +static inline struct aa_policydb *aa_get_pdb(struct aa_policydb *pdb) +{ + if (pdb) + kref_get(&(pdb->count)); + + return pdb; +} + +/** + * aa_put_pdb - put a pdb refcount + * @pdb: pdb to put refcount (MAYBE NULL) + * + * Requires: if @pdb != NULL that a valid refcount be held + */ +static inline void aa_put_pdb(struct aa_policydb *pdb) +{ + if (pdb) + kref_put(&pdb->count, aa_pdb_free_kref); } static inline struct aa_perms *aa_lookup_perms(struct aa_policydb *policy, @@ -139,8 +164,8 @@ struct aa_ruleset { int size; /* TODO: merge policy and file */ - struct aa_policydb policy; - struct aa_policydb file; + struct aa_policydb *policy; + struct aa_policydb *file; struct aa_caps caps; struct aa_rlimit rlimits; @@ -159,7 +184,7 @@ struct aa_ruleset { */ struct aa_attachment { const char *xmatch_str; - struct aa_policydb xmatch; + struct aa_policydb *xmatch; unsigned int xmatch_len; int xattr_count; char **xattrs; @@ -267,10 +292,10 @@ static inline aa_state_t RULE_MEDIATES(struct aa_ruleset *rules, unsigned char class) { if (class <= AA_CLASS_LAST) - return rules->policy.start[class]; + return rules->policy->start[class]; else - return aa_dfa_match_len(rules->policy.dfa, - rules->policy.start[0], &class, 1); + return aa_dfa_match_len(rules->policy->dfa, + rules->policy->start[0], &class, 1); } static inline aa_state_t RULE_MEDIATES_AF(struct aa_ruleset *rules, u16 AF) @@ -280,7 +305,7 @@ static inline aa_state_t RULE_MEDIATES_AF(struct aa_ruleset *rules, u16 AF) if (!state) return DFA_NOMATCH; - return aa_dfa_match_len(rules->policy.dfa, state, (char *) &be_af, 2); + return aa_dfa_match_len(rules->policy->dfa, state, (char *) &be_af, 2); } static inline aa_state_t ANY_RULE_MEDIATES(struct list_head *head, diff --git a/security/apparmor/ipc.c b/security/apparmor/ipc.c index c0d0dbd7b4c4..0cdf4340b02d 100644 --- a/security/apparmor/ipc.c +++ b/security/apparmor/ipc.c @@ -92,8 +92,8 @@ static int profile_signal_perm(const struct cred *cred, ad->subj_cred = cred; ad->peer = peer; /* TODO: secondary cache check */ - state = aa_dfa_next(rules->policy.dfa, - rules->policy.start[AA_CLASS_SIGNAL], + state = aa_dfa_next(rules->policy->dfa, + rules->policy->start[AA_CLASS_SIGNAL], ad->signal); aa_label_match(profile, rules, peer, state, false, request, &perms); aa_apply_modes_to_perms(profile, &perms); diff --git a/security/apparmor/label.c b/security/apparmor/label.c index 8a74e89b05f4..c71e4615dd46 100644 --- a/security/apparmor/label.c +++ b/security/apparmor/label.c @@ -1270,14 +1270,14 @@ static inline aa_state_t match_component(struct aa_profile *profile, const char *ns_name; if (profile->ns == tp->ns) - return aa_dfa_match(rules->policy.dfa, state, tp->base.hname); + return aa_dfa_match(rules->policy->dfa, state, tp->base.hname); /* try matching with namespace name and then profile */ ns_name = aa_ns_name(profile->ns, tp->ns, true); - state = aa_dfa_match_len(rules->policy.dfa, state, ":", 1); - state = aa_dfa_match(rules->policy.dfa, state, ns_name); - state = aa_dfa_match_len(rules->policy.dfa, state, ":", 1); - return aa_dfa_match(rules->policy.dfa, state, tp->base.hname); + state = aa_dfa_match_len(rules->policy->dfa, state, ":", 1); + state = aa_dfa_match(rules->policy->dfa, state, ns_name); + state = aa_dfa_match_len(rules->policy->dfa, state, ":", 1); + return aa_dfa_match(rules->policy->dfa, state, tp->base.hname); } /** @@ -1323,12 +1323,12 @@ next: label_for_each_cont(i, label, tp) { if (!aa_ns_visible(profile->ns, tp->ns, subns)) continue; - state = aa_dfa_match(rules->policy.dfa, state, "//&"); + state = aa_dfa_match(rules->policy->dfa, state, "//&"); state = match_component(profile, rules, tp, state); if (!state) goto fail; } - *perms = *aa_lookup_perms(&rules->policy, state); + *perms = *aa_lookup_perms(rules->policy, state); aa_apply_modes_to_perms(profile, perms); if ((perms->allow & request) != request) return -EACCES; @@ -1381,7 +1381,7 @@ static int label_components_match(struct aa_profile *profile, return 0; next: - tmp = *aa_lookup_perms(&rules->policy, state); + tmp = *aa_lookup_perms(rules->policy, state); aa_apply_modes_to_perms(profile, &tmp); aa_perms_accum(perms, &tmp); label_for_each_cont(i, label, tp) { @@ -1390,7 +1390,7 @@ next: state = match_component(profile, rules, tp, start); if (!state) goto fail; - tmp = *aa_lookup_perms(&rules->policy, state); + tmp = *aa_lookup_perms(rules->policy, state); aa_apply_modes_to_perms(profile, &tmp); aa_perms_accum(perms, &tmp); } diff --git a/security/apparmor/lib.c b/security/apparmor/lib.c index c87bccafff44..4c198d273f09 100644 --- a/security/apparmor/lib.c +++ b/security/apparmor/lib.c @@ -341,8 +341,8 @@ void aa_profile_match_label(struct aa_profile *profile, /* TODO: doesn't yet handle extended types */ aa_state_t state; - state = aa_dfa_next(rules->policy.dfa, - rules->policy.start[AA_CLASS_LABEL], + state = aa_dfa_next(rules->policy->dfa, + rules->policy->start[AA_CLASS_LABEL], type); aa_label_match(profile, rules, label, state, false, request, perms); } diff --git a/security/apparmor/lsm.c b/security/apparmor/lsm.c index 9e31a932a644..bcfe8b9cb4c1 100644 --- a/security/apparmor/lsm.c +++ b/security/apparmor/lsm.c @@ -1887,6 +1887,69 @@ static int __init apparmor_nf_ip_init(void) __initcall(apparmor_nf_ip_init); #endif +static char nulldfa_src[] = { + #include "nulldfa.in" +}; +struct aa_dfa *nulldfa; + +static char stacksplitdfa_src[] = { + #include "stacksplitdfa.in" +}; +struct aa_dfa *stacksplitdfa; +struct aa_policydb *nullpdb; + +static int __init aa_setup_dfa_engine(void) +{ + int error = -ENOMEM; + + nullpdb = aa_alloc_pdb(GFP_KERNEL); + if (!nullpdb) + return -ENOMEM; + + nulldfa = aa_dfa_unpack(nulldfa_src, sizeof(nulldfa_src), + TO_ACCEPT1_FLAG(YYTD_DATA32) | + TO_ACCEPT2_FLAG(YYTD_DATA32)); + if (IS_ERR(nulldfa)) { + error = PTR_ERR(nulldfa); + goto fail; + } + nullpdb->dfa = aa_get_dfa(nulldfa); + nullpdb->perms = kcalloc(2, sizeof(struct aa_perms), GFP_KERNEL); + if (!nullpdb->perms) + goto fail; + nullpdb->size = 2; + + stacksplitdfa = aa_dfa_unpack(stacksplitdfa_src, + sizeof(stacksplitdfa_src), + TO_ACCEPT1_FLAG(YYTD_DATA32) | + TO_ACCEPT2_FLAG(YYTD_DATA32)); + if (IS_ERR(stacksplitdfa)) { + error = PTR_ERR(stacksplitdfa); + goto fail; + } + + return 0; + +fail: + aa_put_pdb(nullpdb); + aa_put_dfa(nulldfa); + nullpdb = NULL; + nulldfa = NULL; + stacksplitdfa = NULL; + + return error; +} + +static void __init aa_teardown_dfa_engine(void) +{ + aa_put_dfa(stacksplitdfa); + aa_put_dfa(nulldfa); + aa_put_pdb(nullpdb); + nullpdb = NULL; + stacksplitdfa = NULL; + nulldfa = NULL; +} + static int __init apparmor_init(void) { int error; diff --git a/security/apparmor/match.c b/security/apparmor/match.c index 7bdcca2aed7d..517d77d3c34c 100644 --- a/security/apparmor/match.c +++ b/security/apparmor/match.c @@ -21,50 +21,6 @@ #define base_idx(X) ((X) & 0xffffff) -static char nulldfa_src[] = { - #include "nulldfa.in" -}; -struct aa_dfa *nulldfa; - -static char stacksplitdfa_src[] = { - #include "stacksplitdfa.in" -}; -struct aa_dfa *stacksplitdfa; - -int __init aa_setup_dfa_engine(void) -{ - int error; - - nulldfa = aa_dfa_unpack(nulldfa_src, sizeof(nulldfa_src), - TO_ACCEPT1_FLAG(YYTD_DATA32) | - TO_ACCEPT2_FLAG(YYTD_DATA32)); - if (IS_ERR(nulldfa)) { - error = PTR_ERR(nulldfa); - nulldfa = NULL; - return error; - } - - stacksplitdfa = aa_dfa_unpack(stacksplitdfa_src, - sizeof(stacksplitdfa_src), - TO_ACCEPT1_FLAG(YYTD_DATA32) | - TO_ACCEPT2_FLAG(YYTD_DATA32)); - if (IS_ERR(stacksplitdfa)) { - aa_put_dfa(nulldfa); - nulldfa = NULL; - error = PTR_ERR(stacksplitdfa); - stacksplitdfa = NULL; - return error; - } - - return 0; -} - -void __init aa_teardown_dfa_engine(void) -{ - aa_put_dfa(stacksplitdfa); - aa_put_dfa(nulldfa); -} - /** * unpack_table - unpack a dfa table (one of accept, default, base, next check) * @blob: data to unpack (NOT NULL) diff --git a/security/apparmor/mount.c b/security/apparmor/mount.c index 2bb77aacc49a..3455dd4b1f99 100644 --- a/security/apparmor/mount.c +++ b/security/apparmor/mount.c @@ -332,8 +332,8 @@ static int match_mnt_path_str(const struct cred *subj_cred, } error = -EACCES; - pos = do_match_mnt(&rules->policy, - rules->policy.start[AA_CLASS_MOUNT], + pos = do_match_mnt(rules->policy, + rules->policy->start[AA_CLASS_MOUNT], mntpnt, devname, type, flags, data, binary, &perms); if (pos) { info = mnt_info_table[pos]; @@ -606,10 +606,10 @@ static int profile_umount(const struct cred *subj_cred, if (error) goto audit; - state = aa_dfa_match(rules->policy.dfa, - rules->policy.start[AA_CLASS_MOUNT], + state = aa_dfa_match(rules->policy->dfa, + rules->policy->start[AA_CLASS_MOUNT], name); - perms = *aa_lookup_perms(&rules->policy, state); + perms = *aa_lookup_perms(rules->policy, state); if (AA_MAY_UMOUNT & ~perms.allow) error = -EACCES; @@ -680,12 +680,12 @@ static struct aa_label *build_pivotroot(const struct cred *subj_cred, goto audit; error = -EACCES; - state = aa_dfa_match(rules->policy.dfa, - rules->policy.start[AA_CLASS_MOUNT], + state = aa_dfa_match(rules->policy->dfa, + rules->policy->start[AA_CLASS_MOUNT], new_name); - state = aa_dfa_null_transition(rules->policy.dfa, state); - state = aa_dfa_match(rules->policy.dfa, state, old_name); - perms = *aa_lookup_perms(&rules->policy, state); + state = aa_dfa_null_transition(rules->policy->dfa, state); + state = aa_dfa_match(rules->policy->dfa, state, old_name); + perms = *aa_lookup_perms(rules->policy, state); if (AA_MAY_PIVOTROOT & perms.allow) error = 0; diff --git a/security/apparmor/net.c b/security/apparmor/net.c index 704c171232ab..87e934b2b548 100644 --- a/security/apparmor/net.c +++ b/security/apparmor/net.c @@ -127,9 +127,9 @@ int aa_profile_af_perm(struct aa_profile *profile, buffer[0] = cpu_to_be16(family); buffer[1] = cpu_to_be16((u16) type); - state = aa_dfa_match_len(rules->policy.dfa, state, (char *) &buffer, + state = aa_dfa_match_len(rules->policy->dfa, state, (char *) &buffer, 4); - perms = *aa_lookup_perms(&rules->policy, state); + perms = *aa_lookup_perms(rules->policy, state); aa_apply_modes_to_perms(profile, &perms); return aa_check_perms(profile, &perms, request, ad, audit_net_cb); diff --git a/security/apparmor/policy.c b/security/apparmor/policy.c index 6f80e1207d24..0b36bd6a6f33 100644 --- a/security/apparmor/policy.c +++ b/security/apparmor/policy.c @@ -98,6 +98,41 @@ const char *const aa_profile_mode_names[] = { }; +static void aa_free_pdb(struct aa_policydb *policy) +{ + if (policy) { + aa_put_dfa(policy->dfa); + if (policy->perms) + kvfree(policy->perms); + aa_free_str_table(&policy->trans); + } +} + +/** + * aa_pdb_free_kref - free aa_policydb by kref (called by aa_put_pdb) + * @kr: kref callback for freeing of a dfa (NOT NULL) + */ +void aa_pdb_free_kref(struct kref *kref) +{ + struct aa_policydb *pdb = container_of(kref, struct aa_policydb, count); + + aa_free_pdb(pdb); +} + + +struct aa_policydb *aa_alloc_pdb(gfp_t gfp) +{ + struct aa_policydb *pdb = kzalloc(sizeof(struct aa_policydb), gfp); + + if (!pdb) + return NULL; + + kref_init(&pdb->count); + + return pdb; +} + + /** * __add_profile - add a profiles to list and label tree * @list: list to add it to (NOT NULL) @@ -200,15 +235,15 @@ static void free_attachment(struct aa_attachment *attach) for (i = 0; i < attach->xattr_count; i++) kfree_sensitive(attach->xattrs[i]); kfree_sensitive(attach->xattrs); - aa_destroy_policydb(&attach->xmatch); + aa_put_pdb(attach->xmatch); } static void free_ruleset(struct aa_ruleset *rules) { int i; - aa_destroy_policydb(&rules->file); - aa_destroy_policydb(&rules->policy); + aa_put_pdb(rules->file); + aa_put_pdb(rules->policy); aa_free_cap_rules(&rules->caps); aa_free_rlimit_rules(&rules->rlimits); @@ -590,16 +625,8 @@ struct aa_profile *aa_alloc_null(struct aa_profile *parent, const char *name, /* TODO: ideally we should inherit abi from parent */ profile->label.flags |= FLAG_NULL; rules = list_first_entry(&profile->rules, typeof(*rules), list); - rules->file.dfa = aa_get_dfa(nulldfa); - rules->file.perms = kcalloc(2, sizeof(struct aa_perms), gfp); - if (!rules->file.perms) - goto fail; - rules->file.size = 2; - rules->policy.dfa = aa_get_dfa(nulldfa); - rules->policy.perms = kcalloc(2, sizeof(struct aa_perms), gfp); - if (!rules->policy.perms) - goto fail; - rules->policy.size = 2; + rules->file = aa_get_pdb(nullpdb); + rules->policy = aa_get_pdb(nullpdb); if (parent) { profile->path_flags = parent->path_flags; @@ -610,11 +637,6 @@ struct aa_profile *aa_alloc_null(struct aa_profile *parent, const char *name, } return profile; - -fail: - aa_free_profile(profile); - - return NULL; } /** diff --git a/security/apparmor/policy_unpack.c b/security/apparmor/policy_unpack.c index 1eb98d6994e8..3fad34b68fdc 100644 --- a/security/apparmor/policy_unpack.c +++ b/security/apparmor/policy_unpack.c @@ -703,24 +703,29 @@ fail_reset: return -EPROTO; } -static int unpack_pdb(struct aa_ext *e, struct aa_policydb *policy, +static int unpack_pdb(struct aa_ext *e, struct aa_policydb **policy, bool required_dfa, bool required_trans, const char **info) { + struct aa_policydb *pdb; void *pos = e->pos; int i, flags, error = -EPROTO; ssize_t size; - size = unpack_perms_table(e, &policy->perms); + pdb = aa_alloc_pdb(GFP_KERNEL); + if (!pdb) + return -ENOMEM; + + size = unpack_perms_table(e, &pdb->perms); if (size < 0) { error = size; - policy->perms = NULL; + pdb->perms = NULL; *info = "failed to unpack - perms"; goto fail; } - policy->size = size; + pdb->size = size; - if (policy->perms) { + if (pdb->perms) { /* perms table present accept is index */ flags = TO_ACCEPT1_FLAG(YYTD_DATA32); } else { @@ -729,13 +734,13 @@ static int unpack_pdb(struct aa_ext *e, struct aa_policydb *policy, TO_ACCEPT2_FLAG(YYTD_DATA32); } - policy->dfa = unpack_dfa(e, flags); - if (IS_ERR(policy->dfa)) { - error = PTR_ERR(policy->dfa); - policy->dfa = NULL; + pdb->dfa = unpack_dfa(e, flags); + if (IS_ERR(pdb->dfa)) { + error = PTR_ERR(pdb->dfa); + pdb->dfa = NULL; *info = "failed to unpack - dfa"; goto fail; - } else if (!policy->dfa) { + } else if (!pdb->dfa) { if (required_dfa) { *info = "missing required dfa"; goto fail; @@ -749,18 +754,18 @@ static int unpack_pdb(struct aa_ext *e, struct aa_policydb *policy, * sadly start was given different names for file and policydb * but since it is optional we can try both */ - if (!aa_unpack_u32(e, &policy->start[0], "start")) + if (!aa_unpack_u32(e, &pdb->start[0], "start")) /* default start state */ - policy->start[0] = DFA_START; - if (!aa_unpack_u32(e, &policy->start[AA_CLASS_FILE], "dfa_start")) { + pdb->start[0] = DFA_START; + if (!aa_unpack_u32(e, &pdb->start[AA_CLASS_FILE], "dfa_start")) { /* default start state for xmatch and file dfa */ - policy->start[AA_CLASS_FILE] = DFA_START; + pdb->start[AA_CLASS_FILE] = DFA_START; } /* setup class index */ for (i = AA_CLASS_FILE + 1; i <= AA_CLASS_LAST; i++) { - policy->start[i] = aa_dfa_next(policy->dfa, policy->start[0], + pdb->start[i] = aa_dfa_next(pdb->dfa, pdb->start[0], i); } - if (!unpack_trans_table(e, &policy->trans) && required_trans) { + if (!unpack_trans_table(e, &pdb->trans) && required_trans) { *info = "failed to unpack profile transition table"; goto fail; } @@ -768,9 +773,11 @@ static int unpack_pdb(struct aa_ext *e, struct aa_policydb *policy, /* TODO: move compat mapping here, requires dfa merging first */ /* TODO: move verify here, it has to be done after compat mappings */ out: + *policy = pdb; return 0; fail: + aa_put_pdb(pdb); e->pos = pos; return error; } @@ -854,15 +861,15 @@ static struct aa_profile *unpack_profile(struct aa_ext *e, char **ns_name) } /* neither xmatch_len not xmatch_perms are optional if xmatch is set */ - if (profile->attach.xmatch.dfa) { + if (profile->attach.xmatch->dfa) { if (!aa_unpack_u32(e, &tmp, NULL)) { info = "missing xmatch len"; goto fail; } profile->attach.xmatch_len = tmp; - profile->attach.xmatch.start[AA_CLASS_XMATCH] = DFA_START; - if (!profile->attach.xmatch.perms) { - error = aa_compat_map_xmatch(&profile->attach.xmatch); + profile->attach.xmatch->start[AA_CLASS_XMATCH] = DFA_START; + if (!profile->attach.xmatch->perms) { + error = aa_compat_map_xmatch(profile->attach.xmatch); if (error) { info = "failed to convert xmatch permission table"; goto fail; @@ -979,16 +986,16 @@ static struct aa_profile *unpack_profile(struct aa_ext *e, char **ns_name) if (error) goto fail; /* Fixup: drop when we get rid of start array */ - if (aa_dfa_next(rules->policy.dfa, rules->policy.start[0], + if (aa_dfa_next(rules->policy->dfa, rules->policy->start[0], AA_CLASS_FILE)) - rules->policy.start[AA_CLASS_FILE] = - aa_dfa_next(rules->policy.dfa, - rules->policy.start[0], + rules->policy->start[AA_CLASS_FILE] = + aa_dfa_next(rules->policy->dfa, + rules->policy->start[0], AA_CLASS_FILE); if (!aa_unpack_nameX(e, AA_STRUCTEND, NULL)) goto fail; - if (!rules->policy.perms) { - error = aa_compat_map_policy(&rules->policy, + if (!rules->policy->perms) { + error = aa_compat_map_policy(rules->policy, e->version); if (error) { info = "failed to remap policydb permission table"; @@ -996,44 +1003,25 @@ static struct aa_profile *unpack_profile(struct aa_ext *e, char **ns_name) } } } else { - rules->policy.dfa = aa_get_dfa(nulldfa); - rules->policy.perms = kcalloc(2, sizeof(struct aa_perms), - GFP_KERNEL); - if (!rules->policy.perms) - goto fail; - rules->policy.size = 2; + rules->policy = aa_get_pdb(nullpdb); } /* get file rules */ error = unpack_pdb(e, &rules->file, false, true, &info); if (error) { goto fail; - } else if (rules->file.dfa) { - if (!rules->file.perms) { - error = aa_compat_map_file(&rules->file); + } else if (rules->file->dfa) { + if (!rules->file->perms) { + error = aa_compat_map_file(rules->file); if (error) { info = "failed to remap file permission table"; goto fail; } } - } else if (rules->policy.dfa && - rules->policy.start[AA_CLASS_FILE]) { - rules->file.dfa = aa_get_dfa(rules->policy.dfa); - rules->file.start[AA_CLASS_FILE] = rules->policy.start[AA_CLASS_FILE]; - rules->file.perms = kcalloc(rules->policy.size, - sizeof(struct aa_perms), - GFP_KERNEL); - if (!rules->file.perms) - goto fail; - memcpy(rules->file.perms, rules->policy.perms, - rules->policy.size * sizeof(struct aa_perms)); - rules->file.size = rules->policy.size; + } else if (rules->policy->dfa && + rules->policy->start[AA_CLASS_FILE]) { + rules->file = aa_get_pdb(rules->policy); } else { - rules->file.dfa = aa_get_dfa(nulldfa); - rules->file.perms = kcalloc(2, sizeof(struct aa_perms), - GFP_KERNEL); - if (!rules->file.perms) - goto fail; - rules->file.size = 2; + rules->file = aa_get_pdb(nullpdb); } error = -EPROTO; if (aa_unpack_nameX(e, AA_STRUCT, "data")) { @@ -1240,32 +1228,32 @@ static int verify_profile(struct aa_profile *profile) if (!rules) return 0; - if (rules->file.dfa && !verify_dfa_accept_index(rules->file.dfa, - rules->file.size)) { + if (rules->file->dfa && !verify_dfa_accept_index(rules->file->dfa, + rules->file->size)) { audit_iface(profile, NULL, NULL, "Unpack: file Invalid named transition", NULL, -EPROTO); return -EPROTO; } - if (rules->policy.dfa && - !verify_dfa_accept_index(rules->policy.dfa, rules->policy.size)) { + if (rules->policy->dfa && + !verify_dfa_accept_index(rules->policy->dfa, rules->policy->size)) { audit_iface(profile, NULL, NULL, "Unpack: policy Invalid named transition", NULL, -EPROTO); return -EPROTO; } - if (!verify_perms(&rules->file)) { + if (!verify_perms(rules->file)) { audit_iface(profile, NULL, NULL, "Unpack: Invalid perm index", NULL, -EPROTO); return -EPROTO; } - if (!verify_perms(&rules->policy)) { + if (!verify_perms(rules->policy)) { audit_iface(profile, NULL, NULL, "Unpack: Invalid perm index", NULL, -EPROTO); return -EPROTO; } - if (!verify_perms(&profile->attach.xmatch)) { + if (!verify_perms(profile->attach.xmatch)) { audit_iface(profile, NULL, NULL, "Unpack: Invalid perm index", NULL, -EPROTO); return -EPROTO; From e105d8079f82819f4773c4853dc199e195fedf40 Mon Sep 17 00:00:00 2001 From: John Johansen Date: Thu, 24 Aug 2023 12:38:01 -0700 Subject: [PATCH 267/475] apparmor: advertise disconnected.path is available While disconnected.path has been available for a while it was never properly advertised as a feature. Fix this so that userspace doesn't need special casing to handle it. Reviewed-by: Georgia Garcia Signed-off-by: John Johansen --- security/apparmor/apparmorfs.c | 1 + 1 file changed, 1 insertion(+) diff --git a/security/apparmor/apparmorfs.c b/security/apparmor/apparmorfs.c index 0bba46ccdb71..b123abbc43d8 100644 --- a/security/apparmor/apparmorfs.c +++ b/security/apparmor/apparmorfs.c @@ -2336,6 +2336,7 @@ static struct aa_sfs_entry aa_sfs_entry_domain[] = { AA_SFS_FILE_BOOLEAN("post_nnp_subset", 1), AA_SFS_FILE_BOOLEAN("computed_longest_left", 1), AA_SFS_DIR("attach_conditions", aa_sfs_entry_attach), + AA_SFS_FILE_BOOLEAN("disconnected.path", 1), AA_SFS_FILE_STRING("version", "1.2"), { } }; From 1f36b190ad2dea68e3a7e84b7b2f24ce8c4063ea Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Tue, 10 Oct 2023 16:42:47 -0700 Subject: [PATCH 268/475] perf tools: Do not ignore the default vmlinux.h The recent change made it possible to generate vmlinux.h from BTF and to ignore the file. But we also have a minimal vmlinux.h that will be used by default. It should not be ignored by GIT. Fixes: b7a2d774c9c5 ("perf build: Add ability to build with a generated vmlinux.h") Reported-by: kernel test robot Reviewed-by: Ian Rogers Closes: https://lore.kernel.org/oe-kbuild-all/202310110451.rvdUZJEY-lkp@intel.com/ Cc: oe-kbuild-all@lists.linux.dev Signed-off-by: Namhyung Kim --- tools/perf/util/bpf_skel/vmlinux/.gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 tools/perf/util/bpf_skel/vmlinux/.gitignore diff --git a/tools/perf/util/bpf_skel/vmlinux/.gitignore b/tools/perf/util/bpf_skel/vmlinux/.gitignore new file mode 100644 index 000000000000..49502c04183a --- /dev/null +++ b/tools/perf/util/bpf_skel/vmlinux/.gitignore @@ -0,0 +1 @@ +!vmlinux.h From 2d9da9b188b8cd3b579d7ef5ba5d334be9dd38fc Mon Sep 17 00:00:00 2001 From: John Johansen Date: Wed, 9 Aug 2023 00:26:36 -0700 Subject: [PATCH 269/475] apparmor: allow restricting unprivileged change_profile unprivileged unconfined can use change_profile to alter the confinement set by the mac admin. Allow restricting unprivileged unconfined by still allowing change_profile but stacking the change against unconfined. This allows unconfined to still apply system policy but allows the task to enter the new confinement. If unprivileged unconfined is required a sysctl is provided to switch to the previous behavior. Reviewed-by: Georgia Garcia Signed-off-by: John Johansen --- security/apparmor/apparmorfs.c | 6 ++++++ security/apparmor/domain.c | 24 ++++++++++++++++++++++++ security/apparmor/include/policy.h | 1 + security/apparmor/lsm.c | 7 +++++++ security/apparmor/policy.c | 1 + 5 files changed, 39 insertions(+) diff --git a/security/apparmor/apparmorfs.c b/security/apparmor/apparmorfs.c index b123abbc43d8..6d0848f10ff0 100644 --- a/security/apparmor/apparmorfs.c +++ b/security/apparmor/apparmorfs.c @@ -2341,6 +2341,11 @@ static struct aa_sfs_entry aa_sfs_entry_domain[] = { { } }; +static struct aa_sfs_entry aa_sfs_entry_unconfined[] = { + AA_SFS_FILE_BOOLEAN("change_profile", 1), + { } +}; + static struct aa_sfs_entry aa_sfs_entry_versions[] = { AA_SFS_FILE_BOOLEAN("v5", 1), AA_SFS_FILE_BOOLEAN("v6", 1), @@ -2358,6 +2363,7 @@ static struct aa_sfs_entry aa_sfs_entry_policy[] = { AA_SFS_FILE_U64("outofband", MAX_OOB_SUPPORTED), AA_SFS_FILE_U64("permstable32_version", 1), AA_SFS_FILE_STRING("permstable32", PERMS32STR), + AA_SFS_DIR("unconfined_restrictions", aa_sfs_entry_unconfined), { } }; diff --git a/security/apparmor/domain.c b/security/apparmor/domain.c index 87dfa0e40398..ed4a13d44894 100644 --- a/security/apparmor/domain.c +++ b/security/apparmor/domain.c @@ -1311,6 +1311,8 @@ static int change_profile_perms_wrapper(const char *op, const char *name, return error; } +const char *stack_msg = "change_profile unprivileged unconfined converted to stacking"; + /** * aa_change_profile - perform a one-way profile transition * @fqname: name of profile may include namespace (NOT NULL) @@ -1370,6 +1372,28 @@ int aa_change_profile(const char *fqname, int flags) op = OP_CHANGE_PROFILE; } + /* This should move to a per profile test. Requires pushing build + * into callback + */ + if (!stack && unconfined(label) && + label == &labels_ns(label)->unconfined->label && + aa_unprivileged_unconfined_restricted && + /* TODO: refactor so this check is a fn */ + cap_capable(current_cred(), &init_user_ns, CAP_MAC_OVERRIDE, + CAP_OPT_NOAUDIT)) { + /* regardless of the request in this case apparmor + * stacks against unconfined so admin set policy can't be + * by-passed + */ + stack = true; + perms.audit = request; + (void) fn_for_each_in_ns(label, profile, + aa_audit_file(subj_cred, profile, &perms, op, + request, auditname, NULL, target, + GLOBAL_ROOT_UID, stack_msg, 0)); + perms.audit = 0; + } + if (*fqname == '&') { stack = true; /* don't have label_parse() do stacking */ diff --git a/security/apparmor/include/policy.h b/security/apparmor/include/policy.h index e69c91619a7c..75088cc310b6 100644 --- a/security/apparmor/include/policy.h +++ b/security/apparmor/include/policy.h @@ -34,6 +34,7 @@ struct aa_ns; extern int unprivileged_userns_apparmor_policy; +extern int aa_unprivileged_unconfined_restricted; extern const char *const aa_profile_mode_names[]; #define APPARMOR_MODE_NAMES_MAX_INDEX 4 diff --git a/security/apparmor/lsm.c b/security/apparmor/lsm.c index bcfe8b9cb4c1..518576ae3cfb 100644 --- a/security/apparmor/lsm.c +++ b/security/apparmor/lsm.c @@ -1798,6 +1798,13 @@ static struct ctl_table apparmor_sysctl_table[] = { .mode = 0600, .proc_handler = apparmor_dointvec, }, + { + .procname = "apparmor_restrict_unprivileged_unconfined", + .data = &aa_unprivileged_unconfined_restricted, + .maxlen = sizeof(int), + .mode = 0600, + .proc_handler = apparmor_dointvec, + }, { } }; diff --git a/security/apparmor/policy.c b/security/apparmor/policy.c index 0b36bd6a6f33..a441d96adcbf 100644 --- a/security/apparmor/policy.c +++ b/security/apparmor/policy.c @@ -88,6 +88,7 @@ #include "include/resource.h" int unprivileged_userns_apparmor_policy = 1; +int aa_unprivileged_unconfined_restricted; const char *const aa_profile_mode_names[] = { "enforce", From fa9b63adabcfa9b724120ef3352cf6fb82b4b9a5 Mon Sep 17 00:00:00 2001 From: John Johansen Date: Fri, 9 Sep 2022 16:00:09 -0700 Subject: [PATCH 270/475] apparmor: add user namespace creation mediation Unprivileged user namespace creation is often used as a first step in privilege escalation attacks. Instead of disabling it at the sysrq level, which blocks its legitimate use as for setting up a sandbox, allow control on a per domain basis. This allows an admin to quickly lock down a system while also still allowing legitimate use. Reviewed-by: Georgia Garcia Signed-off-by: John Johansen --- security/apparmor/apparmorfs.c | 1 + security/apparmor/audit.c | 2 +- security/apparmor/include/apparmor.h | 1 + security/apparmor/include/audit.h | 1 + security/apparmor/include/task.h | 6 ++++ security/apparmor/lsm.c | 25 ++++++++++++++++- security/apparmor/task.c | 41 ++++++++++++++++++++++++++++ 7 files changed, 75 insertions(+), 2 deletions(-) diff --git a/security/apparmor/apparmorfs.c b/security/apparmor/apparmorfs.c index 6d0848f10ff0..7170349c8af0 100644 --- a/security/apparmor/apparmorfs.c +++ b/security/apparmor/apparmorfs.c @@ -2375,6 +2375,7 @@ static struct aa_sfs_entry aa_sfs_entry_mount[] = { static struct aa_sfs_entry aa_sfs_entry_ns[] = { AA_SFS_FILE_BOOLEAN("profile", 1), AA_SFS_FILE_BOOLEAN("pivot_root", 0), + AA_SFS_FILE_STRING("mask", "userns_create"), { } }; diff --git a/security/apparmor/audit.c b/security/apparmor/audit.c index 6933cb2f679b..3b24f4a8c727 100644 --- a/security/apparmor/audit.c +++ b/security/apparmor/audit.c @@ -58,7 +58,7 @@ static const char *const aa_class_names[] = { "io_uring", "module", "lsm", - "unknown", + "namespace", "unknown", "unknown", "unknown", diff --git a/security/apparmor/include/apparmor.h b/security/apparmor/include/apparmor.h index 8a81557c9d59..e2b759f24064 100644 --- a/security/apparmor/include/apparmor.h +++ b/security/apparmor/include/apparmor.h @@ -33,6 +33,7 @@ #define AA_CLASS_IO_URING 18 #define AA_CLASS_MODULE 19 #define AA_CLASS_DISPLAY_LSM 20 +#define AA_CLASS_NS 21 #define AA_CLASS_X 31 #define AA_CLASS_DBUS 32 diff --git a/security/apparmor/include/audit.h b/security/apparmor/include/audit.h index 42d701fec5a6..095707e05b70 100644 --- a/security/apparmor/include/audit.h +++ b/security/apparmor/include/audit.h @@ -103,6 +103,7 @@ enum audit_type { #define OP_PROF_LOAD "profile_load" #define OP_PROF_RM "profile_remove" +#define OP_USERNS_CREATE "userns_create" struct apparmor_audit_data { int error; diff --git a/security/apparmor/include/task.h b/security/apparmor/include/task.h index 13945e2495f0..b1aaaf60fa8b 100644 --- a/security/apparmor/include/task.h +++ b/security/apparmor/include/task.h @@ -96,4 +96,10 @@ int aa_may_ptrace(const struct cred *tracer_cred, struct aa_label *tracer, u32 request); + +#define AA_USERNS_CREATE 8 + +int aa_profile_ns_perm(struct aa_profile *profile, + struct apparmor_audit_data *ad, u32 request); + #endif /* __AA_TASK_H */ diff --git a/security/apparmor/lsm.c b/security/apparmor/lsm.c index 518576ae3cfb..c61835bd7db9 100644 --- a/security/apparmor/lsm.c +++ b/security/apparmor/lsm.c @@ -836,6 +836,27 @@ static int apparmor_task_kill(struct task_struct *target, struct kernel_siginfo return error; } +static int apparmor_userns_create(const struct cred *cred) +{ + struct aa_label *label; + struct aa_profile *profile; + int error = 0; + DEFINE_AUDIT_DATA(ad, LSM_AUDIT_DATA_TASK, AA_CLASS_NS, + OP_USERNS_CREATE); + + ad.subj_cred = current_cred(); + + label = begin_current_label_crit_section(); + if (!unconfined(label)) { + error = fn_for_each(label, profile, + aa_profile_ns_perm(profile, &ad, + AA_USERNS_CREATE)); + } + end_current_label_crit_section(label); + + return error; +} + /** * apparmor_sk_alloc_security - allocate and attach the sk_security field */ @@ -1313,6 +1334,7 @@ static struct security_hook_list apparmor_hooks[] __ro_after_init = { LSM_HOOK_INIT(task_getsecid_obj, apparmor_task_getsecid_obj), LSM_HOOK_INIT(task_setrlimit, apparmor_task_setrlimit), LSM_HOOK_INIT(task_kill, apparmor_task_kill), + LSM_HOOK_INIT(userns_create, apparmor_userns_create), #ifdef CONFIG_AUDIT LSM_HOOK_INIT(audit_rule_init, aa_audit_rule_init), @@ -1784,6 +1806,7 @@ static int apparmor_dointvec(struct ctl_table *table, int write, } static struct ctl_table apparmor_sysctl_table[] = { +#ifdef CONFIG_USER_NS { .procname = "unprivileged_userns_apparmor_policy", .data = &unprivileged_userns_apparmor_policy, @@ -1791,6 +1814,7 @@ static struct ctl_table apparmor_sysctl_table[] = { .mode = 0600, .proc_handler = apparmor_dointvec, }, +#endif /* CONFIG_USER_NS */ { .procname = "apparmor_display_secid_mode", .data = &apparmor_display_secid_mode, @@ -1805,7 +1829,6 @@ static struct ctl_table apparmor_sysctl_table[] = { .mode = 0600, .proc_handler = apparmor_dointvec, }, - { } }; diff --git a/security/apparmor/task.c b/security/apparmor/task.c index 1a7c9d02e31d..f29a2e80e6bf 100644 --- a/security/apparmor/task.c +++ b/security/apparmor/task.c @@ -298,3 +298,44 @@ int aa_may_ptrace(const struct cred *tracer_cred, struct aa_label *tracer, profile_tracee_perm(tracee_cred, profile, tracer, xrequest, &sa)); } + +/* call back to audit ptrace fields */ +static void audit_ns_cb(struct audit_buffer *ab, void *va) +{ + struct apparmor_audit_data *ad = aad_of_va(va); + + if (ad->request & AA_USERNS_CREATE) + audit_log_format(ab, " requested=\"userns_create\""); + + if (ad->denied & AA_USERNS_CREATE) + audit_log_format(ab, " denied=\"userns_create\""); +} + +int aa_profile_ns_perm(struct aa_profile *profile, + struct apparmor_audit_data *ad, + u32 request) +{ + struct aa_perms perms = { }; + int error = 0; + + ad->subj_label = &profile->label; + ad->request = request; + + if (!profile_unconfined(profile)) { + struct aa_ruleset *rules = list_first_entry(&profile->rules, + typeof(*rules), + list); + aa_state_t state; + + state = RULE_MEDIATES(rules, ad->class); + if (!state) + /* TODO: add flag to complain about unmediated */ + return 0; + perms = *aa_lookup_perms(rules->policy, state); + aa_apply_modes_to_perms(profile, &perms); + error = aa_check_perms(profile, &perms, request, ad, + audit_ns_cb); + } + + return error; +} From c4371d90633b73cf6e86aff43ff2b5d95ad2b9eb Mon Sep 17 00:00:00 2001 From: Georgia Garcia Date: Mon, 20 Mar 2023 14:43:41 -0300 Subject: [PATCH 271/475] apparmor: add io_uring mediation For now, the io_uring mediation is limited to sqpoll and override_creds. Signed-off-by: Georgia Garcia Signed-off-by: John Johansen --- security/apparmor/apparmorfs.c | 7 ++ security/apparmor/audit.c | 2 +- security/apparmor/include/apparmor.h | 2 +- security/apparmor/include/audit.h | 6 ++ security/apparmor/include/perms.h | 3 + security/apparmor/lsm.c | 113 +++++++++++++++++++++++++++ 6 files changed, 131 insertions(+), 2 deletions(-) diff --git a/security/apparmor/apparmorfs.c b/security/apparmor/apparmorfs.c index 7170349c8af0..a608a6bd76c5 100644 --- a/security/apparmor/apparmorfs.c +++ b/security/apparmor/apparmorfs.c @@ -2390,6 +2390,12 @@ static struct aa_sfs_entry aa_sfs_entry_query[] = { AA_SFS_DIR("label", aa_sfs_entry_query_label), { } }; + +static struct aa_sfs_entry aa_sfs_entry_io_uring[] = { + AA_SFS_FILE_STRING("mask", "sqpoll override_creds"), + { } +}; + static struct aa_sfs_entry aa_sfs_entry_features[] = { AA_SFS_DIR("policy", aa_sfs_entry_policy), AA_SFS_DIR("domain", aa_sfs_entry_domain), @@ -2403,6 +2409,7 @@ static struct aa_sfs_entry aa_sfs_entry_features[] = { AA_SFS_DIR("ptrace", aa_sfs_entry_ptrace), AA_SFS_DIR("signal", aa_sfs_entry_signal), AA_SFS_DIR("query", aa_sfs_entry_query), + AA_SFS_DIR("io_uring", aa_sfs_entry_io_uring), { } }; diff --git a/security/apparmor/audit.c b/security/apparmor/audit.c index 3b24f4a8c727..45beb1c5f747 100644 --- a/security/apparmor/audit.c +++ b/security/apparmor/audit.c @@ -59,7 +59,7 @@ static const char *const aa_class_names[] = { "module", "lsm", "namespace", - "unknown", + "io_uring", "unknown", "unknown", "unknown", diff --git a/security/apparmor/include/apparmor.h b/security/apparmor/include/apparmor.h index e2b759f24064..f83934913b0f 100644 --- a/security/apparmor/include/apparmor.h +++ b/security/apparmor/include/apparmor.h @@ -30,10 +30,10 @@ #define AA_CLASS_NET 14 #define AA_CLASS_LABEL 16 #define AA_CLASS_POSIX_MQUEUE 17 -#define AA_CLASS_IO_URING 18 #define AA_CLASS_MODULE 19 #define AA_CLASS_DISPLAY_LSM 20 #define AA_CLASS_NS 21 +#define AA_CLASS_IO_URING 22 #define AA_CLASS_X 31 #define AA_CLASS_DBUS 32 diff --git a/security/apparmor/include/audit.h b/security/apparmor/include/audit.h index 095707e05b70..acbb03b9bd25 100644 --- a/security/apparmor/include/audit.h +++ b/security/apparmor/include/audit.h @@ -105,6 +105,9 @@ enum audit_type { #define OP_USERNS_CREATE "userns_create" +#define OP_URING_OVERRIDE "uring_override" +#define OP_URING_SQPOLL "uring_sqpoll" + struct apparmor_audit_data { int error; int type; @@ -153,6 +156,9 @@ struct apparmor_audit_data { const char *data; unsigned long flags; } mnt; + struct { + struct aa_label *target; + } uring; }; struct common_audit_data common; diff --git a/security/apparmor/include/perms.h b/security/apparmor/include/perms.h index 83534df8939f..0f7e913c3fc2 100644 --- a/security/apparmor/include/perms.h +++ b/security/apparmor/include/perms.h @@ -48,6 +48,9 @@ #define AA_LINK_SUBSET AA_MAY_LOCK /* overlaid */ +#define AA_MAY_CREATE_SQPOLL AA_MAY_CREATE +#define AA_MAY_OVERRIDE_CRED AA_MAY_APPEND +#define AA_URING_PERM_MASK (AA_MAY_OVERRIDE_CRED | AA_MAY_CREATE_SQPOLL) #define PERMS_CHRS_MASK (MAY_READ | MAY_WRITE | AA_MAY_CREATE | \ AA_MAY_DELETE | AA_MAY_LINK | AA_MAY_LOCK | \ diff --git a/security/apparmor/lsm.c b/security/apparmor/lsm.c index c61835bd7db9..c80c1bd3024a 100644 --- a/security/apparmor/lsm.c +++ b/security/apparmor/lsm.c @@ -582,6 +582,114 @@ static int apparmor_file_mprotect(struct vm_area_struct *vma, false); } +#ifdef CONFIG_IO_URING +static const char *audit_uring_mask(u32 mask) +{ + if (mask & AA_MAY_CREATE_SQPOLL) + return "sqpoll"; + if (mask & AA_MAY_OVERRIDE_CRED) + return "override_creds"; + return ""; +} + +static void audit_uring_cb(struct audit_buffer *ab, void *va) +{ + struct apparmor_audit_data *ad = aad_of_va(va); + + if (ad->request & AA_URING_PERM_MASK) { + audit_log_format(ab, " requested=\"%s\"", + audit_uring_mask(ad->request)); + if (ad->denied & AA_URING_PERM_MASK) { + audit_log_format(ab, " denied=\"%s\"", + audit_uring_mask(ad->denied)); + } + } + if (ad->uring.target) { + audit_log_format(ab, " tcontext="); + aa_label_xaudit(ab, labels_ns(ad->subj_label), + ad->uring.target, + FLAGS_NONE, GFP_ATOMIC); + } +} + +static int profile_uring(struct aa_profile *profile, u32 request, + struct aa_label *new, int cap, + struct apparmor_audit_data *ad) +{ + unsigned int state; + struct aa_ruleset *rules; + int error = 0; + + AA_BUG(!profile); + + rules = list_first_entry(&profile->rules, typeof(*rules), list); + state = RULE_MEDIATES(rules, AA_CLASS_IO_URING); + if (state) { + struct aa_perms perms = { }; + + if (new) { + aa_label_match(profile, rules, new, state, + false, request, &perms); + } else { + perms = *aa_lookup_perms(rules->policy, state); + } + aa_apply_modes_to_perms(profile, &perms); + error = aa_check_perms(profile, &perms, request, ad, + audit_uring_cb); + } + + return error; +} + +/** + * apparmor_uring_override_creds - check the requested cred override + * @new: the target creds + * + * Check to see if the current task is allowed to override it's credentials + * to service an io_uring operation. + */ +int apparmor_uring_override_creds(const struct cred *new) +{ + struct aa_profile *profile; + struct aa_label *label; + int error; + DEFINE_AUDIT_DATA(ad, LSM_AUDIT_DATA_NONE, AA_CLASS_IO_URING, + OP_URING_OVERRIDE); + + ad.uring.target = cred_label(new); + label = __begin_current_label_crit_section(); + error = fn_for_each(label, profile, + profile_uring(profile, AA_MAY_OVERRIDE_CRED, + cred_label(new), CAP_SYS_ADMIN, &ad)); + __end_current_label_crit_section(label); + + return error; +} + +/** + * apparmor_uring_sqpoll - check if a io_uring polling thread can be created + * + * Check to see if the current task is allowed to create a new io_uring + * kernel polling thread. + */ +int apparmor_uring_sqpoll(void) +{ + struct aa_profile *profile; + struct aa_label *label; + int error; + DEFINE_AUDIT_DATA(ad, LSM_AUDIT_DATA_NONE, AA_CLASS_IO_URING, + OP_URING_SQPOLL); + + label = __begin_current_label_crit_section(); + error = fn_for_each(label, profile, + profile_uring(profile, AA_MAY_CREATE_SQPOLL, + NULL, CAP_SYS_ADMIN, &ad)); + __end_current_label_crit_section(label); + + return error; +} +#endif /* CONFIG_IO_URING */ + static int apparmor_sb_mount(const char *dev_name, const struct path *path, const char *type, unsigned long flags, void *data) { @@ -1346,6 +1454,11 @@ static struct security_hook_list apparmor_hooks[] __ro_after_init = { LSM_HOOK_INIT(secid_to_secctx, apparmor_secid_to_secctx), LSM_HOOK_INIT(secctx_to_secid, apparmor_secctx_to_secid), LSM_HOOK_INIT(release_secctx, apparmor_release_secctx), + +#ifdef CONFIG_IO_URING + LSM_HOOK_INIT(uring_override_creds, apparmor_uring_override_creds), + LSM_HOOK_INIT(uring_sqpoll, apparmor_uring_sqpoll), +#endif }; /* From ea9bae12d02819556db63348db8bd8441eb316f2 Mon Sep 17 00:00:00 2001 From: John Johansen Date: Tue, 25 Oct 2022 01:18:41 -0700 Subject: [PATCH 272/475] apparmor: cache buffers on percpu list if there is lock contention commit df323337e507 ("apparmor: Use a memory pool instead per-CPU caches") changed buffer allocation to use a memory pool, however on a heavily loaded machine there can be lock contention on the global buffers lock. Add a percpu list to cache buffers on when lock contention is encountered. When allocating buffers attempt to use cached buffers first, before taking the global buffers lock. When freeing buffers try to put them back to the global list but if contention is encountered, put the buffer on the percpu list. The length of time a buffer is held on the percpu list is dynamically adjusted based on lock contention. The amount of hold time is increased and decreased linearly. v5: - simplify base patch by removing: improvements can be added later - MAX_LOCAL and must lock - contention scaling. v4: - fix percpu ->count buffer count which had been spliced across a debug patch. - introduce define for MAX_LOCAL_COUNT - rework count check and locking around it. - update commit message to reference commit that introduced the memory. v3: - limit number of buffers that can be pushed onto the percpu list. This avoids a problem on some kernels where one percpu list can inherit buffers from another cpu after a reschedule, causing more kernel memory to used than is necessary. Under normal conditions this should eventually return to normal but under pathelogical conditions the extra memory consumption may have been unbouanded v2: - dynamically adjust buffer hold time on percpu list based on lock contention. v1: - cache buffers on percpu list on lock contention Reported-by: Sergey Senozhatsky Reviewed-by: Georgia Garcia Signed-off-by: John Johansen --- security/apparmor/lsm.c | 67 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 62 insertions(+), 5 deletions(-) diff --git a/security/apparmor/lsm.c b/security/apparmor/lsm.c index c80c1bd3024a..ce4f3e7a784d 100644 --- a/security/apparmor/lsm.c +++ b/security/apparmor/lsm.c @@ -49,12 +49,19 @@ union aa_buffer { DECLARE_FLEX_ARRAY(char, buffer); }; +struct aa_local_cache { + unsigned int hold; + unsigned int count; + struct list_head head; +}; + #define RESERVE_COUNT 2 static int reserve_count = RESERVE_COUNT; static int buffer_count; static LIST_HEAD(aa_global_buffers); static DEFINE_SPINLOCK(aa_buffers_lock); +static DEFINE_PER_CPU(struct aa_local_cache, aa_local_buffers); /* * LSM hook functions @@ -1789,11 +1796,32 @@ static int param_set_mode(const char *val, const struct kernel_param *kp) char *aa_get_buffer(bool in_atomic) { union aa_buffer *aa_buf; + struct aa_local_cache *cache; bool try_again = true; gfp_t flags = (GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN); + /* use per cpu cached buffers first */ + cache = get_cpu_ptr(&aa_local_buffers); + if (!list_empty(&cache->head)) { + aa_buf = list_first_entry(&cache->head, union aa_buffer, list); + list_del(&aa_buf->list); + cache->hold--; + cache->count--; + put_cpu_ptr(&aa_local_buffers); + return &aa_buf->buffer[0]; + } + put_cpu_ptr(&aa_local_buffers); + + if (!spin_trylock(&aa_buffers_lock)) { + cache = get_cpu_ptr(&aa_local_buffers); + cache->hold += 1; + put_cpu_ptr(&aa_local_buffers); + spin_lock(&aa_buffers_lock); + } else { + cache = get_cpu_ptr(&aa_local_buffers); + put_cpu_ptr(&aa_local_buffers); + } retry: - spin_lock(&aa_buffers_lock); if (buffer_count > reserve_count || (in_atomic && !list_empty(&aa_global_buffers))) { aa_buf = list_first_entry(&aa_global_buffers, union aa_buffer, @@ -1819,6 +1847,7 @@ retry: if (!aa_buf) { if (try_again) { try_again = false; + spin_lock(&aa_buffers_lock); goto retry; } pr_warn_once("AppArmor: Failed to allocate a memory buffer.\n"); @@ -1830,15 +1859,34 @@ retry: void aa_put_buffer(char *buf) { union aa_buffer *aa_buf; + struct aa_local_cache *cache; if (!buf) return; aa_buf = container_of(buf, union aa_buffer, buffer[0]); - spin_lock(&aa_buffers_lock); - list_add(&aa_buf->list, &aa_global_buffers); - buffer_count++; - spin_unlock(&aa_buffers_lock); + cache = get_cpu_ptr(&aa_local_buffers); + if (!cache->hold) { + put_cpu_ptr(&aa_local_buffers); + + if (spin_trylock(&aa_buffers_lock)) { + /* put back on global list */ + list_add(&aa_buf->list, &aa_global_buffers); + buffer_count++; + spin_unlock(&aa_buffers_lock); + cache = get_cpu_ptr(&aa_local_buffers); + put_cpu_ptr(&aa_local_buffers); + return; + } + /* contention on global list, fallback to percpu */ + cache = get_cpu_ptr(&aa_local_buffers); + cache->hold += 1; + } + + /* cache in percpu list */ + list_add(&aa_buf->list, &cache->head); + cache->count++; + put_cpu_ptr(&aa_local_buffers); } /* @@ -1880,6 +1928,15 @@ static int __init alloc_buffers(void) union aa_buffer *aa_buf; int i, num; + /* + * per cpu set of cached allocated buffers used to help reduce + * lock contention + */ + for_each_possible_cpu(i) { + per_cpu(aa_local_buffers, i).hold = 0; + per_cpu(aa_local_buffers, i).count = 0; + INIT_LIST_HEAD(&per_cpu(aa_local_buffers, i).head); + } /* * A function may require two buffers at once. Usually the buffers are * used for a short period of time and are shared. On UP kernel buffers From 157a3537d6bc28ceb9a11fc8cb67f2152d860146 Mon Sep 17 00:00:00 2001 From: John Johansen Date: Sun, 10 Sep 2023 03:35:22 -0700 Subject: [PATCH 273/475] apparmor: Fix regression in mount mediation commit 2db154b3ea8e ("vfs: syscall: Add move_mount(2) to move mounts around") introduced a new move_mount(2) system call and a corresponding new LSM security_move_mount hook but did not implement this hook for any existing LSM. This creates a regression for AppArmor mediation of mount. This patch provides a base mapping of the move_mount syscall to the existing mount mediation. In the future we may introduce additional mediations around the new mount calls. Fixes: 2db154b3ea8e ("vfs: syscall: Add move_mount(2) to move mounts around") CC: stable@vger.kernel.org Reported-by: Andreas Steinmetz Signed-off-by: John Johansen --- security/apparmor/include/mount.h | 7 +++-- security/apparmor/lsm.c | 20 ++++++++++++-- security/apparmor/mount.c | 46 +++++++++++++++++++------------ 3 files changed, 51 insertions(+), 22 deletions(-) diff --git a/security/apparmor/include/mount.h b/security/apparmor/include/mount.h index 10c76f906a65..46834f828179 100644 --- a/security/apparmor/include/mount.h +++ b/security/apparmor/include/mount.h @@ -38,9 +38,12 @@ int aa_mount_change_type(const struct cred *subj_cred, struct aa_label *label, const struct path *path, unsigned long flags); +int aa_move_mount_old(const struct cred *subj_cred, + struct aa_label *label, const struct path *path, + const char *old_name); int aa_move_mount(const struct cred *subj_cred, - struct aa_label *label, const struct path *path, - const char *old_name); + struct aa_label *label, const struct path *from_path, + const struct path *to_path); int aa_new_mount(const struct cred *subj_cred, struct aa_label *label, const char *dev_name, diff --git a/security/apparmor/lsm.c b/security/apparmor/lsm.c index ce4f3e7a784d..b047d1d355a9 100644 --- a/security/apparmor/lsm.c +++ b/security/apparmor/lsm.c @@ -722,8 +722,8 @@ static int apparmor_sb_mount(const char *dev_name, const struct path *path, error = aa_mount_change_type(current_cred(), label, path, flags); else if (flags & MS_MOVE) - error = aa_move_mount(current_cred(), label, path, - dev_name); + error = aa_move_mount_old(current_cred(), label, path, + dev_name); else error = aa_new_mount(current_cred(), label, dev_name, path, type, flags, data); @@ -733,6 +733,21 @@ static int apparmor_sb_mount(const char *dev_name, const struct path *path, return error; } +static int apparmor_move_mount(const struct path *from_path, + const struct path *to_path) +{ + struct aa_label *label; + int error = 0; + + label = __begin_current_label_crit_section(); + if (!unconfined(label)) + error = aa_move_mount(current_cred(), label, from_path, + to_path); + __end_current_label_crit_section(label); + + return error; +} + static int apparmor_sb_umount(struct vfsmount *mnt, int flags) { struct aa_label *label; @@ -1376,6 +1391,7 @@ static struct security_hook_list apparmor_hooks[] __ro_after_init = { LSM_HOOK_INIT(capget, apparmor_capget), LSM_HOOK_INIT(capable, apparmor_capable), + LSM_HOOK_INIT(move_mount, apparmor_move_mount), LSM_HOOK_INIT(sb_mount, apparmor_sb_mount), LSM_HOOK_INIT(sb_umount, apparmor_sb_umount), LSM_HOOK_INIT(sb_pivotroot, apparmor_sb_pivotroot), diff --git a/security/apparmor/mount.c b/security/apparmor/mount.c index 3455dd4b1f99..fb30204c761a 100644 --- a/security/apparmor/mount.c +++ b/security/apparmor/mount.c @@ -483,36 +483,46 @@ int aa_mount_change_type(const struct cred *subj_cred, } int aa_move_mount(const struct cred *subj_cred, - struct aa_label *label, const struct path *path, - const char *orig_name) + struct aa_label *label, const struct path *from_path, + const struct path *to_path) { struct aa_profile *profile; - char *buffer = NULL, *old_buffer = NULL; - struct path old_path; + char *to_buffer = NULL, *from_buffer = NULL; int error; AA_BUG(!label); - AA_BUG(!path); + AA_BUG(!from_path); + AA_BUG(!to_path); + + to_buffer = aa_get_buffer(false); + from_buffer = aa_get_buffer(false); + error = -ENOMEM; + if (!to_buffer || !from_buffer) + goto out; + error = fn_for_each_confined(label, profile, + match_mnt(subj_cred, profile, to_path, to_buffer, + from_path, from_buffer, + NULL, MS_MOVE, NULL, false)); +out: + aa_put_buffer(to_buffer); + aa_put_buffer(from_buffer); + + return error; +} + +int aa_move_mount_old(const struct cred *subj_cred, struct aa_label *label, + const struct path *path, const char *orig_name) +{ + struct path old_path; + int error; if (!orig_name || !*orig_name) return -EINVAL; - error = kern_path(orig_name, LOOKUP_FOLLOW, &old_path); if (error) return error; - buffer = aa_get_buffer(false); - old_buffer = aa_get_buffer(false); - error = -ENOMEM; - if (!buffer || !old_buffer) - goto out; - error = fn_for_each_confined(label, profile, - match_mnt(subj_cred, profile, path, buffer, &old_path, - old_buffer, - NULL, MS_MOVE, NULL, false)); -out: - aa_put_buffer(buffer); - aa_put_buffer(old_buffer); + error = aa_move_mount(subj_cred, label, &old_path, path); path_put(&old_path); return error; From bdcb37a5d8de3253da48b120e3f10863696fb654 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 17 Oct 2023 16:33:58 +0200 Subject: [PATCH 274/475] buildid: reduce header file dependencies for module The vmlinux decompressor code intentionally has only a limited set of included header files, but this started running into a build failure because of the bitmap logic needing linux/errno.h: In file included from include/linux/cpumask.h:12, from include/linux/mm_types_task.h:14, from include/linux/mm_types.h:5, from include/linux/buildid.h:5, from include/linux/module.h:14, from arch/arm/boot/compressed/../../../../lib/lz4/lz4_decompress.c:39, from arch/arm/boot/compressed/../../../../lib/decompress_unlz4.c:10, from arch/arm/boot/compressed/decompress.c:60: include/linux/bitmap.h: In function 'bitmap_allocate_region': include/linux/bitmap.h:527:25: error: 'EBUSY' undeclared (first use in this function) 527 | return -EBUSY; | ^~~~~ include/linux/bitmap.h:527:25: note: each undeclared identifier is reported only once for each function it appears in include/linux/bitmap.h: In function 'bitmap_find_free_region': include/linux/bitmap.h:554:17: error: 'ENOMEM' undeclared (first use in this function) 554 | return -ENOMEM; | ^~~~~~ This is easily avoided by changing linux/buildid.h to no longer depend on linux/mm_types.h, a header that pulls in a huge number of indirect dependencies. Fixes: b9c957f554442 ("bitmap: move bitmap_*_region() functions to bitmap.h") Fixes: bd7525dacd7e2 ("bpf: Move stack_map_get_build_id into lib") Signed-off-by: Arnd Bergmann Signed-off-by: Yury Norov --- include/linux/buildid.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/linux/buildid.h b/include/linux/buildid.h index 3b7a0ff4642f..8a582d242f06 100644 --- a/include/linux/buildid.h +++ b/include/linux/buildid.h @@ -2,10 +2,11 @@ #ifndef _LINUX_BUILDID_H #define _LINUX_BUILDID_H -#include +#include #define BUILD_ID_SIZE_MAX 20 +struct vm_area_struct; int build_id_parse(struct vm_area_struct *vma, unsigned char *build_id, __u32 *size); int build_id_parse_buf(const void *buf, unsigned char *build_id, u32 buf_size); From cc8ee288f484a2a59c01ccd4d8a417d6ed3466e3 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 25 Sep 2023 20:31:17 +0200 Subject: [PATCH 275/475] powerpc/40x: Remove stale PTE_ATOMIC_UPDATES macro 40x TLB handlers were reworked by commit 2c74e2586bb9 ("powerpc/40x: Rework 40x PTE access and TLB miss") to not require PTE_ATOMIC_UPDATES anymore. Then commit 4e1df545e2fa ("powerpc/pgtable: Drop PTE_ATOMIC_UPDATES") removed all code related to PTE_ATOMIC_UPDATES. Remove left over PTE_ATOMIC_UPDATES macro. Fixes: 2c74e2586bb9 ("powerpc/40x: Rework 40x PTE access and TLB miss") Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://msgid.link/f061db5857fcd748f84a6707aad01754686ce97e.1695659959.git.christophe.leroy@csgroup.eu --- arch/powerpc/include/asm/nohash/32/pte-40x.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/arch/powerpc/include/asm/nohash/32/pte-40x.h b/arch/powerpc/include/asm/nohash/32/pte-40x.h index 6fe46e754556..0b4e5f8ce3e8 100644 --- a/arch/powerpc/include/asm/nohash/32/pte-40x.h +++ b/arch/powerpc/include/asm/nohash/32/pte-40x.h @@ -69,9 +69,6 @@ #define _PTE_NONE_MASK 0 -/* Until my rework is finished, 40x still needs atomic PTE updates */ -#define PTE_ATOMIC_UPDATES 1 - #define _PAGE_BASE_NC (_PAGE_PRESENT | _PAGE_ACCESSED) #define _PAGE_BASE (_PAGE_BASE_NC) From 3b8547ec4d35778c9f4cc261d85c0cae6c1a8ecb Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 25 Sep 2023 20:31:18 +0200 Subject: [PATCH 276/475] powerpc: Remove pte_ERROR() pte_ERROR() is used neither in powerpc code nor in common mm code. Remove it. Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://msgid.link/bec9eb973ecc1cba091e5c9201d877a7797f3242.1695659959.git.christophe.leroy@csgroup.eu --- arch/powerpc/include/asm/book3s/32/pgtable.h | 3 --- arch/powerpc/include/asm/book3s/64/pgtable.h | 2 -- arch/powerpc/include/asm/nohash/32/pgtable.h | 3 --- arch/powerpc/include/asm/nohash/64/pgtable.h | 2 -- 4 files changed, 10 deletions(-) diff --git a/arch/powerpc/include/asm/book3s/32/pgtable.h b/arch/powerpc/include/asm/book3s/32/pgtable.h index 9b13eb14e21b..543c3691839b 100644 --- a/arch/powerpc/include/asm/book3s/32/pgtable.h +++ b/arch/powerpc/include/asm/book3s/32/pgtable.h @@ -224,9 +224,6 @@ void unmap_kernel_page(unsigned long va); /* Bits to mask out from a PGD to get to the PUD page */ #define PGD_MASKED_BITS 0 -#define pte_ERROR(e) \ - pr_err("%s:%d: bad pte %llx.\n", __FILE__, __LINE__, \ - (unsigned long long)pte_val(e)) #define pgd_ERROR(e) \ pr_err("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e)) /* diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h index 5c497c862d75..7c4ad1e03a49 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -1014,8 +1014,6 @@ static inline pmd_t *pud_pgtable(pud_t pud) return (pmd_t *)__va(pud_val(pud) & ~PUD_MASKED_BITS); } -#define pte_ERROR(e) \ - pr_err("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e)) #define pmd_ERROR(e) \ pr_err("%s:%d: bad pmd %08lx.\n", __FILE__, __LINE__, pmd_val(e)) #define pud_ERROR(e) \ diff --git a/arch/powerpc/include/asm/nohash/32/pgtable.h b/arch/powerpc/include/asm/nohash/32/pgtable.h index f99c53a5f184..868aecbec8d1 100644 --- a/arch/powerpc/include/asm/nohash/32/pgtable.h +++ b/arch/powerpc/include/asm/nohash/32/pgtable.h @@ -55,9 +55,6 @@ extern int icache_44x_need_flush; #define USER_PTRS_PER_PGD (TASK_SIZE / PGDIR_SIZE) -#define pte_ERROR(e) \ - pr_err("%s:%d: bad pte %llx.\n", __FILE__, __LINE__, \ - (unsigned long long)pte_val(e)) #define pgd_ERROR(e) \ pr_err("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e)) diff --git a/arch/powerpc/include/asm/nohash/64/pgtable.h b/arch/powerpc/include/asm/nohash/64/pgtable.h index eb6891e34cbd..8083c04a1e6d 100644 --- a/arch/powerpc/include/asm/nohash/64/pgtable.h +++ b/arch/powerpc/include/asm/nohash/64/pgtable.h @@ -269,8 +269,6 @@ static inline void __ptep_set_access_flags(struct vm_area_struct *vma, flush_tlb_page(vma, address); } -#define pte_ERROR(e) \ - pr_err("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e)) #define pmd_ERROR(e) \ pr_err("%s:%d: bad pmd %08lx.\n", __FILE__, __LINE__, pmd_val(e)) #define pgd_ERROR(e) \ From 93f81f6eea10f497e892c52998a2194b4e16c91d Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 25 Sep 2023 20:31:19 +0200 Subject: [PATCH 277/475] powerpc: Deduplicate prototypes of ptep_set_access_flags() and phys_mem_access_prot() Prototypes of ptep_set_access_flags() and phys_mem_access_prot() are identical for book3s and nohash. Deduplicate them. Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://msgid.link/b846832cce842a2852615b7356937fb9507e436d.1695659959.git.christophe.leroy@csgroup.eu --- arch/powerpc/include/asm/book3s/pgtable.h | 9 --------- arch/powerpc/include/asm/nohash/pgtable.h | 10 ---------- arch/powerpc/include/asm/pgtable.h | 10 ++++++++++ 3 files changed, 10 insertions(+), 19 deletions(-) diff --git a/arch/powerpc/include/asm/book3s/pgtable.h b/arch/powerpc/include/asm/book3s/pgtable.h index 3b7bd36a2321..6f4578daea6c 100644 --- a/arch/powerpc/include/asm/book3s/pgtable.h +++ b/arch/powerpc/include/asm/book3s/pgtable.h @@ -9,15 +9,6 @@ #endif #ifndef __ASSEMBLY__ -#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS -extern int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long address, - pte_t *ptep, pte_t entry, int dirty); - -struct file; -extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, - unsigned long size, pgprot_t vma_prot); -#define __HAVE_PHYS_MEM_ACCESS_PROT - void __update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *ptep); /* diff --git a/arch/powerpc/include/asm/nohash/pgtable.h b/arch/powerpc/include/asm/nohash/pgtable.h index c721478c5934..5b6647fb398b 100644 --- a/arch/powerpc/include/asm/nohash/pgtable.h +++ b/arch/powerpc/include/asm/nohash/pgtable.h @@ -207,11 +207,6 @@ static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr, mb(); } - -#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS -extern int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long address, - pte_t *ptep, pte_t entry, int dirty); - /* * Macro to mark a page protection value as "uncacheable". */ @@ -240,11 +235,6 @@ extern int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long addre #define pgprot_writecombine pgprot_noncached_wc -struct file; -extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, - unsigned long size, pgprot_t vma_prot); -#define __HAVE_PHYS_MEM_ACCESS_PROT - #ifdef CONFIG_HUGETLB_PAGE static inline int hugepd_ok(hugepd_t hpd) { diff --git a/arch/powerpc/include/asm/pgtable.h b/arch/powerpc/include/asm/pgtable.h index d0ee46de248e..bcdbdeda65d3 100644 --- a/arch/powerpc/include/asm/pgtable.h +++ b/arch/powerpc/include/asm/pgtable.h @@ -110,6 +110,16 @@ void mark_initmem_nx(void); static inline void mark_initmem_nx(void) { } #endif +#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS +int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long address, + pte_t *ptep, pte_t entry, int dirty); + +struct file; +pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, + unsigned long size, pgprot_t vma_prot); +#define __HAVE_PHYS_MEM_ACCESS_PROT + + /* * When used, PTE_FRAG_NR is defined in subarch pgtable.h * so we are sure it is included when arriving here. From da9554e0fe3c7b46912a361a803b50f2655ff30f Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 25 Sep 2023 20:31:20 +0200 Subject: [PATCH 278/475] powerpc: Refactor update_mmu_cache_range() On nohash, this function voids except for E500 with hugepages. On book3s, this function is for hash MMUs only. Combine those tests and rename E500 update_mmu_cache_range() as __update_mmu_cache() which gets called by update_mmu_cache_range(). Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://msgid.link/b029842cb6783cbeb43d202e69a90341d65295a4.1695659959.git.christophe.leroy@csgroup.eu --- arch/powerpc/include/asm/book3s/pgtable.h | 24 ----------------------- arch/powerpc/include/asm/nohash/pgtable.h | 15 -------------- arch/powerpc/include/asm/pgtable.h | 19 ++++++++++++++++++ arch/powerpc/mm/nohash/e500_hugetlbpage.c | 3 +-- 4 files changed, 20 insertions(+), 41 deletions(-) diff --git a/arch/powerpc/include/asm/book3s/pgtable.h b/arch/powerpc/include/asm/book3s/pgtable.h index 6f4578daea6c..f42d68c6b314 100644 --- a/arch/powerpc/include/asm/book3s/pgtable.h +++ b/arch/powerpc/include/asm/book3s/pgtable.h @@ -8,28 +8,4 @@ #include #endif -#ifndef __ASSEMBLY__ -void __update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *ptep); - -/* - * This gets called at the end of handling a page fault, when - * the kernel has put a new PTE into the page table for the process. - * We use it to ensure coherency between the i-cache and d-cache - * for the page which has just been mapped in. - * On machines which use an MMU hash table, we use this to put a - * corresponding HPTE into the hash table ahead of time, instead of - * waiting for the inevitable extra hash-table miss exception. - */ -static inline void update_mmu_cache_range(struct vm_fault *vmf, - struct vm_area_struct *vma, unsigned long address, - pte_t *ptep, unsigned int nr) -{ - if (IS_ENABLED(CONFIG_PPC32) && !mmu_has_feature(MMU_FTR_HPTE_TABLE)) - return; - if (radix_enabled()) - return; - __update_mmu_cache(vma, address, ptep); -} - -#endif /* __ASSEMBLY__ */ #endif diff --git a/arch/powerpc/include/asm/nohash/pgtable.h b/arch/powerpc/include/asm/nohash/pgtable.h index 5b6647fb398b..a9056f4fad48 100644 --- a/arch/powerpc/include/asm/nohash/pgtable.h +++ b/arch/powerpc/include/asm/nohash/pgtable.h @@ -259,20 +259,5 @@ static inline int pud_huge(pud_t pud) #define is_hugepd(hpd) (hugepd_ok(hpd)) #endif -/* - * This gets called at the end of handling a page fault, when - * the kernel has put a new PTE into the page table for the process. - * We use it to ensure coherency between the i-cache and d-cache - * for the page which has just been mapped in. - */ -#if defined(CONFIG_PPC_E500) && defined(CONFIG_HUGETLB_PAGE) -void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, - unsigned long address, pte_t *ptep, unsigned int nr); -#else -static inline void update_mmu_cache_range(struct vm_fault *vmf, - struct vm_area_struct *vma, unsigned long address, - pte_t *ptep, unsigned int nr) {} -#endif - #endif /* __ASSEMBLY__ */ #endif diff --git a/arch/powerpc/include/asm/pgtable.h b/arch/powerpc/include/asm/pgtable.h index bcdbdeda65d3..966e7c5119f6 100644 --- a/arch/powerpc/include/asm/pgtable.h +++ b/arch/powerpc/include/asm/pgtable.h @@ -119,6 +119,25 @@ pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, unsigned long size, pgprot_t vma_prot); #define __HAVE_PHYS_MEM_ACCESS_PROT +void __update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *ptep); + +/* + * This gets called at the end of handling a page fault, when + * the kernel has put a new PTE into the page table for the process. + * We use it to ensure coherency between the i-cache and d-cache + * for the page which has just been mapped in. + * On machines which use an MMU hash table, we use this to put a + * corresponding HPTE into the hash table ahead of time, instead of + * waiting for the inevitable extra hash-table miss exception. + */ +static inline void update_mmu_cache_range(struct vm_fault *vmf, + struct vm_area_struct *vma, unsigned long address, + pte_t *ptep, unsigned int nr) +{ + if ((mmu_has_feature(MMU_FTR_HPTE_TABLE) && !radix_enabled()) || + (IS_ENABLED(CONFIG_PPC_E500) && IS_ENABLED(CONFIG_HUGETLB_PAGE))) + __update_mmu_cache(vma, address, ptep); +} /* * When used, PTE_FRAG_NR is defined in subarch pgtable.h diff --git a/arch/powerpc/mm/nohash/e500_hugetlbpage.c b/arch/powerpc/mm/nohash/e500_hugetlbpage.c index 6b30e40d4590..a134d28a0e4d 100644 --- a/arch/powerpc/mm/nohash/e500_hugetlbpage.c +++ b/arch/powerpc/mm/nohash/e500_hugetlbpage.c @@ -178,8 +178,7 @@ book3e_hugetlb_preload(struct vm_area_struct *vma, unsigned long ea, pte_t pte) * * This must always be called with the pte lock held. */ -void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, - unsigned long address, pte_t *ptep, unsigned int nr) +void __update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *ptep) { if (is_vm_hugetlb_page(vma)) book3e_hugetlb_preload(vma, address, *ptep); From d3e01796728add53ab778298573772d44d52d19c Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 25 Sep 2023 20:31:21 +0200 Subject: [PATCH 279/475] powerpc: Untangle fixmap.h and pgtable.h and mmu.h fixmap.h need pgtable.h for [un]map_kernel_page() pgtable.h need fixmap.h for FIXADDR_TOP. Untangle the two files by moving FIXADDR_TOP into pgtable.h Also move VIRT_IMMR_BASE to fixmap.h to avoid fixmap.h in mmu.h Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://msgid.link/5eba12392a018be28ad0a02ed844767b132589e7.1695659959.git.christophe.leroy@csgroup.eu --- arch/powerpc/include/asm/book3s/32/pgtable.h | 9 ++++++++- arch/powerpc/include/asm/book3s/64/pgtable.h | 1 + arch/powerpc/include/asm/fixmap.h | 16 ++++------------ arch/powerpc/include/asm/nohash/32/mmu-8xx.h | 1 - arch/powerpc/include/asm/nohash/32/pgtable.h | 9 ++++++++- arch/powerpc/include/asm/nohash/64/pgtable.h | 1 + arch/powerpc/mm/init_32.c | 1 + arch/powerpc/mm/mem.c | 1 + arch/powerpc/mm/nohash/8xx.c | 2 ++ arch/powerpc/platforms/83xx/misc.c | 2 ++ arch/powerpc/platforms/8xx/cpm1.c | 1 + 11 files changed, 29 insertions(+), 15 deletions(-) diff --git a/arch/powerpc/include/asm/book3s/32/pgtable.h b/arch/powerpc/include/asm/book3s/32/pgtable.h index 543c3691839b..45b69ae2631e 100644 --- a/arch/powerpc/include/asm/book3s/32/pgtable.h +++ b/arch/powerpc/include/asm/book3s/32/pgtable.h @@ -170,7 +170,14 @@ void unmap_kernel_page(unsigned long va); * value (for now) on others, from where we can start layout kernel * virtual space that goes below PKMAP and FIXMAP */ -#include + +#define FIXADDR_SIZE 0 +#ifdef CONFIG_KASAN +#include +#define FIXADDR_TOP (KASAN_SHADOW_START - PAGE_SIZE) +#else +#define FIXADDR_TOP ((unsigned long)(-PAGE_SIZE)) +#endif /* * ioremap_bot starts at that address. Early ioremaps move down from there, diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h index 7c4ad1e03a49..dbd545e73161 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -316,6 +316,7 @@ extern unsigned long pci_io_base; #define IOREMAP_START (ioremap_bot) #define IOREMAP_END (KERN_IO_END - FIXADDR_SIZE) #define FIXADDR_SIZE SZ_32M +#define FIXADDR_TOP (IOREMAP_END + FIXADDR_SIZE) #ifndef __ASSEMBLY__ diff --git a/arch/powerpc/include/asm/fixmap.h b/arch/powerpc/include/asm/fixmap.h index a832aeafe560..f9068dd8dfce 100644 --- a/arch/powerpc/include/asm/fixmap.h +++ b/arch/powerpc/include/asm/fixmap.h @@ -23,18 +23,6 @@ #include #endif -#ifdef CONFIG_PPC64 -#define FIXADDR_TOP (IOREMAP_END + FIXADDR_SIZE) -#else -#define FIXADDR_SIZE 0 -#ifdef CONFIG_KASAN -#include -#define FIXADDR_TOP (KASAN_SHADOW_START - PAGE_SIZE) -#else -#define FIXADDR_TOP ((unsigned long)(-PAGE_SIZE)) -#endif -#endif - /* * Here we define all the compile-time 'special' virtual * addresses. The point is to have a constant address at @@ -119,5 +107,9 @@ static inline void __set_fixmap(enum fixed_addresses idx, #define __early_set_fixmap __set_fixmap +#ifdef CONFIG_PPC_8xx +#define VIRT_IMMR_BASE (__fix_to_virt(FIX_IMMR_BASE)) +#endif + #endif /* !__ASSEMBLY__ */ #endif diff --git a/arch/powerpc/include/asm/nohash/32/mmu-8xx.h b/arch/powerpc/include/asm/nohash/32/mmu-8xx.h index 0e93a4728c9e..141d82e249a8 100644 --- a/arch/powerpc/include/asm/nohash/32/mmu-8xx.h +++ b/arch/powerpc/include/asm/nohash/32/mmu-8xx.h @@ -188,7 +188,6 @@ typedef struct { } mm_context_t; #define PHYS_IMMR_BASE (mfspr(SPRN_IMMR) & 0xfff80000) -#define VIRT_IMMR_BASE (__fix_to_virt(FIX_IMMR_BASE)) /* Page size definitions, common between 32 and 64-bit * diff --git a/arch/powerpc/include/asm/nohash/32/pgtable.h b/arch/powerpc/include/asm/nohash/32/pgtable.h index 868aecbec8d1..c8311ee08811 100644 --- a/arch/powerpc/include/asm/nohash/32/pgtable.h +++ b/arch/powerpc/include/asm/nohash/32/pgtable.h @@ -71,7 +71,14 @@ void unmap_kernel_page(unsigned long va); * value (for now) on others, from where we can start layout kernel * virtual space that goes below PKMAP and FIXMAP */ -#include + +#define FIXADDR_SIZE 0 +#ifdef CONFIG_KASAN +#include +#define FIXADDR_TOP (KASAN_SHADOW_START - PAGE_SIZE) +#else +#define FIXADDR_TOP ((unsigned long)(-PAGE_SIZE)) +#endif /* * ioremap_bot starts at that address. Early ioremaps move down from there, diff --git a/arch/powerpc/include/asm/nohash/64/pgtable.h b/arch/powerpc/include/asm/nohash/64/pgtable.h index 8083c04a1e6d..dee3fc654d40 100644 --- a/arch/powerpc/include/asm/nohash/64/pgtable.h +++ b/arch/powerpc/include/asm/nohash/64/pgtable.h @@ -57,6 +57,7 @@ #define IOREMAP_START (ioremap_bot) #define IOREMAP_END (KERN_IO_START + KERN_IO_SIZE - FIXADDR_SIZE) #define FIXADDR_SIZE SZ_32M +#define FIXADDR_TOP (IOREMAP_END + FIXADDR_SIZE) /* * Defines the address of the vmemap area, in its own region on diff --git a/arch/powerpc/mm/init_32.c b/arch/powerpc/mm/init_32.c index d8adc452f431..4e71dfe7d026 100644 --- a/arch/powerpc/mm/init_32.c +++ b/arch/powerpc/mm/init_32.c @@ -39,6 +39,7 @@ #include #include #include +#include #include diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c index 8b121df7b08f..08f3ec9d522b 100644 --- a/arch/powerpc/mm/mem.c +++ b/arch/powerpc/mm/mem.c @@ -26,6 +26,7 @@ #include #include #include +#include #include diff --git a/arch/powerpc/mm/nohash/8xx.c b/arch/powerpc/mm/nohash/8xx.c index dbbfe897455d..bb9c39b449d1 100644 --- a/arch/powerpc/mm/nohash/8xx.c +++ b/arch/powerpc/mm/nohash/8xx.c @@ -10,6 +10,8 @@ #include #include +#include + #include #define IMMR_SIZE (FIX_IMMR_SIZE << PAGE_SHIFT) diff --git a/arch/powerpc/platforms/83xx/misc.c b/arch/powerpc/platforms/83xx/misc.c index 2fb2a85d131f..1135c1ab923c 100644 --- a/arch/powerpc/platforms/83xx/misc.c +++ b/arch/powerpc/platforms/83xx/misc.c @@ -14,6 +14,8 @@ #include #include #include +#include + #include #include diff --git a/arch/powerpc/platforms/8xx/cpm1.c b/arch/powerpc/platforms/8xx/cpm1.c index ebb5f6a27dbf..b24d4102fbf6 100644 --- a/arch/powerpc/platforms/8xx/cpm1.c +++ b/arch/powerpc/platforms/8xx/cpm1.c @@ -40,6 +40,7 @@ #include #include #include +#include #include From 81fbb9997057b6e6e5795a08d9a8e10e9f48236f Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 25 Sep 2023 20:31:22 +0200 Subject: [PATCH 280/475] powerpc/nohash: Remove {pte/pmd}_protnone() Only book3s/64 selects ARCH_SUPPORTS_NUMA_BALANCING so CONFIG_NUMA_BALANCING can't be selected on nohash targets. Remove pte_protnone() and pmd_protnone(). Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://msgid.link/a50c1a19828a8eced82cbcc5c61754b667037d21.1695659959.git.christophe.leroy@csgroup.eu --- arch/powerpc/include/asm/nohash/pgtable.h | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/arch/powerpc/include/asm/nohash/pgtable.h b/arch/powerpc/include/asm/nohash/pgtable.h index a9056f4fad48..ab26af2b421a 100644 --- a/arch/powerpc/include/asm/nohash/pgtable.h +++ b/arch/powerpc/include/asm/nohash/pgtable.h @@ -35,23 +35,6 @@ static inline bool pte_hashpte(pte_t pte) { return false; } static inline bool pte_ci(pte_t pte) { return pte_val(pte) & _PAGE_NO_CACHE; } static inline bool pte_exec(pte_t pte) { return pte_val(pte) & _PAGE_EXEC; } -#ifdef CONFIG_NUMA_BALANCING -/* - * These work without NUMA balancing but the kernel does not care. See the - * comment in include/linux/pgtable.h . On powerpc, this will only - * work for user pages and always return true for kernel pages. - */ -static inline int pte_protnone(pte_t pte) -{ - return pte_present(pte) && !pte_user(pte); -} - -static inline int pmd_protnone(pmd_t pmd) -{ - return pte_protnone(pmd_pte(pmd)); -} -#endif /* CONFIG_NUMA_BALANCING */ - static inline int pte_present(pte_t pte) { return pte_val(pte) & _PAGE_PRESENT; From 7835006979e5415aa4c9bc0e3e7063b5c5943ed4 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 25 Sep 2023 20:31:23 +0200 Subject: [PATCH 281/475] powerpc/nohash: Refactor declaration of {map/unmap}_kernel_page() map_kernel_page() and unmap_kernel_page() have the same prototypes on nohash/32 and nohash/64, keep only one declaration. Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://msgid.link/7fec5f3288cf0d0eac61b1b3f48c3ea54eb80cad.1695659959.git.christophe.leroy@csgroup.eu --- arch/powerpc/include/asm/nohash/32/pgtable.h | 8 -------- arch/powerpc/include/asm/nohash/64/pgtable.h | 2 -- arch/powerpc/include/asm/nohash/pgtable.h | 3 +++ arch/powerpc/mm/nohash/book3e_pgtable.c | 2 +- 4 files changed, 4 insertions(+), 11 deletions(-) diff --git a/arch/powerpc/include/asm/nohash/32/pgtable.h b/arch/powerpc/include/asm/nohash/32/pgtable.h index c8311ee08811..26289e4e767c 100644 --- a/arch/powerpc/include/asm/nohash/32/pgtable.h +++ b/arch/powerpc/include/asm/nohash/32/pgtable.h @@ -58,14 +58,6 @@ extern int icache_44x_need_flush; #define pgd_ERROR(e) \ pr_err("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e)) -#ifndef __ASSEMBLY__ - -int map_kernel_page(unsigned long va, phys_addr_t pa, pgprot_t prot); -void unmap_kernel_page(unsigned long va); - -#endif /* !__ASSEMBLY__ */ - - /* * This is the bottom of the PKMAP area with HIGHMEM or an arbitrary * value (for now) on others, from where we can start layout kernel diff --git a/arch/powerpc/include/asm/nohash/64/pgtable.h b/arch/powerpc/include/asm/nohash/64/pgtable.h index dee3fc654d40..f5a8e8a9dba4 100644 --- a/arch/powerpc/include/asm/nohash/64/pgtable.h +++ b/arch/powerpc/include/asm/nohash/64/pgtable.h @@ -309,8 +309,6 @@ static inline void __ptep_set_access_flags(struct vm_area_struct *vma, /* We borrow MSB 56 (LSB 7) to store the exclusive marker in swap PTEs. */ #define _PAGE_SWP_EXCLUSIVE 0x80 -int map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot); -void unmap_kernel_page(unsigned long va); extern int __meminit vmemmap_create_mapping(unsigned long start, unsigned long page_size, unsigned long phys); diff --git a/arch/powerpc/include/asm/nohash/pgtable.h b/arch/powerpc/include/asm/nohash/pgtable.h index ab26af2b421a..3d684b500fe6 100644 --- a/arch/powerpc/include/asm/nohash/pgtable.h +++ b/arch/powerpc/include/asm/nohash/pgtable.h @@ -242,5 +242,8 @@ static inline int pud_huge(pud_t pud) #define is_hugepd(hpd) (hugepd_ok(hpd)) #endif +int map_kernel_page(unsigned long va, phys_addr_t pa, pgprot_t prot); +void unmap_kernel_page(unsigned long va); + #endif /* __ASSEMBLY__ */ #endif diff --git a/arch/powerpc/mm/nohash/book3e_pgtable.c b/arch/powerpc/mm/nohash/book3e_pgtable.c index b80fc4a91a53..1c5e4ecbebeb 100644 --- a/arch/powerpc/mm/nohash/book3e_pgtable.c +++ b/arch/powerpc/mm/nohash/book3e_pgtable.c @@ -71,7 +71,7 @@ static void __init *early_alloc_pgtable(unsigned long size) * map_kernel_page adds an entry to the ioremap page table * and adds an entry to the HPT, possibly bolting it */ -int __ref map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot) +int __ref map_kernel_page(unsigned long ea, phys_addr_t pa, pgprot_t prot) { pgd_t *pgdp; p4d_t *p4dp; From 4c1a89d983be951a3e39d7f9c1d6987f3054e32d Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 25 Sep 2023 20:31:24 +0200 Subject: [PATCH 282/475] powerpc/nohash: Move 8xx version of pte_update() into pte-8xx.h No point in having 8xx special pte_update() in common header, move it into pte-8xx.h Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://msgid.link/17e209b1a1a43ed219e9e1f2947ec594ed4f9394.1695659959.git.christophe.leroy@csgroup.eu --- arch/powerpc/include/asm/nohash/32/pgtable.h | 57 +------------------- arch/powerpc/include/asm/nohash/32/pte-8xx.h | 57 ++++++++++++++++++++ 2 files changed, 58 insertions(+), 56 deletions(-) diff --git a/arch/powerpc/include/asm/nohash/32/pgtable.h b/arch/powerpc/include/asm/nohash/32/pgtable.h index 26289e4e767c..be8bca42bdce 100644 --- a/arch/powerpc/include/asm/nohash/32/pgtable.h +++ b/arch/powerpc/include/asm/nohash/32/pgtable.h @@ -219,63 +219,8 @@ static inline void pmd_clear(pmd_t *pmdp) * that an executable user mapping was modified, which is needed * to properly flush the virtually tagged instruction cache of * those implementations. - * - * On the 8xx, the page tables are a bit special. For 16k pages, we have - * 4 identical entries. For 512k pages, we have 128 entries as if it was - * 4k pages, but they are flagged as 512k pages for the hardware. - * For other page sizes, we have a single entry in the table. */ -#ifdef CONFIG_PPC_8xx -static pmd_t *pmd_off(struct mm_struct *mm, unsigned long addr); -static int hugepd_ok(hugepd_t hpd); - -static int number_of_cells_per_pte(pmd_t *pmd, pte_basic_t val, int huge) -{ - if (!huge) - return PAGE_SIZE / SZ_4K; - else if (hugepd_ok(*((hugepd_t *)pmd))) - return 1; - else if (IS_ENABLED(CONFIG_PPC_4K_PAGES) && !(val & _PAGE_HUGE)) - return SZ_16K / SZ_4K; - else - return SZ_512K / SZ_4K; -} - -static inline pte_basic_t pte_update(struct mm_struct *mm, unsigned long addr, pte_t *p, - unsigned long clr, unsigned long set, int huge) -{ - pte_basic_t *entry = (pte_basic_t *)p; - pte_basic_t old = pte_val(*p); - pte_basic_t new = (old & ~(pte_basic_t)clr) | set; - int num, i; - pmd_t *pmd = pmd_off(mm, addr); - - num = number_of_cells_per_pte(pmd, new, huge); - - for (i = 0; i < num; i += PAGE_SIZE / SZ_4K, new += PAGE_SIZE) { - *entry++ = new; - if (IS_ENABLED(CONFIG_PPC_16K_PAGES) && num != 1) { - *entry++ = new; - *entry++ = new; - *entry++ = new; - } - } - - return old; -} - -#ifdef CONFIG_PPC_16K_PAGES -#define ptep_get ptep_get -static inline pte_t ptep_get(pte_t *ptep) -{ - pte_basic_t val = READ_ONCE(ptep->pte); - pte_t pte = {val, val, val, val}; - - return pte; -} -#endif /* CONFIG_PPC_16K_PAGES */ - -#else +#ifndef pte_update static inline pte_basic_t pte_update(struct mm_struct *mm, unsigned long addr, pte_t *p, unsigned long clr, unsigned long set, int huge) { diff --git a/arch/powerpc/include/asm/nohash/32/pte-8xx.h b/arch/powerpc/include/asm/nohash/32/pte-8xx.h index e6fe1d5731f2..52395a5ecd70 100644 --- a/arch/powerpc/include/asm/nohash/32/pte-8xx.h +++ b/arch/powerpc/include/asm/nohash/32/pte-8xx.h @@ -187,6 +187,63 @@ static inline unsigned long pte_leaf_size(pte_t pte) #define pte_leaf_size pte_leaf_size +/* + * On the 8xx, the page tables are a bit special. For 16k pages, we have + * 4 identical entries. For 512k pages, we have 128 entries as if it was + * 4k pages, but they are flagged as 512k pages for the hardware. + * For other page sizes, we have a single entry in the table. + */ +static pmd_t *pmd_off(struct mm_struct *mm, unsigned long addr); +static int hugepd_ok(hugepd_t hpd); + +static inline int number_of_cells_per_pte(pmd_t *pmd, pte_basic_t val, int huge) +{ + if (!huge) + return PAGE_SIZE / SZ_4K; + else if (hugepd_ok(*((hugepd_t *)pmd))) + return 1; + else if (IS_ENABLED(CONFIG_PPC_4K_PAGES) && !(val & _PAGE_HUGE)) + return SZ_16K / SZ_4K; + else + return SZ_512K / SZ_4K; +} + +static inline pte_basic_t pte_update(struct mm_struct *mm, unsigned long addr, pte_t *p, + unsigned long clr, unsigned long set, int huge) +{ + pte_basic_t *entry = (pte_basic_t *)p; + pte_basic_t old = pte_val(*p); + pte_basic_t new = (old & ~(pte_basic_t)clr) | set; + int num, i; + pmd_t *pmd = pmd_off(mm, addr); + + num = number_of_cells_per_pte(pmd, new, huge); + + for (i = 0; i < num; i += PAGE_SIZE / SZ_4K, new += PAGE_SIZE) { + *entry++ = new; + if (IS_ENABLED(CONFIG_PPC_16K_PAGES) && num != 1) { + *entry++ = new; + *entry++ = new; + *entry++ = new; + } + } + + return old; +} + +#define pte_update pte_update + +#ifdef CONFIG_PPC_16K_PAGES +#define ptep_get ptep_get +static inline pte_t ptep_get(pte_t *ptep) +{ + pte_basic_t val = READ_ONCE(ptep->pte); + pte_t pte = {val, val, val, val}; + + return pte; +} +#endif /* CONFIG_PPC_16K_PAGES */ + #endif #endif /* __KERNEL__ */ From 0f4027eab59261f2fb72586f18efb44be3594dd4 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 25 Sep 2023 20:31:25 +0200 Subject: [PATCH 283/475] powerpc/nohash: Replace #ifdef CONFIG_44x by IS_ENABLED(CONFIG_44x) in pgtable.h No need of a #ifdef, use IS_ENABLED(CONFIG_44x) Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://msgid.link/7c7d97322a6a05a9842b1e8c4b41265916f542ca.1695659959.git.christophe.leroy@csgroup.eu --- arch/powerpc/include/asm/nohash/32/pgtable.h | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/arch/powerpc/include/asm/nohash/32/pgtable.h b/arch/powerpc/include/asm/nohash/32/pgtable.h index be8bca42bdce..a74476de1ef6 100644 --- a/arch/powerpc/include/asm/nohash/32/pgtable.h +++ b/arch/powerpc/include/asm/nohash/32/pgtable.h @@ -9,9 +9,7 @@ #include #include /* For sub-arch specific PPC_PIN_SIZE */ -#ifdef CONFIG_44x extern int icache_44x_need_flush; -#endif #endif /* __ASSEMBLY__ */ @@ -229,10 +227,9 @@ static inline pte_basic_t pte_update(struct mm_struct *mm, unsigned long addr, p *p = __pte(new); -#ifdef CONFIG_44x - if ((old & _PAGE_USER) && (old & _PAGE_EXEC)) + if (IS_ENABLED(CONFIG_44x) && (old & _PAGE_USER) && (old & _PAGE_EXEC)) icache_44x_need_flush = 1; -#endif + return old; } #endif From 42a2722319f0d3d5612ca8efd3ce7d7eae512291 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 25 Sep 2023 20:31:26 +0200 Subject: [PATCH 284/475] powerpc/nohash: Refactor pte_update() pte_update() is similar. Take the nohash/32 version which works on nohash/64 and add the debug call to assert_pte_locked() which is only on nohash/64. Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://msgid.link/e01cb630cad42f645915ce7702d23985241b71fc.1695659959.git.christophe.leroy@csgroup.eu --- arch/powerpc/include/asm/nohash/32/pgtable.h | 33 --------------- arch/powerpc/include/asm/nohash/64/pgtable.h | 17 -------- arch/powerpc/include/asm/nohash/pgtable.h | 42 ++++++++++++++++++++ 3 files changed, 42 insertions(+), 50 deletions(-) diff --git a/arch/powerpc/include/asm/nohash/32/pgtable.h b/arch/powerpc/include/asm/nohash/32/pgtable.h index a74476de1ef6..ae7f3c8afd4f 100644 --- a/arch/powerpc/include/asm/nohash/32/pgtable.h +++ b/arch/powerpc/include/asm/nohash/32/pgtable.h @@ -9,8 +9,6 @@ #include #include /* For sub-arch specific PPC_PIN_SIZE */ -extern int icache_44x_need_flush; - #endif /* __ASSEMBLY__ */ #define PTE_INDEX_SIZE PTE_SHIFT @@ -203,37 +201,6 @@ static inline void pmd_clear(pmd_t *pmdp) *pmdp = __pmd(0); } -/* - * PTE updates. This function is called whenever an existing - * valid PTE is updated. This does -not- include set_pte_at() - * which nowadays only sets a new PTE. - * - * Depending on the type of MMU, we may need to use atomic updates - * and the PTE may be either 32 or 64 bit wide. In the later case, - * when using atomic updates, only the low part of the PTE is - * accessed atomically. - * - * In addition, on 44x, we also maintain a global flag indicating - * that an executable user mapping was modified, which is needed - * to properly flush the virtually tagged instruction cache of - * those implementations. - */ -#ifndef pte_update -static inline pte_basic_t pte_update(struct mm_struct *mm, unsigned long addr, pte_t *p, - unsigned long clr, unsigned long set, int huge) -{ - pte_basic_t old = pte_val(*p); - pte_basic_t new = (old & ~(pte_basic_t)clr) | set; - - *p = __pte(new); - - if (IS_ENABLED(CONFIG_44x) && (old & _PAGE_USER) && (old & _PAGE_EXEC)) - icache_44x_need_flush = 1; - - return old; -} -#endif - #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG static inline int __ptep_test_and_clear_young(struct mm_struct *mm, unsigned long addr, pte_t *ptep) diff --git a/arch/powerpc/include/asm/nohash/64/pgtable.h b/arch/powerpc/include/asm/nohash/64/pgtable.h index f5a8e8a9dba4..b149a39f2685 100644 --- a/arch/powerpc/include/asm/nohash/64/pgtable.h +++ b/arch/powerpc/include/asm/nohash/64/pgtable.h @@ -171,23 +171,6 @@ static inline void p4d_set(p4d_t *p4dp, unsigned long val) *p4dp = __p4d(val); } -/* Atomic PTE updates */ -static inline unsigned long pte_update(struct mm_struct *mm, - unsigned long addr, - pte_t *ptep, unsigned long clr, - unsigned long set, - int huge) -{ - unsigned long old = pte_val(*ptep); - *ptep = __pte((old & ~clr) | set); - - /* huge pages use the old page table lock */ - if (!huge) - assert_pte_locked(mm, addr); - - return old; -} - static inline int pte_young(pte_t pte) { return pte_val(pte) & _PAGE_ACCESSED; diff --git a/arch/powerpc/include/asm/nohash/pgtable.h b/arch/powerpc/include/asm/nohash/pgtable.h index 3d684b500fe6..bd5c3a4baabd 100644 --- a/arch/powerpc/include/asm/nohash/pgtable.h +++ b/arch/powerpc/include/asm/nohash/pgtable.h @@ -2,6 +2,11 @@ #ifndef _ASM_POWERPC_NOHASH_PGTABLE_H #define _ASM_POWERPC_NOHASH_PGTABLE_H +#ifndef __ASSEMBLY__ +static inline pte_basic_t pte_update(struct mm_struct *mm, unsigned long addr, pte_t *p, + unsigned long clr, unsigned long set, int huge); +#endif + #if defined(CONFIG_PPC64) #include #else @@ -18,6 +23,43 @@ #ifndef __ASSEMBLY__ +extern int icache_44x_need_flush; + +/* + * PTE updates. This function is called whenever an existing + * valid PTE is updated. This does -not- include set_pte_at() + * which nowadays only sets a new PTE. + * + * Depending on the type of MMU, we may need to use atomic updates + * and the PTE may be either 32 or 64 bit wide. In the later case, + * when using atomic updates, only the low part of the PTE is + * accessed atomically. + * + * In addition, on 44x, we also maintain a global flag indicating + * that an executable user mapping was modified, which is needed + * to properly flush the virtually tagged instruction cache of + * those implementations. + */ +#ifndef pte_update +static inline pte_basic_t pte_update(struct mm_struct *mm, unsigned long addr, pte_t *p, + unsigned long clr, unsigned long set, int huge) +{ + pte_basic_t old = pte_val(*p); + pte_basic_t new = (old & ~(pte_basic_t)clr) | set; + + *p = __pte(new); + + if (IS_ENABLED(CONFIG_44x) && (old & _PAGE_USER) && (old & _PAGE_EXEC)) + icache_44x_need_flush = 1; + + /* huge pages use the old page table lock */ + if (!huge) + assert_pte_locked(mm, addr); + + return old; +} +#endif + /* Generic accessors to PTE bits */ #ifndef pte_write static inline int pte_write(pte_t pte) From 7c929ad0b3167e980a3963e03403a761138a4350 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 25 Sep 2023 20:31:27 +0200 Subject: [PATCH 285/475] powerpc/nohash: Refactor checking of no-change in pte_update() On nohash/64, a few callers of pte_update() check if there is really a change in order to avoid an unnecessary write. Refactor that inside pte_update(). Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://msgid.link/076563e611c2b51036686a8d378bfd5ef1726341.1695659959.git.christophe.leroy@csgroup.eu --- arch/powerpc/include/asm/nohash/64/pgtable.h | 9 --------- arch/powerpc/include/asm/nohash/pgtable.h | 3 +++ 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/arch/powerpc/include/asm/nohash/64/pgtable.h b/arch/powerpc/include/asm/nohash/64/pgtable.h index b149a39f2685..cba08a62c52c 100644 --- a/arch/powerpc/include/asm/nohash/64/pgtable.h +++ b/arch/powerpc/include/asm/nohash/64/pgtable.h @@ -181,8 +181,6 @@ static inline int __ptep_test_and_clear_young(struct mm_struct *mm, { unsigned long old; - if (!pte_young(*ptep)) - return 0; old = pte_update(mm, addr, ptep, _PAGE_ACCESSED, 0, 0); return (old & _PAGE_ACCESSED) != 0; } @@ -198,10 +196,6 @@ static inline int __ptep_test_and_clear_young(struct mm_struct *mm, static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { - - if ((pte_val(*ptep) & _PAGE_RW) == 0) - return; - pte_update(mm, addr, ptep, _PAGE_RW, 0, 0); } @@ -209,9 +203,6 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, static inline void huge_ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { - if ((pte_val(*ptep) & _PAGE_RW) == 0) - return; - pte_update(mm, addr, ptep, _PAGE_RW, 0, 1); } diff --git a/arch/powerpc/include/asm/nohash/pgtable.h b/arch/powerpc/include/asm/nohash/pgtable.h index bd5c3a4baabd..8adaacbbdd1d 100644 --- a/arch/powerpc/include/asm/nohash/pgtable.h +++ b/arch/powerpc/include/asm/nohash/pgtable.h @@ -47,6 +47,9 @@ static inline pte_basic_t pte_update(struct mm_struct *mm, unsigned long addr, p pte_basic_t old = pte_val(*p); pte_basic_t new = (old & ~(pte_basic_t)clr) | set; + if (new == old) + return old; + *p = __pte(new); if (IS_ENABLED(CONFIG_44x) && (old & _PAGE_USER) && (old & _PAGE_EXEC)) From 27672be7751f25566e69bc228c8b8440a0772f8b Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 25 Sep 2023 20:31:28 +0200 Subject: [PATCH 286/475] powerpc/nohash: Deduplicate _PAGE_CHG_MASK _PAGE_CHG_MASK is identical between nohash/32 and nohash/64, deduplicate it. While at it, clean the #ifdef for PTE_RPN_MASK in nohash/32 as it is already CONFIG_PPC32. Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://msgid.link/c1a1893dbba4afb825bfa78b262f0b0e0fc3b490.1695659959.git.christophe.leroy@csgroup.eu --- arch/powerpc/include/asm/nohash/32/pgtable.h | 8 +------- arch/powerpc/include/asm/nohash/64/pgtable.h | 6 ------ arch/powerpc/include/asm/nohash/pgtable.h | 6 ++++++ 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/arch/powerpc/include/asm/nohash/32/pgtable.h b/arch/powerpc/include/asm/nohash/32/pgtable.h index ae7f3c8afd4f..a39ecd498084 100644 --- a/arch/powerpc/include/asm/nohash/32/pgtable.h +++ b/arch/powerpc/include/asm/nohash/32/pgtable.h @@ -143,7 +143,7 @@ * The mask covered by the RPN must be a ULL on 32-bit platforms with * 64-bit PTEs. */ -#if defined(CONFIG_PPC32) && defined(CONFIG_PTE_64BIT) +#ifdef CONFIG_PTE_64BIT #define PTE_RPN_MASK (~((1ULL << PTE_RPN_SHIFT) - 1)) #define MAX_POSSIBLE_PHYSMEM_BITS 36 #else @@ -151,12 +151,6 @@ #define MAX_POSSIBLE_PHYSMEM_BITS 32 #endif -/* - * _PAGE_CHG_MASK masks of bits that are to be preserved across - * pgprot changes. - */ -#define _PAGE_CHG_MASK (PTE_RPN_MASK | _PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_SPECIAL) - #ifndef __ASSEMBLY__ #define pte_clear(mm, addr, ptep) \ diff --git a/arch/powerpc/include/asm/nohash/64/pgtable.h b/arch/powerpc/include/asm/nohash/64/pgtable.h index cba08a62c52c..34a518a1c04d 100644 --- a/arch/powerpc/include/asm/nohash/64/pgtable.h +++ b/arch/powerpc/include/asm/nohash/64/pgtable.h @@ -75,12 +75,6 @@ #define PTE_RPN_MASK (~((1UL << PTE_RPN_SHIFT) - 1)) -/* - * _PAGE_CHG_MASK masks of bits that are to be preserved across - * pgprot changes. - */ -#define _PAGE_CHG_MASK (PTE_RPN_MASK | _PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_SPECIAL) - #define H_PAGE_4K_PFN 0 #ifndef __ASSEMBLY__ diff --git a/arch/powerpc/include/asm/nohash/pgtable.h b/arch/powerpc/include/asm/nohash/pgtable.h index 8adaacbbdd1d..c64a040f4a6a 100644 --- a/arch/powerpc/include/asm/nohash/pgtable.h +++ b/arch/powerpc/include/asm/nohash/pgtable.h @@ -13,6 +13,12 @@ static inline pte_basic_t pte_update(struct mm_struct *mm, unsigned long addr, p #include #endif +/* + * _PAGE_CHG_MASK masks of bits that are to be preserved across + * pgprot changes. + */ +#define _PAGE_CHG_MASK (PTE_RPN_MASK | _PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_SPECIAL) + /* Permission masks used for kernel mappings */ #define PAGE_KERNEL __pgprot(_PAGE_BASE | _PAGE_KERNEL_RW) #define PAGE_KERNEL_NC __pgprot(_PAGE_BASE_NC | _PAGE_KERNEL_RW | _PAGE_NO_CACHE) From 3a4288164d631b88a57119777b15099eb23c6fbf Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 25 Sep 2023 20:31:29 +0200 Subject: [PATCH 287/475] powerpc/nohash: Deduplicate pte helpers Deduplicate following helpers that are identical on nohash/32 and nohash/64: pte_mkwrite_novma() pte_mkdirty() pte_mkyoung() pte_wrprotect() pte_mkexec() pte_young() Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://msgid.link/485f3cbafaa948143f92692e17ca652dfed68da2.1695659959.git.christophe.leroy@csgroup.eu --- arch/powerpc/include/asm/nohash/32/pgtable.h | 36 -------------------- arch/powerpc/include/asm/nohash/64/pgtable.h | 25 -------------- arch/powerpc/include/asm/nohash/pgtable.h | 36 ++++++++++++++++++++ 3 files changed, 36 insertions(+), 61 deletions(-) diff --git a/arch/powerpc/include/asm/nohash/32/pgtable.h b/arch/powerpc/include/asm/nohash/32/pgtable.h index a39ecd498084..de51f78449a0 100644 --- a/arch/powerpc/include/asm/nohash/32/pgtable.h +++ b/arch/powerpc/include/asm/nohash/32/pgtable.h @@ -156,37 +156,6 @@ #define pte_clear(mm, addr, ptep) \ do { pte_update(mm, addr, ptep, ~0, 0, 0); } while (0) -#ifndef pte_mkwrite_novma -static inline pte_t pte_mkwrite_novma(pte_t pte) -{ - return __pte(pte_val(pte) | _PAGE_RW); -} -#endif - -static inline pte_t pte_mkdirty(pte_t pte) -{ - return __pte(pte_val(pte) | _PAGE_DIRTY); -} - -static inline pte_t pte_mkyoung(pte_t pte) -{ - return __pte(pte_val(pte) | _PAGE_ACCESSED); -} - -#ifndef pte_wrprotect -static inline pte_t pte_wrprotect(pte_t pte) -{ - return __pte(pte_val(pte) & ~_PAGE_RW); -} -#endif - -#ifndef pte_mkexec -static inline pte_t pte_mkexec(pte_t pte) -{ - return __pte(pte_val(pte) | _PAGE_EXEC); -} -#endif - #define pmd_none(pmd) (!pmd_val(pmd)) #define pmd_bad(pmd) (pmd_val(pmd) & _PMD_BAD) #define pmd_present(pmd) (pmd_val(pmd) & _PMD_PRESENT_MASK) @@ -238,11 +207,6 @@ static inline void __ptep_set_access_flags(struct vm_area_struct *vma, } #endif -static inline int pte_young(pte_t pte) -{ - return pte_val(pte) & _PAGE_ACCESSED; -} - /* * Note that on Book E processors, the pmd contains the kernel virtual * (lowmem) address of the pte page. The physical address is less useful diff --git a/arch/powerpc/include/asm/nohash/64/pgtable.h b/arch/powerpc/include/asm/nohash/64/pgtable.h index 34a518a1c04d..e8bbc6ec1084 100644 --- a/arch/powerpc/include/asm/nohash/64/pgtable.h +++ b/arch/powerpc/include/asm/nohash/64/pgtable.h @@ -80,26 +80,6 @@ #ifndef __ASSEMBLY__ /* pte_clear moved to later in this file */ -static inline pte_t pte_mkwrite_novma(pte_t pte) -{ - return __pte(pte_val(pte) | _PAGE_RW); -} - -static inline pte_t pte_mkdirty(pte_t pte) -{ - return __pte(pte_val(pte) | _PAGE_DIRTY); -} - -static inline pte_t pte_mkyoung(pte_t pte) -{ - return __pte(pte_val(pte) | _PAGE_ACCESSED); -} - -static inline pte_t pte_wrprotect(pte_t pte) -{ - return __pte(pte_val(pte) & ~_PAGE_RW); -} - #define PMD_BAD_BITS (PTE_TABLE_SIZE-1) #define PUD_BAD_BITS (PMD_TABLE_SIZE-1) @@ -165,11 +145,6 @@ static inline void p4d_set(p4d_t *p4dp, unsigned long val) *p4dp = __p4d(val); } -static inline int pte_young(pte_t pte) -{ - return pte_val(pte) & _PAGE_ACCESSED; -} - static inline int __ptep_test_and_clear_young(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { diff --git a/arch/powerpc/include/asm/nohash/pgtable.h b/arch/powerpc/include/asm/nohash/pgtable.h index c64a040f4a6a..21f232d2e34f 100644 --- a/arch/powerpc/include/asm/nohash/pgtable.h +++ b/arch/powerpc/include/asm/nohash/pgtable.h @@ -70,6 +70,37 @@ static inline pte_basic_t pte_update(struct mm_struct *mm, unsigned long addr, p #endif /* Generic accessors to PTE bits */ +#ifndef pte_mkwrite_novma +static inline pte_t pte_mkwrite_novma(pte_t pte) +{ + return __pte(pte_val(pte) | _PAGE_RW); +} +#endif + +static inline pte_t pte_mkdirty(pte_t pte) +{ + return __pte(pte_val(pte) | _PAGE_DIRTY); +} + +static inline pte_t pte_mkyoung(pte_t pte) +{ + return __pte(pte_val(pte) | _PAGE_ACCESSED); +} + +#ifndef pte_wrprotect +static inline pte_t pte_wrprotect(pte_t pte) +{ + return __pte(pte_val(pte) & ~_PAGE_RW); +} +#endif + +#ifndef pte_mkexec +static inline pte_t pte_mkexec(pte_t pte) +{ + return __pte(pte_val(pte) | _PAGE_EXEC); +} +#endif + #ifndef pte_write static inline int pte_write(pte_t pte) { @@ -96,6 +127,11 @@ static inline bool pte_hw_valid(pte_t pte) return pte_val(pte) & _PAGE_PRESENT; } +static inline int pte_young(pte_t pte) +{ + return pte_val(pte) & _PAGE_ACCESSED; +} + /* * Don't just check for any non zero bits in __PAGE_USER, since for book3e * and PTE_64BIT, PAGE_KERNEL_X contains _PAGE_BAP_SR which is also in From 8c3d9eb323bbf2b37cdc5c01ebf9604175b5970f Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 25 Sep 2023 20:31:30 +0200 Subject: [PATCH 288/475] powerpc/nohash: Refactor ptep_test_and_clear_young() Remove ptep_test_and_clear_young() macro, make __ptep_test_and_clear_young() common to nohash/32 and nohash/64 and change it to become ptep_test_and_clear_young() Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://msgid.link/de90679ae169104fa3c98d0a8828d7ede228fc52.1695659959.git.christophe.leroy@csgroup.eu --- arch/powerpc/include/asm/nohash/32/pgtable.h | 11 ----------- arch/powerpc/include/asm/nohash/64/pgtable.h | 19 +------------------ arch/powerpc/include/asm/nohash/pgtable.h | 11 +++++++++++ 3 files changed, 12 insertions(+), 29 deletions(-) diff --git a/arch/powerpc/include/asm/nohash/32/pgtable.h b/arch/powerpc/include/asm/nohash/32/pgtable.h index de51f78449a0..b7605000bd91 100644 --- a/arch/powerpc/include/asm/nohash/32/pgtable.h +++ b/arch/powerpc/include/asm/nohash/32/pgtable.h @@ -164,17 +164,6 @@ static inline void pmd_clear(pmd_t *pmdp) *pmdp = __pmd(0); } -#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG -static inline int __ptep_test_and_clear_young(struct mm_struct *mm, - unsigned long addr, pte_t *ptep) -{ - unsigned long old; - old = pte_update(mm, addr, ptep, _PAGE_ACCESSED, 0, 0); - return (old & _PAGE_ACCESSED) != 0; -} -#define ptep_test_and_clear_young(__vma, __addr, __ptep) \ - __ptep_test_and_clear_young((__vma)->vm_mm, __addr, __ptep) - #define __HAVE_ARCH_PTEP_GET_AND_CLEAR static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) diff --git a/arch/powerpc/include/asm/nohash/64/pgtable.h b/arch/powerpc/include/asm/nohash/64/pgtable.h index e8bbc6ec1084..56041036fa34 100644 --- a/arch/powerpc/include/asm/nohash/64/pgtable.h +++ b/arch/powerpc/include/asm/nohash/64/pgtable.h @@ -145,22 +145,6 @@ static inline void p4d_set(p4d_t *p4dp, unsigned long val) *p4dp = __p4d(val); } -static inline int __ptep_test_and_clear_young(struct mm_struct *mm, - unsigned long addr, pte_t *ptep) -{ - unsigned long old; - - old = pte_update(mm, addr, ptep, _PAGE_ACCESSED, 0, 0); - return (old & _PAGE_ACCESSED) != 0; -} -#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG -#define ptep_test_and_clear_young(__vma, __addr, __ptep) \ -({ \ - int __r; \ - __r = __ptep_test_and_clear_young((__vma)->vm_mm, __addr, __ptep); \ - __r; \ -}) - #define __HAVE_ARCH_PTEP_SET_WRPROTECT static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_t *ptep) @@ -178,8 +162,7 @@ static inline void huge_ptep_set_wrprotect(struct mm_struct *mm, #define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH #define ptep_clear_flush_young(__vma, __address, __ptep) \ ({ \ - int __young = __ptep_test_and_clear_young((__vma)->vm_mm, __address, \ - __ptep); \ + int __young = ptep_test_and_clear_young(__vma, __address, __ptep);\ __young; \ }) diff --git a/arch/powerpc/include/asm/nohash/pgtable.h b/arch/powerpc/include/asm/nohash/pgtable.h index 21f232d2e34f..2b043b72f642 100644 --- a/arch/powerpc/include/asm/nohash/pgtable.h +++ b/arch/powerpc/include/asm/nohash/pgtable.h @@ -69,6 +69,17 @@ static inline pte_basic_t pte_update(struct mm_struct *mm, unsigned long addr, p } #endif +static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep) +{ + unsigned long old; + + old = pte_update(vma->vm_mm, addr, ptep, _PAGE_ACCESSED, 0, 0); + + return (old & _PAGE_ACCESSED) != 0; +} +#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG + /* Generic accessors to PTE bits */ #ifndef pte_mkwrite_novma static inline pte_t pte_mkwrite_novma(pte_t pte) From cc68d77febe055b6499dda5fa13bda976a12a85c Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 25 Sep 2023 20:31:31 +0200 Subject: [PATCH 289/475] powerpc/nohash: Deduplicate ptep_set_wrprotect() and ptep_get_and_clear() ptep_set_wrprotect() and ptep_get_and_clear are identical for nohash/32 and nohash/64. Make them common. Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://msgid.link/ffe46edecdabce915e2d1a4b79a3b2ab770f2248.1695659959.git.christophe.leroy@csgroup.eu --- arch/powerpc/include/asm/nohash/32/pgtable.h | 16 ---------------- arch/powerpc/include/asm/nohash/64/pgtable.h | 15 --------------- arch/powerpc/include/asm/nohash/pgtable.h | 16 ++++++++++++++++ 3 files changed, 16 insertions(+), 31 deletions(-) diff --git a/arch/powerpc/include/asm/nohash/32/pgtable.h b/arch/powerpc/include/asm/nohash/32/pgtable.h index b7605000bd91..0be464af4cb1 100644 --- a/arch/powerpc/include/asm/nohash/32/pgtable.h +++ b/arch/powerpc/include/asm/nohash/32/pgtable.h @@ -164,22 +164,6 @@ static inline void pmd_clear(pmd_t *pmdp) *pmdp = __pmd(0); } -#define __HAVE_ARCH_PTEP_GET_AND_CLEAR -static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, - pte_t *ptep) -{ - return __pte(pte_update(mm, addr, ptep, ~0, 0, 0)); -} - -#define __HAVE_ARCH_PTEP_SET_WRPROTECT -#ifndef ptep_set_wrprotect -static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, - pte_t *ptep) -{ - pte_update(mm, addr, ptep, _PAGE_RW, 0, 0); -} -#endif - #ifndef __ptep_set_access_flags static inline void __ptep_set_access_flags(struct vm_area_struct *vma, pte_t *ptep, pte_t entry, diff --git a/arch/powerpc/include/asm/nohash/64/pgtable.h b/arch/powerpc/include/asm/nohash/64/pgtable.h index 56041036fa34..dc6e35c3a53f 100644 --- a/arch/powerpc/include/asm/nohash/64/pgtable.h +++ b/arch/powerpc/include/asm/nohash/64/pgtable.h @@ -145,13 +145,6 @@ static inline void p4d_set(p4d_t *p4dp, unsigned long val) *p4dp = __p4d(val); } -#define __HAVE_ARCH_PTEP_SET_WRPROTECT -static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, - pte_t *ptep) -{ - pte_update(mm, addr, ptep, _PAGE_RW, 0, 0); -} - #define __HAVE_ARCH_HUGE_PTEP_SET_WRPROTECT static inline void huge_ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_t *ptep) @@ -166,14 +159,6 @@ static inline void huge_ptep_set_wrprotect(struct mm_struct *mm, __young; \ }) -#define __HAVE_ARCH_PTEP_GET_AND_CLEAR -static inline pte_t ptep_get_and_clear(struct mm_struct *mm, - unsigned long addr, pte_t *ptep) -{ - unsigned long old = pte_update(mm, addr, ptep, ~0UL, 0, 0); - return __pte(old); -} - static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t * ptep) { diff --git a/arch/powerpc/include/asm/nohash/pgtable.h b/arch/powerpc/include/asm/nohash/pgtable.h index 2b043b72f642..7e810a84ac15 100644 --- a/arch/powerpc/include/asm/nohash/pgtable.h +++ b/arch/powerpc/include/asm/nohash/pgtable.h @@ -80,6 +80,22 @@ static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, } #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG +#ifndef ptep_set_wrprotect +static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, + pte_t *ptep) +{ + pte_update(mm, addr, ptep, _PAGE_RW, 0, 0); +} +#endif +#define __HAVE_ARCH_PTEP_SET_WRPROTECT + +static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, + pte_t *ptep) +{ + return __pte(pte_update(mm, addr, ptep, ~0UL, 0, 0)); +} +#define __HAVE_ARCH_PTEP_GET_AND_CLEAR + /* Generic accessors to PTE bits */ #ifndef pte_mkwrite_novma static inline pte_t pte_mkwrite_novma(pte_t pte) From 2ef9f4bb9c47ed30ff3c7961744cae545c034154 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 25 Sep 2023 20:31:32 +0200 Subject: [PATCH 290/475] powerpc/nohash: Refactor pte_clear() pte_clear() are doing the same on nohash/32 and nohash/64, Keep the static inline version of nohash/64, make it common and remove the macro version of nohash/32. Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://msgid.link/818f7df83d7e9e18f55e274cd3c44f2871ade4dd.1695659959.git.christophe.leroy@csgroup.eu --- arch/powerpc/include/asm/nohash/32/pgtable.h | 3 --- arch/powerpc/include/asm/nohash/64/pgtable.h | 7 ------- arch/powerpc/include/asm/nohash/pgtable.h | 5 +++++ 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/arch/powerpc/include/asm/nohash/32/pgtable.h b/arch/powerpc/include/asm/nohash/32/pgtable.h index 0be464af4cb1..481594097f46 100644 --- a/arch/powerpc/include/asm/nohash/32/pgtable.h +++ b/arch/powerpc/include/asm/nohash/32/pgtable.h @@ -153,9 +153,6 @@ #ifndef __ASSEMBLY__ -#define pte_clear(mm, addr, ptep) \ - do { pte_update(mm, addr, ptep, ~0, 0, 0); } while (0) - #define pmd_none(pmd) (!pmd_val(pmd)) #define pmd_bad(pmd) (pmd_val(pmd) & _PMD_BAD) #define pmd_present(pmd) (pmd_val(pmd) & _PMD_PRESENT_MASK) diff --git a/arch/powerpc/include/asm/nohash/64/pgtable.h b/arch/powerpc/include/asm/nohash/64/pgtable.h index dc6e35c3a53f..b59fbf754f82 100644 --- a/arch/powerpc/include/asm/nohash/64/pgtable.h +++ b/arch/powerpc/include/asm/nohash/64/pgtable.h @@ -159,13 +159,6 @@ static inline void huge_ptep_set_wrprotect(struct mm_struct *mm, __young; \ }) -static inline void pte_clear(struct mm_struct *mm, unsigned long addr, - pte_t * ptep) -{ - pte_update(mm, addr, ptep, ~0UL, 0, 0); -} - - /* Set the dirty and/or accessed bits atomically in a linux PTE */ static inline void __ptep_set_access_flags(struct vm_area_struct *vma, pte_t *ptep, pte_t entry, diff --git a/arch/powerpc/include/asm/nohash/pgtable.h b/arch/powerpc/include/asm/nohash/pgtable.h index 7e810a84ac15..b974d1ab8a97 100644 --- a/arch/powerpc/include/asm/nohash/pgtable.h +++ b/arch/powerpc/include/asm/nohash/pgtable.h @@ -96,6 +96,11 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, } #define __HAVE_ARCH_PTEP_GET_AND_CLEAR +static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) +{ + pte_update(mm, addr, ptep, ~0UL, 0, 0); +} + /* Generic accessors to PTE bits */ #ifndef pte_mkwrite_novma static inline pte_t pte_mkwrite_novma(pte_t pte) From 799d8836a7c4f4327833e4a5ca952a1700acdb14 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 25 Sep 2023 20:31:33 +0200 Subject: [PATCH 291/475] powerpc/nohash: Refactor __ptep_set_access_flags() nohash/32 version of __ptep_set_access_flags() does the same as nohash/64 version, the only difference is that nohash/32 version is more complete and uses pte_update(). Make it common and remove the nohash/64 version. Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://msgid.link/e296885df46289d3e5f4cb51efeefe593f76ef24.1695659959.git.christophe.leroy@csgroup.eu --- arch/powerpc/include/asm/nohash/32/pgtable.h | 16 ---------------- arch/powerpc/include/asm/nohash/64/pgtable.h | 15 --------------- arch/powerpc/include/asm/nohash/pgtable.h | 17 +++++++++++++++++ 3 files changed, 17 insertions(+), 31 deletions(-) diff --git a/arch/powerpc/include/asm/nohash/32/pgtable.h b/arch/powerpc/include/asm/nohash/32/pgtable.h index 481594097f46..9164a9e41b02 100644 --- a/arch/powerpc/include/asm/nohash/32/pgtable.h +++ b/arch/powerpc/include/asm/nohash/32/pgtable.h @@ -161,22 +161,6 @@ static inline void pmd_clear(pmd_t *pmdp) *pmdp = __pmd(0); } -#ifndef __ptep_set_access_flags -static inline void __ptep_set_access_flags(struct vm_area_struct *vma, - pte_t *ptep, pte_t entry, - unsigned long address, - int psize) -{ - unsigned long set = pte_val(entry) & - (_PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_RW | _PAGE_EXEC); - int huge = psize > mmu_virtual_psize ? 1 : 0; - - pte_update(vma->vm_mm, address, ptep, 0, set, huge); - - flush_tlb_page(vma, address); -} -#endif - /* * Note that on Book E processors, the pmd contains the kernel virtual * (lowmem) address of the pte page. The physical address is less useful diff --git a/arch/powerpc/include/asm/nohash/64/pgtable.h b/arch/powerpc/include/asm/nohash/64/pgtable.h index b59fbf754f82..36b9bad428cc 100644 --- a/arch/powerpc/include/asm/nohash/64/pgtable.h +++ b/arch/powerpc/include/asm/nohash/64/pgtable.h @@ -159,21 +159,6 @@ static inline void huge_ptep_set_wrprotect(struct mm_struct *mm, __young; \ }) -/* Set the dirty and/or accessed bits atomically in a linux PTE */ -static inline void __ptep_set_access_flags(struct vm_area_struct *vma, - pte_t *ptep, pte_t entry, - unsigned long address, - int psize) -{ - unsigned long bits = pte_val(entry) & - (_PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_RW | _PAGE_EXEC); - - unsigned long old = pte_val(*ptep); - *ptep = __pte(old | bits); - - flush_tlb_page(vma, address); -} - #define pmd_ERROR(e) \ pr_err("%s:%d: bad pmd %08lx.\n", __FILE__, __LINE__, pmd_val(e)) #define pgd_ERROR(e) \ diff --git a/arch/powerpc/include/asm/nohash/pgtable.h b/arch/powerpc/include/asm/nohash/pgtable.h index b974d1ab8a97..c5f21eda2a43 100644 --- a/arch/powerpc/include/asm/nohash/pgtable.h +++ b/arch/powerpc/include/asm/nohash/pgtable.h @@ -101,6 +101,23 @@ static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *pt pte_update(mm, addr, ptep, ~0UL, 0, 0); } +/* Set the dirty and/or accessed bits atomically in a linux PTE */ +#ifndef __ptep_set_access_flags +static inline void __ptep_set_access_flags(struct vm_area_struct *vma, + pte_t *ptep, pte_t entry, + unsigned long address, + int psize) +{ + unsigned long set = pte_val(entry) & + (_PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_RW | _PAGE_EXEC); + int huge = psize > mmu_virtual_psize ? 1 : 0; + + pte_update(vma->vm_mm, address, ptep, 0, set, huge); + + flush_tlb_page(vma, address); +} +#endif + /* Generic accessors to PTE bits */ #ifndef pte_mkwrite_novma static inline pte_t pte_mkwrite_novma(pte_t pte) From 4c8dd6c9872d4e89fd2b3a6fc92fd6cc9cdce347 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 25 Sep 2023 20:31:34 +0200 Subject: [PATCH 292/475] powerpc/e500: Simplify pte_mkexec() Commit b6cb20fdc273 ("powerpc/book3e: Fix set_memory_x() and set_memory_nx()") implemented a more elaborated version of pte_mkwrite() suitable for both kernel and user pages. That was needed because set_memory_x() was using pte_mkwrite(). But since commit a4c182ecf335 ("powerpc/set_memory: Avoid spinlock recursion in change_page_attr()") pte_mkwrite() is not used anymore by set_memory_x() so pte_mkwrite() can be simplified as it is only used for user pages. Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://msgid.link/cdc822322fe2ff4b0f5ecfde71d09d950b1c7557.1695659959.git.christophe.leroy@csgroup.eu --- arch/powerpc/include/asm/nohash/pte-e500.h | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/arch/powerpc/include/asm/nohash/pte-e500.h b/arch/powerpc/include/asm/nohash/pte-e500.h index d8924cbd61e4..99288e26b6b0 100644 --- a/arch/powerpc/include/asm/nohash/pte-e500.h +++ b/arch/powerpc/include/asm/nohash/pte-e500.h @@ -115,10 +115,7 @@ static inline pte_t pte_mkuser(pte_t pte) static inline pte_t pte_mkexec(pte_t pte) { - if (pte_val(pte) & _PAGE_BAP_UR) - return __pte((pte_val(pte) & ~_PAGE_BAP_SX) | _PAGE_BAP_UX); - else - return __pte((pte_val(pte) & ~_PAGE_BAP_UX) | _PAGE_BAP_SX); + return __pte((pte_val(pte) & ~_PAGE_BAP_SX) | _PAGE_BAP_UX); } #define pte_mkexec pte_mkexec From d3c0dfcfc95796701e82719722fd997ec5256013 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 25 Sep 2023 20:31:35 +0200 Subject: [PATCH 293/475] powerpc: Implement and use pgprot_nx() ioremap_page_range() calls pgprot_nx() vmap() and vmap_pfn() clear execute permission by calling pgprot_nx(). When pgprot_nx() is not defined it falls back to a nop. Implement it for powerpc then use it in early_ioremap_range(). Then the call to pte_exprotect() can be removed from ioremap_prot(). Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://msgid.link/5993a7a097e989af1c97fc4a6c011fefc67dbe6e.1695659959.git.christophe.leroy@csgroup.eu --- arch/powerpc/include/asm/pgtable.h | 6 ++++++ arch/powerpc/mm/ioremap.c | 5 ++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/arch/powerpc/include/asm/pgtable.h b/arch/powerpc/include/asm/pgtable.h index 966e7c5119f6..2bfb7dd3b49e 100644 --- a/arch/powerpc/include/asm/pgtable.h +++ b/arch/powerpc/include/asm/pgtable.h @@ -71,6 +71,12 @@ static inline pgprot_t pte_pgprot(pte_t pte) return __pgprot(pte_flags); } +static inline pgprot_t pgprot_nx(pgprot_t prot) +{ + return pte_pgprot(pte_exprotect(__pte(pgprot_val(prot)))); +} +#define pgprot_nx pgprot_nx + #ifndef pmd_page_vaddr static inline const void *pmd_page_vaddr(pmd_t pmd) { diff --git a/arch/powerpc/mm/ioremap.c b/arch/powerpc/mm/ioremap.c index 705e8e8ffde4..d5159f205380 100644 --- a/arch/powerpc/mm/ioremap.c +++ b/arch/powerpc/mm/ioremap.c @@ -50,8 +50,7 @@ void __iomem *ioremap_prot(phys_addr_t addr, size_t size, unsigned long flags) if (pte_write(pte)) pte = pte_mkdirty(pte); - /* we don't want to let _PAGE_USER and _PAGE_EXEC leak out */ - pte = pte_exprotect(pte); + /* we don't want to let _PAGE_USER leak out */ pte = pte_mkprivileged(pte); if (iowa_is_active()) @@ -66,7 +65,7 @@ int early_ioremap_range(unsigned long ea, phys_addr_t pa, unsigned long i; for (i = 0; i < size; i += PAGE_SIZE) { - int err = map_kernel_page(ea + i, pa + i, prot); + int err = map_kernel_page(ea + i, pa + i, pgprot_nx(prot)); if (WARN_ON_ONCE(err)) /* Should clean up */ return err; From c7263f156395d1f2a2142375a75b7b040686a07a Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 25 Sep 2023 20:31:36 +0200 Subject: [PATCH 294/475] powerpc: Fail ioremap() instead of silently ignoring flags when PAGE_USER is set Calling ioremap() with _PAGE_USER (or _PAGE_PRIVILEDGE unset) is wrong. Loudly fail the call to ioremap() instead of blindly clearing the flags. Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://msgid.link/b6dd5485ad00d2aafd2bb9b7c2c4eac3ebf2cdaf.1695659959.git.christophe.leroy@csgroup.eu --- arch/powerpc/mm/ioremap.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/powerpc/mm/ioremap.c b/arch/powerpc/mm/ioremap.c index d5159f205380..7823c38f09de 100644 --- a/arch/powerpc/mm/ioremap.c +++ b/arch/powerpc/mm/ioremap.c @@ -51,7 +51,8 @@ void __iomem *ioremap_prot(phys_addr_t addr, size_t size, unsigned long flags) pte = pte_mkdirty(pte); /* we don't want to let _PAGE_USER leak out */ - pte = pte_mkprivileged(pte); + if (WARN_ON(pte_user(pte))) + return NULL; if (iowa_is_active()) return iowa_ioremap(addr, size, pte_pgprot(pte), caller); From 69339071bb27f0b1371cd23d6dada3f976261c20 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 25 Sep 2023 20:31:37 +0200 Subject: [PATCH 295/475] powerpc: Remove pte_mkuser() and pte_mkpriviledged() pte_mkuser() is never used. Remove it. pte_mkpriviledged() is not used anymore. Remove it too. Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://msgid.link/1a1dc18816456c637dc8a9c38d532f7598b60ac4.1695659959.git.christophe.leroy@csgroup.eu --- arch/powerpc/include/asm/book3s/32/pgtable.h | 10 ---------- arch/powerpc/include/asm/book3s/64/pgtable.h | 10 ---------- arch/powerpc/include/asm/nohash/32/pte-8xx.h | 14 -------------- arch/powerpc/include/asm/nohash/pgtable.h | 14 -------------- arch/powerpc/include/asm/nohash/pte-e500.h | 15 --------------- 5 files changed, 63 deletions(-) diff --git a/arch/powerpc/include/asm/book3s/32/pgtable.h b/arch/powerpc/include/asm/book3s/32/pgtable.h index 45b69ae2631e..80505915c77c 100644 --- a/arch/powerpc/include/asm/book3s/32/pgtable.h +++ b/arch/powerpc/include/asm/book3s/32/pgtable.h @@ -522,16 +522,6 @@ static inline pte_t pte_mkhuge(pte_t pte) return pte; } -static inline pte_t pte_mkprivileged(pte_t pte) -{ - return __pte(pte_val(pte) & ~_PAGE_USER); -} - -static inline pte_t pte_mkuser(pte_t pte) -{ - return __pte(pte_val(pte) | _PAGE_USER); -} - static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) { return __pte((pte_val(pte) & _PAGE_CHG_MASK) | pgprot_val(newprot)); diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h index dbd545e73161..c3b921769ece 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -630,16 +630,6 @@ static inline pte_t pte_mkdevmap(pte_t pte) return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_SPECIAL | _PAGE_DEVMAP)); } -static inline pte_t pte_mkprivileged(pte_t pte) -{ - return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_PRIVILEGED)); -} - -static inline pte_t pte_mkuser(pte_t pte) -{ - return __pte_raw(pte_raw(pte) & cpu_to_be64(~_PAGE_PRIVILEGED)); -} - /* * This is potentially called with a pmd as the argument, in which case it's not * safe to check _PAGE_DEVMAP unless we also confirm that _PAGE_PTE is set. diff --git a/arch/powerpc/include/asm/nohash/32/pte-8xx.h b/arch/powerpc/include/asm/nohash/32/pte-8xx.h index 52395a5ecd70..843fe0138a66 100644 --- a/arch/powerpc/include/asm/nohash/32/pte-8xx.h +++ b/arch/powerpc/include/asm/nohash/32/pte-8xx.h @@ -122,20 +122,6 @@ static inline bool pte_user(pte_t pte) #define pte_user pte_user -static inline pte_t pte_mkprivileged(pte_t pte) -{ - return __pte(pte_val(pte) | _PAGE_SH); -} - -#define pte_mkprivileged pte_mkprivileged - -static inline pte_t pte_mkuser(pte_t pte) -{ - return __pte(pte_val(pte) & ~_PAGE_SH); -} - -#define pte_mkuser pte_mkuser - static inline pte_t pte_mkhuge(pte_t pte) { return __pte(pte_val(pte) | _PAGE_SPS | _PAGE_HUGE); diff --git a/arch/powerpc/include/asm/nohash/pgtable.h b/arch/powerpc/include/asm/nohash/pgtable.h index c5f21eda2a43..18c1c9430a86 100644 --- a/arch/powerpc/include/asm/nohash/pgtable.h +++ b/arch/powerpc/include/asm/nohash/pgtable.h @@ -251,20 +251,6 @@ static inline pte_t pte_mkhuge(pte_t pte) } #endif -#ifndef pte_mkprivileged -static inline pte_t pte_mkprivileged(pte_t pte) -{ - return __pte(pte_val(pte) & ~_PAGE_USER); -} -#endif - -#ifndef pte_mkuser -static inline pte_t pte_mkuser(pte_t pte) -{ - return __pte(pte_val(pte) | _PAGE_USER); -} -#endif - static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) { return __pte((pte_val(pte) & _PAGE_CHG_MASK) | pgprot_val(newprot)); diff --git a/arch/powerpc/include/asm/nohash/pte-e500.h b/arch/powerpc/include/asm/nohash/pte-e500.h index 99288e26b6b0..9f9e3f02d414 100644 --- a/arch/powerpc/include/asm/nohash/pte-e500.h +++ b/arch/powerpc/include/asm/nohash/pte-e500.h @@ -54,7 +54,6 @@ #define _PAGE_KERNEL_RWX (_PAGE_BAP_SW | _PAGE_BAP_SR | _PAGE_DIRTY | _PAGE_BAP_SX) #define _PAGE_KERNEL_ROX (_PAGE_BAP_SR | _PAGE_BAP_SX) #define _PAGE_USER (_PAGE_BAP_UR | _PAGE_BAP_SR) /* Can be read */ -#define _PAGE_PRIVILEGED (_PAGE_BAP_SR) #define _PAGE_SPECIAL _PAGE_SW0 @@ -99,20 +98,6 @@ #define PAGE_READONLY_X __pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_BAP_UX) #ifndef __ASSEMBLY__ -static inline pte_t pte_mkprivileged(pte_t pte) -{ - return __pte((pte_val(pte) & ~_PAGE_USER) | _PAGE_PRIVILEGED); -} - -#define pte_mkprivileged pte_mkprivileged - -static inline pte_t pte_mkuser(pte_t pte) -{ - return __pte((pte_val(pte) & ~_PAGE_PRIVILEGED) | _PAGE_USER); -} - -#define pte_mkuser pte_mkuser - static inline pte_t pte_mkexec(pte_t pte) { return __pte((pte_val(pte) & ~_PAGE_BAP_SX) | _PAGE_BAP_UX); From a78587473642aec302697cdaceb719a7f8791369 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 25 Sep 2023 20:31:38 +0200 Subject: [PATCH 296/475] powerpc: Rely on address instead of pte_user() pte_user() may return 'false' when a user page is PAGE_NONE. In that case it is still a user page and needs to be handled as such. So use is_kernel_addr() instead. And remove "user" text from ptdump as ptdump only dumps kernel tables. Note: no change done for book3s/64 which still has it 'priviledge' bit. Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://msgid.link/c778dad89fad07727c31717a9c62f45357c29ebc.1695659959.git.christophe.leroy@csgroup.eu --- arch/powerpc/include/asm/nohash/pgtable.h | 2 +- arch/powerpc/mm/book3s32/mmu.c | 4 ++-- arch/powerpc/mm/nohash/e500.c | 2 +- arch/powerpc/mm/pgtable.c | 22 +++++++++++----------- arch/powerpc/mm/ptdump/8xx.c | 5 ----- arch/powerpc/mm/ptdump/shared.c | 5 ----- 6 files changed, 15 insertions(+), 25 deletions(-) diff --git a/arch/powerpc/include/asm/nohash/pgtable.h b/arch/powerpc/include/asm/nohash/pgtable.h index 18c1c9430a86..942f46e86b4b 100644 --- a/arch/powerpc/include/asm/nohash/pgtable.h +++ b/arch/powerpc/include/asm/nohash/pgtable.h @@ -58,7 +58,7 @@ static inline pte_basic_t pte_update(struct mm_struct *mm, unsigned long addr, p *p = __pte(new); - if (IS_ENABLED(CONFIG_44x) && (old & _PAGE_USER) && (old & _PAGE_EXEC)) + if (IS_ENABLED(CONFIG_44x) && !is_kernel_addr(addr) && (old & _PAGE_EXEC)) icache_44x_need_flush = 1; /* huge pages use the old page table lock */ diff --git a/arch/powerpc/mm/book3s32/mmu.c b/arch/powerpc/mm/book3s32/mmu.c index 850783cfa9c7..d1041c946ce2 100644 --- a/arch/powerpc/mm/book3s32/mmu.c +++ b/arch/powerpc/mm/book3s32/mmu.c @@ -127,7 +127,7 @@ static void setibat(int index, unsigned long virt, phys_addr_t phys, wimgxpp = (flags & _PAGE_COHERENT) | (_PAGE_EXEC ? BPP_RX : BPP_XX); bat[0].batu = virt | (bl << 2) | 2; /* Vs=1, Vp=0 */ bat[0].batl = BAT_PHYS_ADDR(phys) | wimgxpp; - if (flags & _PAGE_USER) + if (!is_kernel_addr(virt)) bat[0].batu |= 1; /* Vp = 1 */ } @@ -280,7 +280,7 @@ void __init setbat(int index, unsigned long virt, phys_addr_t phys, wimgxpp |= (flags & _PAGE_RW)? BPP_RW: BPP_RX; bat[1].batu = virt | (bl << 2) | 2; /* Vs=1, Vp=0 */ bat[1].batl = BAT_PHYS_ADDR(phys) | wimgxpp; - if (flags & _PAGE_USER) + if (!is_kernel_addr(virt)) bat[1].batu |= 1; /* Vp = 1 */ if (flags & _PAGE_GUARDED) { /* G bit must be zero in IBATs */ diff --git a/arch/powerpc/mm/nohash/e500.c b/arch/powerpc/mm/nohash/e500.c index 40a4e69ae1a9..5b7d7a932bfd 100644 --- a/arch/powerpc/mm/nohash/e500.c +++ b/arch/powerpc/mm/nohash/e500.c @@ -122,7 +122,7 @@ static void settlbcam(int index, unsigned long virt, phys_addr_t phys, TLBCAM[index].MAS7 = (u64)phys >> 32; /* Below is unlikely -- only for large user pages or similar */ - if (pte_user(__pte(flags))) { + if (!is_kernel_addr(virt)) { TLBCAM[index].MAS3 |= MAS3_UR; TLBCAM[index].MAS3 |= (flags & _PAGE_EXEC) ? MAS3_UX : 0; TLBCAM[index].MAS3 |= (flags & _PAGE_RW) ? MAS3_UW : 0; diff --git a/arch/powerpc/mm/pgtable.c b/arch/powerpc/mm/pgtable.c index 3f86fd217690..781a68c69c2f 100644 --- a/arch/powerpc/mm/pgtable.c +++ b/arch/powerpc/mm/pgtable.c @@ -46,13 +46,13 @@ static inline int is_exec_fault(void) * and we avoid _PAGE_SPECIAL and cache inhibited pte. We also only do that * on userspace PTEs */ -static inline int pte_looks_normal(pte_t pte) +static inline int pte_looks_normal(pte_t pte, unsigned long addr) { if (pte_present(pte) && !pte_special(pte)) { if (pte_ci(pte)) return 0; - if (pte_user(pte)) + if (!is_kernel_addr(addr)) return 1; } return 0; @@ -79,11 +79,11 @@ static struct folio *maybe_pte_to_folio(pte_t pte) * support falls into the same category. */ -static pte_t set_pte_filter_hash(pte_t pte) +static pte_t set_pte_filter_hash(pte_t pte, unsigned long addr) { pte = __pte(pte_val(pte) & ~_PAGE_HPTEFLAGS); - if (pte_looks_normal(pte) && !(cpu_has_feature(CPU_FTR_COHERENT_ICACHE) || - cpu_has_feature(CPU_FTR_NOEXECUTE))) { + if (pte_looks_normal(pte, addr) && !(cpu_has_feature(CPU_FTR_COHERENT_ICACHE) || + cpu_has_feature(CPU_FTR_NOEXECUTE))) { struct folio *folio = maybe_pte_to_folio(pte); if (!folio) return pte; @@ -97,7 +97,7 @@ static pte_t set_pte_filter_hash(pte_t pte) #else /* CONFIG_PPC_BOOK3S */ -static pte_t set_pte_filter_hash(pte_t pte) { return pte; } +static pte_t set_pte_filter_hash(pte_t pte, unsigned long addr) { return pte; } #endif /* CONFIG_PPC_BOOK3S */ @@ -105,7 +105,7 @@ static pte_t set_pte_filter_hash(pte_t pte) { return pte; } * as we don't have two bits to spare for _PAGE_EXEC and _PAGE_HWEXEC so * instead we "filter out" the exec permission for non clean pages. */ -static inline pte_t set_pte_filter(pte_t pte) +static inline pte_t set_pte_filter(pte_t pte, unsigned long addr) { struct folio *folio; @@ -113,10 +113,10 @@ static inline pte_t set_pte_filter(pte_t pte) return pte; if (mmu_has_feature(MMU_FTR_HPTE_TABLE)) - return set_pte_filter_hash(pte); + return set_pte_filter_hash(pte, addr); /* No exec permission in the first place, move on */ - if (!pte_exec(pte) || !pte_looks_normal(pte)) + if (!pte_exec(pte) || !pte_looks_normal(pte, addr)) return pte; /* If you set _PAGE_EXEC on weird pages you're on your own */ @@ -200,7 +200,7 @@ void set_ptes(struct mm_struct *mm, unsigned long addr, pte_t *ptep, * this context might not have been activated yet when this * is called. */ - pte = set_pte_filter(pte); + pte = set_pte_filter(pte, addr); /* Perform the setting of the PTE */ arch_enter_lazy_mmu_mode(); @@ -301,7 +301,7 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_ */ VM_WARN_ON(pte_hw_valid(*ptep) && !pte_protnone(*ptep)); - pte = set_pte_filter(pte); + pte = set_pte_filter(pte, addr); val = pte_val(pte); diff --git a/arch/powerpc/mm/ptdump/8xx.c b/arch/powerpc/mm/ptdump/8xx.c index fac932eb8f9a..b5c79b11ea3c 100644 --- a/arch/powerpc/mm/ptdump/8xx.c +++ b/arch/powerpc/mm/ptdump/8xx.c @@ -20,11 +20,6 @@ static const struct flag_info flag_array[] = { #endif .set = "huge", .clear = " ", - }, { - .mask = _PAGE_SH, - .val = 0, - .set = "user", - .clear = " ", }, { .mask = _PAGE_RO | _PAGE_NA, .val = 0, diff --git a/arch/powerpc/mm/ptdump/shared.c b/arch/powerpc/mm/ptdump/shared.c index f884760ca5cf..5ff101654c45 100644 --- a/arch/powerpc/mm/ptdump/shared.c +++ b/arch/powerpc/mm/ptdump/shared.c @@ -11,11 +11,6 @@ static const struct flag_info flag_array[] = { { - .mask = _PAGE_USER, - .val = _PAGE_USER, - .set = "user", - .clear = " ", - }, { .mask = _PAGE_RW, .val = 0, .set = "r ", From a5a08dc90f4513d1a78582ec24b687fad01cc843 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 25 Sep 2023 20:31:39 +0200 Subject: [PATCH 297/475] powerpc: Refactor permission masks used for __P/__S table and kernel memory flags Prepare a common version of the permission masks that will be based on _PAGE_NA, _PAGE_RO, _PAGE_ROX, _PAGE_RW, _PAGE_RWX that will be defined in platform specific headers in later patches. Put them in a new header pgtable-masks.h which will be included by platforms. And prepare a common version of flags used for mapping kernel memory that will be based on _PAGE_RO, _PAGE_ROX, _PAGE_RW, _PAGE_RWX that will be defined in platform specific headers. Put them in unless _PAGE_KERNEL_RO is already defined so that platform specific definitions can be dismantled one by one. Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://msgid.link/d31b59efcdb38e675479563307af891aeadf6ec9.1695659959.git.christophe.leroy@csgroup.eu --- arch/powerpc/include/asm/pgtable-masks.h | 30 ++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 arch/powerpc/include/asm/pgtable-masks.h diff --git a/arch/powerpc/include/asm/pgtable-masks.h b/arch/powerpc/include/asm/pgtable-masks.h new file mode 100644 index 000000000000..808a3b9e8fc0 --- /dev/null +++ b/arch/powerpc/include/asm/pgtable-masks.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_PGTABLE_MASKS_H +#define _ASM_POWERPC_PGTABLE_MASKS_H + +#ifndef _PAGE_NA +#define _PAGE_NA 0 +#define _PAGE_RO _PAGE_READ +#define _PAGE_ROX (_PAGE_READ | _PAGE_EXEC) +#define _PAGE_RW (_PAGE_READ | _PAGE_WRITE) +#define _PAGE_RWX (_PAGE_READ | _PAGE_WRITE | _PAGE_EXEC) +#endif + +/* Permission flags for kernel mappings */ +#ifndef _PAGE_KERNEL_RO +#define _PAGE_KERNEL_RO _PAGE_RO +#define _PAGE_KERNEL_ROX _PAGE_ROX +#define _PAGE_KERNEL_RW (_PAGE_RW | _PAGE_DIRTY) +#define _PAGE_KERNEL_RWX (_PAGE_RWX | _PAGE_DIRTY) +#endif + +/* Permission masks used to generate the __P and __S table */ +#define PAGE_NONE __pgprot(_PAGE_BASE | _PAGE_NA) +#define PAGE_SHARED __pgprot(_PAGE_BASE | _PAGE_RW) +#define PAGE_SHARED_X __pgprot(_PAGE_BASE | _PAGE_RWX) +#define PAGE_COPY __pgprot(_PAGE_BASE | _PAGE_RO) +#define PAGE_COPY_X __pgprot(_PAGE_BASE | _PAGE_ROX) +#define PAGE_READONLY __pgprot(_PAGE_BASE | _PAGE_RO) +#define PAGE_READONLY_X __pgprot(_PAGE_BASE | _PAGE_ROX) + +#endif /* _ASM_POWERPC_PGTABLE_MASKS_H */ From f9f09b93e80148fc5824afb338c318272abde529 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 25 Sep 2023 20:31:40 +0200 Subject: [PATCH 298/475] powerpc/8xx: Use generic permission masks 8xx already has _PAGE_NA and _PAGE_RO. So add _PAGE_ROX, _PAGE_RW and _PAGE_RWX and remove specific permission masks. Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://msgid.link/5d0b5ce43485f697313eee4326ddff97157fb219.1695659959.git.christophe.leroy@csgroup.eu --- arch/powerpc/include/asm/nohash/32/pte-8xx.h | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/arch/powerpc/include/asm/nohash/32/pte-8xx.h b/arch/powerpc/include/asm/nohash/32/pte-8xx.h index 843fe0138a66..62c965a4511a 100644 --- a/arch/powerpc/include/asm/nohash/32/pte-8xx.h +++ b/arch/powerpc/include/asm/nohash/32/pte-8xx.h @@ -48,6 +48,10 @@ #define _PAGE_HUGE 0x0800 /* Copied to L1 PS bit 29 */ +#define _PAGE_ROX (_PAGE_RO | _PAGE_EXEC) +#define _PAGE_RW 0 +#define _PAGE_RWX _PAGE_EXEC + /* cache related flags non existing on 8xx */ #define _PAGE_COHERENT 0 #define _PAGE_WRITETHRU 0 @@ -77,14 +81,7 @@ #define _PAGE_BASE_NC (_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_PSIZE) #define _PAGE_BASE (_PAGE_BASE_NC) -/* Permission masks used to generate the __P and __S table */ -#define PAGE_NONE __pgprot(_PAGE_BASE | _PAGE_NA) -#define PAGE_SHARED __pgprot(_PAGE_BASE) -#define PAGE_SHARED_X __pgprot(_PAGE_BASE | _PAGE_EXEC) -#define PAGE_COPY __pgprot(_PAGE_BASE | _PAGE_RO) -#define PAGE_COPY_X __pgprot(_PAGE_BASE | _PAGE_RO | _PAGE_EXEC) -#define PAGE_READONLY __pgprot(_PAGE_BASE | _PAGE_RO) -#define PAGE_READONLY_X __pgprot(_PAGE_BASE | _PAGE_RO | _PAGE_EXEC) +#include #ifndef __ASSEMBLY__ static inline pte_t pte_wrprotect(pte_t pte) From 58f534623c4d8800c2e5d63da9783530848e570c Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 25 Sep 2023 20:31:41 +0200 Subject: [PATCH 299/475] powerpc/64s: Use generic permission masks book3s64 need specific masks because it needs _PAGE_PRIVILEGED for PAGE_NONE. book3s64 already has _PAGE_RW and _PAGE_RWX. So add _PAGE_NA, _PAGE_RO and _PAGE_ROX and remove specific permission masks. Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://msgid.link/d37a418de52e2c950cad6797e81663b56991366f.1695659959.git.christophe.leroy@csgroup.eu --- arch/powerpc/include/asm/book3s/64/pgtable.h | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h index c3b921769ece..0fd12bdc7b5e 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -17,6 +17,9 @@ #define _PAGE_EXEC 0x00001 /* execute permission */ #define _PAGE_WRITE 0x00002 /* write access allowed */ #define _PAGE_READ 0x00004 /* read access allowed */ +#define _PAGE_NA _PAGE_PRIVILEGED +#define _PAGE_RO _PAGE_READ +#define _PAGE_ROX (_PAGE_READ | _PAGE_EXEC) #define _PAGE_RW (_PAGE_READ | _PAGE_WRITE) #define _PAGE_RWX (_PAGE_READ | _PAGE_WRITE | _PAGE_EXEC) #define _PAGE_PRIVILEGED 0x00008 /* kernel access only */ @@ -136,21 +139,8 @@ #define _PAGE_BASE_NC (_PAGE_PRESENT | _PAGE_ACCESSED) #define _PAGE_BASE (_PAGE_BASE_NC) -/* Permission masks used to generate the __P and __S table, - * - * Note:__pgprot is defined in arch/powerpc/include/asm/page.h - * - * Write permissions imply read permissions for now (we could make write-only - * pages on BookE but we don't bother for now). Execute permission control is - * possible on platforms that define _PAGE_EXEC - */ -#define PAGE_NONE __pgprot(_PAGE_BASE | _PAGE_PRIVILEGED) -#define PAGE_SHARED __pgprot(_PAGE_BASE | _PAGE_RW) -#define PAGE_SHARED_X __pgprot(_PAGE_BASE | _PAGE_RW | _PAGE_EXEC) -#define PAGE_COPY __pgprot(_PAGE_BASE | _PAGE_READ) -#define PAGE_COPY_X __pgprot(_PAGE_BASE | _PAGE_READ | _PAGE_EXEC) -#define PAGE_READONLY __pgprot(_PAGE_BASE | _PAGE_READ) -#define PAGE_READONLY_X __pgprot(_PAGE_BASE | _PAGE_READ | _PAGE_EXEC) +#include + /* Radix only, Hash uses PAGE_READONLY_X + execute-only pkey instead */ #define PAGE_EXECONLY __pgprot(_PAGE_BASE | _PAGE_EXEC) From d20506d4728c3b7408e84d9aececbcb78c3061ee Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 25 Sep 2023 20:31:42 +0200 Subject: [PATCH 300/475] powerpc/nohash: Add _PAGE_WRITE to supplement _PAGE_RW Several places, _PAGE_RW maps to write permission and don't always imply read. To make it more clear, do as book3s/64 in commit c7d54842deb1 ("powerpc/mm: Use _PAGE_READ to indicate Read access") and use _PAGE_WRITE when more relevant. For the time being _PAGE_WRITE is equivalent to _PAGE_RW but that will change when _PAGE_READ gets added in following patches. Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://msgid.link/1f79b88db54d030ada776dc9845e0e88345bfc28.1695659959.git.christophe.leroy@csgroup.eu --- arch/powerpc/include/asm/nohash/32/pte-40x.h | 2 ++ arch/powerpc/include/asm/nohash/32/pte-44x.h | 2 ++ arch/powerpc/include/asm/nohash/32/pte-85xx.h | 2 ++ arch/powerpc/include/asm/nohash/64/pgtable.h | 2 +- arch/powerpc/include/asm/nohash/pgtable.h | 9 ++++++--- arch/powerpc/include/asm/nohash/pte-e500.h | 2 ++ arch/powerpc/kernel/head_40x.S | 12 ++++++------ arch/powerpc/kernel/head_44x.S | 4 ++-- arch/powerpc/kernel/head_85xx.S | 2 +- arch/powerpc/mm/nohash/e500.c | 4 ++-- 10 files changed, 26 insertions(+), 15 deletions(-) diff --git a/arch/powerpc/include/asm/nohash/32/pte-40x.h b/arch/powerpc/include/asm/nohash/32/pte-40x.h index 0b4e5f8ce3e8..e28ef0f5781e 100644 --- a/arch/powerpc/include/asm/nohash/32/pte-40x.h +++ b/arch/powerpc/include/asm/nohash/32/pte-40x.h @@ -49,6 +49,8 @@ #define _PAGE_EXEC 0x200 /* hardware: EX permission */ #define _PAGE_ACCESSED 0x400 /* software: R: page referenced */ +#define _PAGE_WRITE _PAGE_RW + /* No page size encoding in the linux PTE */ #define _PAGE_PSIZE 0 diff --git a/arch/powerpc/include/asm/nohash/32/pte-44x.h b/arch/powerpc/include/asm/nohash/32/pte-44x.h index b7ed13cee137..fc0c075006ea 100644 --- a/arch/powerpc/include/asm/nohash/32/pte-44x.h +++ b/arch/powerpc/include/asm/nohash/32/pte-44x.h @@ -75,6 +75,8 @@ #define _PAGE_NO_CACHE 0x00000400 /* H: I bit */ #define _PAGE_WRITETHRU 0x00000800 /* H: W bit */ +#define _PAGE_WRITE _PAGE_RW + /* No page size encoding in the linux PTE */ #define _PAGE_PSIZE 0 diff --git a/arch/powerpc/include/asm/nohash/32/pte-85xx.h b/arch/powerpc/include/asm/nohash/32/pte-85xx.h index 16451df5ddb0..462acf69e302 100644 --- a/arch/powerpc/include/asm/nohash/32/pte-85xx.h +++ b/arch/powerpc/include/asm/nohash/32/pte-85xx.h @@ -31,6 +31,8 @@ #define _PAGE_WRITETHRU 0x00400 /* H: W bit */ #define _PAGE_SPECIAL 0x00800 /* S: Special page */ +#define _PAGE_WRITE _PAGE_RW + #define _PAGE_KERNEL_RO 0 #define _PAGE_KERNEL_ROX _PAGE_EXEC #define _PAGE_KERNEL_RW (_PAGE_DIRTY | _PAGE_RW) diff --git a/arch/powerpc/include/asm/nohash/64/pgtable.h b/arch/powerpc/include/asm/nohash/64/pgtable.h index 36b9bad428cc..2202c78730e8 100644 --- a/arch/powerpc/include/asm/nohash/64/pgtable.h +++ b/arch/powerpc/include/asm/nohash/64/pgtable.h @@ -149,7 +149,7 @@ static inline void p4d_set(p4d_t *p4dp, unsigned long val) static inline void huge_ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { - pte_update(mm, addr, ptep, _PAGE_RW, 0, 1); + pte_update(mm, addr, ptep, _PAGE_WRITE, 0, 1); } #define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH diff --git a/arch/powerpc/include/asm/nohash/pgtable.h b/arch/powerpc/include/asm/nohash/pgtable.h index 942f46e86b4b..9a91efa6455b 100644 --- a/arch/powerpc/include/asm/nohash/pgtable.h +++ b/arch/powerpc/include/asm/nohash/pgtable.h @@ -84,7 +84,7 @@ static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { - pte_update(mm, addr, ptep, _PAGE_RW, 0, 0); + pte_update(mm, addr, ptep, _PAGE_WRITE, 0, 0); } #endif #define __HAVE_ARCH_PTEP_SET_WRPROTECT @@ -122,6 +122,9 @@ static inline void __ptep_set_access_flags(struct vm_area_struct *vma, #ifndef pte_mkwrite_novma static inline pte_t pte_mkwrite_novma(pte_t pte) { + /* + * write implies read, hence set both + */ return __pte(pte_val(pte) | _PAGE_RW); } #endif @@ -139,7 +142,7 @@ static inline pte_t pte_mkyoung(pte_t pte) #ifndef pte_wrprotect static inline pte_t pte_wrprotect(pte_t pte) { - return __pte(pte_val(pte) & ~_PAGE_RW); + return __pte(pte_val(pte) & ~_PAGE_WRITE); } #endif @@ -153,7 +156,7 @@ static inline pte_t pte_mkexec(pte_t pte) #ifndef pte_write static inline int pte_write(pte_t pte) { - return pte_val(pte) & _PAGE_RW; + return pte_val(pte) & _PAGE_WRITE; } #endif #ifndef pte_read diff --git a/arch/powerpc/include/asm/nohash/pte-e500.h b/arch/powerpc/include/asm/nohash/pte-e500.h index 9f9e3f02d414..b775c7d465a4 100644 --- a/arch/powerpc/include/asm/nohash/pte-e500.h +++ b/arch/powerpc/include/asm/nohash/pte-e500.h @@ -55,6 +55,8 @@ #define _PAGE_KERNEL_ROX (_PAGE_BAP_SR | _PAGE_BAP_SX) #define _PAGE_USER (_PAGE_BAP_UR | _PAGE_BAP_SR) /* Can be read */ +#define _PAGE_WRITE _PAGE_RW + #define _PAGE_SPECIAL _PAGE_SW0 /* Base page size */ diff --git a/arch/powerpc/kernel/head_40x.S b/arch/powerpc/kernel/head_40x.S index b32e7b2ebdcf..9f92f5c5e6aa 100644 --- a/arch/powerpc/kernel/head_40x.S +++ b/arch/powerpc/kernel/head_40x.S @@ -316,9 +316,9 @@ _ASM_NOKPROBE_SYMBOL(\name\()_virt) andc. r9, r9, r11 /* Check permission */ bne 5f - rlwinm r9, r11, 1, _PAGE_RW /* dirty => rw */ - and r9, r9, r11 /* hwwrite = dirty & rw */ - rlwimi r11, r9, 0, _PAGE_RW /* replace rw by hwwrite */ + rlwinm r9, r11, 1, _PAGE_WRITE /* dirty => w */ + and r9, r9, r11 /* hwwrite = dirty & w */ + rlwimi r11, r9, 0, _PAGE_WRITE /* replace w by hwwrite */ /* Create TLB tag. This is the faulting address plus a static * set of bits. These are size, valid, E, U0. @@ -400,9 +400,9 @@ _ASM_NOKPROBE_SYMBOL(\name\()_virt) andc. r9, r9, r11 /* Check permission */ bne 5f - rlwinm r9, r11, 1, _PAGE_RW /* dirty => rw */ - and r9, r9, r11 /* hwwrite = dirty & rw */ - rlwimi r11, r9, 0, _PAGE_RW /* replace rw by hwwrite */ + rlwinm r9, r11, 1, _PAGE_WRITE /* dirty => w */ + and r9, r9, r11 /* hwwrite = dirty & w */ + rlwimi r11, r9, 0, _PAGE_WRITE /* replace w by hwwrite */ /* Create TLB tag. This is the faulting address plus a static * set of bits. These are size, valid, E, U0. diff --git a/arch/powerpc/kernel/head_44x.S b/arch/powerpc/kernel/head_44x.S index a3197c9f721c..858dabf84432 100644 --- a/arch/powerpc/kernel/head_44x.S +++ b/arch/powerpc/kernel/head_44x.S @@ -342,7 +342,7 @@ interrupt_base: mtspr SPRN_MMUCR,r12 /* Mask of required permission bits. Note that while we - * do copy ESR:ST to _PAGE_RW position as trying to write + * do copy ESR:ST to _PAGE_WRITE position as trying to write * to an RO page is pretty common, we don't do it with * _PAGE_DIRTY. We could do it, but it's a fairly rare * event so I'd rather take the overhead when it happens @@ -586,7 +586,7 @@ finish_tlb_load_44x: 4: mtspr SPRN_MMUCR,r12 /* Set MMUCR */ /* Mask of required permission bits. Note that while we - * do copy ESR:ST to _PAGE_RW position as trying to write + * do copy ESR:ST to _PAGE_WRITE position as trying to write * to an RO page is pretty common, we don't do it with * _PAGE_DIRTY. We could do it, but it's a fairly rare * event so I'd rather take the overhead when it happens diff --git a/arch/powerpc/kernel/head_85xx.S b/arch/powerpc/kernel/head_85xx.S index 0f1641a31250..1b122cba557b 100644 --- a/arch/powerpc/kernel/head_85xx.S +++ b/arch/powerpc/kernel/head_85xx.S @@ -471,7 +471,7 @@ END_BTB_FLUSH_SECTION 4: /* Mask of required permission bits. Note that while we - * do copy ESR:ST to _PAGE_RW position as trying to write + * do copy ESR:ST to _PAGE_WRITE position as trying to write * to an RO page is pretty common, we don't do it with * _PAGE_DIRTY. We could do it, but it's a fairly rare * event so I'd rather take the overhead when it happens diff --git a/arch/powerpc/mm/nohash/e500.c b/arch/powerpc/mm/nohash/e500.c index 5b7d7a932bfd..921c3521ec11 100644 --- a/arch/powerpc/mm/nohash/e500.c +++ b/arch/powerpc/mm/nohash/e500.c @@ -117,7 +117,7 @@ static void settlbcam(int index, unsigned long virt, phys_addr_t phys, TLBCAM[index].MAS2 |= (flags & _PAGE_ENDIAN) ? MAS2_E : 0; TLBCAM[index].MAS3 = (phys & MAS3_RPN) | MAS3_SR; - TLBCAM[index].MAS3 |= (flags & _PAGE_RW) ? MAS3_SW : 0; + TLBCAM[index].MAS3 |= (flags & _PAGE_WRITE) ? MAS3_SW : 0; if (mmu_has_feature(MMU_FTR_BIG_PHYS)) TLBCAM[index].MAS7 = (u64)phys >> 32; @@ -125,7 +125,7 @@ static void settlbcam(int index, unsigned long virt, phys_addr_t phys, if (!is_kernel_addr(virt)) { TLBCAM[index].MAS3 |= MAS3_UR; TLBCAM[index].MAS3 |= (flags & _PAGE_EXEC) ? MAS3_UX : 0; - TLBCAM[index].MAS3 |= (flags & _PAGE_RW) ? MAS3_UW : 0; + TLBCAM[index].MAS3 |= (flags & _PAGE_WRITE) ? MAS3_UW : 0; } else { TLBCAM[index].MAS3 |= (flags & _PAGE_EXEC) ? MAS3_SX : 0; } From 8e9bd41e4ce1001f5b89e4c9a69f870f39d56c12 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 25 Sep 2023 20:31:43 +0200 Subject: [PATCH 301/475] powerpc/nohash: Replace pte_user() by pte_read() pte_user() is now only used in pte_access_permitted() to check access on vmas. User flag is cleared to make a page unreadable. So rename it pte_read() and remove pte_user() which isn't used anymore. For the time being it checks _PAGE_USER but in the near futur all plateforms will be converted to _PAGE_READ so lets support both for now. Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://msgid.link/72cbb5be595e9ef884140def73815ed0b0b37010.1695659959.git.christophe.leroy@csgroup.eu --- arch/powerpc/include/asm/nohash/32/pte-8xx.h | 7 ------- arch/powerpc/include/asm/nohash/pgtable.h | 13 +++++++------ arch/powerpc/mm/ioremap.c | 4 ---- 3 files changed, 7 insertions(+), 17 deletions(-) diff --git a/arch/powerpc/include/asm/nohash/32/pte-8xx.h b/arch/powerpc/include/asm/nohash/32/pte-8xx.h index 62c965a4511a..1ee38befd29a 100644 --- a/arch/powerpc/include/asm/nohash/32/pte-8xx.h +++ b/arch/powerpc/include/asm/nohash/32/pte-8xx.h @@ -112,13 +112,6 @@ static inline pte_t pte_mkwrite_novma(pte_t pte) #define pte_mkwrite_novma pte_mkwrite_novma -static inline bool pte_user(pte_t pte) -{ - return !(pte_val(pte) & _PAGE_SH); -} - -#define pte_user pte_user - static inline pte_t pte_mkhuge(pte_t pte) { return __pte(pte_val(pte) | _PAGE_SPS | _PAGE_HUGE); diff --git a/arch/powerpc/include/asm/nohash/pgtable.h b/arch/powerpc/include/asm/nohash/pgtable.h index 9a91efa6455b..84b6a160d2fc 100644 --- a/arch/powerpc/include/asm/nohash/pgtable.h +++ b/arch/powerpc/include/asm/nohash/pgtable.h @@ -159,9 +159,6 @@ static inline int pte_write(pte_t pte) return pte_val(pte) & _PAGE_WRITE; } #endif -#ifndef pte_read -static inline int pte_read(pte_t pte) { return 1; } -#endif static inline int pte_dirty(pte_t pte) { return pte_val(pte) & _PAGE_DIRTY; } static inline int pte_special(pte_t pte) { return pte_val(pte) & _PAGE_SPECIAL; } static inline int pte_none(pte_t pte) { return (pte_val(pte) & ~_PTE_NONE_MASK) == 0; } @@ -189,10 +186,14 @@ static inline int pte_young(pte_t pte) * and PTE_64BIT, PAGE_KERNEL_X contains _PAGE_BAP_SR which is also in * _PAGE_USER. Need to explicitly match _PAGE_BAP_UR bit in that case too. */ -#ifndef pte_user -static inline bool pte_user(pte_t pte) +#ifndef pte_read +static inline bool pte_read(pte_t pte) { +#ifdef _PAGE_READ + return (pte_val(pte) & _PAGE_READ) == _PAGE_READ; +#else return (pte_val(pte) & _PAGE_USER) == _PAGE_USER; +#endif } #endif @@ -207,7 +208,7 @@ static inline bool pte_access_permitted(pte_t pte, bool write) * A read-only access is controlled by _PAGE_USER bit. * We have _PAGE_READ set for WRITE and EXECUTE */ - if (!pte_present(pte) || !pte_user(pte) || !pte_read(pte)) + if (!pte_present(pte) || !pte_read(pte)) return false; if (write && !pte_write(pte)) diff --git a/arch/powerpc/mm/ioremap.c b/arch/powerpc/mm/ioremap.c index 7823c38f09de..7b0afcabd89f 100644 --- a/arch/powerpc/mm/ioremap.c +++ b/arch/powerpc/mm/ioremap.c @@ -50,10 +50,6 @@ void __iomem *ioremap_prot(phys_addr_t addr, size_t size, unsigned long flags) if (pte_write(pte)) pte = pte_mkdirty(pte); - /* we don't want to let _PAGE_USER leak out */ - if (WARN_ON(pte_user(pte))) - return NULL; - if (iowa_is_active()) return iowa_ioremap(addr, size, pte_pgprot(pte), caller); return __ioremap_caller(addr, size, pte_pgprot(pte), caller); From 48cf93bb168d506a8278a6fb25c2f88c1c93ce6e Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 25 Sep 2023 20:31:44 +0200 Subject: [PATCH 302/475] powerpc/e500: Introduce _PAGE_READ and remove _PAGE_USER e500 MMU has 6 page protection bits: - R, W, X for supervisor - R, W, X for user It means that it can support X without R. To do that, _PAGE_READ flag is needed. With 32 bits PTE there is no bit available for it in PTE. On the other hand the only real use of _PAGE_USER is to implement PAGE_NONE by clearing _PAGE_USER. As _PAGE_NONE can also be implemented by clearing _PAGE_READ, remove _PAGE_USER and add _PAGE_READ. Move _PAGE_PRESENT into bit 30 so that _PAGE_READ can match SR bit. With 64 bits PTE _PAGE_USER is already the combination of SR and UR so all we need to do is to rename it _PAGE_READ. Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://msgid.link/0849ab6bf7ae2af23f94b0457fa40d0ea3983fe4.1695659959.git.christophe.leroy@csgroup.eu --- arch/powerpc/include/asm/nohash/32/pte-85xx.h | 22 ++++--------------- arch/powerpc/include/asm/nohash/pte-e500.h | 20 ++++++++--------- arch/powerpc/kernel/head_85xx.S | 10 ++++----- 3 files changed, 18 insertions(+), 34 deletions(-) diff --git a/arch/powerpc/include/asm/nohash/32/pte-85xx.h b/arch/powerpc/include/asm/nohash/32/pte-85xx.h index 462acf69e302..653a342d3b25 100644 --- a/arch/powerpc/include/asm/nohash/32/pte-85xx.h +++ b/arch/powerpc/include/asm/nohash/32/pte-85xx.h @@ -17,9 +17,9 @@ */ /* Definitions for FSL Book-E Cores */ -#define _PAGE_PRESENT 0x00001 /* S: PTE contains a translation */ -#define _PAGE_USER 0x00002 /* S: User page (maps to UR) */ -#define _PAGE_RW 0x00004 /* S: Write permission (SW) */ +#define _PAGE_READ 0x00001 /* H: Read permission (SR) */ +#define _PAGE_PRESENT 0x00002 /* S: PTE contains a translation */ +#define _PAGE_WRITE 0x00004 /* S: Write permission (SW) */ #define _PAGE_DIRTY 0x00008 /* S: Page dirty */ #define _PAGE_EXEC 0x00010 /* H: SX permission */ #define _PAGE_ACCESSED 0x00020 /* S: Page referenced */ @@ -31,13 +31,6 @@ #define _PAGE_WRITETHRU 0x00400 /* H: W bit */ #define _PAGE_SPECIAL 0x00800 /* S: Special page */ -#define _PAGE_WRITE _PAGE_RW - -#define _PAGE_KERNEL_RO 0 -#define _PAGE_KERNEL_ROX _PAGE_EXEC -#define _PAGE_KERNEL_RW (_PAGE_DIRTY | _PAGE_RW) -#define _PAGE_KERNEL_RWX (_PAGE_DIRTY | _PAGE_RW | _PAGE_EXEC) - /* No page size encoding in the linux PTE */ #define _PAGE_PSIZE 0 @@ -63,14 +56,7 @@ #define _PAGE_BASE (_PAGE_BASE_NC) #endif -/* Permission masks used to generate the __P and __S table */ -#define PAGE_NONE __pgprot(_PAGE_BASE) -#define PAGE_SHARED __pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_RW) -#define PAGE_SHARED_X __pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_RW | _PAGE_EXEC) -#define PAGE_COPY __pgprot(_PAGE_BASE | _PAGE_USER) -#define PAGE_COPY_X __pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_EXEC) -#define PAGE_READONLY __pgprot(_PAGE_BASE | _PAGE_USER) -#define PAGE_READONLY_X __pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_EXEC) +#include #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_NOHASH_32_PTE_FSL_85xx_H */ diff --git a/arch/powerpc/include/asm/nohash/pte-e500.h b/arch/powerpc/include/asm/nohash/pte-e500.h index b775c7d465a4..31d2c3ea7df8 100644 --- a/arch/powerpc/include/asm/nohash/pte-e500.h +++ b/arch/powerpc/include/asm/nohash/pte-e500.h @@ -48,14 +48,19 @@ /* "Higher level" linux bit combinations */ #define _PAGE_EXEC (_PAGE_BAP_SX | _PAGE_BAP_UX) /* .. and was cache cleaned */ -#define _PAGE_RW (_PAGE_BAP_SW | _PAGE_BAP_UW) /* User write permission */ +#define _PAGE_READ (_PAGE_BAP_SR | _PAGE_BAP_UR) /* User read permission */ +#define _PAGE_WRITE (_PAGE_BAP_SW | _PAGE_BAP_UW) /* User write permission */ + #define _PAGE_KERNEL_RW (_PAGE_BAP_SW | _PAGE_BAP_SR | _PAGE_DIRTY) #define _PAGE_KERNEL_RO (_PAGE_BAP_SR) #define _PAGE_KERNEL_RWX (_PAGE_BAP_SW | _PAGE_BAP_SR | _PAGE_DIRTY | _PAGE_BAP_SX) #define _PAGE_KERNEL_ROX (_PAGE_BAP_SR | _PAGE_BAP_SX) -#define _PAGE_USER (_PAGE_BAP_UR | _PAGE_BAP_SR) /* Can be read */ -#define _PAGE_WRITE _PAGE_RW +#define _PAGE_NA 0 +#define _PAGE_RO _PAGE_READ +#define _PAGE_ROX (_PAGE_READ | _PAGE_BAP_UX) +#define _PAGE_RW (_PAGE_READ | _PAGE_WRITE) +#define _PAGE_RWX (_PAGE_READ | _PAGE_WRITE | _PAGE_BAP_UX) #define _PAGE_SPECIAL _PAGE_SW0 @@ -90,14 +95,7 @@ #define _PAGE_BASE (_PAGE_BASE_NC) #endif -/* Permission masks used to generate the __P and __S table */ -#define PAGE_NONE __pgprot(_PAGE_BASE) -#define PAGE_SHARED __pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_RW) -#define PAGE_SHARED_X __pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_RW | _PAGE_BAP_UX) -#define PAGE_COPY __pgprot(_PAGE_BASE | _PAGE_USER) -#define PAGE_COPY_X __pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_BAP_UX) -#define PAGE_READONLY __pgprot(_PAGE_BASE | _PAGE_USER) -#define PAGE_READONLY_X __pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_BAP_UX) +#include #ifndef __ASSEMBLY__ static inline pte_t pte_mkexec(pte_t pte) diff --git a/arch/powerpc/kernel/head_85xx.S b/arch/powerpc/kernel/head_85xx.S index 1b122cba557b..39724ff5ae1f 100644 --- a/arch/powerpc/kernel/head_85xx.S +++ b/arch/powerpc/kernel/head_85xx.S @@ -485,10 +485,10 @@ END_BTB_FLUSH_SECTION */ mfspr r12,SPRN_ESR #ifdef CONFIG_PTE_64BIT - li r13,_PAGE_PRESENT + li r13,_PAGE_PRESENT|_PAGE_BAP_SR oris r13,r13,_PAGE_ACCESSED@h #else - li r13,_PAGE_PRESENT|_PAGE_ACCESSED + li r13,_PAGE_PRESENT|_PAGE_READ|_PAGE_ACCESSED #endif rlwimi r13,r12,11,29,29 @@ -783,15 +783,15 @@ BEGIN_MMU_FTR_SECTION mtspr SPRN_MAS7, r10 END_MMU_FTR_SECTION_IFSET(MMU_FTR_BIG_PHYS) #else - li r10, (_PAGE_EXEC | _PAGE_PRESENT) + li r10, (_PAGE_EXEC | _PAGE_READ) mr r13, r11 rlwimi r10, r11, 31, 29, 29 /* extract _PAGE_DIRTY into SW */ and r12, r11, r10 - andi. r10, r11, _PAGE_USER /* Test for _PAGE_USER */ + mcrf cr0, cr5 /* Test for user page */ slwi r10, r12, 1 or r10, r10, r12 rlwinm r10, r10, 0, ~_PAGE_EXEC /* Clear SX on user pages */ - iseleq r12, r12, r10 + isellt r12, r10, r12 rlwimi r13, r12, 0, 20, 31 /* Get RPN from PTE, merge w/ perms */ mtspr SPRN_MAS3, r13 #endif From 93820bfeefc4a125a6cedd1ee1a956eeb3eb2580 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 25 Sep 2023 20:31:45 +0200 Subject: [PATCH 303/475] powerpc/44x: Introduce _PAGE_READ and remove _PAGE_USER 44x MMU has 6 page protection bits: - R, W, X for supervisor - R, W, X for user It means that it can support X without R. To do that, _PAGE_READ flag is needed but there is no bit available for it in PTE. On the other hand the only real use of _PAGE_USER is to implement PAGE_NONE by clearing _PAGE_USER. As _PAGE_NONE can also be implemented by clearing _PAGE_READ, remove _PAGE_USER and add _PAGE_READ. In order to insert bits in one go during TLB miss, move _PAGE_ACCESSED and put _PAGE_READ just after _PAGE_DIRTY so that _PAGE_DIRTY is copied into SW and _PAGE_READ into SR at once. With that change, 44x now also honors execute-only protection. Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://msgid.link/043e17987b260b99b45094138c6cb2e89e63d499.1695659959.git.christophe.leroy@csgroup.eu --- arch/powerpc/include/asm/nohash/32/pte-44x.h | 22 +++--------- arch/powerpc/kernel/head_44x.S | 36 ++++++++++---------- 2 files changed, 22 insertions(+), 36 deletions(-) diff --git a/arch/powerpc/include/asm/nohash/32/pte-44x.h b/arch/powerpc/include/asm/nohash/32/pte-44x.h index fc0c075006ea..851813725237 100644 --- a/arch/powerpc/include/asm/nohash/32/pte-44x.h +++ b/arch/powerpc/include/asm/nohash/32/pte-44x.h @@ -63,28 +63,21 @@ */ #define _PAGE_PRESENT 0x00000001 /* S: PTE valid */ -#define _PAGE_RW 0x00000002 /* S: Write permission */ +#define _PAGE_WRITE 0x00000002 /* S: Write permission */ #define _PAGE_EXEC 0x00000004 /* H: Execute permission */ -#define _PAGE_ACCESSED 0x00000008 /* S: Page referenced */ +#define _PAGE_READ 0x00000008 /* S: Read permission */ #define _PAGE_DIRTY 0x00000010 /* S: Page dirty */ #define _PAGE_SPECIAL 0x00000020 /* S: Special page */ -#define _PAGE_USER 0x00000040 /* S: User page */ +#define _PAGE_ACCESSED 0x00000040 /* S: Page referenced */ #define _PAGE_ENDIAN 0x00000080 /* H: E bit */ #define _PAGE_GUARDED 0x00000100 /* H: G bit */ #define _PAGE_COHERENT 0x00000200 /* H: M bit */ #define _PAGE_NO_CACHE 0x00000400 /* H: I bit */ #define _PAGE_WRITETHRU 0x00000800 /* H: W bit */ -#define _PAGE_WRITE _PAGE_RW - /* No page size encoding in the linux PTE */ #define _PAGE_PSIZE 0 -#define _PAGE_KERNEL_RO 0 -#define _PAGE_KERNEL_ROX _PAGE_EXEC -#define _PAGE_KERNEL_RW (_PAGE_DIRTY | _PAGE_RW) -#define _PAGE_KERNEL_RWX (_PAGE_DIRTY | _PAGE_RW | _PAGE_EXEC) - /* TODO: Add large page lowmem mapping support */ #define _PMD_PRESENT 0 #define _PMD_PRESENT_MASK (PAGE_MASK) @@ -107,14 +100,7 @@ #define _PAGE_BASE (_PAGE_BASE_NC) #endif -/* Permission masks used to generate the __P and __S table */ -#define PAGE_NONE __pgprot(_PAGE_BASE) -#define PAGE_SHARED __pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_RW) -#define PAGE_SHARED_X __pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_RW | _PAGE_EXEC) -#define PAGE_COPY __pgprot(_PAGE_BASE | _PAGE_USER) -#define PAGE_COPY_X __pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_EXEC) -#define PAGE_READONLY __pgprot(_PAGE_BASE | _PAGE_USER) -#define PAGE_READONLY_X __pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_EXEC) +#include #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_NOHASH_32_PTE_44x_H */ diff --git a/arch/powerpc/kernel/head_44x.S b/arch/powerpc/kernel/head_44x.S index 858dabf84432..25642e802ed3 100644 --- a/arch/powerpc/kernel/head_44x.S +++ b/arch/powerpc/kernel/head_44x.S @@ -314,8 +314,8 @@ interrupt_base: * kernel page tables. */ lis r11, PAGE_OFFSET@h - cmplw r10, r11 - blt+ 3f + cmplw cr7, r10, r11 + blt+ cr7, 3f lis r11, swapper_pg_dir@h ori r11, r11, swapper_pg_dir@l @@ -355,7 +355,7 @@ interrupt_base: * place or can we save a couple of instructions here ? */ mfspr r12,SPRN_ESR - li r13,_PAGE_PRESENT|_PAGE_ACCESSED + li r13,_PAGE_PRESENT|_PAGE_ACCESSED|_PAGE_READ rlwimi r13,r12,10,30,30 /* Load the PTE */ @@ -428,8 +428,8 @@ interrupt_base: * kernel page tables. */ lis r11, PAGE_OFFSET@h - cmplw r10, r11 - blt+ 3f + cmplw cr7, r10, r11 + blt+ cr7, 3f lis r11, swapper_pg_dir@h ori r11, r11, swapper_pg_dir@l @@ -515,6 +515,7 @@ interrupt_base: * r11 - PTE high word value * r12 - PTE low word value * r13 - TLB index + * cr7 - Result of comparison with PAGE_OFFSET * MMUCR - loaded with proper value when we get here * Upon exit, we reload everything and RFI. */ @@ -533,11 +534,10 @@ finish_tlb_load_44x: tlbwe r10,r13,PPC44x_TLB_PAGEID /* Write PAGEID */ /* And WS 2 */ - li r10,0xf85 /* Mask to apply from PTE */ - rlwimi r10,r12,29,30,30 /* DIRTY -> SW position */ + li r10,0xf84 /* Mask to apply from PTE */ + rlwimi r10,r12,29,30,31 /* DIRTY,READ -> SW,SR position */ and r11,r12,r10 /* Mask PTE bits to keep */ - andi. r10,r12,_PAGE_USER /* User page ? */ - beq 1f /* nope, leave U bits empty */ + bge cr7,1f /* User page ? no, leave U bits empty */ rlwimi r11,r11,3,26,28 /* yes, copy S bits to U */ rlwinm r11,r11,0,~PPC44x_TLB_SX /* Clear SX if User page */ 1: tlbwe r11,r13,PPC44x_TLB_ATTRIB /* Write ATTRIB */ @@ -568,8 +568,8 @@ finish_tlb_load_44x: * kernel page tables. */ lis r11,PAGE_OFFSET@h - cmplw cr0,r10,r11 - blt+ 3f + cmplw cr7,r10,r11 + blt+ cr7,3f lis r11,swapper_pg_dir@h ori r11,r11, swapper_pg_dir@l li r12,0 /* MMUCR = 0 */ @@ -599,7 +599,7 @@ finish_tlb_load_44x: * place or can we save a couple of instructions here ? */ mfspr r12,SPRN_ESR - li r13,_PAGE_PRESENT|_PAGE_ACCESSED + li r13,_PAGE_PRESENT|_PAGE_ACCESSED|_PAGE_READ rlwimi r13,r12,10,30,30 /* Load the PTE */ @@ -669,8 +669,8 @@ finish_tlb_load_44x: * kernel page tables. */ lis r11,PAGE_OFFSET@h - cmplw cr0,r10,r11 - blt+ 3f + cmplw cr7,r10,r11 + blt+ cr7,3f lis r11,swapper_pg_dir@h ori r11,r11, swapper_pg_dir@l li r12,0 /* MMUCR = 0 */ @@ -744,6 +744,7 @@ finish_tlb_load_44x: * r11 - PTE high word value * r12 - PTE low word value * r13 - free to use + * cr7 - Result of comparison with PAGE_OFFSET * MMUCR - loaded with proper value when we get here * Upon exit, we reload everything and RFI. */ @@ -753,11 +754,10 @@ finish_tlb_load_47x: tlbwe r11,r13,1 /* And make up word 2 */ - li r10,0xf85 /* Mask to apply from PTE */ - rlwimi r10,r12,29,30,30 /* DIRTY -> SW position */ + li r10,0xf84 /* Mask to apply from PTE */ + rlwimi r10,r12,29,30,31 /* DIRTY,READ -> SW,SR position */ and r11,r12,r10 /* Mask PTE bits to keep */ - andi. r10,r12,_PAGE_USER /* User page ? */ - beq 1f /* nope, leave U bits empty */ + bge cr7,1f /* User page ? no, leave U bits empty */ rlwimi r11,r11,3,26,28 /* yes, copy S bits to U */ rlwinm r11,r11,0,~PPC47x_TLB2_SX /* Clear SX if User page */ 1: tlbwe r11,r13,2 From ed815bd3fe9b14a742e2ae094f7f55f70918dbbc Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 25 Sep 2023 20:31:46 +0200 Subject: [PATCH 304/475] powerpc/40x: Introduce _PAGE_READ and remove _PAGE_USER _PAGE_USER is used to select the zone. Today zone 0 is kernel and zone 1 is user. To implement _PAGE_NONE, _PAGE_USER is cleared, leading to no access for user but kernel still has access to the page so it's possible for a user application to write in that page by using a kernel function as trampoline. What is really wanted is to have user rights on pages below TASK_SIZE and no user rights on pages above TASK_SIZE. Use zones for that. There are 16 zones so lets use the 4 upper address bits to set the zone and declare zone rights based on TASK_SIZE. Then drop _PAGE_USER and reuse it as _PAGE_READ that will be checked in Data TLB miss handler. That will properly handle PAGE_NONE for both kernel and user. In addition, it partially implements execute-only right. The implementation won't be complete because once a TLB has been loaded via the Instruction TLB miss handler, it will be possible to read the page. But at least it can't be read unless it is executed first. Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://msgid.link/2a13e3ba8a5dec43143cc1f9a91ec71ea1529f3c.1695659959.git.christophe.leroy@csgroup.eu --- arch/powerpc/include/asm/nohash/32/pte-40x.h | 20 +++----------------- arch/powerpc/kernel/head_40x.S | 7 ++++--- arch/powerpc/mm/nohash/40x.c | 19 ++++++++++++------- 3 files changed, 19 insertions(+), 27 deletions(-) diff --git a/arch/powerpc/include/asm/nohash/32/pte-40x.h b/arch/powerpc/include/asm/nohash/32/pte-40x.h index e28ef0f5781e..d759cfd74754 100644 --- a/arch/powerpc/include/asm/nohash/32/pte-40x.h +++ b/arch/powerpc/include/asm/nohash/32/pte-40x.h @@ -42,26 +42,19 @@ #define _PAGE_PRESENT 0x002 /* software: PTE contains a translation */ #define _PAGE_NO_CACHE 0x004 /* I: caching is inhibited */ #define _PAGE_WRITETHRU 0x008 /* W: caching is write-through */ -#define _PAGE_USER 0x010 /* matches one of the zone permission bits */ +#define _PAGE_READ 0x010 /* software: read permission */ #define _PAGE_SPECIAL 0x020 /* software: Special page */ #define _PAGE_DIRTY 0x080 /* software: dirty page */ -#define _PAGE_RW 0x100 /* hardware: WR, anded with dirty in exception */ +#define _PAGE_WRITE 0x100 /* hardware: WR, anded with dirty in exception */ #define _PAGE_EXEC 0x200 /* hardware: EX permission */ #define _PAGE_ACCESSED 0x400 /* software: R: page referenced */ -#define _PAGE_WRITE _PAGE_RW - /* No page size encoding in the linux PTE */ #define _PAGE_PSIZE 0 /* cache related flags non existing on 40x */ #define _PAGE_COHERENT 0 -#define _PAGE_KERNEL_RO 0 -#define _PAGE_KERNEL_ROX _PAGE_EXEC -#define _PAGE_KERNEL_RW (_PAGE_DIRTY | _PAGE_RW) -#define _PAGE_KERNEL_RWX (_PAGE_DIRTY | _PAGE_RW | _PAGE_EXEC) - #define _PMD_PRESENT 0x400 /* PMD points to page of PTEs */ #define _PMD_PRESENT_MASK _PMD_PRESENT #define _PMD_BAD 0x802 @@ -74,14 +67,7 @@ #define _PAGE_BASE_NC (_PAGE_PRESENT | _PAGE_ACCESSED) #define _PAGE_BASE (_PAGE_BASE_NC) -/* Permission masks used to generate the __P and __S table */ -#define PAGE_NONE __pgprot(_PAGE_BASE) -#define PAGE_SHARED __pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_RW) -#define PAGE_SHARED_X __pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_RW | _PAGE_EXEC) -#define PAGE_COPY __pgprot(_PAGE_BASE | _PAGE_USER) -#define PAGE_COPY_X __pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_EXEC) -#define PAGE_READONLY __pgprot(_PAGE_BASE | _PAGE_USER) -#define PAGE_READONLY_X __pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_EXEC) +#include #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_NOHASH_32_PTE_40x_H */ diff --git a/arch/powerpc/kernel/head_40x.S b/arch/powerpc/kernel/head_40x.S index 9f92f5c5e6aa..9fc90410b385 100644 --- a/arch/powerpc/kernel/head_40x.S +++ b/arch/powerpc/kernel/head_40x.S @@ -312,7 +312,7 @@ _ASM_NOKPROBE_SYMBOL(\name\()_virt) rlwimi r11, r10, 22, 20, 29 /* Compute PTE address */ lwz r11, 0(r11) /* Get Linux PTE */ - li r9, _PAGE_PRESENT | _PAGE_ACCESSED + li r9, _PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_READ andc. r9, r9, r11 /* Check permission */ bne 5f @@ -561,10 +561,11 @@ finish_tlb_load: /* * Clear out the software-only bits in the PTE to generate the * TLB_DATA value. These are the bottom 2 bits of the RPM, the - * top 3 bits of the zone field, and M. + * 4 bits of the zone field, and M. */ - li r9, 0x0ce2 + li r9, 0x0cf2 andc r11, r11, r9 + rlwimi r11, r10, 8, 24, 27 /* Copy 4 upper address bit into zone */ /* load the next available TLB index. */ lwz r9, tlb_4xx_index@l(0) diff --git a/arch/powerpc/mm/nohash/40x.c b/arch/powerpc/mm/nohash/40x.c index 3684d6e570fb..e835e80c09db 100644 --- a/arch/powerpc/mm/nohash/40x.c +++ b/arch/powerpc/mm/nohash/40x.c @@ -48,20 +48,25 @@ */ void __init MMU_init_hw(void) { + int i; + unsigned long zpr; + /* * The Zone Protection Register (ZPR) defines how protection will - * be applied to every page which is a member of a given zone. At - * present, we utilize only two of the 4xx's zones. + * be applied to every page which is a member of a given zone. * The zone index bits (of ZSEL) in the PTE are used for software - * indicators, except the LSB. For user access, zone 1 is used, - * for kernel access, zone 0 is used. We set all but zone 1 - * to zero, allowing only kernel access as indicated in the PTE. - * For zone 1, we set a 01 binary (a value of 10 will not work) + * indicators. We use the 4 upper bits of virtual address to select + * the zone. We set all zones above TASK_SIZE to zero, allowing + * only kernel access as indicated in the PTE. For zones below + * TASK_SIZE, we set a 01 binary (a value of 10 will not work) * to allow user access as indicated in the PTE. This also allows * kernel access as indicated in the PTE. */ - mtspr(SPRN_ZPR, 0x10000000); + for (i = 0, zpr = 0; i < TASK_SIZE >> 28; i++) + zpr |= 1 << (30 - i * 2); + + mtspr(SPRN_ZPR, zpr); flush_instruction_cache(); From 46ebef51fd92f52ba7dca21d3c4332e4127de515 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 25 Sep 2023 20:31:47 +0200 Subject: [PATCH 305/475] powerpc/32s: Add _PAGE_WRITE to supplement _PAGE_RW Several places, _PAGE_RW maps to write permission and don't always imply read. To make it more clear, do as book3s/64 in commit c7d54842deb1 ("powerpc/mm: Use _PAGE_READ to indicate Read access") and use _PAGE_WRITE when more relevant. For the time being _PAGE_WRITE is equivalent to _PAGE_RW but that will change when _PAGE_READ gets added in following patches. Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://msgid.link/5798782869fe4d2698f104948dabd17657b89395.1695659959.git.christophe.leroy@csgroup.eu --- arch/powerpc/include/asm/book3s/32/pgtable.h | 15 ++++++++++++--- arch/powerpc/kernel/head_book3s_32.S | 6 +++--- arch/powerpc/mm/book3s32/hash_low.S | 12 ++++++------ arch/powerpc/mm/book3s32/mmu.c | 2 +- 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/arch/powerpc/include/asm/book3s/32/pgtable.h b/arch/powerpc/include/asm/book3s/32/pgtable.h index 80505915c77c..480ad6b4fd6f 100644 --- a/arch/powerpc/include/asm/book3s/32/pgtable.h +++ b/arch/powerpc/include/asm/book3s/32/pgtable.h @@ -31,6 +31,8 @@ #define _PAGE_RW 0x400 /* software: user write access allowed */ #define _PAGE_SPECIAL 0x800 /* software: Special page */ +#define _PAGE_WRITE _PAGE_RW + #ifdef CONFIG_PTE_64BIT /* We never clear the high word of the pte */ #define _PTE_NONE_MASK (0xffffffff00000000ULL | _PAGE_HASHPTE) @@ -347,7 +349,7 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { - pte_update(mm, addr, ptep, _PAGE_RW, 0, 0); + pte_update(mm, addr, ptep, _PAGE_WRITE, 0, 0); } static inline void __ptep_set_access_flags(struct vm_area_struct *vma, @@ -406,7 +408,11 @@ static inline pte_t pte_swp_clear_exclusive(pte_t pte) } /* Generic accessors to PTE bits */ -static inline int pte_write(pte_t pte) { return !!(pte_val(pte) & _PAGE_RW);} +static inline bool pte_write(pte_t pte) +{ + return !!(pte_val(pte) & _PAGE_WRITE); +} + static inline int pte_read(pte_t pte) { return 1; } static inline int pte_dirty(pte_t pte) { return !!(pte_val(pte) & _PAGE_DIRTY); } static inline int pte_young(pte_t pte) { return !!(pte_val(pte) & _PAGE_ACCESSED); } @@ -469,7 +475,7 @@ static inline pte_t pfn_pte(unsigned long pfn, pgprot_t pgprot) /* Generic modifiers for PTE bits */ static inline pte_t pte_wrprotect(pte_t pte) { - return __pte(pte_val(pte) & ~_PAGE_RW); + return __pte(pte_val(pte) & ~_PAGE_WRITE); } static inline pte_t pte_exprotect(pte_t pte) @@ -499,6 +505,9 @@ static inline pte_t pte_mkpte(pte_t pte) static inline pte_t pte_mkwrite_novma(pte_t pte) { + /* + * write implies read, hence set both + */ return __pte(pte_val(pte) | _PAGE_RW); } diff --git a/arch/powerpc/kernel/head_book3s_32.S b/arch/powerpc/kernel/head_book3s_32.S index 6764b98ca360..615d429d7bd1 100644 --- a/arch/powerpc/kernel/head_book3s_32.S +++ b/arch/powerpc/kernel/head_book3s_32.S @@ -503,9 +503,9 @@ DataLoadTLBMiss: andc. r1,r1,r0 /* check access & ~permission */ bne- DataAddressInvalid /* return if access not permitted */ /* Convert linux-style PTE to low word of PPC-style PTE */ - rlwinm r1,r0,32-9,30,30 /* _PAGE_RW -> PP msb */ + rlwinm r1,r0,32-9,30,30 /* _PAGE_WRITE -> PP msb */ rlwimi r0,r0,32-1,30,30 /* _PAGE_USER -> PP msb */ - rlwimi r1,r0,32-3,24,24 /* _PAGE_RW -> _PAGE_DIRTY */ + rlwimi r1,r0,32-3,24,24 /* _PAGE_WRITE -> _PAGE_DIRTY */ rlwimi r0,r0,32-1,31,31 /* _PAGE_USER -> PP lsb */ xori r1,r1,_PAGE_DIRTY /* clear dirty when not rw */ ori r1,r1,0xe04 /* clear out reserved bits */ @@ -689,7 +689,7 @@ hash_page_dsi: mfdar r4 mfsrr0 r5 mfsrr1 r9 - rlwinm r3, r3, 32 - 15, _PAGE_RW /* DSISR_STORE -> _PAGE_RW */ + rlwinm r3, r3, 32 - 15, _PAGE_WRITE /* DSISR_STORE -> _PAGE_WRITE */ bl hash_page mfspr r10, SPRN_SPRG_THREAD restore_regs_thread r10 diff --git a/arch/powerpc/mm/book3s32/hash_low.S b/arch/powerpc/mm/book3s32/hash_low.S index 8b804e1a9fa4..acb0584c174c 100644 --- a/arch/powerpc/mm/book3s32/hash_low.S +++ b/arch/powerpc/mm/book3s32/hash_low.S @@ -37,7 +37,7 @@ /* * Load a PTE into the hash table, if possible. * The address is in r4, and r3 contains an access flag: - * _PAGE_RW (0x400) if a write. + * _PAGE_WRITE (0x400) if a write. * r9 contains the SRR1 value, from which we use the MSR_PR bit. * SPRG_THREAD contains the physical address of the current task's thread. * @@ -113,15 +113,15 @@ _GLOBAL(hash_page) lwarx r6,0,r8 /* get linux-style pte, flag word */ #ifdef CONFIG_PPC_KUAP mfsrin r5,r4 - rlwinm r0,r9,28,_PAGE_RW /* MSR[PR] => _PAGE_RW */ - rlwinm r5,r5,12,_PAGE_RW /* Ks => _PAGE_RW */ + rlwinm r0,r9,28,_PAGE_WRITE /* MSR[PR] => _PAGE_WRITE */ + rlwinm r5,r5,12,_PAGE_WRITE /* Ks => _PAGE_WRITE */ andc r5,r5,r0 /* Ks & ~MSR[PR] */ - andc r5,r6,r5 /* Clear _PAGE_RW when Ks = 1 && MSR[PR] = 0 */ + andc r5,r6,r5 /* Clear _PAGE_WRITE when Ks = 1 && MSR[PR] = 0 */ andc. r5,r3,r5 /* check access & ~permission */ #else andc. r5,r3,r6 /* check access & ~permission */ #endif - rlwinm r0,r3,32-3,24,24 /* _PAGE_RW access -> _PAGE_DIRTY */ + rlwinm r0,r3,32-3,24,24 /* _PAGE_WRITE access -> _PAGE_DIRTY */ ori r0,r0,_PAGE_ACCESSED|_PAGE_HASHPTE #ifdef CONFIG_SMP bne- .Lhash_page_out /* return if access not permitted */ @@ -307,7 +307,7 @@ Hash_msk = (((1 << Hash_bits) - 1) * 64) __REF _GLOBAL(create_hpte) /* Convert linux-style PTE (r5) to low word of PPC-style PTE (r8) */ - rlwinm r8,r5,32-9,30,30 /* _PAGE_RW -> PP msb */ + rlwinm r8,r5,32-9,30,30 /* _PAGE_WRITE -> PP msb */ rlwinm r0,r5,32-6,30,30 /* _PAGE_DIRTY -> PP msb */ and r8,r8,r0 /* writable if _RW & _DIRTY */ rlwimi r5,r5,32-1,30,30 /* _PAGE_USER -> PP msb */ diff --git a/arch/powerpc/mm/book3s32/mmu.c b/arch/powerpc/mm/book3s32/mmu.c index d1041c946ce2..5445587bfe84 100644 --- a/arch/powerpc/mm/book3s32/mmu.c +++ b/arch/powerpc/mm/book3s32/mmu.c @@ -277,7 +277,7 @@ void __init setbat(int index, unsigned long virt, phys_addr_t phys, /* Do DBAT first */ wimgxpp = flags & (_PAGE_WRITETHRU | _PAGE_NO_CACHE | _PAGE_COHERENT | _PAGE_GUARDED); - wimgxpp |= (flags & _PAGE_RW)? BPP_RW: BPP_RX; + wimgxpp |= (flags & _PAGE_WRITE) ? BPP_RW : BPP_RX; bat[1].batu = virt | (bl << 2) | 2; /* Vs=1, Vp=0 */ bat[1].batl = BAT_PHYS_ADDR(phys) | wimgxpp; if (!is_kernel_addr(virt)) From bac4cffc7c4a009cf0d16f1785a275e0a7715e8d Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 25 Sep 2023 20:31:48 +0200 Subject: [PATCH 306/475] powerpc/32s: Introduce _PAGE_READ and remove _PAGE_USER On 603 MMU, TLB missed are handled by SW and there are separated DTLB and ITLB. It is therefore possible to implement execute-only protection by not loading DTLB when read access is not permitted. To do that, _PAGE_READ flag is needed but there is no bit available for it in PTE. On the other hand the only real use of _PAGE_USER is to implement PAGE_NONE by clearing _PAGE_USER. As _PAGE_NONE can also be implemented by clearing _PAGE_READ, remove _PAGE_USER and add _PAGE_READ. Then use the virtual address to know whether user rights or kernel rights are to be used. With that change, 603 MMU now honors execute-only protection. For hash (604) MMU it is more tricky because hash table is common to load/store and execute. Nevertheless it is still possible to check whether _PAGE_READ is set before loading hash table for a load/store access. At least it can't be read unless it is executed first. Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://msgid.link/b7702dd5a041ec59055ed2880f4952e94c087a2e.1695659959.git.christophe.leroy@csgroup.eu --- arch/powerpc/include/asm/book3s/32/pgtable.h | 48 ++++----------- arch/powerpc/kernel/head_book3s_32.S | 61 +++++++++++--------- arch/powerpc/mm/book3s32/hash_low.S | 22 ++++--- 3 files changed, 60 insertions(+), 71 deletions(-) diff --git a/arch/powerpc/include/asm/book3s/32/pgtable.h b/arch/powerpc/include/asm/book3s/32/pgtable.h index 480ad6b4fd6f..244621c88510 100644 --- a/arch/powerpc/include/asm/book3s/32/pgtable.h +++ b/arch/powerpc/include/asm/book3s/32/pgtable.h @@ -20,7 +20,7 @@ #define _PAGE_PRESENT 0x001 /* software: pte contains a translation */ #define _PAGE_HASHPTE 0x002 /* hash_page has made an HPTE for this pte */ -#define _PAGE_USER 0x004 /* usermode access allowed */ +#define _PAGE_READ 0x004 /* software: read access allowed */ #define _PAGE_GUARDED 0x008 /* G: prohibit speculative access */ #define _PAGE_COHERENT 0x010 /* M: enforce memory coherence (SMP systems) */ #define _PAGE_NO_CACHE 0x020 /* I: cache inhibit */ @@ -28,11 +28,9 @@ #define _PAGE_DIRTY 0x080 /* C: page changed */ #define _PAGE_ACCESSED 0x100 /* R: page referenced */ #define _PAGE_EXEC 0x200 /* software: exec allowed */ -#define _PAGE_RW 0x400 /* software: user write access allowed */ +#define _PAGE_WRITE 0x400 /* software: user write access allowed */ #define _PAGE_SPECIAL 0x800 /* software: Special page */ -#define _PAGE_WRITE _PAGE_RW - #ifdef CONFIG_PTE_64BIT /* We never clear the high word of the pte */ #define _PTE_NONE_MASK (0xffffffff00000000ULL | _PAGE_HASHPTE) @@ -44,26 +42,13 @@ #define _PMD_PRESENT_MASK (PAGE_MASK) #define _PMD_BAD (~PAGE_MASK) -/* We borrow the _PAGE_USER bit to store the exclusive marker in swap PTEs. */ -#define _PAGE_SWP_EXCLUSIVE _PAGE_USER +/* We borrow the _PAGE_READ bit to store the exclusive marker in swap PTEs. */ +#define _PAGE_SWP_EXCLUSIVE _PAGE_READ /* And here we include common definitions */ -#define _PAGE_KERNEL_RO 0 -#define _PAGE_KERNEL_ROX (_PAGE_EXEC) -#define _PAGE_KERNEL_RW (_PAGE_DIRTY | _PAGE_RW) -#define _PAGE_KERNEL_RWX (_PAGE_DIRTY | _PAGE_RW | _PAGE_EXEC) - #define _PAGE_HPTEFLAGS _PAGE_HASHPTE -#ifndef __ASSEMBLY__ - -static inline bool pte_user(pte_t pte) -{ - return pte_val(pte) & _PAGE_USER; -} -#endif /* __ASSEMBLY__ */ - /* * Location of the PFN in the PTE. Most 32-bit platforms use the same * as _PAGE_SHIFT here (ie, naturally aligned). @@ -99,20 +84,7 @@ static inline bool pte_user(pte_t pte) #define _PAGE_BASE_NC (_PAGE_PRESENT | _PAGE_ACCESSED) #define _PAGE_BASE (_PAGE_BASE_NC | _PAGE_COHERENT) -/* - * Permission masks used to generate the __P and __S table. - * - * Note:__pgprot is defined in arch/powerpc/include/asm/page.h - * - * Write permissions imply read permissions for now. - */ -#define PAGE_NONE __pgprot(_PAGE_BASE) -#define PAGE_SHARED __pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_RW) -#define PAGE_SHARED_X __pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_RW | _PAGE_EXEC) -#define PAGE_COPY __pgprot(_PAGE_BASE | _PAGE_USER) -#define PAGE_COPY_X __pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_EXEC) -#define PAGE_READONLY __pgprot(_PAGE_BASE | _PAGE_USER) -#define PAGE_READONLY_X __pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_EXEC) +#include /* Permission masks used for kernel mappings */ #define PAGE_KERNEL __pgprot(_PAGE_BASE | _PAGE_KERNEL_RW) @@ -408,12 +380,16 @@ static inline pte_t pte_swp_clear_exclusive(pte_t pte) } /* Generic accessors to PTE bits */ +static inline bool pte_read(pte_t pte) +{ + return !!(pte_val(pte) & _PAGE_READ); +} + static inline bool pte_write(pte_t pte) { return !!(pte_val(pte) & _PAGE_WRITE); } -static inline int pte_read(pte_t pte) { return 1; } static inline int pte_dirty(pte_t pte) { return !!(pte_val(pte) & _PAGE_DIRTY); } static inline int pte_young(pte_t pte) { return !!(pte_val(pte) & _PAGE_ACCESSED); } static inline int pte_special(pte_t pte) { return !!(pte_val(pte) & _PAGE_SPECIAL); } @@ -448,10 +424,10 @@ static inline bool pte_ci(pte_t pte) static inline bool pte_access_permitted(pte_t pte, bool write) { /* - * A read-only access is controlled by _PAGE_USER bit. + * A read-only access is controlled by _PAGE_READ bit. * We have _PAGE_READ set for WRITE and EXECUTE */ - if (!pte_present(pte) || !pte_user(pte) || !pte_read(pte)) + if (!pte_present(pte) || !pte_read(pte)) return false; if (write && !pte_write(pte)) diff --git a/arch/powerpc/kernel/head_book3s_32.S b/arch/powerpc/kernel/head_book3s_32.S index 615d429d7bd1..c1d89764dd22 100644 --- a/arch/powerpc/kernel/head_book3s_32.S +++ b/arch/powerpc/kernel/head_book3s_32.S @@ -412,10 +412,10 @@ END_FTR_SECTION_IFSET(CPU_FTR_FPU_UNAVAILABLE) . = INTERRUPT_INST_TLB_MISS_603 InstructionTLBMiss: /* - * r0: scratch + * r0: userspace flag (later scratch) * r1: linux style pte ( later becomes ppc hardware pte ) * r2: ptr to linux-style pte - * r3: scratch + * r3: fault address */ /* Get PTE (linux-style) and check access */ mfspr r3,SPRN_IMISS @@ -424,12 +424,13 @@ InstructionTLBMiss: cmplw 0,r1,r3 #endif mfspr r2, SPRN_SDR1 - li r1,_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_EXEC | _PAGE_USER + li r1,_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_EXEC rlwinm r2, r2, 28, 0xfffff000 #ifdef CONFIG_MODULES + li r0, 3 bgt- 112f lis r2, (swapper_pg_dir - PAGE_OFFSET)@ha /* if kernel address, use */ - li r1,_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_EXEC + li r0, 0 addi r2, r2, (swapper_pg_dir - PAGE_OFFSET)@l /* kernel page table */ #endif 112: rlwimi r2,r3,12,20,29 /* insert top 10 bits of address */ @@ -437,13 +438,15 @@ InstructionTLBMiss: rlwinm. r2,r2,0,0,19 /* extract address of pte page */ beq- InstructionAddressInvalid /* return if no mapping */ rlwimi r2,r3,22,20,29 /* insert next 10 bits of address */ - lwz r0,0(r2) /* get linux-style pte */ - andc. r1,r1,r0 /* check access & ~permission */ + lwz r2,0(r2) /* get linux-style pte */ + andc. r1,r1,r2 /* check access & ~permission */ bne- InstructionAddressInvalid /* return if access not permitted */ /* Convert linux-style PTE to low word of PPC-style PTE */ - rlwimi r0,r0,32-2,31,31 /* _PAGE_USER -> PP lsb */ +#ifdef CONFIG_MODULES + rlwimi r2, r0, 0, 31, 31 /* userspace ? -> PP lsb */ +#endif ori r1, r1, 0xe06 /* clear out reserved bits */ - andc r1, r0, r1 /* PP = user? 1 : 0 */ + andc r1, r2, r1 /* PP = user? 1 : 0 */ BEGIN_FTR_SECTION rlwinm r1,r1,0,~_PAGE_COHERENT /* clear M (coherence not required) */ END_FTR_SECTION_IFCLR(CPU_FTR_NEED_COHERENT) @@ -478,38 +481,38 @@ InstructionAddressInvalid: . = INTERRUPT_DATA_LOAD_TLB_MISS_603 DataLoadTLBMiss: /* - * r0: scratch + * r0: userspace flag (later scratch) * r1: linux style pte ( later becomes ppc hardware pte ) * r2: ptr to linux-style pte - * r3: scratch + * r3: fault address */ /* Get PTE (linux-style) and check access */ mfspr r3,SPRN_DMISS lis r1, TASK_SIZE@h /* check if kernel address */ cmplw 0,r1,r3 mfspr r2, SPRN_SDR1 - li r1, _PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_USER + li r1, _PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_READ rlwinm r2, r2, 28, 0xfffff000 + li r0, 3 bgt- 112f lis r2, (swapper_pg_dir - PAGE_OFFSET)@ha /* if kernel address, use */ - li r1, _PAGE_PRESENT | _PAGE_ACCESSED + li r0, 0 addi r2, r2, (swapper_pg_dir - PAGE_OFFSET)@l /* kernel page table */ 112: rlwimi r2,r3,12,20,29 /* insert top 10 bits of address */ lwz r2,0(r2) /* get pmd entry */ rlwinm. r2,r2,0,0,19 /* extract address of pte page */ beq- DataAddressInvalid /* return if no mapping */ rlwimi r2,r3,22,20,29 /* insert next 10 bits of address */ - lwz r0,0(r2) /* get linux-style pte */ - andc. r1,r1,r0 /* check access & ~permission */ + lwz r2,0(r2) /* get linux-style pte */ + andc. r1,r1,r2 /* check access & ~permission */ bne- DataAddressInvalid /* return if access not permitted */ /* Convert linux-style PTE to low word of PPC-style PTE */ - rlwinm r1,r0,32-9,30,30 /* _PAGE_WRITE -> PP msb */ - rlwimi r0,r0,32-1,30,30 /* _PAGE_USER -> PP msb */ - rlwimi r1,r0,32-3,24,24 /* _PAGE_WRITE -> _PAGE_DIRTY */ - rlwimi r0,r0,32-1,31,31 /* _PAGE_USER -> PP lsb */ + rlwinm r1,r2,32-9,30,30 /* _PAGE_WRITE -> PP msb */ + rlwimi r2,r0,0,30,31 /* userspace ? -> PP */ + rlwimi r1,r2,32-3,24,24 /* _PAGE_WRITE -> _PAGE_DIRTY */ xori r1,r1,_PAGE_DIRTY /* clear dirty when not rw */ ori r1,r1,0xe04 /* clear out reserved bits */ - andc r1,r0,r1 /* PP = user? rw? 1: 3: 0 */ + andc r1,r2,r1 /* PP = user? rw? 1: 3: 0 */ BEGIN_FTR_SECTION rlwinm r1,r1,0,~_PAGE_COHERENT /* clear M (coherence not required) */ END_FTR_SECTION_IFCLR(CPU_FTR_NEED_COHERENT) @@ -558,34 +561,35 @@ DataAddressInvalid: . = INTERRUPT_DATA_STORE_TLB_MISS_603 DataStoreTLBMiss: /* - * r0: scratch + * r0: userspace flag (later scratch) * r1: linux style pte ( later becomes ppc hardware pte ) * r2: ptr to linux-style pte - * r3: scratch + * r3: fault address */ /* Get PTE (linux-style) and check access */ mfspr r3,SPRN_DMISS lis r1, TASK_SIZE@h /* check if kernel address */ cmplw 0,r1,r3 mfspr r2, SPRN_SDR1 - li r1, _PAGE_RW | _PAGE_DIRTY | _PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_USER + li r1, _PAGE_RW | _PAGE_DIRTY | _PAGE_PRESENT | _PAGE_ACCESSED rlwinm r2, r2, 28, 0xfffff000 + li r0, 3 bgt- 112f lis r2, (swapper_pg_dir - PAGE_OFFSET)@ha /* if kernel address, use */ - li r1, _PAGE_RW | _PAGE_DIRTY | _PAGE_PRESENT | _PAGE_ACCESSED + li r0, 0 addi r2, r2, (swapper_pg_dir - PAGE_OFFSET)@l /* kernel page table */ 112: rlwimi r2,r3,12,20,29 /* insert top 10 bits of address */ lwz r2,0(r2) /* get pmd entry */ rlwinm. r2,r2,0,0,19 /* extract address of pte page */ beq- DataAddressInvalid /* return if no mapping */ rlwimi r2,r3,22,20,29 /* insert next 10 bits of address */ - lwz r0,0(r2) /* get linux-style pte */ - andc. r1,r1,r0 /* check access & ~permission */ + lwz r2,0(r2) /* get linux-style pte */ + andc. r1,r1,r2 /* check access & ~permission */ bne- DataAddressInvalid /* return if access not permitted */ /* Convert linux-style PTE to low word of PPC-style PTE */ - rlwimi r0,r0,32-2,31,31 /* _PAGE_USER -> PP lsb */ + rlwimi r2,r0,0,31,31 /* userspace ? -> PP lsb */ li r1,0xe06 /* clear out reserved bits & PP msb */ - andc r1,r0,r1 /* PP = user? 1: 0 */ + andc r1,r2,r1 /* PP = user? 1: 0 */ BEGIN_FTR_SECTION rlwinm r1,r1,0,~_PAGE_COHERENT /* clear M (coherence not required) */ END_FTR_SECTION_IFCLR(CPU_FTR_NEED_COHERENT) @@ -690,6 +694,7 @@ hash_page_dsi: mfsrr0 r5 mfsrr1 r9 rlwinm r3, r3, 32 - 15, _PAGE_WRITE /* DSISR_STORE -> _PAGE_WRITE */ + ori r3, r3, _PAGE_PRESENT | _PAGE_READ bl hash_page mfspr r10, SPRN_SPRG_THREAD restore_regs_thread r10 @@ -699,7 +704,7 @@ hash_page_isi: mr r11, r10 mfspr r10, SPRN_SPRG_THREAD save_regs_thread r10 - li r3, 0 + li r3, _PAGE_PRESENT | _PAGE_EXEC lwz r4, SRR0(r10) lwz r9, SRR1(r10) bl hash_page diff --git a/arch/powerpc/mm/book3s32/hash_low.S b/arch/powerpc/mm/book3s32/hash_low.S index acb0584c174c..4ed0efd03db5 100644 --- a/arch/powerpc/mm/book3s32/hash_low.S +++ b/arch/powerpc/mm/book3s32/hash_low.S @@ -36,8 +36,9 @@ /* * Load a PTE into the hash table, if possible. - * The address is in r4, and r3 contains an access flag: - * _PAGE_WRITE (0x400) if a write. + * The address is in r4, and r3 contains required access flags: + * - For ISI: _PAGE_PRESENT | _PAGE_EXEC + * - For DSI: _PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE if a write. * r9 contains the SRR1 value, from which we use the MSR_PR bit. * SPRG_THREAD contains the physical address of the current task's thread. * @@ -67,12 +68,16 @@ _GLOBAL(hash_page) lis r0, TASK_SIZE@h /* check if kernel address */ cmplw 0,r4,r0 mfspr r8,SPRN_SPRG_THREAD /* current task's THREAD (phys) */ - ori r3,r3,_PAGE_USER|_PAGE_PRESENT /* test low addresses as user */ lwz r5,PGDIR(r8) /* virt page-table root */ blt+ 112f /* assume user more likely */ lis r5,swapper_pg_dir@ha /* if kernel address, use */ + andi. r0,r9,MSR_PR /* Check usermode */ addi r5,r5,swapper_pg_dir@l /* kernel page table */ - rlwimi r3,r9,32-12,29,29 /* MSR_PR -> _PAGE_USER */ +#ifdef CONFIG_SMP + bne- .Lhash_page_out /* return if usermode */ +#else + bnelr- +#endif 112: tophys(r5, r5) #ifndef CONFIG_PTE_64BIT rlwimi r5,r4,12,20,29 /* insert top 10 bits of address */ @@ -307,12 +312,15 @@ Hash_msk = (((1 << Hash_bits) - 1) * 64) __REF _GLOBAL(create_hpte) /* Convert linux-style PTE (r5) to low word of PPC-style PTE (r8) */ + lis r0, TASK_SIZE@h + rlwinm r5,r5,0,~3 /* Clear PP bits */ + cmplw r4,r0 rlwinm r8,r5,32-9,30,30 /* _PAGE_WRITE -> PP msb */ rlwinm r0,r5,32-6,30,30 /* _PAGE_DIRTY -> PP msb */ and r8,r8,r0 /* writable if _RW & _DIRTY */ - rlwimi r5,r5,32-1,30,30 /* _PAGE_USER -> PP msb */ - rlwimi r5,r5,32-2,31,31 /* _PAGE_USER -> PP lsb */ - ori r8,r8,0xe04 /* clear out reserved bits */ + bge- 1f /* Kernelspace ? Skip */ + ori r5,r5,3 /* Userspace ? PP = 3 */ +1: ori r8,r8,0xe04 /* clear out reserved bits */ andc r8,r5,r8 /* PP = user? (rw&dirty? 1: 3): 0 */ BEGIN_FTR_SECTION rlwinm r8,r8,0,~_PAGE_COHERENT /* clear M (coherence not required) */ From ceaba662c06598e52cbe4b90fef6b71b7f965cf9 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 25 Sep 2023 20:31:49 +0200 Subject: [PATCH 307/475] powerpc/ptdump: Display _PAGE_READ and _PAGE_WRITE Instead of always displaying either 'rw' or 'r ' depending on _PAGE_RW, display 'r' or ' ' for _PAGE_READ and 'w' or ' ' for _PAGE_WRITE. Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://msgid.link/dd8201a0f8fd87ce62a7ff2edc958b604b8ec3c0.1695659959.git.christophe.leroy@csgroup.eu --- arch/powerpc/mm/ptdump/shared.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/arch/powerpc/mm/ptdump/shared.c b/arch/powerpc/mm/ptdump/shared.c index 5ff101654c45..39c30c62b7ea 100644 --- a/arch/powerpc/mm/ptdump/shared.c +++ b/arch/powerpc/mm/ptdump/shared.c @@ -11,10 +11,15 @@ static const struct flag_info flag_array[] = { { - .mask = _PAGE_RW, + .mask = _PAGE_READ, .val = 0, - .set = "r ", - .clear = "rw", + .set = " ", + .clear = "r", + }, { + .mask = _PAGE_WRITE, + .val = 0, + .set = " ", + .clear = "w", }, { .mask = _PAGE_EXEC, .val = _PAGE_EXEC, From 163a72fa89161b57b05d848aedfbd5103fac9dd7 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 25 Sep 2023 20:31:50 +0200 Subject: [PATCH 308/475] powerpc: Finally remove _PAGE_USER _PAGE_USER is now gone on all targets. Remove it completely. Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://msgid.link/76ebe74fdaed4297a1d8203a61174650c1d8d278.1695659959.git.christophe.leroy@csgroup.eu --- arch/powerpc/include/asm/nohash/pgtable.h | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/arch/powerpc/include/asm/nohash/pgtable.h b/arch/powerpc/include/asm/nohash/pgtable.h index 84b6a160d2fc..8804bcc260cb 100644 --- a/arch/powerpc/include/asm/nohash/pgtable.h +++ b/arch/powerpc/include/asm/nohash/pgtable.h @@ -182,18 +182,14 @@ static inline int pte_young(pte_t pte) } /* - * Don't just check for any non zero bits in __PAGE_USER, since for book3e + * Don't just check for any non zero bits in __PAGE_READ, since for book3e * and PTE_64BIT, PAGE_KERNEL_X contains _PAGE_BAP_SR which is also in - * _PAGE_USER. Need to explicitly match _PAGE_BAP_UR bit in that case too. + * _PAGE_READ. Need to explicitly match _PAGE_BAP_UR bit in that case too. */ #ifndef pte_read static inline bool pte_read(pte_t pte) { -#ifdef _PAGE_READ return (pte_val(pte) & _PAGE_READ) == _PAGE_READ; -#else - return (pte_val(pte) & _PAGE_USER) == _PAGE_USER; -#endif } #endif @@ -205,7 +201,7 @@ static inline bool pte_read(pte_t pte) static inline bool pte_access_permitted(pte_t pte, bool write) { /* - * A read-only access is controlled by _PAGE_USER bit. + * A read-only access is controlled by _PAGE_READ bit. * We have _PAGE_READ set for WRITE and EXECUTE */ if (!pte_present(pte) || !pte_read(pte)) From b1fba034a6793e9601d581594a781b46c255471f Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 25 Sep 2023 20:31:51 +0200 Subject: [PATCH 309/475] powerpc: Support execute-only on all powerpc Introduce PAGE_EXECONLY_X macro which provides exec-only rights. The _X may be seen as redundant with the EXECONLY but it helps keep consistency, all macros having the EXEC right have _X. And put it next to PAGE_NONE as PAGE_EXECONLY_X is somehow PAGE_NONE + EXEC just like all other SOMETHING_X are just SOMETHING + EXEC. On book3s/64 PAGE_EXECONLY becomes PAGE_READONLY_X. On book3s/64, as PAGE_EXECONLY is only valid for Radix add VM_READ flag in vm_get_page_prot() for non-Radix. And update access_error() so that a non exec fault on a VM_EXEC only mapping is always invalid, even when the underlying layer don't always generate a fault for that. For 8xx, set PAGE_EXECONLY_X as _PAGE_NA | _PAGE_EXEC. For others, only set it as just _PAGE_EXEC With that change, 8xx, e500 and 44x fully honor execute-only protection. On 40x that is a partial implementation of execute-only. The implementation won't be complete because once a TLB has been loaded via the Instruction TLB miss handler, it will be possible to read the page. But at least it can't be read unless it is executed first. On 603 MMU, TLB missed are handled by SW and there are separate DTLB and ITLB. Execute-only is therefore now supported by not loading DTLB when read access is not permitted. On hash (604) MMU it is more tricky because hash table is common to load/store and execute. Nevertheless it is still possible to check whether _PAGE_READ is set before loading hash table for a load/store access. At least it can't be read unless it is executed first. Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://msgid.link/4283ea9cbef9ff2fbee468904800e1962bc8fc18.1695659959.git.christophe.leroy@csgroup.eu --- arch/powerpc/include/asm/book3s/32/pgtable.h | 2 +- arch/powerpc/include/asm/book3s/64/pgtable.h | 4 +--- arch/powerpc/include/asm/nohash/32/pte-8xx.h | 1 + arch/powerpc/include/asm/nohash/pgtable.h | 2 +- arch/powerpc/include/asm/nohash/pte-e500.h | 1 + arch/powerpc/include/asm/pgtable-masks.h | 2 ++ arch/powerpc/mm/book3s64/pgtable.c | 10 ++++------ arch/powerpc/mm/fault.c | 9 +++++---- arch/powerpc/mm/pgtable.c | 4 ++-- 9 files changed, 18 insertions(+), 17 deletions(-) diff --git a/arch/powerpc/include/asm/book3s/32/pgtable.h b/arch/powerpc/include/asm/book3s/32/pgtable.h index 244621c88510..52971ee30717 100644 --- a/arch/powerpc/include/asm/book3s/32/pgtable.h +++ b/arch/powerpc/include/asm/book3s/32/pgtable.h @@ -425,7 +425,7 @@ static inline bool pte_access_permitted(pte_t pte, bool write) { /* * A read-only access is controlled by _PAGE_READ bit. - * We have _PAGE_READ set for WRITE and EXECUTE + * We have _PAGE_READ set for WRITE */ if (!pte_present(pte) || !pte_read(pte)) return false; diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h index 0fd12bdc7b5e..751b01227e36 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -18,6 +18,7 @@ #define _PAGE_WRITE 0x00002 /* write access allowed */ #define _PAGE_READ 0x00004 /* read access allowed */ #define _PAGE_NA _PAGE_PRIVILEGED +#define _PAGE_NAX _PAGE_EXEC #define _PAGE_RO _PAGE_READ #define _PAGE_ROX (_PAGE_READ | _PAGE_EXEC) #define _PAGE_RW (_PAGE_READ | _PAGE_WRITE) @@ -141,9 +142,6 @@ #include -/* Radix only, Hash uses PAGE_READONLY_X + execute-only pkey instead */ -#define PAGE_EXECONLY __pgprot(_PAGE_BASE | _PAGE_EXEC) - /* Permission masks used for kernel mappings */ #define PAGE_KERNEL __pgprot(_PAGE_BASE | _PAGE_KERNEL_RW) #define PAGE_KERNEL_NC __pgprot(_PAGE_BASE_NC | _PAGE_KERNEL_RW | _PAGE_TOLERANT) diff --git a/arch/powerpc/include/asm/nohash/32/pte-8xx.h b/arch/powerpc/include/asm/nohash/32/pte-8xx.h index 1ee38befd29a..137dc3c84e45 100644 --- a/arch/powerpc/include/asm/nohash/32/pte-8xx.h +++ b/arch/powerpc/include/asm/nohash/32/pte-8xx.h @@ -48,6 +48,7 @@ #define _PAGE_HUGE 0x0800 /* Copied to L1 PS bit 29 */ +#define _PAGE_NAX (_PAGE_NA | _PAGE_EXEC) #define _PAGE_ROX (_PAGE_RO | _PAGE_EXEC) #define _PAGE_RW 0 #define _PAGE_RWX _PAGE_EXEC diff --git a/arch/powerpc/include/asm/nohash/pgtable.h b/arch/powerpc/include/asm/nohash/pgtable.h index 8804bcc260cb..427db14292c9 100644 --- a/arch/powerpc/include/asm/nohash/pgtable.h +++ b/arch/powerpc/include/asm/nohash/pgtable.h @@ -202,7 +202,7 @@ static inline bool pte_access_permitted(pte_t pte, bool write) { /* * A read-only access is controlled by _PAGE_READ bit. - * We have _PAGE_READ set for WRITE and EXECUTE + * We have _PAGE_READ set for WRITE */ if (!pte_present(pte) || !pte_read(pte)) return false; diff --git a/arch/powerpc/include/asm/nohash/pte-e500.h b/arch/powerpc/include/asm/nohash/pte-e500.h index 31d2c3ea7df8..f516f0b5b7a8 100644 --- a/arch/powerpc/include/asm/nohash/pte-e500.h +++ b/arch/powerpc/include/asm/nohash/pte-e500.h @@ -57,6 +57,7 @@ #define _PAGE_KERNEL_ROX (_PAGE_BAP_SR | _PAGE_BAP_SX) #define _PAGE_NA 0 +#define _PAGE_NAX _PAGE_BAP_UX #define _PAGE_RO _PAGE_READ #define _PAGE_ROX (_PAGE_READ | _PAGE_BAP_UX) #define _PAGE_RW (_PAGE_READ | _PAGE_WRITE) diff --git a/arch/powerpc/include/asm/pgtable-masks.h b/arch/powerpc/include/asm/pgtable-masks.h index 808a3b9e8fc0..6e8e2db26a5a 100644 --- a/arch/powerpc/include/asm/pgtable-masks.h +++ b/arch/powerpc/include/asm/pgtable-masks.h @@ -4,6 +4,7 @@ #ifndef _PAGE_NA #define _PAGE_NA 0 +#define _PAGE_NAX _PAGE_EXEC #define _PAGE_RO _PAGE_READ #define _PAGE_ROX (_PAGE_READ | _PAGE_EXEC) #define _PAGE_RW (_PAGE_READ | _PAGE_WRITE) @@ -20,6 +21,7 @@ /* Permission masks used to generate the __P and __S table */ #define PAGE_NONE __pgprot(_PAGE_BASE | _PAGE_NA) +#define PAGE_EXECONLY_X __pgprot(_PAGE_BASE | _PAGE_NAX) #define PAGE_SHARED __pgprot(_PAGE_BASE | _PAGE_RW) #define PAGE_SHARED_X __pgprot(_PAGE_BASE | _PAGE_RWX) #define PAGE_COPY __pgprot(_PAGE_BASE | _PAGE_RO) diff --git a/arch/powerpc/mm/book3s64/pgtable.c b/arch/powerpc/mm/book3s64/pgtable.c index 8f8a62d3ff4d..be229290a6a7 100644 --- a/arch/powerpc/mm/book3s64/pgtable.c +++ b/arch/powerpc/mm/book3s64/pgtable.c @@ -635,12 +635,10 @@ pgprot_t vm_get_page_prot(unsigned long vm_flags) unsigned long prot; /* Radix supports execute-only, but protection_map maps X -> RX */ - if (radix_enabled() && ((vm_flags & VM_ACCESS_FLAGS) == VM_EXEC)) { - prot = pgprot_val(PAGE_EXECONLY); - } else { - prot = pgprot_val(protection_map[vm_flags & - (VM_ACCESS_FLAGS | VM_SHARED)]); - } + if (!radix_enabled() && ((vm_flags & VM_ACCESS_FLAGS) == VM_EXEC)) + vm_flags |= VM_READ; + + prot = pgprot_val(protection_map[vm_flags & (VM_ACCESS_FLAGS | VM_SHARED)]); if (vm_flags & VM_SAO) prot |= _PAGE_SAO; diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c index b1723094d464..9e49ede2bc1c 100644 --- a/arch/powerpc/mm/fault.c +++ b/arch/powerpc/mm/fault.c @@ -266,14 +266,15 @@ static bool access_error(bool is_write, bool is_exec, struct vm_area_struct *vma } /* - * VM_READ, VM_WRITE and VM_EXEC all imply read permissions, as - * defined in protection_map[]. Read faults can only be caused by - * a PROT_NONE mapping, or with a PROT_EXEC-only mapping on Radix. + * VM_READ, VM_WRITE and VM_EXEC may imply read permissions, as + * defined in protection_map[]. In that case Read faults can only be + * caused by a PROT_NONE mapping. However a non exec access on a + * VM_EXEC only mapping is invalid anyway, so report it as such. */ if (unlikely(!vma_is_accessible(vma))) return true; - if (unlikely(radix_enabled() && ((vma->vm_flags & VM_ACCESS_FLAGS) == VM_EXEC))) + if ((vma->vm_flags & VM_ACCESS_FLAGS) == VM_EXEC) return true; /* diff --git a/arch/powerpc/mm/pgtable.c b/arch/powerpc/mm/pgtable.c index 781a68c69c2f..79508c1d15d7 100644 --- a/arch/powerpc/mm/pgtable.c +++ b/arch/powerpc/mm/pgtable.c @@ -492,7 +492,7 @@ const pgprot_t protection_map[16] = { [VM_READ] = PAGE_READONLY, [VM_WRITE] = PAGE_COPY, [VM_WRITE | VM_READ] = PAGE_COPY, - [VM_EXEC] = PAGE_READONLY_X, + [VM_EXEC] = PAGE_EXECONLY_X, [VM_EXEC | VM_READ] = PAGE_READONLY_X, [VM_EXEC | VM_WRITE] = PAGE_COPY_X, [VM_EXEC | VM_WRITE | VM_READ] = PAGE_COPY_X, @@ -500,7 +500,7 @@ const pgprot_t protection_map[16] = { [VM_SHARED | VM_READ] = PAGE_READONLY, [VM_SHARED | VM_WRITE] = PAGE_SHARED, [VM_SHARED | VM_WRITE | VM_READ] = PAGE_SHARED, - [VM_SHARED | VM_EXEC] = PAGE_READONLY_X, + [VM_SHARED | VM_EXEC] = PAGE_EXECONLY_X, [VM_SHARED | VM_EXEC | VM_READ] = PAGE_READONLY_X, [VM_SHARED | VM_EXEC | VM_WRITE] = PAGE_SHARED_X, [VM_SHARED | VM_EXEC | VM_WRITE | VM_READ] = PAGE_SHARED_X From ff7a60ab1e065257a0e467c13b519f4debcd7fcf Mon Sep 17 00:00:00 2001 From: Benjamin Gray Date: Wed, 11 Oct 2023 16:37:00 +1100 Subject: [PATCH 310/475] powerpc/xive: Fix endian conversion size Sparse reports a size mismatch in the endian swap. The Opal implementation[1] passes the value as a __be64, and the receiving variable out_qsize is a u64, so the use of be32_to_cpu() appears to be an error. [1]: https://github.com/open-power/skiboot/blob/80e2b1dc73/hw/xive.c#L3854 Fixes: 88ec6b93c8e7 ("powerpc/xive: add OPAL extensions for the XIVE native exploitation support") Signed-off-by: Benjamin Gray Signed-off-by: Michael Ellerman Link: https://msgid.link/20231011053711.93427-2-bgray@linux.ibm.com --- arch/powerpc/sysdev/xive/native.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/powerpc/sysdev/xive/native.c b/arch/powerpc/sysdev/xive/native.c index 9f0af4d795d8..f1c0fa6ece21 100644 --- a/arch/powerpc/sysdev/xive/native.c +++ b/arch/powerpc/sysdev/xive/native.c @@ -802,7 +802,7 @@ int xive_native_get_queue_info(u32 vp_id, u32 prio, if (out_qpage) *out_qpage = be64_to_cpu(qpage); if (out_qsize) - *out_qsize = be32_to_cpu(qsize); + *out_qsize = be64_to_cpu(qsize); if (out_qeoi_page) *out_qeoi_page = be64_to_cpu(qeoi_page); if (out_escalate_irq) From 340a60e3725b9a229eaf03a9b3f8538f22f6ac16 Mon Sep 17 00:00:00 2001 From: Benjamin Gray Date: Wed, 11 Oct 2023 16:37:02 +1100 Subject: [PATCH 311/475] powerpc: Explicitly reverse bytes when checking for byte reversal Sparse reports an invalid endian cast here. The code is written for big endian platforms, so le32_to_cpu() acts as a byte reversal. This file is checked by sparse on a little endian build though, so replace the reverse function with the dedicated swab32() function to better express the intent of the code. Signed-off-by: Benjamin Gray Signed-off-by: Michael Ellerman Link: https://msgid.link/20231011053711.93427-4-bgray@linux.ibm.com --- arch/powerpc/sysdev/mpic.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/powerpc/sysdev/mpic.c b/arch/powerpc/sysdev/mpic.c index ba287abcb008..dabbdd356664 100644 --- a/arch/powerpc/sysdev/mpic.c +++ b/arch/powerpc/sysdev/mpic.c @@ -355,7 +355,7 @@ static void __init mpic_test_broken_ipi(struct mpic *mpic) mpic_write(mpic->gregs, MPIC_INFO(GREG_IPI_VECTOR_PRI_0), MPIC_VECPRI_MASK); r = mpic_read(mpic->gregs, MPIC_INFO(GREG_IPI_VECTOR_PRI_0)); - if (r == le32_to_cpu(MPIC_VECPRI_MASK)) { + if (r == swab32(MPIC_VECPRI_MASK)) { printk(KERN_INFO "mpic: Detected reversed IPI registers\n"); mpic->flags |= MPIC_BROKEN_IPI; } From ddfb7d9db843fd4fbdff81fb8a8743f865c3dd96 Mon Sep 17 00:00:00 2001 From: Benjamin Gray Date: Wed, 11 Oct 2023 16:37:03 +1100 Subject: [PATCH 312/475] powerpc: Use NULL instead of 0 for null pointers Sparse reports several uses of 0 for pointer arguments and comparisons. Replace with NULL to better convey the intent. Remove entirely if a comparison to follow the kernel style of implicit boolean conversions. Signed-off-by: Benjamin Gray Signed-off-by: Michael Ellerman Link: https://msgid.link/20231011053711.93427-5-bgray@linux.ibm.com --- arch/powerpc/kernel/setup_64.c | 2 +- arch/powerpc/kvm/book3s_xive_native.c | 2 +- arch/powerpc/net/bpf_jit_comp.c | 4 ++-- arch/powerpc/perf/imc-pmu.c | 2 +- arch/powerpc/platforms/4xx/soc.c | 2 +- arch/powerpc/platforms/pseries/lpar.c | 8 ++++---- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c index 246201d0d879..2f19d5e94485 100644 --- a/arch/powerpc/kernel/setup_64.c +++ b/arch/powerpc/kernel/setup_64.c @@ -364,7 +364,7 @@ void __init early_setup(unsigned long dt_ptr) */ initialise_paca(&boot_paca, 0); fixup_boot_paca(&boot_paca); - WARN_ON(local_paca != 0); + WARN_ON(local_paca); setup_paca(&boot_paca); /* install the paca into registers */ /* -------- printk is now safe to use ------- */ diff --git a/arch/powerpc/kvm/book3s_xive_native.c b/arch/powerpc/kvm/book3s_xive_native.c index 712ab91ced39..6e2ebbd8aaac 100644 --- a/arch/powerpc/kvm/book3s_xive_native.c +++ b/arch/powerpc/kvm/book3s_xive_native.c @@ -567,7 +567,7 @@ static int kvmppc_xive_native_set_queue_config(struct kvmppc_xive *xive, u8 priority; struct kvm_ppc_xive_eq kvm_eq; int rc; - __be32 *qaddr = 0; + __be32 *qaddr = NULL; struct page *page; struct xive_q *q; gfn_t gfn; diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index 37043dfc1add..471e37cf8e2a 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -146,9 +146,9 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) * update ctgtx.idx as it pretends to output instructions, then we can * calculate total size from idx. */ - bpf_jit_build_prologue(0, &cgctx); + bpf_jit_build_prologue(NULL, &cgctx); addrs[fp->len] = cgctx.idx * 4; - bpf_jit_build_epilogue(0, &cgctx); + bpf_jit_build_epilogue(NULL, &cgctx); fixup_len = fp->aux->num_exentries * BPF_FIXUP_LEN * 4; extable_len = fp->aux->num_exentries * sizeof(struct exception_table_entry); diff --git a/arch/powerpc/perf/imc-pmu.c b/arch/powerpc/perf/imc-pmu.c index 9d229ef7f86e..eba2baabccb0 100644 --- a/arch/powerpc/perf/imc-pmu.c +++ b/arch/powerpc/perf/imc-pmu.c @@ -544,7 +544,7 @@ static int nest_imc_event_init(struct perf_event *event) break; } pcni++; - } while (pcni->vbase != 0); + } while (pcni->vbase); if (!flag) return -ENODEV; diff --git a/arch/powerpc/platforms/4xx/soc.c b/arch/powerpc/platforms/4xx/soc.c index b2d940437a66..5412e6b21e10 100644 --- a/arch/powerpc/platforms/4xx/soc.c +++ b/arch/powerpc/platforms/4xx/soc.c @@ -112,7 +112,7 @@ static int __init ppc4xx_l2c_probe(void) } /* Install error handler */ - if (request_irq(irq, l2c_error_handler, 0, "L2C", 0) < 0) { + if (request_irq(irq, l2c_error_handler, 0, "L2C", NULL) < 0) { printk(KERN_ERR "Cannot install L2C error handler" ", cache is not enabled\n"); of_node_put(np); diff --git a/arch/powerpc/platforms/pseries/lpar.c b/arch/powerpc/platforms/pseries/lpar.c index f2cb62148f36..b4e86b7ea584 100644 --- a/arch/powerpc/platforms/pseries/lpar.c +++ b/arch/powerpc/platforms/pseries/lpar.c @@ -192,9 +192,9 @@ static void free_dtl_buffers(unsigned long *time_limit) continue; kmem_cache_free(dtl_cache, pp->dispatch_log); pp->dtl_ridx = 0; - pp->dispatch_log = 0; - pp->dispatch_log_end = 0; - pp->dtl_curr = 0; + pp->dispatch_log = NULL; + pp->dispatch_log_end = NULL; + pp->dtl_curr = NULL; if (time_limit && time_after(jiffies, *time_limit)) { cond_resched(); @@ -223,7 +223,7 @@ static void destroy_cpu_associativity(void) { kfree(vcpu_associativity); kfree(pcpu_associativity); - vcpu_associativity = pcpu_associativity = 0; + vcpu_associativity = pcpu_associativity = NULL; } static __be32 *__get_cpu_associativity(int cpu, __be32 *cpu_assoc, int flag) From 419d5d112c2e1e78beda9c3299f71c35141d8dba Mon Sep 17 00:00:00 2001 From: Benjamin Gray Date: Wed, 11 Oct 2023 16:37:04 +1100 Subject: [PATCH 313/475] powerpc: Remove extern from function implementations Sparse reports several function implementations annotated with extern. This is clearly incorrect, likely just copied from an actual extern declaration in another file. Fix the sparse warnings by removing extern. Signed-off-by: Benjamin Gray Signed-off-by: Michael Ellerman Link: https://msgid.link/20231011053711.93427-6-bgray@linux.ibm.com --- arch/powerpc/kernel/iommu.c | 8 ++++---- arch/powerpc/kernel/traps.c | 4 ++-- arch/powerpc/kvm/book3s_64_vio.c | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/arch/powerpc/kernel/iommu.c b/arch/powerpc/kernel/iommu.c index 14251bc5219e..3e28579f7c62 100644 --- a/arch/powerpc/kernel/iommu.c +++ b/arch/powerpc/kernel/iommu.c @@ -1074,10 +1074,10 @@ int iommu_tce_check_gpa(unsigned long page_shift, unsigned long gpa) } EXPORT_SYMBOL_GPL(iommu_tce_check_gpa); -extern long iommu_tce_xchg_no_kill(struct mm_struct *mm, - struct iommu_table *tbl, - unsigned long entry, unsigned long *hpa, - enum dma_data_direction *direction) +long iommu_tce_xchg_no_kill(struct mm_struct *mm, + struct iommu_table *tbl, + unsigned long entry, unsigned long *hpa, + enum dma_data_direction *direction) { long ret; unsigned long size = 0; diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c index fe3f720c9cd6..5ea2014aff90 100644 --- a/arch/powerpc/kernel/traps.c +++ b/arch/powerpc/kernel/traps.c @@ -157,7 +157,7 @@ static int die_owner = -1; static unsigned int die_nest_count; static int die_counter; -extern void panic_flush_kmsg_start(void) +void panic_flush_kmsg_start(void) { /* * These are mostly taken from kernel/panic.c, but tries to do @@ -170,7 +170,7 @@ extern void panic_flush_kmsg_start(void) bust_spinlocks(1); } -extern void panic_flush_kmsg_end(void) +void panic_flush_kmsg_end(void) { kmsg_dump(KMSG_DUMP_PANIC); bust_spinlocks(0); diff --git a/arch/powerpc/kvm/book3s_64_vio.c b/arch/powerpc/kvm/book3s_64_vio.c index 93b695b289e9..15200d766fc5 100644 --- a/arch/powerpc/kvm/book3s_64_vio.c +++ b/arch/powerpc/kvm/book3s_64_vio.c @@ -77,8 +77,8 @@ static void kvm_spapr_tce_liobn_put(struct kref *kref) call_rcu(&stit->rcu, kvm_spapr_tce_iommu_table_free); } -extern void kvm_spapr_tce_release_iommu_group(struct kvm *kvm, - struct iommu_group *grp) +void kvm_spapr_tce_release_iommu_group(struct kvm *kvm, + struct iommu_group *grp) { int i; struct kvmppc_spapr_tce_table *stt; @@ -105,8 +105,8 @@ extern void kvm_spapr_tce_release_iommu_group(struct kvm *kvm, rcu_read_unlock(); } -extern long kvm_spapr_tce_attach_iommu_group(struct kvm *kvm, int tablefd, - struct iommu_group *grp) +long kvm_spapr_tce_attach_iommu_group(struct kvm *kvm, int tablefd, + struct iommu_group *grp) { struct kvmppc_spapr_tce_table *stt = NULL; bool found = false; From 2b4a6cc9a1a7cf6958c8b11f94e61c8e81b60b88 Mon Sep 17 00:00:00 2001 From: Benjamin Gray Date: Wed, 11 Oct 2023 16:37:05 +1100 Subject: [PATCH 314/475] powerpc: Annotate endianness of various variables and functions Sparse reports several endianness warnings on variables and functions that are consistently treated as big endian. There are no multi-endianness shenanigans going on here so fix these low hanging fruit up in one patch. All changes are just type annotations; no endianness switching operations are introduced by this patch. Signed-off-by: Benjamin Gray Signed-off-by: Michael Ellerman Link: https://msgid.link/20231011053711.93427-7-bgray@linux.ibm.com --- arch/powerpc/include/asm/book3s/64/pgtable.h | 2 +- arch/powerpc/include/asm/imc-pmu.h | 16 ++++++++-------- arch/powerpc/kernel/prom_init.c | 2 +- arch/powerpc/kexec/core_64.c | 4 ++-- arch/powerpc/kexec/file_load_64.c | 6 +++--- arch/powerpc/mm/drmem.c | 2 +- arch/powerpc/perf/hv-24x7.c | 2 +- arch/powerpc/perf/imc-pmu.c | 9 +++++---- arch/powerpc/platforms/powermac/feature.c | 3 ++- arch/powerpc/platforms/pseries/hotplug-memory.c | 3 ++- 10 files changed, 26 insertions(+), 23 deletions(-) diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h index 751b01227e36..cb77eddca54b 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -626,7 +626,7 @@ static inline pte_t pte_mkdevmap(pte_t pte) */ static inline int pte_devmap(pte_t pte) { - u64 mask = cpu_to_be64(_PAGE_DEVMAP | _PAGE_PTE); + __be64 mask = cpu_to_be64(_PAGE_DEVMAP | _PAGE_PTE); return (pte_raw(pte) & mask) == mask; } diff --git a/arch/powerpc/include/asm/imc-pmu.h b/arch/powerpc/include/asm/imc-pmu.h index 699a88584ae1..a656635df386 100644 --- a/arch/powerpc/include/asm/imc-pmu.h +++ b/arch/powerpc/include/asm/imc-pmu.h @@ -74,14 +74,14 @@ struct imc_events { * The following is the data structure to hold trace imc data. */ struct trace_imc_data { - u64 tb1; - u64 ip; - u64 val; - u64 cpmc1; - u64 cpmc2; - u64 cpmc3; - u64 cpmc4; - u64 tb2; + __be64 tb1; + __be64 ip; + __be64 val; + __be64 cpmc1; + __be64 cpmc2; + __be64 cpmc3; + __be64 cpmc4; + __be64 tb2; }; /* Event attribute array index */ diff --git a/arch/powerpc/kernel/prom_init.c b/arch/powerpc/kernel/prom_init.c index d464ba412084..e67effdba85c 100644 --- a/arch/powerpc/kernel/prom_init.c +++ b/arch/powerpc/kernel/prom_init.c @@ -947,7 +947,7 @@ struct option_vector7 { } __packed; struct ibm_arch_vec { - struct { u32 mask, val; } pvrs[14]; + struct { __be32 mask, val; } pvrs[14]; u8 num_vectors; diff --git a/arch/powerpc/kexec/core_64.c b/arch/powerpc/kexec/core_64.c index a79e28c91e2b..0bee7ca9a77c 100644 --- a/arch/powerpc/kexec/core_64.c +++ b/arch/powerpc/kexec/core_64.c @@ -379,8 +379,8 @@ void default_machine_kexec(struct kimage *image) #ifdef CONFIG_PPC_64S_HASH_MMU /* Values we need to export to the second kernel via the device tree. */ -static unsigned long htab_base; -static unsigned long htab_size; +static __be64 htab_base; +static __be64 htab_size; static struct property htab_base_prop = { .name = "linux,htab-base", diff --git a/arch/powerpc/kexec/file_load_64.c b/arch/powerpc/kexec/file_load_64.c index 09187aca3d1f..961a6dd67365 100644 --- a/arch/powerpc/kexec/file_load_64.c +++ b/arch/powerpc/kexec/file_load_64.c @@ -32,7 +32,7 @@ #include struct umem_info { - u64 *buf; /* data buffer for usable-memory property */ + __be64 *buf; /* data buffer for usable-memory property */ u32 size; /* size allocated for the data buffer */ u32 max_entries; /* maximum no. of entries */ u32 idx; /* index of current entry */ @@ -443,10 +443,10 @@ static int locate_mem_hole_bottom_up_ppc64(struct kexec_buf *kbuf, * * Returns buffer on success, NULL on error. */ -static u64 *check_realloc_usable_mem(struct umem_info *um_info, int cnt) +static __be64 *check_realloc_usable_mem(struct umem_info *um_info, int cnt) { u32 new_size; - u64 *tbuf; + __be64 *tbuf; if ((um_info->idx + cnt) <= um_info->max_entries) return um_info->buf; diff --git a/arch/powerpc/mm/drmem.c b/arch/powerpc/mm/drmem.c index 2369d1bf2411..fde7790277f7 100644 --- a/arch/powerpc/mm/drmem.c +++ b/arch/powerpc/mm/drmem.c @@ -67,7 +67,7 @@ static int drmem_update_dt_v1(struct device_node *memory, struct property *new_prop; struct of_drconf_cell_v1 *dr_cell; struct drmem_lmb *lmb; - u32 *p; + __be32 *p; new_prop = clone_property(prop, prop->length); if (!new_prop) diff --git a/arch/powerpc/perf/hv-24x7.c b/arch/powerpc/perf/hv-24x7.c index 3449be7c0d51..057ec2e3451d 100644 --- a/arch/powerpc/perf/hv-24x7.c +++ b/arch/powerpc/perf/hv-24x7.c @@ -1338,7 +1338,7 @@ static int get_count_from_result(struct perf_event *event, for (i = count = 0, element_data = res->elements + data_offset; i < num_elements; i++, element_data += data_size + data_offset) - count += be64_to_cpu(*((u64 *) element_data)); + count += be64_to_cpu(*((__be64 *)element_data)); *countp = count; diff --git a/arch/powerpc/perf/imc-pmu.c b/arch/powerpc/perf/imc-pmu.c index eba2baabccb0..2016aee27037 100644 --- a/arch/powerpc/perf/imc-pmu.c +++ b/arch/powerpc/perf/imc-pmu.c @@ -1025,16 +1025,16 @@ static bool is_thread_imc_pmu(struct perf_event *event) return false; } -static u64 * get_event_base_addr(struct perf_event *event) +static __be64 *get_event_base_addr(struct perf_event *event) { u64 addr; if (is_thread_imc_pmu(event)) { addr = (u64)per_cpu(thread_imc_mem, smp_processor_id()); - return (u64 *)(addr + (event->attr.config & IMC_EVENT_OFFSET_MASK)); + return (__be64 *)(addr + (event->attr.config & IMC_EVENT_OFFSET_MASK)); } - return (u64 *)event->hw.event_base; + return (__be64 *)event->hw.event_base; } static void thread_imc_pmu_start_txn(struct pmu *pmu, @@ -1058,7 +1058,8 @@ static int thread_imc_pmu_commit_txn(struct pmu *pmu) static u64 imc_read_counter(struct perf_event *event) { - u64 *addr, data; + __be64 *addr; + u64 data; /* * In-Memory Collection (IMC) counters are free flowing counters. diff --git a/arch/powerpc/platforms/powermac/feature.c b/arch/powerpc/platforms/powermac/feature.c index ae62d432db8b..81c9fbae88b1 100644 --- a/arch/powerpc/platforms/powermac/feature.c +++ b/arch/powerpc/platforms/powermac/feature.c @@ -2614,7 +2614,8 @@ static void __init probe_one_macio(const char *name, const char *compat, int typ struct device_node* node; int i; volatile u32 __iomem *base; - const u32 *addrp, *revp; + const __be32 *addrp; + const u32 *revp; phys_addr_t addr; u64 size; diff --git a/arch/powerpc/platforms/pseries/hotplug-memory.c b/arch/powerpc/platforms/pseries/hotplug-memory.c index aa4042dcd6d4..a43bfb01720a 100644 --- a/arch/powerpc/platforms/pseries/hotplug-memory.c +++ b/arch/powerpc/platforms/pseries/hotplug-memory.c @@ -55,7 +55,8 @@ static bool find_aa_index(struct device_node *dr_node, struct property *ala_prop, const u32 *lmb_assoc, u32 *aa_index) { - u32 *assoc_arrays, new_prop_size; + __be32 *assoc_arrays; + u32 new_prop_size; struct property *new_prop; int aa_arrays, aa_array_entries, aa_array_sz; int i, index; From b7bce570430e42229fb63f775fcbb10f38b83c71 Mon Sep 17 00:00:00 2001 From: Benjamin Gray Date: Wed, 11 Oct 2023 16:37:06 +1100 Subject: [PATCH 315/475] powerpc/kvm: Force cast endianness of KVM shared regs Sparse reports endianness mismatches in the KVM shared regs getter and setter helpers. This code has dynamic endianness behind a safe interface, so a force is warranted here to tell sparse this is OK. Signed-off-by: Benjamin Gray Signed-off-by: Michael Ellerman Link: https://msgid.link/20231011053711.93427-8-bgray@linux.ibm.com --- arch/powerpc/include/asm/kvm_ppc.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/powerpc/include/asm/kvm_ppc.h b/arch/powerpc/include/asm/kvm_ppc.h index db8bfaaa4536..3281215097cc 100644 --- a/arch/powerpc/include/asm/kvm_ppc.h +++ b/arch/powerpc/include/asm/kvm_ppc.h @@ -981,18 +981,18 @@ static inline u##size kvmppc_get_##reg(struct kvm_vcpu *vcpu) \ if (iden) \ WARN_ON(kvmhv_nestedv2_cached_reload(vcpu, iden) < 0); \ if (kvmppc_shared_big_endian(vcpu)) \ - return be##size##_to_cpu(vcpu->arch.shared->reg); \ + return be##size##_to_cpu((__be##size __force)vcpu->arch.shared->reg); \ else \ - return le##size##_to_cpu(vcpu->arch.shared->reg); \ + return le##size##_to_cpu((__le##size __force)vcpu->arch.shared->reg); \ } \ #define KVMPPC_VCPU_SHARED_REGS_ACCESSOR_SET(reg, size, iden) \ static inline void kvmppc_set_##reg(struct kvm_vcpu *vcpu, u##size val) \ { \ if (kvmppc_shared_big_endian(vcpu)) \ - vcpu->arch.shared->reg = cpu_to_be##size(val); \ + vcpu->arch.shared->reg = (u##size __force)cpu_to_be##size(val); \ else \ - vcpu->arch.shared->reg = cpu_to_le##size(val); \ + vcpu->arch.shared->reg = (u##size __force)cpu_to_le##size(val); \ \ if (iden) \ kvmhv_nestedv2_mark_dirty(vcpu, iden); \ From 8577dd00a6ba335bc359313599d6100522a1931c Mon Sep 17 00:00:00 2001 From: Benjamin Gray Date: Wed, 11 Oct 2023 16:37:07 +1100 Subject: [PATCH 316/475] powerpc/opal: Annotate out param endianness Sparse reports an endian mismatch with args to opal_int_get_xirr(). Checking the skiboot source[1] shows the function takes a __be32* (as expected), so update the function declaration to reflect this. [1]: https://github.com/open-power/skiboot/blob/80e2b1dc73/hw/xive.c#L3479 Signed-off-by: Benjamin Gray Signed-off-by: Michael Ellerman Link: https://msgid.link/20231011053711.93427-9-bgray@linux.ibm.com --- arch/powerpc/include/asm/opal.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/powerpc/include/asm/opal.h b/arch/powerpc/include/asm/opal.h index a9b31cc258fc..b66b0c615f4f 100644 --- a/arch/powerpc/include/asm/opal.h +++ b/arch/powerpc/include/asm/opal.h @@ -227,7 +227,7 @@ int64_t opal_pci_set_power_state(uint64_t async_token, uint64_t id, uint64_t data); int64_t opal_pci_poll2(uint64_t id, uint64_t data); -int64_t opal_int_get_xirr(uint32_t *out_xirr, bool just_poll); +int64_t opal_int_get_xirr(__be32 *out_xirr, bool just_poll); int64_t opal_int_set_cppr(uint8_t cppr); int64_t opal_int_eoi(uint32_t xirr); int64_t opal_int_set_mfrr(uint32_t cpu, uint8_t mfrr); From c6519c6df0722e432f330afbc7c00d16d5be5c58 Mon Sep 17 00:00:00 2001 From: Benjamin Gray Date: Wed, 11 Oct 2023 16:37:08 +1100 Subject: [PATCH 317/475] powerpc/uaccess: Cast away __user annotation after verification Sparse reports dereference of a __user pointer. copy_mc_to_user() takes a __user pointer, verifies it, then calls the generic copy routine copy_mc_generic(). As we have verified the pointer, cast out the __user annotation when passing to copy_mc_generic(). Signed-off-by: Benjamin Gray Signed-off-by: Michael Ellerman Link: https://msgid.link/20231011053711.93427-10-bgray@linux.ibm.com --- arch/powerpc/include/asm/uaccess.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/powerpc/include/asm/uaccess.h b/arch/powerpc/include/asm/uaccess.h index fb725ec77926..f1f9890f50d3 100644 --- a/arch/powerpc/include/asm/uaccess.h +++ b/arch/powerpc/include/asm/uaccess.h @@ -374,7 +374,7 @@ copy_mc_to_user(void __user *to, const void *from, unsigned long n) if (check_copy_size(from, n, true)) { if (access_ok(to, n)) { allow_write_to_user(to, n); - n = copy_mc_generic((void *)to, from, n); + n = copy_mc_generic((void __force *)to, from, n); prevent_write_to_user(to, n); } } From 2c4ce3e65b1a543123ffcec4b021ad6ebd4e4e4e Mon Sep 17 00:00:00 2001 From: Benjamin Gray Date: Wed, 11 Oct 2023 16:37:09 +1100 Subject: [PATCH 318/475] powerpc: Cast away __iomem in low level IO routines Sparse reports dereferencing an __iomem pointer. These routines are clearly low level handlers for IO memory, so force cast away the __iomem annotation to tell sparse the dereferences are safe. Signed-off-by: Benjamin Gray Signed-off-by: Michael Ellerman Link: https://msgid.link/20231011053711.93427-11-bgray@linux.ibm.com --- arch/powerpc/kernel/io.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/powerpc/kernel/io.c b/arch/powerpc/kernel/io.c index 2f29b7d432de..6af535905984 100644 --- a/arch/powerpc/kernel/io.c +++ b/arch/powerpc/kernel/io.c @@ -33,7 +33,7 @@ void _insb(const volatile u8 __iomem *port, void *buf, long count) return; asm volatile("sync"); do { - tmp = *port; + tmp = *(const volatile u8 __force *)port; eieio(); *tbuf++ = tmp; } while (--count != 0); @@ -49,7 +49,7 @@ void _outsb(volatile u8 __iomem *port, const void *buf, long count) return; asm volatile("sync"); do { - *port = *tbuf++; + *(volatile u8 __force *)port = *tbuf++; } while (--count != 0); asm volatile("sync"); } @@ -64,7 +64,7 @@ void _insw_ns(const volatile u16 __iomem *port, void *buf, long count) return; asm volatile("sync"); do { - tmp = *port; + tmp = *(const volatile u16 __force *)port; eieio(); *tbuf++ = tmp; } while (--count != 0); @@ -80,7 +80,7 @@ void _outsw_ns(volatile u16 __iomem *port, const void *buf, long count) return; asm volatile("sync"); do { - *port = *tbuf++; + *(volatile u16 __force *)port = *tbuf++; } while (--count != 0); asm volatile("sync"); } @@ -95,7 +95,7 @@ void _insl_ns(const volatile u32 __iomem *port, void *buf, long count) return; asm volatile("sync"); do { - tmp = *port; + tmp = *(const volatile u32 __force *)port; eieio(); *tbuf++ = tmp; } while (--count != 0); @@ -111,7 +111,7 @@ void _outsl_ns(volatile u32 __iomem *port, const void *buf, long count) return; asm volatile("sync"); do { - *port = *tbuf++; + *(volatile u32 __force *)port = *tbuf++; } while (--count != 0); asm volatile("sync"); } From 82f635243f209a85d3deb9f64439c3ea84cd4ecb Mon Sep 17 00:00:00 2001 From: Benjamin Gray Date: Wed, 11 Oct 2023 16:37:10 +1100 Subject: [PATCH 319/475] powerpc/eeh: Remove unnecessary cast Sparse reports a warning when casting to an int. There is no need to cast in the first place, so drop them. Signed-off-by: Benjamin Gray Signed-off-by: Michael Ellerman Link: https://msgid.link/20231011053711.93427-12-bgray@linux.ibm.com --- arch/powerpc/kernel/eeh_driver.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/powerpc/kernel/eeh_driver.c b/arch/powerpc/kernel/eeh_driver.c index 438568a472d0..48773d2d9be3 100644 --- a/arch/powerpc/kernel/eeh_driver.c +++ b/arch/powerpc/kernel/eeh_driver.c @@ -39,7 +39,7 @@ static int eeh_result_priority(enum pci_ers_result result) case PCI_ERS_RESULT_NEED_RESET: return 6; default: - WARN_ONCE(1, "Unknown pci_ers_result value: %d\n", (int)result); + WARN_ONCE(1, "Unknown pci_ers_result value: %d\n", result); return 0; } }; @@ -60,7 +60,7 @@ static const char *pci_ers_result_name(enum pci_ers_result result) case PCI_ERS_RESULT_NO_AER_DRIVER: return "no AER driver"; default: - WARN_ONCE(1, "Unknown result type: %d\n", (int)result); + WARN_ONCE(1, "Unknown result type: %d\n", result); return "unknown"; } }; From b574b817cc7bfcc87bbc92b4b19525442401ae5e Mon Sep 17 00:00:00 2001 From: Benjamin Gray Date: Wed, 11 Oct 2023 16:37:11 +1100 Subject: [PATCH 320/475] powerpc/fadump: Annotate endianness cast with __force Sparse reports an endianness error with the else case of val = (cpu_endian ? be64_to_cpu(reg_entry->reg_val) : (u64)(reg_entry->reg_val)); This is a safe operation because the code is explicitly working with dynamic endianness, so add the __force annotation to tell Sparse to ignore it. Signed-off-by: Benjamin Gray Signed-off-by: Michael Ellerman Link: https://msgid.link/20231011053711.93427-13-bgray@linux.ibm.com --- arch/powerpc/platforms/powernv/opal-fadump.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/powerpc/platforms/powernv/opal-fadump.h b/arch/powerpc/platforms/powernv/opal-fadump.h index 3f715efb0aa6..5eeb794b5eb1 100644 --- a/arch/powerpc/platforms/powernv/opal-fadump.h +++ b/arch/powerpc/platforms/powernv/opal-fadump.h @@ -135,7 +135,7 @@ static inline void opal_fadump_read_regs(char *bufp, unsigned int regs_cnt, for (i = 0; i < regs_cnt; i++, bufp += reg_entry_size) { reg_entry = (struct hdat_fadump_reg_entry *)bufp; val = (cpu_endian ? be64_to_cpu(reg_entry->reg_val) : - (u64)(reg_entry->reg_val)); + (u64 __force)(reg_entry->reg_val)); opal_fadump_set_regval_regnum(regs, be32_to_cpu(reg_entry->reg_type), be32_to_cpu(reg_entry->reg_num), From 6db51ff905362e6c79593dbda08f61f24b25971c Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Thu, 5 Jan 2023 14:41:45 +0800 Subject: [PATCH 321/475] macintosh/macio-adb: add missing iounmap() on error in macio_init() Add missing iounmap(), if request_irq() fails. Signed-off-by: Yang Yingliang Signed-off-by: Michael Ellerman Link: https://msgid.link/20230105064145.3879356-1-yangyingliang@huawei.com --- drivers/macintosh/macio-adb.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/macintosh/macio-adb.c b/drivers/macintosh/macio-adb.c index 55a9f8c3a150..779f1268286e 100644 --- a/drivers/macintosh/macio-adb.c +++ b/drivers/macintosh/macio-adb.c @@ -123,6 +123,7 @@ int macio_init(void) irq = irq_of_parse_and_map(adbs, 0); of_node_put(adbs); if (request_irq(irq, macio_adb_interrupt, 0, "ADB", (void *)0)) { + iounmap(adb); printk(KERN_ERR "ADB: can't get irq %d\n", irq); return -EAGAIN; } From b28d1ccf921a4333be14017d82066386d419e638 Mon Sep 17 00:00:00 2001 From: Stanislav Kinsburskii Date: Sat, 15 Apr 2023 04:17:53 -0700 Subject: [PATCH 322/475] powerpc/io: Expect immutable pointer in virt_to_phys() prototype MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit virt_to_phys() doesn't need the address pointer to be mutable. At the same time allowing it to be mutable leads to the following build warning for constant pointers: warning: passing argument 1 of ‘virt_to_phys’ discards ‘const’ qualifier from pointer target type Signed-off-by: Stanislav Kinsburskii Reviewed-by: Linus Walleij Signed-off-by: Michael Ellerman Link: https://msgid.link/168155747391.13678.10634415747614468991.stgit@skinsburskii.localdomain --- arch/powerpc/include/asm/io.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/powerpc/include/asm/io.h b/arch/powerpc/include/asm/io.h index 0732b743e099..5220274a6277 100644 --- a/arch/powerpc/include/asm/io.h +++ b/arch/powerpc/include/asm/io.h @@ -950,7 +950,7 @@ extern void __iomem *__ioremap_caller(phys_addr_t, unsigned long size, * almost all conceivable cases a device driver should not be using * this function */ -static inline unsigned long virt_to_phys(volatile void * address) +static inline unsigned long virt_to_phys(const volatile void * address) { WARN_ON(IS_ENABLED(CONFIG_DEBUG_VIRTUAL) && !virt_addr_valid(address)); From d45c4b48dafb5820e5cc267ff9a6d7784d13a43c Mon Sep 17 00:00:00 2001 From: Michael Ellerman Date: Thu, 24 Aug 2023 16:42:10 +1000 Subject: [PATCH 323/475] powerpc: Hide empty pt_regs at base of the stack A thread started via eg. user_mode_thread() runs in the kernel to begin with and then may later return to userspace. While it's running in the kernel it has a pt_regs at the base of its kernel stack, but that pt_regs is all zeroes. If the thread oopses in that state, it leads to an ugly stack trace with a big block of zero GPRs, as reported by Joel: Kernel panic - not syncing: VFS: Unable to mount root fs on unknown-block(0,0) CPU: 0 PID: 1 Comm: swapper/0 Not tainted 6.5.0-rc7-00004-gf7757129e3de-dirty #3 Hardware name: IBM PowerNV (emulated by qemu) POWER9 0x4e1200 opal:v7.0 PowerNV Call Trace: [c0000000036afb00] [c0000000010dd058] dump_stack_lvl+0x6c/0x9c (unreliable) [c0000000036afb30] [c00000000013c524] panic+0x178/0x424 [c0000000036afbd0] [c000000002005100] mount_root_generic+0x250/0x324 [c0000000036afca0] [c0000000020057d0] prepare_namespace+0x2d4/0x344 [c0000000036afd20] [c0000000020049c0] kernel_init_freeable+0x358/0x3ac [c0000000036afdf0] [c0000000000111b0] kernel_init+0x30/0x1a0 [c0000000036afe50] [c00000000000debc] ret_from_kernel_user_thread+0x14/0x1c --- interrupt: 0 at 0x0 NIP: 0000000000000000 LR: 0000000000000000 CTR: 0000000000000000 REGS: c0000000036afe80 TRAP: 0000 Not tainted (6.5.0-rc7-00004-gf7757129e3de-dirty) MSR: 0000000000000000 <> CR: 00000000 XER: 00000000 CFAR: 0000000000000000 IRQMASK: 0 GPR00: 0000000000000000 0000000000000000 0000000000000000 0000000000000000 GPR04: 0000000000000000 0000000000000000 0000000000000000 0000000000000000 GPR08: 0000000000000000 0000000000000000 0000000000000000 0000000000000000 GPR12: 0000000000000000 0000000000000000 0000000000000000 0000000000000000 GPR16: 0000000000000000 0000000000000000 0000000000000000 0000000000000000 GPR20: 0000000000000000 0000000000000000 0000000000000000 0000000000000000 GPR24: 0000000000000000 0000000000000000 0000000000000000 0000000000000000 GPR28: 0000000000000000 0000000000000000 0000000000000000 0000000000000000 NIP [0000000000000000] 0x0 LR [0000000000000000] 0x0 --- interrupt: 0 The all-zero pt_regs looks ugly and conveys no useful information, other than its presence. So detect that case and just show the presence of the frame by printing the interrupt marker, eg: Kernel panic - not syncing: VFS: Unable to mount root fs on unknown-block(0,0) CPU: 0 PID: 1 Comm: swapper/0 Not tainted 6.5.0-rc3-00126-g18e9506562a0-dirty #301 Hardware name: IBM pSeries (emulated by qemu) POWER9 (raw) 0x4e1202 0xf000005 of:SLOF,HEAD hv:linux,kvm pSeries Call Trace: [c000000003aabb00] [c000000001143db8] dump_stack_lvl+0x6c/0x9c (unreliable) [c000000003aabb30] [c00000000014c624] panic+0x178/0x424 [c000000003aabbd0] [c0000000020050fc] mount_root_generic+0x250/0x324 [c000000003aabca0] [c0000000020057cc] prepare_namespace+0x2d4/0x344 [c000000003aabd20] [c0000000020049bc] kernel_init_freeable+0x358/0x3ac [c000000003aabdf0] [c0000000000111b0] kernel_init+0x30/0x1a0 [c000000003aabe50] [c00000000000debc] ret_from_kernel_user_thread+0x14/0x1c --- interrupt: 0 at 0x0 To avoid ever suppressing a valid pt_regs make sure the pt_regs has a zero MSR and TRAP value, and is located at the very base of the stack. Fixes: 6895dfc04741 ("powerpc: copy_thread fill in interrupt frame marker and back chain") Reported-by: Joel Stanley Reported-by: Nicholas Piggin Reviewed-by: Joel Stanley Signed-off-by: Michael Ellerman Link: https://msgid.link/20230824064210.907266-1-mpe@ellerman.id.au --- arch/powerpc/kernel/process.c | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c index b68898ac07e1..392404688cec 100644 --- a/arch/powerpc/kernel/process.c +++ b/arch/powerpc/kernel/process.c @@ -2258,6 +2258,22 @@ unsigned long __get_wchan(struct task_struct *p) return ret; } +static bool empty_user_regs(struct pt_regs *regs, struct task_struct *tsk) +{ + unsigned long stack_page; + + // A non-empty pt_regs should never have a zero MSR or TRAP value. + if (regs->msr || regs->trap) + return false; + + // Check it sits at the very base of the stack + stack_page = (unsigned long)task_stack_page(tsk); + if ((unsigned long)(regs + 1) != stack_page + THREAD_SIZE) + return false; + + return true; +} + static int kstack_depth_to_print = CONFIG_PRINT_STACK_DEPTH; void __no_sanitize_address show_stack(struct task_struct *tsk, @@ -2322,9 +2338,13 @@ void __no_sanitize_address show_stack(struct task_struct *tsk, lr = regs->link; printk("%s--- interrupt: %lx at %pS\n", loglvl, regs->trap, (void *)regs->nip); - __show_regs(regs); - printk("%s--- interrupt: %lx\n", - loglvl, regs->trap); + + // Detect the case of an empty pt_regs at the very base + // of the stack and suppress showing it in full. + if (!empty_user_regs(regs, tsk)) { + __show_regs(regs); + printk("%s--- interrupt: %lx\n", loglvl, regs->trap); + } firstframe = 1; } From e08c43e6c3eb5d805b61d981f1e8286ee0dc6d1a Mon Sep 17 00:00:00 2001 From: Kuan-Wei Chiu Date: Sat, 14 Oct 2023 01:57:14 +0800 Subject: [PATCH 324/475] powerpc/perf: Optimize find_alternatives_list() using binary search This patch improves the performance of event alternative lookup by replacing the previous linear search with a more efficient binary search. This change reduces the time complexity for the search process from O(n) to O(log(n)). A pre-sorted table of event values and their corresponding indices has been introduced to expedite the search process. Signed-off-by: Kuan-Wei Chiu [mpe: Call the array "presort*ed*_event_table", minor formatting] Signed-off-by: Michael Ellerman Link: https://msgid.link/20231013175714.2142775-1-visitorckw@gmail.com --- arch/powerpc/perf/power6-pmu.c | 44 +++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/arch/powerpc/perf/power6-pmu.c b/arch/powerpc/perf/power6-pmu.c index 5729b6e059de..9f720b522e17 100644 --- a/arch/powerpc/perf/power6-pmu.c +++ b/arch/powerpc/perf/power6-pmu.c @@ -335,26 +335,38 @@ static const unsigned int event_alternatives[][MAX_ALT] = { { 0x3000fe, 0x400056 }, /* PM_DATA_FROM_L3MISS */ }; -/* - * This could be made more efficient with a binary search on - * a presorted list, if necessary - */ static int find_alternatives_list(u64 event) { - int i, j; - unsigned int alt; + const unsigned int presorted_event_table[] = { + 0x0130e8, 0x080080, 0x080088, 0x10000a, 0x10000b, 0x10000d, 0x10000e, + 0x100010, 0x10001a, 0x100026, 0x100054, 0x100056, 0x1000f0, 0x1000f8, + 0x1000fc, 0x200008, 0x20000e, 0x200010, 0x200012, 0x200054, 0x2000f0, + 0x2000f2, 0x2000f4, 0x2000f5, 0x2000f6, 0x2000f8, 0x2000fc, 0x2000fe, + 0x2d0030, 0x30000a, 0x30000c, 0x300010, 0x300012, 0x30001a, 0x300056, + 0x3000f0, 0x3000f2, 0x3000f6, 0x3000f8, 0x3000fc, 0x3000fe, 0x400006, + 0x400007, 0x40000a, 0x40000e, 0x400010, 0x400018, 0x400056, 0x4000f0, + 0x4000f8, 0x600005 + }; + const unsigned int event_index_table[] = { + 0, 1, 2, 3, 4, 1, 5, 6, 7, 8, 9, 10, 11, 12, 13, 12, 14, + 7, 15, 2, 9, 16, 3, 4, 0, 17, 10, 18, 19, 20, 1, 17, 15, 19, + 18, 2, 16, 21, 8, 0, 22, 13, 14, 11, 21, 5, 20, 22, 1, 6, 3 + }; + int hi = ARRAY_SIZE(presorted_event_table) - 1; + int lo = 0; - for (i = 0; i < ARRAY_SIZE(event_alternatives); ++i) { - if (event < event_alternatives[i][0]) - return -1; - for (j = 0; j < MAX_ALT; ++j) { - alt = event_alternatives[i][j]; - if (!alt || event < alt) - break; - if (event == alt) - return i; - } + while (lo <= hi) { + int mid = lo + (hi - lo) / 2; + unsigned int alt = presorted_event_table[mid]; + + if (alt < event) + lo = mid + 1; + else if (alt > event) + hi = mid - 1; + else + return event_index_table[mid]; } + return -1; } From efce8422dd53fae6cdbd37741034ac4eb4730819 Mon Sep 17 00:00:00 2001 From: Srikar Dronamraju Date: Thu, 19 Oct 2023 14:44:52 +0530 Subject: [PATCH 325/475] powerpc/paravirt: Improve vcpu_is_preempted PowerVM Hypervisor dispatches on a whole core basis. In a shared LPAR, a CPU from a core that is CEDED or preempted may have a larger latency. In such a scenario, its preferable to choose a different CPU to run. If one of the CPUs in the core is active, i.e neither CEDED nor preempted, then consider this CPU as not preempted. Also if any of the CPUs in the core has yielded but OS has not requested CEDE or CONFER, then consider this CPU to be preempted. Correct detection of preempted CPUs is important for detecting idle CPUs/cores in task scheduler. Tested-by: Aboorva Devarajan Reviewed-by: Shrikanth Hegde Signed-off-by: Srikar Dronamraju Signed-off-by: Michael Ellerman Link: https://msgid.link/20231019091452.95260-1-srikar@linux.vnet.ibm.com --- arch/powerpc/include/asm/paravirt.h | 47 +++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/arch/powerpc/include/asm/paravirt.h b/arch/powerpc/include/asm/paravirt.h index e08513d73119..ac4279208d63 100644 --- a/arch/powerpc/include/asm/paravirt.h +++ b/arch/powerpc/include/asm/paravirt.h @@ -71,6 +71,11 @@ static inline void yield_to_any(void) { plpar_hcall_norets_notrace(H_CONFER, -1, 0); } + +static inline bool is_vcpu_idle(int vcpu) +{ + return lppaca_of(vcpu).idle; +} #else static inline bool is_shared_processor(void) { @@ -100,6 +105,10 @@ static inline void prod_cpu(int cpu) ___bad_prod_cpu(); /* This would be a bug */ } +static inline bool is_vcpu_idle(int vcpu) +{ + return false; +} #endif #define vcpu_is_preempted vcpu_is_preempted @@ -121,9 +130,23 @@ static inline bool vcpu_is_preempted(int cpu) if (!is_shared_processor()) return false; + /* + * If the hypervisor has dispatched the target CPU on a physical + * processor, then the target CPU is definitely not preempted. + */ + if (!(yield_count_of(cpu) & 1)) + return false; + + /* + * If the target CPU has yielded to Hypervisor but OS has not + * requested idle then the target CPU is definitely preempted. + */ + if (!is_vcpu_idle(cpu)) + return true; + #ifdef CONFIG_PPC_SPLPAR if (!is_kvm_guest()) { - int first_cpu; + int first_cpu, i; /* * The result of vcpu_is_preempted() is used in a @@ -149,11 +172,29 @@ static inline bool vcpu_is_preempted(int cpu) */ if (cpu_first_thread_sibling(cpu) == first_cpu) return false; + + /* + * If any of the threads of the target CPU's core are not + * preempted or ceded, then consider target CPU to be + * non-preempted. + */ + first_cpu = cpu_first_thread_sibling(cpu); + for (i = first_cpu; i < first_cpu + threads_per_core; i++) { + if (i == cpu) + continue; + if (!(yield_count_of(i) & 1)) + return false; + if (!is_vcpu_idle(i)) + return true; + } } #endif - if (yield_count_of(cpu) & 1) - return true; + /* + * None of the threads in target CPU's core are running but none of + * them were preempted too. Hence assume the target CPU to be + * non-preempted. + */ return false; } From 3bf983e4e93ce8e6d69e9d63f52a66ec0856672e Mon Sep 17 00:00:00 2001 From: Gaurav Batra Date: Mon, 2 Oct 2023 22:08:02 -0500 Subject: [PATCH 326/475] powerpc/pseries/iommu: enable_ddw incorrectly returns direct mapping for SR-IOV device When a device is initialized, the driver invokes dma_supported() twice - first for streaming mappings followed by coherent mappings. For an SR-IOV device, default window is deleted and DDW created. With vPMEM enabled, TCE mappings are dynamically created for both vPMEM and SR-IOV device. There are no direct mappings. First time when dma_supported() is called with 64 bit mask, DDW is created and marked as dynamic window. The second time dma_supported() is called, enable_ddw() finds existing window for the device and incorrectly returns it as "direct mapping". This only happens when size of DDW is big enough to map max LPAR memory. This results in streaming TCEs to not get dynamically mapped, since code incorrently assumes these are already pre-mapped. The adapter initially comes up but goes down due to EEH. Fixes: 381ceda88c4c ("powerpc/pseries/iommu: Make use of DDW for indirect mapping") Cc: stable@vger.kernel.org # v5.15+ Signed-off-by: Gaurav Batra Signed-off-by: Michael Ellerman Link: https://msgid.link/20231003030802.47914-1-gbatra@linux.vnet.ibm.com --- arch/powerpc/platforms/pseries/iommu.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/powerpc/platforms/pseries/iommu.c b/arch/powerpc/platforms/pseries/iommu.c index 16d93b580f61..496e16c588aa 100644 --- a/arch/powerpc/platforms/pseries/iommu.c +++ b/arch/powerpc/platforms/pseries/iommu.c @@ -914,7 +914,8 @@ static int remove_ddw(struct device_node *np, bool remove_prop, const char *win_ return 0; } -static bool find_existing_ddw(struct device_node *pdn, u64 *dma_addr, int *window_shift) +static bool find_existing_ddw(struct device_node *pdn, u64 *dma_addr, int *window_shift, + bool *direct_mapping) { struct dma_win *window; const struct dynamic_dma_window_prop *dma64; @@ -927,6 +928,7 @@ static bool find_existing_ddw(struct device_node *pdn, u64 *dma_addr, int *windo dma64 = window->prop; *dma_addr = be64_to_cpu(dma64->dma_base); *window_shift = be32_to_cpu(dma64->window_shift); + *direct_mapping = window->direct; found = true; break; } @@ -1270,10 +1272,8 @@ static bool enable_ddw(struct pci_dev *dev, struct device_node *pdn) mutex_lock(&dma_win_init_mutex); - if (find_existing_ddw(pdn, &dev->dev.archdata.dma_offset, &len)) { - direct_mapping = (len >= max_ram_len); + if (find_existing_ddw(pdn, &dev->dev.archdata.dma_offset, &len, &direct_mapping)) goto out_unlock; - } /* * If we already went through this for a previous function of From 7bc8b8eb2b1e7b06d14bacf964fa25f74a2b87e5 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 16 Oct 2023 12:10:04 +0200 Subject: [PATCH 327/475] s390/mm: remove __GFP_HIGHMEM masking Remove unnecessary __GFP_HIGHMEM masking, which was introduced with commit 6326c26c1514 ("s390: convert various pgalloc functions to use ptdescs"). Also remove a whitespace change which was introduced with the same commit. Link: https://lore.kernel.org/all/CAOzc2px-SFSnmjcPriiB3cm1fNj3+YC8S0VSp4t1QvDR0f4E2A@mail.gmail.com Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/pgalloc.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/arch/s390/mm/pgalloc.c b/arch/s390/mm/pgalloc.c index 61fb157029c8..a6b71bc71896 100644 --- a/arch/s390/mm/pgalloc.c +++ b/arch/s390/mm/pgalloc.c @@ -488,11 +488,10 @@ static unsigned long *base_crst_alloc(unsigned long val) unsigned long *table; struct ptdesc *ptdesc; - ptdesc = pagetable_alloc(GFP_KERNEL & ~__GFP_HIGHMEM, CRST_ALLOC_ORDER); + ptdesc = pagetable_alloc(GFP_KERNEL, CRST_ALLOC_ORDER); if (!ptdesc) return NULL; table = ptdesc_address(ptdesc); - crst_table_init(table, val); return table; } From 4f62c6e30155c7a85ed74e52bb8c71e7b0879cb3 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 16 Oct 2023 12:17:59 +0200 Subject: [PATCH 328/475] s390/mm: make vmemmap_free() only for CONFIG_MEMORY_HOTPLUG available MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Get rid of this W=1 compile warning: arch/s390/mm/vmem.c:502:6: warning: no previous prototype for ‘vmemmap_free’ [-Wmissing-prototypes] 502 | void vmemmap_free(unsigned long start, unsigned long end, | ^~~~~~~~~~~~ Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/vmem.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c index 5ed2e9254649..86384586eb20 100644 --- a/arch/s390/mm/vmem.c +++ b/arch/s390/mm/vmem.c @@ -499,6 +499,8 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, return ret; } +#ifdef CONFIG_MEMORY_HOTPLUG + void vmemmap_free(unsigned long start, unsigned long end, struct vmem_altmap *altmap) { @@ -507,6 +509,8 @@ void vmemmap_free(unsigned long start, unsigned long end, mutex_unlock(&vmem_mutex); } +#endif + void vmem_remove_mapping(unsigned long start, unsigned long size) { mutex_lock(&vmem_mutex); From 60f8f641f3db85edf20f9612f4801563119d7fd6 Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Mon, 21 Aug 2023 16:49:05 +0200 Subject: [PATCH 329/475] s390/pai_crypto: dynamically allocate percpu pai crypto map data structure Struct paicrypt_map is a data structure and is statically defined for each possible CPU. Rework this and replace it by dynamically allocated data structures created when a perf_event_open() system call is invoked. It is replaced by an array of pointers to all possible CPUs and reference counting. The array of pointers is allocated when the first event is created. For each online CPU an event is installed on, a struct paicrypt_map is allocated and a pointer to struct cpu_cf_events is stored in the array: CPU 0 1 2 3 ... N +---+---+---+---+---+---+ paicrypt_root::mapptr--> | * | | | |...| | +-|-+---+---+---+---+---+ | | \|/ +--------------+ | paicrypt_map | +--------------+ With this approach the large data structure is only allocated when an event is actually installed and used. Also implement proper reference counting for allocation and removal. PAI crypto counter events can not be created when a CPU hot plug add is processed. This means a CPU hot plug add does not get the necessary PAI event to record PAI cryptography counter increments on the newly added CPU. There is no possibility to notify user space of a new CPU and the necessary event infrastructure assoiciated with the file descriptor returned by perf_event_open() system call. However system call perf_event_open() can use the newly added CPU when issued after the CPU hot plug add. Kernel CPU hot plug remove deletes the CPU and stops the PAI counters on that CPU. When the process closes the file descriptor associated with that event, the event's destroy() function removes any allocated data structures and adjusts the reference counts. Signed-off-by: Thomas Richter Acked-by: Sumanth Korikkar Acked-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/kernel/perf_pai_crypto.c | 127 +++++++++++++++++++++++------ 1 file changed, 100 insertions(+), 27 deletions(-) diff --git a/arch/s390/kernel/perf_pai_crypto.c b/arch/s390/kernel/perf_pai_crypto.c index db470243966f..19d53edf50d2 100644 --- a/arch/s390/kernel/perf_pai_crypto.c +++ b/arch/s390/kernel/perf_pai_crypto.c @@ -40,7 +40,43 @@ struct paicrypt_map { struct perf_event *event; /* Perf event for sampling */ }; -static DEFINE_PER_CPU(struct paicrypt_map, paicrypt_map); +struct paicrypt_mapptr { + struct paicrypt_map *mapptr; +}; + +static struct paicrypt_root { /* Anchor to per CPU data */ + refcount_t refcnt; /* Overall active events */ + struct paicrypt_mapptr __percpu *mapptr; +} paicrypt_root; + +/* Free per CPU data when the last event is removed. */ +static void paicrypt_root_free(void) +{ + if (refcount_dec_and_test(&paicrypt_root.refcnt)) { + free_percpu(paicrypt_root.mapptr); + paicrypt_root.mapptr = NULL; + } + debug_sprintf_event(cfm_dbg, 5, "%s root.refcount %d\n", __func__, + refcount_read(&paicrypt_root.refcnt)); +} + +/* + * On initialization of first event also allocate per CPU data dynamically. + * Start with an array of pointers, the array size is the maximum number of + * CPUs possible, which might be larger than the number of CPUs currently + * online. + */ +static int paicrypt_root_alloc(void) +{ + if (!refcount_inc_not_zero(&paicrypt_root.refcnt)) { + /* The memory is already zeroed. */ + paicrypt_root.mapptr = alloc_percpu(struct paicrypt_mapptr); + if (!paicrypt_root.mapptr) + return -ENOMEM; + refcount_set(&paicrypt_root.refcnt, 1); + } + return 0; +} /* Release the PMU if event is the last perf event */ static DEFINE_MUTEX(pai_reserve_mutex); @@ -50,7 +86,9 @@ static DEFINE_MUTEX(pai_reserve_mutex); */ static void paicrypt_event_destroy(struct perf_event *event) { - struct paicrypt_map *cpump = per_cpu_ptr(&paicrypt_map, event->cpu); + struct paicrypt_mapptr *mp = per_cpu_ptr(paicrypt_root.mapptr, + event->cpu); + struct paicrypt_map *cpump = mp->mapptr; cpump->event = NULL; static_branch_dec(&pai_key); @@ -65,11 +103,11 @@ static void paicrypt_event_destroy(struct perf_event *event) __func__, (unsigned long)cpump->page, cpump->save); free_page((unsigned long)cpump->page); - cpump->page = NULL; kvfree(cpump->save); - cpump->save = NULL; - cpump->mode = PAI_MODE_NONE; + kfree(cpump); + mp->mapptr = NULL; } + paicrypt_root_free(); mutex_unlock(&pai_reserve_mutex); } @@ -85,7 +123,8 @@ static u64 paicrypt_getctr(struct paicrypt_map *cpump, int nr, bool kernel) */ static u64 paicrypt_getdata(struct perf_event *event, bool kernel) { - struct paicrypt_map *cpump = this_cpu_ptr(&paicrypt_map); + struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr); + struct paicrypt_map *cpump = mp->mapptr; u64 sum = 0; int i; @@ -131,11 +170,31 @@ static u64 paicrypt_getall(struct perf_event *event) * * Allocate the memory for the event. */ -static int paicrypt_busy(struct perf_event_attr *a, struct paicrypt_map *cpump) +static struct paicrypt_map *paicrypt_busy(struct perf_event *event) { - int rc = 0; + struct perf_event_attr *a = &event->attr; + struct paicrypt_map *cpump = NULL; + struct paicrypt_mapptr *mp; + int rc; mutex_lock(&pai_reserve_mutex); + + /* Allocate root node */ + rc = paicrypt_root_alloc(); + if (rc) + goto unlock; + + /* Allocate node for this event */ + mp = per_cpu_ptr(paicrypt_root.mapptr, event->cpu); + cpump = mp->mapptr; + if (!cpump) { /* Paicrypt_map allocated? */ + cpump = kzalloc(sizeof(*cpump), GFP_KERNEL); + if (!cpump) { + rc = -ENOMEM; + goto free_root; + } + } + if (a->sample_period) { /* Sampling requested */ if (cpump->mode != PAI_MODE_NONE) rc = -EBUSY; /* ... sampling/counting active */ @@ -143,8 +202,15 @@ static int paicrypt_busy(struct perf_event_attr *a, struct paicrypt_map *cpump) if (cpump->mode == PAI_MODE_SAMPLING) rc = -EBUSY; /* ... and sampling active */ } + /* + * This error case triggers when there is a conflict: + * Either sampling requested and counting already active, or visa + * versa. Therefore the struct paicrypto_map for this CPU is + * needed or the error could not have occurred. Only adjust root + * node refcount. + */ if (rc) - goto unlock; + goto free_root; /* Allocate memory for counter page and counter extraction. * Only the first counting event has to allocate a page. @@ -157,30 +223,36 @@ static int paicrypt_busy(struct perf_event_attr *a, struct paicrypt_map *cpump) rc = -ENOMEM; cpump->page = (unsigned long *)get_zeroed_page(GFP_KERNEL); if (!cpump->page) - goto unlock; + goto free_paicrypt_map; cpump->save = kvmalloc_array(paicrypt_cnt + 1, sizeof(struct pai_userdata), GFP_KERNEL); if (!cpump->save) { free_page((unsigned long)cpump->page); cpump->page = NULL; - goto unlock; + goto free_paicrypt_map; } + + /* Set mode and reference count */ rc = 0; refcount_set(&cpump->refcnt, 1); - -unlock: - /* If rc is non-zero, do not set mode and reference count */ - if (!rc) { - cpump->mode = a->sample_period ? PAI_MODE_SAMPLING - : PAI_MODE_COUNTING; - } + cpump->mode = a->sample_period ? PAI_MODE_SAMPLING : PAI_MODE_COUNTING; + mp->mapptr = cpump; debug_sprintf_event(cfm_dbg, 5, "%s sample_period %#llx users %d" " mode %d refcnt %u page %#lx save %p rc %d\n", __func__, a->sample_period, cpump->active_events, cpump->mode, refcount_read(&cpump->refcnt), (unsigned long)cpump->page, cpump->save, rc); + goto unlock; + +free_paicrypt_map: + kfree(cpump); + mp->mapptr = NULL; +free_root: + paicrypt_root_free(); + +unlock: mutex_unlock(&pai_reserve_mutex); - return rc; + return rc ? ERR_PTR(rc) : cpump; } /* Might be called on different CPU than the one the event is intended for. */ @@ -188,7 +260,6 @@ static int paicrypt_event_init(struct perf_event *event) { struct perf_event_attr *a = &event->attr; struct paicrypt_map *cpump; - int rc; /* PAI crypto PMU registered as PERF_TYPE_RAW, check event type */ if (a->type != PERF_TYPE_RAW && event->pmu->type != a->type) @@ -204,10 +275,9 @@ static int paicrypt_event_init(struct perf_event *event) if (a->sample_period && a->config != PAI_CRYPTO_BASE) return -EINVAL; - cpump = per_cpu_ptr(&paicrypt_map, event->cpu); - rc = paicrypt_busy(a, cpump); - if (rc) - return rc; + cpump = paicrypt_busy(event); + if (IS_ERR(cpump)) + return PTR_ERR(cpump); /* Event initialization sets last_tag to 0. When later on the events * are deleted and re-added, do not reset the event count value to zero. @@ -259,7 +329,8 @@ static void paicrypt_start(struct perf_event *event, int flags) static int paicrypt_add(struct perf_event *event, int flags) { - struct paicrypt_map *cpump = this_cpu_ptr(&paicrypt_map); + struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr); + struct paicrypt_map *cpump = mp->mapptr; unsigned long ccd; if (++cpump->active_events == 1) { @@ -286,7 +357,8 @@ static void paicrypt_stop(struct perf_event *event, int flags) static void paicrypt_del(struct perf_event *event, int flags) { - struct paicrypt_map *cpump = this_cpu_ptr(&paicrypt_map); + struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr); + struct paicrypt_map *cpump = mp->mapptr; if (event->attr.sample_period) perf_sched_cb_dec(event->pmu); @@ -328,7 +400,8 @@ static size_t paicrypt_copy(struct pai_userdata *userdata, static int paicrypt_push_sample(void) { - struct paicrypt_map *cpump = this_cpu_ptr(&paicrypt_map); + struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr); + struct paicrypt_map *cpump = mp->mapptr; struct perf_event *event = cpump->event; struct perf_sample_data data; struct perf_raw_record raw; From 5069211e2f0b47e75119805e23ae6352d871e263 Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Thu, 19 Oct 2023 10:26:42 +0200 Subject: [PATCH 330/475] perf trace: Use the right bpf_probe_read(_str) variant for reading user data Perf test case 111 Check open filename arg using perf trace + vfs_getname fails on s390. This is caused by a failing function bpf_probe_read() in file util/bpf_skel/augmented_raw_syscalls.bpf.c. The root cause is the lookup by address. Function bpf_probe_read() is used. This function works only for architectures with ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE. On s390 is not possible to determine from the address to which address space the address belongs to (user or kernel space). Replace bpf_probe_read() by bpf_probe_read_kernel() and bpf_probe_read_str() by bpf_probe_read_user_str() to explicity specify the address space the address refers to. Output before: # ./perf trace -eopen,openat -- touch /tmp/111 libbpf: prog 'sys_enter': BPF program load failed: Invalid argument libbpf: prog 'sys_enter': -- BEGIN PROG LOAD LOG -- reg type unsupported for arg#0 function sys_enter#75 0: R1=ctx(off=0,imm=0) R10=fp0 ; int sys_enter(struct syscall_enter_args *args) 0: (bf) r6 = r1 ; R1=ctx(off=0,imm=0) R6_w=ctx(off=0,imm=0) ; return bpf_get_current_pid_tgid(); 1: (85) call bpf_get_current_pid_tgid#14 ; R0_w=scalar() 2: (63) *(u32 *)(r10 -8) = r0 ; R0_w=scalar() R10=fp0 fp-8=????mmmm 3: (bf) r2 = r10 ; R2_w=fp0 R10=fp0 ; ..... lines deleted here ..... 23: (bf) r3 = r6 ; R3_w=ctx(off=0,imm=0) R6=ctx(off=0,imm=0) 24: (85) call bpf_probe_read#4 unknown func bpf_probe_read#4 processed 23 insns (limit 1000000) max_states_per_insn 0 \ total_states 2 peak_states 2 mark_read 2 -- END PROG LOAD LOG -- libbpf: prog 'sys_enter': failed to load: -22 libbpf: failed to load object 'augmented_raw_syscalls_bpf' libbpf: failed to load BPF skeleton 'augmented_raw_syscalls_bpf': -22 .... Output after: # ./perf test -Fv 111 111: Check open filename arg using perf trace + vfs_getname : --- start --- 1.085 ( 0.011 ms): touch/320753 openat(dfd: CWD, filename: \ "/tmp/temporary_file.SWH85", \ flags: CREAT|NOCTTY|NONBLOCK|WRONLY, mode: IRUGO|IWUGO) = 3 ---- end ---- Check open filename arg using perf trace + vfs_getname: Ok # Test with the sleep command shows: Output before: # ./perf trace -e *sleep sleep 1.234567890 0.000 (1234.681 ms): sleep/63114 clock_nanosleep(rqtp: \ { .tv_sec: 0, .tv_nsec: 0 }, rmtp: 0x3ffe0979720) = 0 # Output after: # ./perf trace -e *sleep sleep 1.234567890 0.000 (1234.686 ms): sleep/64277 clock_nanosleep(rqtp: \ { .tv_sec: 1, .tv_nsec: 234567890 }, rmtp: 0x3fff3df9ea0) = 0 # Fixes: 14e4b9f4289a ("perf trace: Raw augmented syscalls fix libbpf 1.0+ compatibility") Signed-off-by: Thomas Richter Co-developed-by: Arnaldo Carvalho de Melo Acked-by: Ilya Leoshkevich Tested-by: Arnaldo Carvalho de Melo Cc: Ian Rogers Cc: gor@linux.ibm.com Cc: hca@linux.ibm.com Cc: sumanthk@linux.ibm.com Cc: svens@linux.ibm.com Link: https://lore.kernel.org/r/20231019082642.3286650-1-tmricht@linux.ibm.com Signed-off-by: Namhyung Kim --- .../util/bpf_skel/augmented_raw_syscalls.bpf.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tools/perf/util/bpf_skel/augmented_raw_syscalls.bpf.c b/tools/perf/util/bpf_skel/augmented_raw_syscalls.bpf.c index 939ec769bf4a..52c270330ae0 100644 --- a/tools/perf/util/bpf_skel/augmented_raw_syscalls.bpf.c +++ b/tools/perf/util/bpf_skel/augmented_raw_syscalls.bpf.c @@ -153,7 +153,7 @@ static inline unsigned int augmented_arg__read_str(struct augmented_arg *augmented_arg, const void *arg, unsigned int arg_len) { unsigned int augmented_len = sizeof(*augmented_arg); - int string_len = bpf_probe_read_str(&augmented_arg->value, arg_len, arg); + int string_len = bpf_probe_read_user_str(&augmented_arg->value, arg_len, arg); augmented_arg->size = augmented_arg->err = 0; /* @@ -203,7 +203,7 @@ int sys_enter_connect(struct syscall_enter_args *args) _Static_assert(is_power_of_2(sizeof(augmented_args->saddr)), "sizeof(augmented_args->saddr) needs to be a power of two"); socklen &= sizeof(augmented_args->saddr) - 1; - bpf_probe_read(&augmented_args->saddr, socklen, sockaddr_arg); + bpf_probe_read_user(&augmented_args->saddr, socklen, sockaddr_arg); return augmented__output(args, augmented_args, len + socklen); } @@ -221,7 +221,7 @@ int sys_enter_sendto(struct syscall_enter_args *args) socklen &= sizeof(augmented_args->saddr) - 1; - bpf_probe_read(&augmented_args->saddr, socklen, sockaddr_arg); + bpf_probe_read_user(&augmented_args->saddr, socklen, sockaddr_arg); return augmented__output(args, augmented_args, len + socklen); } @@ -311,7 +311,7 @@ int sys_enter_perf_event_open(struct syscall_enter_args *args) if (augmented_args == NULL) goto failure; - if (bpf_probe_read(&augmented_args->__data, sizeof(*attr), attr) < 0) + if (bpf_probe_read_user(&augmented_args->__data, sizeof(*attr), attr) < 0) goto failure; attr_read = (const struct perf_event_attr_size *)augmented_args->__data; @@ -325,7 +325,7 @@ int sys_enter_perf_event_open(struct syscall_enter_args *args) goto failure; // Now that we read attr->size and tested it against the size limits, read it completely - if (bpf_probe_read(&augmented_args->__data, size, attr) < 0) + if (bpf_probe_read_user(&augmented_args->__data, size, attr) < 0) goto failure; return augmented__output(args, augmented_args, len + size); @@ -347,7 +347,7 @@ int sys_enter_clock_nanosleep(struct syscall_enter_args *args) if (size > sizeof(augmented_args->__data)) goto failure; - bpf_probe_read(&augmented_args->__data, size, rqtp_arg); + bpf_probe_read_user(&augmented_args->__data, size, rqtp_arg); return augmented__output(args, augmented_args, len + size); failure: @@ -385,7 +385,7 @@ int sys_enter(struct syscall_enter_args *args) if (augmented_args == NULL) return 1; - bpf_probe_read(&augmented_args->args, sizeof(augmented_args->args), args); + bpf_probe_read_kernel(&augmented_args->args, sizeof(augmented_args->args), args); /* * Jump to syscall specific augmenter, even if the default one, @@ -406,7 +406,7 @@ int sys_exit(struct syscall_exit_args *args) if (pid_filter__has(&pids_filtered, getpid())) return 0; - bpf_probe_read(&exit_args, sizeof(exit_args), args); + bpf_probe_read_kernel(&exit_args, sizeof(exit_args), args); /* * Jump to syscall specific return augmenter, even if the default one, * "!raw_syscalls:unaugmented" that will just return 1 to return the From 73b25505ce043b561028e5571d84dc82aa53c2b4 Mon Sep 17 00:00:00 2001 From: Haren Myneni Date: Thu, 19 Oct 2023 14:50:33 -0700 Subject: [PATCH 331/475] powerpc/vas: Limit open window failure messages in log bufffer The VAS open window call prints error message and returns -EBUSY after the migration suspend event initiated and until the resume event completed on the destination system. It can cause the log buffer filled with these error messages if the user space issues continuous open window calls. Similar case even for DLPAR CPU remove event when no credits are available until the credits are freed or with the other DLPAR CPU add event. So changes in the patch to use pr_err_ratelimited() instead of pr_err() to display open window failure and not-available credits error messages. Use pr_fmt() and make the corresponding changes to have the consistencein prefix all pr_*() messages (vas-api.c). Fixes: 37e6764895ef ("powerpc/pseries/vas: Add VAS migration handler") Signed-off-by: Haren Myneni [mpe: Use "vas-api" as the prefix to match the file name.] Signed-off-by: Michael Ellerman Link: https://msgid.link/20231019215033.1335251-1-haren@linux.ibm.com --- arch/powerpc/platforms/book3s/vas-api.c | 34 ++++++++++++------------- arch/powerpc/platforms/pseries/vas.c | 4 +-- 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/arch/powerpc/platforms/book3s/vas-api.c b/arch/powerpc/platforms/book3s/vas-api.c index 77ea9335fd04..f381b177ea06 100644 --- a/arch/powerpc/platforms/book3s/vas-api.c +++ b/arch/powerpc/platforms/book3s/vas-api.c @@ -4,6 +4,8 @@ * Copyright (C) 2019 Haren Myneni, IBM Corp */ +#define pr_fmt(fmt) "vas-api: " fmt + #include #include #include @@ -78,7 +80,7 @@ int get_vas_user_win_ref(struct vas_user_win_ref *task_ref) task_ref->mm = get_task_mm(current); if (!task_ref->mm) { put_pid(task_ref->pid); - pr_err("VAS: pid(%d): mm_struct is not found\n", + pr_err("pid(%d): mm_struct is not found\n", current->pid); return -EPERM; } @@ -235,8 +237,7 @@ void vas_update_csb(struct coprocessor_request_block *crb, rc = kill_pid_info(SIGSEGV, &info, pid); rcu_read_unlock(); - pr_devel("%s(): pid %d kill_proc_info() rc %d\n", __func__, - pid_vnr(pid), rc); + pr_devel("pid %d kill_proc_info() rc %d\n", pid_vnr(pid), rc); } void vas_dump_crb(struct coprocessor_request_block *crb) @@ -294,7 +295,7 @@ static int coproc_ioc_tx_win_open(struct file *fp, unsigned long arg) rc = copy_from_user(&uattr, uptr, sizeof(uattr)); if (rc) { - pr_err("%s(): copy_from_user() returns %d\n", __func__, rc); + pr_err("copy_from_user() returns %d\n", rc); return -EFAULT; } @@ -311,7 +312,7 @@ static int coproc_ioc_tx_win_open(struct file *fp, unsigned long arg) txwin = cp_inst->coproc->vops->open_win(uattr.vas_id, uattr.flags, cp_inst->coproc->cop_type); if (IS_ERR(txwin)) { - pr_err("%s() VAS window open failed, %ld\n", __func__, + pr_err_ratelimited("VAS window open failed rc=%ld\n", PTR_ERR(txwin)); return PTR_ERR(txwin); } @@ -405,8 +406,7 @@ static vm_fault_t vas_mmap_fault(struct vm_fault *vmf) * window is not opened. Shouldn't expect this error. */ if (!cp_inst || !cp_inst->txwin) { - pr_err("%s(): Unexpected fault on paste address with TX window closed\n", - __func__); + pr_err("Unexpected fault on paste address with TX window closed\n"); return VM_FAULT_SIGBUS; } @@ -421,8 +421,7 @@ static vm_fault_t vas_mmap_fault(struct vm_fault *vmf) * issue NX request. */ if (txwin->task_ref.vma != vmf->vma) { - pr_err("%s(): No previous mapping with paste address\n", - __func__); + pr_err("No previous mapping with paste address\n"); return VM_FAULT_SIGBUS; } @@ -481,19 +480,19 @@ static int coproc_mmap(struct file *fp, struct vm_area_struct *vma) txwin = cp_inst->txwin; if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) { - pr_debug("%s(): size 0x%zx, PAGE_SIZE 0x%zx\n", __func__, + pr_debug("size 0x%zx, PAGE_SIZE 0x%zx\n", (vma->vm_end - vma->vm_start), PAGE_SIZE); return -EINVAL; } /* Ensure instance has an open send window */ if (!txwin) { - pr_err("%s(): No send window open?\n", __func__); + pr_err("No send window open?\n"); return -EINVAL; } if (!cp_inst->coproc->vops || !cp_inst->coproc->vops->paste_addr) { - pr_err("%s(): VAS API is not registered\n", __func__); + pr_err("VAS API is not registered\n"); return -EACCES; } @@ -510,14 +509,14 @@ static int coproc_mmap(struct file *fp, struct vm_area_struct *vma) */ mutex_lock(&txwin->task_ref.mmap_mutex); if (txwin->status != VAS_WIN_ACTIVE) { - pr_err("%s(): Window is not active\n", __func__); + pr_err("Window is not active\n"); rc = -EACCES; goto out; } paste_addr = cp_inst->coproc->vops->paste_addr(txwin); if (!paste_addr) { - pr_err("%s(): Window paste address failed\n", __func__); + pr_err("Window paste address failed\n"); rc = -EINVAL; goto out; } @@ -533,8 +532,8 @@ static int coproc_mmap(struct file *fp, struct vm_area_struct *vma) rc = remap_pfn_range(vma, vma->vm_start, pfn + vma->vm_pgoff, vma->vm_end - vma->vm_start, prot); - pr_devel("%s(): paste addr %llx at %lx, rc %d\n", __func__, - paste_addr, vma->vm_start, rc); + pr_devel("paste addr %llx at %lx, rc %d\n", paste_addr, + vma->vm_start, rc); txwin->task_ref.vma = vma; vma->vm_ops = &vas_vm_ops; @@ -609,8 +608,7 @@ int vas_register_coproc_api(struct module *mod, enum vas_cop_type cop_type, goto err; } - pr_devel("%s: Added dev [%d,%d]\n", __func__, MAJOR(devno), - MINOR(devno)); + pr_devel("Added dev [%d,%d]\n", MAJOR(devno), MINOR(devno)); return 0; diff --git a/arch/powerpc/platforms/pseries/vas.c b/arch/powerpc/platforms/pseries/vas.c index e25ac52acf50..b1f25bac280b 100644 --- a/arch/powerpc/platforms/pseries/vas.c +++ b/arch/powerpc/platforms/pseries/vas.c @@ -341,7 +341,7 @@ static struct vas_window *vas_allocate_window(int vas_id, u64 flags, if (atomic_inc_return(&cop_feat_caps->nr_used_credits) > atomic_read(&cop_feat_caps->nr_total_credits)) { - pr_err("Credits are not available to allocate window\n"); + pr_err_ratelimited("Credits are not available to allocate window\n"); rc = -EINVAL; goto out; } @@ -424,7 +424,7 @@ static struct vas_window *vas_allocate_window(int vas_id, u64 flags, put_vas_user_win_ref(&txwin->vas_win.task_ref); rc = -EBUSY; - pr_err("No credit is available to allocate window\n"); + pr_err_ratelimited("No credit is available to allocate window\n"); out_free: /* From 007240d59c11f87ac4f6cfc6a1d116630b6b634c Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 9 Mar 2023 14:48:31 +0100 Subject: [PATCH 332/475] powerpc/imc-pmu: Use the correct spinlock initializer. The macro __SPIN_LOCK_INITIALIZER() is implementation specific. Users that desire to initialize a spinlock in a struct must use __SPIN_LOCK_UNLOCKED(). Use __SPIN_LOCK_UNLOCKED() for the spinlock_t in imc_global_refc. Fixes: 76d588dddc459 ("powerpc/imc-pmu: Fix use of mutex in IRQs disabled section") Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Michael Ellerman Link: https://msgid.link/20230309134831.Nz12nqsU@linutronix.de --- arch/powerpc/perf/imc-pmu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/powerpc/perf/imc-pmu.c b/arch/powerpc/perf/imc-pmu.c index 2016aee27037..5d12ca386c1f 100644 --- a/arch/powerpc/perf/imc-pmu.c +++ b/arch/powerpc/perf/imc-pmu.c @@ -51,7 +51,7 @@ static int trace_imc_mem_size; * core and trace-imc */ static struct imc_pmu_ref imc_global_refc = { - .lock = __SPIN_LOCK_INITIALIZER(imc_global_refc.lock), + .lock = __SPIN_LOCK_UNLOCKED(imc_global_refc.lock), .id = 0, .refc = 0, }; From 95f1a128cd728a7257d78e868f1f5a145fc43736 Mon Sep 17 00:00:00 2001 From: Wang Yufen Date: Wed, 14 Dec 2022 15:46:23 +0800 Subject: [PATCH 333/475] powerpc/pseries: fix potential memory leak in init_cpu_associativity() If the vcpu_associativity alloc memory successfully but the pcpu_associativity fails to alloc memory, the vcpu_associativity memory leaks. Fixes: d62c8deeb6e6 ("powerpc/pseries: Provide vcpu dispatch statistics") Signed-off-by: Wang Yufen Reviewed-by: "Naveen N. Rao" Signed-off-by: Michael Ellerman Link: https://msgid.link/1671003983-10794-1-git-send-email-wangyufen@huawei.com --- arch/powerpc/platforms/pseries/lpar.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/powerpc/platforms/pseries/lpar.c b/arch/powerpc/platforms/pseries/lpar.c index b4e86b7ea584..4561667832ed 100644 --- a/arch/powerpc/platforms/pseries/lpar.c +++ b/arch/powerpc/platforms/pseries/lpar.c @@ -526,8 +526,10 @@ static ssize_t vcpudispatch_stats_write(struct file *file, const char __user *p, if (cmd) { rc = init_cpu_associativity(); - if (rc) + if (rc) { + destroy_cpu_associativity(); goto out; + } for_each_possible_cpu(cpu) { disp = per_cpu_ptr(&vcpu_disp_data, cpu); From 269d79fb30f6b2419b8a67d99d8bdf58ced44d72 Mon Sep 17 00:00:00 2001 From: Nick Child Date: Tue, 14 Mar 2023 11:44:42 -0500 Subject: [PATCH 334/475] powerpc/boot: Add version to install filenames Rather than replacing the versionless vmlinux and System.map files, copy to files with the version info appended. Additionally, since executing the script is a last resort option, inform the user about the missing `installkernel` command and the location of the installation. This work is adapted from `arch/s390/boot/install.sh`, and also matches the behaviour of arm, arm64 and riscv. Signed-off-by: Nick Child Signed-off-by: Michael Ellerman Link: https://msgid.link/20230314164442.124929-1-nnac123@linux.ibm.com --- arch/powerpc/boot/install.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/arch/powerpc/boot/install.sh b/arch/powerpc/boot/install.sh index 461902c8a46d..101fcb397a0f 100755 --- a/arch/powerpc/boot/install.sh +++ b/arch/powerpc/boot/install.sh @@ -21,13 +21,17 @@ set -e # this should work for both the pSeries zImage and the iSeries vmlinux.sm image_name=`basename $2` -if [ -f $4/$image_name ]; then - mv $4/$image_name $4/$image_name.old + +echo "Warning: '${INSTALLKERNEL}' command not available... Copying" \ + "directly to $4/$image_name-$1" >&2 + +if [ -f $4/$image_name-$1 ]; then + mv $4/$image_name-$1 $4/$image_name-$1.old fi -if [ -f $4/System.map ]; then - mv $4/System.map $4/System.old +if [ -f $4/System.map-$1 ]; then + mv $4/System.map-$1 $4/System-$1.old fi -cat $2 > $4/$image_name -cp $3 $4/System.map +cat $2 > $4/$image_name-$1 +cp $3 $4/System.map-$1 From 3f8b6e5b11192dacb721d2d28ea4589917f5e822 Mon Sep 17 00:00:00 2001 From: Kajol Jain Date: Mon, 16 Oct 2023 20:01:10 +0530 Subject: [PATCH 335/475] perf vendor events: Update PMC used in PM_RUN_INST_CMPL event for power10 platform The CPI_STALL_RATIO metric group can be used to present the high level CPI stall breakdown metrics in powerpc, which will show: - DISPATCH_STALL_CPI ( Dispatch stall cycles per insn ) - ISSUE_STALL_CPI ( Issue stall cycles per insn ) - EXECUTION_STALL_CPI ( Execution stall cycles per insn ) - COMPLETION_STALL_CPI ( Completion stall cycles per insn ) Commit cf26e043c2a9 ("perf vendor events power10: Add JSON metric events to present CPI stall cycles in powerpc)" which added the CPI_STALL_RATIO metric group, also modified the PMC value used in PM_RUN_INST_CMPL event from PMC4 to PMC5, to avoid multiplexing of events. But that got revert in recent changes. Fix this issue by changing back the PMC value used in PM_RUN_INST_CMPL to PMC5. Result with the fix: ./perf stat --metric-no-group -M CPI_STALL_RATIO Performance counter stats for 'workload': 68,745,426 PM_CMPL_STALL # 0.21 COMPLETION_STALL_CPI 7,692,827 PM_ISSUE_STALL # 0.02 ISSUE_STALL_CPI 322,638,223 PM_RUN_INST_CMPL # 0.05 DISPATCH_STALL_CPI # 0.48 EXECUTION_STALL_CPI 16,858,553 PM_DISP_STALL_CYC 153,880,133 PM_EXEC_STALL 0.089774592 seconds time elapsed "--metric-no-group" is used for forcing PM_RUN_INST_CMPL to be scheduled in all group for more accuracy. Fixes: 7d473f475b2a ("perf vendor events: Move JSON/events to appropriate files for power10 platform") Reported-by: Disha Goel Signed-off-by: Kajol Jain Reviewed-by: Athira Rajeev Tested-by: Disha Goel Cc: maddy@linux.ibm.com Link: https://lore.kernel.org/r/20231016143110.244255-1-kjain@linux.ibm.com Signed-off-by: Namhyung Kim --- tools/perf/pmu-events/arch/powerpc/power10/pmc.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/perf/pmu-events/arch/powerpc/power10/pmc.json b/tools/perf/pmu-events/arch/powerpc/power10/pmc.json index c606ae03cd27..0e0253d0e757 100644 --- a/tools/perf/pmu-events/arch/powerpc/power10/pmc.json +++ b/tools/perf/pmu-events/arch/powerpc/power10/pmc.json @@ -195,7 +195,7 @@ "BriefDescription": "Threshold counter exceeded a value of 128." }, { - "EventCode": "0x400FA", + "EventCode": "0x500FA", "EventName": "PM_RUN_INST_CMPL", "BriefDescription": "PowerPC instruction completed while the run latch is set." } From c4a852635eddcf4130a93e687bc7b120b19fd1b7 Mon Sep 17 00:00:00 2001 From: Yang Jihong Date: Fri, 13 Oct 2023 07:59:45 +0000 Subject: [PATCH 336/475] perf data: Increase RLIMIT_NOFILE limit when open too many files in perf_data__create_dir() If using parallel threads to collect data, perf record needs at least 6 fds per CPU. (one for sys_perf_event_open, four for pipe msg and ack of the pipe, see record__thread_data_open_pipes(), and one for open perf.data.XXX) For an environment with more than 100 cores, if perf record uses both `-a` and `--threads` options, it is easy to exceed the upper limit of the file descriptor number, when we run out of them try to increase the limits. Before: $ ulimit -n 1024 $ lscpu | grep 'On-line CPU(s)' On-line CPU(s) list: 0-159 $ perf record --threads -a sleep 1 Failed to create data directory: Too many open files After: $ ulimit -n 1024 $ lscpu | grep 'On-line CPU(s)' On-line CPU(s) list: 0-159 $ perf record --threads -a sleep 1 [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.394 MB perf.data (1576 samples) ] Signed-off-by: Yang Jihong Acked-by: Namhyung Kim Link: https://lore.kernel.org/r/20231013075945.698874-1-yangjihong1@huawei.com Signed-off-by: Namhyung Kim --- tools/perf/util/data.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tools/perf/util/data.c b/tools/perf/util/data.c index fc16299c915f..098f9e3bb2e7 100644 --- a/tools/perf/util/data.c +++ b/tools/perf/util/data.c @@ -17,6 +17,7 @@ #include "util.h" // rm_rf_perf_data() #include "debug.h" #include "header.h" +#include "evsel.h" #include static void close_dir(struct perf_data_file *files, int nr) @@ -35,6 +36,7 @@ void perf_data__close_dir(struct perf_data *data) int perf_data__create_dir(struct perf_data *data, int nr) { + enum rlimit_action set_rlimit = NO_CHANGE; struct perf_data_file *files = NULL; int i, ret; @@ -54,11 +56,21 @@ int perf_data__create_dir(struct perf_data *data, int nr) goto out_err; } +retry_open: ret = open(file->path, O_RDWR|O_CREAT|O_TRUNC, S_IRUSR|S_IWUSR); if (ret < 0) { + /* + * If using parallel threads to collect data, + * perf record needs at least 6 fds per CPU. + * When we run out of them try to increase the limits. + */ + if (errno == EMFILE && evsel__increase_rlimit(&set_rlimit)) + goto retry_open; + ret = -errno; goto out_err; } + set_rlimit = NO_CHANGE; file->fd = ret; } From d42f55e8ae741540d0d2ebab8ff02f68fb0c802f Mon Sep 17 00:00:00 2001 From: Naveen N Rao Date: Tue, 30 May 2023 15:08:21 +0530 Subject: [PATCH 337/475] powerpc/tools: Pass -mabi=elfv2 to gcc-check-mprofile-kernel.sh Toolchains don't always default to the ELFv2 ABI. This is true with at least the kernel.org toolchains. As such, pass -mabi=elfv2 explicitly to ensure that we are testing against the correct compiler output. Signed-off-by: Naveen N Rao [mpe: Tweak comment wording] Signed-off-by: Michael Ellerman Link: https://msgid.link/20230530093821.298590-1-naveen@kernel.org --- arch/powerpc/tools/gcc-check-mprofile-kernel.sh | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/arch/powerpc/tools/gcc-check-mprofile-kernel.sh b/arch/powerpc/tools/gcc-check-mprofile-kernel.sh index a31a56016c09..73e331e7660e 100755 --- a/arch/powerpc/tools/gcc-check-mprofile-kernel.sh +++ b/arch/powerpc/tools/gcc-check-mprofile-kernel.sh @@ -7,21 +7,20 @@ set -o pipefail # To debug, uncomment the following line # set -x -# -mprofile-kernel is only supported on 64-bit, so this should not be invoked -# for 32-bit. We pass in -m64 explicitly, and -mbig-endian and -mlittle-endian -# are passed in from Kconfig, which takes care of toolchains defaulting to -# other targets. +# -mprofile-kernel is only supported on 64-bit with ELFv2, so this should not +# be invoked for other targets. Therefore we can pass in -m64 and -mabi +# explicitly, to take care of toolchains defaulting to other targets. # Test whether the compile option -mprofile-kernel exists and generates # profiling code (ie. a call to _mcount()). echo "int func() { return 0; }" | \ - $* -m64 -S -x c -O2 -p -mprofile-kernel - -o - \ + $* -m64 -mabi=elfv2 -S -x c -O2 -p -mprofile-kernel - -o - \ 2> /dev/null | grep -q "_mcount" # Test whether the notrace attribute correctly suppresses calls to _mcount(). echo -e "#include \nnotrace int func() { return 0; }" | \ - $* -m64 -S -x c -O2 -p -mprofile-kernel - -o - \ + $* -m64 -mabi=elfv2 -S -x c -O2 -p -mprofile-kernel - -o - \ 2> /dev/null | grep -q "_mcount" && \ exit 1 From f01b0edd562ef5a05bae31fe3a296721f89af7d9 Mon Sep 17 00:00:00 2001 From: Naveen N Rao Date: Wed, 14 Jun 2023 14:29:26 +0530 Subject: [PATCH 338/475] powerpc/trace: Add support for HAVE_FUNCTION_ARG_ACCESS_API When creating a kprobe on function entry through tracefs, enable arguments to be recorded to be specified using $argN syntax. Signed-off-by: Naveen N Rao Signed-off-by: Michael Ellerman Link: https://msgid.link/20230614085926.2176641-1-naveen@kernel.org --- arch/powerpc/Kconfig | 1 + arch/powerpc/include/asm/ptrace.h | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index d5d5388973ac..6f105ee4f3cf 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -237,6 +237,7 @@ config PPC select HAVE_EFFICIENT_UNALIGNED_ACCESS select HAVE_FAST_GUP select HAVE_FTRACE_MCOUNT_RECORD + select HAVE_FUNCTION_ARG_ACCESS_API select HAVE_FUNCTION_DESCRIPTORS if PPC64_ELF_ABI_V1 select HAVE_FUNCTION_ERROR_INJECTION select HAVE_FUNCTION_GRAPH_TRACER diff --git a/arch/powerpc/include/asm/ptrace.h b/arch/powerpc/include/asm/ptrace.h index 9db8b16567e2..ea8f91fbc62f 100644 --- a/arch/powerpc/include/asm/ptrace.h +++ b/arch/powerpc/include/asm/ptrace.h @@ -397,6 +397,23 @@ static inline unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, return 0; } +/** + * regs_get_kernel_argument() - get Nth function argument in kernel + * @regs: pt_regs of that context + * @n: function argument number (start from 0) + * + * We support up to 8 arguments and assume they are sent in through the GPRs. + * This will fail for fp/vector arguments, but those aren't usually found in + * kernel code. This is expected to be called from kprobes or ftrace with regs. + */ +static inline unsigned long regs_get_kernel_argument(struct pt_regs *regs, unsigned int n) +{ +#define NR_REG_ARGUMENTS 8 + if (n < NR_REG_ARGUMENTS) + return regs_get_register(regs, offsetof(struct pt_regs, gpr[3 + n])); + return 0; +} + #endif /* __ASSEMBLY__ */ #ifndef __powerpc64__ From ea142e590aec55ba40c5affb4d49e68c713c63dc Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Thu, 19 Oct 2023 01:34:23 +1000 Subject: [PATCH 339/475] powerpc/perf: Fix disabling BHRB and instruction sampling When the PMU is disabled, MMCRA is not updated to disable BHRB and instruction sampling. This can lead to those features remaining enabled, which can slow down a real or emulated CPU. Fixes: 1cade527f6e9 ("powerpc/perf: BHRB control to disable BHRB logic when not used") Cc: stable@vger.kernel.org # v5.9+ Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman Link: https://msgid.link/20231018153423.298373-1-npiggin@gmail.com --- arch/powerpc/perf/core-book3s.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/arch/powerpc/perf/core-book3s.c b/arch/powerpc/perf/core-book3s.c index 8c1f7def596e..10b946e9c6e7 100644 --- a/arch/powerpc/perf/core-book3s.c +++ b/arch/powerpc/perf/core-book3s.c @@ -1371,8 +1371,7 @@ static void power_pmu_disable(struct pmu *pmu) /* * Disable instruction sampling if it was enabled */ - if (cpuhw->mmcr.mmcra & MMCRA_SAMPLE_ENABLE) - val &= ~MMCRA_SAMPLE_ENABLE; + val &= ~MMCRA_SAMPLE_ENABLE; /* Disable BHRB via mmcra (BHRBRD) for p10 */ if (ppmu->flags & PPMU_ARCH_31) @@ -1383,7 +1382,7 @@ static void power_pmu_disable(struct pmu *pmu) * instruction sampling or BHRB. */ if (val != mmcra) { - mtspr(SPRN_MMCRA, mmcra); + mtspr(SPRN_MMCRA, val); mb(); isync(); } From f6568647382c667912245c8d07aa26c9c6d4d0c8 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Mon, 16 Oct 2023 22:43:01 +1000 Subject: [PATCH 340/475] powerpc/qspinlock: stop queued waiters trying to set lock sleepy If a queued waiter notices the lock owner or the previous waiter has been preempted, it attempts to mark the lock sleepy, but it does this as a try-set operation using the original lock value it got when queueing, which will become stale as the queue progresses, and the try-set will fail. Drop this and just set the sleepy seen clock. Signed-off-by: Nicholas Piggin Tested-by: Shrikanth Hegde Reviewed-by: "Nysal Jan K.A" Signed-off-by: Michael Ellerman Link: https://msgid.link/20231016124305.139923-3-npiggin@gmail.com --- arch/powerpc/lib/qspinlock.c | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/arch/powerpc/lib/qspinlock.c b/arch/powerpc/lib/qspinlock.c index 6dd2f46bd3ef..75608ced14c2 100644 --- a/arch/powerpc/lib/qspinlock.c +++ b/arch/powerpc/lib/qspinlock.c @@ -247,22 +247,18 @@ static __always_inline void seen_sleepy_lock(void) this_cpu_write(sleepy_lock_seen_clock, sched_clock()); } -static __always_inline void seen_sleepy_node(struct qspinlock *lock, u32 val) +static __always_inline void seen_sleepy_node(void) { if (pv_sleepy_lock) { if (pv_sleepy_lock_interval_ns) this_cpu_write(sleepy_lock_seen_clock, sched_clock()); - if (val & _Q_LOCKED_VAL) { - if (!(val & _Q_SLEEPY_VAL)) - try_set_sleepy(lock, val); - } + /* Don't set sleepy because we likely have a stale val */ } } -static struct qnode *get_tail_qnode(struct qspinlock *lock, u32 val) +static struct qnode *get_tail_qnode(struct qspinlock *lock, int prev_cpu) { - int cpu = decode_tail_cpu(val); - struct qnodes *qnodesp = per_cpu_ptr(&qnodes, cpu); + struct qnodes *qnodesp = per_cpu_ptr(&qnodes, prev_cpu); int idx; /* @@ -381,9 +377,8 @@ static __always_inline void propagate_yield_cpu(struct qnode *node, u32 val, int } /* Called inside spin_begin() */ -static __always_inline bool yield_to_prev(struct qspinlock *lock, struct qnode *node, u32 val, bool paravirt) +static __always_inline bool yield_to_prev(struct qspinlock *lock, struct qnode *node, int prev_cpu, bool paravirt) { - int prev_cpu = decode_tail_cpu(val); u32 yield_count; int yield_cpu; bool preempted = false; @@ -412,7 +407,7 @@ static __always_inline bool yield_to_prev(struct qspinlock *lock, struct qnode * spin_end(); preempted = true; - seen_sleepy_node(lock, val); + seen_sleepy_node(); smp_rmb(); @@ -436,7 +431,7 @@ yield_prev: spin_end(); preempted = true; - seen_sleepy_node(lock, val); + seen_sleepy_node(); smp_rmb(); /* See __yield_to_locked_owner comment */ @@ -587,7 +582,8 @@ static __always_inline void queued_spin_lock_mcs_queue(struct qspinlock *lock, b * head of the waitqueue. */ if (old & _Q_TAIL_CPU_MASK) { - struct qnode *prev = get_tail_qnode(lock, old); + int prev_cpu = decode_tail_cpu(old); + struct qnode *prev = get_tail_qnode(lock, prev_cpu); /* Link @node into the waitqueue. */ WRITE_ONCE(prev->next, node); @@ -597,7 +593,7 @@ static __always_inline void queued_spin_lock_mcs_queue(struct qspinlock *lock, b while (!READ_ONCE(node->locked)) { spec_barrier(); - if (yield_to_prev(lock, node, old, paravirt)) + if (yield_to_prev(lock, node, prev_cpu, paravirt)) seen_preempted = true; } spec_barrier(); From fd8fae50c9c6117c4e05954deab2cc515508666b Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Mon, 16 Oct 2023 22:43:02 +1000 Subject: [PATCH 341/475] powerpc/qspinlock: propagate owner preemptedness rather than CPU number Rather than propagating the CPU number of the preempted lock owner, just propagate whether the owner was preempted. Waiters must read the lock value when yielding to it to prevent races anyway, so might as well always load the owner CPU from the lock. To further simplify the code, also don't propagate the -1 (or sleepy=false in the new scheme) down the queue. Instead, have the waiters clear it themselves when finding the lock owner is not preempted. Signed-off-by: Nicholas Piggin Tested-by: Shrikanth Hegde Reviewed-by: "Nysal Jan K.A" Signed-off-by: Michael Ellerman Link: https://msgid.link/20231016124305.139923-4-npiggin@gmail.com --- arch/powerpc/lib/qspinlock.c | 82 ++++++++++++++++-------------------- 1 file changed, 37 insertions(+), 45 deletions(-) diff --git a/arch/powerpc/lib/qspinlock.c b/arch/powerpc/lib/qspinlock.c index 75608ced14c2..0932d24a6b07 100644 --- a/arch/powerpc/lib/qspinlock.c +++ b/arch/powerpc/lib/qspinlock.c @@ -16,7 +16,8 @@ struct qnode { struct qnode *next; struct qspinlock *lock; int cpu; - int yield_cpu; + u8 sleepy; /* 1 if the previous vCPU was preempted or + * if the previous node was sleepy */ u8 locked; /* 1 if lock acquired */ }; @@ -349,7 +350,7 @@ static __always_inline bool yield_head_to_locked_owner(struct qspinlock *lock, u return __yield_to_locked_owner(lock, val, paravirt, mustq); } -static __always_inline void propagate_yield_cpu(struct qnode *node, u32 val, int *set_yield_cpu, bool paravirt) +static __always_inline void propagate_sleepy(struct qnode *node, u32 val, bool *set_sleepy, bool paravirt) { struct qnode *next; int owner; @@ -358,21 +359,17 @@ static __always_inline void propagate_yield_cpu(struct qnode *node, u32 val, int return; if (!pv_yield_propagate_owner) return; - - owner = get_owner_cpu(val); - if (*set_yield_cpu == owner) + if (*set_sleepy) return; next = READ_ONCE(node->next); if (!next) return; + owner = get_owner_cpu(val); if (vcpu_is_preempted(owner)) { - next->yield_cpu = owner; - *set_yield_cpu = owner; - } else if (*set_yield_cpu != -1) { - next->yield_cpu = owner; - *set_yield_cpu = owner; + next->sleepy = 1; + *set_sleepy = true; } } @@ -380,7 +377,6 @@ static __always_inline void propagate_yield_cpu(struct qnode *node, u32 val, int static __always_inline bool yield_to_prev(struct qspinlock *lock, struct qnode *node, int prev_cpu, bool paravirt) { u32 yield_count; - int yield_cpu; bool preempted = false; if (!paravirt) @@ -389,37 +385,33 @@ static __always_inline bool yield_to_prev(struct qspinlock *lock, struct qnode * if (!pv_yield_propagate_owner) goto yield_prev; - yield_cpu = READ_ONCE(node->yield_cpu); - if (yield_cpu == -1) { - /* Propagate back the -1 CPU */ - if (node->next && node->next->yield_cpu != -1) - node->next->yield_cpu = yield_cpu; + if (!READ_ONCE(node->sleepy)) { + /* Propagate back sleepy==false */ + if (node->next && node->next->sleepy) + node->next->sleepy = 0; goto yield_prev; + } else { + u32 val = READ_ONCE(lock->val); + + if (val & _Q_LOCKED_VAL) { + if (node->next && !node->next->sleepy) { + /* + * Propagate sleepy to next waiter. Only if + * owner is preempted, which allows the queue + * to become "non-sleepy" if vCPU preemption + * ceases to occur, even if the lock remains + * highly contended. + */ + if (vcpu_is_preempted(get_owner_cpu(val))) + node->next->sleepy = 1; + } + + preempted = yield_to_locked_owner(lock, val, paravirt); + if (preempted) + return preempted; + } } - yield_count = yield_count_of(yield_cpu); - if ((yield_count & 1) == 0) - goto yield_prev; /* owner vcpu is running */ - - if (get_owner_cpu(READ_ONCE(lock->val)) != yield_cpu) - goto yield_prev; /* re-sample lock owner */ - - spin_end(); - - preempted = true; - seen_sleepy_node(); - - smp_rmb(); - - if (yield_cpu == node->yield_cpu) { - if (node->next && node->next->yield_cpu != yield_cpu) - node->next->yield_cpu = yield_cpu; - yield_to_preempted(yield_cpu, yield_count); - spin_begin(); - return preempted; - } - spin_begin(); - yield_prev: if (!pv_yield_prev) goto relax; @@ -541,7 +533,7 @@ static __always_inline void queued_spin_lock_mcs_queue(struct qspinlock *lock, b bool sleepy = false; bool mustq = false; int idx; - int set_yield_cpu = -1; + bool set_sleepy = false; int iters = 0; BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS)); @@ -565,7 +557,7 @@ static __always_inline void queued_spin_lock_mcs_queue(struct qspinlock *lock, b node->next = NULL; node->lock = lock; node->cpu = smp_processor_id(); - node->yield_cpu = -1; + node->sleepy = 0; node->locked = 0; tail = encode_tail_cpu(node->cpu); @@ -599,9 +591,9 @@ static __always_inline void queued_spin_lock_mcs_queue(struct qspinlock *lock, b spec_barrier(); spin_end(); - /* Clear out stale propagated yield_cpu */ - if (paravirt && pv_yield_propagate_owner && node->yield_cpu != -1) - node->yield_cpu = -1; + /* Clear out stale propagated sleepy */ + if (paravirt && pv_yield_propagate_owner && node->sleepy) + node->sleepy = 0; smp_rmb(); /* acquire barrier for the mcs lock */ @@ -644,7 +636,7 @@ again: } } - propagate_yield_cpu(node, val, &set_yield_cpu, paravirt); + propagate_sleepy(node, val, &set_sleepy, paravirt); preempted = yield_head_to_locked_owner(lock, val, paravirt); if (!maybe_stealers) continue; From fcf77d44274b96a55cc74043561a4b3151b9ad24 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Mon, 16 Oct 2023 22:43:03 +1000 Subject: [PATCH 342/475] powerpc/qspinlock: don't propagate the not-sleepy state To simplify things, don't propagate the not-sleepy condition back down the queue. Instead, have the waiters clear their own node->sleepy when finding the lock owner is not preempted. Signed-off-by: Nicholas Piggin Tested-by: Shrikanth Hegde Reviewed-by: "Nysal Jan K.A" Signed-off-by: Michael Ellerman Link: https://msgid.link/20231016124305.139923-5-npiggin@gmail.com --- arch/powerpc/lib/qspinlock.c | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/arch/powerpc/lib/qspinlock.c b/arch/powerpc/lib/qspinlock.c index 0932d24a6b07..6bb627e90a32 100644 --- a/arch/powerpc/lib/qspinlock.c +++ b/arch/powerpc/lib/qspinlock.c @@ -350,7 +350,7 @@ static __always_inline bool yield_head_to_locked_owner(struct qspinlock *lock, u return __yield_to_locked_owner(lock, val, paravirt, mustq); } -static __always_inline void propagate_sleepy(struct qnode *node, u32 val, bool *set_sleepy, bool paravirt) +static __always_inline void propagate_sleepy(struct qnode *node, u32 val, bool paravirt) { struct qnode *next; int owner; @@ -359,18 +359,17 @@ static __always_inline void propagate_sleepy(struct qnode *node, u32 val, bool * return; if (!pv_yield_propagate_owner) return; - if (*set_sleepy) - return; next = READ_ONCE(node->next); if (!next) return; + if (next->sleepy) + return; + owner = get_owner_cpu(val); - if (vcpu_is_preempted(owner)) { + if (vcpu_is_preempted(owner)) next->sleepy = 1; - *set_sleepy = true; - } } /* Called inside spin_begin() */ @@ -385,12 +384,7 @@ static __always_inline bool yield_to_prev(struct qspinlock *lock, struct qnode * if (!pv_yield_propagate_owner) goto yield_prev; - if (!READ_ONCE(node->sleepy)) { - /* Propagate back sleepy==false */ - if (node->next && node->next->sleepy) - node->next->sleepy = 0; - goto yield_prev; - } else { + if (node->sleepy) { u32 val = READ_ONCE(lock->val); if (val & _Q_LOCKED_VAL) { @@ -410,6 +404,7 @@ static __always_inline bool yield_to_prev(struct qspinlock *lock, struct qnode * if (preempted) return preempted; } + node->sleepy = false; } yield_prev: @@ -533,7 +528,6 @@ static __always_inline void queued_spin_lock_mcs_queue(struct qspinlock *lock, b bool sleepy = false; bool mustq = false; int idx; - bool set_sleepy = false; int iters = 0; BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS)); @@ -591,10 +585,6 @@ static __always_inline void queued_spin_lock_mcs_queue(struct qspinlock *lock, b spec_barrier(); spin_end(); - /* Clear out stale propagated sleepy */ - if (paravirt && pv_yield_propagate_owner && node->sleepy) - node->sleepy = 0; - smp_rmb(); /* acquire barrier for the mcs lock */ /* @@ -636,7 +626,7 @@ again: } } - propagate_sleepy(node, val, &set_sleepy, paravirt); + propagate_sleepy(node, val, paravirt); preempted = yield_head_to_locked_owner(lock, val, paravirt); if (!maybe_stealers) continue; From 1e6d5f725738fff83e1c3cbdb8547142eb8820c2 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Mon, 16 Oct 2023 22:43:04 +1000 Subject: [PATCH 343/475] powerpc/qspinlock: Propagate sleepy if previous waiter is preempted The sleepy (aka lock-owner-is-preempted) condition is propagated down the queue by each waiter. If a waiter becomes preempted, it can no longer propagate sleepy. To allow subsequent waiters to yield to the lock owner, also check the lock owner in this case. Signed-off-by: Nicholas Piggin Tested-by: Shrikanth Hegde Reviewed-by: "Nysal Jan K.A" Signed-off-by: Michael Ellerman Link: https://msgid.link/20231016124305.139923-6-npiggin@gmail.com --- arch/powerpc/lib/qspinlock.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/powerpc/lib/qspinlock.c b/arch/powerpc/lib/qspinlock.c index 6bb627e90a32..c68c2bd7b853 100644 --- a/arch/powerpc/lib/qspinlock.c +++ b/arch/powerpc/lib/qspinlock.c @@ -384,7 +384,11 @@ static __always_inline bool yield_to_prev(struct qspinlock *lock, struct qnode * if (!pv_yield_propagate_owner) goto yield_prev; - if (node->sleepy) { + /* + * If the previous waiter was preempted it might not be able to + * propagate sleepy to us, so check the lock in that case too. + */ + if (node->sleepy || vcpu_is_preempted(prev_cpu)) { u32 val = READ_ONCE(lock->val); if (val & _Q_LOCKED_VAL) { From b629b541702b082d161c307c9d7d9867911fc933 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Mon, 16 Oct 2023 22:43:05 +1000 Subject: [PATCH 344/475] powerpc/qspinlock: Rename yield_propagate_owner tunable Rename yield_propagate_owner to yield_sleepy_owner, which better describes what it does (what, not how). Signed-off-by: Nicholas Piggin Tested-by: Shrikanth Hegde Reviewed-by: "Nysal Jan K.A" Signed-off-by: Michael Ellerman Link: https://msgid.link/20231016124305.139923-7-npiggin@gmail.com --- arch/powerpc/lib/qspinlock.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/arch/powerpc/lib/qspinlock.c b/arch/powerpc/lib/qspinlock.c index c68c2bd7b853..5de4dd549f6e 100644 --- a/arch/powerpc/lib/qspinlock.c +++ b/arch/powerpc/lib/qspinlock.c @@ -44,7 +44,7 @@ static bool pv_sleepy_lock_sticky __read_mostly = false; static u64 pv_sleepy_lock_interval_ns __read_mostly = 0; static int pv_sleepy_lock_factor __read_mostly = 256; static bool pv_yield_prev __read_mostly = true; -static bool pv_yield_propagate_owner __read_mostly = true; +static bool pv_yield_sleepy_owner __read_mostly = true; static bool pv_prod_head __read_mostly = false; static DEFINE_PER_CPU_ALIGNED(struct qnodes, qnodes); @@ -357,7 +357,7 @@ static __always_inline void propagate_sleepy(struct qnode *node, u32 val, bool p if (!paravirt) return; - if (!pv_yield_propagate_owner) + if (!pv_yield_sleepy_owner) return; next = READ_ONCE(node->next); @@ -381,7 +381,7 @@ static __always_inline bool yield_to_prev(struct qspinlock *lock, struct qnode * if (!paravirt) goto relax; - if (!pv_yield_propagate_owner) + if (!pv_yield_sleepy_owner) goto yield_prev; /* @@ -934,21 +934,21 @@ static int pv_yield_prev_get(void *data, u64 *val) DEFINE_SIMPLE_ATTRIBUTE(fops_pv_yield_prev, pv_yield_prev_get, pv_yield_prev_set, "%llu\n"); -static int pv_yield_propagate_owner_set(void *data, u64 val) +static int pv_yield_sleepy_owner_set(void *data, u64 val) { - pv_yield_propagate_owner = !!val; + pv_yield_sleepy_owner = !!val; return 0; } -static int pv_yield_propagate_owner_get(void *data, u64 *val) +static int pv_yield_sleepy_owner_get(void *data, u64 *val) { - *val = pv_yield_propagate_owner; + *val = pv_yield_sleepy_owner; return 0; } -DEFINE_SIMPLE_ATTRIBUTE(fops_pv_yield_propagate_owner, pv_yield_propagate_owner_get, pv_yield_propagate_owner_set, "%llu\n"); +DEFINE_SIMPLE_ATTRIBUTE(fops_pv_yield_sleepy_owner, pv_yield_sleepy_owner_get, pv_yield_sleepy_owner_set, "%llu\n"); static int pv_prod_head_set(void *data, u64 val) { @@ -980,7 +980,7 @@ static __init int spinlock_debugfs_init(void) debugfs_create_file("qspl_pv_sleepy_lock_interval_ns", 0600, arch_debugfs_dir, NULL, &fops_pv_sleepy_lock_interval_ns); debugfs_create_file("qspl_pv_sleepy_lock_factor", 0600, arch_debugfs_dir, NULL, &fops_pv_sleepy_lock_factor); debugfs_create_file("qspl_pv_yield_prev", 0600, arch_debugfs_dir, NULL, &fops_pv_yield_prev); - debugfs_create_file("qspl_pv_yield_propagate_owner", 0600, arch_debugfs_dir, NULL, &fops_pv_yield_propagate_owner); + debugfs_create_file("qspl_pv_yield_sleepy_owner", 0600, arch_debugfs_dir, NULL, &fops_pv_yield_sleepy_owner); debugfs_create_file("qspl_pv_prod_head", 0600, arch_debugfs_dir, NULL, &fops_pv_prod_head); } From ad496f8f83960c211b00fdbd8ae4ac1e55e4f3d5 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Wed, 11 Oct 2023 08:27:03 +0200 Subject: [PATCH 345/475] powerpc: Remove cpm_dp...() macros Since commit d3c511ac1d72 ("powerpc/cpm: Remove !CONFIG_PPC_CPM_NEW_BINDING code") cpm_dp...() macros have no added value anymore. Last user of those macros were fixed by commit 5e6cb39a256d ("net: fs_enet: Use cpm_muram_xxx() functions instead of cpm_dpxxx() macros") Remove them. Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://msgid.link/3aaa40bf706afeab8fe9a74b8437704a4269a6a2.1697005615.git.christophe.leroy@csgroup.eu --- arch/powerpc/include/asm/cpm1.h | 5 ----- arch/powerpc/include/asm/cpm2.h | 4 ---- drivers/soc/fsl/qe/qe_common.c | 4 ++-- 3 files changed, 2 insertions(+), 11 deletions(-) diff --git a/arch/powerpc/include/asm/cpm1.h b/arch/powerpc/include/asm/cpm1.h index 3bdd74739cb8..e3c6969853ef 100644 --- a/arch/powerpc/include/asm/cpm1.h +++ b/arch/powerpc/include/asm/cpm1.h @@ -49,11 +49,6 @@ */ extern cpm8xx_t __iomem *cpmp; /* Pointer to comm processor */ -#define cpm_dpalloc cpm_muram_alloc -#define cpm_dpfree cpm_muram_free -#define cpm_dpram_addr cpm_muram_addr -#define cpm_dpram_phys cpm_muram_dma - extern void cpm_setbrg(uint brg, uint rate); extern void __init cpm_load_patch(cpm8xx_t *cp); diff --git a/arch/powerpc/include/asm/cpm2.h b/arch/powerpc/include/asm/cpm2.h index 249d43cc6427..a22acc36eb9b 100644 --- a/arch/powerpc/include/asm/cpm2.h +++ b/arch/powerpc/include/asm/cpm2.h @@ -87,10 +87,6 @@ */ extern cpm_cpm2_t __iomem *cpmp; /* Pointer to comm processor */ -#define cpm_dpalloc cpm_muram_alloc -#define cpm_dpfree cpm_muram_free -#define cpm_dpram_addr cpm_muram_addr - extern void cpm2_reset(void); /* Baud rate generators. diff --git a/drivers/soc/fsl/qe/qe_common.c b/drivers/soc/fsl/qe/qe_common.c index 9729ce86db59..a877347d37d3 100644 --- a/drivers/soc/fsl/qe/qe_common.c +++ b/drivers/soc/fsl/qe/qe_common.c @@ -141,7 +141,7 @@ static s32 cpm_muram_alloc_common(unsigned long size, * * This function returns a non-negative offset into the muram area, or * a negative errno on failure. - * Use cpm_dpram_addr() to get the virtual address of the area. + * Use cpm_muram_addr() to get the virtual address of the area. * Use cpm_muram_free() to free the allocation. */ s32 cpm_muram_alloc(unsigned long size, unsigned long align) @@ -193,7 +193,7 @@ EXPORT_SYMBOL(cpm_muram_free); * @size: number of bytes to allocate * This function returns @offset if the area was available, a negative * errno otherwise. - * Use cpm_dpram_addr() to get the virtual address of the area. + * Use cpm_muram_addr() to get the virtual address of the area. * Use cpm_muram_free() to free the allocation. */ s32 cpm_muram_alloc_fixed(unsigned long offset, unsigned long size) From 89f17016a85280e1b36a6c0305e0191594cbe6ea Mon Sep 17 00:00:00 2001 From: Rob Herring Date: Fri, 6 Oct 2023 16:45:16 -0500 Subject: [PATCH 346/475] powerpc/fsl_msi: Use device_get_match_data() Use preferred device_get_match_data() instead of of_match_device() to get the driver match data. With this, adjust the includes to explicitly include the correct headers. Signed-off-by: Rob Herring Signed-off-by: Michael Ellerman Link: https://msgid.link/20231006214516.340589-1-robh@kernel.org --- arch/powerpc/sysdev/fsl_msi.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/arch/powerpc/sysdev/fsl_msi.c b/arch/powerpc/sysdev/fsl_msi.c index 57978a44d55b..558ec68d768e 100644 --- a/arch/powerpc/sysdev/fsl_msi.c +++ b/arch/powerpc/sysdev/fsl_msi.c @@ -11,9 +11,11 @@ #include #include #include +#include #include #include -#include +#include +#include #include #include #include @@ -392,7 +394,6 @@ static int fsl_msi_setup_hwirq(struct fsl_msi *msi, struct platform_device *dev, static const struct of_device_id fsl_of_msi_ids[]; static int fsl_of_msi_probe(struct platform_device *dev) { - const struct of_device_id *match; struct fsl_msi *msi; struct resource res, msiir; int err, i, j, irq_index, count; @@ -402,10 +403,7 @@ static int fsl_of_msi_probe(struct platform_device *dev) u32 offset; struct pci_controller *phb; - match = of_match_device(fsl_of_msi_ids, &dev->dev); - if (!match) - return -EINVAL; - features = match->data; + features = device_get_match_data(&dev->dev); printk(KERN_DEBUG "Setting up Freescale MSI support\n"); From 74726fda9fe306f848088ef73ec266cae0470d5b Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Sat, 7 Oct 2023 12:46:19 +0200 Subject: [PATCH 347/475] powerpc/code-patching: Perform hwsync in __patch_instruction() in case of failure Commit c28c15b6d28a ("powerpc/code-patching: Use temporary mm for Radix MMU") added a hwsync for when __patch_instruction() fails, we results in a quite odd unbalanced logic. Instead of calling mb() when __patch_instruction() returns an error, call mb() in the __patch_instruction()'s error path directly. Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman Link: https://msgid.link/e88b154eaf2efd9ff177d472d3411dcdec8ff4f5.1696675567.git.christophe.leroy@csgroup.eu --- arch/powerpc/lib/code-patching.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/arch/powerpc/lib/code-patching.c b/arch/powerpc/lib/code-patching.c index b00112d7ad46..7a47a871e6b8 100644 --- a/arch/powerpc/lib/code-patching.c +++ b/arch/powerpc/lib/code-patching.c @@ -38,6 +38,7 @@ static int __patch_instruction(u32 *exec_addr, ppc_inst_t instr, u32 *patch_addr return 0; failed: + mb(); /* sync */ return -EPERM; } @@ -309,10 +310,6 @@ static int __do_patch_instruction_mm(u32 *addr, ppc_inst_t instr) err = __patch_instruction(addr, instr, patch_addr); - /* hwsync performed by __patch_instruction (sync) if successful */ - if (err) - mb(); /* sync */ - /* context synchronisation performed by __patch_instruction (isync or exception) */ stop_using_temp_mm(patching_mm, orig_mm); From ca2b746d5f91a37f01baedff54b9315a50ee617d Mon Sep 17 00:00:00 2001 From: Minjie Du Date: Mon, 17 Jul 2023 17:26:48 +0800 Subject: [PATCH 348/475] powerpc/pseries: use kfree_sensitive() in plpks_gen_password() password might contain private information, so better use kfree_sensitive to free it. In plpks_gen_password() use kfree_sensitive(). Signed-off-by: Minjie Du Signed-off-by: Michael Ellerman Link: https://msgid.link/20230717092648.9752-1-duminjie@vivo.com --- arch/powerpc/platforms/pseries/plpks.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/powerpc/platforms/pseries/plpks.c b/arch/powerpc/platforms/pseries/plpks.c index 2d40304eb6c1..febe18f251d0 100644 --- a/arch/powerpc/platforms/pseries/plpks.c +++ b/arch/powerpc/platforms/pseries/plpks.c @@ -150,7 +150,7 @@ static int plpks_gen_password(void) ospasswordlength = maxpwsize; ospassword = kzalloc(maxpwsize, GFP_KERNEL); if (!ospassword) { - kfree(password); + kfree_sensitive(password); return -ENOMEM; } memcpy(ospassword, password, ospasswordlength); @@ -163,7 +163,7 @@ static int plpks_gen_password(void) } } out: - kfree(password); + kfree_sensitive(password); return pseries_status_to_err(rc); } From 7e8ad67c9b5c11e990c320ed7e7563f2301672a7 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Thu, 19 Oct 2023 20:41:32 -0400 Subject: [PATCH 349/475] eventfs: Fix failure path in eventfs_create_events_dir() The failure path of allocating ei goes to a path that dereferences ei. Add another label that skips over the ei dereferences to do the rest of the clean up. Link: https://lore.kernel.org/all/70e7bace-561c-95f-1117-706c2c220bc@inria.fr/ Link: https://lore.kernel.org/linux-trace-kernel/20231019204132.6662fef0@gandalf.local.home Cc: Masami Hiramatsu Cc: Mark Rutland Fixes: 5790b1fb3d67 ("eventfs: Remove eventfs_file and just use eventfs_inode") Reported-by: Julia Lawall Signed-off-by: Steven Rostedt (Google) --- fs/tracefs/event_inode.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fs/tracefs/event_inode.c b/fs/tracefs/event_inode.c index 9f19b6608954..1885f1f1f339 100644 --- a/fs/tracefs/event_inode.c +++ b/fs/tracefs/event_inode.c @@ -735,7 +735,7 @@ struct eventfs_inode *eventfs_create_events_dir(const char *name, struct dentry ei = kzalloc(sizeof(*ei), GFP_KERNEL); if (!ei) - goto fail; + goto fail_ei; inode = tracefs_get_inode(dentry->d_sb); if (unlikely(!inode)) @@ -781,6 +781,7 @@ struct eventfs_inode *eventfs_create_events_dir(const char *name, struct dentry fail: kfree(ei->d_children); kfree(ei); + fail_ei: tracefs_failed_creating(dentry); return ERR_PTR(-ENOMEM); } From 5264a2f4bb3baf712e19f1f053caaa8d7d3afa2e Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Fri, 20 Oct 2023 16:52:45 +0300 Subject: [PATCH 350/475] tracing: Fix a NULL vs IS_ERR() bug in event_subsystem_dir() The eventfs_create_dir() function returns error pointers, it never returns NULL. Update the check to reflect that. Link: https://lore.kernel.org/linux-trace-kernel/ff641474-84e2-46a7-9d7a-62b251a1050c@moroto.mountain Cc: Masami Hiramatsu Fixes: 5790b1fb3d67 ("eventfs: Remove eventfs_file and just use eventfs_inode") Signed-off-by: Dan Carpenter Signed-off-by: Steven Rostedt (Google) --- kernel/trace/trace_events.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index db46d2116500..f9e3e24d8796 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c @@ -2354,7 +2354,7 @@ event_subsystem_dir(struct trace_array *tr, const char *name, nr_entries = ARRAY_SIZE(system_entries); ei = eventfs_create_dir(name, parent, system_entries, nr_entries, dir); - if (!ei) { + if (IS_ERR(ei)) { pr_warn("Failed to create system directory %s\n", name); __put_system(system); goto out_free; From 64bf2f685c795e75dd855761c75a193ee5998731 Mon Sep 17 00:00:00 2001 From: Jiapeng Chong Date: Thu, 19 Oct 2023 11:13:53 +0800 Subject: [PATCH 351/475] tracefs/eventfs: Modify mismatched function name No functional modification involved. fs/tracefs/event_inode.c:864: warning: expecting prototype for eventfs_remove(). Prototype was for eventfs_remove_dir() instead. Link: https://lore.kernel.org/linux-trace-kernel/20231019031353.73846-1-jiapeng.chong@linux.alibaba.com Reported-by: Abaci Robot Closes: https://bugzilla.openanolis.cn/show_bug.cgi?id=6939 Signed-off-by: Jiapeng Chong Signed-off-by: Steven Rostedt (Google) --- fs/tracefs/event_inode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/tracefs/event_inode.c b/fs/tracefs/event_inode.c index 1885f1f1f339..09ab93357957 100644 --- a/fs/tracefs/event_inode.c +++ b/fs/tracefs/event_inode.c @@ -856,7 +856,7 @@ static void unhook_dentry(struct dentry **dentry, struct dentry **list) } } /** - * eventfs_remove - remove eventfs dir or file from list + * eventfs_remove_dir - remove eventfs dir or file from list * @ei: eventfs_inode to be removed. * * This function acquire the eventfs_mutex lock and call eventfs_remove_rec() From d0ed46b60396cfa7e0056f55e1ce0b43c7db57b6 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 20 Oct 2023 04:35:45 +0100 Subject: [PATCH 352/475] tracing: Move readpos from seq_buf to trace_seq To make seq_buf more lightweight as a string buf, move the readpos member from seq_buf to its container, trace_seq. That puts the responsibility of maintaining the readpos entirely in the tracing code. If some future users want to package up the readpos with a seq_buf, we can define a new struct then. Link: https://lore.kernel.org/linux-trace-kernel/20231020033545.2587554-2-willy@infradead.org Cc: Kees Cook Cc: Justin Stitt Cc: Kent Overstreet Cc: Petr Mladek Cc: Andy Shevchenko Cc: Rasmus Villemoes Cc: Sergey Senozhatsky Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Christoph Hellwig Acked-by: Greg Kroah-Hartman Signed-off-by: Steven Rostedt (Google) --- include/linux/seq_buf.h | 5 +---- include/linux/trace_seq.h | 2 ++ kernel/trace/trace.c | 10 +++++----- kernel/trace/trace_seq.c | 6 +++++- lib/seq_buf.c | 22 ++++++++++------------ 5 files changed, 23 insertions(+), 22 deletions(-) diff --git a/include/linux/seq_buf.h b/include/linux/seq_buf.h index 515d7fcb9634..a0fb013cebdf 100644 --- a/include/linux/seq_buf.h +++ b/include/linux/seq_buf.h @@ -14,19 +14,16 @@ * @buffer: pointer to the buffer * @size: size of the buffer * @len: the amount of data inside the buffer - * @readpos: The next position to read in the buffer. */ struct seq_buf { char *buffer; size_t size; size_t len; - loff_t readpos; }; static inline void seq_buf_clear(struct seq_buf *s) { s->len = 0; - s->readpos = 0; } static inline void @@ -143,7 +140,7 @@ extern __printf(2, 0) int seq_buf_vprintf(struct seq_buf *s, const char *fmt, va_list args); extern int seq_buf_print_seq(struct seq_file *m, struct seq_buf *s); extern int seq_buf_to_user(struct seq_buf *s, char __user *ubuf, - int cnt); + size_t start, int cnt); extern int seq_buf_puts(struct seq_buf *s, const char *str); extern int seq_buf_putc(struct seq_buf *s, unsigned char c); extern int seq_buf_putmem(struct seq_buf *s, const void *mem, unsigned int len); diff --git a/include/linux/trace_seq.h b/include/linux/trace_seq.h index 6be92bf559fe..3691e0e76a1a 100644 --- a/include/linux/trace_seq.h +++ b/include/linux/trace_seq.h @@ -14,6 +14,7 @@ struct trace_seq { char buffer[PAGE_SIZE]; struct seq_buf seq; + size_t readpos; int full; }; @@ -22,6 +23,7 @@ trace_seq_init(struct trace_seq *s) { seq_buf_init(&s->seq, s->buffer, PAGE_SIZE); s->full = 0; + s->readpos = 0; } /** diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 4383be8fa1b0..d629065c2383 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -1731,15 +1731,15 @@ static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt) { int len; - if (trace_seq_used(s) <= s->seq.readpos) + if (trace_seq_used(s) <= s->readpos) return -EBUSY; - len = trace_seq_used(s) - s->seq.readpos; + len = trace_seq_used(s) - s->readpos; if (cnt > len) cnt = len; - memcpy(buf, s->buffer + s->seq.readpos, cnt); + memcpy(buf, s->buffer + s->readpos, cnt); - s->seq.readpos += cnt; + s->readpos += cnt; return cnt; } @@ -7008,7 +7008,7 @@ waitagain: /* Now copy what we have to the user */ sret = trace_seq_to_user(&iter->seq, ubuf, cnt); - if (iter->seq.seq.readpos >= trace_seq_used(&iter->seq)) + if (iter->seq.readpos >= trace_seq_used(&iter->seq)) trace_seq_init(&iter->seq); /* diff --git a/kernel/trace/trace_seq.c b/kernel/trace/trace_seq.c index bac06ee3b98b..7be97229ddf8 100644 --- a/kernel/trace/trace_seq.c +++ b/kernel/trace/trace_seq.c @@ -370,8 +370,12 @@ EXPORT_SYMBOL_GPL(trace_seq_path); */ int trace_seq_to_user(struct trace_seq *s, char __user *ubuf, int cnt) { + int ret; __trace_seq_init(s); - return seq_buf_to_user(&s->seq, ubuf, cnt); + ret = seq_buf_to_user(&s->seq, ubuf, s->readpos, cnt); + if (ret > 0) + s->readpos += ret; + return ret; } EXPORT_SYMBOL_GPL(trace_seq_to_user); diff --git a/lib/seq_buf.c b/lib/seq_buf.c index 45c450f423fa..b7477aefff53 100644 --- a/lib/seq_buf.c +++ b/lib/seq_buf.c @@ -324,23 +324,24 @@ int seq_buf_path(struct seq_buf *s, const struct path *path, const char *esc) * seq_buf_to_user - copy the sequence buffer to user space * @s: seq_buf descriptor * @ubuf: The userspace memory location to copy to + * @start: The first byte in the buffer to copy * @cnt: The amount to copy * * Copies the sequence buffer into the userspace memory pointed to - * by @ubuf. It starts from the last read position (@s->readpos) - * and writes up to @cnt characters or till it reaches the end of - * the content in the buffer (@s->len), which ever comes first. + * by @ubuf. It starts from @start and writes up to @cnt characters + * or until it reaches the end of the content in the buffer (@s->len), + * whichever comes first. * * On success, it returns a positive number of the number of bytes * it copied. * * On failure it returns -EBUSY if all of the content in the * sequence has been already read, which includes nothing in the - * sequence (@s->len == @s->readpos). + * sequence (@s->len == @start). * * Returns -EFAULT if the copy to userspace fails. */ -int seq_buf_to_user(struct seq_buf *s, char __user *ubuf, int cnt) +int seq_buf_to_user(struct seq_buf *s, char __user *ubuf, size_t start, int cnt) { int len; int ret; @@ -350,20 +351,17 @@ int seq_buf_to_user(struct seq_buf *s, char __user *ubuf, int cnt) len = seq_buf_used(s); - if (len <= s->readpos) + if (len <= start) return -EBUSY; - len -= s->readpos; + len -= start; if (cnt > len) cnt = len; - ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt); + ret = copy_to_user(ubuf, s->buffer + start, cnt); if (ret == cnt) return -EFAULT; - cnt -= ret; - - s->readpos += cnt; - return cnt; + return cnt - ret; } /** From 845e31e1101fc8533be52aff42d8f1ff48636024 Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Fri, 20 Oct 2023 14:38:49 -0600 Subject: [PATCH 353/475] seq_buf: fix a misleading comment The comment for seq_buf_has_overflowed() says that an overflow condition is marked by len == size, but that's not what the code is testing. Make the comment match reality. Link: https://lkml.kernel.org/r/87pm19kp0m.fsf@meer.lwn.net Fixes: 8cd709ae7658a ("tracing: Have seq_buf use full buffer") Signed-off-by: Jonathan Corbet Signed-off-by: Steven Rostedt (Google) --- include/linux/seq_buf.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/seq_buf.h b/include/linux/seq_buf.h index a0fb013cebdf..8483e4b2d0d2 100644 --- a/include/linux/seq_buf.h +++ b/include/linux/seq_buf.h @@ -36,7 +36,7 @@ seq_buf_init(struct seq_buf *s, char *buf, unsigned int size) /* * seq_buf have a buffer that might overflow. When this happens - * the len and size are set to be equal. + * len is set to be greater than size. */ static inline bool seq_buf_has_overflowed(struct seq_buf *s) From 7060d3ccdd4cf042749664d2fe93f978a777e3e9 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Fri, 20 Oct 2023 15:12:57 +0200 Subject: [PATCH 354/475] apparmor: mark new functions static Two new functions were introduced as global functions when they are only called from inside the file that defines them and should have been static: security/apparmor/lsm.c:658:5: error: no previous prototype for 'apparmor_uring_override_creds' [-Werror=missing-prototypes] security/apparmor/lsm.c:682:5: error: no previous prototype for 'apparmor_uring_sqpoll' [-Werror=missing-prototypes] Fixes: c4371d90633b7 ("apparmor: add io_uring mediation") Signed-off-by: Arnd Bergmann Signed-off-by: John Johansen --- security/apparmor/lsm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/security/apparmor/lsm.c b/security/apparmor/lsm.c index b047d1d355a9..4d34180e9799 100644 --- a/security/apparmor/lsm.c +++ b/security/apparmor/lsm.c @@ -655,7 +655,7 @@ static int profile_uring(struct aa_profile *profile, u32 request, * Check to see if the current task is allowed to override it's credentials * to service an io_uring operation. */ -int apparmor_uring_override_creds(const struct cred *new) +static int apparmor_uring_override_creds(const struct cred *new) { struct aa_profile *profile; struct aa_label *label; @@ -679,7 +679,7 @@ int apparmor_uring_override_creds(const struct cred *new) * Check to see if the current task is allowed to create a new io_uring * kernel polling thread. */ -int apparmor_uring_sqpoll(void) +static int apparmor_uring_sqpoll(void) { struct aa_profile *profile; struct aa_label *label; From 6a81051398bc3fa84199fc701b94aadc37da1684 Mon Sep 17 00:00:00 2001 From: Yang Li Date: Mon, 23 Oct 2023 08:55:17 +0800 Subject: [PATCH 355/475] apparmor: Fix some kernel-doc comments Fix some kernel-doc comments to silence the warnings: security/apparmor/capability.c:66: warning: Function parameter or member 'ad' not described in 'audit_caps' security/apparmor/capability.c:66: warning: Excess function parameter 'as' description in 'audit_caps' security/apparmor/capability.c:154: warning: Function parameter or member 'subj_cred' not described in 'aa_capable' security/apparmor/capability.c:154: warning: Excess function parameter 'subj_cread' description in 'aa_capable' Reported-by: Abaci Robot Closes: https://bugzilla.openanolis.cn/show_bug.cgi?id=7035 Signed-off-by: Yang Li Signed-off-by: John Johansen --- security/apparmor/capability.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/security/apparmor/capability.c b/security/apparmor/capability.c index 0b7d2b1086c9..9934df16c843 100644 --- a/security/apparmor/capability.c +++ b/security/apparmor/capability.c @@ -51,7 +51,7 @@ static void audit_cb(struct audit_buffer *ab, void *va) /** * audit_caps - audit a capability - * @as: audit data + * @ad: audit data * @profile: profile being tested for confinement (NOT NULL) * @cap: capability tested * @error: error code returned by test @@ -140,7 +140,7 @@ static int profile_capable(struct aa_profile *profile, int cap, /** * aa_capable - test permission to use capability - * @subj_cread: cred we are testing capability against + * @subj_cred: cred we are testing capability against * @label: label being tested for capability (NOT NULL) * @cap: capability to be tested * @opts: CAP_OPT_NOAUDIT bit determines whether audit record is generated From cd269ca9a7b90e8b7b11eec09f1caab622338840 Mon Sep 17 00:00:00 2001 From: Yang Li Date: Mon, 23 Oct 2023 09:11:23 +0800 Subject: [PATCH 356/475] apparmor: Fix one kernel-doc comment Fix one kernel-doc comment to silence the warnings: security/apparmor/domain.c:46: warning: Function parameter or member 'to_cred' not described in 'may_change_ptraced_domain' security/apparmor/domain.c:46: warning: Excess function parameter 'cred' description in 'may_change_ptraced_domain' Reported-by: Abaci Robot Closes: https://bugzilla.openanolis.cn/show_bug.cgi?id=7036 Signed-off-by: Yang Li Signed-off-by: John Johansen --- security/apparmor/domain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/security/apparmor/domain.c b/security/apparmor/domain.c index ed4a13d44894..89fbeab4b33b 100644 --- a/security/apparmor/domain.c +++ b/security/apparmor/domain.c @@ -31,7 +31,7 @@ /** * may_change_ptraced_domain - check if can change profile on ptraced task - * @cred: cred of task changing domain + * @to_cred: cred of task changing domain * @to_label: profile to change to (NOT NULL) * @info: message if there is an error * From 6cede10161be00d129a24e8b84c2674785a32cf8 Mon Sep 17 00:00:00 2001 From: Yang Li Date: Mon, 23 Oct 2023 09:11:24 +0800 Subject: [PATCH 357/475] apparmor: Fix some kernel-doc comments Fix some kernel-doc comments to silence the warnings: security/apparmor/policy.c:117: warning: Function parameter or member 'kref' not described in 'aa_pdb_free_kref' security/apparmor/policy.c:117: warning: Excess function parameter 'kr' description in 'aa_pdb_free_kref' security/apparmor/policy.c:882: warning: Function parameter or member 'subj_cred' not described in 'aa_may_manage_policy' Reported-by: Abaci Robot Closes: https://bugzilla.openanolis.cn/show_bug.cgi?id=7037 Signed-off-by: Yang Li Signed-off-by: John Johansen --- security/apparmor/policy.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/security/apparmor/policy.c b/security/apparmor/policy.c index a441d96adcbf..ed4c9803c8fa 100644 --- a/security/apparmor/policy.c +++ b/security/apparmor/policy.c @@ -111,7 +111,7 @@ static void aa_free_pdb(struct aa_policydb *policy) /** * aa_pdb_free_kref - free aa_policydb by kref (called by aa_put_pdb) - * @kr: kref callback for freeing of a dfa (NOT NULL) + * @kref: kref callback for freeing of a dfa (NOT NULL) */ void aa_pdb_free_kref(struct kref *kref) { @@ -870,7 +870,7 @@ bool aa_current_policy_admin_capable(struct aa_ns *ns) /** * aa_may_manage_policy - can the current task manage policy - * @subj_cred; subjects cred + * @subj_cred: subjects cred * @label: label to check if it can manage policy * @ns: namespace being managed by @label (may be NULL if @label's ns) * @mask: contains the policy manipulation operation being done From aad26d3b6af13c055b1d05dd253d2484551bde55 Mon Sep 17 00:00:00 2001 From: Michael Ellerman Date: Mon, 23 Oct 2023 20:23:19 +1100 Subject: [PATCH 358/475] powerpc/32s: Implement local_flush_tlb_page_psize() There's a single call to local_flush_tlb_page_psize() in the code patching code. That call is never executed on 32-bit Book3S, because it's guarded by mm_patch_enabled() which is essentially a radix_enabled() check, which is always false on 32s. However depending on how the optimiser sees things it may still trip over the BUILD_BUG() in the 32s stub of local_flush_tlb_page_psize(). To avoid that, implement it in terms of flush_range() so that if it ever becomes called it should function, even if not optimally. Note that flush_range() deals with page aligning the address and so on, and that 32s doesn't support huge pages so there should be no issue with non-standard page sizes needing to be flushed. Signed-off-by: Michael Ellerman Link: https://msgid.link/20231023092319.1507325-1-mpe@ellerman.id.au --- arch/powerpc/include/asm/book3s/32/tlbflush.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/powerpc/include/asm/book3s/32/tlbflush.h b/arch/powerpc/include/asm/book3s/32/tlbflush.h index 4be572908124..e43534da5207 100644 --- a/arch/powerpc/include/asm/book3s/32/tlbflush.h +++ b/arch/powerpc/include/asm/book3s/32/tlbflush.h @@ -80,7 +80,7 @@ static inline void local_flush_tlb_page(struct vm_area_struct *vma, static inline void local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr, int psize) { - BUILD_BUG(); + flush_range(mm, vmaddr, vmaddr); } static inline void local_flush_tlb_mm(struct mm_struct *mm) From 465cabc97b42405eb89380ea6ba8d8b03e4ae1a2 Mon Sep 17 00:00:00 2001 From: Hari Bathini Date: Fri, 20 Oct 2023 19:43:54 +0530 Subject: [PATCH 359/475] powerpc/code-patching: introduce patch_instructions() patch_instruction() entails setting up pte, patching the instruction, clearing the pte and flushing the tlb. If multiple instructions need to be patched, every instruction would have to go through the above drill unnecessarily. Instead, introduce patch_instructions() function that sets up the pte, clears the pte and flushes the tlb only once per page range of instructions to be patched. Duplicate most of the patch_instruction() code instead of merging with it, to avoid the performance degradation observed on ppc32, for patch_instruction(), with the code path merged. Also, setup poking_init() always as BPF expects poking_init() to be setup even when STRICT_KERNEL_RWX is off. Signed-off-by: Hari Bathini Acked-by: Song Liu Signed-off-by: Michael Ellerman Link: https://msgid.link/20231020141358.643575-2-hbathini@linux.ibm.com --- arch/powerpc/include/asm/code-patching.h | 1 + arch/powerpc/lib/code-patching.c | 141 ++++++++++++++++++++++- 2 files changed, 139 insertions(+), 3 deletions(-) diff --git a/arch/powerpc/include/asm/code-patching.h b/arch/powerpc/include/asm/code-patching.h index 3f881548fb61..0e29ccf903d0 100644 --- a/arch/powerpc/include/asm/code-patching.h +++ b/arch/powerpc/include/asm/code-patching.h @@ -74,6 +74,7 @@ int create_cond_branch(ppc_inst_t *instr, const u32 *addr, int patch_branch(u32 *addr, unsigned long target, int flags); int patch_instruction(u32 *addr, ppc_inst_t instr); int raw_patch_instruction(u32 *addr, ppc_inst_t instr); +int patch_instructions(u32 *addr, u32 *code, size_t len, bool repeat_instr); static inline unsigned long patch_site_addr(s32 *site) { diff --git a/arch/powerpc/lib/code-patching.c b/arch/powerpc/lib/code-patching.c index 7a47a871e6b8..c6ab46156cda 100644 --- a/arch/powerpc/lib/code-patching.c +++ b/arch/powerpc/lib/code-patching.c @@ -205,9 +205,6 @@ void __init poking_init(void) { int ret; - if (!IS_ENABLED(CONFIG_STRICT_KERNEL_RWX)) - return; - if (mm_patch_enabled()) ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "powerpc/text_poke_mm:online", @@ -375,6 +372,144 @@ int patch_instruction(u32 *addr, ppc_inst_t instr) } NOKPROBE_SYMBOL(patch_instruction); +static int __patch_instructions(u32 *patch_addr, u32 *code, size_t len, bool repeat_instr) +{ + unsigned long start = (unsigned long)patch_addr; + + /* Repeat instruction */ + if (repeat_instr) { + ppc_inst_t instr = ppc_inst_read(code); + + if (ppc_inst_prefixed(instr)) { + u64 val = ppc_inst_as_ulong(instr); + + memset64((u64 *)patch_addr, val, len / 8); + } else { + u32 val = ppc_inst_val(instr); + + memset32(patch_addr, val, len / 4); + } + } else { + memcpy(patch_addr, code, len); + } + + smp_wmb(); /* smp write barrier */ + flush_icache_range(start, start + len); + return 0; +} + +/* + * A page is mapped and instructions that fit the page are patched. + * Assumes 'len' to be (PAGE_SIZE - offset_in_page(addr)) or below. + */ +static int __do_patch_instructions_mm(u32 *addr, u32 *code, size_t len, bool repeat_instr) +{ + struct mm_struct *patching_mm, *orig_mm; + unsigned long pfn = get_patch_pfn(addr); + unsigned long text_poke_addr; + spinlock_t *ptl; + u32 *patch_addr; + pte_t *pte; + int err; + + patching_mm = __this_cpu_read(cpu_patching_context.mm); + text_poke_addr = __this_cpu_read(cpu_patching_context.addr); + patch_addr = (u32 *)(text_poke_addr + offset_in_page(addr)); + + pte = get_locked_pte(patching_mm, text_poke_addr, &ptl); + if (!pte) + return -ENOMEM; + + __set_pte_at(patching_mm, text_poke_addr, pte, pfn_pte(pfn, PAGE_KERNEL), 0); + + /* order PTE update before use, also serves as the hwsync */ + asm volatile("ptesync" ::: "memory"); + + /* order context switch after arbitrary prior code */ + isync(); + + orig_mm = start_using_temp_mm(patching_mm); + + err = __patch_instructions(patch_addr, code, len, repeat_instr); + + /* context synchronisation performed by __patch_instructions */ + stop_using_temp_mm(patching_mm, orig_mm); + + pte_clear(patching_mm, text_poke_addr, pte); + /* + * ptesync to order PTE update before TLB invalidation done + * by radix__local_flush_tlb_page_psize (in _tlbiel_va) + */ + local_flush_tlb_page_psize(patching_mm, text_poke_addr, mmu_virtual_psize); + + pte_unmap_unlock(pte, ptl); + + return err; +} + +/* + * A page is mapped and instructions that fit the page are patched. + * Assumes 'len' to be (PAGE_SIZE - offset_in_page(addr)) or below. + */ +static int __do_patch_instructions(u32 *addr, u32 *code, size_t len, bool repeat_instr) +{ + unsigned long pfn = get_patch_pfn(addr); + unsigned long text_poke_addr; + u32 *patch_addr; + pte_t *pte; + int err; + + text_poke_addr = (unsigned long)__this_cpu_read(cpu_patching_context.addr) & PAGE_MASK; + patch_addr = (u32 *)(text_poke_addr + offset_in_page(addr)); + + pte = __this_cpu_read(cpu_patching_context.pte); + __set_pte_at(&init_mm, text_poke_addr, pte, pfn_pte(pfn, PAGE_KERNEL), 0); + /* See ptesync comment in radix__set_pte_at() */ + if (radix_enabled()) + asm volatile("ptesync" ::: "memory"); + + err = __patch_instructions(patch_addr, code, len, repeat_instr); + + pte_clear(&init_mm, text_poke_addr, pte); + flush_tlb_kernel_range(text_poke_addr, text_poke_addr + PAGE_SIZE); + + return err; +} + +/* + * Patch 'addr' with 'len' bytes of instructions from 'code'. + * + * If repeat_instr is true, the same instruction is filled for + * 'len' bytes. + */ +int patch_instructions(u32 *addr, u32 *code, size_t len, bool repeat_instr) +{ + while (len > 0) { + unsigned long flags; + size_t plen; + int err; + + plen = min_t(size_t, PAGE_SIZE - offset_in_page(addr), len); + + local_irq_save(flags); + if (mm_patch_enabled()) + err = __do_patch_instructions_mm(addr, code, plen, repeat_instr); + else + err = __do_patch_instructions(addr, code, plen, repeat_instr); + local_irq_restore(flags); + if (err) + return err; + + len -= plen; + addr = (u32 *)((unsigned long)addr + plen); + if (!repeat_instr) + code = (u32 *)((unsigned long)code + plen); + } + + return 0; +} +NOKPROBE_SYMBOL(patch_instructions); + int patch_branch(u32 *addr, unsigned long target, int flags) { ppc_inst_t instr; From 6efc1675acb88eef45ef0156b93f95d66a8ee759 Mon Sep 17 00:00:00 2001 From: Hari Bathini Date: Fri, 20 Oct 2023 19:43:55 +0530 Subject: [PATCH 360/475] powerpc/bpf: implement bpf_arch_text_copy bpf_arch_text_copy is used to dump JITed binary to RX page, allowing multiple BPF programs to share the same page. Use the newly introduced patch_instructions() to implement it. Signed-off-by: Hari Bathini Acked-by: Song Liu Signed-off-by: Michael Ellerman Link: https://msgid.link/20231020141358.643575-3-hbathini@linux.ibm.com --- arch/powerpc/net/bpf_jit_comp.c | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index 471e37cf8e2a..c21cb88b09e4 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -13,9 +13,13 @@ #include #include #include -#include +#include +#include #include +#include +#include + #include "bpf_jit.h" static void bpf_jit_fill_ill_insns(void *area, unsigned int size) @@ -274,3 +278,17 @@ int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, int pass, struct code ctx->exentry_idx++; return 0; } + +void *bpf_arch_text_copy(void *dst, void *src, size_t len) +{ + int err; + + if (WARN_ON_ONCE(core_kernel_text((unsigned long)dst))) + return ERR_PTR(-EINVAL); + + mutex_lock(&text_mutex); + err = patch_instructions(dst, src, len, false); + mutex_unlock(&text_mutex); + + return err ? ERR_PTR(err) : dst; +} From 033ffaf0af1f974ecf401db3f70aae6fe1a90fc5 Mon Sep 17 00:00:00 2001 From: Hari Bathini Date: Fri, 20 Oct 2023 19:43:56 +0530 Subject: [PATCH 361/475] powerpc/bpf: implement bpf_arch_text_invalidate for bpf_prog_pack Implement bpf_arch_text_invalidate and use it to fill unused part of the bpf_prog_pack with trap instructions when a BPF program is freed. Signed-off-by: Hari Bathini Acked-by: Song Liu Signed-off-by: Michael Ellerman Link: https://msgid.link/20231020141358.643575-4-hbathini@linux.ibm.com --- arch/powerpc/net/bpf_jit_comp.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index c21cb88b09e4..0a5a9758903a 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -292,3 +292,18 @@ void *bpf_arch_text_copy(void *dst, void *src, size_t len) return err ? ERR_PTR(err) : dst; } + +int bpf_arch_text_invalidate(void *dst, size_t len) +{ + u32 insn = BREAKPOINT_INSTRUCTION; + int ret; + + if (WARN_ON_ONCE(core_kernel_text((unsigned long)dst))) + return -EINVAL; + + mutex_lock(&text_mutex); + ret = patch_instructions(dst, &insn, len, true); + mutex_unlock(&text_mutex); + + return ret; +} From de04e40600ae15fa5e484be242e74aad6de7418f Mon Sep 17 00:00:00 2001 From: Hari Bathini Date: Fri, 20 Oct 2023 19:43:57 +0530 Subject: [PATCH 362/475] powerpc/bpf: rename powerpc64_jit_data to powerpc_jit_data powerpc64_jit_data is a misnomer as it is meant for both ppc32 and ppc64. Rename it to powerpc_jit_data. Signed-off-by: Hari Bathini Acked-by: Song Liu Signed-off-by: Michael Ellerman Link: https://msgid.link/20231020141358.643575-5-hbathini@linux.ibm.com --- arch/powerpc/net/bpf_jit_comp.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index 0a5a9758903a..7a01bbc2513a 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -43,7 +43,7 @@ int bpf_jit_emit_exit_insn(u32 *image, struct codegen_context *ctx, int tmp_reg, return 0; } -struct powerpc64_jit_data { +struct powerpc_jit_data { struct bpf_binary_header *header; u32 *addrs; u8 *image; @@ -63,7 +63,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) u8 *image = NULL; u32 *code_base; u32 *addrs; - struct powerpc64_jit_data *jit_data; + struct powerpc_jit_data *jit_data; struct codegen_context cgctx; int pass; int flen; From 90d862f370b6e9de1b5d607843c5a2f9823990f3 Mon Sep 17 00:00:00 2001 From: Hari Bathini Date: Fri, 20 Oct 2023 19:43:58 +0530 Subject: [PATCH 363/475] powerpc/bpf: use bpf_jit_binary_pack_[alloc|finalize|free] Use bpf_jit_binary_pack_alloc in powerpc jit. The jit engine first writes the program to the rw buffer. When the jit is done, the program is copied to the final location with bpf_jit_binary_pack_finalize. With multiple jit_subprogs, bpf_jit_free is called on some subprograms that haven't got bpf_jit_binary_pack_finalize() yet. Implement custom bpf_jit_free() like in commit 1d5f82d9dd47 ("bpf, x86: fix freeing of not-finalized bpf_prog_pack") to call bpf_jit_binary_pack_finalize(), if necessary. As bpf_flush_icache() is not needed anymore, remove it. Signed-off-by: Hari Bathini Acked-by: Song Liu Signed-off-by: Michael Ellerman Link: https://msgid.link/20231020141358.643575-6-hbathini@linux.ibm.com --- arch/powerpc/net/bpf_jit.h | 18 ++--- arch/powerpc/net/bpf_jit_comp.c | 106 ++++++++++++++++++++++-------- arch/powerpc/net/bpf_jit_comp32.c | 13 ++-- arch/powerpc/net/bpf_jit_comp64.c | 10 +-- 4 files changed, 96 insertions(+), 51 deletions(-) diff --git a/arch/powerpc/net/bpf_jit.h b/arch/powerpc/net/bpf_jit.h index 72b7bb34fade..cdea5dccaefe 100644 --- a/arch/powerpc/net/bpf_jit.h +++ b/arch/powerpc/net/bpf_jit.h @@ -36,9 +36,6 @@ EMIT(PPC_RAW_BRANCH(offset)); \ } while (0) -/* bl (unconditional 'branch' with link) */ -#define PPC_BL(dest) EMIT(PPC_RAW_BL((dest) - (unsigned long)(image + ctx->idx))) - /* "cond" here covers BO:BI fields. */ #define PPC_BCC_SHORT(cond, dest) \ do { \ @@ -147,12 +144,6 @@ struct codegen_context { #define BPF_FIXUP_LEN 2 /* Two instructions => 8 bytes */ #endif -static inline void bpf_flush_icache(void *start, void *end) -{ - smp_wmb(); /* smp write barrier */ - flush_icache_range((unsigned long)start, (unsigned long)end); -} - static inline bool bpf_is_seen_register(struct codegen_context *ctx, int i) { return ctx->seen & (1 << (31 - i)); @@ -169,16 +160,17 @@ static inline void bpf_clear_seen_register(struct codegen_context *ctx, int i) } void bpf_jit_init_reg_mapping(struct codegen_context *ctx); -int bpf_jit_emit_func_call_rel(u32 *image, struct codegen_context *ctx, u64 func); -int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *ctx, +int bpf_jit_emit_func_call_rel(u32 *image, u32 *fimage, struct codegen_context *ctx, u64 func); +int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct codegen_context *ctx, u32 *addrs, int pass, bool extra_pass); void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx); void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx); void bpf_jit_realloc_regs(struct codegen_context *ctx); int bpf_jit_emit_exit_insn(u32 *image, struct codegen_context *ctx, int tmp_reg, long exit_addr); -int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, int pass, struct codegen_context *ctx, - int insn_idx, int jmp_off, int dst_reg); +int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, u32 *fimage, int pass, + struct codegen_context *ctx, int insn_idx, + int jmp_off, int dst_reg); #endif diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index 7a01bbc2513a..0f9a21783329 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -44,9 +44,12 @@ int bpf_jit_emit_exit_insn(u32 *image, struct codegen_context *ctx, int tmp_reg, } struct powerpc_jit_data { - struct bpf_binary_header *header; + /* address of rw header */ + struct bpf_binary_header *hdr; + /* address of ro final header */ + struct bpf_binary_header *fhdr; u32 *addrs; - u8 *image; + u8 *fimage; u32 proglen; struct codegen_context ctx; }; @@ -67,11 +70,14 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) struct codegen_context cgctx; int pass; int flen; - struct bpf_binary_header *bpf_hdr; + struct bpf_binary_header *fhdr = NULL; + struct bpf_binary_header *hdr = NULL; struct bpf_prog *org_fp = fp; struct bpf_prog *tmp_fp; bool bpf_blinded = false; bool extra_pass = false; + u8 *fimage = NULL; + u32 *fcode_base; u32 extable_len; u32 fixup_len; @@ -101,9 +107,16 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) addrs = jit_data->addrs; if (addrs) { cgctx = jit_data->ctx; - image = jit_data->image; - bpf_hdr = jit_data->header; + /* + * JIT compiled to a writable location (image/code_base) first. + * It is then moved to the readonly final location (fimage/fcode_base) + * using instruction patching. + */ + fimage = jit_data->fimage; + fhdr = jit_data->fhdr; proglen = jit_data->proglen; + hdr = jit_data->hdr; + image = (void *)hdr + ((void *)fimage - (void *)fhdr); extra_pass = true; /* During extra pass, ensure index is reset before repopulating extable entries */ cgctx.exentry_idx = 0; @@ -123,7 +136,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) cgctx.stack_size = round_up(fp->aux->stack_depth, 16); /* Scouting faux-generate pass 0 */ - if (bpf_jit_build_body(fp, 0, &cgctx, addrs, 0, false)) { + if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) { /* We hit something illegal or unsupported. */ fp = org_fp; goto out_addrs; @@ -138,7 +151,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) */ if (cgctx.seen & SEEN_TAILCALL || !is_offset_in_branch_range((long)cgctx.idx * 4)) { cgctx.idx = 0; - if (bpf_jit_build_body(fp, 0, &cgctx, addrs, 0, false)) { + if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) { fp = org_fp; goto out_addrs; } @@ -160,17 +173,19 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) proglen = cgctx.idx * 4; alloclen = proglen + FUNCTION_DESCR_SIZE + fixup_len + extable_len; - bpf_hdr = bpf_jit_binary_alloc(alloclen, &image, 4, bpf_jit_fill_ill_insns); - if (!bpf_hdr) { + fhdr = bpf_jit_binary_pack_alloc(alloclen, &fimage, 4, &hdr, &image, + bpf_jit_fill_ill_insns); + if (!fhdr) { fp = org_fp; goto out_addrs; } if (extable_len) - fp->aux->extable = (void *)image + FUNCTION_DESCR_SIZE + proglen + fixup_len; + fp->aux->extable = (void *)fimage + FUNCTION_DESCR_SIZE + proglen + fixup_len; skip_init_ctx: code_base = (u32 *)(image + FUNCTION_DESCR_SIZE); + fcode_base = (u32 *)(fimage + FUNCTION_DESCR_SIZE); /* Code generation passes 1-2 */ for (pass = 1; pass < 3; pass++) { @@ -178,8 +193,10 @@ skip_init_ctx: cgctx.idx = 0; cgctx.alt_exit_addr = 0; bpf_jit_build_prologue(code_base, &cgctx); - if (bpf_jit_build_body(fp, code_base, &cgctx, addrs, pass, extra_pass)) { - bpf_jit_binary_free(bpf_hdr); + if (bpf_jit_build_body(fp, code_base, fcode_base, &cgctx, addrs, pass, + extra_pass)) { + bpf_arch_text_copy(&fhdr->size, &hdr->size, sizeof(hdr->size)); + bpf_jit_binary_pack_free(fhdr, hdr); fp = org_fp; goto out_addrs; } @@ -199,17 +216,19 @@ skip_init_ctx: #ifdef CONFIG_PPC64_ELF_ABI_V1 /* Function descriptor nastiness: Address + TOC */ - ((u64 *)image)[0] = (u64)code_base; + ((u64 *)image)[0] = (u64)fcode_base; ((u64 *)image)[1] = local_paca->kernel_toc; #endif - fp->bpf_func = (void *)image; + fp->bpf_func = (void *)fimage; fp->jited = 1; fp->jited_len = proglen + FUNCTION_DESCR_SIZE; - bpf_flush_icache(bpf_hdr, (u8 *)bpf_hdr + bpf_hdr->size); if (!fp->is_func || extra_pass) { - bpf_jit_binary_lock_ro(bpf_hdr); + if (bpf_jit_binary_pack_finalize(fp, fhdr, hdr)) { + fp = org_fp; + goto out_addrs; + } bpf_prog_fill_jited_linfo(fp, addrs); out_addrs: kfree(addrs); @@ -219,8 +238,9 @@ out_addrs: jit_data->addrs = addrs; jit_data->ctx = cgctx; jit_data->proglen = proglen; - jit_data->image = image; - jit_data->header = bpf_hdr; + jit_data->fimage = fimage; + jit_data->fhdr = fhdr; + jit_data->hdr = hdr; } out: @@ -234,12 +254,13 @@ out: * The caller should check for (BPF_MODE(code) == BPF_PROBE_MEM) before calling * this function, as this only applies to BPF_PROBE_MEM, for now. */ -int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, int pass, struct codegen_context *ctx, - int insn_idx, int jmp_off, int dst_reg) +int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, u32 *fimage, int pass, + struct codegen_context *ctx, int insn_idx, int jmp_off, + int dst_reg) { off_t offset; unsigned long pc; - struct exception_table_entry *ex; + struct exception_table_entry *ex, *ex_entry; u32 *fixup; /* Populate extable entries only in the last pass */ @@ -250,9 +271,16 @@ int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, int pass, struct code WARN_ON_ONCE(ctx->exentry_idx >= fp->aux->num_exentries)) return -EINVAL; + /* + * Program is first written to image before copying to the + * final location (fimage). Accordingly, update in the image first. + * As all offsets used are relative, copying as is to the + * final location should be alright. + */ pc = (unsigned long)&image[insn_idx]; + ex = (void *)fp->aux->extable - (void *)fimage + (void *)image; - fixup = (void *)fp->aux->extable - + fixup = (void *)ex - (fp->aux->num_exentries * BPF_FIXUP_LEN * 4) + (ctx->exentry_idx * BPF_FIXUP_LEN * 4); @@ -263,17 +291,17 @@ int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, int pass, struct code fixup[BPF_FIXUP_LEN - 1] = PPC_RAW_BRANCH((long)(pc + jmp_off) - (long)&fixup[BPF_FIXUP_LEN - 1]); - ex = &fp->aux->extable[ctx->exentry_idx]; + ex_entry = &ex[ctx->exentry_idx]; - offset = pc - (long)&ex->insn; + offset = pc - (long)&ex_entry->insn; if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN)) return -ERANGE; - ex->insn = offset; + ex_entry->insn = offset; - offset = (long)fixup - (long)&ex->fixup; + offset = (long)fixup - (long)&ex_entry->fixup; if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN)) return -ERANGE; - ex->fixup = offset; + ex_entry->fixup = offset; ctx->exentry_idx++; return 0; @@ -307,3 +335,27 @@ int bpf_arch_text_invalidate(void *dst, size_t len) return ret; } + +void bpf_jit_free(struct bpf_prog *fp) +{ + if (fp->jited) { + struct powerpc_jit_data *jit_data = fp->aux->jit_data; + struct bpf_binary_header *hdr; + + /* + * If we fail the final pass of JIT (from jit_subprogs), + * the program may not be finalized yet. Call finalize here + * before freeing it. + */ + if (jit_data) { + bpf_jit_binary_pack_finalize(fp, jit_data->fhdr, jit_data->hdr); + kvfree(jit_data->addrs); + kfree(jit_data); + } + hdr = bpf_jit_binary_pack_hdr(fp); + bpf_jit_binary_pack_free(hdr, NULL); + WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(fp)); + } + + bpf_prog_unlock_free(fp); +} diff --git a/arch/powerpc/net/bpf_jit_comp32.c b/arch/powerpc/net/bpf_jit_comp32.c index bc7f92ec7f2d..2f39c50ca729 100644 --- a/arch/powerpc/net/bpf_jit_comp32.c +++ b/arch/powerpc/net/bpf_jit_comp32.c @@ -200,12 +200,13 @@ void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx) EMIT(PPC_RAW_BLR()); } -int bpf_jit_emit_func_call_rel(u32 *image, struct codegen_context *ctx, u64 func) +/* Relative offset needs to be calculated based on final image location */ +int bpf_jit_emit_func_call_rel(u32 *image, u32 *fimage, struct codegen_context *ctx, u64 func) { - s32 rel = (s32)func - (s32)(image + ctx->idx); + s32 rel = (s32)func - (s32)(fimage + ctx->idx); if (image && rel < 0x2000000 && rel >= -0x2000000) { - PPC_BL(func); + EMIT(PPC_RAW_BL(rel)); } else { /* Load function address into r0 */ EMIT(PPC_RAW_LIS(_R0, IMM_H(func))); @@ -278,7 +279,7 @@ static int bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 o } /* Assemble the body code between the prologue & epilogue */ -int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *ctx, +int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct codegen_context *ctx, u32 *addrs, int pass, bool extra_pass) { const struct bpf_insn *insn = fp->insnsi; @@ -997,7 +998,7 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context * jmp_off += 4; } - ret = bpf_add_extable_entry(fp, image, pass, ctx, insn_idx, + ret = bpf_add_extable_entry(fp, image, fimage, pass, ctx, insn_idx, jmp_off, dst_reg); if (ret) return ret; @@ -1053,7 +1054,7 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context * EMIT(PPC_RAW_STW(bpf_to_ppc(BPF_REG_5), _R1, 12)); } - ret = bpf_jit_emit_func_call_rel(image, ctx, func_addr); + ret = bpf_jit_emit_func_call_rel(image, fimage, ctx, func_addr); if (ret) return ret; diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c index 0f8048f6dad6..79f23974a320 100644 --- a/arch/powerpc/net/bpf_jit_comp64.c +++ b/arch/powerpc/net/bpf_jit_comp64.c @@ -240,7 +240,7 @@ static int bpf_jit_emit_func_call_hlp(u32 *image, struct codegen_context *ctx, u return 0; } -int bpf_jit_emit_func_call_rel(u32 *image, struct codegen_context *ctx, u64 func) +int bpf_jit_emit_func_call_rel(u32 *image, u32 *fimage, struct codegen_context *ctx, u64 func) { unsigned int i, ctx_idx = ctx->idx; @@ -361,7 +361,7 @@ asm ( ); /* Assemble the body code between the prologue & epilogue */ -int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *ctx, +int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct codegen_context *ctx, u32 *addrs, int pass, bool extra_pass) { enum stf_barrier_type stf_barrier = stf_barrier_type_get(); @@ -940,8 +940,8 @@ emit_clear: addrs[++i] = ctx->idx * 4; if (BPF_MODE(code) == BPF_PROBE_MEM) { - ret = bpf_add_extable_entry(fp, image, pass, ctx, ctx->idx - 1, - 4, dst_reg); + ret = bpf_add_extable_entry(fp, image, fimage, pass, ctx, + ctx->idx - 1, 4, dst_reg); if (ret) return ret; } @@ -995,7 +995,7 @@ emit_clear: if (func_addr_fixed) ret = bpf_jit_emit_func_call_hlp(image, ctx, func_addr); else - ret = bpf_jit_emit_func_call_rel(image, ctx, func_addr); + ret = bpf_jit_emit_func_call_rel(image, fimage, ctx, func_addr); if (ret) return ret; From 3aad8c044297799bee37492fc9695aafd7513d0b Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:30 +0200 Subject: [PATCH 364/475] s390/mm,fault: remove and improve comments, adjust whitespace Remove wrong, outdated, and pointless comments. Adjust wording for some comments, and adjust whitespace at some places. Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 86 ++++++++++++-------------------------------- 1 file changed, 23 insertions(+), 63 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 587b6b64185f..797e6e2f1d11 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -3,7 +3,7 @@ * S390 version * Copyright IBM Corp. 1999 * Author(s): Hartmut Penner (hp@de.ibm.com) - * Ulrich Weigand (uweigand@de.ibm.com) + * Ulrich Weigand (uweigand@de.ibm.com) * * Derived from "arch/i386/mm/fault.c" * Copyright (C) 1995 Linus Torvalds @@ -45,8 +45,8 @@ #define __FAIL_ADDR_MASK -4096L /* - * Allocate private vm_fault_reason from top. Please make sure it won't - * collide with vm_fault_reason. + * Allocate private vm_fault_reason from top. + * Please make sure it won't collide with vm_fault_reason. */ #define VM_FAULT_BADCONTEXT ((__force vm_fault_t)0x80000000) #define VM_FAULT_BADMAP ((__force vm_fault_t)0x40000000) @@ -78,8 +78,8 @@ static enum fault_type get_fault_type(struct pt_regs *regs) unsigned long trans_exc_code; trans_exc_code = regs->int_parm_long & 3; + /* Primary address space */ if (likely(trans_exc_code == 0)) { - /* primary space exception */ if (user_mode(regs)) return USER_FAULT; if (!IS_ENABLED(CONFIG_PGSTE)) @@ -90,11 +90,11 @@ static enum fault_type get_fault_type(struct pt_regs *regs) } if (trans_exc_code == 2) return USER_FAULT; + /* Access register mode, not used in the kernel */ if (trans_exc_code == 1) { - /* access register mode, not used in the kernel */ return USER_FAULT; } - /* home space exception -> access via kernel ASCE */ + /* Home space -> access via kernel ASCE */ return KERNEL_FAULT; } @@ -233,10 +233,6 @@ void report_user_fault(struct pt_regs *regs, long signr, int is_mm_fault) show_regs(regs); } -/* - * Send SIGSEGV to task. This is an external routine - * to keep the stack usage of do_page_fault small. - */ static noinline void do_sigsegv(struct pt_regs *regs, int si_code) { report_user_fault(regs, SIGSEGV, 1); @@ -259,10 +255,6 @@ static noinline void do_no_context(struct pt_regs *regs, vm_fault_t fault) if (kfence_handle_page_fault(address, is_write, regs)) return; } - /* - * Oops. The kernel tried to access some bad page. We'll have to - * terminate things with extreme prejudice. - */ if (fault_type == KERNEL_FAULT) printk(KERN_ALERT "Unable to handle kernel pointer dereference" " in virtual kernel address space\n"); @@ -275,10 +267,12 @@ static noinline void do_no_context(struct pt_regs *regs, vm_fault_t fault) static noinline void do_low_address(struct pt_regs *regs) { - /* Low-address protection hit in kernel mode means - NULL pointer write access in kernel mode. */ + /* + * Low-address protection hit in kernel mode means + * NULL pointer write access in kernel mode. + */ if (regs->psw.mask & PSW_MASK_PSTATE) { - /* Low-address protection hit in user mode 'cannot happen'. */ + /* Low-address protection hit in user mode: 'cannot happen' */ die (regs, "Low-address protection"); } @@ -287,10 +281,6 @@ static noinline void do_low_address(struct pt_regs *regs) static noinline void do_sigbus(struct pt_regs *regs) { - /* - * Send a sigbus, regardless of whether we were in kernel - * or user mode. - */ force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)(regs->int_parm_long & __FAIL_ADDR_MASK)); } @@ -349,10 +339,10 @@ static noinline void do_fault_error(struct pt_regs *regs, vm_fault_t fault) * routines. * * interruption code (int_code): - * 04 Protection -> Write-Protection (suppression) - * 10 Segment translation -> Not present (nullification) - * 11 Page translation -> Not present (nullification) - * 3b Region third trans. -> Not present (nullification) + * 04 Protection -> Write-Protection (suppression) + * 10 Segment translation -> Not present (nullification) + * 11 Page translation -> Not present (nullification) + * 3b Region third trans. -> Not present (nullification) */ static inline vm_fault_t do_exception(struct pt_regs *regs, int access) { @@ -372,19 +362,11 @@ static inline vm_fault_t do_exception(struct pt_regs *regs, int access) * been nullified. Don't signal single step via SIGTRAP. */ clear_thread_flag(TIF_PER_TRAP); - if (kprobe_page_fault(regs, 14)) return 0; - mm = tsk->mm; address = get_fault_address(regs); is_write = fault_is_write(regs); - - /* - * Verify that the fault happened in user space, that - * we are not in an interrupt and that there is a - * user context. - */ fault = VM_FAULT_BADCONTEXT; type = get_fault_type(regs); switch (type) { @@ -396,7 +378,6 @@ static inline vm_fault_t do_exception(struct pt_regs *regs, int access) goto out; break; } - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); flags = FAULT_FLAG_DEFAULT; if (user_mode(regs)) @@ -431,7 +412,6 @@ static inline vm_fault_t do_exception(struct pt_regs *regs, int access) } lock_mmap: mmap_read_lock(mm); - gmap = NULL; if (IS_ENABLED(CONFIG_PGSTE) && type == GMAP_FAULT) { gmap = (struct gmap *) S390_lowcore.gmap; @@ -446,13 +426,11 @@ lock_mmap: if (gmap->pfault_enabled) flags |= FAULT_FLAG_RETRY_NOWAIT; } - retry: fault = VM_FAULT_BADMAP; vma = find_vma(mm, address); if (!vma) goto out_up; - if (unlikely(vma->vm_start > address)) { if (!(vma->vm_flags & VM_GROWSDOWN)) goto out_up; @@ -460,20 +438,9 @@ retry: if (!vma) goto out; } - - /* - * Ok, we have a good vm_area for this memory access, so - * we can handle it.. - */ fault = VM_FAULT_BADACCESS; if (unlikely(!(vma->vm_flags & access))) goto out_up; - - /* - * If for any reason at all we couldn't handle the fault, - * make sure we exit gracefully rather than endlessly redo - * the fault. - */ fault = handle_mm_fault(vma, address, flags, regs); if (fault_signal_pending(fault, regs)) { fault = VM_FAULT_SIGNAL; @@ -481,7 +448,6 @@ retry: goto out_up; goto out; } - /* The fault is fully completed (including releasing mmap lock) */ if (fault & VM_FAULT_COMPLETED) { if (gmap) { @@ -491,16 +457,14 @@ retry: fault = 0; goto out; } - if (unlikely(fault & VM_FAULT_ERROR)) goto out_up; - if (fault & VM_FAULT_RETRY) { if (IS_ENABLED(CONFIG_PGSTE) && gmap && (flags & FAULT_FLAG_RETRY_NOWAIT)) { /* - * FAULT_FLAG_RETRY_NOWAIT has been set, mmap_lock has - * not been released + * FAULT_FLAG_RETRY_NOWAIT has been set, + * mmap_lock has not been released */ current->thread.gmap_pfault = 1; fault = VM_FAULT_PFAULT; @@ -592,11 +556,10 @@ void do_secure_storage_access(struct pt_regs *regs) int rc; /* - * bit 61 tells us if the address is valid, if it's not we - * have a major problem and should stop the kernel or send a - * SIGSEGV to the process. Unfortunately bit 61 is not - * reliable without the misc UV feature so we need to check - * for that as well. + * Bit 61 indicates if the address is valid, if it is not the + * kernel should be stopped or SIGSEGV should be sent to the + * process. Bit 61 is not reliable without the misc UV feature, + * therefore this needs to be checked too. */ if (uv_has_feature(BIT_UV_FEAT_MISC) && !test_bit_inv(61, ®s->int_parm_long)) { @@ -609,14 +572,12 @@ void do_secure_storage_access(struct pt_regs *regs) send_sig(SIGSEGV, current, 0); return; } - /* - * The kernel should never run into this case and we - * have no way out of this situation. + * The kernel should never run into this case and + * there is no way out of this situation. */ panic("Unexpected PGM 0x3d with TEID bit 61=0"); } - switch (get_fault_type(regs)) { case GMAP_FAULT: mm = current->mm; @@ -674,7 +635,6 @@ void do_non_secure_storage_access(struct pt_regs *regs) WARN_ON_ONCE(1); return; } - if (gmap_convert_to_secure(gmap, gaddr) == -EINVAL) send_sig(SIGSEGV, current, 0); } From 7c915a84e5e2cacf71f7328b940aa783c1b7f2e4 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:31 +0200 Subject: [PATCH 365/475] s390/mm,fault: reverse x-mas tree coding style Have reverse x-mas tree coding style for variables everywhere. Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 797e6e2f1d11..c13999aa0526 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -346,13 +346,13 @@ static noinline void do_fault_error(struct pt_regs *regs, vm_fault_t fault) */ static inline vm_fault_t do_exception(struct pt_regs *regs, int access) { - struct gmap *gmap; - struct task_struct *tsk; - struct mm_struct *mm; struct vm_area_struct *vma; - enum fault_type type; + struct task_struct *tsk; unsigned long address; + struct mm_struct *mm; + enum fault_type type; unsigned int flags; + struct gmap *gmap; vm_fault_t fault; bool is_write; @@ -498,8 +498,8 @@ out: void do_protection_exception(struct pt_regs *regs) { unsigned long trans_exc_code; - int access; vm_fault_t fault; + int access; trans_exc_code = regs->int_parm_long; /* @@ -534,8 +534,8 @@ NOKPROBE_SYMBOL(do_protection_exception); void do_dat_exception(struct pt_regs *regs) { - int access; vm_fault_t fault; + int access; access = VM_ACCESS_FLAGS; fault = do_exception(regs, access); From 28f3e0002aeaf9b02ace41551c92dc1c04f0de6e Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:32 +0200 Subject: [PATCH 366/475] s390/mm,fault: use __ratelimit() instead of printk_ratelimit() Just like other architectures make use __ratelimit() instead of printk_ratelimit(). Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index c13999aa0526..d22a8fe5e4fc 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -218,11 +218,13 @@ int show_unhandled_signals = 1; void report_user_fault(struct pt_regs *regs, long signr, int is_mm_fault) { + static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, DEFAULT_RATELIMIT_BURST); + if ((task_pid_nr(current) > 1) && !show_unhandled_signals) return; if (!unhandled_signal(current, signr)) return; - if (!printk_ratelimit()) + if (!__ratelimit(&rs)) return; printk(KERN_ALERT "User process fault: interruption code %04x ilc:%d ", regs->int_code & 0xffff, regs->int_code >> 17); From c9b611bf6e576aa1a03ac097b74322af56cb7052 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:33 +0200 Subject: [PATCH 367/475] s390/mm,fault: use pr_warn_ratelimited() Use pr_warn_ratelimited() instead of printk_ratelimited(). checkpatch reports: WARNING: Prefer ... pr_warn_ratelimited(... to printk_ratelimited(KERN_WARNING ... + printk_ratelimited(KERN_WARNING Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index d22a8fe5e4fc..dcea87f9ef68 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -661,9 +661,8 @@ void do_secure_storage_violation(struct pt_regs *regs) * This exception is only triggered when a guest 2 is running * and can therefore never occur in kernel context. */ - printk_ratelimited(KERN_WARNING - "Secure storage violation in task: %s, pid %d\n", - current->comm, current->pid); + pr_warn_ratelimited("Secure storage violation in task: %s, pid %d\n", + current->comm, current->pid); send_sig(SIGSEGV, current, 0); } From 760f6511096c5e37e9c9cf240d1ad525259bac86 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:34 +0200 Subject: [PATCH 368/475] s390/mm,fault: use pr_warn(), pr_cont(), ... instead of open-coding Use pr_warn() and friends instead of open-coding with printk(). Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index dcea87f9ef68..2d629c1693e0 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -226,10 +226,10 @@ void report_user_fault(struct pt_regs *regs, long signr, int is_mm_fault) return; if (!__ratelimit(&rs)) return; - printk(KERN_ALERT "User process fault: interruption code %04x ilc:%d ", - regs->int_code & 0xffff, regs->int_code >> 17); + pr_alert("User process fault: interruption code %04x ilc:%d ", + regs->int_code & 0xffff, regs->int_code >> 17); print_vma_addr(KERN_CONT "in ", regs->psw.addr); - printk(KERN_CONT "\n"); + pr_cont("\n"); if (is_mm_fault) dump_fault_info(regs); show_regs(regs); @@ -258,11 +258,9 @@ static noinline void do_no_context(struct pt_regs *regs, vm_fault_t fault) return; } if (fault_type == KERNEL_FAULT) - printk(KERN_ALERT "Unable to handle kernel pointer dereference" - " in virtual kernel address space\n"); + pr_alert("Unable to handle kernel pointer dereference in virtual kernel address space\n"); else - printk(KERN_ALERT "Unable to handle kernel paging request" - " in virtual user address space\n"); + pr_alert("Unable to handle kernel paging request in virtual user address space\n"); dump_fault_info(regs); die(regs, "Oops"); } From 8dbc33dc81638cfcd8d3b4103ba315d76c3c2aff Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:35 +0200 Subject: [PATCH 369/475] s390/mm,fault: have balanced braces, remove unnecessary blanks Remove unnecessary braces and also blanks after casts. Add braces to have balanced braces where missing. Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 2d629c1693e0..e7f5330dd363 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -91,9 +91,8 @@ static enum fault_type get_fault_type(struct pt_regs *regs) if (trans_exc_code == 2) return USER_FAULT; /* Access register mode, not used in the kernel */ - if (trans_exc_code == 1) { + if (trans_exc_code == 1) return USER_FAULT; - } /* Home space -> access via kernel ASCE */ return KERNEL_FAULT; } @@ -200,7 +199,7 @@ static void dump_fault_info(struct pt_regs *regs) pr_cont("user "); break; case GMAP_FAULT: - asce = ((struct gmap *) S390_lowcore.gmap)->asce; + asce = ((struct gmap *)S390_lowcore.gmap)->asce; pr_cont("gmap "); break; case KERNEL_FAULT: @@ -273,7 +272,7 @@ static noinline void do_low_address(struct pt_regs *regs) */ if (regs->psw.mask & PSW_MASK_PSTATE) { /* Low-address protection hit in user mode: 'cannot happen' */ - die (regs, "Low-address protection"); + die(regs, "Low-address protection"); } do_no_context(regs, VM_FAULT_BADACCESS); @@ -327,8 +326,9 @@ static noinline void do_fault_error(struct pt_regs *regs, vm_fault_t fault) do_no_context(regs, fault); else do_sigbus(regs); - } else + } else { BUG(); + } break; } } @@ -414,7 +414,7 @@ lock_mmap: mmap_read_lock(mm); gmap = NULL; if (IS_ENABLED(CONFIG_PGSTE) && type == GMAP_FAULT) { - gmap = (struct gmap *) S390_lowcore.gmap; + gmap = (struct gmap *)S390_lowcore.gmap; current->thread.gmap_addr = address; current->thread.gmap_write_flag = !!(flags & FAULT_FLAG_WRITE); current->thread.gmap_int_code = regs->int_code & 0xffff; From e23c53467ba03a538f4e8f837ca80f0bb058a4ee Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:36 +0200 Subject: [PATCH 370/475] s390/mm,fault: include linux/mmu_context.h Include linux/mmu_context.h instead asm/mmu_context.h. checkpatch reports: CHECK: Consider using #include instead of +#include Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index e7f5330dd363..e457d87e53d2 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -10,6 +10,7 @@ */ #include +#include #include #include #include @@ -37,7 +38,6 @@ #include #include #include -#include #include #include #include "../kernel/entry.h" From 4df5ec98581eb2271d940975186fbe5716158491 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:37 +0200 Subject: [PATCH 371/475] s390/mm,fault: remove line break chechpatch reports: CHECK: Alignment should match open parenthesis + if (IS_ENABLED(CONFIG_PGSTE) && gmap && + (flags & FAULT_FLAG_RETRY_NOWAIT)) { Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index e457d87e53d2..9545789d34ef 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -460,8 +460,7 @@ retry: if (unlikely(fault & VM_FAULT_ERROR)) goto out_up; if (fault & VM_FAULT_RETRY) { - if (IS_ENABLED(CONFIG_PGSTE) && gmap && - (flags & FAULT_FLAG_RETRY_NOWAIT)) { + if (IS_ENABLED(CONFIG_PGSTE) && gmap && (flags & FAULT_FLAG_RETRY_NOWAIT)) { /* * FAULT_FLAG_RETRY_NOWAIT has been set, * mmap_lock has not been released From 5c845de331d980d90c6d18028942812fa491fe69 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:38 +0200 Subject: [PATCH 372/475] s390/mm,fault: remove noinline attribute from all functions Remove all noinline attribute from all functions and leave the inlining decisions up to the compiler. Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 9545789d34ef..4ab698988618 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -234,14 +234,14 @@ void report_user_fault(struct pt_regs *regs, long signr, int is_mm_fault) show_regs(regs); } -static noinline void do_sigsegv(struct pt_regs *regs, int si_code) +static void do_sigsegv(struct pt_regs *regs, int si_code) { report_user_fault(regs, SIGSEGV, 1); force_sig_fault(SIGSEGV, si_code, (void __user *)(regs->int_parm_long & __FAIL_ADDR_MASK)); } -static noinline void do_no_context(struct pt_regs *regs, vm_fault_t fault) +static void do_no_context(struct pt_regs *regs, vm_fault_t fault) { enum fault_type fault_type; unsigned long address; @@ -264,7 +264,7 @@ static noinline void do_no_context(struct pt_regs *regs, vm_fault_t fault) die(regs, "Oops"); } -static noinline void do_low_address(struct pt_regs *regs) +static void do_low_address(struct pt_regs *regs) { /* * Low-address protection hit in kernel mode means @@ -278,13 +278,13 @@ static noinline void do_low_address(struct pt_regs *regs) do_no_context(regs, VM_FAULT_BADACCESS); } -static noinline void do_sigbus(struct pt_regs *regs) +static void do_sigbus(struct pt_regs *regs) { force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)(regs->int_parm_long & __FAIL_ADDR_MASK)); } -static noinline void do_fault_error(struct pt_regs *regs, vm_fault_t fault) +static void do_fault_error(struct pt_regs *regs, vm_fault_t fault) { int si_code; From ae626f686351d220d5eab746dc3a19e90bb1de9a Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:39 +0200 Subject: [PATCH 373/475] s390/mm,fault: replace WARN_ON_ONCE() with unreachable() do_secure_storage_access() contains a switch statements which handles all possible return values from get_fault_type(). Therefore remove the pointless default case error handling and replace it with unreachable(). Reviewed-by: Janosch Frank Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 4ab698988618..dd1dca10bd6b 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -618,8 +618,7 @@ void do_secure_storage_access(struct pt_regs *regs) BUG(); break; default: - do_fault_error(regs, VM_FAULT_BADMAP); - WARN_ON_ONCE(1); + unreachable(); } } NOKPROBE_SYMBOL(do_secure_storage_access); From 9641613f48bbf8626355118de58f96eec5aebf49 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:40 +0200 Subject: [PATCH 374/475] s390/mm,fault: use get_fault_address() everywhere Use the get_fault_address() helper function instead of open-coding it at many locations. Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index dd1dca10bd6b..ba2056b35910 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -176,7 +176,7 @@ static void dump_fault_info(struct pt_regs *regs) unsigned long asce; pr_alert("Failing address: %016lx TEID: %016lx\n", - regs->int_parm_long & __FAIL_ADDR_MASK, regs->int_parm_long); + get_fault_address(regs), regs->int_parm_long); pr_alert("Fault in "); switch (regs->int_parm_long & 3) { case 3: @@ -210,7 +210,7 @@ static void dump_fault_info(struct pt_regs *regs) unreachable(); } pr_cont("ASCE.\n"); - dump_pagetable(asce, regs->int_parm_long & __FAIL_ADDR_MASK); + dump_pagetable(asce, get_fault_address(regs)); } int show_unhandled_signals = 1; @@ -237,8 +237,7 @@ void report_user_fault(struct pt_regs *regs, long signr, int is_mm_fault) static void do_sigsegv(struct pt_regs *regs, int si_code) { report_user_fault(regs, SIGSEGV, 1); - force_sig_fault(SIGSEGV, si_code, - (void __user *)(regs->int_parm_long & __FAIL_ADDR_MASK)); + force_sig_fault(SIGSEGV, si_code, (void __user *)get_fault_address(regs)); } static void do_no_context(struct pt_regs *regs, vm_fault_t fault) @@ -280,8 +279,7 @@ static void do_low_address(struct pt_regs *regs) static void do_sigbus(struct pt_regs *regs) { - force_sig_fault(SIGBUS, BUS_ADRERR, - (void __user *)(regs->int_parm_long & __FAIL_ADDR_MASK)); + force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)get_fault_address(regs)); } static void do_fault_error(struct pt_regs *regs, vm_fault_t fault) @@ -547,7 +545,7 @@ NOKPROBE_SYMBOL(do_dat_exception); void do_secure_storage_access(struct pt_regs *regs) { - unsigned long addr = regs->int_parm_long & __FAIL_ADDR_MASK; + unsigned long addr = get_fault_address(regs); struct vm_area_struct *vma; struct mm_struct *mm; struct page *page; @@ -625,8 +623,8 @@ NOKPROBE_SYMBOL(do_secure_storage_access); void do_non_secure_storage_access(struct pt_regs *regs) { - unsigned long gaddr = regs->int_parm_long & __FAIL_ADDR_MASK; struct gmap *gmap = (struct gmap *)S390_lowcore.gmap; + unsigned long gaddr = get_fault_address(regs); if (get_fault_type(regs) != GMAP_FAULT) { do_fault_error(regs, VM_FAULT_BADMAP); @@ -640,8 +638,8 @@ NOKPROBE_SYMBOL(do_non_secure_storage_access); void do_secure_storage_violation(struct pt_regs *regs) { - unsigned long gaddr = regs->int_parm_long & __FAIL_ADDR_MASK; struct gmap *gmap = (struct gmap *)S390_lowcore.gmap; + unsigned long gaddr = get_fault_address(regs); /* * If the VM has been rebooted, its address space might still contain From 4416d2ed816699b8e7491f1e8e8b15f69685af57 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:41 +0200 Subject: [PATCH 375/475] s390/mm,fault: use static key for store indication Generate slightly better code by using a static key to implement store indication. This allows to get rid of a memory access on the hot path. Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index ba2056b35910..5466a30d1ec4 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -60,12 +61,12 @@ enum fault_type { GMAP_FAULT, }; -static unsigned long store_indication __read_mostly; +static DEFINE_STATIC_KEY_FALSE(have_store_indication); static int __init fault_init(void) { if (test_facility(75)) - store_indication = 0xc00; + static_branch_enable(&have_store_indication); return 0; } early_initcall(fault_init); @@ -104,11 +105,13 @@ static unsigned long get_fault_address(struct pt_regs *regs) return trans_exc_code & __FAIL_ADDR_MASK; } -static bool fault_is_write(struct pt_regs *regs) +static __always_inline bool fault_is_write(struct pt_regs *regs) { unsigned long trans_exc_code = regs->int_parm_long; - return (trans_exc_code & store_indication) == 0x400; + if (static_branch_likely(&have_store_indication)) + return (trans_exc_code & 0xc00) == 0x400; + return false; } static int bad_address(void *p) From 44ae766353af6eb3007308b8565f39cd8d0dd19a Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:42 +0200 Subject: [PATCH 376/475] s390/mm: move translation-exception identification structure to fault.h Move translation-exception identification structure to new fault.h header file, change it to a union, and change existing kvm code accordingly. The new union will be used by subsequent patches. Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/fault.h | 28 ++++++++++++++++++++++ arch/s390/kvm/gaccess.c | 45 +++++++++++------------------------ 2 files changed, 42 insertions(+), 31 deletions(-) create mode 100644 arch/s390/include/asm/fault.h diff --git a/arch/s390/include/asm/fault.h b/arch/s390/include/asm/fault.h new file mode 100644 index 000000000000..d326f56603d6 --- /dev/null +++ b/arch/s390/include/asm/fault.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright IBM Corp. 1999, 2023 + */ +#ifndef _ASM_S390_FAULT_H +#define _ASM_S390_FAULT_H + +union teid { + unsigned long val; + struct { + unsigned long addr : 52; /* Translation-exception Address */ + unsigned long fsi : 2; /* Access Exception Fetch/Store Indication */ + unsigned long : 2; + unsigned long b56 : 1; + unsigned long : 3; + unsigned long b60 : 1; + unsigned long b61 : 1; + unsigned long as : 2; /* ASCE Identifier */ + }; +}; + +enum { + TEID_FSI_UNKNOWN = 0, /* Unknown whether fetch or store */ + TEID_FSI_STORE = 1, /* Exception was due to store operation */ + TEID_FSI_FETCH = 2 /* Exception was due to fetch operation */ +}; + +#endif /* _ASM_S390_FAULT_H */ diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c index 6d6bc19b37dc..8d7763a13671 100644 --- a/arch/s390/kvm/gaccess.c +++ b/arch/s390/kvm/gaccess.c @@ -11,7 +11,7 @@ #include #include #include - +#include #include #include "kvm-s390.h" #include "gaccess.h" @@ -466,23 +466,6 @@ static int ar_translation(struct kvm_vcpu *vcpu, union asce *asce, u8 ar, return 0; } -struct trans_exc_code_bits { - unsigned long addr : 52; /* Translation-exception Address */ - unsigned long fsi : 2; /* Access Exception Fetch/Store Indication */ - unsigned long : 2; - unsigned long b56 : 1; - unsigned long : 3; - unsigned long b60 : 1; - unsigned long b61 : 1; - unsigned long as : 2; /* ASCE Identifier */ -}; - -enum { - FSI_UNKNOWN = 0, /* Unknown whether fetch or store */ - FSI_STORE = 1, /* Exception was due to store operation */ - FSI_FETCH = 2 /* Exception was due to fetch operation */ -}; - enum prot_type { PROT_TYPE_LA = 0, PROT_TYPE_KEYC = 1, @@ -497,11 +480,11 @@ static int trans_exc_ending(struct kvm_vcpu *vcpu, int code, unsigned long gva, enum gacc_mode mode, enum prot_type prot, bool terminate) { struct kvm_s390_pgm_info *pgm = &vcpu->arch.pgm; - struct trans_exc_code_bits *tec; + union teid *teid; memset(pgm, 0, sizeof(*pgm)); pgm->code = code; - tec = (struct trans_exc_code_bits *)&pgm->trans_exc_code; + teid = (union teid *)&pgm->trans_exc_code; switch (code) { case PGM_PROTECTION: @@ -511,25 +494,25 @@ static int trans_exc_ending(struct kvm_vcpu *vcpu, int code, unsigned long gva, WARN_ON_ONCE(1); break; case PROT_TYPE_IEP: - tec->b61 = 1; + teid->b61 = 1; fallthrough; case PROT_TYPE_LA: - tec->b56 = 1; + teid->b56 = 1; break; case PROT_TYPE_KEYC: - tec->b60 = 1; + teid->b60 = 1; break; case PROT_TYPE_ALC: - tec->b60 = 1; + teid->b60 = 1; fallthrough; case PROT_TYPE_DAT: - tec->b61 = 1; + teid->b61 = 1; break; } if (terminate) { - tec->b56 = 0; - tec->b60 = 0; - tec->b61 = 0; + teid->b56 = 0; + teid->b60 = 0; + teid->b61 = 0; } fallthrough; case PGM_ASCE_TYPE: @@ -543,9 +526,9 @@ static int trans_exc_ending(struct kvm_vcpu *vcpu, int code, unsigned long gva, * exc_access_id has to be set to 0 for some instructions. Both * cases have to be handled by the caller. */ - tec->addr = gva >> PAGE_SHIFT; - tec->fsi = mode == GACC_STORE ? FSI_STORE : FSI_FETCH; - tec->as = psw_bits(vcpu->arch.sie_block->gpsw).as; + teid->addr = gva >> PAGE_SHIFT; + teid->fsi = mode == GACC_STORE ? TEID_FSI_STORE : TEID_FSI_FETCH; + teid->as = psw_bits(vcpu->arch.sie_block->gpsw).as; fallthrough; case PGM_ALEN_TRANSLATION: case PGM_ALE_SEQUENCE: From 5be05c35e72f7492105987e0231afab46b69935e Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:43 +0200 Subject: [PATCH 377/475] s390/mm,fault: improve readability by using teid union Get rid of some magic numbers, and use the teid union and also some ptrace PSW defines to improve readability. Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 49 +++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 5466a30d1ec4..bcaac0f939df 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -36,6 +36,8 @@ #include #include #include +#include +#include #include #include #include @@ -43,8 +45,6 @@ #include #include "../kernel/entry.h" -#define __FAIL_ADDR_MASK -4096L - /* * Allocate private vm_fault_reason from top. * Please make sure it won't collide with vm_fault_reason. @@ -76,11 +76,9 @@ early_initcall(fault_init); */ static enum fault_type get_fault_type(struct pt_regs *regs) { - unsigned long trans_exc_code; + union teid teid = { .val = regs->int_parm_long }; - trans_exc_code = regs->int_parm_long & 3; - /* Primary address space */ - if (likely(trans_exc_code == 0)) { + if (likely(teid.as == PSW_BITS_AS_PRIMARY)) { if (user_mode(regs)) return USER_FAULT; if (!IS_ENABLED(CONFIG_PGSTE)) @@ -89,10 +87,10 @@ static enum fault_type get_fault_type(struct pt_regs *regs) return GMAP_FAULT; return KERNEL_FAULT; } - if (trans_exc_code == 2) + if (teid.as == PSW_BITS_AS_SECONDARY) return USER_FAULT; /* Access register mode, not used in the kernel */ - if (trans_exc_code == 1) + if (teid.as == PSW_BITS_AS_ACCREG) return USER_FAULT; /* Home space -> access via kernel ASCE */ return KERNEL_FAULT; @@ -100,17 +98,17 @@ static enum fault_type get_fault_type(struct pt_regs *regs) static unsigned long get_fault_address(struct pt_regs *regs) { - unsigned long trans_exc_code = regs->int_parm_long; + union teid teid = { .val = regs->int_parm_long }; - return trans_exc_code & __FAIL_ADDR_MASK; + return teid.addr * PAGE_SIZE; } static __always_inline bool fault_is_write(struct pt_regs *regs) { - unsigned long trans_exc_code = regs->int_parm_long; + union teid teid = { .val = regs->int_parm_long }; if (static_branch_likely(&have_store_indication)) - return (trans_exc_code & 0xc00) == 0x400; + return teid.fsi == TEID_FSI_STORE; return false; } @@ -176,22 +174,23 @@ bad: static void dump_fault_info(struct pt_regs *regs) { + union teid teid = { .val = regs->int_parm_long }; unsigned long asce; pr_alert("Failing address: %016lx TEID: %016lx\n", - get_fault_address(regs), regs->int_parm_long); + get_fault_address(regs), teid.val); pr_alert("Fault in "); - switch (regs->int_parm_long & 3) { - case 3: + switch (teid.as) { + case PSW_BITS_AS_HOME: pr_cont("home space "); break; - case 2: + case PSW_BITS_AS_SECONDARY: pr_cont("secondary space "); break; - case 1: + case PSW_BITS_AS_ACCREG: pr_cont("access register "); break; - case 0: + case PSW_BITS_AS_PRIMARY: pr_cont("primary space "); break; } @@ -497,11 +496,10 @@ out: void do_protection_exception(struct pt_regs *regs) { - unsigned long trans_exc_code; + union teid teid = { .val = regs->int_parm_long }; vm_fault_t fault; int access; - trans_exc_code = regs->int_parm_long; /* * Protection exceptions are suppressing, decrement psw address. * The exception to this rule are aborted transactions, for these @@ -514,13 +512,12 @@ void do_protection_exception(struct pt_regs *regs) * as a special case because the translation exception code * field is not guaranteed to contain valid data in this case. */ - if (unlikely(!(trans_exc_code & 4))) { + if (unlikely(!teid.b61)) { do_low_address(regs); return; } - if (unlikely(MACHINE_HAS_NX && (trans_exc_code & 0x80))) { - regs->int_parm_long = (trans_exc_code & ~PAGE_MASK) | - (regs->psw.addr & PAGE_MASK); + if (unlikely(MACHINE_HAS_NX && teid.b56)) { + regs->int_parm_long = (teid.addr * PAGE_SIZE) | (regs->psw.addr & PAGE_MASK); access = VM_EXEC; fault = VM_FAULT_BADACCESS; } else { @@ -548,6 +545,7 @@ NOKPROBE_SYMBOL(do_dat_exception); void do_secure_storage_access(struct pt_regs *regs) { + union teid teid = { .val = regs->int_parm_long }; unsigned long addr = get_fault_address(regs); struct vm_area_struct *vma; struct mm_struct *mm; @@ -561,8 +559,7 @@ void do_secure_storage_access(struct pt_regs *regs) * process. Bit 61 is not reliable without the misc UV feature, * therefore this needs to be checked too. */ - if (uv_has_feature(BIT_UV_FEAT_MISC) && - !test_bit_inv(61, ®s->int_parm_long)) { + if (uv_has_feature(BIT_UV_FEAT_MISC) && !teid.b61) { /* * When this happens, userspace did something that it * was not supposed to do, e.g. branching into secure From f67c2da9f1c2d56e3adab342fc39c00c0c5ec790 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:44 +0200 Subject: [PATCH 378/475] s390/mm,fault: use get_kernel_nofault() to dereference in dump_pagetable() The page table dumper uses get_kernel_nofault() to test if dereferencing page table entries is possible. Use the result, which is the required page table entry, instead of throwing it away and dereferencing a second time without any safe guard. Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 45 +++++++++++++++++++------------------------- 1 file changed, 19 insertions(+), 26 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index bcaac0f939df..c37e28fbfbef 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -112,59 +112,52 @@ static __always_inline bool fault_is_write(struct pt_regs *regs) return false; } -static int bad_address(void *p) -{ - unsigned long dummy; - - return get_kernel_nofault(dummy, (unsigned long *)p); -} - static void dump_pagetable(unsigned long asce, unsigned long address) { - unsigned long *table = __va(asce & _ASCE_ORIGIN); + unsigned long entry, *table = __va(asce & _ASCE_ORIGIN); pr_alert("AS:%016lx ", asce); switch (asce & _ASCE_TYPE_MASK) { case _ASCE_TYPE_REGION1: table += (address & _REGION1_INDEX) >> _REGION1_SHIFT; - if (bad_address(table)) + if (get_kernel_nofault(entry, table)) goto bad; - pr_cont("R1:%016lx ", *table); - if (*table & _REGION_ENTRY_INVALID) + pr_cont("R1:%016lx ", entry); + if (entry & _REGION_ENTRY_INVALID) goto out; - table = __va(*table & _REGION_ENTRY_ORIGIN); + table = __va(entry & _REGION_ENTRY_ORIGIN); fallthrough; case _ASCE_TYPE_REGION2: table += (address & _REGION2_INDEX) >> _REGION2_SHIFT; - if (bad_address(table)) + if (get_kernel_nofault(entry, table)) goto bad; - pr_cont("R2:%016lx ", *table); - if (*table & _REGION_ENTRY_INVALID) + pr_cont("R2:%016lx ", entry); + if (entry & _REGION_ENTRY_INVALID) goto out; - table = __va(*table & _REGION_ENTRY_ORIGIN); + table = __va(entry & _REGION_ENTRY_ORIGIN); fallthrough; case _ASCE_TYPE_REGION3: table += (address & _REGION3_INDEX) >> _REGION3_SHIFT; - if (bad_address(table)) + if (get_kernel_nofault(entry, table)) goto bad; - pr_cont("R3:%016lx ", *table); - if (*table & (_REGION_ENTRY_INVALID | _REGION3_ENTRY_LARGE)) + pr_cont("R3:%016lx ", entry); + if (entry & (_REGION_ENTRY_INVALID | _REGION3_ENTRY_LARGE)) goto out; - table = __va(*table & _REGION_ENTRY_ORIGIN); + table = __va(entry & _REGION_ENTRY_ORIGIN); fallthrough; case _ASCE_TYPE_SEGMENT: table += (address & _SEGMENT_INDEX) >> _SEGMENT_SHIFT; - if (bad_address(table)) + if (get_kernel_nofault(entry, table)) goto bad; - pr_cont("S:%016lx ", *table); - if (*table & (_SEGMENT_ENTRY_INVALID | _SEGMENT_ENTRY_LARGE)) + pr_cont("S:%016lx ", entry); + if (entry & (_SEGMENT_ENTRY_INVALID | _SEGMENT_ENTRY_LARGE)) goto out; - table = __va(*table & _SEGMENT_ENTRY_ORIGIN); + table = __va(entry & _SEGMENT_ENTRY_ORIGIN); } table += (address & _PAGE_INDEX) >> _PAGE_SHIFT; - if (bad_address(table)) + if (get_kernel_nofault(entry, table)) goto bad; - pr_cont("P:%016lx ", *table); + pr_cont("P:%016lx ", entry); out: pr_cont("\n"); return; From cca12b427d43bb57be207d634e1d44e9792d1cb4 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:45 +0200 Subject: [PATCH 379/475] s390/mm,fault: remove VM_FAULT_PFAULT Handling of VM_FAULT_PFAULT and VM_FAULT_BADCONTEXT is nearly identical; the only difference is within do_no_context() where however the fault_type (KERNEL_FAULT vs GMAP_FAULT) makes sure that both types will be handled differently. Therefore it is possible to get rid of VM_FAULT_PFAULT and use VM_FAULT_BADCONTEXT instead. Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index c37e28fbfbef..ee8a40917592 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -53,7 +53,6 @@ #define VM_FAULT_BADMAP ((__force vm_fault_t)0x40000000) #define VM_FAULT_BADACCESS ((__force vm_fault_t)0x20000000) #define VM_FAULT_SIGNAL ((__force vm_fault_t)0x10000000) -#define VM_FAULT_PFAULT ((__force vm_fault_t)0x8000000) enum fault_type { KERNEL_FAULT, @@ -294,7 +293,6 @@ static void do_fault_error(struct pt_regs *regs, vm_fault_t fault) } fallthrough; case VM_FAULT_BADCONTEXT: - case VM_FAULT_PFAULT: do_no_context(regs, fault); break; case VM_FAULT_SIGNAL: @@ -459,7 +457,7 @@ retry: * mmap_lock has not been released */ current->thread.gmap_pfault = 1; - fault = VM_FAULT_PFAULT; + fault = VM_FAULT_BADCONTEXT; goto out_up; } flags &= ~FAULT_FLAG_RETRY_NOWAIT; From 5db06565cad67442e4677e7c0da7f7dad1eccb03 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:46 +0200 Subject: [PATCH 380/475] s390/mm,fault: get rid of do_low_address() There is only one caller of do_low_address(). Given that this code is quite special just get rid of do_low_address, and add it to do_protection_exception() in order to make the code a bit more readable. Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index ee8a40917592..87bfcf4d3082 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -257,20 +257,6 @@ static void do_no_context(struct pt_regs *regs, vm_fault_t fault) die(regs, "Oops"); } -static void do_low_address(struct pt_regs *regs) -{ - /* - * Low-address protection hit in kernel mode means - * NULL pointer write access in kernel mode. - */ - if (regs->psw.mask & PSW_MASK_PSTATE) { - /* Low-address protection hit in user mode: 'cannot happen' */ - die(regs, "Low-address protection"); - } - - do_no_context(regs, VM_FAULT_BADACCESS); -} - static void do_sigbus(struct pt_regs *regs) { force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)get_fault_address(regs)); @@ -504,8 +490,15 @@ void do_protection_exception(struct pt_regs *regs) * field is not guaranteed to contain valid data in this case. */ if (unlikely(!teid.b61)) { - do_low_address(regs); - return; + if (user_mode(regs)) { + /* Low-address protection in user mode: cannot happen */ + die(regs, "Low-address protection"); + } + /* + * Low-address protection in kernel mode means + * NULL pointer write access in kernel mode. + */ + return do_no_context(regs, VM_FAULT_BADACCESS); } if (unlikely(MACHINE_HAS_NX && teid.b56)) { regs->int_parm_long = (teid.addr * PAGE_SIZE) | (regs->psw.addr & PAGE_MASK); From 64ea33fb09f8fdcc9eb9f253906886a5801bb48a Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:47 +0200 Subject: [PATCH 381/475] s390/mm,fault: call do_fault_error() only from do_exception() Remove duplicated fault error handling and handle it only once within do_exception(). Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 87bfcf4d3082..9ed979690665 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -321,7 +321,7 @@ static void do_fault_error(struct pt_regs *regs, vm_fault_t fault) * 11 Page translation -> Not present (nullification) * 3b Region third trans. -> Not present (nullification) */ -static inline vm_fault_t do_exception(struct pt_regs *regs, int access) +static void do_exception(struct pt_regs *regs, int access) { struct vm_area_struct *vma; struct task_struct *tsk; @@ -340,7 +340,7 @@ static inline vm_fault_t do_exception(struct pt_regs *regs, int access) */ clear_thread_flag(TIF_PER_TRAP); if (kprobe_page_fault(regs, 14)) - return 0; + return; mm = tsk->mm; address = get_fault_address(regs); is_write = fault_is_write(regs); @@ -468,14 +468,13 @@ out_gmap: out_up: mmap_read_unlock(mm); out: - return fault; + if (unlikely(fault)) + do_fault_error(regs, fault); } void do_protection_exception(struct pt_regs *regs) { union teid teid = { .val = regs->int_parm_long }; - vm_fault_t fault; - int access; /* * Protection exceptions are suppressing, decrement psw address. @@ -502,26 +501,16 @@ void do_protection_exception(struct pt_regs *regs) } if (unlikely(MACHINE_HAS_NX && teid.b56)) { regs->int_parm_long = (teid.addr * PAGE_SIZE) | (regs->psw.addr & PAGE_MASK); - access = VM_EXEC; - fault = VM_FAULT_BADACCESS; - } else { - access = VM_WRITE; - fault = do_exception(regs, access); + do_fault_error(regs, VM_FAULT_BADACCESS); + return; } - if (unlikely(fault)) - do_fault_error(regs, fault); + do_exception(regs, VM_WRITE); } NOKPROBE_SYMBOL(do_protection_exception); void do_dat_exception(struct pt_regs *regs) { - vm_fault_t fault; - int access; - - access = VM_ACCESS_FLAGS; - fault = do_exception(regs, access); - if (unlikely(fault)) - do_fault_error(regs, fault); + do_exception(regs, VM_ACCESS_FLAGS); } NOKPROBE_SYMBOL(do_dat_exception); From 0f1a14e0348eb48e93e139af569944349b725f3f Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:48 +0200 Subject: [PATCH 382/475] s390/mm,fault: simplify kfence fault handling do_no_context() can be simplified by removing its fault parameter, which is only used to decide if kfence_handle_page_fault() should be called. If the fault happened within the kernel space it is ok to always check if this happened on a page which was unmapped because of the kfence feature. Limiting the check to the VM_FAULT_BADCONTEXT case doesn't add any value. Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 9ed979690665..e0dbc231828c 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -234,7 +234,7 @@ static void do_sigsegv(struct pt_regs *regs, int si_code) force_sig_fault(SIGSEGV, si_code, (void __user *)get_fault_address(regs)); } -static void do_no_context(struct pt_regs *regs, vm_fault_t fault) +static void do_no_context(struct pt_regs *regs) { enum fault_type fault_type; unsigned long address; @@ -243,7 +243,7 @@ static void do_no_context(struct pt_regs *regs, vm_fault_t fault) if (fixup_exception(regs)) return; fault_type = get_fault_type(regs); - if ((fault_type == KERNEL_FAULT) && (fault == VM_FAULT_BADCONTEXT)) { + if (fault_type == KERNEL_FAULT) { address = get_fault_address(regs); is_write = fault_is_write(regs); if (kfence_handle_page_fault(address, is_write, regs)) @@ -279,28 +279,28 @@ static void do_fault_error(struct pt_regs *regs, vm_fault_t fault) } fallthrough; case VM_FAULT_BADCONTEXT: - do_no_context(regs, fault); + do_no_context(regs); break; case VM_FAULT_SIGNAL: if (!user_mode(regs)) - do_no_context(regs, fault); + do_no_context(regs); break; default: /* fault & VM_FAULT_ERROR */ if (fault & VM_FAULT_OOM) { if (!user_mode(regs)) - do_no_context(regs, fault); + do_no_context(regs); else pagefault_out_of_memory(); } else if (fault & VM_FAULT_SIGSEGV) { /* Kernel mode? Handle exceptions or die */ if (!user_mode(regs)) - do_no_context(regs, fault); + do_no_context(regs); else do_sigsegv(regs, SEGV_MAPERR); } else if (fault & VM_FAULT_SIGBUS) { /* Kernel mode? Handle exceptions or die */ if (!user_mode(regs)) - do_no_context(regs, fault); + do_no_context(regs); else do_sigbus(regs); } else { @@ -497,7 +497,7 @@ void do_protection_exception(struct pt_regs *regs) * Low-address protection in kernel mode means * NULL pointer write access in kernel mode. */ - return do_no_context(regs, VM_FAULT_BADACCESS); + return do_no_context(regs); } if (unlikely(MACHINE_HAS_NX && teid.b56)) { regs->int_parm_long = (teid.addr * PAGE_SIZE) | (regs->psw.addr & PAGE_MASK); From 0f86ac4ba71329936c44ef71222f1c50284a8928 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:49 +0200 Subject: [PATCH 383/475] s390/mm,fault: remove VM_FAULT_BADCONTEXT Remove VM_FAULT_BADCONTEXT and instead call do_no_context() via wrappers. This adds two new wrappers similar to what x86 has: handle_fault_error() and handle_fault_error_nolock(). Both of them simply call do_no_context(), while handle_fault_error() also unlocks mmap lock, which avoids adding lots of mmap_read_unlock() calls with this and subsequent patches. Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index e0dbc231828c..05c908051e93 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -49,7 +49,6 @@ * Allocate private vm_fault_reason from top. * Please make sure it won't collide with vm_fault_reason. */ -#define VM_FAULT_BADCONTEXT ((__force vm_fault_t)0x80000000) #define VM_FAULT_BADMAP ((__force vm_fault_t)0x40000000) #define VM_FAULT_BADACCESS ((__force vm_fault_t)0x20000000) #define VM_FAULT_SIGNAL ((__force vm_fault_t)0x10000000) @@ -257,6 +256,19 @@ static void do_no_context(struct pt_regs *regs) die(regs, "Oops"); } +static inline void handle_fault_error_nolock(struct pt_regs *regs) +{ + do_no_context(regs); +} + +static void handle_fault_error(struct pt_regs *regs) +{ + struct mm_struct *mm = current->mm; + + mmap_read_unlock(mm); + handle_fault_error_nolock(regs); +} + static void do_sigbus(struct pt_regs *regs) { force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)get_fault_address(regs)); @@ -277,8 +289,6 @@ static void do_fault_error(struct pt_regs *regs, vm_fault_t fault) do_sigsegv(regs, si_code); break; } - fallthrough; - case VM_FAULT_BADCONTEXT: do_no_context(regs); break; case VM_FAULT_SIGNAL: @@ -344,15 +354,14 @@ static void do_exception(struct pt_regs *regs, int access) mm = tsk->mm; address = get_fault_address(regs); is_write = fault_is_write(regs); - fault = VM_FAULT_BADCONTEXT; type = get_fault_type(regs); switch (type) { case KERNEL_FAULT: - goto out; + return handle_fault_error_nolock(regs); case USER_FAULT: case GMAP_FAULT: if (faulthandler_disabled() || !mm) - goto out; + return handle_fault_error_nolock(regs); break; } perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); @@ -443,8 +452,7 @@ retry: * mmap_lock has not been released */ current->thread.gmap_pfault = 1; - fault = VM_FAULT_BADCONTEXT; - goto out_up; + return handle_fault_error(regs); } flags &= ~FAULT_FLAG_RETRY_NOWAIT; flags |= FAULT_FLAG_TRIED; From b61a0922b6dc7dfa6cef5c6d8ab1036826c43b8d Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:50 +0200 Subject: [PATCH 384/475] s390/mm,fault: remove VM_FAULT_SIGNAL Remove VM_FAULT_SIGNAL and open-code it at the only two locations where it is used. Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 05c908051e93..7dd965d153d5 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -51,7 +51,6 @@ */ #define VM_FAULT_BADMAP ((__force vm_fault_t)0x40000000) #define VM_FAULT_BADACCESS ((__force vm_fault_t)0x20000000) -#define VM_FAULT_SIGNAL ((__force vm_fault_t)0x10000000) enum fault_type { KERNEL_FAULT, @@ -291,10 +290,6 @@ static void do_fault_error(struct pt_regs *regs, vm_fault_t fault) } do_no_context(regs); break; - case VM_FAULT_SIGNAL: - if (!user_mode(regs)) - do_no_context(regs); - break; default: /* fault & VM_FAULT_ERROR */ if (fault & VM_FAULT_OOM) { if (!user_mode(regs)) @@ -393,8 +388,9 @@ static void do_exception(struct pt_regs *regs, int access) count_vm_vma_lock_event(VMA_LOCK_RETRY); /* Quick path to respond to signals */ if (fault_signal_pending(fault, regs)) { - fault = VM_FAULT_SIGNAL; - goto out; + if (!user_mode(regs)) + handle_fault_error_nolock(regs); + return; } lock_mmap: mmap_read_lock(mm); @@ -429,10 +425,11 @@ retry: goto out_up; fault = handle_mm_fault(vma, address, flags, regs); if (fault_signal_pending(fault, regs)) { - fault = VM_FAULT_SIGNAL; if (flags & FAULT_FLAG_RETRY_NOWAIT) - goto out_up; - goto out; + mmap_read_unlock(mm); + if (!user_mode(regs)) + handle_fault_error_nolock(regs); + return; } /* The fault is fully completed (including releasing mmap lock) */ if (fault & VM_FAULT_COMPLETED) { From 7c194d84a9ce662426b2ecb59da54bb80c6b1d91 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:51 +0200 Subject: [PATCH 385/475] s390/mm,fault: remove VM_FAULT_BADMAP and VM_FAULT_BADACCESS Remove the last two private vm_fault reasons: VM_FAULT_BADMAP and VM_FAULT_BADACCESS. In order to achieve this add an si_code parameter to do_no_context() and it's wrappers and directly call the wrappers instead of relying on do_fault_error() handling. Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 136 ++++++++++++++++--------------------------- 1 file changed, 51 insertions(+), 85 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 7dd965d153d5..bee22e91e14b 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -45,13 +45,6 @@ #include #include "../kernel/entry.h" -/* - * Allocate private vm_fault_reason from top. - * Please make sure it won't collide with vm_fault_reason. - */ -#define VM_FAULT_BADMAP ((__force vm_fault_t)0x40000000) -#define VM_FAULT_BADACCESS ((__force vm_fault_t)0x20000000) - enum fault_type { KERNEL_FAULT, USER_FAULT, @@ -232,12 +225,17 @@ static void do_sigsegv(struct pt_regs *regs, int si_code) force_sig_fault(SIGSEGV, si_code, (void __user *)get_fault_address(regs)); } -static void do_no_context(struct pt_regs *regs) +static void do_no_context(struct pt_regs *regs, int si_code) { enum fault_type fault_type; unsigned long address; bool is_write; + if (user_mode(regs)) { + if (WARN_ON_ONCE(!si_code)) + si_code = SEGV_MAPERR; + return do_sigsegv(regs, si_code); + } if (fixup_exception(regs)) return; fault_type = get_fault_type(regs); @@ -255,17 +253,17 @@ static void do_no_context(struct pt_regs *regs) die(regs, "Oops"); } -static inline void handle_fault_error_nolock(struct pt_regs *regs) +static inline void handle_fault_error_nolock(struct pt_regs *regs, int si_code) { - do_no_context(regs); + do_no_context(regs, si_code); } -static void handle_fault_error(struct pt_regs *regs) +static void handle_fault_error(struct pt_regs *regs, int si_code) { struct mm_struct *mm = current->mm; mmap_read_unlock(mm); - handle_fault_error_nolock(regs); + handle_fault_error_nolock(regs, si_code); } static void do_sigbus(struct pt_regs *regs) @@ -275,43 +273,26 @@ static void do_sigbus(struct pt_regs *regs) static void do_fault_error(struct pt_regs *regs, vm_fault_t fault) { - int si_code; - - switch (fault) { - case VM_FAULT_BADACCESS: - case VM_FAULT_BADMAP: - /* Bad memory access. Check if it is kernel or user space. */ - if (user_mode(regs)) { - /* User mode accesses just cause a SIGSEGV */ - si_code = (fault == VM_FAULT_BADMAP) ? - SEGV_MAPERR : SEGV_ACCERR; - do_sigsegv(regs, si_code); - break; - } - do_no_context(regs); - break; - default: /* fault & VM_FAULT_ERROR */ - if (fault & VM_FAULT_OOM) { - if (!user_mode(regs)) - do_no_context(regs); - else - pagefault_out_of_memory(); - } else if (fault & VM_FAULT_SIGSEGV) { - /* Kernel mode? Handle exceptions or die */ - if (!user_mode(regs)) - do_no_context(regs); - else - do_sigsegv(regs, SEGV_MAPERR); - } else if (fault & VM_FAULT_SIGBUS) { - /* Kernel mode? Handle exceptions or die */ - if (!user_mode(regs)) - do_no_context(regs); - else - do_sigbus(regs); - } else { - BUG(); - } - break; + /* fault & VM_FAULT_ERROR */ + if (fault & VM_FAULT_OOM) { + if (!user_mode(regs)) + do_no_context(regs, 0); + else + pagefault_out_of_memory(); + } else if (fault & VM_FAULT_SIGSEGV) { + /* Kernel mode? Handle exceptions or die */ + if (!user_mode(regs)) + do_no_context(regs, 0); + else + do_sigsegv(regs, SEGV_MAPERR); + } else if (fault & VM_FAULT_SIGBUS) { + /* Kernel mode? Handle exceptions or die */ + if (!user_mode(regs)) + do_no_context(regs, 0); + else + do_sigbus(regs); + } else { + BUG(); } } @@ -352,11 +333,11 @@ static void do_exception(struct pt_regs *regs, int access) type = get_fault_type(regs); switch (type) { case KERNEL_FAULT: - return handle_fault_error_nolock(regs); + return handle_fault_error_nolock(regs, 0); case USER_FAULT: case GMAP_FAULT: if (faulthandler_disabled() || !mm) - return handle_fault_error_nolock(regs); + return handle_fault_error_nolock(regs, 0); break; } perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); @@ -389,7 +370,7 @@ static void do_exception(struct pt_regs *regs, int access) /* Quick path to respond to signals */ if (fault_signal_pending(fault, regs)) { if (!user_mode(regs)) - handle_fault_error_nolock(regs); + handle_fault_error_nolock(regs, 0); return; } lock_mmap: @@ -401,34 +382,30 @@ lock_mmap: current->thread.gmap_write_flag = !!(flags & FAULT_FLAG_WRITE); current->thread.gmap_int_code = regs->int_code & 0xffff; address = __gmap_translate(gmap, address); - if (address == -EFAULT) { - fault = VM_FAULT_BADMAP; - goto out_up; - } + if (address == -EFAULT) + return handle_fault_error(regs, SEGV_MAPERR); if (gmap->pfault_enabled) flags |= FAULT_FLAG_RETRY_NOWAIT; } retry: - fault = VM_FAULT_BADMAP; vma = find_vma(mm, address); if (!vma) - goto out_up; + return handle_fault_error(regs, SEGV_MAPERR); if (unlikely(vma->vm_start > address)) { if (!(vma->vm_flags & VM_GROWSDOWN)) - goto out_up; + return handle_fault_error(regs, SEGV_MAPERR); vma = expand_stack(mm, address); if (!vma) - goto out; + return handle_fault_error_nolock(regs, SEGV_MAPERR); } - fault = VM_FAULT_BADACCESS; if (unlikely(!(vma->vm_flags & access))) - goto out_up; + return handle_fault_error(regs, SEGV_ACCERR); fault = handle_mm_fault(vma, address, flags, regs); if (fault_signal_pending(fault, regs)) { if (flags & FAULT_FLAG_RETRY_NOWAIT) mmap_read_unlock(mm); if (!user_mode(regs)) - handle_fault_error_nolock(regs); + handle_fault_error_nolock(regs, 0); return; } /* The fault is fully completed (including releasing mmap lock) */ @@ -449,7 +426,7 @@ retry: * mmap_lock has not been released */ current->thread.gmap_pfault = 1; - return handle_fault_error(regs); + return handle_fault_error(regs, 0); } flags &= ~FAULT_FLAG_RETRY_NOWAIT; flags |= FAULT_FLAG_TRIED; @@ -460,10 +437,8 @@ out_gmap: if (IS_ENABLED(CONFIG_PGSTE) && gmap) { address = __gmap_link(gmap, current->thread.gmap_addr, address); - if (address == -EFAULT) { - fault = VM_FAULT_BADMAP; - goto out_up; - } + if (address == -EFAULT) + return handle_fault_error(regs, SEGV_MAPERR); if (address == -ENOMEM) { fault = VM_FAULT_OOM; goto out_up; @@ -502,12 +477,11 @@ void do_protection_exception(struct pt_regs *regs) * Low-address protection in kernel mode means * NULL pointer write access in kernel mode. */ - return do_no_context(regs); + return do_no_context(regs, 0); } if (unlikely(MACHINE_HAS_NX && teid.b56)) { regs->int_parm_long = (teid.addr * PAGE_SIZE) | (regs->psw.addr & PAGE_MASK); - do_fault_error(regs, VM_FAULT_BADACCESS); - return; + return handle_fault_error_nolock(regs, SEGV_ACCERR); } do_exception(regs, VM_WRITE); } @@ -560,20 +534,15 @@ void do_secure_storage_access(struct pt_regs *regs) mmap_read_lock(mm); addr = __gmap_translate(gmap, addr); mmap_read_unlock(mm); - if (IS_ERR_VALUE(addr)) { - do_fault_error(regs, VM_FAULT_BADMAP); - break; - } + if (IS_ERR_VALUE(addr)) + return handle_fault_error_nolock(regs, SEGV_MAPERR); fallthrough; case USER_FAULT: mm = current->mm; mmap_read_lock(mm); vma = find_vma(mm, addr); - if (!vma) { - mmap_read_unlock(mm); - do_fault_error(regs, VM_FAULT_BADMAP); - break; - } + if (!vma) + return handle_fault_error(regs, SEGV_MAPERR); page = follow_page(vma, addr, FOLL_WRITE | FOLL_GET); if (IS_ERR_OR_NULL(page)) { mmap_read_unlock(mm); @@ -604,11 +573,8 @@ void do_non_secure_storage_access(struct pt_regs *regs) struct gmap *gmap = (struct gmap *)S390_lowcore.gmap; unsigned long gaddr = get_fault_address(regs); - if (get_fault_type(regs) != GMAP_FAULT) { - do_fault_error(regs, VM_FAULT_BADMAP); - WARN_ON_ONCE(1); - return; - } + if (WARN_ON_ONCE(get_fault_type(regs) != GMAP_FAULT)) + return handle_fault_error_nolock(regs, SEGV_MAPERR); if (gmap_convert_to_secure(gmap, gaddr) == -EINVAL) send_sig(SIGSEGV, current, 0); } From b20c8216c1e0d5a5a5c9f40df0cf9bbc795e84f1 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 12 Oct 2023 09:40:52 +0200 Subject: [PATCH 386/475] s390/mm,fault: move VM_FAULT_ERROR handling to do_exception() Get rid of do_fault_error() and move its contents to do_exception(), which makes do_exception(). With removing do_fault_error() it is also possible to get rid of the handle_fault_error_nolock() wrapper. Instead rename do_no_context() to handle_fault_error_nolock(). In result the whole fault handling looks much more like on other architectures. Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/fault.c | 81 +++++++++++++++++++------------------------- 1 file changed, 34 insertions(+), 47 deletions(-) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index bee22e91e14b..249aefcf7c4e 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -225,7 +225,7 @@ static void do_sigsegv(struct pt_regs *regs, int si_code) force_sig_fault(SIGSEGV, si_code, (void __user *)get_fault_address(regs)); } -static void do_no_context(struct pt_regs *regs, int si_code) +static void handle_fault_error_nolock(struct pt_regs *regs, int si_code) { enum fault_type fault_type; unsigned long address; @@ -253,11 +253,6 @@ static void do_no_context(struct pt_regs *regs, int si_code) die(regs, "Oops"); } -static inline void handle_fault_error_nolock(struct pt_regs *regs, int si_code) -{ - do_no_context(regs, si_code); -} - static void handle_fault_error(struct pt_regs *regs, int si_code) { struct mm_struct *mm = current->mm; @@ -271,31 +266,6 @@ static void do_sigbus(struct pt_regs *regs) force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)get_fault_address(regs)); } -static void do_fault_error(struct pt_regs *regs, vm_fault_t fault) -{ - /* fault & VM_FAULT_ERROR */ - if (fault & VM_FAULT_OOM) { - if (!user_mode(regs)) - do_no_context(regs, 0); - else - pagefault_out_of_memory(); - } else if (fault & VM_FAULT_SIGSEGV) { - /* Kernel mode? Handle exceptions or die */ - if (!user_mode(regs)) - do_no_context(regs, 0); - else - do_sigsegv(regs, SEGV_MAPERR); - } else if (fault & VM_FAULT_SIGBUS) { - /* Kernel mode? Handle exceptions or die */ - if (!user_mode(regs)) - do_no_context(regs, 0); - else - do_sigbus(regs); - } else { - BUG(); - } -} - /* * This routine handles page faults. It determines the address, * and the problem, and then passes it off to one of the appropriate @@ -362,9 +332,9 @@ static void do_exception(struct pt_regs *regs, int access) vma_end_read(vma); if (!(fault & VM_FAULT_RETRY)) { count_vm_vma_lock_event(VMA_LOCK_SUCCESS); - if (likely(!(fault & VM_FAULT_ERROR))) - fault = 0; - goto out; + if (unlikely(fault & VM_FAULT_ERROR)) + goto error; + return; } count_vm_vma_lock_event(VMA_LOCK_RETRY); /* Quick path to respond to signals */ @@ -412,13 +382,14 @@ retry: if (fault & VM_FAULT_COMPLETED) { if (gmap) { mmap_read_lock(mm); - goto out_gmap; + goto gmap; } - fault = 0; - goto out; + return; + } + if (unlikely(fault & VM_FAULT_ERROR)) { + mmap_read_unlock(mm); + goto error; } - if (unlikely(fault & VM_FAULT_ERROR)) - goto out_up; if (fault & VM_FAULT_RETRY) { if (IS_ENABLED(CONFIG_PGSTE) && gmap && (flags & FAULT_FLAG_RETRY_NOWAIT)) { /* @@ -433,7 +404,7 @@ retry: mmap_read_lock(mm); goto retry; } -out_gmap: +gmap: if (IS_ENABLED(CONFIG_PGSTE) && gmap) { address = __gmap_link(gmap, current->thread.gmap_addr, address); @@ -441,15 +412,31 @@ out_gmap: return handle_fault_error(regs, SEGV_MAPERR); if (address == -ENOMEM) { fault = VM_FAULT_OOM; - goto out_up; + mmap_read_unlock(mm); + goto error; } } - fault = 0; -out_up: mmap_read_unlock(mm); -out: - if (unlikely(fault)) - do_fault_error(regs, fault); + return; +error: + if (fault & VM_FAULT_OOM) { + if (!user_mode(regs)) + handle_fault_error_nolock(regs, 0); + else + pagefault_out_of_memory(); + } else if (fault & VM_FAULT_SIGSEGV) { + if (!user_mode(regs)) + handle_fault_error_nolock(regs, 0); + else + do_sigsegv(regs, SEGV_MAPERR); + } else if (fault & VM_FAULT_SIGBUS) { + if (!user_mode(regs)) + handle_fault_error_nolock(regs, 0); + else + do_sigbus(regs); + } else { + BUG(); + } } void do_protection_exception(struct pt_regs *regs) @@ -477,7 +464,7 @@ void do_protection_exception(struct pt_regs *regs) * Low-address protection in kernel mode means * NULL pointer write access in kernel mode. */ - return do_no_context(regs, 0); + return handle_fault_error_nolock(regs, 0); } if (unlikely(MACHINE_HAS_NX && teid.b56)) { regs->int_parm_long = (teid.addr * PAGE_SIZE) | (regs->psw.addr & PAGE_MASK); From c8b5d574fcea49e9974b7658d5337f93d1921f8c Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Wed, 18 Oct 2023 12:33:05 +0200 Subject: [PATCH 387/475] s390/diag: add missing virt_to_phys() translation to diag224() Diagnose 224 expects a physical address, but all users pass a virtual address. Translate the address to fix this. Reported-by: Mete Durlu Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/kernel/diag.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/s390/kernel/diag.c b/arch/s390/kernel/diag.c index f9f06cd8fcee..92fdc35f028c 100644 --- a/arch/s390/kernel/diag.c +++ b/arch/s390/kernel/diag.c @@ -245,6 +245,7 @@ EXPORT_SYMBOL(diag8c); int diag224(void *ptr) { + unsigned long addr = __pa(ptr); int rc = -EOPNOTSUPP; diag_stat_inc(DIAG_STAT_X224); @@ -253,7 +254,7 @@ int diag224(void *ptr) "0: lhi %0,0x0\n" "1:\n" EX_TABLE(0b,1b) - : "+d" (rc) :"d" (0), "d" (ptr) : "memory"); + : "+d" (rc) :"d" (0), "d" (addr) : "memory"); return rc; } EXPORT_SYMBOL(diag224); From 16ba44826a04834d3eeeda4b731c2ea3481062b7 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Tue, 17 Oct 2023 21:07:03 +0200 Subject: [PATCH 388/475] s390/cmma: fix initial kernel address space page table walk If the cmma no-dat feature is available the kernel page tables are walked to identify and mark all pages which are used for address translation (all region, segment, and page tables). In a subsequent loop all other pages are marked as "no-dat" pages with the ESSA instruction. This information is visible to the hypervisor, so that the hypervisor can optimize purging of guest TLB entries. The initial loop however does not cover the complete kernel address space. This can result in pages being marked as not being used for dynamic address translation, even though they are. In turn guest TLB entries incorrectly may not be purged. Fix this by adjusting the end address of the kernel address range being walked. Cc: Reviewed-by: Claudio Imbrenda Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/page-states.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/arch/s390/mm/page-states.c b/arch/s390/mm/page-states.c index 1e2ea706aa22..c3db3c0e75c0 100644 --- a/arch/s390/mm/page-states.c +++ b/arch/s390/mm/page-states.c @@ -151,15 +151,22 @@ static void mark_kernel_p4d(pgd_t *pgd, unsigned long addr, unsigned long end) static void mark_kernel_pgd(void) { - unsigned long addr, next; + unsigned long addr, next, max_addr; struct page *page; pgd_t *pgd; int i; addr = 0; + /* + * Figure out maximum virtual address accessible with the + * kernel ASCE. This is required to keep the page table walker + * from accessing non-existent entries. + */ + max_addr = (S390_lowcore.kernel_asce.val & _ASCE_TYPE_MASK) >> 2; + max_addr = 1UL << (max_addr * 11 + 31); pgd = pgd_offset_k(addr); do { - next = pgd_addr_end(addr, MODULES_END); + next = pgd_addr_end(addr, max_addr); if (pgd_none(*pgd)) continue; if (!pgd_folded(*pgd)) { @@ -168,7 +175,7 @@ static void mark_kernel_pgd(void) set_bit(PG_arch_1, &page[i].flags); } mark_kernel_p4d(pgd, addr, next); - } while (pgd++, addr = next, addr != MODULES_END); + } while (pgd++, addr = next, addr != max_addr); } void __init cmma_init_nodat(void) From 09cda0a400519b1541591c506e54c9c48e3101bf Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Tue, 17 Oct 2023 21:07:04 +0200 Subject: [PATCH 389/475] s390/mm: add missing arch_set_page_dat() call to vmem_crst_alloc() If the cmma no-dat feature is available all pages that are not used for dynamic address translation are marked as "no-dat" with the ESSA instruction. This information is visible to the hypervisor, so that the hypervisor can optimize purging of guest TLB entries. This also means that pages which are used for dynamic address translation must not be marked as "no-dat", since the hypervisor may then incorrectly not purge guest TLB entries. Region and segment tables allocated via vmem_crst_alloc() are incorrectly marked as "no-dat", as soon as slab_is_available() returns true. Such tables are allocated e.g. when kernel page tables are split, memory is hotplugged, or a DCSS segment is loaded. Fix this by adding the missing arch_set_page_dat() call. Cc: Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/vmem.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c index 86384586eb20..2e8a1064f103 100644 --- a/arch/s390/mm/vmem.c +++ b/arch/s390/mm/vmem.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -46,8 +47,11 @@ void *vmem_crst_alloc(unsigned long val) unsigned long *table; table = vmem_alloc_pages(CRST_ALLOC_ORDER); - if (table) - crst_table_init(table, val); + if (!table) + return NULL; + crst_table_init(table, val); + if (slab_is_available()) + arch_set_page_dat(virt_to_page(table), CRST_ALLOC_ORDER); return table; } From 545db7e21e64766e6b7cb987fbfd3e79419726ce Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Sat, 21 Oct 2023 16:42:41 +0200 Subject: [PATCH 390/475] tracing/histograms: Simplify last_cmd_set() Turn a kzalloc()+strcpy()+strncat() into an equivalent and less verbose kasprintf(). Link: https://lore.kernel.org/linux-trace-kernel/30b6fb04dadc10a03cc1ad08f5d8a93ef623a167.1697899346.git.christophe.jaillet@wanadoo.fr Cc: Masami Hiramatsu Signed-off-by: Christophe JAILLET Reviewed-by: Mukesh ojha Signed-off-by: Steven Rostedt (Google) --- kernel/trace/trace_events_hist.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c index d06938ae0717..1abc07fba1b9 100644 --- a/kernel/trace/trace_events_hist.c +++ b/kernel/trace/trace_events_hist.c @@ -774,23 +774,16 @@ static void last_cmd_set(struct trace_event_file *file, char *str) { const char *system = NULL, *name = NULL; struct trace_event_call *call; - int len; if (!str) return; - /* sizeof() contains the nul byte */ - len = sizeof(HIST_PREFIX) + strlen(str); kfree(last_cmd); - last_cmd = kzalloc(len, GFP_KERNEL); + + last_cmd = kasprintf(GFP_KERNEL, HIST_PREFIX "%s", str); if (!last_cmd) return; - strcpy(last_cmd, HIST_PREFIX); - /* Again, sizeof() contains the nul byte */ - len -= sizeof(HIST_PREFIX); - strncat(last_cmd, str, len); - if (file) { call = file->event_call; system = call->class->system; From 3f6074cf467f6c2beb7db0b0d685fb53f25ae9aa Mon Sep 17 00:00:00 2001 From: Li kunyu Date: Mon, 23 Oct 2023 14:23:59 +0800 Subject: [PATCH 391/475] =?UTF-8?q?printk:=20printk:=20Remove=20unnecessar?= =?UTF-8?q?y=20statements=EF=BC=87len=20=3D=200;=EF=BC=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In the following two functions, len has already been assigned a value of 0 when defining the variable, so remove 'len=0;'. Signed-off-by: Li kunyu Reviewed-by: Petr Mladek Signed-off-by: Petr Mladek Link: https://lore.kernel.org/r/20231023062359.130633-1-kunyu@nfschina.com --- kernel/printk/printk.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 96fc38cb2e84..1c6ed598b480 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -1671,7 +1671,6 @@ static int syslog_print_all(char __user *buf, int size, bool clear) prb_rec_init_rd(&r, &info, text, PRINTK_MESSAGE_MAX); - len = 0; prb_for_each_record(seq, prb, seq, &r) { int textlen; @@ -4190,7 +4189,6 @@ bool kmsg_dump_get_buffer(struct kmsg_dump_iter *iter, bool syslog, prb_rec_init_rd(&r, &info, buf, size); - len = 0; prb_for_each_record(seq, prb, seq, &r) { if (r.info->seq >= iter->next_seq) break; From 357673120af6bd2939b9c99dec58962fff65a110 Mon Sep 17 00:00:00 2001 From: Michael Ellerman Date: Mon, 23 Oct 2023 20:37:06 +1100 Subject: [PATCH 392/475] Revert "powerpc: add `cur_cpu_spec` symbol to vmcoreinfo" This reverts commit 7135b921b32966d7602ede396b7286d372aee63f. I applied this commit prematurely while there was still discussion ongoing. Revert it so the final patch can be applied cleanly. Signed-off-by: Michael Ellerman --- arch/powerpc/kexec/core.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/powerpc/kexec/core.c b/arch/powerpc/kexec/core.c index 369b8334a4f0..de64c7962991 100644 --- a/arch/powerpc/kexec/core.c +++ b/arch/powerpc/kexec/core.c @@ -63,8 +63,6 @@ void arch_crash_save_vmcoreinfo(void) #ifndef CONFIG_NUMA VMCOREINFO_SYMBOL(contig_page_data); #endif - VMCOREINFO_SYMBOL(cur_cpu_spec); - VMCOREINFO_OFFSET(cpu_spec, mmu_features); #if defined(CONFIG_PPC64) && defined(CONFIG_SPARSEMEM_VMEMMAP) VMCOREINFO_SYMBOL(vmemmap_list); VMCOREINFO_SYMBOL(mmu_vmemmap_psize); From 36e826b568e412f61d68fedc02a67b4d8b7583cc Mon Sep 17 00:00:00 2001 From: Aditya Gupta Date: Mon, 23 Oct 2023 12:56:12 +0530 Subject: [PATCH 393/475] powerpc/vmcore: Add MMU information to vmcoreinfo Since below commit, address mapping for vmemmap has changed for Radix MMU, where address mapping is stored in kernel page table itself, instead of earlier used 'vmemmap_list'. commit 368a0590d954 ("powerpc/book3s64/vmemmap: switch radix to use a different vmemmap handling function") Hence with upstream kernel, in case of Radix MMU, makedumpfile fails to do address translation for vmemmap addresses, as it depended on vmemmap_list, which can now be empty. While fixing the address translation in makedumpfile, it was identified that currently makedumpfile cannot distinguish between Hash MMU and Radix MMU, unless VMLINUX is passed with -x flag to makedumpfile. And hence fails to assign offsets and shifts correctly (such as in L4 to PGDIR offset calculation in makedumpfile). For getting the MMU, makedumpfile uses `cur_cpu_spec.mmu_features`. Add `cur_cpu_spec` symbol and offset of `mmu_features` in the `cpu_spec` struct, to VMCOREINFO, so that makedumpfile can assign the offsets correctly, without needing a VMLINUX. Also, even along with `cur_cpu_spec->mmu_features` makedumpfile has to depend on the 'MMU_FTR_TYPE_RADIX' flag in mmu_features, implying kernel developers need to be cautious of changes to 'MMU_FTR_*' defines. A more stable approach was suggested in the below thread by contributors: https://lore.kernel.org/linuxppc-dev/20230920105706.853626-1-adityag@linux.ibm.com/ The suggestion was to add whether 'RADIX_MMU' is enabled in vmcoreinfo This patch also implements the suggestion, by adding 'RADIX_MMU' in vmcoreinfo, which makedumpfile can use to get whether the crashed system had RADIX MMU (in which case 'NUMBER(RADIX_MMU)=1') or not (in which case 'NUMBER(RADIX_MMU)=0') Fixes: 368a0590d954 ("powerpc/book3s64/vmemmap: switch radix to use a different vmemmap handling function") Reported-by: Sachin Sant Signed-off-by: Aditya Gupta Signed-off-by: Michael Ellerman Link: https://msgid.link/20231023072612.50874-1-adityag@linux.ibm.com --- arch/powerpc/kexec/core.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/powerpc/kexec/core.c b/arch/powerpc/kexec/core.c index de64c7962991..005269ac3244 100644 --- a/arch/powerpc/kexec/core.c +++ b/arch/powerpc/kexec/core.c @@ -74,6 +74,9 @@ void arch_crash_save_vmcoreinfo(void) VMCOREINFO_STRUCT_SIZE(mmu_psize_def); VMCOREINFO_OFFSET(mmu_psize_def, shift); #endif + VMCOREINFO_SYMBOL(cur_cpu_spec); + VMCOREINFO_OFFSET(cpu_spec, mmu_features); + vmcoreinfo_append_str("NUMBER(RADIX_MMU)=%d\n", early_radix_enabled()); vmcoreinfo_append_str("KERNELOFFSET=%lx\n", kaslr_offset()); } From 1954da4a2b621a3328a63382cae7e5f5e2af502c Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Fri, 20 Oct 2023 17:26:50 +0200 Subject: [PATCH 394/475] s390/mm: add missing arch_set_page_dat() call to gmap allocations If the cmma no-dat feature is available all pages that are not used for dynamic address translation are marked as "no-dat" with the ESSA instruction. This information is visible to the hypervisor, so that the hypervisor can optimize purging of guest TLB entries. This also means that pages which are used for dynamic address translation must not be marked as "no-dat", since the hypervisor may then incorrectly not purge guest TLB entries. Region, segment, and page tables allocated within the gmap code are incorrectly marked as "no-dat", since an explicit call to arch_set_page_dat() is missing, which would remove the "no-dat" mark. In order to fix this add a new gmap_alloc_crst() function which should be used to allocate region and segment tables, and which also calls arch_set_page_dat(). Also add the arch_set_page_dat() call to page_table_alloc_pgste(). Cc: Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/gmap.c | 24 ++++++++++++++++++------ arch/s390/mm/pgalloc.c | 1 + 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/arch/s390/mm/gmap.c b/arch/s390/mm/gmap.c index 906a7bfc2a78..20786f6883b2 100644 --- a/arch/s390/mm/gmap.c +++ b/arch/s390/mm/gmap.c @@ -21,10 +21,22 @@ #include #include +#include #include #define GMAP_SHADOW_FAKE_TABLE 1ULL +static struct page *gmap_alloc_crst(void) +{ + struct page *page; + + page = alloc_pages(GFP_KERNEL_ACCOUNT, CRST_ALLOC_ORDER); + if (!page) + return NULL; + arch_set_page_dat(page, CRST_ALLOC_ORDER); + return page; +} + /** * gmap_alloc - allocate and initialize a guest address space * @limit: maximum address of the gmap address space @@ -67,7 +79,7 @@ static struct gmap *gmap_alloc(unsigned long limit) spin_lock_init(&gmap->guest_table_lock); spin_lock_init(&gmap->shadow_lock); refcount_set(&gmap->ref_count, 1); - page = alloc_pages(GFP_KERNEL_ACCOUNT, CRST_ALLOC_ORDER); + page = gmap_alloc_crst(); if (!page) goto out_free; page->index = 0; @@ -308,7 +320,7 @@ static int gmap_alloc_table(struct gmap *gmap, unsigned long *table, unsigned long *new; /* since we dont free the gmap table until gmap_free we can unlock */ - page = alloc_pages(GFP_KERNEL_ACCOUNT, CRST_ALLOC_ORDER); + page = gmap_alloc_crst(); if (!page) return -ENOMEM; new = page_to_virt(page); @@ -1759,7 +1771,7 @@ int gmap_shadow_r2t(struct gmap *sg, unsigned long saddr, unsigned long r2t, BUG_ON(!gmap_is_shadow(sg)); /* Allocate a shadow region second table */ - page = alloc_pages(GFP_KERNEL_ACCOUNT, CRST_ALLOC_ORDER); + page = gmap_alloc_crst(); if (!page) return -ENOMEM; page->index = r2t & _REGION_ENTRY_ORIGIN; @@ -1843,7 +1855,7 @@ int gmap_shadow_r3t(struct gmap *sg, unsigned long saddr, unsigned long r3t, BUG_ON(!gmap_is_shadow(sg)); /* Allocate a shadow region second table */ - page = alloc_pages(GFP_KERNEL_ACCOUNT, CRST_ALLOC_ORDER); + page = gmap_alloc_crst(); if (!page) return -ENOMEM; page->index = r3t & _REGION_ENTRY_ORIGIN; @@ -1927,7 +1939,7 @@ int gmap_shadow_sgt(struct gmap *sg, unsigned long saddr, unsigned long sgt, BUG_ON(!gmap_is_shadow(sg) || (sgt & _REGION3_ENTRY_LARGE)); /* Allocate a shadow segment table */ - page = alloc_pages(GFP_KERNEL_ACCOUNT, CRST_ALLOC_ORDER); + page = gmap_alloc_crst(); if (!page) return -ENOMEM; page->index = sgt & _REGION_ENTRY_ORIGIN; @@ -2855,7 +2867,7 @@ int s390_replace_asce(struct gmap *gmap) if ((gmap->asce & _ASCE_TYPE_MASK) == _ASCE_TYPE_SEGMENT) return -EINVAL; - page = alloc_pages(GFP_KERNEL_ACCOUNT, CRST_ALLOC_ORDER); + page = gmap_alloc_crst(); if (!page) return -ENOMEM; page->index = 0; diff --git a/arch/s390/mm/pgalloc.c b/arch/s390/mm/pgalloc.c index a6b71bc71896..a7e4b6e11cc5 100644 --- a/arch/s390/mm/pgalloc.c +++ b/arch/s390/mm/pgalloc.c @@ -146,6 +146,7 @@ struct page *page_table_alloc_pgste(struct mm_struct *mm) ptdesc = pagetable_alloc(GFP_KERNEL, 0); if (ptdesc) { table = (u64 *)ptdesc_to_virt(ptdesc); + arch_set_page_dat(virt_to_page(table), 0); memset64(table, _PAGE_INVALID, PTRS_PER_PTE); memset64(table + PTRS_PER_PTE, 0, PTRS_PER_PTE); } From a13e8bdf0f788552aa37e1f9a6fe5fcca4095d2b Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Tue, 10 Oct 2023 16:14:22 +0200 Subject: [PATCH 395/475] s390/pai_crypto: use PERF_ATTACH_TASK define for per task detection Use define PERF_ATTACH_TASK bit in event->attach_state to determine system wide invocation or per task invocation in event initialization. This bit is set in common code and before calling PMU device driver specific event initialization. It is set once and never changes. It is save to use and also in sync with other PMU specific code. No functional change. Acked-by: Sumanth Korikkar Signed-off-by: Thomas Richter Signed-off-by: Vasily Gorbik --- arch/s390/kernel/perf_pai_crypto.c | 2 +- arch/s390/kernel/perf_pai_ext.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/s390/kernel/perf_pai_crypto.c b/arch/s390/kernel/perf_pai_crypto.c index 19d53edf50d2..85d7398ac279 100644 --- a/arch/s390/kernel/perf_pai_crypto.c +++ b/arch/s390/kernel/perf_pai_crypto.c @@ -269,7 +269,7 @@ static int paicrypt_event_init(struct perf_event *event) a->config > PAI_CRYPTO_BASE + paicrypt_cnt) return -EINVAL; /* Allow only CPU wide operation, no process context for now. */ - if (event->hw.target || event->cpu == -1) + if ((event->attach_state & PERF_ATTACH_TASK) || event->cpu == -1) return -ENOENT; /* Allow only CRYPTO_ALL for sampling. */ if (a->sample_period && a->config != PAI_CRYPTO_BASE) diff --git a/arch/s390/kernel/perf_pai_ext.c b/arch/s390/kernel/perf_pai_ext.c index e91c77155f08..edceff2143b9 100644 --- a/arch/s390/kernel/perf_pai_ext.c +++ b/arch/s390/kernel/perf_pai_ext.c @@ -248,7 +248,7 @@ static int paiext_event_init(struct perf_event *event) if (rc) return rc; /* Allow only CPU wide operation, no process context for now. */ - if (event->hw.target || event->cpu == -1) + if ((event->attach_state & PERF_ATTACH_TASK) || event->cpu == -1) return -ENOENT; /* Allow only event NNPA_ALL for sampling. */ if (a->sample_period && a->config != PAI_NNPA_BASE) From b286997e83dcf7b498329a66a8a22fc8a5bf50f0 Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Wed, 11 Oct 2023 12:09:30 +0200 Subject: [PATCH 396/475] s390/pai: initialize event count once at initialization Event count value is initialized and set to zero in function paicrypt_start(). This function is called once per CPU when an event is started on that CPU. This leads to event count value being set to zero as many times as there are online CPUs. This is not necessary. The event count value is bound to the event and it is sufficient to initialize the event counter once at event creation time. This is done when the event structure is dynamicly allocated with __GFP_ZERO flag. This sets member count to zero. Acked-by: Sumanth Korikkar Signed-off-by: Thomas Richter Signed-off-by: Vasily Gorbik --- arch/s390/kernel/perf_pai_crypto.c | 1 - arch/s390/kernel/perf_pai_ext.c | 1 - 2 files changed, 2 deletions(-) diff --git a/arch/s390/kernel/perf_pai_crypto.c b/arch/s390/kernel/perf_pai_crypto.c index 85d7398ac279..35940234da75 100644 --- a/arch/s390/kernel/perf_pai_crypto.c +++ b/arch/s390/kernel/perf_pai_crypto.c @@ -322,7 +322,6 @@ static void paicrypt_start(struct perf_event *event, int flags) if (!event->hw.last_tag) { event->hw.last_tag = 1; sum = paicrypt_getall(event); /* Get current value */ - local64_set(&event->count, 0); local64_set(&event->hw.prev_count, sum); } } diff --git a/arch/s390/kernel/perf_pai_ext.c b/arch/s390/kernel/perf_pai_ext.c index edceff2143b9..8ba0f1a3a39d 100644 --- a/arch/s390/kernel/perf_pai_ext.c +++ b/arch/s390/kernel/perf_pai_ext.c @@ -326,7 +326,6 @@ static void paiext_start(struct perf_event *event, int flags) event->hw.last_tag = 1; sum = paiext_getall(event); /* Get current value */ local64_set(&event->hw.prev_count, sum); - local64_set(&event->count, 0); } static int paiext_add(struct perf_event *event, int flags) From aecd5a37b5ef4de4f6402dc079672e4243cc4c13 Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Wed, 18 Oct 2023 11:53:52 +0200 Subject: [PATCH 397/475] s390/pai_crypto: remove per-cpu variable assignement in event initialization Function paicrypt_event_init() initializes the PMU device driver specific details for an event. It is called once per event creation. The function paicrypt_event_init() is not necessarily executed on that CPU the event will be used for. When an event is activated, function paicrypt_start() is used to start the event on that CPU. The per CPU data structure struct paicrypt_map has a pointer to the event which is active for a particular CPU. This pointer is set in function paicrypt_start() to point to the currently installed event. There is no need to also set this pointer in function paicrypt_event_init() where is might be assigned to the wrong CPU. Therefore remove this assignment in paicrypt_event_init(). Acked-by: Sumanth Korikkar Signed-off-by: Thomas Richter Signed-off-by: Vasily Gorbik --- arch/s390/kernel/perf_pai_crypto.c | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/s390/kernel/perf_pai_crypto.c b/arch/s390/kernel/perf_pai_crypto.c index 35940234da75..77fd24e6cbb6 100644 --- a/arch/s390/kernel/perf_pai_crypto.c +++ b/arch/s390/kernel/perf_pai_crypto.c @@ -285,7 +285,6 @@ static int paicrypt_event_init(struct perf_event *event) * are active at the same time. */ event->hw.last_tag = 0; - cpump->event = event; event->destroy = paicrypt_event_destroy; if (a->sample_period) { From e3f4170ccf20ebe2985b8e166184dd2d54220b60 Mon Sep 17 00:00:00 2001 From: Sumanth Korikkar Date: Fri, 29 Sep 2023 19:12:04 +0200 Subject: [PATCH 398/475] s390/sclp: handle default case in sclp memory notifier When new memory notifier types are added in common code and unimplemented on s390, return success instead of EINVAL return code. This prevents breakage when new memory notifier types are added. Reviewed-by: Heiko Carstens Signed-off-by: Sumanth Korikkar Signed-off-by: Vasily Gorbik --- drivers/s390/char/sclp_cmd.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/s390/char/sclp_cmd.c b/drivers/s390/char/sclp_cmd.c index b73edf0cd725..11c428f4c7cf 100644 --- a/drivers/s390/char/sclp_cmd.c +++ b/drivers/s390/char/sclp_cmd.c @@ -353,7 +353,6 @@ static int sclp_mem_notifier(struct notifier_block *nb, sclp_mem_change_state(start, size, 0); break; default: - rc = -EINVAL; break; } mutex_unlock(&sclp_mem_mutex); From 44d93045247661acbd50b1629e62f415f2747577 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Tue, 24 Oct 2023 10:15:19 +0200 Subject: [PATCH 399/475] s390/cmma: fix detection of DAT pages If the cmma no-dat feature is available the kernel page tables are walked to identify and mark all pages which are used for address translation (all region, segment, and page tables). In a subsequent loop all other pages are marked as "no-dat" pages with the ESSA instruction. This information is visible to the hypervisor, so that the hypervisor can optimize purging of guest TLB entries. The initial loop however is incorrect: only the first three of the four pages which belong to segment and region tables will be marked as being used for DAT. The last page is incorrectly marked as no-dat. This can result in incorrect guest TLB flushes. Fix this by simply marking all four pages. Cc: Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/page-states.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/s390/mm/page-states.c b/arch/s390/mm/page-states.c index c3db3c0e75c0..20c0b160efee 100644 --- a/arch/s390/mm/page-states.c +++ b/arch/s390/mm/page-states.c @@ -121,7 +121,7 @@ static void mark_kernel_pud(p4d_t *p4d, unsigned long addr, unsigned long end) continue; if (!pud_folded(*pud)) { page = phys_to_page(pud_val(*pud)); - for (i = 0; i < 3; i++) + for (i = 0; i < 4; i++) set_bit(PG_arch_1, &page[i].flags); } mark_kernel_pmd(pud, addr, next); @@ -142,7 +142,7 @@ static void mark_kernel_p4d(pgd_t *pgd, unsigned long addr, unsigned long end) continue; if (!p4d_folded(*p4d)) { page = phys_to_page(p4d_val(*p4d)); - for (i = 0; i < 3; i++) + for (i = 0; i < 4; i++) set_bit(PG_arch_1, &page[i].flags); } mark_kernel_pud(p4d, addr, next); @@ -171,7 +171,7 @@ static void mark_kernel_pgd(void) continue; if (!pgd_folded(*pgd)) { page = phys_to_page(pgd_val(*pgd)); - for (i = 0; i < 3; i++) + for (i = 0; i < 4; i++) set_bit(PG_arch_1, &page[i].flags); } mark_kernel_p4d(pgd, addr, next); From 84bb41d5df48868055d159d9247b80927f1f70f9 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Tue, 24 Oct 2023 10:15:20 +0200 Subject: [PATCH 400/475] s390/cmma: fix handling of swapper_pg_dir and invalid_pg_dir If the cmma no-dat feature is available the kernel page tables are walked to identify and mark all pages which are used for address translation (all region, segment, and page tables). In a subsequent loop all other pages are marked as "no-dat" pages with the ESSA instruction. This information is visible to the hypervisor, so that the hypervisor can optimize purging of guest TLB entries. All pages used for swapper_pg_dir and invalid_pg_dir are incorrectly marked as no-dat, which in turn can result in incorrect guest TLB flushes. Fix this by marking those pages correctly as being used for DAT. Cc: Reviewed-by: Claudio Imbrenda Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/mm/page-states.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/s390/mm/page-states.c b/arch/s390/mm/page-states.c index 20c0b160efee..a31acb2c4ef2 100644 --- a/arch/s390/mm/page-states.c +++ b/arch/s390/mm/page-states.c @@ -188,6 +188,12 @@ void __init cmma_init_nodat(void) return; /* Mark pages used in kernel page tables */ mark_kernel_pgd(); + page = virt_to_page(&swapper_pg_dir); + for (i = 0; i < 4; i++) + set_bit(PG_arch_1, &page[i].flags); + page = virt_to_page(&invalid_pg_dir); + for (i = 0; i < 4; i++) + set_bit(PG_arch_1, &page[i].flags); /* Set all kernel pages not used for page tables to stable/no-dat */ for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, NULL) { From f139a7a2f88190ef28162d3e485324bab8a4e12e Mon Sep 17 00:00:00 2001 From: Ilya Leoshkevich Date: Mon, 31 Jul 2023 17:07:02 +0200 Subject: [PATCH 401/475] s390: delete the unused store_prefix() function Its last usage was deleted in commit 4df29d2b9024 ("s390/smp: rework absolute lowcore access"). Reviewed-by: Alexander Gordeev Signed-off-by: Ilya Leoshkevich Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/lowcore.h | 8 -------- 1 file changed, 8 deletions(-) diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h index 3366431dcad5..5dc1b6345006 100644 --- a/arch/s390/include/asm/lowcore.h +++ b/arch/s390/include/asm/lowcore.h @@ -222,12 +222,4 @@ static inline void set_prefix(__u32 address) asm volatile("spx %0" : : "Q" (address) : "memory"); } -static inline __u32 store_prefix(void) -{ - __u32 address; - - asm volatile("stpx %0" : "=Q" (address)); - return address; -} - #endif /* _ASM_S390_LOWCORE_H */ From f48781d220ee6bb59816383237bf0abd1c65c493 Mon Sep 17 00:00:00 2001 From: Peter Oberparleiter Date: Fri, 18 Aug 2023 15:11:18 +0200 Subject: [PATCH 402/475] s390/cio: export CMG value as decimal Change format of the "cmg" sysfs attribute from hex to decimal to make it easier to consume. Note that this should not break any existing users since only values 2 and 3 are currently exported. Also the main user already assumes decimal notation [1]. [1] https://sourceforge.net/p/sblim/gather/ci/master/tree/plugin/metriczCH.c Reviewed-by: Vineeth Vijayan Signed-off-by: Peter Oberparleiter Signed-off-by: Vasily Gorbik --- drivers/s390/cio/chp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/s390/cio/chp.c b/drivers/s390/cio/chp.c index 5440f285f349..675d7ed82356 100644 --- a/drivers/s390/cio/chp.c +++ b/drivers/s390/cio/chp.c @@ -334,7 +334,7 @@ static ssize_t chp_cmg_show(struct device *dev, struct device_attribute *attr, return 0; if (chp->cmg == -1) /* channel measurements not available */ return sprintf(buf, "unknown\n"); - return sprintf(buf, "%x\n", chp->cmg); + return sprintf(buf, "%d\n", chp->cmg); } static DEVICE_ATTR(cmg, 0444, chp_cmg_show, NULL); From e78002aa9a5a9eaf18bc29ccaef7f405616fd68e Mon Sep 17 00:00:00 2001 From: Peter Oberparleiter Date: Fri, 18 Aug 2023 15:14:02 +0200 Subject: [PATCH 403/475] s390/cio: fix virtual vs physical address confusion Fix virtual vs physical address confusion (which currently are the same). Reviewed-by: Vineeth Vijayan Signed-off-by: Peter Oberparleiter Signed-off-by: Vasily Gorbik --- drivers/s390/cio/chsc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/s390/cio/chsc.c b/drivers/s390/cio/chsc.c index 0abd77f4b664..cc6dc5076ffc 100644 --- a/drivers/s390/cio/chsc.c +++ b/drivers/s390/cio/chsc.c @@ -881,8 +881,8 @@ int __chsc_do_secm(struct channel_subsystem *css, int enable) secm_area->request.code = 0x0016; secm_area->key = PAGE_DEFAULT_KEY >> 4; - secm_area->cub_addr1 = (u64)(unsigned long)css->cub_addr1; - secm_area->cub_addr2 = (u64)(unsigned long)css->cub_addr2; + secm_area->cub_addr1 = virt_to_phys(css->cub_addr1); + secm_area->cub_addr2 = virt_to_phys(css->cub_addr2); secm_area->operation_code = enable ? 0 : 1; From e37988bcd1fddb726cf08e4f04a1b8dc4a2f80aa Mon Sep 17 00:00:00 2001 From: Justin Stitt Date: Mon, 23 Oct 2023 19:14:49 +0000 Subject: [PATCH 404/475] s390/sclp: replace deprecated strncpy with strtomem Let's move away from using strncpy() as it is deprecated [1]. Instead use strtomem() as `e.id` is already marked as nonstring: | char id[4] __nonstring; We don't need strtomem_pad() because `e` is already memset to 0 -- rendering any additional NUL-padding useless. Link: https://www.kernel.org/doc/html/latest/process/deprecated.html#strncpy-on-nul-terminated-strings [1] Link: https://github.com/KSPP/linux/issues/90 Cc: linux-hardening@vger.kernel.org Signed-off-by: Justin Stitt Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20231023-strncpy-drivers-s390-char-sclp-c-v1-1-eaeef80522bb@google.com Signed-off-by: Vasily Gorbik --- drivers/s390/char/sclp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/s390/char/sclp.c b/drivers/s390/char/sclp.c index ba9b202c5dee..d53ee34d398f 100644 --- a/drivers/s390/char/sclp.c +++ b/drivers/s390/char/sclp.c @@ -81,7 +81,7 @@ static inline void sclp_trace(int prio, char *id, u32 a, u64 b, bool err) struct sclp_trace_entry e; memset(&e, 0, sizeof(e)); - strncpy(e.id, id, sizeof(e.id)); + strtomem(e.id, id); e.a = a; e.b = b; debug_event(&sclp_debug, prio, &e, sizeof(e)); From 991a211aa99f468cd291a97b8dcb448ebc77f6c4 Mon Sep 17 00:00:00 2001 From: Justin Stitt Date: Mon, 23 Oct 2023 19:24:38 +0000 Subject: [PATCH 405/475] s390/cio: replace deprecated strncpy with strscpy strncpy() is deprecated for use on NUL-terminated destination strings [1] and as such we should prefer more robust and less ambiguous string interfaces. We expect both `params` and `id` to be NUL-terminated based on their usage with format strings: format_node_data(iuparams, iunodeid, &lir->incident_node); format_node_data(auparams, aunodeid, &lir->attached_node); switch (lir->iq.class) { case LIR_IQ_CLASS_DEGRADED: pr_warn("Link degraded: RS=%02x RSID=%04x IC=%02x " "IUPARAMS=%s IUNODEID=%s AUPARAMS=%s AUNODEID=%s\n", sei_area->rs, sei_area->rsid, lir->ic, iuparams, iunodeid, auparams, aunodeid); NUL-padding is not required as both `params` and `id` have been memset to 0: memset(params, 0, PARAMS_LEN); memset(id, 0, NODEID_LEN); Considering the above, a suitable replacement is `strscpy` [2] due to the fact that it guarantees NUL-termination on the destination buffer without unnecessarily NUL-padding. Note that there's no overread bugs in the current implementation as the string literal "n/a" has a size much smaller than PARAMS_LEN or NODEID_LEN. Nonetheless, let's favor strscpy(). Link: https://www.kernel.org/doc/html/latest/process/deprecated.html#strncpy-on-nul-terminated-strings [1] Link: https://manpages.debian.org/testing/linux-manual-4.8/strscpy.9.en.html [2] Link: https://github.com/KSPP/linux/issues/90 Cc: linux-hardening@vger.kernel.org Signed-off-by: Justin Stitt Reviewed-by: Vineeth Vijayan Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20231023-strncpy-drivers-s390-cio-chsc-c-v1-1-8b76a7b83260@google.com Signed-off-by: Vasily Gorbik --- drivers/s390/cio/chsc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/s390/cio/chsc.c b/drivers/s390/cio/chsc.c index cc6dc5076ffc..f8b04ce61556 100644 --- a/drivers/s390/cio/chsc.c +++ b/drivers/s390/cio/chsc.c @@ -393,8 +393,8 @@ static void format_node_data(char *params, char *id, struct node_descriptor *nd) memset(id, 0, NODEID_LEN); if (nd->validity != ND_VALIDITY_VALID) { - strncpy(params, "n/a", PARAMS_LEN - 1); - strncpy(id, "n/a", NODEID_LEN - 1); + strscpy(params, "n/a", PARAMS_LEN); + strscpy(id, "n/a", NODEID_LEN); return; } From cbf5f58461b25f3f3e19704e2e51ad002c11080c Mon Sep 17 00:00:00 2001 From: Michael Petlan Date: Thu, 19 Oct 2023 11:11:37 +0200 Subject: [PATCH 406/475] perf test: Skip CoreSight tests if cs_etm// event is not available CoreSight might be not available, in such case, skip the tests. Signed-off-by: Michael Petlan Reviewed-by: Leo Yan Reviewed-by: Carsten Haitzler Cc: vmolnaro@redhat.com Link: https://lore.kernel.org/r/20231019091137.22525-1-mpetlan@redhat.com Signed-off-by: Namhyung Kim --- tools/perf/tests/shell/lib/coresight.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/perf/tests/shell/lib/coresight.sh b/tools/perf/tests/shell/lib/coresight.sh index 6c3d34ec64d8..11ed2c25ed91 100644 --- a/tools/perf/tests/shell/lib/coresight.sh +++ b/tools/perf/tests/shell/lib/coresight.sh @@ -17,6 +17,8 @@ DIR="$TOOLS/$TEST" BIN="$DIR/$TEST" # If the test tool/binary does not exist and is executable then skip the test if ! test -x "$BIN"; then exit 2; fi +# If CoreSight is not available, skip the test +perf list cs_etm | grep -q cs_etm || exit 2 DATD="." # If the data dir env is set then make the data dir use that instead of ./ if test -n "$PERF_TEST_CORESIGHT_DATADIR"; then From 79a3371bdf453bedb9d2c80df5adc201dddc11b5 Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Tue, 17 Oct 2023 13:23:42 -0700 Subject: [PATCH 407/475] perf bench sched pipe: Add -G/--cgroups option The -G/--cgroups option is to put sender and receiver in different cgroups in order to measure cgroup context switch overheads. Users need to make sure the cgroups exist and accessible. The following example should the effect of this change. Please don't forget taskset before the perf bench to measure cgroup switches properly. Otherwise each task would run on a different CPU and generate cgroup switches regardless of this change. # perf stat -e context-switches,cgroup-switches \ > taskset -c 0 perf bench sched pipe -l 10000 > /dev/null Performance counter stats for 'taskset -c 0 perf bench sched pipe -l 10000': 20,001 context-switches 2 cgroup-switches 0.053449651 seconds time elapsed 0.011286000 seconds user 0.041869000 seconds sys # perf stat -e context-switches,cgroup-switches \ > taskset -c 0 perf bench sched pipe -l 10000 -G AAA,BBB > /dev/null Performance counter stats for 'taskset -c 0 perf bench sched pipe -l 10000 -G AAA,BBB': 20,001 context-switches 20,001 cgroup-switches 0.052768627 seconds time elapsed 0.006284000 seconds user 0.046266000 seconds sys Signed-off-by: Namhyung Kim Tested-by: Arnaldo Carvalho de Melo Link: https://lore.kernel.org/r/20231017202342.1353124-1-namhyung@kernel.org --- tools/perf/Documentation/perf-bench.txt | 19 ++++ tools/perf/bench/sched-pipe.c | 132 +++++++++++++++++++++++- 2 files changed, 147 insertions(+), 4 deletions(-) diff --git a/tools/perf/Documentation/perf-bench.txt b/tools/perf/Documentation/perf-bench.txt index ca5789625cd2..8331bd28b10e 100644 --- a/tools/perf/Documentation/perf-bench.txt +++ b/tools/perf/Documentation/perf-bench.txt @@ -124,6 +124,14 @@ Options of *pipe* --loop=:: Specify number of loops. +-G:: +--cgroups=:: +Names of cgroups for sender and receiver, separated by a comma. +This is useful to check cgroup context switching overhead. +Note that perf doesn't create nor delete the cgroups, so users should +make sure that the cgroups exist and are accessible before use. + + Example of *pipe* ^^^^^^^^^^^^^^^^^ @@ -141,6 +149,17 @@ Example of *pipe* Total time:0.016 sec 16.948000 usecs/op 59004 ops/sec + +% perf bench sched pipe -G AAA,BBB +(executing 1000000 pipe operations between cgroups) +# Running 'sched/pipe' benchmark: +# Executed 1000000 pipe operations between two processes + + Total time: 6.886 [sec] + + 6.886208 usecs/op + 145217 ops/sec + --------------------- SUITES FOR 'syscall' diff --git a/tools/perf/bench/sched-pipe.c b/tools/perf/bench/sched-pipe.c index a960e7a93aec..3af6d3c55aba 100644 --- a/tools/perf/bench/sched-pipe.c +++ b/tools/perf/bench/sched-pipe.c @@ -10,7 +10,9 @@ * Ported to perf by Hitoshi Mitake */ #include +#include #include "bench.h" +#include "util/cgroup.h" #include #include @@ -19,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -31,6 +34,7 @@ struct thread_data { int nr; int pipe_read; int pipe_write; + bool cgroup_failed; pthread_t pthread; }; @@ -40,9 +44,48 @@ static int loops = LOOPS_DEFAULT; /* Use processes by default: */ static bool threaded; +static char *cgrp_names[2]; +static struct cgroup *cgrps[2]; + +static int parse_two_cgroups(const struct option *opt __maybe_unused, + const char *str, int unset __maybe_unused) +{ + char *p = strdup(str); + char *q; + int ret = -1; + + if (p == NULL) { + fprintf(stderr, "memory allocation failure\n"); + return -1; + } + + q = strchr(p, ','); + if (q == NULL) { + fprintf(stderr, "it should have two cgroup names: %s\n", p); + goto out; + } + *q = '\0'; + + cgrp_names[0] = strdup(p); + cgrp_names[1] = strdup(q + 1); + + if (cgrp_names[0] == NULL || cgrp_names[1] == NULL) { + fprintf(stderr, "memory allocation failure\n"); + goto out; + } + ret = 0; + +out: + free(p); + return ret; +} + static const struct option options[] = { OPT_INTEGER('l', "loop", &loops, "Specify number of loops"), OPT_BOOLEAN('T', "threaded", &threaded, "Specify threads/process based task setup"), + OPT_CALLBACK('G', "cgroups", NULL, "SEND,RECV", + "Put sender and receivers in given cgroups", + parse_two_cgroups), OPT_END() }; @@ -51,12 +94,89 @@ static const char * const bench_sched_pipe_usage[] = { NULL }; +static int enter_cgroup(int nr) +{ + char buf[32]; + int fd, len, ret; + int saved_errno; + struct cgroup *cgrp; + pid_t pid; + + if (cgrp_names[nr] == NULL) + return 0; + + if (cgrps[nr] == NULL) { + cgrps[nr] = cgroup__new(cgrp_names[nr], /*do_open=*/true); + if (cgrps[nr] == NULL) + goto err; + } + cgrp = cgrps[nr]; + + if (threaded) + pid = syscall(__NR_gettid); + else + pid = getpid(); + + snprintf(buf, sizeof(buf), "%d\n", pid); + len = strlen(buf); + + /* try cgroup v2 interface first */ + if (threaded) + fd = openat(cgrp->fd, "cgroup.threads", O_WRONLY); + else + fd = openat(cgrp->fd, "cgroup.procs", O_WRONLY); + + /* try cgroup v1 if failed */ + if (fd < 0 && errno == ENOENT) + fd = openat(cgrp->fd, "tasks", O_WRONLY); + + if (fd < 0) + goto err; + + ret = write(fd, buf, len); + close(fd); + + if (ret != len) { + printf("Cannot enter to cgroup: %s\n", cgrp->name); + return -1; + } + return 0; + +err: + saved_errno = errno; + printf("Failed to open cgroup file in %s\n", cgrp_names[nr]); + + if (saved_errno == ENOENT) { + char mnt[PATH_MAX]; + + if (cgroupfs_find_mountpoint(mnt, sizeof(mnt), "perf_event") == 0) + printf(" Hint: create the cgroup first, like 'mkdir %s/%s'\n", + mnt, cgrp_names[nr]); + } else if (saved_errno == EACCES && geteuid() > 0) { + printf(" Hint: try to run as root\n"); + } + + return -1; +} + +static void exit_cgroup(int nr) +{ + cgroup__put(cgrps[nr]); + free(cgrp_names[nr]); +} + static void *worker_thread(void *__tdata) { struct thread_data *td = __tdata; int m = 0, i; int ret; + ret = enter_cgroup(td->nr); + if (ret < 0) { + td->cgroup_failed = true; + return NULL; + } + for (i = 0; i < loops; i++) { if (!td->nr) { ret = read(td->pipe_read, &m, sizeof(int)); @@ -76,7 +196,8 @@ static void *worker_thread(void *__tdata) int bench_sched_pipe(int argc, const char **argv) { - struct thread_data threads[2], *td; + struct thread_data threads[2] = {}; + struct thread_data *td; int pipe_1[2], pipe_2[2]; struct timeval start, stop, diff; unsigned long long result_usec = 0; @@ -112,9 +233,7 @@ int bench_sched_pipe(int argc, const char **argv) } } - if (threaded) { - for (t = 0; t < nr_threads; t++) { td = threads + t; @@ -128,7 +247,6 @@ int bench_sched_pipe(int argc, const char **argv) ret = pthread_join(td->pthread, NULL); BUG_ON(ret); } - } else { pid = fork(); assert(pid >= 0); @@ -147,6 +265,12 @@ int bench_sched_pipe(int argc, const char **argv) gettimeofday(&stop, NULL); timersub(&stop, &start, &diff); + exit_cgroup(0); + exit_cgroup(1); + + if (threads[0].cgroup_failed || threads[1].cgroup_failed) + return 0; + switch (bench_format) { case BENCH_FORMAT_DEFAULT: printf("# Executed %d pipe operations between two %s\n\n", From e093a222d7cba1eb6c36887e58ce8a4ff249f1c6 Mon Sep 17 00:00:00 2001 From: Yang Jihong Date: Mon, 23 Oct 2023 03:31:44 +0000 Subject: [PATCH 408/475] perf evsel: Rename evsel__increase_rlimit to rlimit__increase_nofile evsel__increase_rlimit() helper does nothing with evsel, and description of the functionality is inaccurate, rename it and move to util/rlimit.c. By the way, fix a checkppatch warning about misplaced license tag: WARNING: Misplaced SPDX-License-Identifier tag - use line 1 instead #160: FILE: tools/perf/util/rlimit.h:3: /* SPDX-License-Identifier: LGPL-2.1 */ No functional change. Signed-off-by: Yang Jihong Link: https://lore.kernel.org/r/20231023033144.1011896-1-yangjihong1@huawei.com Signed-off-by: Namhyung Kim --- tools/perf/util/data.c | 4 ++-- tools/perf/util/evsel.c | 30 ++---------------------------- tools/perf/util/evsel.h | 3 --- tools/perf/util/rlimit.c | 28 ++++++++++++++++++++++++++++ tools/perf/util/rlimit.h | 11 ++++++++++- 5 files changed, 42 insertions(+), 34 deletions(-) diff --git a/tools/perf/util/data.c b/tools/perf/util/data.c index 098f9e3bb2e7..c29d8a382b19 100644 --- a/tools/perf/util/data.c +++ b/tools/perf/util/data.c @@ -17,7 +17,7 @@ #include "util.h" // rm_rf_perf_data() #include "debug.h" #include "header.h" -#include "evsel.h" +#include "rlimit.h" #include static void close_dir(struct perf_data_file *files, int nr) @@ -64,7 +64,7 @@ retry_open: * perf record needs at least 6 fds per CPU. * When we run out of them try to increase the limits. */ - if (errno == EMFILE && evsel__increase_rlimit(&set_rlimit)) + if (errno == EMFILE && rlimit__increase_nofile(&set_rlimit)) goto retry_open; ret = -errno; diff --git a/tools/perf/util/evsel.c b/tools/perf/util/evsel.c index d5363d23f5d3..72a5dfc38d38 100644 --- a/tools/perf/util/evsel.c +++ b/tools/perf/util/evsel.c @@ -49,6 +49,7 @@ #include "off_cpu.h" #include "pmu.h" #include "pmus.h" +#include "rlimit.h" #include "../perf-sys.h" #include "util/parse-branch-options.h" #include "util/bpf-filter.h" @@ -1989,33 +1990,6 @@ bool evsel__detect_missing_features(struct evsel *evsel) } } -bool evsel__increase_rlimit(enum rlimit_action *set_rlimit) -{ - int old_errno; - struct rlimit l; - - if (*set_rlimit < INCREASED_MAX) { - old_errno = errno; - - if (getrlimit(RLIMIT_NOFILE, &l) == 0) { - if (*set_rlimit == NO_CHANGE) { - l.rlim_cur = l.rlim_max; - } else { - l.rlim_cur = l.rlim_max + 1000; - l.rlim_max = l.rlim_cur; - } - if (setrlimit(RLIMIT_NOFILE, &l) == 0) { - (*set_rlimit) += 1; - errno = old_errno; - return true; - } - } - errno = old_errno; - } - - return false; -} - static int evsel__open_cpu(struct evsel *evsel, struct perf_cpu_map *cpus, struct perf_thread_map *threads, int start_cpu_map_idx, int end_cpu_map_idx) @@ -2143,7 +2117,7 @@ try_fallback: * perf stat needs between 5 and 22 fds per CPU. When we run out * of them try to increase the limits. */ - if (err == -EMFILE && evsel__increase_rlimit(&set_rlimit)) + if (err == -EMFILE && rlimit__increase_nofile(&set_rlimit)) goto retry_open; if (err != -EINVAL || idx > 0 || thread > 0) diff --git a/tools/perf/util/evsel.h b/tools/perf/util/evsel.h index 815be2491938..d791316a1792 100644 --- a/tools/perf/util/evsel.h +++ b/tools/perf/util/evsel.h @@ -330,9 +330,6 @@ int evsel__prepare_open(struct evsel *evsel, struct perf_cpu_map *cpus, struct perf_thread_map *threads); bool evsel__detect_missing_features(struct evsel *evsel); -enum rlimit_action { NO_CHANGE, SET_TO_MAX, INCREASED_MAX }; -bool evsel__increase_rlimit(enum rlimit_action *set_rlimit); - bool evsel__precise_ip_fallback(struct evsel *evsel); struct perf_sample; diff --git a/tools/perf/util/rlimit.c b/tools/perf/util/rlimit.c index 13521d392a22..f857405fe1aa 100644 --- a/tools/perf/util/rlimit.c +++ b/tools/perf/util/rlimit.c @@ -1,5 +1,6 @@ /* SPDX-License-Identifier: LGPL-2.1 */ +#include #include "util/debug.h" #include "util/rlimit.h" #include @@ -27,3 +28,30 @@ void rlimit__bump_memlock(void) } } } + +bool rlimit__increase_nofile(enum rlimit_action *set_rlimit) +{ + int old_errno; + struct rlimit l; + + if (*set_rlimit < INCREASED_MAX) { + old_errno = errno; + + if (getrlimit(RLIMIT_NOFILE, &l) == 0) { + if (*set_rlimit == NO_CHANGE) { + l.rlim_cur = l.rlim_max; + } else { + l.rlim_cur = l.rlim_max + 1000; + l.rlim_max = l.rlim_cur; + } + if (setrlimit(RLIMIT_NOFILE, &l) == 0) { + (*set_rlimit) += 1; + errno = old_errno; + return true; + } + } + errno = old_errno; + } + + return false; +} diff --git a/tools/perf/util/rlimit.h b/tools/perf/util/rlimit.h index 9f59d8e710a3..19050d7fb9d7 100644 --- a/tools/perf/util/rlimit.h +++ b/tools/perf/util/rlimit.h @@ -1,6 +1,15 @@ +/* SPDX-License-Identifier: LGPL-2.1 */ #ifndef __PERF_RLIMIT_H_ #define __PERF_RLIMIT_H_ -/* SPDX-License-Identifier: LGPL-2.1 */ + +enum rlimit_action { + NO_CHANGE, + SET_TO_MAX, + INCREASED_MAX +}; void rlimit__bump_memlock(void); + +bool rlimit__increase_nofile(enum rlimit_action *set_rlimit); + #endif // __PERF_RLIMIT_H_ From d99317f214ca3d381f7b17a75c41263885664e06 Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Fri, 20 Oct 2023 13:47:39 -0700 Subject: [PATCH 409/475] perf lock contention: Clear lock addr after use It checks the current lock to calculated the delta of contention time. The address is saved in the tstamp map which is allocated at begining of contention and released at end of contention. But it's possible for bpf_map_delete_elem() to fail. In that case, the element in the tstamp map kept for the current lock and it makes the next contention for the same lock tracked incorrectly. Specificially the next contention begin will see the existing element for the task and it'd just return. Then the next contention end will see the element and calculate the time using the timestamp for the previous begin. This can result in a large value for two small contentions happened from time to time. Let's clear the lock address so that it can be updated next time even if the bpf_map_delete_elem() failed. Signed-off-by: Namhyung Kim Acked-by: Ian Rogers Cc: Hao Luo Cc: Song Liu Cc: bpf@vger.kernel.org Link: https://lore.kernel.org/r/20231020204741.1869520-1-namhyung@kernel.org --- tools/perf/util/bpf_skel/lock_contention.bpf.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/perf/util/bpf_skel/lock_contention.bpf.c b/tools/perf/util/bpf_skel/lock_contention.bpf.c index 4900a5dfb4a4..b11179452e19 100644 --- a/tools/perf/util/bpf_skel/lock_contention.bpf.c +++ b/tools/perf/util/bpf_skel/lock_contention.bpf.c @@ -389,6 +389,7 @@ int contention_end(u64 *ctx) duration = bpf_ktime_get_ns() - pelem->timestamp; if ((__s64)duration < 0) { + pelem->lock = 0; bpf_map_delete_elem(&tstamp, &pid); __sync_fetch_and_add(&time_fail, 1); return 0; @@ -422,6 +423,7 @@ int contention_end(u64 *ctx) data = bpf_map_lookup_elem(&lock_stat, &key); if (!data) { if (data_map_full) { + pelem->lock = 0; bpf_map_delete_elem(&tstamp, &pid); __sync_fetch_and_add(&data_fail, 1); return 0; @@ -445,6 +447,7 @@ int contention_end(u64 *ctx) data_map_full = 1; __sync_fetch_and_add(&data_fail, 1); } + pelem->lock = 0; bpf_map_delete_elem(&tstamp, &pid); return 0; } @@ -458,6 +461,7 @@ int contention_end(u64 *ctx) if (data->min_time > duration) data->min_time = duration; + pelem->lock = 0; bpf_map_delete_elem(&tstamp, &pid); return 0; } From 6a070573f290f99a6129ac3e13b9df521a1a65de Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Fri, 20 Oct 2023 13:47:40 -0700 Subject: [PATCH 410/475] perf lock contention: Check race in tstamp elem creation When pelem is NULL, it'd create a new entry with zero data. But it might be preempted by IRQ/NMI just before calling bpf_map_update_elem() then there's a chance to call it twice for the same pid. So it'd be better to use BPF_NOEXIST flag and check the return value to prevent the race. Signed-off-by: Namhyung Kim Acked-by: Ian Rogers Cc: Hao Luo Cc: Song Liu Cc: bpf@vger.kernel.org Link: https://lore.kernel.org/r/20231020204741.1869520-2-namhyung@kernel.org --- tools/perf/util/bpf_skel/lock_contention.bpf.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/perf/util/bpf_skel/lock_contention.bpf.c b/tools/perf/util/bpf_skel/lock_contention.bpf.c index b11179452e19..69d31fd77cd0 100644 --- a/tools/perf/util/bpf_skel/lock_contention.bpf.c +++ b/tools/perf/util/bpf_skel/lock_contention.bpf.c @@ -328,7 +328,11 @@ int contention_begin(u64 *ctx) if (pelem == NULL) { struct tstamp_data zero = {}; - bpf_map_update_elem(&tstamp, &pid, &zero, BPF_ANY); + if (bpf_map_update_elem(&tstamp, &pid, &zero, BPF_NOEXIST) < 0) { + __sync_fetch_and_add(&task_fail, 1); + return 0; + } + pelem = bpf_map_lookup_elem(&tstamp, &pid); if (pelem == NULL) { __sync_fetch_and_add(&task_fail, 1); From b5711042a1c8cc88ed40a5ebf612b36e83a4e2e4 Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Fri, 20 Oct 2023 13:47:41 -0700 Subject: [PATCH 411/475] perf lock contention: Use per-cpu array map for spinlocks Currently lock contention timestamp is maintained in a hash map keyed by pid. That means it needs to get and release a map element (which is proctected by spinlock!) on each contention begin and end pair. This can impact on performance if there are a lot of contention (usually from spinlocks). It used to go with task local storage but it had an issue on memory allocation in some critical paths. Although it's addressed in recent kernels IIUC, the tool should support old kernels too. So it cannot simply switch to the task local storage at least for now. As spinlocks create lots of contention and they disabled preemption during the spinning, it can use per-cpu array to keep the timestamp to avoid overhead in hashmap update and delete. In contention_begin, it's easy to check the lock types since it can see the flags. But contention_end cannot see it. So let's try to per-cpu array first (unconditionally) if it has an active element (lock != 0). Then it should be used and per-task tstamp map should not be used until the per-cpu array element is cleared which means nested spinlock contention (if any) was finished and it nows see (the outer) lock. Signed-off-by: Namhyung Kim Acked-by: Ian Rogers Cc: Hao Luo Cc: Song Liu Cc: bpf@vger.kernel.org Link: https://lore.kernel.org/r/20231020204741.1869520-3-namhyung@kernel.org --- .../perf/util/bpf_skel/lock_contention.bpf.c | 89 +++++++++++++++---- 1 file changed, 72 insertions(+), 17 deletions(-) diff --git a/tools/perf/util/bpf_skel/lock_contention.bpf.c b/tools/perf/util/bpf_skel/lock_contention.bpf.c index 69d31fd77cd0..95cd8414f6ef 100644 --- a/tools/perf/util/bpf_skel/lock_contention.bpf.c +++ b/tools/perf/util/bpf_skel/lock_contention.bpf.c @@ -42,6 +42,14 @@ struct { __uint(max_entries, MAX_ENTRIES); } tstamp SEC(".maps"); +/* maintain per-CPU timestamp at the beginning of contention */ +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(struct tstamp_data)); + __uint(max_entries, 1); +} tstamp_cpu SEC(".maps"); + /* actual lock contention statistics */ struct { __uint(type, BPF_MAP_TYPE_HASH); @@ -311,34 +319,57 @@ static inline __u32 check_lock_type(__u64 lock, __u32 flags) return 0; } -SEC("tp_btf/contention_begin") -int contention_begin(u64 *ctx) +static inline struct tstamp_data *get_tstamp_elem(__u32 flags) { __u32 pid; struct tstamp_data *pelem; - if (!enabled || !can_record(ctx)) - return 0; + /* Use per-cpu array map for spinlock and rwlock */ + if (flags == (LCB_F_SPIN | LCB_F_READ) || flags == LCB_F_SPIN || + flags == (LCB_F_SPIN | LCB_F_WRITE)) { + __u32 idx = 0; + + pelem = bpf_map_lookup_elem(&tstamp_cpu, &idx); + /* Do not update the element for nested locks */ + if (pelem && pelem->lock) + pelem = NULL; + return pelem; + } pid = bpf_get_current_pid_tgid(); pelem = bpf_map_lookup_elem(&tstamp, &pid); + /* Do not update the element for nested locks */ if (pelem && pelem->lock) - return 0; + return NULL; if (pelem == NULL) { struct tstamp_data zero = {}; if (bpf_map_update_elem(&tstamp, &pid, &zero, BPF_NOEXIST) < 0) { __sync_fetch_and_add(&task_fail, 1); - return 0; + return NULL; } pelem = bpf_map_lookup_elem(&tstamp, &pid); if (pelem == NULL) { __sync_fetch_and_add(&task_fail, 1); - return 0; + return NULL; } } + return pelem; +} + +SEC("tp_btf/contention_begin") +int contention_begin(u64 *ctx) +{ + struct tstamp_data *pelem; + + if (!enabled || !can_record(ctx)) + return 0; + + pelem = get_tstamp_elem(ctx[1]); + if (pelem == NULL) + return 0; pelem->timestamp = bpf_ktime_get_ns(); pelem->lock = (__u64)ctx[0]; @@ -377,24 +408,42 @@ int contention_begin(u64 *ctx) SEC("tp_btf/contention_end") int contention_end(u64 *ctx) { - __u32 pid; + __u32 pid = 0, idx = 0; struct tstamp_data *pelem; struct contention_key key = {}; struct contention_data *data; __u64 duration; + bool need_delete = false; if (!enabled) return 0; - pid = bpf_get_current_pid_tgid(); - pelem = bpf_map_lookup_elem(&tstamp, &pid); - if (!pelem || pelem->lock != ctx[0]) - return 0; + /* + * For spinlock and rwlock, it needs to get the timestamp for the + * per-cpu map. However, contention_end does not have the flags + * so it cannot know whether it reads percpu or hash map. + * + * Try per-cpu map first and check if there's active contention. + * If it is, do not read hash map because it cannot go to sleeping + * locks before releasing the spinning locks. + */ + pelem = bpf_map_lookup_elem(&tstamp_cpu, &idx); + if (pelem && pelem->lock) { + if (pelem->lock != ctx[0]) + return 0; + } else { + pid = bpf_get_current_pid_tgid(); + pelem = bpf_map_lookup_elem(&tstamp, &pid); + if (!pelem || pelem->lock != ctx[0]) + return 0; + need_delete = true; + } duration = bpf_ktime_get_ns() - pelem->timestamp; if ((__s64)duration < 0) { pelem->lock = 0; - bpf_map_delete_elem(&tstamp, &pid); + if (need_delete) + bpf_map_delete_elem(&tstamp, &pid); __sync_fetch_and_add(&time_fail, 1); return 0; } @@ -406,8 +455,11 @@ int contention_end(u64 *ctx) case LOCK_AGGR_TASK: if (lock_owner) key.pid = pelem->flags; - else + else { + if (!need_delete) + pid = bpf_get_current_pid_tgid(); key.pid = pid; + } if (needs_callstack) key.stack_id = pelem->stack_id; break; @@ -428,7 +480,8 @@ int contention_end(u64 *ctx) if (!data) { if (data_map_full) { pelem->lock = 0; - bpf_map_delete_elem(&tstamp, &pid); + if (need_delete) + bpf_map_delete_elem(&tstamp, &pid); __sync_fetch_and_add(&data_fail, 1); return 0; } @@ -452,7 +505,8 @@ int contention_end(u64 *ctx) __sync_fetch_and_add(&data_fail, 1); } pelem->lock = 0; - bpf_map_delete_elem(&tstamp, &pid); + if (need_delete) + bpf_map_delete_elem(&tstamp, &pid); return 0; } @@ -466,7 +520,8 @@ int contention_end(u64 *ctx) data->min_time = duration; pelem->lock = 0; - bpf_map_delete_elem(&tstamp, &pid); + if (need_delete) + bpf_map_delete_elem(&tstamp, &pid); return 0; } From a6e4a4a14a8eb6376278db0d3441a2ba124cc30c Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Tue, 24 Oct 2023 17:31:21 -0700 Subject: [PATCH 412/475] perf report: Fix hierarchy mode on pipe input The hierarchy mode needs to setup output formats for each evsel. Normally setup_sorting() handles this at the beginning, but it cannot do that if data comes from a pipe since there's no evsel info before reading the data. And then perf report cannot process the samples in hierarchy mode and think as if there's no sample. Let's check the condition and setup the output formats after reading data so that it can find evsels. Before: $ ./perf record -o- true | ./perf report -i- --hierarchy -q [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.000 MB - ] Error: The - data has no samples! After: $ ./perf record -o- true | ./perf report -i- --hierarchy -q [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.000 MB - ] 94.76% true 94.76% [kernel.kallsyms] 94.76% [k] filemap_fault 5.24% perf-ex 5.24% [kernel.kallsyms] 5.06% [k] __memset 0.18% [k] native_write_msr Acked-by: Ian Rogers Link: https://lore.kernel.org/r/20231025003121.2811738-1-namhyung@kernel.org Signed-off-by: Namhyung Kim --- tools/perf/builtin-report.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tools/perf/builtin-report.c b/tools/perf/builtin-report.c index dcedfe00f04d..ca8f2331795c 100644 --- a/tools/perf/builtin-report.c +++ b/tools/perf/builtin-report.c @@ -691,10 +691,25 @@ static int report__browse_hists(struct report *rep) static int report__collapse_hists(struct report *rep) { + struct perf_session *session = rep->session; + struct evlist *evlist = session->evlist; struct ui_progress prog; struct evsel *pos; int ret = 0; + /* + * The pipe data needs to setup hierarchy hpp formats now, because it + * cannot know about evsels in the data before reading the data. The + * normal file data saves the event (attribute) info in the header + * section, but pipe does not have the luxury. + */ + if (perf_data__is_pipe(session->data)) { + if (perf_hpp__setup_hists_formats(&perf_hpp_list, evlist) < 0) { + ui__error("Failed to setup hierachy output formats\n"); + return -1; + } + } + ui_progress__init(&prog, rep->nr_entries, "Merging related events..."); evlist__for_each_entry(rep->session->evlist, pos) { From b27778ed5d1d21cdb4fa83ff5185e7e438fd653f Mon Sep 17 00:00:00 2001 From: Arnaldo Carvalho de Melo Date: Wed, 25 Oct 2023 08:23:05 -0300 Subject: [PATCH 413/475] perf build: Address stray '\' before # that is warned about since grep 3.8 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To address this grep 3.8 warning: grep: warning: stray \ before # We needed to remove the '' around the grep expression and keep the \ before # so that it is escaped by the $(shell grep ...) and thus doesn't get to grep. We need that \ before the #, otherwise we get this: Makefile.perf:364: *** unterminated call to function 'shell': missing ')'. Stop. As everything after the # will be considered a comment. Removing the single quotes needs some more escaping so that _some_ of the escaped chars gets to grep, like the '\|' that becomes '\\\|´. Running on debian:10, where there is no libtraceevent-devel available, we get: Makefile.perf:367: *** PYTHON_EXT_SRCS= util/python.c ../lib/ctype.c util/cap.c util/evlist.c util/evsel.c util/evsel_fprintf.c util/perf_event_attr_fprintf.c util/cpumap.c util/memswap.c util/mmap.c util/namespaces.c ../lib/bitmap.c ../lib/find_bit.c ../lib/list_sort.c ../lib/hweight.c ../lib/string.c ../lib/vsprintf.c util/thread_map.c util/util.c util/cgroup.c util/parse-branch-options.c util/rblist.c util/counts.c util/print_binary.c util/strlist.c ../lib/rbtree.c util/string.c util/symbol_fprintf.c util/units.c util/affinity.c util/rwsem.c util/hashmap.c util/perf_regs.c util/fncache.c util/perf-regs-arch/perf_regs_aarch64.c util/perf-regs-arch/perf_regs_arm.c util/perf-regs-arch/perf_regs_csky.c util/perf-regs-arch/perf_regs_loongarch.c util/perf-regs-arch/perf_regs_mips.c util/perf-regs-arch/perf_regs_powerpc.c util/perf-regs-arch/perf_regs_riscv.c util/perf-regs-arch/perf_regs_s390.c util/perf-regs-arch/perf_regs_x86.c. Stop. make[1]: *** [Makefile.perf:242: sub-make] Error 2 I.e. both the comments and the util/trace-event.c were removed. When using: msg := $(error PYTHON_EXT_SRCS=$(PYTHON_EXT_SRCS)) While on the more recent fedora:38, with the new grep and make packages and libtraceevent-devel installed: Makefile.perf:367: *** PYTHON_EXT_SRCS= util/python.c ../lib/ctype.c util/cap.c util/evlist.c util/evsel.c util/evsel_fprintf.c util/perf_event_attr_fprintf.c util/cpumap.c util/memswap.c util/mmap.c util/namespaces.c ../lib/bitmap.c ../lib/find_bit.c ../lib/list_sort.c ../lib/hweight.c ../lib/string.c ../lib/vsprintf.c util/thread_map.c util/util.c util/cgroup.c util/parse-branch-options.c util/rblist.c util/counts.c util/print_binary.c util/strlist.c util/trace-event.c ../lib/rbtree.c util/string.c util/symbol_fprintf.c util/units.c util/affinity.c util/rwsem.c util/hashmap.c util/perf_regs.c util/fncache.c util/perf-regs-arch/perf_regs_aarch64.c util/perf-regs-arch/perf_regs_arm.c util/perf-regs-arch/perf_regs_csky.c util/perf-regs-arch/perf_regs_loongarch.c util/perf-regs-arch/perf_regs_mips.c util/perf-regs-arch/perf_regs_powerpc.c util/perf-regs-arch/perf_regs_riscv.c util/perf-regs-arch/perf_regs_s390.c util/perf-regs-arch/perf_regs_x86.c. Stop. make[1]: *** [Makefile.perf:242: sub-make] Error 2 make: *** [Makefile:113: install-bin] Error 2 make: Leaving directory '/home/acme/git/perf-tools-next/tools/perf' $ I.e. only the comments were removed. If we build it on the same fedora:38 system, but using NO_LIBTRACEEVENT=1 $ make NO_LIBTRACEEVENT=1 CORESIGHT=1 O=/tmp/build/$(basename $PWD) -C tools/perf install-bin Makefile.perf:367: *** PYTHON_EXT_SRCS= util/python.c ../lib/ctype.c util/cap.c util/evlist.c util/evsel.c util/evsel_fprintf.c util/perf_event_attr_fprintf.c util/cpumap.c util/memswap.c util/mmap.c util/namespaces.c ../lib/bitmap.c ../lib/find_bit.c ../lib/list_sort.c ../lib/hweight.c ../lib/string.c ../lib/vsprintf.c util/thread_map.c util/util.c util/cgroup.c util/parse-branch-options.c util/rblist.c util/counts.c util/print_binary.c util/strlist.c ../lib/rbtree.c util/string.c util/symbol_fprintf.c util/units.c util/affinity.c util/rwsem.c util/hashmap.c util/perf_regs.c util/fncache.c util/perf-regs-arch/perf_regs_aarch64.c util/perf-regs-arch/perf_regs_arm.c util/perf-regs-arch/perf_regs_csky.c util/perf-regs-arch/perf_regs_loongarch.c util/perf-regs-arch/perf_regs_mips.c util/perf-regs-arch/perf_regs_powerpc.c util/perf-regs-arch/perf_regs_riscv.c util/perf-regs-arch/perf_regs_s390.c util/perf-regs-arch/perf_regs_x86.c. Stop. make[1]: *** [Makefile.perf:242: sub-make] Error 2 make: *** [Makefile:113: install-bin] Error 2 make: Leaving directory '/home/acme/git/perf-tools-next/tools/perf' $ Both comments and the util/trace-event.c file removed. Signed-off-by: Arnaldo Carvalho de Melo Acked-by: Ian Rogers Link: https://lore.kernel.org/r/ZTj6mfM9UqY2DggC@kernel.org Signed-off-by: Namhyung Kim --- tools/perf/Makefile.perf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/perf/Makefile.perf b/tools/perf/Makefile.perf index 456872ac410d..d80dcaa5a1e3 100644 --- a/tools/perf/Makefile.perf +++ b/tools/perf/Makefile.perf @@ -361,7 +361,7 @@ python-clean := $(call QUIET_CLEAN, python) $(RM) -r $(PYTHON_EXTBUILD) $(OUTPUT ifeq ($(CONFIG_LIBTRACEEVENT),y) PYTHON_EXT_SRCS := $(shell grep -v ^\# util/python-ext-sources) else - PYTHON_EXT_SRCS := $(shell grep -v '^\#\|util/trace-event.c' util/python-ext-sources) + PYTHON_EXT_SRCS := $(shell grep -v ^\#\\\|util/trace-event.c util/python-ext-sources) endif PYTHON_EXT_DEPS := util/python-ext-sources util/setup.py $(LIBAPI) From 7a8f349e9d14da1633e021c2fe87234b21ab181d Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Tue, 24 Oct 2023 15:23:04 -0700 Subject: [PATCH 414/475] perf rwsem: Add debug mode that uses a mutex Mutex error check will capture trying to take the lock recursively and other problems that rwlock won't. At the expense of concurrency, adda debug mode that uses a mutex in place of a rwsem. Signed-off-by: Ian Rogers Cc: K Prateek Nayak Cc: Ravi Bangoria Cc: Sandipan Das Cc: Anshuman Khandual Cc: German Gomez Cc: James Clark Cc: Nick Terrell Cc: Sean Christopherson Cc: Changbin Du Cc: liuwenyu Cc: Yang Jihong Cc: Masami Hiramatsu Cc: Miguel Ojeda Cc: Song Liu Cc: Leo Yan Cc: Kajol Jain Cc: Andi Kleen Cc: Kan Liang Cc: Athira Rajeev Cc: Yanteng Si Cc: Liam Howlett Cc: Paolo Bonzini Link: https://lore.kernel.org/r/20231024222353.3024098-2-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/util/rwsem.c | 34 ++++++++++++++++++++++++++++++++++ tools/perf/util/rwsem.h | 11 +++++++++++ 2 files changed, 45 insertions(+) diff --git a/tools/perf/util/rwsem.c b/tools/perf/util/rwsem.c index f3d29d8ddc99..5109167f27f7 100644 --- a/tools/perf/util/rwsem.c +++ b/tools/perf/util/rwsem.c @@ -2,32 +2,66 @@ #include "util.h" #include "rwsem.h" +#if RWS_ERRORCHECK +#include "mutex.h" +#endif + int init_rwsem(struct rw_semaphore *sem) { +#if RWS_ERRORCHECK + mutex_init(&sem->mtx); + return 0; +#else return pthread_rwlock_init(&sem->lock, NULL); +#endif } int exit_rwsem(struct rw_semaphore *sem) { +#if RWS_ERRORCHECK + mutex_destroy(&sem->mtx); + return 0; +#else return pthread_rwlock_destroy(&sem->lock); +#endif } int down_read(struct rw_semaphore *sem) { +#if RWS_ERRORCHECK + mutex_lock(&sem->mtx); + return 0; +#else return perf_singlethreaded ? 0 : pthread_rwlock_rdlock(&sem->lock); +#endif } int up_read(struct rw_semaphore *sem) { +#if RWS_ERRORCHECK + mutex_unlock(&sem->mtx); + return 0; +#else return perf_singlethreaded ? 0 : pthread_rwlock_unlock(&sem->lock); +#endif } int down_write(struct rw_semaphore *sem) { +#if RWS_ERRORCHECK + mutex_lock(&sem->mtx); + return 0; +#else return perf_singlethreaded ? 0 : pthread_rwlock_wrlock(&sem->lock); +#endif } int up_write(struct rw_semaphore *sem) { +#if RWS_ERRORCHECK + mutex_unlock(&sem->mtx); + return 0; +#else return perf_singlethreaded ? 0 : pthread_rwlock_unlock(&sem->lock); +#endif } diff --git a/tools/perf/util/rwsem.h b/tools/perf/util/rwsem.h index 94565ad4d494..ef5cbc31d967 100644 --- a/tools/perf/util/rwsem.h +++ b/tools/perf/util/rwsem.h @@ -2,9 +2,20 @@ #define _PERF_RWSEM_H #include +#include "mutex.h" + +/* + * Mutexes have additional error checking. Enable to use a mutex rather than a + * rwlock for debugging. + */ +#define RWS_ERRORCHECK 0 struct rw_semaphore { +#if RWS_ERRORCHECK + struct mutex mtx; +#else pthread_rwlock_t lock; +#endif }; int init_rwsem(struct rw_semaphore *sem); From ab8ce150781d326c6bfbe1e09f175ffde1186f80 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Tue, 24 Oct 2023 15:23:05 -0700 Subject: [PATCH 415/475] perf machine: Avoid out of bounds LBR memory read Running perf top with address sanitizer and "--call-graph=lbr" fails due to reading sample 0 when no samples exist. Add a guard to prevent this. Fixes: e2b23483eb1d ("perf machine: Factor out lbr_callchain_add_lbr_ip()") Signed-off-by: Ian Rogers Cc: K Prateek Nayak Cc: Ravi Bangoria Cc: Sandipan Das Cc: Anshuman Khandual Cc: German Gomez Cc: James Clark Cc: Nick Terrell Cc: Sean Christopherson Cc: Changbin Du Cc: liuwenyu Cc: Yang Jihong Cc: Masami Hiramatsu Cc: Miguel Ojeda Cc: Song Liu Cc: Leo Yan Cc: Kajol Jain Cc: Andi Kleen Cc: Kan Liang Cc: Athira Rajeev Cc: Yanteng Si Cc: Liam Howlett Cc: Paolo Bonzini Link: https://lore.kernel.org/r/20231024222353.3024098-3-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/util/machine.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/tools/perf/util/machine.c b/tools/perf/util/machine.c index addfae2f63ef..e0e2c4a943e4 100644 --- a/tools/perf/util/machine.c +++ b/tools/perf/util/machine.c @@ -2622,16 +2622,18 @@ static int lbr_callchain_add_lbr_ip(struct thread *thread, save_lbr_cursor_node(thread, cursor, i); } - /* Add LBR ip from first entries.to */ - ip = entries[0].to; - flags = &entries[0].flags; - *branch_from = entries[0].from; - err = add_callchain_ip(thread, cursor, parent, - root_al, &cpumode, ip, - true, flags, NULL, - *branch_from); - if (err) - return err; + if (lbr_nr > 0) { + /* Add LBR ip from first entries.to */ + ip = entries[0].to; + flags = &entries[0].flags; + *branch_from = entries[0].from; + err = add_callchain_ip(thread, cursor, parent, + root_al, &cpumode, ip, + true, flags, NULL, + *branch_from); + if (err) + return err; + } return 0; } From 75265320d290c5f5891f16967b94883676c46705 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Tue, 24 Oct 2023 15:23:06 -0700 Subject: [PATCH 416/475] libperf rc_check: Make implicit enabling work for GCC Make the implicit REFCOUNT_CHECKING robust to when building with GCC. Fixes: 9be6ab181b7b ("libperf rc_check: Enable implicitly with sanitizers") Signed-off-by: Ian Rogers Cc: K Prateek Nayak Cc: Ravi Bangoria Cc: Sandipan Das Cc: Anshuman Khandual Cc: German Gomez Cc: James Clark Cc: Nick Terrell Cc: Sean Christopherson Cc: Changbin Du Cc: liuwenyu Cc: Yang Jihong Cc: Masami Hiramatsu Cc: Miguel Ojeda Cc: Song Liu Cc: Leo Yan Cc: Kajol Jain Cc: Andi Kleen Cc: Kan Liang Cc: Athira Rajeev Cc: Yanteng Si Cc: Liam Howlett Cc: Paolo Bonzini Link: https://lore.kernel.org/r/20231024222353.3024098-4-irogers@google.com Signed-off-by: Namhyung Kim --- tools/lib/perf/include/internal/rc_check.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/lib/perf/include/internal/rc_check.h b/tools/lib/perf/include/internal/rc_check.h index d5d771ccdc7b..e88a6d8a0b0f 100644 --- a/tools/lib/perf/include/internal/rc_check.h +++ b/tools/lib/perf/include/internal/rc_check.h @@ -9,8 +9,12 @@ * Enable reference count checking implicitly with leak checking, which is * integrated into address sanitizer. */ -#if defined(LEAK_SANITIZER) || defined(ADDRESS_SANITIZER) +#if defined(__SANITIZE_ADDRESS__) || defined(LEAK_SANITIZER) || defined(ADDRESS_SANITIZER) #define REFCNT_CHECKING 1 +#elif defined(__has_feature) +#if __has_feature(address_sanitizer) || __has_feature(leak_sanitizer) +#define REFCNT_CHECKING 1 +#endif #endif /* From 78c32f4cb12f9430e29402118635dcf972b7d325 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Tue, 24 Oct 2023 15:23:07 -0700 Subject: [PATCH 417/475] libperf rc_check: Add RC_CHK_EQUAL Comparing pointers with reference count checking is tricky to avoid a SEGV. Add a convenience macro to simplify and use. Signed-off-by: Ian Rogers Cc: K Prateek Nayak Cc: Ravi Bangoria Cc: Sandipan Das Cc: Anshuman Khandual Cc: German Gomez Cc: James Clark Cc: Nick Terrell Cc: Sean Christopherson Cc: Changbin Du Cc: liuwenyu Cc: Yang Jihong Cc: Masami Hiramatsu Cc: Miguel Ojeda Cc: Song Liu Cc: Leo Yan Cc: Kajol Jain Cc: Andi Kleen Cc: Kan Liang Cc: Athira Rajeev Cc: Yanteng Si Cc: Liam Howlett Cc: Paolo Bonzini Link: https://lore.kernel.org/r/20231024222353.3024098-5-irogers@google.com Signed-off-by: Namhyung Kim --- tools/lib/perf/include/internal/rc_check.h | 7 +++++++ tools/perf/builtin-sched.c | 2 +- tools/perf/tests/hists_link.c | 4 ++-- tools/perf/tests/thread-maps-share.c | 9 ++++----- tools/perf/util/callchain.c | 2 +- tools/perf/util/hist.c | 2 +- tools/perf/util/machine.c | 4 ++-- tools/perf/util/sort.c | 2 +- tools/perf/util/symbol.c | 4 ++-- 9 files changed, 21 insertions(+), 15 deletions(-) diff --git a/tools/lib/perf/include/internal/rc_check.h b/tools/lib/perf/include/internal/rc_check.h index e88a6d8a0b0f..f80ddfc80129 100644 --- a/tools/lib/perf/include/internal/rc_check.h +++ b/tools/lib/perf/include/internal/rc_check.h @@ -54,6 +54,9 @@ /* A put operation removing the indirection layer. */ #define RC_CHK_PUT(object) {} +/* Pointer equality when the indirection may or may not be there. */ +#define RC_CHK_EQUAL(object1, object2) (object1 == object2) + #else /* Replaces "struct foo" so that the pointer may be interposed. */ @@ -101,6 +104,10 @@ } \ } while(0) +/* Pointer equality when the indirection may or may not be there. */ +#define RC_CHK_EQUAL(object1, object2) (object1 == object2 || \ + (object1 && object2 && object1->orig == object2->orig)) + #endif #endif /* __LIBPERF_INTERNAL_RC_CHECK_H */ diff --git a/tools/perf/builtin-sched.c b/tools/perf/builtin-sched.c index 9ab300b6f131..dd6065afbbaf 100644 --- a/tools/perf/builtin-sched.c +++ b/tools/perf/builtin-sched.c @@ -1385,7 +1385,7 @@ static int pid_cmp(struct work_atoms *l, struct work_atoms *r) { pid_t l_tid, r_tid; - if (RC_CHK_ACCESS(l->thread) == RC_CHK_ACCESS(r->thread)) + if (RC_CHK_EQUAL(l->thread, r->thread)) return 0; l_tid = thread__tid(l->thread); r_tid = thread__tid(r->thread); diff --git a/tools/perf/tests/hists_link.c b/tools/perf/tests/hists_link.c index 2d19657ab5e0..5b6f1e883466 100644 --- a/tools/perf/tests/hists_link.c +++ b/tools/perf/tests/hists_link.c @@ -148,8 +148,8 @@ static int find_sample(struct sample *samples, size_t nr_samples, struct thread *t, struct map *m, struct symbol *s) { while (nr_samples--) { - if (RC_CHK_ACCESS(samples->thread) == RC_CHK_ACCESS(t) && - RC_CHK_ACCESS(samples->map) == RC_CHK_ACCESS(m) && + if (RC_CHK_EQUAL(samples->thread, t) && + RC_CHK_EQUAL(samples->map, m) && samples->sym == s) return 1; samples++; diff --git a/tools/perf/tests/thread-maps-share.c b/tools/perf/tests/thread-maps-share.c index faf980b26252..7fa6f7c568e2 100644 --- a/tools/perf/tests/thread-maps-share.c +++ b/tools/perf/tests/thread-maps-share.c @@ -46,9 +46,9 @@ static int test__thread_maps_share(struct test_suite *test __maybe_unused, int s TEST_ASSERT_EQUAL("wrong refcnt", refcount_read(maps__refcnt(maps)), 4); /* test the maps pointer is shared */ - TEST_ASSERT_VAL("maps don't match", RC_CHK_ACCESS(maps) == RC_CHK_ACCESS(thread__maps(t1))); - TEST_ASSERT_VAL("maps don't match", RC_CHK_ACCESS(maps) == RC_CHK_ACCESS(thread__maps(t2))); - TEST_ASSERT_VAL("maps don't match", RC_CHK_ACCESS(maps) == RC_CHK_ACCESS(thread__maps(t3))); + TEST_ASSERT_VAL("maps don't match", RC_CHK_EQUAL(maps, thread__maps(t1))); + TEST_ASSERT_VAL("maps don't match", RC_CHK_EQUAL(maps, thread__maps(t2))); + TEST_ASSERT_VAL("maps don't match", RC_CHK_EQUAL(maps, thread__maps(t3))); /* * Verify the other leader was created by previous call. @@ -73,8 +73,7 @@ static int test__thread_maps_share(struct test_suite *test __maybe_unused, int s other_maps = thread__maps(other); TEST_ASSERT_EQUAL("wrong refcnt", refcount_read(maps__refcnt(other_maps)), 2); - TEST_ASSERT_VAL("maps don't match", RC_CHK_ACCESS(other_maps) == - RC_CHK_ACCESS(thread__maps(other_leader))); + TEST_ASSERT_VAL("maps don't match", RC_CHK_EQUAL(other_maps, thread__maps(other_leader))); /* release thread group */ thread__put(t3); diff --git a/tools/perf/util/callchain.c b/tools/perf/util/callchain.c index aee937d14fbb..18d545c0629e 100644 --- a/tools/perf/util/callchain.c +++ b/tools/perf/util/callchain.c @@ -1142,7 +1142,7 @@ int fill_callchain_info(struct addr_location *al, struct callchain_cursor_node * if (al->map == NULL) goto out; } - if (RC_CHK_ACCESS(al->maps) == RC_CHK_ACCESS(machine__kernel_maps(machine))) { + if (RC_CHK_EQUAL(al->maps, machine__kernel_maps(machine))) { if (machine__is_host(machine)) { al->cpumode = PERF_RECORD_MISC_KERNEL; al->level = 'k'; diff --git a/tools/perf/util/hist.c b/tools/perf/util/hist.c index 3dc8a4968beb..cde0078e6c90 100644 --- a/tools/perf/util/hist.c +++ b/tools/perf/util/hist.c @@ -2142,7 +2142,7 @@ static bool hists__filter_entry_by_thread(struct hists *hists, struct hist_entry *he) { if (hists->thread_filter != NULL && - RC_CHK_ACCESS(he->thread) != RC_CHK_ACCESS(hists->thread_filter)) { + !RC_CHK_EQUAL(he->thread, hists->thread_filter)) { he->filtered |= (1 << HIST_FILTER__THREAD); return true; } diff --git a/tools/perf/util/machine.c b/tools/perf/util/machine.c index e0e2c4a943e4..098600d983c5 100644 --- a/tools/perf/util/machine.c +++ b/tools/perf/util/machine.c @@ -969,7 +969,7 @@ static int machine__process_ksymbol_unregister(struct machine *machine, if (!map) return 0; - if (RC_CHK_ACCESS(map) != RC_CHK_ACCESS(machine->vmlinux_map)) + if (!RC_CHK_EQUAL(map, machine->vmlinux_map)) maps__remove(machine__kernel_maps(machine), map); else { struct dso *dso = map__dso(map); @@ -2058,7 +2058,7 @@ static void __machine__remove_thread(struct machine *machine, struct thread_rb_n if (!nd) nd = thread_rb_node__find(th, &threads->entries.rb_root); - if (threads->last_match && RC_CHK_ACCESS(threads->last_match) == RC_CHK_ACCESS(th)) + if (threads->last_match && RC_CHK_EQUAL(threads->last_match, th)) threads__set_last_match(threads, NULL); if (lock) diff --git a/tools/perf/util/sort.c b/tools/perf/util/sort.c index 6aa1c7f2b444..80e4f6132740 100644 --- a/tools/perf/util/sort.c +++ b/tools/perf/util/sort.c @@ -128,7 +128,7 @@ static int hist_entry__thread_filter(struct hist_entry *he, int type, const void if (type != HIST_FILTER__THREAD) return -1; - return th && RC_CHK_ACCESS(he->thread) != RC_CHK_ACCESS(th); + return th && !RC_CHK_EQUAL(he->thread, th); } struct sort_entry sort_thread = { diff --git a/tools/perf/util/symbol.c b/tools/perf/util/symbol.c index 96587fd7a5a2..822f4dcebfe6 100644 --- a/tools/perf/util/symbol.c +++ b/tools/perf/util/symbol.c @@ -877,7 +877,7 @@ static int maps__split_kallsyms(struct maps *kmaps, struct dso *dso, u64 delta, *module++ = '\0'; curr_map_dso = map__dso(curr_map); if (strcmp(curr_map_dso->short_name, module)) { - if (RC_CHK_ACCESS(curr_map) != RC_CHK_ACCESS(initial_map) && + if (!RC_CHK_EQUAL(curr_map, initial_map) && dso->kernel == DSO_SPACE__KERNEL_GUEST && machine__is_default_guest(machine)) { /* @@ -1469,7 +1469,7 @@ static int dso__load_kcore(struct dso *dso, struct map *map, list_del_init(&new_node->node); - if (RC_CHK_ACCESS(new_map) == RC_CHK_ACCESS(replacement_map)) { + if (RC_CHK_EQUAL(new_map, replacement_map)) { struct map *map_ref; map__set_start(map, map__start(new_map)); From c1149037f65bcf0334886180ebe3d5efcf214912 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Tue, 24 Oct 2023 15:23:08 -0700 Subject: [PATCH 418/475] perf hist: Add missing puts to hist__account_cycles Caught using reference count checking on perf top with "--call-graph=lbr". After this no memory leaks were detected. Fixes: 57849998e2cd ("perf report: Add processing for cycle histograms") Signed-off-by: Ian Rogers Cc: K Prateek Nayak Cc: Ravi Bangoria Cc: Sandipan Das Cc: Anshuman Khandual Cc: German Gomez Cc: James Clark Cc: Nick Terrell Cc: Sean Christopherson Cc: Changbin Du Cc: liuwenyu Cc: Yang Jihong Cc: Masami Hiramatsu Cc: Miguel Ojeda Cc: Song Liu Cc: Leo Yan Cc: Kajol Jain Cc: Andi Kleen Cc: Kan Liang Cc: Athira Rajeev Cc: Yanteng Si Cc: Liam Howlett Cc: Paolo Bonzini Link: https://lore.kernel.org/r/20231024222353.3024098-6-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/util/hist.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tools/perf/util/hist.c b/tools/perf/util/hist.c index cde0078e6c90..0aa7e231172a 100644 --- a/tools/perf/util/hist.c +++ b/tools/perf/util/hist.c @@ -2676,8 +2676,6 @@ void hist__account_cycles(struct branch_stack *bs, struct addr_location *al, /* If we have branch cycles always annotate them. */ if (bs && bs->nr && entries[0].flags.cycles) { - int i; - bi = sample__resolve_bstack(sample, al); if (bi) { struct addr_map_symbol *prev = NULL; @@ -2692,7 +2690,7 @@ void hist__account_cycles(struct branch_stack *bs, struct addr_location *al, * Note that perf stores branches reversed from * program order! */ - for (i = bs->nr - 1; i >= 0; i--) { + for (int i = bs->nr - 1; i >= 0; i--) { addr_map_symbol__account_cycles(&bi[i].from, nonany_branch_mode ? NULL : prev, bi[i].flags.cycles); @@ -2701,6 +2699,12 @@ void hist__account_cycles(struct branch_stack *bs, struct addr_location *al, if (total_cycles) *total_cycles += bi[i].flags.cycles; } + for (unsigned int i = 0; i < bs->nr; i++) { + map__put(bi[i].to.ms.map); + maps__put(bi[i].to.ms.maps); + map__put(bi[i].from.ms.map); + maps__put(bi[i].from.ms.maps); + } free(bi); } } From 7b2e444b76ceff71dad53fb8705780741421b570 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Tue, 24 Oct 2023 15:23:09 -0700 Subject: [PATCH 419/475] perf threads: Remove unused dead thread list Commit 40826c45eb0b ("perf thread: Remove notion of dead threads") removed dead threads but the list head wasn't removed. Remove it here. Signed-off-by: Ian Rogers Cc: K Prateek Nayak Cc: Ravi Bangoria Cc: Sandipan Das Cc: Anshuman Khandual Cc: German Gomez Cc: James Clark Cc: Nick Terrell Cc: Sean Christopherson Cc: Changbin Du Cc: liuwenyu Cc: Yang Jihong Cc: Masami Hiramatsu Cc: Miguel Ojeda Cc: Song Liu Cc: Leo Yan Cc: Kajol Jain Cc: Andi Kleen Cc: Kan Liang Cc: Athira Rajeev Cc: Yanteng Si Cc: Liam Howlett Cc: Paolo Bonzini Link: https://lore.kernel.org/r/20231024222353.3024098-7-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/util/machine.c | 1 - tools/perf/util/machine.h | 1 - 2 files changed, 2 deletions(-) diff --git a/tools/perf/util/machine.c b/tools/perf/util/machine.c index 098600d983c5..31fdbb0f27af 100644 --- a/tools/perf/util/machine.c +++ b/tools/perf/util/machine.c @@ -67,7 +67,6 @@ static void machine__threads_init(struct machine *machine) threads->entries = RB_ROOT_CACHED; init_rwsem(&threads->lock); threads->nr = 0; - INIT_LIST_HEAD(&threads->dead); threads->last_match = NULL; } } diff --git a/tools/perf/util/machine.h b/tools/perf/util/machine.h index d034ecaf89c1..1279acda6a8a 100644 --- a/tools/perf/util/machine.h +++ b/tools/perf/util/machine.h @@ -35,7 +35,6 @@ struct threads { struct rb_root_cached entries; struct rw_semaphore lock; unsigned int nr; - struct list_head dead; struct thread *last_match; }; From 67a3ebf1c3588ee2ab994a4c96ef19179209c132 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Tue, 24 Oct 2023 15:23:10 -0700 Subject: [PATCH 420/475] perf offcpu: Add missed btf_free Caught by address/leak sanitizer. Signed-off-by: Ian Rogers Cc: K Prateek Nayak Cc: Ravi Bangoria Cc: Sandipan Das Cc: Anshuman Khandual Cc: German Gomez Cc: James Clark Cc: Nick Terrell Cc: Sean Christopherson Cc: Changbin Du Cc: liuwenyu Cc: Yang Jihong Cc: Masami Hiramatsu Cc: Miguel Ojeda Cc: Song Liu Cc: Leo Yan Cc: Kajol Jain Cc: Andi Kleen Cc: Kan Liang Cc: Athira Rajeev Cc: Yanteng Si Cc: Liam Howlett Cc: Paolo Bonzini Link: https://lore.kernel.org/r/20231024222353.3024098-8-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/util/bpf_off_cpu.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tools/perf/util/bpf_off_cpu.c b/tools/perf/util/bpf_off_cpu.c index 21f4d9ba023d..6af36142dc5a 100644 --- a/tools/perf/util/bpf_off_cpu.c +++ b/tools/perf/util/bpf_off_cpu.c @@ -98,22 +98,22 @@ static void off_cpu_finish(void *arg __maybe_unused) /* v5.18 kernel added prev_state arg, so it needs to check the signature */ static void check_sched_switch_args(void) { - const struct btf *btf = btf__load_vmlinux_btf(); + struct btf *btf = btf__load_vmlinux_btf(); const struct btf_type *t1, *t2, *t3; u32 type_id; type_id = btf__find_by_name_kind(btf, "btf_trace_sched_switch", BTF_KIND_TYPEDEF); if ((s32)type_id < 0) - return; + goto cleanup; t1 = btf__type_by_id(btf, type_id); if (t1 == NULL) - return; + goto cleanup; t2 = btf__type_by_id(btf, t1->type); if (t2 == NULL || !btf_is_ptr(t2)) - return; + goto cleanup; t3 = btf__type_by_id(btf, t2->type); /* btf_trace func proto has one more argument for the context */ @@ -121,6 +121,8 @@ static void check_sched_switch_args(void) /* new format: pass prev_state as 4th arg */ skel->rodata->has_prev_state = true; } +cleanup: + btf__free(btf); } int off_cpu_prepare(struct evlist *evlist, struct target *target, From d47d876d72624ee1ef31f2e491be4393d4eacaf0 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Tue, 24 Oct 2023 15:23:11 -0700 Subject: [PATCH 421/475] perf callchain: Make display use of branch_type_stat const Display code doesn't modify the branch_type_stat so switch uses to const. This is done to aid refactoring struct callchain_list where current the branch_type_stat is embedded even if not used. Signed-off-by: Ian Rogers Cc: K Prateek Nayak Cc: Ravi Bangoria Cc: Sandipan Das Cc: Anshuman Khandual Cc: German Gomez Cc: James Clark Cc: Nick Terrell Cc: Sean Christopherson Cc: Changbin Du Cc: liuwenyu Cc: Yang Jihong Cc: Masami Hiramatsu Cc: Miguel Ojeda Cc: Song Liu Cc: Leo Yan Cc: Kajol Jain Cc: Andi Kleen Cc: Kan Liang Cc: Athira Rajeev Cc: Yanteng Si Cc: Liam Howlett Cc: Paolo Bonzini Link: https://lore.kernel.org/r/20231024222353.3024098-9-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/util/branch.c | 4 ++-- tools/perf/util/branch.h | 4 ++-- tools/perf/util/callchain.c | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tools/perf/util/branch.c b/tools/perf/util/branch.c index 378f16a24751..ab760e267d41 100644 --- a/tools/perf/util/branch.c +++ b/tools/perf/util/branch.c @@ -109,7 +109,7 @@ const char *get_branch_type(struct branch_entry *e) return branch_type_name(e->flags.type); } -void branch_type_stat_display(FILE *fp, struct branch_type_stat *st) +void branch_type_stat_display(FILE *fp, const struct branch_type_stat *st) { u64 total = 0; int i; @@ -171,7 +171,7 @@ static int count_str_scnprintf(int idx, const char *str, char *bf, int size) return scnprintf(bf, size, "%s%s", (idx) ? " " : " (", str); } -int branch_type_str(struct branch_type_stat *st, char *bf, int size) +int branch_type_str(const struct branch_type_stat *st, char *bf, int size) { int i, j = 0, printed = 0; u64 total = 0; diff --git a/tools/perf/util/branch.h b/tools/perf/util/branch.h index e41bfffe2217..87704d713ff6 100644 --- a/tools/perf/util/branch.h +++ b/tools/perf/util/branch.h @@ -86,8 +86,8 @@ void branch_type_count(struct branch_type_stat *st, struct branch_flags *flags, const char *branch_type_name(int type); const char *branch_new_type_name(int new_type); const char *get_branch_type(struct branch_entry *e); -void branch_type_stat_display(FILE *fp, struct branch_type_stat *st); -int branch_type_str(struct branch_type_stat *st, char *bf, int bfsize); +void branch_type_stat_display(FILE *fp, const struct branch_type_stat *st); +int branch_type_str(const struct branch_type_stat *st, char *bf, int bfsize); const char *branch_spec_desc(int spec); diff --git a/tools/perf/util/callchain.c b/tools/perf/util/callchain.c index 18d545c0629e..cde4860e6f28 100644 --- a/tools/perf/util/callchain.c +++ b/tools/perf/util/callchain.c @@ -1339,7 +1339,7 @@ static int count_float_printf(int idx, const char *str, float value, static int branch_to_str(char *bf, int bfsize, u64 branch_count, u64 predicted_count, u64 abort_count, - struct branch_type_stat *brtype_stat) + const struct branch_type_stat *brtype_stat) { int printed, i = 0; @@ -1403,7 +1403,7 @@ static int counts_str_build(char *bf, int bfsize, u64 abort_count, u64 cycles_count, u64 iter_count, u64 iter_cycles, u64 from_count, - struct branch_type_stat *brtype_stat) + const struct branch_type_stat *brtype_stat) { int printed; @@ -1430,7 +1430,7 @@ static int callchain_counts_printf(FILE *fp, char *bf, int bfsize, u64 abort_count, u64 cycles_count, u64 iter_count, u64 iter_cycles, u64 from_count, - struct branch_type_stat *brtype_stat) + const struct branch_type_stat *brtype_stat) { char str[256]; From 6ba29fbb0b3863fa50c809d098680a0f3e0bd625 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Tue, 24 Oct 2023 15:23:12 -0700 Subject: [PATCH 422/475] perf callchain: Make brtype_stat in callchain_list optional struct callchain_list is 352bytes in size, 232 of which are brtype_stat. brtype_stat is only used for certain callchain_list items so make it optional, allocating when necessary. So that printing doesn't need to deal with an optional brtype_stat, pass an empty/zero version. Before: ``` struct callchain_list { u64 ip; /* 0 8 */ struct map_symbol ms; /* 8 24 */ struct { _Bool unfolded; /* 32 1 */ _Bool has_children; /* 33 1 */ }; /* 32 2 */ /* XXX 6 bytes hole, try to pack */ u64 branch_count; /* 40 8 */ u64 from_count; /* 48 8 */ u64 predicted_count; /* 56 8 */ /* --- cacheline 1 boundary (64 bytes) --- */ u64 abort_count; /* 64 8 */ u64 cycles_count; /* 72 8 */ u64 iter_count; /* 80 8 */ u64 iter_cycles; /* 88 8 */ struct branch_type_stat brtype_stat; /* 96 232 */ /* --- cacheline 5 boundary (320 bytes) was 8 bytes ago --- */ const char * srcline; /* 328 8 */ struct list_head list; /* 336 16 */ /* size: 352, cachelines: 6, members: 13 */ /* sum members: 346, holes: 1, sum holes: 6 */ /* last cacheline: 32 bytes */ }; ``` After: ``` struct callchain_list { u64 ip; /* 0 8 */ struct map_symbol ms; /* 8 24 */ struct { _Bool unfolded; /* 32 1 */ _Bool has_children; /* 33 1 */ }; /* 32 2 */ /* XXX 6 bytes hole, try to pack */ u64 branch_count; /* 40 8 */ u64 from_count; /* 48 8 */ u64 predicted_count; /* 56 8 */ /* --- cacheline 1 boundary (64 bytes) --- */ u64 abort_count; /* 64 8 */ u64 cycles_count; /* 72 8 */ u64 iter_count; /* 80 8 */ u64 iter_cycles; /* 88 8 */ struct branch_type_stat * brtype_stat; /* 96 8 */ const char * srcline; /* 104 8 */ struct list_head list; /* 112 16 */ /* size: 128, cachelines: 2, members: 13 */ /* sum members: 122, holes: 1, sum holes: 6 */ }; ``` Signed-off-by: Ian Rogers Cc: K Prateek Nayak Cc: Ravi Bangoria Cc: Sandipan Das Cc: Anshuman Khandual Cc: German Gomez Cc: James Clark Cc: Nick Terrell Cc: Sean Christopherson Cc: Changbin Du Cc: liuwenyu Cc: Yang Jihong Cc: Masami Hiramatsu Cc: Miguel Ojeda Cc: Song Liu Cc: Leo Yan Cc: Kajol Jain Cc: Andi Kleen Cc: Kan Liang Cc: Athira Rajeev Cc: Yanteng Si Cc: Liam Howlett Cc: Paolo Bonzini Link: https://lore.kernel.org/r/20231024222353.3024098-10-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/util/callchain.c | 41 +++++++++++++++++++++++++++++-------- tools/perf/util/callchain.h | 2 +- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/tools/perf/util/callchain.c b/tools/perf/util/callchain.c index cde4860e6f28..5349c6a21849 100644 --- a/tools/perf/util/callchain.c +++ b/tools/perf/util/callchain.c @@ -586,7 +586,7 @@ fill_node(struct callchain_node *node, struct callchain_cursor *cursor) call = zalloc(sizeof(*call)); if (!call) { perror("not enough memory for the code path tree"); - return -1; + return -ENOMEM; } call->ip = cursor_node->ip; call->ms = cursor_node->ms; @@ -602,7 +602,15 @@ fill_node(struct callchain_node *node, struct callchain_cursor *cursor) * branch_from is set with value somewhere else * to imply it's "to" of a branch. */ - call->brtype_stat.branch_to = true; + if (!call->brtype_stat) { + call->brtype_stat = zalloc(sizeof(*call->brtype_stat)); + if (!call->brtype_stat) { + perror("not enough memory for the code path branch statisitcs"); + free(call->brtype_stat); + return -ENOMEM; + } + } + call->brtype_stat->branch_to = true; if (cursor_node->branch_flags.predicted) call->predicted_count = 1; @@ -610,7 +618,7 @@ fill_node(struct callchain_node *node, struct callchain_cursor *cursor) if (cursor_node->branch_flags.abort) call->abort_count = 1; - branch_type_count(&call->brtype_stat, + branch_type_count(call->brtype_stat, &cursor_node->branch_flags, cursor_node->branch_from, cursor_node->ip); @@ -618,7 +626,8 @@ fill_node(struct callchain_node *node, struct callchain_cursor *cursor) /* * It's "from" of a branch */ - call->brtype_stat.branch_to = false; + if (call->brtype_stat && call->brtype_stat->branch_to) + call->brtype_stat->branch_to = false; call->cycles_count = cursor_node->branch_flags.cycles; call->iter_count = cursor_node->nr_loop_iter; @@ -652,6 +661,7 @@ add_child(struct callchain_node *parent, list_del_init(&call->list); map__zput(call->ms.map); maps__zput(call->ms.maps); + zfree(&call->brtype_stat); free(call); } free(new); @@ -762,7 +772,14 @@ static enum match_result match_chain(struct callchain_cursor_node *node, /* * It's "to" of a branch */ - cnode->brtype_stat.branch_to = true; + if (!cnode->brtype_stat) { + cnode->brtype_stat = zalloc(sizeof(*cnode->brtype_stat)); + if (!cnode->brtype_stat) { + perror("not enough memory for the code path branch statisitcs"); + return MATCH_ERROR; + } + } + cnode->brtype_stat->branch_to = true; if (node->branch_flags.predicted) cnode->predicted_count++; @@ -770,7 +787,7 @@ static enum match_result match_chain(struct callchain_cursor_node *node, if (node->branch_flags.abort) cnode->abort_count++; - branch_type_count(&cnode->brtype_stat, + branch_type_count(cnode->brtype_stat, &node->branch_flags, node->branch_from, node->ip); @@ -778,7 +795,8 @@ static enum match_result match_chain(struct callchain_cursor_node *node, /* * It's "from" of a branch */ - cnode->brtype_stat.branch_to = false; + if (cnode->brtype_stat && cnode->brtype_stat->branch_to) + cnode->brtype_stat->branch_to = false; cnode->cycles_count += node->branch_flags.cycles; cnode->iter_count += node->nr_loop_iter; cnode->iter_cycles += node->iter_cycles; @@ -1026,6 +1044,7 @@ merge_chain_branch(struct callchain_cursor *cursor, maps__zput(ms.maps); map__zput(list->ms.map); maps__zput(list->ms.maps); + zfree(&list->brtype_stat); free(list); } @@ -1447,11 +1466,14 @@ static int callchain_counts_printf(FILE *fp, char *bf, int bfsize, int callchain_list_counts__printf_value(struct callchain_list *clist, FILE *fp, char *bf, int bfsize) { + static const struct branch_type_stat empty_brtype_stat = {}; + const struct branch_type_stat *brtype_stat; u64 branch_count, predicted_count; u64 abort_count, cycles_count; u64 iter_count, iter_cycles; u64 from_count; + brtype_stat = clist->brtype_stat ?: &empty_brtype_stat; branch_count = clist->branch_count; predicted_count = clist->predicted_count; abort_count = clist->abort_count; @@ -1463,7 +1485,7 @@ int callchain_list_counts__printf_value(struct callchain_list *clist, return callchain_counts_printf(fp, bf, bfsize, branch_count, predicted_count, abort_count, cycles_count, iter_count, iter_cycles, - from_count, &clist->brtype_stat); + from_count, brtype_stat); } static void free_callchain_node(struct callchain_node *node) @@ -1476,6 +1498,7 @@ static void free_callchain_node(struct callchain_node *node) list_del_init(&list->list); map__zput(list->ms.map); maps__zput(list->ms.maps); + zfree(&list->brtype_stat); free(list); } @@ -1483,6 +1506,7 @@ static void free_callchain_node(struct callchain_node *node) list_del_init(&list->list); map__zput(list->ms.map); maps__zput(list->ms.maps); + zfree(&list->brtype_stat); free(list); } @@ -1569,6 +1593,7 @@ out: list_del_init(&chain->list); map__zput(chain->ms.map); maps__zput(chain->ms.maps); + zfree(&chain->brtype_stat); free(chain); } return -ENOMEM; diff --git a/tools/perf/util/callchain.h b/tools/perf/util/callchain.h index d2618a47deca..86e8a9e81456 100644 --- a/tools/perf/util/callchain.h +++ b/tools/perf/util/callchain.h @@ -129,7 +129,7 @@ struct callchain_list { u64 cycles_count; u64 iter_count; u64 iter_cycles; - struct branch_type_stat brtype_stat; + struct branch_type_stat *brtype_stat; const char *srcline; struct list_head list; }; From dec07fe5d4fd29aed2faf17f56140cd402175d72 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Tue, 24 Oct 2023 15:23:13 -0700 Subject: [PATCH 423/475] perf callchain: Minor layout changes to callchain_list Avoid 6 byte hole for padding. Place more frequently used fields first in an attempt to use just 1 cacheline in the common case. Before: ``` struct callchain_list { u64 ip; /* 0 8 */ struct map_symbol ms; /* 8 24 */ struct { _Bool unfolded; /* 32 1 */ _Bool has_children; /* 33 1 */ }; /* 32 2 */ /* XXX 6 bytes hole, try to pack */ u64 branch_count; /* 40 8 */ u64 from_count; /* 48 8 */ u64 predicted_count; /* 56 8 */ /* --- cacheline 1 boundary (64 bytes) --- */ u64 abort_count; /* 64 8 */ u64 cycles_count; /* 72 8 */ u64 iter_count; /* 80 8 */ u64 iter_cycles; /* 88 8 */ struct branch_type_stat * brtype_stat; /* 96 8 */ const char * srcline; /* 104 8 */ struct list_head list; /* 112 16 */ /* size: 128, cachelines: 2, members: 13 */ /* sum members: 122, holes: 1, sum holes: 6 */ }; ``` After: ``` struct callchain_list { struct list_head list; /* 0 16 */ u64 ip; /* 16 8 */ struct map_symbol ms; /* 24 24 */ const char * srcline; /* 48 8 */ u64 branch_count; /* 56 8 */ /* --- cacheline 1 boundary (64 bytes) --- */ u64 from_count; /* 64 8 */ u64 cycles_count; /* 72 8 */ u64 iter_count; /* 80 8 */ u64 iter_cycles; /* 88 8 */ struct branch_type_stat * brtype_stat; /* 96 8 */ u64 predicted_count; /* 104 8 */ u64 abort_count; /* 112 8 */ struct { _Bool unfolded; /* 120 1 */ _Bool has_children; /* 121 1 */ }; /* 120 2 */ /* size: 128, cachelines: 2, members: 13 */ /* padding: 6 */ }; ``` Signed-off-by: Ian Rogers Cc: K Prateek Nayak Cc: Ravi Bangoria Cc: Sandipan Das Cc: Anshuman Khandual Cc: German Gomez Cc: James Clark Cc: Nick Terrell Cc: Sean Christopherson Cc: Changbin Du Cc: liuwenyu Cc: Yang Jihong Cc: Masami Hiramatsu Cc: Miguel Ojeda Cc: Song Liu Cc: Leo Yan Cc: Kajol Jain Cc: Andi Kleen Cc: Kan Liang Cc: Athira Rajeev Cc: Yanteng Si Cc: Liam Howlett Cc: Paolo Bonzini Link: https://lore.kernel.org/r/20231024222353.3024098-11-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/util/callchain.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tools/perf/util/callchain.h b/tools/perf/util/callchain.h index 86e8a9e81456..d5c66345ae31 100644 --- a/tools/perf/util/callchain.h +++ b/tools/perf/util/callchain.h @@ -116,22 +116,22 @@ extern struct callchain_param callchain_param; extern struct callchain_param callchain_param_default; struct callchain_list { + struct list_head list; u64 ip; struct map_symbol ms; - struct /* for TUI */ { - bool unfolded; - bool has_children; - }; + const char *srcline; u64 branch_count; u64 from_count; - u64 predicted_count; - u64 abort_count; u64 cycles_count; u64 iter_count; u64 iter_cycles; struct branch_type_stat *brtype_stat; - const char *srcline; - struct list_head list; + u64 predicted_count; + u64 abort_count; + struct /* for TUI */ { + bool unfolded; + bool has_children; + }; }; /* From 56e144fe98260a0f8a17326993ceb576ef859ed5 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Tue, 24 Oct 2023 15:23:14 -0700 Subject: [PATCH 424/475] perf mem_info: Add and use map_symbol__exit and addr_map_symbol__exit Fix leak where mem_info__put wouldn't release the maps/map as used by perf mem. Add exit functions and use elsewhere that the maps and map are released. Signed-off-by: Ian Rogers Cc: K Prateek Nayak Cc: Ravi Bangoria Cc: Sandipan Das Cc: Anshuman Khandual Cc: German Gomez Cc: James Clark Cc: Nick Terrell Cc: Sean Christopherson Cc: Changbin Du Cc: liuwenyu Cc: Yang Jihong Cc: Masami Hiramatsu Cc: Miguel Ojeda Cc: Song Liu Cc: Leo Yan Cc: Kajol Jain Cc: Andi Kleen Cc: Kan Liang Cc: Athira Rajeev Cc: Yanteng Si Cc: Liam Howlett Cc: Paolo Bonzini Link: https://lore.kernel.org/r/20231024222353.3024098-12-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/util/Build | 1 + tools/perf/util/callchain.c | 27 +++++++++------------------ tools/perf/util/hist.c | 28 ++++++++++++---------------- tools/perf/util/machine.c | 6 ++---- tools/perf/util/map_symbol.c | 15 +++++++++++++++ tools/perf/util/map_symbol.h | 4 ++++ tools/perf/util/symbol.c | 5 ++++- 7 files changed, 47 insertions(+), 39 deletions(-) create mode 100644 tools/perf/util/map_symbol.c diff --git a/tools/perf/util/Build b/tools/perf/util/Build index 0ea5a9d368d4..96058f949ec9 100644 --- a/tools/perf/util/Build +++ b/tools/perf/util/Build @@ -49,6 +49,7 @@ perf-y += dso.o perf-y += dsos.o perf-y += symbol.o perf-y += symbol_fprintf.o +perf-y += map_symbol.o perf-y += color.o perf-y += color_config.o perf-y += metricgroup.o diff --git a/tools/perf/util/callchain.c b/tools/perf/util/callchain.c index 5349c6a21849..99cce43ba152 100644 --- a/tools/perf/util/callchain.c +++ b/tools/perf/util/callchain.c @@ -659,8 +659,7 @@ add_child(struct callchain_node *parent, list_for_each_entry_safe(call, tmp, &new->val, list) { list_del_init(&call->list); - map__zput(call->ms.map); - maps__zput(call->ms.maps); + map_symbol__exit(&call->ms); zfree(&call->brtype_stat); free(call); } @@ -1040,10 +1039,8 @@ merge_chain_branch(struct callchain_cursor *cursor, }; callchain_cursor_append(cursor, list->ip, &ms, false, NULL, 0, 0, 0, list->srcline); list_del_init(&list->list); - map__zput(ms.map); - maps__zput(ms.maps); - map__zput(list->ms.map); - maps__zput(list->ms.maps); + map_symbol__exit(&ms); + map_symbol__exit(&list->ms); zfree(&list->brtype_stat); free(list); } @@ -1096,8 +1093,7 @@ int callchain_cursor_append(struct callchain_cursor *cursor, } node->ip = ip; - maps__zput(node->ms.maps); - map__zput(node->ms.map); + map_symbol__exit(&node->ms); node->ms = *ms; node->ms.maps = maps__get(ms->maps); node->ms.map = map__get(ms->map); @@ -1496,16 +1492,14 @@ static void free_callchain_node(struct callchain_node *node) list_for_each_entry_safe(list, tmp, &node->parent_val, list) { list_del_init(&list->list); - map__zput(list->ms.map); - maps__zput(list->ms.maps); + map_symbol__exit(&list->ms); zfree(&list->brtype_stat); free(list); } list_for_each_entry_safe(list, tmp, &node->val, list) { list_del_init(&list->list); - map__zput(list->ms.map); - maps__zput(list->ms.maps); + map_symbol__exit(&list->ms); zfree(&list->brtype_stat); free(list); } @@ -1591,8 +1585,7 @@ int callchain_node__make_parent_list(struct callchain_node *node) out: list_for_each_entry_safe(chain, new, &head, list) { list_del_init(&chain->list); - map__zput(chain->ms.map); - maps__zput(chain->ms.maps); + map_symbol__exit(&chain->ms); zfree(&chain->brtype_stat); free(chain); } @@ -1676,10 +1669,8 @@ void callchain_cursor_reset(struct callchain_cursor *cursor) cursor->nr = 0; cursor->last = &cursor->first; - for (node = cursor->first; node != NULL; node = node->next) { - map__zput(node->ms.map); - maps__zput(node->ms.maps); - } + for (node = cursor->first; node != NULL; node = node->next) + map_symbol__exit(&node->ms); } void callchain_param_setup(u64 sample_type, const char *arch) diff --git a/tools/perf/util/hist.c b/tools/perf/util/hist.c index 0aa7e231172a..0888b7163b7c 100644 --- a/tools/perf/util/hist.c +++ b/tools/perf/util/hist.c @@ -515,17 +515,16 @@ err_rawdata: err_infos: if (he->branch_info) { - map__put(he->branch_info->from.ms.map); - map__put(he->branch_info->to.ms.map); + map_symbol__exit(&he->branch_info->from.ms); + map_symbol__exit(&he->branch_info->to.ms); zfree(&he->branch_info); } if (he->mem_info) { - map__put(he->mem_info->iaddr.ms.map); - map__put(he->mem_info->daddr.ms.map); + map_symbol__exit(&he->mem_info->iaddr.ms); + map_symbol__exit(&he->mem_info->daddr.ms); } err: - maps__zput(he->ms.maps); - map__zput(he->ms.map); + map_symbol__exit(&he->ms); zfree(&he->stat_acc); return -ENOMEM; } @@ -1317,20 +1316,19 @@ void hist_entry__delete(struct hist_entry *he) struct hist_entry_ops *ops = he->ops; thread__zput(he->thread); - maps__zput(he->ms.maps); - map__zput(he->ms.map); + map_symbol__exit(&he->ms); if (he->branch_info) { - map__zput(he->branch_info->from.ms.map); - map__zput(he->branch_info->to.ms.map); + map_symbol__exit(&he->branch_info->from.ms); + map_symbol__exit(&he->branch_info->to.ms); zfree_srcline(&he->branch_info->srcline_from); zfree_srcline(&he->branch_info->srcline_to); zfree(&he->branch_info); } if (he->mem_info) { - map__zput(he->mem_info->iaddr.ms.map); - map__zput(he->mem_info->daddr.ms.map); + map_symbol__exit(&he->mem_info->iaddr.ms); + map_symbol__exit(&he->mem_info->daddr.ms); mem_info__zput(he->mem_info); } @@ -2700,10 +2698,8 @@ void hist__account_cycles(struct branch_stack *bs, struct addr_location *al, *total_cycles += bi[i].flags.cycles; } for (unsigned int i = 0; i < bs->nr; i++) { - map__put(bi[i].to.ms.map); - maps__put(bi[i].to.ms.maps); - map__put(bi[i].from.ms.map); - maps__put(bi[i].from.ms.maps); + map_symbol__exit(&bi[i].to.ms); + map_symbol__exit(&bi[i].from.ms); } free(bi); } diff --git a/tools/perf/util/machine.c b/tools/perf/util/machine.c index 31fdbb0f27af..90c750150b19 100644 --- a/tools/perf/util/machine.c +++ b/tools/perf/util/machine.c @@ -2389,8 +2389,7 @@ static int add_callchain_ip(struct thread *thread, iter_cycles, branch_from, srcline); out: addr_location__exit(&al); - maps__put(ms.maps); - map__put(ms.map); + map_symbol__exit(&ms); return err; } @@ -3116,8 +3115,7 @@ static int append_inlines(struct callchain_cursor *cursor, struct map_symbol *ms if (ret != 0) return ret; } - map__put(ilist_ms.map); - maps__put(ilist_ms.maps); + map_symbol__exit(&ilist_ms); return ret; } diff --git a/tools/perf/util/map_symbol.c b/tools/perf/util/map_symbol.c new file mode 100644 index 000000000000..bef5079f2403 --- /dev/null +++ b/tools/perf/util/map_symbol.c @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "map_symbol.h" +#include "maps.h" +#include "map.h" + +void map_symbol__exit(struct map_symbol *ms) +{ + maps__zput(ms->maps); + map__zput(ms->map); +} + +void addr_map_symbol__exit(struct addr_map_symbol *ams) +{ + map_symbol__exit(&ams->ms); +} diff --git a/tools/perf/util/map_symbol.h b/tools/perf/util/map_symbol.h index e08817b0c30f..72d5ed938ed6 100644 --- a/tools/perf/util/map_symbol.h +++ b/tools/perf/util/map_symbol.h @@ -22,4 +22,8 @@ struct addr_map_symbol { u64 phys_addr; u64 data_page_size; }; + +void map_symbol__exit(struct map_symbol *ms); +void addr_map_symbol__exit(struct addr_map_symbol *ams); + #endif // __PERF_MAP_SYMBOL diff --git a/tools/perf/util/symbol.c b/tools/perf/util/symbol.c index 822f4dcebfe6..82cc74b9358e 100644 --- a/tools/perf/util/symbol.c +++ b/tools/perf/util/symbol.c @@ -2791,8 +2791,11 @@ struct mem_info *mem_info__get(struct mem_info *mi) void mem_info__put(struct mem_info *mi) { - if (mi && refcount_dec_and_test(&mi->refcnt)) + if (mi && refcount_dec_and_test(&mi->refcnt)) { + addr_map_symbol__exit(&mi->iaddr); + addr_map_symbol__exit(&mi->daddr); free(mi); + } } struct mem_info *mem_info__new(void) From 3779416eed25a843364b940decee452620a1de4b Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Mon, 25 Sep 2023 20:10:34 -0700 Subject: [PATCH 425/475] perf vendor events intel: Fix broadwellde tma_info_system_dram_bw_use metric Broadwell-de has a consumer core and server uncore. The uncore_arb PMU isn't present and the broadwellx style cbox PMU should be used instead. Fix the tma_info_system_dram_bw_use metric to use the server metric rather than client. The associated converter script fix is in: https://github.com/intel/perfmon/pull/111 Fixes: 7d124303d620 ("perf vendor events intel: Update broadwell variant events/metrics") Signed-off-by: Ian Rogers Reviewed-by: Kan Liang Cc: Caleb Biggers Cc: Perry Taylor Link: https://lore.kernel.org/r/20230926031034.1201145-1-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/pmu-events/arch/x86/broadwellde/bdwde-metrics.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/perf/pmu-events/arch/x86/broadwellde/bdwde-metrics.json b/tools/perf/pmu-events/arch/x86/broadwellde/bdwde-metrics.json index 8fc62b8f667d..d0ef46c9bb61 100644 --- a/tools/perf/pmu-events/arch/x86/broadwellde/bdwde-metrics.json +++ b/tools/perf/pmu-events/arch/x86/broadwellde/bdwde-metrics.json @@ -652,7 +652,7 @@ }, { "BriefDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]", - "MetricExpr": "64 * (arb@event\\=0x81\\,umask\\=0x1@ + arb@event\\=0x84\\,umask\\=0x1@) / 1e6 / duration_time / 1e3", + "MetricExpr": "64 * (UNC_M_CAS_COUNT.RD + UNC_M_CAS_COUNT.WR) / 1e9 / duration_time", "MetricGroup": "HPC;Mem;MemoryBW;SoC;tma_issueBW", "MetricName": "tma_info_system_dram_bw_use", "PublicDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]. Related metrics: tma_fb_full, tma_mem_bandwidth, tma_sq_full" From 0f7f544af60a6082cfaa3ed4c8f4ca1a858807ee Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 24 Oct 2023 15:55:59 +0100 Subject: [PATCH 426/475] powerpc: Remove initialisation of readpos While powerpc doesn't use the seq_buf readpos, it did explicitly initialise it for no good reason. Link: https://lore.kernel.org/linux-trace-kernel/20231024145600.739451-1-willy@infradead.org Cc: Christoph Hellwig Cc: Justin Stitt Cc: Kent Overstreet Cc: Petr Mladek Cc: Andy Shevchenko Cc: Rasmus Villemoes Cc: Sergey Senozhatsky Cc: Michael Ellerman Reviewed-by: Kees Cook Fixes: d0ed46b60396 ("tracing: Move readpos from seq_buf to trace_seq") Signed-off-by: Matthew Wilcox (Oracle) Signed-off-by: Steven Rostedt (Google) --- arch/powerpc/kernel/setup-common.c | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/powerpc/kernel/setup-common.c b/arch/powerpc/kernel/setup-common.c index 2f1026fba00d..34975532e44c 100644 --- a/arch/powerpc/kernel/setup-common.c +++ b/arch/powerpc/kernel/setup-common.c @@ -601,7 +601,6 @@ struct seq_buf ppc_hw_desc __initdata = { .buffer = ppc_hw_desc_buf, .size = sizeof(ppc_hw_desc_buf), .len = 0, - .readpos = 0, }; static __init void probe_machine(void) From a9de4eb15ad430fe45747c211e367da745a90093 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Tue, 24 Oct 2023 12:36:28 -0400 Subject: [PATCH 427/475] eventfs: Fix WARN_ON() in create_file_dentry() As the comment right above a WARN_ON() in create_file_dentry() states: * Note, with the mutex held, the e_dentry cannot have content * and the ei->is_freed be true at the same time. But the WARN_ON() only has: WARN_ON_ONCE(ei->is_free); Where to match the comment (and what it should actually do) is: dentry = *e_dentry; WARN_ON_ONCE(dentry && ei->is_free) Also in that case, set dentry to NULL (although it should never happen). Link: https://lore.kernel.org/linux-trace-kernel/20231024123628.62b88755@gandalf.local.home Cc: Masami Hiramatsu Cc: Mark Rutland Fixes: 5790b1fb3d672 ("eventfs: Remove eventfs_file and just use eventfs_inode") Signed-off-by: Steven Rostedt (Google) --- fs/tracefs/event_inode.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fs/tracefs/event_inode.c b/fs/tracefs/event_inode.c index 09ab93357957..4d2da7480e5f 100644 --- a/fs/tracefs/event_inode.c +++ b/fs/tracefs/event_inode.c @@ -264,8 +264,9 @@ create_file_dentry(struct eventfs_inode *ei, struct dentry **e_dentry, * Note, with the mutex held, the e_dentry cannot have content * and the ei->is_freed be true at the same time. */ - WARN_ON_ONCE(ei->is_freed); dentry = *e_dentry; + if (WARN_ON_ONCE(dentry && ei->is_freed)) + dentry = NULL; /* The lookup does not need to up the dentry refcount */ if (dentry && !lookup) dget(dentry); From 29e06c10702e81a7d0b75020ca514d2f2962704a Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Tue, 24 Oct 2023 13:10:24 -0400 Subject: [PATCH 428/475] eventfs: Fix typo in eventfs_inode union comment It's eventfs_inode not eventfs_indoe. There's no deer involved! Link: https://lore.kernel.org/linux-trace-kernel/20231024131024.5634c743@gandalf.local.home Cc: Masami Hiramatsu Cc: Mark Rutland Fixes: 5790b1fb3d672 ("eventfs: Remove eventfs_file and just use eventfs_inode") Signed-off-by: Steven Rostedt (Google) --- fs/tracefs/internal.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/tracefs/internal.h b/fs/tracefs/internal.h index 298d3ecaf621..64fde9490f52 100644 --- a/fs/tracefs/internal.h +++ b/fs/tracefs/internal.h @@ -37,7 +37,7 @@ struct eventfs_inode { /* * Union - used for deletion * @del_list: list of eventfs_inode to delete - * @rcu: eventfs_indoe to delete in RCU + * @rcu: eventfs_inode to delete in RCU * @is_freed: node is freed if one of the above is set */ union { From 19a214bffdf7abb8d472895bb944d9c269ab1699 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Tue, 26 Sep 2023 13:59:47 -0700 Subject: [PATCH 429/475] perf vendor events intel: Add broadwellde two metrics Add tma_info_system_socket_clks and uncore_freq metrics that require a broadwellx style uncore event for UNC_CLOCK. The associated converter script fix is in: https://github.com/intel/perfmon/pull/112 Fixes: 7d124303d620 ("perf vendor events intel: Update broadwell variant events/metrics") Signed-off-by: Ian Rogers Reviewed-by: Kan Liang Cc: Caleb Biggers Cc: Perry Taylor Link: https://lore.kernel.org/r/20230926205948.1399594-2-irogers@google.com Signed-off-by: Namhyung Kim --- .../arch/x86/broadwellde/bdwde-metrics.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tools/perf/pmu-events/arch/x86/broadwellde/bdwde-metrics.json b/tools/perf/pmu-events/arch/x86/broadwellde/bdwde-metrics.json index d0ef46c9bb61..e1f55fcfa0d0 100644 --- a/tools/perf/pmu-events/arch/x86/broadwellde/bdwde-metrics.json +++ b/tools/perf/pmu-events/arch/x86/broadwellde/bdwde-metrics.json @@ -48,6 +48,12 @@ "MetricName": "C7_Pkg_Residency", "ScaleUnit": "100%" }, + { + "BriefDescription": "Uncore frequency per die [GHZ]", + "MetricExpr": "tma_info_system_socket_clks / #num_dies / duration_time / 1e9", + "MetricGroup": "SoC", + "MetricName": "UNCORE_FREQ" + }, { "BriefDescription": "Percentage of cycles spent in System Management Interrupts.", "MetricExpr": "((msr@aperf@ - cycles) / msr@aperf@ if msr@smi@ > 0 else 0)", @@ -690,6 +696,12 @@ "MetricGroup": "SMT", "MetricName": "tma_info_system_smt_2t_utilization" }, + { + "BriefDescription": "Socket actual clocks when any core is active on that socket", + "MetricExpr": "cbox_0@event\\=0x0@", + "MetricGroup": "SoC", + "MetricName": "tma_info_system_socket_clks" + }, { "BriefDescription": "Average Frequency Utilization relative nominal frequency", "MetricExpr": "tma_info_thread_clks / CPU_CLK_UNHALTED.REF_TSC", From 4ece2a7e88e1624f52c56d44394e4af1a13b1137 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Tue, 26 Sep 2023 13:59:48 -0700 Subject: [PATCH 430/475] perf vendor events intel: Add tigerlake two metrics Add tma_info_system_socket_clks and uncore_freq metrics. The associated converter script fix is in: https://github.com/intel/perfmon/pull/112 Signed-off-by: Ian Rogers Reviewed-by: Kan Liang Cc: Caleb Biggers Cc: Perry Taylor Link: https://lore.kernel.org/r/20230926205948.1399594-3-irogers@google.com Signed-off-by: Namhyung Kim --- .../pmu-events/arch/x86/tigerlake/tgl-metrics.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tools/perf/pmu-events/arch/x86/tigerlake/tgl-metrics.json b/tools/perf/pmu-events/arch/x86/tigerlake/tgl-metrics.json index fab084e1bc69..820d85c2933b 100644 --- a/tools/perf/pmu-events/arch/x86/tigerlake/tgl-metrics.json +++ b/tools/perf/pmu-events/arch/x86/tigerlake/tgl-metrics.json @@ -62,6 +62,12 @@ "MetricName": "C9_Pkg_Residency", "ScaleUnit": "100%" }, + { + "BriefDescription": "Uncore frequency per die [GHZ]", + "MetricExpr": "tma_info_system_socket_clks / #num_dies / duration_time / 1e9", + "MetricGroup": "SoC", + "MetricName": "UNCORE_FREQ" + }, { "BriefDescription": "Percentage of cycles spent in System Management Interrupts.", "MetricExpr": "((msr@aperf@ - cycles) / msr@aperf@ if msr@smi@ > 0 else 0)", @@ -1026,6 +1032,12 @@ "MetricGroup": "SMT", "MetricName": "tma_info_system_smt_2t_utilization" }, + { + "BriefDescription": "Socket actual clocks when any core is active on that socket", + "MetricExpr": "UNC_CLOCK.SOCKET", + "MetricGroup": "SoC", + "MetricName": "tma_info_system_socket_clks" + }, { "BriefDescription": "Average Frequency Utilization relative nominal frequency", "MetricExpr": "tma_info_thread_clks / CPU_CLK_UNHALTED.REF_TSC", From 0b783d2e82d827af73c779e8e8f95d07e992b451 Mon Sep 17 00:00:00 2001 From: James Clark Date: Mon, 23 Oct 2023 14:15:49 +0100 Subject: [PATCH 431/475] perf tests: test_arm_coresight: Simplify source iteration There are two reasons to do this, firstly there is a shellcheck warning in cs_etm_dev_name(), which can be completely deleted. And secondly the current iteration method doesn't support systems with both ETE and ETM because it picks one or the other. There isn't a known system with this configuration, but it could happen in the future. Iterating over all the sources for each CPU can be done by going through /sys/bus/event_source/devices/cs_etm/cpu* and following the symlink back to the Coresight device in /sys/bus/coresight/devices. This will work whether the device is ETE, ETM or any future name, and is much simpler and doesn't require any hard coded version numbers Suggested-by: Suzuki K Poulose Signed-off-by: James Clark Acked-by: Ian Rogers Tested-by: Leo Yan Cc: tianruidong@linux.alibaba.com Cc: Kajol Jain Cc: Anushree Mathur Cc: Tiezhu Yang Cc: atrajeev@linux.vnet.ibm.com Cc: coresight@lists.linaro.org Link: https://lore.kernel.org/r/20231023131550.487760-1-james.clark@arm.com Signed-off-by: Namhyung Kim --- tools/perf/tests/shell/test_arm_coresight.sh | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/tools/perf/tests/shell/test_arm_coresight.sh b/tools/perf/tests/shell/test_arm_coresight.sh index fe78c4626e45..65dd85207125 100755 --- a/tools/perf/tests/shell/test_arm_coresight.sh +++ b/tools/perf/tests/shell/test_arm_coresight.sh @@ -11,19 +11,6 @@ glb_err=0 -cs_etm_dev_name() { - cs_etm_path=$(find /sys/bus/event_source/devices/cs_etm/ -name cpu* -print -quit) - trcdevarch=$(cat ${cs_etm_path}/mgmt/trcdevarch) - archhver=$((($trcdevarch >> 12) & 0xf)) - archpart=$(($trcdevarch & 0xfff)) - - if [ $archhver -eq 5 -a "$(printf "0x%X\n" $archpart)" = "0xA13" ] ; then - echo "ete" - else - echo "etm" - fi -} - skip_if_no_cs_etm_event() { perf list | grep -q 'cs_etm//' && return 0 @@ -149,7 +136,9 @@ arm_cs_iterate_devices() { arm_cs_etm_traverse_path_test() { # Iterate for every ETM device - for dev in /sys/bus/coresight/devices/$(cs_etm_dev_name)*; do + for dev in /sys/bus/event_source/devices/cs_etm/cpu*; do + # Canonicalize the path + dev=`readlink -f $dev` # Find the ETM device belonging to which CPU cpu=`cat $dev/cpu` From 13fc6455fa19b0859e1b9640bf09903bec8df4f4 Mon Sep 17 00:00:00 2001 From: Konstantin Meskhidze Date: Thu, 26 Oct 2023 09:47:40 +0800 Subject: [PATCH 432/475] landlock: Make ruleset's access masks more generic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename ruleset's access masks and modify it's type to access_masks_t to support network type rules in following commits. Add filesystem helper functions to add and get filesystem mask. Signed-off-by: Konstantin Meskhidze Link: https://lore.kernel.org/r/20231026014751.414649-2-konstantin.meskhidze@huawei.com Signed-off-by: Mickaël Salaün --- security/landlock/fs.c | 10 +++++----- security/landlock/limits.h | 1 + security/landlock/ruleset.c | 17 +++++++++-------- security/landlock/ruleset.h | 35 +++++++++++++++++++++++++++++++---- security/landlock/syscalls.c | 7 ++++--- 5 files changed, 50 insertions(+), 20 deletions(-) diff --git a/security/landlock/fs.c b/security/landlock/fs.c index 1c0c198f6fdb..6953801619ac 100644 --- a/security/landlock/fs.c +++ b/security/landlock/fs.c @@ -178,9 +178,9 @@ int landlock_append_fs_rule(struct landlock_ruleset *const ruleset, return -EINVAL; /* Transforms relative access rights to absolute ones. */ - access_rights |= - LANDLOCK_MASK_ACCESS_FS & - ~(ruleset->fs_access_masks[0] | ACCESS_INITIALLY_DENIED); + access_rights |= LANDLOCK_MASK_ACCESS_FS & + ~(landlock_get_fs_access_mask(ruleset, 0) | + ACCESS_INITIALLY_DENIED); object = get_inode_object(d_backing_inode(path->dentry)); if (IS_ERR(object)) return PTR_ERR(object); @@ -294,7 +294,7 @@ get_handled_accesses(const struct landlock_ruleset *const domain) size_t layer_level; for (layer_level = 0; layer_level < domain->num_layers; layer_level++) - access_dom |= domain->fs_access_masks[layer_level]; + access_dom |= landlock_get_fs_access_mask(domain, layer_level); return access_dom & LANDLOCK_MASK_ACCESS_FS; } @@ -336,7 +336,7 @@ init_layer_masks(const struct landlock_ruleset *const domain, * access rights. */ if (BIT_ULL(access_bit) & - (domain->fs_access_masks[layer_level] | + (landlock_get_fs_access_mask(domain, layer_level) | ACCESS_INITIALLY_DENIED)) { (*layer_masks)[access_bit] |= BIT_ULL(layer_level); diff --git a/security/landlock/limits.h b/security/landlock/limits.h index 82288f0e9e5e..bafb3b8dc677 100644 --- a/security/landlock/limits.h +++ b/security/landlock/limits.h @@ -21,6 +21,7 @@ #define LANDLOCK_LAST_ACCESS_FS LANDLOCK_ACCESS_FS_TRUNCATE #define LANDLOCK_MASK_ACCESS_FS ((LANDLOCK_LAST_ACCESS_FS << 1) - 1) #define LANDLOCK_NUM_ACCESS_FS __const_hweight64(LANDLOCK_MASK_ACCESS_FS) +#define LANDLOCK_SHIFT_ACCESS_FS 0 /* clang-format on */ diff --git a/security/landlock/ruleset.c b/security/landlock/ruleset.c index 996484f98bfd..1f3188b4e313 100644 --- a/security/landlock/ruleset.c +++ b/security/landlock/ruleset.c @@ -29,7 +29,7 @@ static struct landlock_ruleset *create_ruleset(const u32 num_layers) struct landlock_ruleset *new_ruleset; new_ruleset = - kzalloc(struct_size(new_ruleset, fs_access_masks, num_layers), + kzalloc(struct_size(new_ruleset, access_masks, num_layers), GFP_KERNEL_ACCOUNT); if (!new_ruleset) return ERR_PTR(-ENOMEM); @@ -40,7 +40,7 @@ static struct landlock_ruleset *create_ruleset(const u32 num_layers) /* * hierarchy = NULL * num_rules = 0 - * fs_access_masks[] = 0 + * access_masks[] = 0 */ return new_ruleset; } @@ -55,7 +55,7 @@ landlock_create_ruleset(const access_mask_t fs_access_mask) return ERR_PTR(-ENOMSG); new_ruleset = create_ruleset(1); if (!IS_ERR(new_ruleset)) - new_ruleset->fs_access_masks[0] = fs_access_mask; + landlock_add_fs_access_mask(new_ruleset, fs_access_mask, 0); return new_ruleset; } @@ -117,11 +117,12 @@ static void build_check_ruleset(void) .num_rules = ~0, .num_layers = ~0, }; - typeof(ruleset.fs_access_masks[0]) fs_access_mask = ~0; + typeof(ruleset.access_masks[0]) access_masks = ~0; BUILD_BUG_ON(ruleset.num_rules < LANDLOCK_MAX_NUM_RULES); BUILD_BUG_ON(ruleset.num_layers < LANDLOCK_MAX_NUM_LAYERS); - BUILD_BUG_ON(fs_access_mask < LANDLOCK_MASK_ACCESS_FS); + BUILD_BUG_ON(access_masks < + (LANDLOCK_MASK_ACCESS_FS << LANDLOCK_SHIFT_ACCESS_FS)); } /** @@ -281,7 +282,7 @@ static int merge_ruleset(struct landlock_ruleset *const dst, err = -EINVAL; goto out_unlock; } - dst->fs_access_masks[dst->num_layers - 1] = src->fs_access_masks[0]; + dst->access_masks[dst->num_layers - 1] = src->access_masks[0]; /* Merges the @src tree. */ rbtree_postorder_for_each_entry_safe(walker_rule, next_rule, &src->root, @@ -340,8 +341,8 @@ static int inherit_ruleset(struct landlock_ruleset *const parent, goto out_unlock; } /* Copies the parent layer stack and leaves a space for the new layer. */ - memcpy(child->fs_access_masks, parent->fs_access_masks, - flex_array_size(parent, fs_access_masks, parent->num_layers)); + memcpy(child->access_masks, parent->access_masks, + flex_array_size(parent, access_masks, parent->num_layers)); if (WARN_ON_ONCE(!parent->hierarchy)) { err = -EINVAL; diff --git a/security/landlock/ruleset.h b/security/landlock/ruleset.h index 55b1df8f66a8..8c99c323b06c 100644 --- a/security/landlock/ruleset.h +++ b/security/landlock/ruleset.h @@ -25,6 +25,11 @@ static_assert(BITS_PER_TYPE(access_mask_t) >= LANDLOCK_NUM_ACCESS_FS); /* Makes sure for_each_set_bit() and for_each_clear_bit() calls are OK. */ static_assert(sizeof(unsigned long) >= sizeof(access_mask_t)); +/* Ruleset access masks. */ +typedef u16 access_masks_t; +/* Makes sure all ruleset access rights can be stored. */ +static_assert(BITS_PER_TYPE(access_masks_t) >= LANDLOCK_NUM_ACCESS_FS); + typedef u16 layer_mask_t; /* Makes sure all layers can be checked. */ static_assert(BITS_PER_TYPE(layer_mask_t) >= LANDLOCK_MAX_NUM_LAYERS); @@ -110,7 +115,7 @@ struct landlock_ruleset { * section. This is only used by * landlock_put_ruleset_deferred() when @usage reaches zero. * The fields @lock, @usage, @num_rules, @num_layers and - * @fs_access_masks are then unused. + * @access_masks are then unused. */ struct work_struct work_free; struct { @@ -137,7 +142,7 @@ struct landlock_ruleset { */ u32 num_layers; /** - * @fs_access_masks: Contains the subset of filesystem + * @access_masks: Contains the subset of filesystem * actions that are restricted by a ruleset. A domain * saves all layers of merged rulesets in a stack * (FAM), starting from the first layer to the last @@ -148,13 +153,13 @@ struct landlock_ruleset { * layers are set once and never changed for the * lifetime of the ruleset. */ - access_mask_t fs_access_masks[]; + access_masks_t access_masks[]; }; }; }; struct landlock_ruleset * -landlock_create_ruleset(const access_mask_t fs_access_mask); +landlock_create_ruleset(const access_mask_t access_mask); void landlock_put_ruleset(struct landlock_ruleset *const ruleset); void landlock_put_ruleset_deferred(struct landlock_ruleset *const ruleset); @@ -177,4 +182,26 @@ static inline void landlock_get_ruleset(struct landlock_ruleset *const ruleset) refcount_inc(&ruleset->usage); } +static inline void +landlock_add_fs_access_mask(struct landlock_ruleset *const ruleset, + const access_mask_t fs_access_mask, + const u16 layer_level) +{ + access_mask_t fs_mask = fs_access_mask & LANDLOCK_MASK_ACCESS_FS; + + /* Should already be checked in sys_landlock_create_ruleset(). */ + WARN_ON_ONCE(fs_access_mask != fs_mask); + ruleset->access_masks[layer_level] |= + (fs_mask << LANDLOCK_SHIFT_ACCESS_FS); +} + +static inline access_mask_t +landlock_get_fs_access_mask(const struct landlock_ruleset *const ruleset, + const u16 layer_level) +{ + return (ruleset->access_masks[layer_level] >> + LANDLOCK_SHIFT_ACCESS_FS) & + LANDLOCK_MASK_ACCESS_FS; +} + #endif /* _SECURITY_LANDLOCK_RULESET_H */ diff --git a/security/landlock/syscalls.c b/security/landlock/syscalls.c index 245cc650a4dc..7ec6bbed7117 100644 --- a/security/landlock/syscalls.c +++ b/security/landlock/syscalls.c @@ -310,6 +310,7 @@ SYSCALL_DEFINE4(landlock_add_rule, const int, ruleset_fd, struct path path; struct landlock_ruleset *ruleset; int res, err; + access_mask_t mask; if (!landlock_initialized) return -EOPNOTSUPP; @@ -346,10 +347,10 @@ SYSCALL_DEFINE4(landlock_add_rule, const int, ruleset_fd, } /* * Checks that allowed_access matches the @ruleset constraints - * (ruleset->fs_access_masks[0] is automatically upgraded to 64-bits). + * (ruleset->access_masks[0] is automatically upgraded to 64-bits). */ - if ((path_beneath_attr.allowed_access | ruleset->fs_access_masks[0]) != - ruleset->fs_access_masks[0]) { + mask = landlock_get_fs_access_mask(ruleset, 0); + if ((path_beneath_attr.allowed_access | mask) != mask) { err = -EINVAL; goto out_put_ruleset; } From d7220364039f6beb76f311c05f74cad89da5fad5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Sala=C3=BCn?= Date: Thu, 26 Oct 2023 09:47:41 +0800 Subject: [PATCH 433/475] landlock: Allow FS topology changes for domains without such rule type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allow mount point and root directory changes when there is no filesystem rule tied to the current Landlock domain. This doesn't change anything for now because a domain must have at least a (filesystem) rule, but this will change when other rule types will come. For instance, a domain only restricting the network should have no impact on filesystem restrictions. Add a new get_current_fs_domain() helper to quickly check filesystem rule existence for all filesystem LSM hooks. Remove unnecessary inlining. Link: https://lore.kernel.org/r/20231026014751.414649-3-konstantin.meskhidze@huawei.com Signed-off-by: Mickaël Salaün --- Documentation/userspace-api/landlock.rst | 6 +- security/landlock/fs.c | 74 ++++++++++++------------ security/landlock/ruleset.h | 24 +++++++- security/landlock/syscalls.c | 2 +- 4 files changed, 63 insertions(+), 43 deletions(-) diff --git a/Documentation/userspace-api/landlock.rst b/Documentation/userspace-api/landlock.rst index d8cd8cd9ce25..f6a7da21708a 100644 --- a/Documentation/userspace-api/landlock.rst +++ b/Documentation/userspace-api/landlock.rst @@ -387,9 +387,9 @@ Current limitations Filesystem topology modification -------------------------------- -As for file renaming and linking, a sandboxed thread cannot modify its -filesystem topology, whether via :manpage:`mount(2)` or -:manpage:`pivot_root(2)`. However, :manpage:`chroot(2)` calls are not denied. +Threads sandboxed with filesystem restrictions cannot modify filesystem +topology, whether via :manpage:`mount(2)` or :manpage:`pivot_root(2)`. +However, :manpage:`chroot(2)` calls are not denied. Special filesystems ------------------- diff --git a/security/landlock/fs.c b/security/landlock/fs.c index 6953801619ac..e8d680c2c80a 100644 --- a/security/landlock/fs.c +++ b/security/landlock/fs.c @@ -150,16 +150,6 @@ retry: LANDLOCK_ACCESS_FS_TRUNCATE) /* clang-format on */ -/* - * All access rights that are denied by default whether they are handled or not - * by a ruleset/layer. This must be ORed with all ruleset->fs_access_masks[] - * entries when we need to get the absolute handled access masks. - */ -/* clang-format off */ -#define ACCESS_INITIALLY_DENIED ( \ - LANDLOCK_ACCESS_FS_REFER) -/* clang-format on */ - /* * @path: Should have been checked by get_path_from_fd(). */ @@ -179,8 +169,7 @@ int landlock_append_fs_rule(struct landlock_ruleset *const ruleset, /* Transforms relative access rights to absolute ones. */ access_rights |= LANDLOCK_MASK_ACCESS_FS & - ~(landlock_get_fs_access_mask(ruleset, 0) | - ACCESS_INITIALLY_DENIED); + ~landlock_get_fs_access_mask(ruleset, 0); object = get_inode_object(d_backing_inode(path->dentry)); if (IS_ERR(object)) return PTR_ERR(object); @@ -287,15 +276,16 @@ static inline bool is_nouser_or_private(const struct dentry *dentry) unlikely(IS_PRIVATE(d_backing_inode(dentry)))); } -static inline access_mask_t -get_handled_accesses(const struct landlock_ruleset *const domain) +static access_mask_t +get_raw_handled_fs_accesses(const struct landlock_ruleset *const domain) { - access_mask_t access_dom = ACCESS_INITIALLY_DENIED; + access_mask_t access_dom = 0; size_t layer_level; for (layer_level = 0; layer_level < domain->num_layers; layer_level++) - access_dom |= landlock_get_fs_access_mask(domain, layer_level); - return access_dom & LANDLOCK_MASK_ACCESS_FS; + access_dom |= + landlock_get_raw_fs_access_mask(domain, layer_level); + return access_dom; } /** @@ -331,13 +321,8 @@ init_layer_masks(const struct landlock_ruleset *const domain, for_each_set_bit(access_bit, &access_req, ARRAY_SIZE(*layer_masks)) { - /* - * Artificially handles all initially denied by default - * access rights. - */ if (BIT_ULL(access_bit) & - (landlock_get_fs_access_mask(domain, layer_level) | - ACCESS_INITIALLY_DENIED)) { + landlock_get_fs_access_mask(domain, layer_level)) { (*layer_masks)[access_bit] |= BIT_ULL(layer_level); handled_accesses |= BIT_ULL(access_bit); @@ -347,6 +332,25 @@ init_layer_masks(const struct landlock_ruleset *const domain, return handled_accesses; } +static access_mask_t +get_handled_fs_accesses(const struct landlock_ruleset *const domain) +{ + /* Handles all initially denied by default access rights. */ + return get_raw_handled_fs_accesses(domain) | + LANDLOCK_ACCESS_FS_INITIALLY_DENIED; +} + +static const struct landlock_ruleset *get_current_fs_domain(void) +{ + const struct landlock_ruleset *const dom = + landlock_get_current_domain(); + + if (!dom || !get_raw_handled_fs_accesses(dom)) + return NULL; + + return dom; +} + /* * Check that a destination file hierarchy has more restrictions than a source * file hierarchy. This is only used for link and rename actions. @@ -519,7 +523,7 @@ static bool is_access_to_paths_allowed( * a superset of the meaningful requested accesses). */ access_masked_parent1 = access_masked_parent2 = - get_handled_accesses(domain); + get_handled_fs_accesses(domain); is_dom_check = true; } else { if (WARN_ON_ONCE(dentry_child1 || dentry_child2)) @@ -651,8 +655,7 @@ static inline int check_access_path(const struct landlock_ruleset *const domain, static inline int current_check_access_path(const struct path *const path, const access_mask_t access_request) { - const struct landlock_ruleset *const dom = - landlock_get_current_domain(); + const struct landlock_ruleset *const dom = get_current_fs_domain(); if (!dom) return 0; @@ -815,8 +818,7 @@ static int current_check_refer_path(struct dentry *const old_dentry, struct dentry *const new_dentry, const bool removable, const bool exchange) { - const struct landlock_ruleset *const dom = - landlock_get_current_domain(); + const struct landlock_ruleset *const dom = get_current_fs_domain(); bool allow_parent1, allow_parent2; access_mask_t access_request_parent1, access_request_parent2; struct path mnt_dir; @@ -1050,7 +1052,7 @@ static int hook_sb_mount(const char *const dev_name, const struct path *const path, const char *const type, const unsigned long flags, void *const data) { - if (!landlock_get_current_domain()) + if (!get_current_fs_domain()) return 0; return -EPERM; } @@ -1058,7 +1060,7 @@ static int hook_sb_mount(const char *const dev_name, static int hook_move_mount(const struct path *const from_path, const struct path *const to_path) { - if (!landlock_get_current_domain()) + if (!get_current_fs_domain()) return 0; return -EPERM; } @@ -1069,14 +1071,14 @@ static int hook_move_mount(const struct path *const from_path, */ static int hook_sb_umount(struct vfsmount *const mnt, const int flags) { - if (!landlock_get_current_domain()) + if (!get_current_fs_domain()) return 0; return -EPERM; } static int hook_sb_remount(struct super_block *const sb, void *const mnt_opts) { - if (!landlock_get_current_domain()) + if (!get_current_fs_domain()) return 0; return -EPERM; } @@ -1092,7 +1094,7 @@ static int hook_sb_remount(struct super_block *const sb, void *const mnt_opts) static int hook_sb_pivotroot(const struct path *const old_path, const struct path *const new_path) { - if (!landlock_get_current_domain()) + if (!get_current_fs_domain()) return 0; return -EPERM; } @@ -1128,8 +1130,7 @@ static int hook_path_mknod(const struct path *const dir, struct dentry *const dentry, const umode_t mode, const unsigned int dev) { - const struct landlock_ruleset *const dom = - landlock_get_current_domain(); + const struct landlock_ruleset *const dom = get_current_fs_domain(); if (!dom) return 0; @@ -1208,8 +1209,7 @@ static int hook_file_open(struct file *const file) layer_mask_t layer_masks[LANDLOCK_NUM_ACCESS_FS] = {}; access_mask_t open_access_request, full_access_request, allowed_access; const access_mask_t optional_access = LANDLOCK_ACCESS_FS_TRUNCATE; - const struct landlock_ruleset *const dom = - landlock_get_current_domain(); + const struct landlock_ruleset *const dom = get_current_fs_domain(); if (!dom) return 0; diff --git a/security/landlock/ruleset.h b/security/landlock/ruleset.h index 8c99c323b06c..6e2ad1546ab3 100644 --- a/security/landlock/ruleset.h +++ b/security/landlock/ruleset.h @@ -15,10 +15,21 @@ #include #include #include +#include #include "limits.h" #include "object.h" +/* + * All access rights that are denied by default whether they are handled or not + * by a ruleset/layer. This must be ORed with all ruleset->access_masks[] + * entries when we need to get the absolute handled access masks. + */ +/* clang-format off */ +#define LANDLOCK_ACCESS_FS_INITIALLY_DENIED ( \ + LANDLOCK_ACCESS_FS_REFER) +/* clang-format on */ + typedef u16 access_mask_t; /* Makes sure all filesystem access rights can be stored. */ static_assert(BITS_PER_TYPE(access_mask_t) >= LANDLOCK_NUM_ACCESS_FS); @@ -196,12 +207,21 @@ landlock_add_fs_access_mask(struct landlock_ruleset *const ruleset, } static inline access_mask_t -landlock_get_fs_access_mask(const struct landlock_ruleset *const ruleset, - const u16 layer_level) +landlock_get_raw_fs_access_mask(const struct landlock_ruleset *const ruleset, + const u16 layer_level) { return (ruleset->access_masks[layer_level] >> LANDLOCK_SHIFT_ACCESS_FS) & LANDLOCK_MASK_ACCESS_FS; } +static inline access_mask_t +landlock_get_fs_access_mask(const struct landlock_ruleset *const ruleset, + const u16 layer_level) +{ + /* Handles all initially denied by default access rights. */ + return landlock_get_raw_fs_access_mask(ruleset, layer_level) | + LANDLOCK_ACCESS_FS_INITIALLY_DENIED; +} + #endif /* _SECURITY_LANDLOCK_RULESET_H */ diff --git a/security/landlock/syscalls.c b/security/landlock/syscalls.c index 7ec6bbed7117..d35cd5d304db 100644 --- a/security/landlock/syscalls.c +++ b/security/landlock/syscalls.c @@ -349,7 +349,7 @@ SYSCALL_DEFINE4(landlock_add_rule, const int, ruleset_fd, * Checks that allowed_access matches the @ruleset constraints * (ruleset->access_masks[0] is automatically upgraded to 64-bits). */ - mask = landlock_get_fs_access_mask(ruleset, 0); + mask = landlock_get_raw_fs_access_mask(ruleset, 0); if ((path_beneath_attr.allowed_access | mask) != mask) { err = -EINVAL; goto out_put_ruleset; From a4ac404b3032562ed6a34232b5266d0f446dd799 Mon Sep 17 00:00:00 2001 From: Konstantin Meskhidze Date: Thu, 26 Oct 2023 09:47:42 +0800 Subject: [PATCH 434/475] landlock: Refactor landlock_find_rule/insert_rule helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a new landlock_key union and landlock_id structure to support a socket port rule type. A struct landlock_id identifies a unique entry in a ruleset: either a kernel object (e.g. inode) or typed data (e.g. TCP port). There is one red-black tree per key type. Add is_object_pointer() and get_root() helpers. is_object_pointer() returns true if key type is LANDLOCK_KEY_INODE. get_root() helper returns a red-black tree root pointer according to a key type. Refactor landlock_insert_rule() and landlock_find_rule() to support coming network modifications. Adding or searching a rule in ruleset can now be done thanks to a Landlock ID argument passed to these helpers. Signed-off-by: Konstantin Meskhidze Link: https://lore.kernel.org/r/20231026014751.414649-4-konstantin.meskhidze@huawei.com [mic: Fix commit message typo] Co-developed-by: Mickaël Salaün Signed-off-by: Mickaël Salaün --- security/landlock/fs.c | 21 +++--- security/landlock/ruleset.c | 133 ++++++++++++++++++++++++++---------- security/landlock/ruleset.h | 65 +++++++++++++++--- 3 files changed, 165 insertions(+), 54 deletions(-) diff --git a/security/landlock/fs.c b/security/landlock/fs.c index e8d680c2c80a..9edb64ac8251 100644 --- a/security/landlock/fs.c +++ b/security/landlock/fs.c @@ -158,7 +158,9 @@ int landlock_append_fs_rule(struct landlock_ruleset *const ruleset, access_mask_t access_rights) { int err; - struct landlock_object *object; + struct landlock_id id = { + .type = LANDLOCK_KEY_INODE, + }; /* Files only get access rights that make sense. */ if (!d_is_dir(path->dentry) && @@ -170,17 +172,17 @@ int landlock_append_fs_rule(struct landlock_ruleset *const ruleset, /* Transforms relative access rights to absolute ones. */ access_rights |= LANDLOCK_MASK_ACCESS_FS & ~landlock_get_fs_access_mask(ruleset, 0); - object = get_inode_object(d_backing_inode(path->dentry)); - if (IS_ERR(object)) - return PTR_ERR(object); + id.key.object = get_inode_object(d_backing_inode(path->dentry)); + if (IS_ERR(id.key.object)) + return PTR_ERR(id.key.object); mutex_lock(&ruleset->lock); - err = landlock_insert_rule(ruleset, object, access_rights); + err = landlock_insert_rule(ruleset, id, access_rights); mutex_unlock(&ruleset->lock); /* * No need to check for an error because landlock_insert_rule() * increments the refcount for the new object if needed. */ - landlock_put_object(object); + landlock_put_object(id.key.object); return err; } @@ -197,6 +199,9 @@ find_rule(const struct landlock_ruleset *const domain, { const struct landlock_rule *rule; const struct inode *inode; + struct landlock_id id = { + .type = LANDLOCK_KEY_INODE, + }; /* Ignores nonexistent leafs. */ if (d_is_negative(dentry)) @@ -204,8 +209,8 @@ find_rule(const struct landlock_ruleset *const domain, inode = d_backing_inode(dentry); rcu_read_lock(); - rule = landlock_find_rule( - domain, rcu_dereference(landlock_inode(inode)->object)); + id.key.object = rcu_dereference(landlock_inode(inode)->object); + rule = landlock_find_rule(domain, id); rcu_read_unlock(); return rule; } diff --git a/security/landlock/ruleset.c b/security/landlock/ruleset.c index 1f3188b4e313..3553632e76d9 100644 --- a/security/landlock/ruleset.c +++ b/security/landlock/ruleset.c @@ -35,7 +35,7 @@ static struct landlock_ruleset *create_ruleset(const u32 num_layers) return ERR_PTR(-ENOMEM); refcount_set(&new_ruleset->usage, 1); mutex_init(&new_ruleset->lock); - new_ruleset->root = RB_ROOT; + new_ruleset->root_inode = RB_ROOT; new_ruleset->num_layers = num_layers; /* * hierarchy = NULL @@ -68,8 +68,20 @@ static void build_check_rule(void) BUILD_BUG_ON(rule.num_layers < LANDLOCK_MAX_NUM_LAYERS); } +static bool is_object_pointer(const enum landlock_key_type key_type) +{ + switch (key_type) { + case LANDLOCK_KEY_INODE: + return true; + + default: + WARN_ON_ONCE(1); + return false; + } +} + static struct landlock_rule * -create_rule(struct landlock_object *const object, +create_rule(const struct landlock_id id, const struct landlock_layer (*const layers)[], const u32 num_layers, const struct landlock_layer *const new_layer) { @@ -90,8 +102,13 @@ create_rule(struct landlock_object *const object, if (!new_rule) return ERR_PTR(-ENOMEM); RB_CLEAR_NODE(&new_rule->node); - landlock_get_object(object); - new_rule->object = object; + if (is_object_pointer(id.type)) { + /* This should be catched by insert_rule(). */ + WARN_ON_ONCE(!id.key.object); + landlock_get_object(id.key.object); + } + + new_rule->key = id.key; new_rule->num_layers = new_num_layers; /* Copies the original layer stack. */ memcpy(new_rule->layers, layers, @@ -102,12 +119,27 @@ create_rule(struct landlock_object *const object, return new_rule; } -static void free_rule(struct landlock_rule *const rule) +static struct rb_root *get_root(struct landlock_ruleset *const ruleset, + const enum landlock_key_type key_type) +{ + switch (key_type) { + case LANDLOCK_KEY_INODE: + return &ruleset->root_inode; + + default: + WARN_ON_ONCE(1); + return ERR_PTR(-EINVAL); + } +} + +static void free_rule(struct landlock_rule *const rule, + const enum landlock_key_type key_type) { might_sleep(); if (!rule) return; - landlock_put_object(rule->object); + if (is_object_pointer(key_type)) + landlock_put_object(rule->key.object); kfree(rule); } @@ -129,8 +161,8 @@ static void build_check_ruleset(void) * insert_rule - Create and insert a rule in a ruleset * * @ruleset: The ruleset to be updated. - * @object: The object to build the new rule with. The underlying kernel - * object must be held by the caller. + * @id: The ID to build the new rule with. The underlying kernel object, if + * any, must be held by the caller. * @layers: One or multiple layers to be copied into the new rule. * @num_layers: The number of @layers entries. * @@ -144,26 +176,35 @@ static void build_check_ruleset(void) * access rights. */ static int insert_rule(struct landlock_ruleset *const ruleset, - struct landlock_object *const object, + const struct landlock_id id, const struct landlock_layer (*const layers)[], - size_t num_layers) + const size_t num_layers) { struct rb_node **walker_node; struct rb_node *parent_node = NULL; struct landlock_rule *new_rule; + struct rb_root *root; might_sleep(); lockdep_assert_held(&ruleset->lock); - if (WARN_ON_ONCE(!object || !layers)) + if (WARN_ON_ONCE(!layers)) return -ENOENT; - walker_node = &(ruleset->root.rb_node); + + if (is_object_pointer(id.type) && WARN_ON_ONCE(!id.key.object)) + return -ENOENT; + + root = get_root(ruleset, id.type); + if (IS_ERR(root)) + return PTR_ERR(root); + + walker_node = &root->rb_node; while (*walker_node) { struct landlock_rule *const this = rb_entry(*walker_node, struct landlock_rule, node); - if (this->object != object) { + if (this->key.data != id.key.data) { parent_node = *walker_node; - if (this->object < object) + if (this->key.data < id.key.data) walker_node = &((*walker_node)->rb_right); else walker_node = &((*walker_node)->rb_left); @@ -195,24 +236,24 @@ static int insert_rule(struct landlock_ruleset *const ruleset, * Intersects access rights when it is a merge between a * ruleset and a domain. */ - new_rule = create_rule(object, &this->layers, this->num_layers, + new_rule = create_rule(id, &this->layers, this->num_layers, &(*layers)[0]); if (IS_ERR(new_rule)) return PTR_ERR(new_rule); - rb_replace_node(&this->node, &new_rule->node, &ruleset->root); - free_rule(this); + rb_replace_node(&this->node, &new_rule->node, root); + free_rule(this, id.type); return 0; } - /* There is no match for @object. */ + /* There is no match for @id. */ build_check_ruleset(); if (ruleset->num_rules >= LANDLOCK_MAX_NUM_RULES) return -E2BIG; - new_rule = create_rule(object, layers, num_layers, NULL); + new_rule = create_rule(id, layers, num_layers, NULL); if (IS_ERR(new_rule)) return PTR_ERR(new_rule); rb_link_node(&new_rule->node, parent_node, walker_node); - rb_insert_color(&new_rule->node, &ruleset->root); + rb_insert_color(&new_rule->node, root); ruleset->num_rules++; return 0; } @@ -230,7 +271,7 @@ static void build_check_layer(void) /* @ruleset must be locked by the caller. */ int landlock_insert_rule(struct landlock_ruleset *const ruleset, - struct landlock_object *const object, + const struct landlock_id id, const access_mask_t access) { struct landlock_layer layers[] = { { @@ -240,7 +281,7 @@ int landlock_insert_rule(struct landlock_ruleset *const ruleset, } }; build_check_layer(); - return insert_rule(ruleset, object, &layers, ARRAY_SIZE(layers)); + return insert_rule(ruleset, id, &layers, ARRAY_SIZE(layers)); } static inline void get_hierarchy(struct landlock_hierarchy *const hierarchy) @@ -263,6 +304,7 @@ static int merge_ruleset(struct landlock_ruleset *const dst, struct landlock_ruleset *const src) { struct landlock_rule *walker_rule, *next_rule; + struct rb_root *src_root; int err = 0; might_sleep(); @@ -273,6 +315,10 @@ static int merge_ruleset(struct landlock_ruleset *const dst, if (WARN_ON_ONCE(!dst || !dst->hierarchy)) return -EINVAL; + src_root = get_root(src, LANDLOCK_KEY_INODE); + if (IS_ERR(src_root)) + return PTR_ERR(src_root); + /* Locks @dst first because we are its only owner. */ mutex_lock(&dst->lock); mutex_lock_nested(&src->lock, SINGLE_DEPTH_NESTING); @@ -285,11 +331,15 @@ static int merge_ruleset(struct landlock_ruleset *const dst, dst->access_masks[dst->num_layers - 1] = src->access_masks[0]; /* Merges the @src tree. */ - rbtree_postorder_for_each_entry_safe(walker_rule, next_rule, &src->root, + rbtree_postorder_for_each_entry_safe(walker_rule, next_rule, src_root, node) { struct landlock_layer layers[] = { { .level = dst->num_layers, } }; + const struct landlock_id id = { + .key = walker_rule->key, + .type = LANDLOCK_KEY_INODE, + }; if (WARN_ON_ONCE(walker_rule->num_layers != 1)) { err = -EINVAL; @@ -300,8 +350,8 @@ static int merge_ruleset(struct landlock_ruleset *const dst, goto out_unlock; } layers[0].access = walker_rule->layers[0].access; - err = insert_rule(dst, walker_rule->object, &layers, - ARRAY_SIZE(layers)); + + err = insert_rule(dst, id, &layers, ARRAY_SIZE(layers)); if (err) goto out_unlock; } @@ -316,21 +366,30 @@ static int inherit_ruleset(struct landlock_ruleset *const parent, struct landlock_ruleset *const child) { struct landlock_rule *walker_rule, *next_rule; + struct rb_root *parent_root; int err = 0; might_sleep(); if (!parent) return 0; + parent_root = get_root(parent, LANDLOCK_KEY_INODE); + if (IS_ERR(parent_root)) + return PTR_ERR(parent_root); + /* Locks @child first because we are its only owner. */ mutex_lock(&child->lock); mutex_lock_nested(&parent->lock, SINGLE_DEPTH_NESTING); /* Copies the @parent tree. */ rbtree_postorder_for_each_entry_safe(walker_rule, next_rule, - &parent->root, node) { - err = insert_rule(child, walker_rule->object, - &walker_rule->layers, + parent_root, node) { + const struct landlock_id id = { + .key = walker_rule->key, + .type = LANDLOCK_KEY_INODE, + }; + + err = insert_rule(child, id, &walker_rule->layers, walker_rule->num_layers); if (err) goto out_unlock; @@ -362,8 +421,9 @@ static void free_ruleset(struct landlock_ruleset *const ruleset) struct landlock_rule *freeme, *next; might_sleep(); - rbtree_postorder_for_each_entry_safe(freeme, next, &ruleset->root, node) - free_rule(freeme); + rbtree_postorder_for_each_entry_safe(freeme, next, &ruleset->root_inode, + node) + free_rule(freeme, LANDLOCK_KEY_INODE); put_hierarchy(ruleset->hierarchy); kfree(ruleset); } @@ -454,20 +514,23 @@ out_put_dom: */ const struct landlock_rule * landlock_find_rule(const struct landlock_ruleset *const ruleset, - const struct landlock_object *const object) + const struct landlock_id id) { + const struct rb_root *root; const struct rb_node *node; - if (!object) + root = get_root((struct landlock_ruleset *)ruleset, id.type); + if (IS_ERR(root)) return NULL; - node = ruleset->root.rb_node; + node = root->rb_node; + while (node) { struct landlock_rule *this = rb_entry(node, struct landlock_rule, node); - if (this->object == object) + if (this->key.data == id.key.data) return this; - if (this->object < object) + if (this->key.data < id.key.data) node = node->rb_right; else node = node->rb_left; diff --git a/security/landlock/ruleset.h b/security/landlock/ruleset.h index 6e2ad1546ab3..9e04c666b23c 100644 --- a/security/landlock/ruleset.h +++ b/security/landlock/ruleset.h @@ -60,6 +60,47 @@ struct landlock_layer { access_mask_t access; }; +/** + * union landlock_key - Key of a ruleset's red-black tree + */ +union landlock_key { + /** + * @object: Pointer to identify a kernel object (e.g. an inode). + */ + struct landlock_object *object; + /** + * @data: Raw data to identify an arbitrary 32-bit value + * (e.g. a TCP port). + */ + uintptr_t data; +}; + +/** + * enum landlock_key_type - Type of &union landlock_key + */ +enum landlock_key_type { + /** + * @LANDLOCK_KEY_INODE: Type of &landlock_ruleset.root_inode's node + * keys. + */ + LANDLOCK_KEY_INODE = 1, +}; + +/** + * struct landlock_id - Unique rule identifier for a ruleset + */ +struct landlock_id { + /** + * @key: Identifies either a kernel object (e.g. an inode) or + * a raw value (e.g. a TCP port). + */ + union landlock_key key; + /** + * @type: Type of a landlock_ruleset's root tree. + */ + const enum landlock_key_type type; +}; + /** * struct landlock_rule - Access rights tied to an object */ @@ -69,12 +110,13 @@ struct landlock_rule { */ struct rb_node node; /** - * @object: Pointer to identify a kernel object (e.g. an inode). This - * is used as a key for this ruleset element. This pointer is set once - * and never modified. It always points to an allocated object because - * each rule increments the refcount of its object. + * @key: A union to identify either a kernel object (e.g. an inode) or + * a raw data value (e.g. a network socket port). This is used as a key + * for this ruleset element. The pointer is set once and never + * modified. It always points to an allocated object because each rule + * increments the refcount of its object. */ - struct landlock_object *object; + union landlock_key key; /** * @num_layers: Number of entries in @layers. */ @@ -110,11 +152,12 @@ struct landlock_hierarchy { */ struct landlock_ruleset { /** - * @root: Root of a red-black tree containing &struct landlock_rule - * nodes. Once a ruleset is tied to a process (i.e. as a domain), this - * tree is immutable until @usage reaches zero. + * @root_inode: Root of a red-black tree containing &struct + * landlock_rule nodes with inode object. Once a ruleset is tied to a + * process (i.e. as a domain), this tree is immutable until @usage + * reaches zero. */ - struct rb_root root; + struct rb_root root_inode; /** * @hierarchy: Enables hierarchy identification even when a parent * domain vanishes. This is needed for the ptrace protection. @@ -176,7 +219,7 @@ void landlock_put_ruleset(struct landlock_ruleset *const ruleset); void landlock_put_ruleset_deferred(struct landlock_ruleset *const ruleset); int landlock_insert_rule(struct landlock_ruleset *const ruleset, - struct landlock_object *const object, + const struct landlock_id id, const access_mask_t access); struct landlock_ruleset * @@ -185,7 +228,7 @@ landlock_merge_ruleset(struct landlock_ruleset *const parent, const struct landlock_rule * landlock_find_rule(const struct landlock_ruleset *const ruleset, - const struct landlock_object *const object); + const struct landlock_id id); static inline void landlock_get_ruleset(struct landlock_ruleset *const ruleset) { From 6146b6141770206792d0e1155794cc48697c7985 Mon Sep 17 00:00:00 2001 From: Konstantin Meskhidze Date: Thu, 26 Oct 2023 09:47:43 +0800 Subject: [PATCH 435/475] landlock: Refactor merge/inherit_ruleset helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refactor merge_ruleset() and inherit_ruleset() functions to support new rule types. Add merge_tree() and inherit_tree() helpers. They use a specific ruleset's red-black tree according to a key type argument. Signed-off-by: Konstantin Meskhidze Link: https://lore.kernel.org/r/20231026014751.414649-5-konstantin.meskhidze@huawei.com Signed-off-by: Mickaël Salaün --- security/landlock/ruleset.c | 118 +++++++++++++++++++++++------------- 1 file changed, 75 insertions(+), 43 deletions(-) diff --git a/security/landlock/ruleset.c b/security/landlock/ruleset.c index 3553632e76d9..cf88754a3f30 100644 --- a/security/landlock/ruleset.c +++ b/security/landlock/ruleset.c @@ -300,13 +300,53 @@ static void put_hierarchy(struct landlock_hierarchy *hierarchy) } } -static int merge_ruleset(struct landlock_ruleset *const dst, - struct landlock_ruleset *const src) +static int merge_tree(struct landlock_ruleset *const dst, + struct landlock_ruleset *const src, + const enum landlock_key_type key_type) { struct landlock_rule *walker_rule, *next_rule; struct rb_root *src_root; int err = 0; + might_sleep(); + lockdep_assert_held(&dst->lock); + lockdep_assert_held(&src->lock); + + src_root = get_root(src, key_type); + if (IS_ERR(src_root)) + return PTR_ERR(src_root); + + /* Merges the @src tree. */ + rbtree_postorder_for_each_entry_safe(walker_rule, next_rule, src_root, + node) { + struct landlock_layer layers[] = { { + .level = dst->num_layers, + } }; + const struct landlock_id id = { + .key = walker_rule->key, + .type = key_type, + }; + + if (WARN_ON_ONCE(walker_rule->num_layers != 1)) + return -EINVAL; + + if (WARN_ON_ONCE(walker_rule->layers[0].level != 0)) + return -EINVAL; + + layers[0].access = walker_rule->layers[0].access; + + err = insert_rule(dst, id, &layers, ARRAY_SIZE(layers)); + if (err) + return err; + } + return err; +} + +static int merge_ruleset(struct landlock_ruleset *const dst, + struct landlock_ruleset *const src) +{ + int err = 0; + might_sleep(); /* Should already be checked by landlock_merge_ruleset() */ if (WARN_ON_ONCE(!src)) @@ -315,10 +355,6 @@ static int merge_ruleset(struct landlock_ruleset *const dst, if (WARN_ON_ONCE(!dst || !dst->hierarchy)) return -EINVAL; - src_root = get_root(src, LANDLOCK_KEY_INODE); - if (IS_ERR(src_root)) - return PTR_ERR(src_root); - /* Locks @dst first because we are its only owner. */ mutex_lock(&dst->lock); mutex_lock_nested(&src->lock, SINGLE_DEPTH_NESTING); @@ -330,31 +366,10 @@ static int merge_ruleset(struct landlock_ruleset *const dst, } dst->access_masks[dst->num_layers - 1] = src->access_masks[0]; - /* Merges the @src tree. */ - rbtree_postorder_for_each_entry_safe(walker_rule, next_rule, src_root, - node) { - struct landlock_layer layers[] = { { - .level = dst->num_layers, - } }; - const struct landlock_id id = { - .key = walker_rule->key, - .type = LANDLOCK_KEY_INODE, - }; - - if (WARN_ON_ONCE(walker_rule->num_layers != 1)) { - err = -EINVAL; - goto out_unlock; - } - if (WARN_ON_ONCE(walker_rule->layers[0].level != 0)) { - err = -EINVAL; - goto out_unlock; - } - layers[0].access = walker_rule->layers[0].access; - - err = insert_rule(dst, id, &layers, ARRAY_SIZE(layers)); - if (err) - goto out_unlock; - } + /* Merges the @src inode tree. */ + err = merge_tree(dst, src, LANDLOCK_KEY_INODE); + if (err) + goto out_unlock; out_unlock: mutex_unlock(&src->lock); @@ -362,38 +377,55 @@ out_unlock: return err; } -static int inherit_ruleset(struct landlock_ruleset *const parent, - struct landlock_ruleset *const child) +static int inherit_tree(struct landlock_ruleset *const parent, + struct landlock_ruleset *const child, + const enum landlock_key_type key_type) { struct landlock_rule *walker_rule, *next_rule; struct rb_root *parent_root; int err = 0; might_sleep(); - if (!parent) - return 0; + lockdep_assert_held(&parent->lock); + lockdep_assert_held(&child->lock); - parent_root = get_root(parent, LANDLOCK_KEY_INODE); + parent_root = get_root(parent, key_type); if (IS_ERR(parent_root)) return PTR_ERR(parent_root); - /* Locks @child first because we are its only owner. */ - mutex_lock(&child->lock); - mutex_lock_nested(&parent->lock, SINGLE_DEPTH_NESTING); - - /* Copies the @parent tree. */ + /* Copies the @parent inode or network tree. */ rbtree_postorder_for_each_entry_safe(walker_rule, next_rule, parent_root, node) { const struct landlock_id id = { .key = walker_rule->key, - .type = LANDLOCK_KEY_INODE, + .type = key_type, }; err = insert_rule(child, id, &walker_rule->layers, walker_rule->num_layers); if (err) - goto out_unlock; + return err; } + return err; +} + +static int inherit_ruleset(struct landlock_ruleset *const parent, + struct landlock_ruleset *const child) +{ + int err = 0; + + might_sleep(); + if (!parent) + return 0; + + /* Locks @child first because we are its only owner. */ + mutex_lock(&child->lock); + mutex_lock_nested(&parent->lock, SINGLE_DEPTH_NESTING); + + /* Copies the @parent inode tree. */ + err = inherit_tree(parent, child, LANDLOCK_KEY_INODE); + if (err) + goto out_unlock; if (WARN_ON_ONCE(child->num_layers <= parent->num_layers)) { err = -EINVAL; From 0e7410112964168a65578002269ae3b80b207936 Mon Sep 17 00:00:00 2001 From: Konstantin Meskhidze Date: Thu, 26 Oct 2023 09:47:44 +0800 Subject: [PATCH 436/475] landlock: Move and rename layer helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move and rename landlock_unmask_layers() and landlock_init_layer_masks() helpers to ruleset.c to share them with Landlock network implementation in following commits. Signed-off-by: Konstantin Meskhidze Link: https://lore.kernel.org/r/20231026014751.414649-6-konstantin.meskhidze@huawei.com Signed-off-by: Mickaël Salaün --- security/landlock/fs.c | 136 ++++++------------------------------ security/landlock/ruleset.c | 98 ++++++++++++++++++++++++++ security/landlock/ruleset.h | 10 +++ 3 files changed, 129 insertions(+), 115 deletions(-) diff --git a/security/landlock/fs.c b/security/landlock/fs.c index 9edb64ac8251..e6a19ff1765a 100644 --- a/security/landlock/fs.c +++ b/security/landlock/fs.c @@ -215,60 +215,6 @@ find_rule(const struct landlock_ruleset *const domain, return rule; } -/* - * @layer_masks is read and may be updated according to the access request and - * the matching rule. - * - * Returns true if the request is allowed (i.e. relevant layer masks for the - * request are empty). - */ -static inline bool -unmask_layers(const struct landlock_rule *const rule, - const access_mask_t access_request, - layer_mask_t (*const layer_masks)[LANDLOCK_NUM_ACCESS_FS]) -{ - size_t layer_level; - - if (!access_request || !layer_masks) - return true; - if (!rule) - return false; - - /* - * An access is granted if, for each policy layer, at least one rule - * encountered on the pathwalk grants the requested access, - * regardless of its position in the layer stack. We must then check - * the remaining layers for each inode, from the first added layer to - * the last one. When there is multiple requested accesses, for each - * policy layer, the full set of requested accesses may not be granted - * by only one rule, but by the union (binary OR) of multiple rules. - * E.g. /a/b + /a => /a/b - */ - for (layer_level = 0; layer_level < rule->num_layers; layer_level++) { - const struct landlock_layer *const layer = - &rule->layers[layer_level]; - const layer_mask_t layer_bit = BIT_ULL(layer->level - 1); - const unsigned long access_req = access_request; - unsigned long access_bit; - bool is_empty; - - /* - * Records in @layer_masks which layer grants access to each - * requested access. - */ - is_empty = true; - for_each_set_bit(access_bit, &access_req, - ARRAY_SIZE(*layer_masks)) { - if (layer->access & BIT_ULL(access_bit)) - (*layer_masks)[access_bit] &= ~layer_bit; - is_empty = is_empty && !(*layer_masks)[access_bit]; - } - if (is_empty) - return true; - } - return false; -} - /* * Allows access to pseudo filesystems that will never be mountable (e.g. * sockfs, pipefs), but can still be reachable through @@ -293,50 +239,6 @@ get_raw_handled_fs_accesses(const struct landlock_ruleset *const domain) return access_dom; } -/** - * init_layer_masks - Initialize layer masks from an access request - * - * Populates @layer_masks such that for each access right in @access_request, - * the bits for all the layers are set where this access right is handled. - * - * @domain: The domain that defines the current restrictions. - * @access_request: The requested access rights to check. - * @layer_masks: The layer masks to populate. - * - * Returns: An access mask where each access right bit is set which is handled - * in any of the active layers in @domain. - */ -static inline access_mask_t -init_layer_masks(const struct landlock_ruleset *const domain, - const access_mask_t access_request, - layer_mask_t (*const layer_masks)[LANDLOCK_NUM_ACCESS_FS]) -{ - access_mask_t handled_accesses = 0; - size_t layer_level; - - memset(layer_masks, 0, sizeof(*layer_masks)); - /* An empty access request can happen because of O_WRONLY | O_RDWR. */ - if (!access_request) - return 0; - - /* Saves all handled accesses per layer. */ - for (layer_level = 0; layer_level < domain->num_layers; layer_level++) { - const unsigned long access_req = access_request; - unsigned long access_bit; - - for_each_set_bit(access_bit, &access_req, - ARRAY_SIZE(*layer_masks)) { - if (BIT_ULL(access_bit) & - landlock_get_fs_access_mask(domain, layer_level)) { - (*layer_masks)[access_bit] |= - BIT_ULL(layer_level); - handled_accesses |= BIT_ULL(access_bit); - } - } - } - return handled_accesses; -} - static access_mask_t get_handled_fs_accesses(const struct landlock_ruleset *const domain) { @@ -540,18 +442,20 @@ static bool is_access_to_paths_allowed( } if (unlikely(dentry_child1)) { - unmask_layers(find_rule(domain, dentry_child1), - init_layer_masks(domain, LANDLOCK_MASK_ACCESS_FS, + landlock_unmask_layers(find_rule(domain, dentry_child1), + landlock_init_layer_masks( + domain, LANDLOCK_MASK_ACCESS_FS, &_layer_masks_child1), - &_layer_masks_child1); + &_layer_masks_child1); layer_masks_child1 = &_layer_masks_child1; child1_is_directory = d_is_dir(dentry_child1); } if (unlikely(dentry_child2)) { - unmask_layers(find_rule(domain, dentry_child2), - init_layer_masks(domain, LANDLOCK_MASK_ACCESS_FS, + landlock_unmask_layers(find_rule(domain, dentry_child2), + landlock_init_layer_masks( + domain, LANDLOCK_MASK_ACCESS_FS, &_layer_masks_child2), - &_layer_masks_child2); + &_layer_masks_child2); layer_masks_child2 = &_layer_masks_child2; child2_is_directory = d_is_dir(dentry_child2); } @@ -603,10 +507,10 @@ static bool is_access_to_paths_allowed( } rule = find_rule(domain, walker_path.dentry); - allowed_parent1 = unmask_layers(rule, access_masked_parent1, - layer_masks_parent1); - allowed_parent2 = unmask_layers(rule, access_masked_parent2, - layer_masks_parent2); + allowed_parent1 = landlock_unmask_layers( + rule, access_masked_parent1, layer_masks_parent1); + allowed_parent2 = landlock_unmask_layers( + rule, access_masked_parent2, layer_masks_parent2); /* Stops when a rule from each layer grants access. */ if (allowed_parent1 && allowed_parent2) @@ -650,7 +554,8 @@ static inline int check_access_path(const struct landlock_ruleset *const domain, { layer_mask_t layer_masks[LANDLOCK_NUM_ACCESS_FS] = {}; - access_request = init_layer_masks(domain, access_request, &layer_masks); + access_request = + landlock_init_layer_masks(domain, access_request, &layer_masks); if (is_access_to_paths_allowed(domain, path, access_request, &layer_masks, NULL, 0, NULL, NULL)) return 0; @@ -735,16 +640,16 @@ static bool collect_domain_accesses( if (is_nouser_or_private(dir)) return true; - access_dom = init_layer_masks(domain, LANDLOCK_MASK_ACCESS_FS, - layer_masks_dom); + access_dom = landlock_init_layer_masks(domain, LANDLOCK_MASK_ACCESS_FS, + layer_masks_dom); dget(dir); while (true) { struct dentry *parent_dentry; /* Gets all layers allowing all domain accesses. */ - if (unmask_layers(find_rule(domain, dir), access_dom, - layer_masks_dom)) { + if (landlock_unmask_layers(find_rule(domain, dir), access_dom, + layer_masks_dom)) { /* * Stops when all handled accesses are allowed by at * least one rule in each layer. @@ -857,7 +762,7 @@ static int current_check_refer_path(struct dentry *const old_dentry, * The LANDLOCK_ACCESS_FS_REFER access right is not required * for same-directory referer (i.e. no reparenting). */ - access_request_parent1 = init_layer_masks( + access_request_parent1 = landlock_init_layer_masks( dom, access_request_parent1 | access_request_parent2, &layer_masks_parent1); if (is_access_to_paths_allowed( @@ -1234,7 +1139,8 @@ static int hook_file_open(struct file *const file) if (is_access_to_paths_allowed( dom, &file->f_path, - init_layer_masks(dom, full_access_request, &layer_masks), + landlock_init_layer_masks(dom, full_access_request, + &layer_masks), &layer_masks, NULL, 0, NULL, NULL)) { allowed_access = full_access_request; } else { diff --git a/security/landlock/ruleset.c b/security/landlock/ruleset.c index cf88754a3f30..28ffeedd21d6 100644 --- a/security/landlock/ruleset.c +++ b/security/landlock/ruleset.c @@ -569,3 +569,101 @@ landlock_find_rule(const struct landlock_ruleset *const ruleset, } return NULL; } + +/* + * @layer_masks is read and may be updated according to the access request and + * the matching rule. + * + * Returns true if the request is allowed (i.e. relevant layer masks for the + * request are empty). + */ +bool landlock_unmask_layers( + const struct landlock_rule *const rule, + const access_mask_t access_request, + layer_mask_t (*const layer_masks)[LANDLOCK_NUM_ACCESS_FS]) +{ + size_t layer_level; + + if (!access_request || !layer_masks) + return true; + if (!rule) + return false; + + /* + * An access is granted if, for each policy layer, at least one rule + * encountered on the pathwalk grants the requested access, + * regardless of its position in the layer stack. We must then check + * the remaining layers for each inode, from the first added layer to + * the last one. When there is multiple requested accesses, for each + * policy layer, the full set of requested accesses may not be granted + * by only one rule, but by the union (binary OR) of multiple rules. + * E.g. /a/b + /a => /a/b + */ + for (layer_level = 0; layer_level < rule->num_layers; layer_level++) { + const struct landlock_layer *const layer = + &rule->layers[layer_level]; + const layer_mask_t layer_bit = BIT_ULL(layer->level - 1); + const unsigned long access_req = access_request; + unsigned long access_bit; + bool is_empty; + + /* + * Records in @layer_masks which layer grants access to each + * requested access. + */ + is_empty = true; + for_each_set_bit(access_bit, &access_req, + ARRAY_SIZE(*layer_masks)) { + if (layer->access & BIT_ULL(access_bit)) + (*layer_masks)[access_bit] &= ~layer_bit; + is_empty = is_empty && !(*layer_masks)[access_bit]; + } + if (is_empty) + return true; + } + return false; +} + +/** + * landlock_init_layer_masks - Initialize layer masks from an access request + * + * Populates @layer_masks such that for each access right in @access_request, + * the bits for all the layers are set where this access right is handled. + * + * @domain: The domain that defines the current restrictions. + * @access_request: The requested access rights to check. + * @layer_masks: The layer masks to populate. + * + * Returns: An access mask where each access right bit is set which is handled + * in any of the active layers in @domain. + */ +access_mask_t landlock_init_layer_masks( + const struct landlock_ruleset *const domain, + const access_mask_t access_request, + layer_mask_t (*const layer_masks)[LANDLOCK_NUM_ACCESS_FS]) +{ + access_mask_t handled_accesses = 0; + size_t layer_level; + + memset(layer_masks, 0, sizeof(*layer_masks)); + /* An empty access request can happen because of O_WRONLY | O_RDWR. */ + if (!access_request) + return 0; + + /* Saves all handled accesses per layer. */ + for (layer_level = 0; layer_level < domain->num_layers; layer_level++) { + const unsigned long access_req = access_request; + unsigned long access_bit; + + for_each_set_bit(access_bit, &access_req, + ARRAY_SIZE(*layer_masks)) { + if (BIT_ULL(access_bit) & + landlock_get_fs_access_mask(domain, layer_level)) { + (*layer_masks)[access_bit] |= + BIT_ULL(layer_level); + handled_accesses |= BIT_ULL(access_bit); + } + } + } + return handled_accesses; +} diff --git a/security/landlock/ruleset.h b/security/landlock/ruleset.h index 9e04c666b23c..760e049f3e0a 100644 --- a/security/landlock/ruleset.h +++ b/security/landlock/ruleset.h @@ -267,4 +267,14 @@ landlock_get_fs_access_mask(const struct landlock_ruleset *const ruleset, LANDLOCK_ACCESS_FS_INITIALLY_DENIED; } +bool landlock_unmask_layers( + const struct landlock_rule *const rule, + const access_mask_t access_request, + layer_mask_t (*const layer_masks)[LANDLOCK_NUM_ACCESS_FS]); + +access_mask_t landlock_init_layer_masks( + const struct landlock_ruleset *const domain, + const access_mask_t access_request, + layer_mask_t (*const layer_masks)[LANDLOCK_NUM_ACCESS_FS]); + #endif /* _SECURITY_LANDLOCK_RULESET_H */ From 7a11275c378753fbdf00f2cfc80bc86a119ca67d Mon Sep 17 00:00:00 2001 From: Konstantin Meskhidze Date: Thu, 26 Oct 2023 09:47:45 +0800 Subject: [PATCH 437/475] landlock: Refactor layer helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a new key_type argument to the landlock_init_layer_masks() helper. Add a masks_array_size argument to the landlock_unmask_layers() helper. These modifications support implementing new rule types in the next Landlock versions. Signed-off-by: Konstantin Meskhidze Link: https://lore.kernel.org/r/20231026014751.414649-7-konstantin.meskhidze@huawei.com Signed-off-by: Mickaël Salaün --- security/landlock/fs.c | 43 ++++++++++++++++++--------------- security/landlock/ruleset.c | 48 +++++++++++++++++++++++++------------ security/landlock/ruleset.h | 17 ++++++------- 3 files changed, 66 insertions(+), 42 deletions(-) diff --git a/security/landlock/fs.c b/security/landlock/fs.c index e6a19ff1765a..bc7c126deea2 100644 --- a/security/landlock/fs.c +++ b/security/landlock/fs.c @@ -442,20 +442,22 @@ static bool is_access_to_paths_allowed( } if (unlikely(dentry_child1)) { - landlock_unmask_layers(find_rule(domain, dentry_child1), - landlock_init_layer_masks( - domain, LANDLOCK_MASK_ACCESS_FS, - &_layer_masks_child1), - &_layer_masks_child1); + landlock_unmask_layers( + find_rule(domain, dentry_child1), + landlock_init_layer_masks( + domain, LANDLOCK_MASK_ACCESS_FS, + &_layer_masks_child1, LANDLOCK_KEY_INODE), + &_layer_masks_child1, ARRAY_SIZE(_layer_masks_child1)); layer_masks_child1 = &_layer_masks_child1; child1_is_directory = d_is_dir(dentry_child1); } if (unlikely(dentry_child2)) { - landlock_unmask_layers(find_rule(domain, dentry_child2), - landlock_init_layer_masks( - domain, LANDLOCK_MASK_ACCESS_FS, - &_layer_masks_child2), - &_layer_masks_child2); + landlock_unmask_layers( + find_rule(domain, dentry_child2), + landlock_init_layer_masks( + domain, LANDLOCK_MASK_ACCESS_FS, + &_layer_masks_child2, LANDLOCK_KEY_INODE), + &_layer_masks_child2, ARRAY_SIZE(_layer_masks_child2)); layer_masks_child2 = &_layer_masks_child2; child2_is_directory = d_is_dir(dentry_child2); } @@ -508,14 +510,15 @@ static bool is_access_to_paths_allowed( rule = find_rule(domain, walker_path.dentry); allowed_parent1 = landlock_unmask_layers( - rule, access_masked_parent1, layer_masks_parent1); + rule, access_masked_parent1, layer_masks_parent1, + ARRAY_SIZE(*layer_masks_parent1)); allowed_parent2 = landlock_unmask_layers( - rule, access_masked_parent2, layer_masks_parent2); + rule, access_masked_parent2, layer_masks_parent2, + ARRAY_SIZE(*layer_masks_parent2)); /* Stops when a rule from each layer grants access. */ if (allowed_parent1 && allowed_parent2) break; - jump_up: if (walker_path.dentry == walker_path.mnt->mnt_root) { if (follow_up(&walker_path)) { @@ -554,8 +557,8 @@ static inline int check_access_path(const struct landlock_ruleset *const domain, { layer_mask_t layer_masks[LANDLOCK_NUM_ACCESS_FS] = {}; - access_request = - landlock_init_layer_masks(domain, access_request, &layer_masks); + access_request = landlock_init_layer_masks( + domain, access_request, &layer_masks, LANDLOCK_KEY_INODE); if (is_access_to_paths_allowed(domain, path, access_request, &layer_masks, NULL, 0, NULL, NULL)) return 0; @@ -641,7 +644,8 @@ static bool collect_domain_accesses( return true; access_dom = landlock_init_layer_masks(domain, LANDLOCK_MASK_ACCESS_FS, - layer_masks_dom); + layer_masks_dom, + LANDLOCK_KEY_INODE); dget(dir); while (true) { @@ -649,7 +653,8 @@ static bool collect_domain_accesses( /* Gets all layers allowing all domain accesses. */ if (landlock_unmask_layers(find_rule(domain, dir), access_dom, - layer_masks_dom)) { + layer_masks_dom, + ARRAY_SIZE(*layer_masks_dom))) { /* * Stops when all handled accesses are allowed by at * least one rule in each layer. @@ -764,7 +769,7 @@ static int current_check_refer_path(struct dentry *const old_dentry, */ access_request_parent1 = landlock_init_layer_masks( dom, access_request_parent1 | access_request_parent2, - &layer_masks_parent1); + &layer_masks_parent1, LANDLOCK_KEY_INODE); if (is_access_to_paths_allowed( dom, new_dir, access_request_parent1, &layer_masks_parent1, NULL, 0, NULL, NULL)) @@ -1140,7 +1145,7 @@ static int hook_file_open(struct file *const file) if (is_access_to_paths_allowed( dom, &file->f_path, landlock_init_layer_masks(dom, full_access_request, - &layer_masks), + &layer_masks, LANDLOCK_KEY_INODE), &layer_masks, NULL, 0, NULL, NULL)) { allowed_access = full_access_request; } else { diff --git a/security/landlock/ruleset.c b/security/landlock/ruleset.c index 28ffeedd21d6..e36c5c332242 100644 --- a/security/landlock/ruleset.c +++ b/security/landlock/ruleset.c @@ -573,14 +573,15 @@ landlock_find_rule(const struct landlock_ruleset *const ruleset, /* * @layer_masks is read and may be updated according to the access request and * the matching rule. + * @masks_array_size must be equal to ARRAY_SIZE(*layer_masks). * * Returns true if the request is allowed (i.e. relevant layer masks for the * request are empty). */ -bool landlock_unmask_layers( - const struct landlock_rule *const rule, - const access_mask_t access_request, - layer_mask_t (*const layer_masks)[LANDLOCK_NUM_ACCESS_FS]) +bool landlock_unmask_layers(const struct landlock_rule *const rule, + const access_mask_t access_request, + layer_mask_t (*const layer_masks)[], + const size_t masks_array_size) { size_t layer_level; @@ -612,8 +613,7 @@ bool landlock_unmask_layers( * requested access. */ is_empty = true; - for_each_set_bit(access_bit, &access_req, - ARRAY_SIZE(*layer_masks)) { + for_each_set_bit(access_bit, &access_req, masks_array_size) { if (layer->access & BIT_ULL(access_bit)) (*layer_masks)[access_bit] &= ~layer_bit; is_empty = is_empty && !(*layer_masks)[access_bit]; @@ -624,6 +624,10 @@ bool landlock_unmask_layers( return false; } +typedef access_mask_t +get_access_mask_t(const struct landlock_ruleset *const ruleset, + const u16 layer_level); + /** * landlock_init_layer_masks - Initialize layer masks from an access request * @@ -633,19 +637,34 @@ bool landlock_unmask_layers( * @domain: The domain that defines the current restrictions. * @access_request: The requested access rights to check. * @layer_masks: The layer masks to populate. + * @key_type: The key type to switch between access masks of different types. * * Returns: An access mask where each access right bit is set which is handled * in any of the active layers in @domain. */ -access_mask_t landlock_init_layer_masks( - const struct landlock_ruleset *const domain, - const access_mask_t access_request, - layer_mask_t (*const layer_masks)[LANDLOCK_NUM_ACCESS_FS]) +access_mask_t +landlock_init_layer_masks(const struct landlock_ruleset *const domain, + const access_mask_t access_request, + layer_mask_t (*const layer_masks)[], + const enum landlock_key_type key_type) { access_mask_t handled_accesses = 0; - size_t layer_level; + size_t layer_level, num_access; + get_access_mask_t *get_access_mask; + + switch (key_type) { + case LANDLOCK_KEY_INODE: + get_access_mask = landlock_get_fs_access_mask; + num_access = LANDLOCK_NUM_ACCESS_FS; + break; + default: + WARN_ON_ONCE(1); + return 0; + } + + memset(layer_masks, 0, + array_size(sizeof((*layer_masks)[0]), num_access)); - memset(layer_masks, 0, sizeof(*layer_masks)); /* An empty access request can happen because of O_WRONLY | O_RDWR. */ if (!access_request) return 0; @@ -655,10 +674,9 @@ access_mask_t landlock_init_layer_masks( const unsigned long access_req = access_request; unsigned long access_bit; - for_each_set_bit(access_bit, &access_req, - ARRAY_SIZE(*layer_masks)) { + for_each_set_bit(access_bit, &access_req, num_access) { if (BIT_ULL(access_bit) & - landlock_get_fs_access_mask(domain, layer_level)) { + get_access_mask(domain, layer_level)) { (*layer_masks)[access_bit] |= BIT_ULL(layer_level); handled_accesses |= BIT_ULL(access_bit); diff --git a/security/landlock/ruleset.h b/security/landlock/ruleset.h index 760e049f3e0a..7107ce7da39d 100644 --- a/security/landlock/ruleset.h +++ b/security/landlock/ruleset.h @@ -267,14 +267,15 @@ landlock_get_fs_access_mask(const struct landlock_ruleset *const ruleset, LANDLOCK_ACCESS_FS_INITIALLY_DENIED; } -bool landlock_unmask_layers( - const struct landlock_rule *const rule, - const access_mask_t access_request, - layer_mask_t (*const layer_masks)[LANDLOCK_NUM_ACCESS_FS]); +bool landlock_unmask_layers(const struct landlock_rule *const rule, + const access_mask_t access_request, + layer_mask_t (*const layer_masks)[], + const size_t masks_array_size); -access_mask_t landlock_init_layer_masks( - const struct landlock_ruleset *const domain, - const access_mask_t access_request, - layer_mask_t (*const layer_masks)[LANDLOCK_NUM_ACCESS_FS]); +access_mask_t +landlock_init_layer_masks(const struct landlock_ruleset *const domain, + const access_mask_t access_request, + layer_mask_t (*const layer_masks)[], + const enum landlock_key_type key_type); #endif /* _SECURITY_LANDLOCK_RULESET_H */ From 0e0fc7e8eb4a11bd9f89a9c74bc7c0e144c56203 Mon Sep 17 00:00:00 2001 From: Konstantin Meskhidze Date: Thu, 26 Oct 2023 09:47:46 +0800 Subject: [PATCH 438/475] landlock: Refactor landlock_add_rule() syscall MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change the landlock_add_rule() syscall to support new rule types with next commits. Add the add_rule_path_beneath() helper to support current filesystem rules. Signed-off-by: Konstantin Meskhidze Link: https://lore.kernel.org/r/20231026014751.414649-8-konstantin.meskhidze@huawei.com Signed-off-by: Mickaël Salaün --- security/landlock/syscalls.c | 89 ++++++++++++++++++------------------ 1 file changed, 45 insertions(+), 44 deletions(-) diff --git a/security/landlock/syscalls.c b/security/landlock/syscalls.c index d35cd5d304db..e87f572e0251 100644 --- a/security/landlock/syscalls.c +++ b/security/landlock/syscalls.c @@ -274,6 +274,44 @@ out_fdput: return err; } +static int add_rule_path_beneath(struct landlock_ruleset *const ruleset, + const void __user *const rule_attr) +{ + struct landlock_path_beneath_attr path_beneath_attr; + struct path path; + int res, err; + access_mask_t mask; + + /* Copies raw user space buffer, only one type for now. */ + res = copy_from_user(&path_beneath_attr, rule_attr, + sizeof(path_beneath_attr)); + if (res) + return -EFAULT; + + /* + * Informs about useless rule: empty allowed_access (i.e. deny rules) + * are ignored in path walks. + */ + if (!path_beneath_attr.allowed_access) + return -ENOMSG; + + /* Checks that allowed_access matches the @ruleset constraints. */ + mask = landlock_get_raw_fs_access_mask(ruleset, 0); + if ((path_beneath_attr.allowed_access | mask) != mask) + return -EINVAL; + + /* Gets and checks the new rule. */ + err = get_path_from_fd(path_beneath_attr.parent_fd, &path); + if (err) + return err; + + /* Imports the new rule. */ + err = landlock_append_fs_rule(ruleset, &path, + path_beneath_attr.allowed_access); + path_put(&path); + return err; +} + /** * sys_landlock_add_rule - Add a new rule to a ruleset * @@ -306,11 +344,8 @@ SYSCALL_DEFINE4(landlock_add_rule, const int, ruleset_fd, const enum landlock_rule_type, rule_type, const void __user *const, rule_attr, const __u32, flags) { - struct landlock_path_beneath_attr path_beneath_attr; - struct path path; struct landlock_ruleset *ruleset; - int res, err; - access_mask_t mask; + int err; if (!landlock_initialized) return -EOPNOTSUPP; @@ -324,48 +359,14 @@ SYSCALL_DEFINE4(landlock_add_rule, const int, ruleset_fd, if (IS_ERR(ruleset)) return PTR_ERR(ruleset); - if (rule_type != LANDLOCK_RULE_PATH_BENEATH) { + switch (rule_type) { + case LANDLOCK_RULE_PATH_BENEATH: + err = add_rule_path_beneath(ruleset, rule_attr); + break; + default: err = -EINVAL; - goto out_put_ruleset; + break; } - - /* Copies raw user space buffer, only one type for now. */ - res = copy_from_user(&path_beneath_attr, rule_attr, - sizeof(path_beneath_attr)); - if (res) { - err = -EFAULT; - goto out_put_ruleset; - } - - /* - * Informs about useless rule: empty allowed_access (i.e. deny rules) - * are ignored in path walks. - */ - if (!path_beneath_attr.allowed_access) { - err = -ENOMSG; - goto out_put_ruleset; - } - /* - * Checks that allowed_access matches the @ruleset constraints - * (ruleset->access_masks[0] is automatically upgraded to 64-bits). - */ - mask = landlock_get_raw_fs_access_mask(ruleset, 0); - if ((path_beneath_attr.allowed_access | mask) != mask) { - err = -EINVAL; - goto out_put_ruleset; - } - - /* Gets and checks the new rule. */ - err = get_path_from_fd(path_beneath_attr.parent_fd, &path); - if (err) - goto out_put_ruleset; - - /* Imports the new rule. */ - err = landlock_append_fs_rule(ruleset, &path, - path_beneath_attr.allowed_access); - path_put(&path); - -out_put_ruleset: landlock_put_ruleset(ruleset); return err; } From fff69fb03dde1dfa348cfdb74b13287dabe42c25 Mon Sep 17 00:00:00 2001 From: Konstantin Meskhidze Date: Thu, 26 Oct 2023 09:47:47 +0800 Subject: [PATCH 439/475] landlock: Support network rules with TCP bind and connect MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add network rules support in the ruleset management helpers and the landlock_create_ruleset() syscall. Extend user space API to support network actions: * Add new network access rights: LANDLOCK_ACCESS_NET_BIND_TCP and LANDLOCK_ACCESS_NET_CONNECT_TCP. * Add a new network rule type: LANDLOCK_RULE_NET_PORT tied to struct landlock_net_port_attr. The allowed_access field contains the network access rights, and the port field contains the port value according to the controlled protocol. This field can take up to a 64-bit value but the maximum value depends on the related protocol (e.g. 16-bit value for TCP). Network port is in host endianness [1]. * Add a new handled_access_net field to struct landlock_ruleset_attr that contains network access rights. * Increment the Landlock ABI version to 4. Implement socket_bind() and socket_connect() LSM hooks, which enable to control TCP socket binding and connection to specific ports. Expand access_masks_t from u16 to u32 to be able to store network access rights alongside filesystem access rights for rulesets' handled access rights. Access rights are not tied to socket file descriptors but checked at bind() or connect() call time against the caller's Landlock domain. For the filesystem, a file descriptor is a direct access to a file/data. However, for network sockets, we cannot identify for which data or peer a newly created socket will give access to. Indeed, we need to wait for a connect or bind request to identify the use case for this socket. Likewise a directory file descriptor may enable to open another file (i.e. a new data item), but this opening is also restricted by the caller's domain, not the file descriptor's access rights [2]. [1] https://lore.kernel.org/r/278ab07f-7583-a4e0-3d37-1bacd091531d@digikod.net [2] https://lore.kernel.org/r/263c1eb3-602f-57fe-8450-3f138581bee7@digikod.net Signed-off-by: Konstantin Meskhidze Link: https://lore.kernel.org/r/20231026014751.414649-9-konstantin.meskhidze@huawei.com [mic: Extend commit message, fix typo in comments, and specify endianness in the documentation] Co-developed-by: Mickaël Salaün Signed-off-by: Mickaël Salaün --- include/uapi/linux/landlock.h | 55 +++++ security/landlock/Kconfig | 1 + security/landlock/Makefile | 2 + security/landlock/limits.h | 5 + security/landlock/net.c | 200 +++++++++++++++++++ security/landlock/net.h | 33 +++ security/landlock/ruleset.c | 61 +++++- security/landlock/ruleset.h | 62 +++++- security/landlock/setup.c | 2 + security/landlock/syscalls.c | 72 ++++++- tools/testing/selftests/landlock/base_test.c | 2 +- 11 files changed, 470 insertions(+), 25 deletions(-) create mode 100644 security/landlock/net.c create mode 100644 security/landlock/net.h diff --git a/include/uapi/linux/landlock.h b/include/uapi/linux/landlock.h index 81d09ef9aa50..25c8d7677539 100644 --- a/include/uapi/linux/landlock.h +++ b/include/uapi/linux/landlock.h @@ -31,6 +31,12 @@ struct landlock_ruleset_attr { * this access right. */ __u64 handled_access_fs; + /** + * @handled_access_net: Bitmask of actions (cf. `Network flags`_) + * that is handled by this ruleset and should then be forbidden if no + * rule explicitly allow them. + */ + __u64 handled_access_net; }; /* @@ -54,6 +60,11 @@ enum landlock_rule_type { * landlock_path_beneath_attr . */ LANDLOCK_RULE_PATH_BENEATH = 1, + /** + * @LANDLOCK_RULE_NET_PORT: Type of a &struct + * landlock_net_port_attr . + */ + LANDLOCK_RULE_NET_PORT, }; /** @@ -79,6 +90,31 @@ struct landlock_path_beneath_attr { */ } __attribute__((packed)); +/** + * struct landlock_net_port_attr - Network port definition + * + * Argument of sys_landlock_add_rule(). + */ +struct landlock_net_port_attr { + /** + * @allowed_access: Bitmask of allowed access network for a port + * (cf. `Network flags`_). + */ + __u64 allowed_access; + /** + * @port: Network port in host endianness. + * + * It should be noted that port 0 passed to :manpage:`bind(2)` will + * bind to an available port from a specific port range. This can be + * configured thanks to the ``/proc/sys/net/ipv4/ip_local_port_range`` + * sysctl (also used for IPv6). A Landlock rule with port 0 and the + * ``LANDLOCK_ACCESS_NET_BIND_TCP`` right means that requesting to bind + * on port 0 is allowed and it will automatically translate to binding + * on the related port range. + */ + __u64 port; +}; + /** * DOC: fs_access * @@ -189,4 +225,23 @@ struct landlock_path_beneath_attr { #define LANDLOCK_ACCESS_FS_TRUNCATE (1ULL << 14) /* clang-format on */ +/** + * DOC: net_access + * + * Network flags + * ~~~~~~~~~~~~~~~~ + * + * These flags enable to restrict a sandboxed process to a set of network + * actions. This is supported since the Landlock ABI version 4. + * + * TCP sockets with allowed actions: + * + * - %LANDLOCK_ACCESS_NET_BIND_TCP: Bind a TCP socket to a local port. + * - %LANDLOCK_ACCESS_NET_CONNECT_TCP: Connect an active TCP socket to + * a remote port. + */ +/* clang-format off */ +#define LANDLOCK_ACCESS_NET_BIND_TCP (1ULL << 0) +#define LANDLOCK_ACCESS_NET_CONNECT_TCP (1ULL << 1) +/* clang-format on */ #endif /* _UAPI_LINUX_LANDLOCK_H */ diff --git a/security/landlock/Kconfig b/security/landlock/Kconfig index c1e862a38410..c4bf0d5eff39 100644 --- a/security/landlock/Kconfig +++ b/security/landlock/Kconfig @@ -3,6 +3,7 @@ config SECURITY_LANDLOCK bool "Landlock support" depends on SECURITY + select SECURITY_NETWORK select SECURITY_PATH help Landlock is a sandboxing mechanism that enables processes to restrict diff --git a/security/landlock/Makefile b/security/landlock/Makefile index 7bbd2f413b3e..c2e116f2a299 100644 --- a/security/landlock/Makefile +++ b/security/landlock/Makefile @@ -2,3 +2,5 @@ obj-$(CONFIG_SECURITY_LANDLOCK) := landlock.o landlock-y := setup.o syscalls.o object.o ruleset.o \ cred.o ptrace.o fs.o + +landlock-$(CONFIG_INET) += net.o diff --git a/security/landlock/limits.h b/security/landlock/limits.h index bafb3b8dc677..93c9c6f91556 100644 --- a/security/landlock/limits.h +++ b/security/landlock/limits.h @@ -23,6 +23,11 @@ #define LANDLOCK_NUM_ACCESS_FS __const_hweight64(LANDLOCK_MASK_ACCESS_FS) #define LANDLOCK_SHIFT_ACCESS_FS 0 +#define LANDLOCK_LAST_ACCESS_NET LANDLOCK_ACCESS_NET_CONNECT_TCP +#define LANDLOCK_MASK_ACCESS_NET ((LANDLOCK_LAST_ACCESS_NET << 1) - 1) +#define LANDLOCK_NUM_ACCESS_NET __const_hweight64(LANDLOCK_MASK_ACCESS_NET) +#define LANDLOCK_SHIFT_ACCESS_NET LANDLOCK_NUM_ACCESS_FS + /* clang-format on */ #endif /* _SECURITY_LANDLOCK_LIMITS_H */ diff --git a/security/landlock/net.c b/security/landlock/net.c new file mode 100644 index 000000000000..aaa92c2b1f08 --- /dev/null +++ b/security/landlock/net.c @@ -0,0 +1,200 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Landlock LSM - Network management and hooks + * + * Copyright © 2022-2023 Huawei Tech. Co., Ltd. + * Copyright © 2022-2023 Microsoft Corporation + */ + +#include +#include +#include +#include + +#include "common.h" +#include "cred.h" +#include "limits.h" +#include "net.h" +#include "ruleset.h" + +int landlock_append_net_rule(struct landlock_ruleset *const ruleset, + const u16 port, access_mask_t access_rights) +{ + int err; + const struct landlock_id id = { + .key.data = (__force uintptr_t)htons(port), + .type = LANDLOCK_KEY_NET_PORT, + }; + + BUILD_BUG_ON(sizeof(port) > sizeof(id.key.data)); + + /* Transforms relative access rights to absolute ones. */ + access_rights |= LANDLOCK_MASK_ACCESS_NET & + ~landlock_get_net_access_mask(ruleset, 0); + + mutex_lock(&ruleset->lock); + err = landlock_insert_rule(ruleset, id, access_rights); + mutex_unlock(&ruleset->lock); + + return err; +} + +static access_mask_t +get_raw_handled_net_accesses(const struct landlock_ruleset *const domain) +{ + access_mask_t access_dom = 0; + size_t layer_level; + + for (layer_level = 0; layer_level < domain->num_layers; layer_level++) + access_dom |= landlock_get_net_access_mask(domain, layer_level); + return access_dom; +} + +static const struct landlock_ruleset *get_current_net_domain(void) +{ + const struct landlock_ruleset *const dom = + landlock_get_current_domain(); + + if (!dom || !get_raw_handled_net_accesses(dom)) + return NULL; + + return dom; +} + +static int current_check_access_socket(struct socket *const sock, + struct sockaddr *const address, + const int addrlen, + const access_mask_t access_request) +{ + __be16 port; + layer_mask_t layer_masks[LANDLOCK_NUM_ACCESS_NET] = {}; + const struct landlock_rule *rule; + access_mask_t handled_access; + struct landlock_id id = { + .type = LANDLOCK_KEY_NET_PORT, + }; + const struct landlock_ruleset *const dom = get_current_net_domain(); + + if (!dom) + return 0; + if (WARN_ON_ONCE(dom->num_layers < 1)) + return -EACCES; + + /* Checks if it's a (potential) TCP socket. */ + if (sock->type != SOCK_STREAM) + return 0; + + /* Checks for minimal header length to safely read sa_family. */ + if (addrlen < offsetofend(typeof(*address), sa_family)) + return -EINVAL; + + switch (address->sa_family) { + case AF_UNSPEC: + case AF_INET: + if (addrlen < sizeof(struct sockaddr_in)) + return -EINVAL; + port = ((struct sockaddr_in *)address)->sin_port; + break; + +#if IS_ENABLED(CONFIG_IPV6) + case AF_INET6: + if (addrlen < SIN6_LEN_RFC2133) + return -EINVAL; + port = ((struct sockaddr_in6 *)address)->sin6_port; + break; +#endif /* IS_ENABLED(CONFIG_IPV6) */ + + default: + return 0; + } + + /* Specific AF_UNSPEC handling. */ + if (address->sa_family == AF_UNSPEC) { + /* + * Connecting to an address with AF_UNSPEC dissolves the TCP + * association, which have the same effect as closing the + * connection while retaining the socket object (i.e., the file + * descriptor). As for dropping privileges, closing + * connections is always allowed. + * + * For a TCP access control system, this request is legitimate. + * Let the network stack handle potential inconsistencies and + * return -EINVAL if needed. + */ + if (access_request == LANDLOCK_ACCESS_NET_CONNECT_TCP) + return 0; + + /* + * For compatibility reason, accept AF_UNSPEC for bind + * accesses (mapped to AF_INET) only if the address is + * INADDR_ANY (cf. __inet_bind). Checking the address is + * required to not wrongfully return -EACCES instead of + * -EAFNOSUPPORT. + * + * We could return 0 and let the network stack handle these + * checks, but it is safer to return a proper error and test + * consistency thanks to kselftest. + */ + if (access_request == LANDLOCK_ACCESS_NET_BIND_TCP) { + /* addrlen has already been checked for AF_UNSPEC. */ + const struct sockaddr_in *const sockaddr = + (struct sockaddr_in *)address; + + if (sock->sk->__sk_common.skc_family != AF_INET) + return -EINVAL; + + if (sockaddr->sin_addr.s_addr != htonl(INADDR_ANY)) + return -EAFNOSUPPORT; + } + } else { + /* + * Checks sa_family consistency to not wrongfully return + * -EACCES instead of -EINVAL. Valid sa_family changes are + * only (from AF_INET or AF_INET6) to AF_UNSPEC. + * + * We could return 0 and let the network stack handle this + * check, but it is safer to return a proper error and test + * consistency thanks to kselftest. + */ + if (address->sa_family != sock->sk->__sk_common.skc_family) + return -EINVAL; + } + + id.key.data = (__force uintptr_t)port; + BUILD_BUG_ON(sizeof(port) > sizeof(id.key.data)); + + rule = landlock_find_rule(dom, id); + handled_access = landlock_init_layer_masks( + dom, access_request, &layer_masks, LANDLOCK_KEY_NET_PORT); + if (landlock_unmask_layers(rule, handled_access, &layer_masks, + ARRAY_SIZE(layer_masks))) + return 0; + + return -EACCES; +} + +static int hook_socket_bind(struct socket *const sock, + struct sockaddr *const address, const int addrlen) +{ + return current_check_access_socket(sock, address, addrlen, + LANDLOCK_ACCESS_NET_BIND_TCP); +} + +static int hook_socket_connect(struct socket *const sock, + struct sockaddr *const address, + const int addrlen) +{ + return current_check_access_socket(sock, address, addrlen, + LANDLOCK_ACCESS_NET_CONNECT_TCP); +} + +static struct security_hook_list landlock_hooks[] __ro_after_init = { + LSM_HOOK_INIT(socket_bind, hook_socket_bind), + LSM_HOOK_INIT(socket_connect, hook_socket_connect), +}; + +__init void landlock_add_net_hooks(void) +{ + security_add_hooks(landlock_hooks, ARRAY_SIZE(landlock_hooks), + LANDLOCK_NAME); +} diff --git a/security/landlock/net.h b/security/landlock/net.h new file mode 100644 index 000000000000..09960c237a13 --- /dev/null +++ b/security/landlock/net.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Landlock LSM - Network management and hooks + * + * Copyright © 2022-2023 Huawei Tech. Co., Ltd. + */ + +#ifndef _SECURITY_LANDLOCK_NET_H +#define _SECURITY_LANDLOCK_NET_H + +#include "common.h" +#include "ruleset.h" +#include "setup.h" + +#if IS_ENABLED(CONFIG_INET) +__init void landlock_add_net_hooks(void); + +int landlock_append_net_rule(struct landlock_ruleset *const ruleset, + const u16 port, access_mask_t access_rights); +#else /* IS_ENABLED(CONFIG_INET) */ +static inline void landlock_add_net_hooks(void) +{ +} + +static inline int +landlock_append_net_rule(struct landlock_ruleset *const ruleset, const u16 port, + access_mask_t access_rights) +{ + return -EAFNOSUPPORT; +} +#endif /* IS_ENABLED(CONFIG_INET) */ + +#endif /* _SECURITY_LANDLOCK_NET_H */ diff --git a/security/landlock/ruleset.c b/security/landlock/ruleset.c index e36c5c332242..ffedc99f2b68 100644 --- a/security/landlock/ruleset.c +++ b/security/landlock/ruleset.c @@ -36,6 +36,11 @@ static struct landlock_ruleset *create_ruleset(const u32 num_layers) refcount_set(&new_ruleset->usage, 1); mutex_init(&new_ruleset->lock); new_ruleset->root_inode = RB_ROOT; + +#if IS_ENABLED(CONFIG_INET) + new_ruleset->root_net_port = RB_ROOT; +#endif /* IS_ENABLED(CONFIG_INET) */ + new_ruleset->num_layers = num_layers; /* * hierarchy = NULL @@ -46,16 +51,21 @@ static struct landlock_ruleset *create_ruleset(const u32 num_layers) } struct landlock_ruleset * -landlock_create_ruleset(const access_mask_t fs_access_mask) +landlock_create_ruleset(const access_mask_t fs_access_mask, + const access_mask_t net_access_mask) { struct landlock_ruleset *new_ruleset; /* Informs about useless ruleset. */ - if (!fs_access_mask) + if (!fs_access_mask && !net_access_mask) return ERR_PTR(-ENOMSG); new_ruleset = create_ruleset(1); - if (!IS_ERR(new_ruleset)) + if (IS_ERR(new_ruleset)) + return new_ruleset; + if (fs_access_mask) landlock_add_fs_access_mask(new_ruleset, fs_access_mask, 0); + if (net_access_mask) + landlock_add_net_access_mask(new_ruleset, net_access_mask, 0); return new_ruleset; } @@ -74,6 +84,11 @@ static bool is_object_pointer(const enum landlock_key_type key_type) case LANDLOCK_KEY_INODE: return true; +#if IS_ENABLED(CONFIG_INET) + case LANDLOCK_KEY_NET_PORT: + return false; +#endif /* IS_ENABLED(CONFIG_INET) */ + default: WARN_ON_ONCE(1); return false; @@ -126,6 +141,11 @@ static struct rb_root *get_root(struct landlock_ruleset *const ruleset, case LANDLOCK_KEY_INODE: return &ruleset->root_inode; +#if IS_ENABLED(CONFIG_INET) + case LANDLOCK_KEY_NET_PORT: + return &ruleset->root_net_port; +#endif /* IS_ENABLED(CONFIG_INET) */ + default: WARN_ON_ONCE(1); return ERR_PTR(-EINVAL); @@ -154,7 +174,8 @@ static void build_check_ruleset(void) BUILD_BUG_ON(ruleset.num_rules < LANDLOCK_MAX_NUM_RULES); BUILD_BUG_ON(ruleset.num_layers < LANDLOCK_MAX_NUM_LAYERS); BUILD_BUG_ON(access_masks < - (LANDLOCK_MASK_ACCESS_FS << LANDLOCK_SHIFT_ACCESS_FS)); + ((LANDLOCK_MASK_ACCESS_FS << LANDLOCK_SHIFT_ACCESS_FS) | + (LANDLOCK_MASK_ACCESS_NET << LANDLOCK_SHIFT_ACCESS_NET))); } /** @@ -371,6 +392,13 @@ static int merge_ruleset(struct landlock_ruleset *const dst, if (err) goto out_unlock; +#if IS_ENABLED(CONFIG_INET) + /* Merges the @src network port tree. */ + err = merge_tree(dst, src, LANDLOCK_KEY_NET_PORT); + if (err) + goto out_unlock; +#endif /* IS_ENABLED(CONFIG_INET) */ + out_unlock: mutex_unlock(&src->lock); mutex_unlock(&dst->lock); @@ -427,6 +455,13 @@ static int inherit_ruleset(struct landlock_ruleset *const parent, if (err) goto out_unlock; +#if IS_ENABLED(CONFIG_INET) + /* Copies the @parent network port tree. */ + err = inherit_tree(parent, child, LANDLOCK_KEY_NET_PORT); + if (err) + goto out_unlock; +#endif /* IS_ENABLED(CONFIG_INET) */ + if (WARN_ON_ONCE(child->num_layers <= parent->num_layers)) { err = -EINVAL; goto out_unlock; @@ -456,6 +491,13 @@ static void free_ruleset(struct landlock_ruleset *const ruleset) rbtree_postorder_for_each_entry_safe(freeme, next, &ruleset->root_inode, node) free_rule(freeme, LANDLOCK_KEY_INODE); + +#if IS_ENABLED(CONFIG_INET) + rbtree_postorder_for_each_entry_safe(freeme, next, + &ruleset->root_net_port, node) + free_rule(freeme, LANDLOCK_KEY_NET_PORT); +#endif /* IS_ENABLED(CONFIG_INET) */ + put_hierarchy(ruleset->hierarchy); kfree(ruleset); } @@ -636,7 +678,8 @@ get_access_mask_t(const struct landlock_ruleset *const ruleset, * * @domain: The domain that defines the current restrictions. * @access_request: The requested access rights to check. - * @layer_masks: The layer masks to populate. + * @layer_masks: It must contain %LANDLOCK_NUM_ACCESS_FS or + * %LANDLOCK_NUM_ACCESS_NET elements according to @key_type. * @key_type: The key type to switch between access masks of different types. * * Returns: An access mask where each access right bit is set which is handled @@ -657,6 +700,14 @@ landlock_init_layer_masks(const struct landlock_ruleset *const domain, get_access_mask = landlock_get_fs_access_mask; num_access = LANDLOCK_NUM_ACCESS_FS; break; + +#if IS_ENABLED(CONFIG_INET) + case LANDLOCK_KEY_NET_PORT: + get_access_mask = landlock_get_net_access_mask; + num_access = LANDLOCK_NUM_ACCESS_NET; + break; +#endif /* IS_ENABLED(CONFIG_INET) */ + default: WARN_ON_ONCE(1); return 0; diff --git a/security/landlock/ruleset.h b/security/landlock/ruleset.h index 7107ce7da39d..c7f1526784fd 100644 --- a/security/landlock/ruleset.h +++ b/security/landlock/ruleset.h @@ -33,13 +33,16 @@ typedef u16 access_mask_t; /* Makes sure all filesystem access rights can be stored. */ static_assert(BITS_PER_TYPE(access_mask_t) >= LANDLOCK_NUM_ACCESS_FS); +/* Makes sure all network access rights can be stored. */ +static_assert(BITS_PER_TYPE(access_mask_t) >= LANDLOCK_NUM_ACCESS_NET); /* Makes sure for_each_set_bit() and for_each_clear_bit() calls are OK. */ static_assert(sizeof(unsigned long) >= sizeof(access_mask_t)); /* Ruleset access masks. */ -typedef u16 access_masks_t; +typedef u32 access_masks_t; /* Makes sure all ruleset access rights can be stored. */ -static_assert(BITS_PER_TYPE(access_masks_t) >= LANDLOCK_NUM_ACCESS_FS); +static_assert(BITS_PER_TYPE(access_masks_t) >= + LANDLOCK_NUM_ACCESS_FS + LANDLOCK_NUM_ACCESS_NET); typedef u16 layer_mask_t; /* Makes sure all layers can be checked. */ @@ -84,6 +87,11 @@ enum landlock_key_type { * keys. */ LANDLOCK_KEY_INODE = 1, + /** + * @LANDLOCK_KEY_NET_PORT: Type of &landlock_ruleset.root_net_port's + * node keys. + */ + LANDLOCK_KEY_NET_PORT, }; /** @@ -158,6 +166,17 @@ struct landlock_ruleset { * reaches zero. */ struct rb_root root_inode; + +#if IS_ENABLED(CONFIG_INET) + /** + * @root_net_port: Root of a red-black tree containing &struct + * landlock_rule nodes with network port. Once a ruleset is tied to a + * process (i.e. as a domain), this tree is immutable until @usage + * reaches zero. + */ + struct rb_root root_net_port; +#endif /* IS_ENABLED(CONFIG_INET) */ + /** * @hierarchy: Enables hierarchy identification even when a parent * domain vanishes. This is needed for the ptrace protection. @@ -196,13 +215,13 @@ struct landlock_ruleset { */ u32 num_layers; /** - * @access_masks: Contains the subset of filesystem - * actions that are restricted by a ruleset. A domain - * saves all layers of merged rulesets in a stack - * (FAM), starting from the first layer to the last - * one. These layers are used when merging rulesets, - * for user space backward compatibility (i.e. - * future-proof), and to properly handle merged + * @access_masks: Contains the subset of filesystem and + * network actions that are restricted by a ruleset. + * A domain saves all layers of merged rulesets in a + * stack (FAM), starting from the first layer to the + * last one. These layers are used when merging + * rulesets, for user space backward compatibility + * (i.e. future-proof), and to properly handle merged * rulesets without overlapping access rights. These * layers are set once and never changed for the * lifetime of the ruleset. @@ -213,7 +232,8 @@ struct landlock_ruleset { }; struct landlock_ruleset * -landlock_create_ruleset(const access_mask_t access_mask); +landlock_create_ruleset(const access_mask_t access_mask_fs, + const access_mask_t access_mask_net); void landlock_put_ruleset(struct landlock_ruleset *const ruleset); void landlock_put_ruleset_deferred(struct landlock_ruleset *const ruleset); @@ -249,6 +269,19 @@ landlock_add_fs_access_mask(struct landlock_ruleset *const ruleset, (fs_mask << LANDLOCK_SHIFT_ACCESS_FS); } +static inline void +landlock_add_net_access_mask(struct landlock_ruleset *const ruleset, + const access_mask_t net_access_mask, + const u16 layer_level) +{ + access_mask_t net_mask = net_access_mask & LANDLOCK_MASK_ACCESS_NET; + + /* Should already be checked in sys_landlock_create_ruleset(). */ + WARN_ON_ONCE(net_access_mask != net_mask); + ruleset->access_masks[layer_level] |= + (net_mask << LANDLOCK_SHIFT_ACCESS_NET); +} + static inline access_mask_t landlock_get_raw_fs_access_mask(const struct landlock_ruleset *const ruleset, const u16 layer_level) @@ -267,6 +300,15 @@ landlock_get_fs_access_mask(const struct landlock_ruleset *const ruleset, LANDLOCK_ACCESS_FS_INITIALLY_DENIED; } +static inline access_mask_t +landlock_get_net_access_mask(const struct landlock_ruleset *const ruleset, + const u16 layer_level) +{ + return (ruleset->access_masks[layer_level] >> + LANDLOCK_SHIFT_ACCESS_NET) & + LANDLOCK_MASK_ACCESS_NET; +} + bool landlock_unmask_layers(const struct landlock_rule *const rule, const access_mask_t access_request, layer_mask_t (*const layer_masks)[], diff --git a/security/landlock/setup.c b/security/landlock/setup.c index 0f6113528fa4..3e11d303542f 100644 --- a/security/landlock/setup.c +++ b/security/landlock/setup.c @@ -12,6 +12,7 @@ #include "common.h" #include "cred.h" #include "fs.h" +#include "net.h" #include "ptrace.h" #include "setup.h" @@ -29,6 +30,7 @@ static int __init landlock_init(void) landlock_add_cred_hooks(); landlock_add_ptrace_hooks(); landlock_add_fs_hooks(); + landlock_add_net_hooks(); landlock_initialized = true; pr_info("Up and running.\n"); return 0; diff --git a/security/landlock/syscalls.c b/security/landlock/syscalls.c index e87f572e0251..898358f57fa0 100644 --- a/security/landlock/syscalls.c +++ b/security/landlock/syscalls.c @@ -29,6 +29,7 @@ #include "cred.h" #include "fs.h" #include "limits.h" +#include "net.h" #include "ruleset.h" #include "setup.h" @@ -74,7 +75,8 @@ static void build_check_abi(void) { struct landlock_ruleset_attr ruleset_attr; struct landlock_path_beneath_attr path_beneath_attr; - size_t ruleset_size, path_beneath_size; + struct landlock_net_port_attr net_port_attr; + size_t ruleset_size, path_beneath_size, net_port_size; /* * For each user space ABI structures, first checks that there is no @@ -82,13 +84,19 @@ static void build_check_abi(void) * struct size. */ ruleset_size = sizeof(ruleset_attr.handled_access_fs); + ruleset_size += sizeof(ruleset_attr.handled_access_net); BUILD_BUG_ON(sizeof(ruleset_attr) != ruleset_size); - BUILD_BUG_ON(sizeof(ruleset_attr) != 8); + BUILD_BUG_ON(sizeof(ruleset_attr) != 16); path_beneath_size = sizeof(path_beneath_attr.allowed_access); path_beneath_size += sizeof(path_beneath_attr.parent_fd); BUILD_BUG_ON(sizeof(path_beneath_attr) != path_beneath_size); BUILD_BUG_ON(sizeof(path_beneath_attr) != 12); + + net_port_size = sizeof(net_port_attr.allowed_access); + net_port_size += sizeof(net_port_attr.port); + BUILD_BUG_ON(sizeof(net_port_attr) != net_port_size); + BUILD_BUG_ON(sizeof(net_port_attr) != 16); } /* Ruleset handling */ @@ -129,7 +137,7 @@ static const struct file_operations ruleset_fops = { .write = fop_dummy_write, }; -#define LANDLOCK_ABI_VERSION 3 +#define LANDLOCK_ABI_VERSION 4 /** * sys_landlock_create_ruleset - Create a new ruleset @@ -188,8 +196,14 @@ SYSCALL_DEFINE3(landlock_create_ruleset, LANDLOCK_MASK_ACCESS_FS) return -EINVAL; + /* Checks network content (and 32-bits cast). */ + if ((ruleset_attr.handled_access_net | LANDLOCK_MASK_ACCESS_NET) != + LANDLOCK_MASK_ACCESS_NET) + return -EINVAL; + /* Checks arguments and transforms to kernel struct. */ - ruleset = landlock_create_ruleset(ruleset_attr.handled_access_fs); + ruleset = landlock_create_ruleset(ruleset_attr.handled_access_fs, + ruleset_attr.handled_access_net); if (IS_ERR(ruleset)) return PTR_ERR(ruleset); @@ -282,7 +296,7 @@ static int add_rule_path_beneath(struct landlock_ruleset *const ruleset, int res, err; access_mask_t mask; - /* Copies raw user space buffer, only one type for now. */ + /* Copies raw user space buffer. */ res = copy_from_user(&path_beneath_attr, rule_attr, sizeof(path_beneath_attr)); if (res) @@ -312,13 +326,46 @@ static int add_rule_path_beneath(struct landlock_ruleset *const ruleset, return err; } +static int add_rule_net_port(struct landlock_ruleset *ruleset, + const void __user *const rule_attr) +{ + struct landlock_net_port_attr net_port_attr; + int res; + access_mask_t mask; + + /* Copies raw user space buffer. */ + res = copy_from_user(&net_port_attr, rule_attr, sizeof(net_port_attr)); + if (res) + return -EFAULT; + + /* + * Informs about useless rule: empty allowed_access (i.e. deny rules) + * are ignored by network actions. + */ + if (!net_port_attr.allowed_access) + return -ENOMSG; + + /* Checks that allowed_access matches the @ruleset constraints. */ + mask = landlock_get_net_access_mask(ruleset, 0); + if ((net_port_attr.allowed_access | mask) != mask) + return -EINVAL; + + /* Denies inserting a rule with port greater than 65535. */ + if (net_port_attr.port > U16_MAX) + return -EINVAL; + + /* Imports the new rule. */ + return landlock_append_net_rule(ruleset, net_port_attr.port, + net_port_attr.allowed_access); +} + /** * sys_landlock_add_rule - Add a new rule to a ruleset * * @ruleset_fd: File descriptor tied to the ruleset that should be extended * with the new rule. - * @rule_type: Identify the structure type pointed to by @rule_attr (only - * %LANDLOCK_RULE_PATH_BENEATH for now). + * @rule_type: Identify the structure type pointed to by @rule_attr: + * %LANDLOCK_RULE_PATH_BENEATH or %LANDLOCK_RULE_NET_PORT. * @rule_attr: Pointer to a rule (only of type &struct * landlock_path_beneath_attr for now). * @flags: Must be 0. @@ -329,9 +376,13 @@ static int add_rule_path_beneath(struct landlock_ruleset *const ruleset, * Possible returned errors are: * * - %EOPNOTSUPP: Landlock is supported by the kernel but disabled at boot time; + * - %EAFNOSUPPORT: @rule_type is %LANDLOCK_RULE_NET_PORT but TCP/IP is not + * supported by the running kernel; * - %EINVAL: @flags is not 0, or inconsistent access in the rule (i.e. - * &landlock_path_beneath_attr.allowed_access is not a subset of the - * ruleset handled accesses); + * &landlock_path_beneath_attr.allowed_access or + * &landlock_net_port_attr.allowed_access is not a subset of the + * ruleset handled accesses), or &landlock_net_port_attr.port is + * greater than 65535; * - %ENOMSG: Empty accesses (e.g. &landlock_path_beneath_attr.allowed_access); * - %EBADF: @ruleset_fd is not a file descriptor for the current thread, or a * member of @rule_attr is not a file descriptor as expected; @@ -363,6 +414,9 @@ SYSCALL_DEFINE4(landlock_add_rule, const int, ruleset_fd, case LANDLOCK_RULE_PATH_BENEATH: err = add_rule_path_beneath(ruleset, rule_attr); break; + case LANDLOCK_RULE_NET_PORT: + err = add_rule_net_port(ruleset, rule_attr); + break; default: err = -EINVAL; break; diff --git a/tools/testing/selftests/landlock/base_test.c b/tools/testing/selftests/landlock/base_test.c index 792c3f0a59b4..646f778dfb1e 100644 --- a/tools/testing/selftests/landlock/base_test.c +++ b/tools/testing/selftests/landlock/base_test.c @@ -75,7 +75,7 @@ TEST(abi_version) const struct landlock_ruleset_attr ruleset_attr = { .handled_access_fs = LANDLOCK_ACCESS_FS_READ_FILE, }; - ASSERT_EQ(3, landlock_create_ruleset(NULL, 0, + ASSERT_EQ(4, landlock_create_ruleset(NULL, 0, LANDLOCK_CREATE_RULESET_VERSION)); ASSERT_EQ(-1, landlock_create_ruleset(&ruleset_attr, 0, From 1fa335209f6ad9d554655bb802c5b49b855d6237 Mon Sep 17 00:00:00 2001 From: Konstantin Meskhidze Date: Thu, 26 Oct 2023 09:47:48 +0800 Subject: [PATCH 440/475] selftests/landlock: Share enforce_ruleset() helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move enforce_ruleset() helper function to common.h so that it can be used both by filesystem tests and network ones. Signed-off-by: Konstantin Meskhidze Link: https://lore.kernel.org/r/20231026014751.414649-10-konstantin.meskhidze@huawei.com Signed-off-by: Mickaël Salaün --- tools/testing/selftests/landlock/common.h | 10 ++++++++++ tools/testing/selftests/landlock/fs_test.c | 10 ---------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tools/testing/selftests/landlock/common.h b/tools/testing/selftests/landlock/common.h index d7987ae8d7fc..0fd6c4cf5e6f 100644 --- a/tools/testing/selftests/landlock/common.h +++ b/tools/testing/selftests/landlock/common.h @@ -256,3 +256,13 @@ static int __maybe_unused send_fd(int usock, int fd_tx) return -errno; return 0; } + +static void __maybe_unused +enforce_ruleset(struct __test_metadata *const _metadata, const int ruleset_fd) +{ + ASSERT_EQ(0, prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)); + ASSERT_EQ(0, landlock_restrict_self(ruleset_fd, 0)) + { + TH_LOG("Failed to enforce ruleset: %s", strerror(errno)); + } +} diff --git a/tools/testing/selftests/landlock/fs_test.c b/tools/testing/selftests/landlock/fs_test.c index 251594306d40..68b7a89cf65b 100644 --- a/tools/testing/selftests/landlock/fs_test.c +++ b/tools/testing/selftests/landlock/fs_test.c @@ -677,16 +677,6 @@ static int create_ruleset(struct __test_metadata *const _metadata, return ruleset_fd; } -static void enforce_ruleset(struct __test_metadata *const _metadata, - const int ruleset_fd) -{ - ASSERT_EQ(0, prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)); - ASSERT_EQ(0, landlock_restrict_self(ruleset_fd, 0)) - { - TH_LOG("Failed to enforce ruleset: %s", strerror(errno)); - } -} - TEST_F_FORK(layout0, proc_nsfs) { const struct rule rules[] = { From a549d055a22e4fc5559ef642b30721899fa07b75 Mon Sep 17 00:00:00 2001 From: Konstantin Meskhidze Date: Thu, 26 Oct 2023 09:47:49 +0800 Subject: [PATCH 441/475] selftests/landlock: Add network tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add 82 test suites to check edge cases related to bind() and connect() actions. They are defined with 6 fixtures and their variants: The "protocol" fixture is extended with 12 variants defined as a matrix of: sandboxed/not-sandboxed, IPv4/IPv6/unix network domain, and stream/datagram socket. 4 related tests suites are defined: * bind: Tests bind action. * connect: Tests connect action. * bind_unspec: Tests bind action with the AF_UNSPEC socket family. * connect_unspec: Tests connect action with the AF_UNSPEC socket family. The "ipv4" fixture is extended with 4 variants defined as a matrix of: sandboxed/not-sandboxed, and stream/datagram socket. 1 related test suite is defined: * from_unix_to_inet: Tests to make sure unix sockets' actions are not restricted by Landlock rules applied to TCP ones. The "tcp_layers" fixture is extended with 8 variants defined as a matrix of: IPv4/IPv6 network domain, and different number of landlock rule layers. 2 related tests suites are defined: * ruleset_overlap: Tests nested layers with less constraints. * ruleset_expand: Tests nested layers with more constraints. In the "mini" fixture 4 tests suites are defined: * network_access_rights: Tests handling of known access rights. * unknown_access_rights: Tests handling of unknown access rights. * inval: Tests unhandled allowed access and zero access value. * tcp_port_overflow: Tests with port values greater than 65535. The "ipv4_tcp" fixture supports IPv4 network domain with stream socket. 2 tests suites are defined: * port_endianness: Tests with big/little endian port formats. * with_fs: Tests a ruleset with both filesystem and network restrictions. The "port_specific" fixture is extended with 4 variants defined as a matrix of: sandboxed/not-sandboxed, IPv4/IPv6 network domain, and stream socket. 2 related tests suites are defined: * bind_connect_zero: Tests with port 0. * bind_connect_1023: Tests with port 1023. Test coverage for security/landlock is 92.4% of 710 lines according to gcc/gcov-13. Signed-off-by: Konstantin Meskhidze Link: https://lore.kernel.org/r/20231026014751.414649-11-konstantin.meskhidze@huawei.com [mic: Extend commit message, update test coverage, clean up capability use, fix useless TEST_F_FORK, and improve ipv4_tcp.with_fs] Co-developed-by: Mickaël Salaün Signed-off-by: Mickaël Salaün --- tools/testing/selftests/landlock/common.h | 3 + tools/testing/selftests/landlock/config | 4 + tools/testing/selftests/landlock/net_test.c | 1738 +++++++++++++++++++ 3 files changed, 1745 insertions(+) create mode 100644 tools/testing/selftests/landlock/net_test.c diff --git a/tools/testing/selftests/landlock/common.h b/tools/testing/selftests/landlock/common.h index 0fd6c4cf5e6f..5b79758cae62 100644 --- a/tools/testing/selftests/landlock/common.h +++ b/tools/testing/selftests/landlock/common.h @@ -112,10 +112,13 @@ static void _init_caps(struct __test_metadata *const _metadata, bool drop_all) cap_t cap_p; /* Only these three capabilities are useful for the tests. */ const cap_value_t caps[] = { + /* clang-format off */ CAP_DAC_OVERRIDE, CAP_MKNOD, CAP_SYS_ADMIN, CAP_SYS_CHROOT, + CAP_NET_BIND_SERVICE, + /* clang-format on */ }; cap_p = cap_get_proc(); diff --git a/tools/testing/selftests/landlock/config b/tools/testing/selftests/landlock/config index 3dc9e438eab1..0086efaa7b68 100644 --- a/tools/testing/selftests/landlock/config +++ b/tools/testing/selftests/landlock/config @@ -1,5 +1,9 @@ CONFIG_CGROUPS=y CONFIG_CGROUP_SCHED=y +CONFIG_INET=y +CONFIG_IPV6=y +CONFIG_NET=y +CONFIG_NET_NS=y CONFIG_OVERLAY_FS=y CONFIG_PROC_FS=y CONFIG_SECURITY=y diff --git a/tools/testing/selftests/landlock/net_test.c b/tools/testing/selftests/landlock/net_test.c new file mode 100644 index 000000000000..929e21c4db05 --- /dev/null +++ b/tools/testing/selftests/landlock/net_test.c @@ -0,0 +1,1738 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Landlock tests - Network + * + * Copyright © 2022-2023 Huawei Tech. Co., Ltd. + * Copyright © 2023 Microsoft Corporation + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common.h" + +const short sock_port_start = (1 << 10); + +static const char loopback_ipv4[] = "127.0.0.1"; +static const char loopback_ipv6[] = "::1"; + +/* Number pending connections queue to be hold. */ +const short backlog = 10; + +enum sandbox_type { + NO_SANDBOX, + /* This may be used to test rules that allow *and* deny accesses. */ + TCP_SANDBOX, +}; + +struct protocol_variant { + int domain; + int type; +}; + +struct service_fixture { + struct protocol_variant protocol; + /* port is also stored in ipv4_addr.sin_port or ipv6_addr.sin6_port */ + unsigned short port; + union { + struct sockaddr_in ipv4_addr; + struct sockaddr_in6 ipv6_addr; + struct { + struct sockaddr_un unix_addr; + socklen_t unix_addr_len; + }; + }; +}; + +static int set_service(struct service_fixture *const srv, + const struct protocol_variant prot, + const unsigned short index) +{ + memset(srv, 0, sizeof(*srv)); + + /* + * Copies all protocol properties in case of the variant only contains + * a subset of them. + */ + srv->protocol = prot; + + /* Checks for port overflow. */ + if (index > 2) + return 1; + srv->port = sock_port_start << (2 * index); + + switch (prot.domain) { + case AF_UNSPEC: + case AF_INET: + srv->ipv4_addr.sin_family = prot.domain; + srv->ipv4_addr.sin_port = htons(srv->port); + srv->ipv4_addr.sin_addr.s_addr = inet_addr(loopback_ipv4); + return 0; + + case AF_INET6: + srv->ipv6_addr.sin6_family = prot.domain; + srv->ipv6_addr.sin6_port = htons(srv->port); + inet_pton(AF_INET6, loopback_ipv6, &srv->ipv6_addr.sin6_addr); + return 0; + + case AF_UNIX: + srv->unix_addr.sun_family = prot.domain; + sprintf(srv->unix_addr.sun_path, + "_selftests-landlock-net-tid%d-index%d", gettid(), + index); + srv->unix_addr_len = SUN_LEN(&srv->unix_addr); + srv->unix_addr.sun_path[0] = '\0'; + return 0; + } + return 1; +} + +static void setup_loopback(struct __test_metadata *const _metadata) +{ + set_cap(_metadata, CAP_SYS_ADMIN); + ASSERT_EQ(0, unshare(CLONE_NEWNET)); + ASSERT_EQ(0, system("ip link set dev lo up")); + clear_cap(_metadata, CAP_SYS_ADMIN); +} + +static bool is_restricted(const struct protocol_variant *const prot, + const enum sandbox_type sandbox) +{ + switch (prot->domain) { + case AF_INET: + case AF_INET6: + switch (prot->type) { + case SOCK_STREAM: + return sandbox == TCP_SANDBOX; + } + break; + } + return false; +} + +static int socket_variant(const struct service_fixture *const srv) +{ + int ret; + + ret = socket(srv->protocol.domain, srv->protocol.type | SOCK_CLOEXEC, + 0); + if (ret < 0) + return -errno; + return ret; +} + +#ifndef SIN6_LEN_RFC2133 +#define SIN6_LEN_RFC2133 24 +#endif + +static socklen_t get_addrlen(const struct service_fixture *const srv, + const bool minimal) +{ + switch (srv->protocol.domain) { + case AF_UNSPEC: + case AF_INET: + return sizeof(srv->ipv4_addr); + + case AF_INET6: + if (minimal) + return SIN6_LEN_RFC2133; + return sizeof(srv->ipv6_addr); + + case AF_UNIX: + if (minimal) + return sizeof(srv->unix_addr) - + sizeof(srv->unix_addr.sun_path); + return srv->unix_addr_len; + + default: + return 0; + } +} + +static void set_port(struct service_fixture *const srv, uint16_t port) +{ + switch (srv->protocol.domain) { + case AF_UNSPEC: + case AF_INET: + srv->ipv4_addr.sin_port = htons(port); + return; + + case AF_INET6: + srv->ipv6_addr.sin6_port = htons(port); + return; + + default: + return; + } +} + +static uint16_t get_binded_port(int socket_fd, + const struct protocol_variant *const prot) +{ + struct sockaddr_in ipv4_addr; + struct sockaddr_in6 ipv6_addr; + socklen_t ipv4_addr_len, ipv6_addr_len; + + /* Gets binded port. */ + switch (prot->domain) { + case AF_UNSPEC: + case AF_INET: + ipv4_addr_len = sizeof(ipv4_addr); + getsockname(socket_fd, &ipv4_addr, &ipv4_addr_len); + return ntohs(ipv4_addr.sin_port); + + case AF_INET6: + ipv6_addr_len = sizeof(ipv6_addr); + getsockname(socket_fd, &ipv6_addr, &ipv6_addr_len); + return ntohs(ipv6_addr.sin6_port); + + default: + return 0; + } +} + +static int bind_variant_addrlen(const int sock_fd, + const struct service_fixture *const srv, + const socklen_t addrlen) +{ + int ret; + + switch (srv->protocol.domain) { + case AF_UNSPEC: + case AF_INET: + ret = bind(sock_fd, &srv->ipv4_addr, addrlen); + break; + + case AF_INET6: + ret = bind(sock_fd, &srv->ipv6_addr, addrlen); + break; + + case AF_UNIX: + ret = bind(sock_fd, &srv->unix_addr, addrlen); + break; + + default: + errno = EAFNOSUPPORT; + return -errno; + } + + if (ret < 0) + return -errno; + return ret; +} + +static int bind_variant(const int sock_fd, + const struct service_fixture *const srv) +{ + return bind_variant_addrlen(sock_fd, srv, get_addrlen(srv, false)); +} + +static int connect_variant_addrlen(const int sock_fd, + const struct service_fixture *const srv, + const socklen_t addrlen) +{ + int ret; + + switch (srv->protocol.domain) { + case AF_UNSPEC: + case AF_INET: + ret = connect(sock_fd, &srv->ipv4_addr, addrlen); + break; + + case AF_INET6: + ret = connect(sock_fd, &srv->ipv6_addr, addrlen); + break; + + case AF_UNIX: + ret = connect(sock_fd, &srv->unix_addr, addrlen); + break; + + default: + errno = -EAFNOSUPPORT; + return -errno; + } + + if (ret < 0) + return -errno; + return ret; +} + +static int connect_variant(const int sock_fd, + const struct service_fixture *const srv) +{ + return connect_variant_addrlen(sock_fd, srv, get_addrlen(srv, false)); +} + +FIXTURE(protocol) +{ + struct service_fixture srv0, srv1, srv2, unspec_any0, unspec_srv0; +}; + +FIXTURE_VARIANT(protocol) +{ + const enum sandbox_type sandbox; + const struct protocol_variant prot; +}; + +FIXTURE_SETUP(protocol) +{ + const struct protocol_variant prot_unspec = { + .domain = AF_UNSPEC, + .type = SOCK_STREAM, + }; + + disable_caps(_metadata); + + ASSERT_EQ(0, set_service(&self->srv0, variant->prot, 0)); + ASSERT_EQ(0, set_service(&self->srv1, variant->prot, 1)); + ASSERT_EQ(0, set_service(&self->srv2, variant->prot, 2)); + + ASSERT_EQ(0, set_service(&self->unspec_srv0, prot_unspec, 0)); + + ASSERT_EQ(0, set_service(&self->unspec_any0, prot_unspec, 0)); + self->unspec_any0.ipv4_addr.sin_addr.s_addr = htonl(INADDR_ANY); + + setup_loopback(_metadata); +}; + +FIXTURE_TEARDOWN(protocol) +{ +} + +/* clang-format off */ +FIXTURE_VARIANT_ADD(protocol, no_sandbox_with_ipv4_tcp) { + /* clang-format on */ + .sandbox = NO_SANDBOX, + .prot = { + .domain = AF_INET, + .type = SOCK_STREAM, + }, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(protocol, no_sandbox_with_ipv6_tcp) { + /* clang-format on */ + .sandbox = NO_SANDBOX, + .prot = { + .domain = AF_INET6, + .type = SOCK_STREAM, + }, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(protocol, no_sandbox_with_ipv4_udp) { + /* clang-format on */ + .sandbox = NO_SANDBOX, + .prot = { + .domain = AF_INET, + .type = SOCK_DGRAM, + }, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(protocol, no_sandbox_with_ipv6_udp) { + /* clang-format on */ + .sandbox = NO_SANDBOX, + .prot = { + .domain = AF_INET6, + .type = SOCK_DGRAM, + }, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(protocol, no_sandbox_with_unix_stream) { + /* clang-format on */ + .sandbox = NO_SANDBOX, + .prot = { + .domain = AF_UNIX, + .type = SOCK_STREAM, + }, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(protocol, no_sandbox_with_unix_datagram) { + /* clang-format on */ + .sandbox = NO_SANDBOX, + .prot = { + .domain = AF_UNIX, + .type = SOCK_DGRAM, + }, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(protocol, tcp_sandbox_with_ipv4_tcp) { + /* clang-format on */ + .sandbox = TCP_SANDBOX, + .prot = { + .domain = AF_INET, + .type = SOCK_STREAM, + }, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(protocol, tcp_sandbox_with_ipv6_tcp) { + /* clang-format on */ + .sandbox = TCP_SANDBOX, + .prot = { + .domain = AF_INET6, + .type = SOCK_STREAM, + }, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(protocol, tcp_sandbox_with_ipv4_udp) { + /* clang-format on */ + .sandbox = TCP_SANDBOX, + .prot = { + .domain = AF_INET, + .type = SOCK_DGRAM, + }, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(protocol, tcp_sandbox_with_ipv6_udp) { + /* clang-format on */ + .sandbox = TCP_SANDBOX, + .prot = { + .domain = AF_INET6, + .type = SOCK_DGRAM, + }, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(protocol, tcp_sandbox_with_unix_stream) { + /* clang-format on */ + .sandbox = TCP_SANDBOX, + .prot = { + .domain = AF_UNIX, + .type = SOCK_STREAM, + }, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(protocol, tcp_sandbox_with_unix_datagram) { + /* clang-format on */ + .sandbox = TCP_SANDBOX, + .prot = { + .domain = AF_UNIX, + .type = SOCK_DGRAM, + }, +}; + +static void test_bind_and_connect(struct __test_metadata *const _metadata, + const struct service_fixture *const srv, + const bool deny_bind, const bool deny_connect) +{ + char buf = '\0'; + int inval_fd, bind_fd, client_fd, status, ret; + pid_t child; + + /* Starts invalid addrlen tests with bind. */ + inval_fd = socket_variant(srv); + ASSERT_LE(0, inval_fd) + { + TH_LOG("Failed to create socket: %s", strerror(errno)); + } + + /* Tries to bind with zero as addrlen. */ + EXPECT_EQ(-EINVAL, bind_variant_addrlen(inval_fd, srv, 0)); + + /* Tries to bind with too small addrlen. */ + EXPECT_EQ(-EINVAL, bind_variant_addrlen(inval_fd, srv, + get_addrlen(srv, true) - 1)); + + /* Tries to bind with minimal addrlen. */ + ret = bind_variant_addrlen(inval_fd, srv, get_addrlen(srv, true)); + if (deny_bind) { + EXPECT_EQ(-EACCES, ret); + } else { + EXPECT_EQ(0, ret) + { + TH_LOG("Failed to bind to socket: %s", strerror(errno)); + } + } + EXPECT_EQ(0, close(inval_fd)); + + /* Starts invalid addrlen tests with connect. */ + inval_fd = socket_variant(srv); + ASSERT_LE(0, inval_fd); + + /* Tries to connect with zero as addrlen. */ + EXPECT_EQ(-EINVAL, connect_variant_addrlen(inval_fd, srv, 0)); + + /* Tries to connect with too small addrlen. */ + EXPECT_EQ(-EINVAL, connect_variant_addrlen(inval_fd, srv, + get_addrlen(srv, true) - 1)); + + /* Tries to connect with minimal addrlen. */ + ret = connect_variant_addrlen(inval_fd, srv, get_addrlen(srv, true)); + if (srv->protocol.domain == AF_UNIX) { + EXPECT_EQ(-EINVAL, ret); + } else if (deny_connect) { + EXPECT_EQ(-EACCES, ret); + } else if (srv->protocol.type == SOCK_STREAM) { + /* No listening server, whatever the value of deny_bind. */ + EXPECT_EQ(-ECONNREFUSED, ret); + } else { + EXPECT_EQ(0, ret) + { + TH_LOG("Failed to connect to socket: %s", + strerror(errno)); + } + } + EXPECT_EQ(0, close(inval_fd)); + + /* Starts connection tests. */ + bind_fd = socket_variant(srv); + ASSERT_LE(0, bind_fd); + + ret = bind_variant(bind_fd, srv); + if (deny_bind) { + EXPECT_EQ(-EACCES, ret); + } else { + EXPECT_EQ(0, ret); + + /* Creates a listening socket. */ + if (srv->protocol.type == SOCK_STREAM) + EXPECT_EQ(0, listen(bind_fd, backlog)); + } + + child = fork(); + ASSERT_LE(0, child); + if (child == 0) { + int connect_fd, ret; + + /* Closes listening socket for the child. */ + EXPECT_EQ(0, close(bind_fd)); + + /* Starts connection tests. */ + connect_fd = socket_variant(srv); + ASSERT_LE(0, connect_fd); + ret = connect_variant(connect_fd, srv); + if (deny_connect) { + EXPECT_EQ(-EACCES, ret); + } else if (deny_bind) { + /* No listening server. */ + EXPECT_EQ(-ECONNREFUSED, ret); + } else { + EXPECT_EQ(0, ret); + EXPECT_EQ(1, write(connect_fd, ".", 1)); + } + + EXPECT_EQ(0, close(connect_fd)); + _exit(_metadata->passed ? EXIT_SUCCESS : EXIT_FAILURE); + return; + } + + /* Accepts connection from the child. */ + client_fd = bind_fd; + if (!deny_bind && !deny_connect) { + if (srv->protocol.type == SOCK_STREAM) { + client_fd = accept(bind_fd, NULL, 0); + ASSERT_LE(0, client_fd); + } + + EXPECT_EQ(1, read(client_fd, &buf, 1)); + EXPECT_EQ('.', buf); + } + + EXPECT_EQ(child, waitpid(child, &status, 0)); + EXPECT_EQ(1, WIFEXITED(status)); + EXPECT_EQ(EXIT_SUCCESS, WEXITSTATUS(status)); + + /* Closes connection, if any. */ + if (client_fd != bind_fd) + EXPECT_LE(0, close(client_fd)); + + /* Closes listening socket. */ + EXPECT_EQ(0, close(bind_fd)); +} + +TEST_F(protocol, bind) +{ + if (variant->sandbox == TCP_SANDBOX) { + const struct landlock_ruleset_attr ruleset_attr = { + .handled_access_net = LANDLOCK_ACCESS_NET_BIND_TCP | + LANDLOCK_ACCESS_NET_CONNECT_TCP, + }; + const struct landlock_net_port_attr tcp_bind_connect_p0 = { + .allowed_access = LANDLOCK_ACCESS_NET_BIND_TCP | + LANDLOCK_ACCESS_NET_CONNECT_TCP, + .port = self->srv0.port, + }; + const struct landlock_net_port_attr tcp_connect_p1 = { + .allowed_access = LANDLOCK_ACCESS_NET_CONNECT_TCP, + .port = self->srv1.port, + }; + int ruleset_fd; + + ruleset_fd = landlock_create_ruleset(&ruleset_attr, + sizeof(ruleset_attr), 0); + ASSERT_LE(0, ruleset_fd); + + /* Allows connect and bind for the first port. */ + ASSERT_EQ(0, + landlock_add_rule(ruleset_fd, LANDLOCK_RULE_NET_PORT, + &tcp_bind_connect_p0, 0)); + + /* Allows connect and denies bind for the second port. */ + ASSERT_EQ(0, + landlock_add_rule(ruleset_fd, LANDLOCK_RULE_NET_PORT, + &tcp_connect_p1, 0)); + + enforce_ruleset(_metadata, ruleset_fd); + EXPECT_EQ(0, close(ruleset_fd)); + } + + /* Binds a socket to the first port. */ + test_bind_and_connect(_metadata, &self->srv0, false, false); + + /* Binds a socket to the second port. */ + test_bind_and_connect(_metadata, &self->srv1, + is_restricted(&variant->prot, variant->sandbox), + false); + + /* Binds a socket to the third port. */ + test_bind_and_connect(_metadata, &self->srv2, + is_restricted(&variant->prot, variant->sandbox), + is_restricted(&variant->prot, variant->sandbox)); +} + +TEST_F(protocol, connect) +{ + if (variant->sandbox == TCP_SANDBOX) { + const struct landlock_ruleset_attr ruleset_attr = { + .handled_access_net = LANDLOCK_ACCESS_NET_BIND_TCP | + LANDLOCK_ACCESS_NET_CONNECT_TCP, + }; + const struct landlock_net_port_attr tcp_bind_connect_p0 = { + .allowed_access = LANDLOCK_ACCESS_NET_BIND_TCP | + LANDLOCK_ACCESS_NET_CONNECT_TCP, + .port = self->srv0.port, + }; + const struct landlock_net_port_attr tcp_bind_p1 = { + .allowed_access = LANDLOCK_ACCESS_NET_BIND_TCP, + .port = self->srv1.port, + }; + int ruleset_fd; + + ruleset_fd = landlock_create_ruleset(&ruleset_attr, + sizeof(ruleset_attr), 0); + ASSERT_LE(0, ruleset_fd); + + /* Allows connect and bind for the first port. */ + ASSERT_EQ(0, + landlock_add_rule(ruleset_fd, LANDLOCK_RULE_NET_PORT, + &tcp_bind_connect_p0, 0)); + + /* Allows bind and denies connect for the second port. */ + ASSERT_EQ(0, + landlock_add_rule(ruleset_fd, LANDLOCK_RULE_NET_PORT, + &tcp_bind_p1, 0)); + + enforce_ruleset(_metadata, ruleset_fd); + EXPECT_EQ(0, close(ruleset_fd)); + } + + test_bind_and_connect(_metadata, &self->srv0, false, false); + + test_bind_and_connect(_metadata, &self->srv1, false, + is_restricted(&variant->prot, variant->sandbox)); + + test_bind_and_connect(_metadata, &self->srv2, + is_restricted(&variant->prot, variant->sandbox), + is_restricted(&variant->prot, variant->sandbox)); +} + +TEST_F(protocol, bind_unspec) +{ + const struct landlock_ruleset_attr ruleset_attr = { + .handled_access_net = LANDLOCK_ACCESS_NET_BIND_TCP, + }; + const struct landlock_net_port_attr tcp_bind = { + .allowed_access = LANDLOCK_ACCESS_NET_BIND_TCP, + .port = self->srv0.port, + }; + int bind_fd, ret; + + if (variant->sandbox == TCP_SANDBOX) { + const int ruleset_fd = landlock_create_ruleset( + &ruleset_attr, sizeof(ruleset_attr), 0); + ASSERT_LE(0, ruleset_fd); + + /* Allows bind. */ + ASSERT_EQ(0, + landlock_add_rule(ruleset_fd, LANDLOCK_RULE_NET_PORT, + &tcp_bind, 0)); + enforce_ruleset(_metadata, ruleset_fd); + EXPECT_EQ(0, close(ruleset_fd)); + } + + bind_fd = socket_variant(&self->srv0); + ASSERT_LE(0, bind_fd); + + /* Allowed bind on AF_UNSPEC/INADDR_ANY. */ + ret = bind_variant(bind_fd, &self->unspec_any0); + if (variant->prot.domain == AF_INET) { + EXPECT_EQ(0, ret) + { + TH_LOG("Failed to bind to unspec/any socket: %s", + strerror(errno)); + } + } else { + EXPECT_EQ(-EINVAL, ret); + } + EXPECT_EQ(0, close(bind_fd)); + + if (variant->sandbox == TCP_SANDBOX) { + const int ruleset_fd = landlock_create_ruleset( + &ruleset_attr, sizeof(ruleset_attr), 0); + ASSERT_LE(0, ruleset_fd); + + /* Denies bind. */ + enforce_ruleset(_metadata, ruleset_fd); + EXPECT_EQ(0, close(ruleset_fd)); + } + + bind_fd = socket_variant(&self->srv0); + ASSERT_LE(0, bind_fd); + + /* Denied bind on AF_UNSPEC/INADDR_ANY. */ + ret = bind_variant(bind_fd, &self->unspec_any0); + if (variant->prot.domain == AF_INET) { + if (is_restricted(&variant->prot, variant->sandbox)) { + EXPECT_EQ(-EACCES, ret); + } else { + EXPECT_EQ(0, ret); + } + } else { + EXPECT_EQ(-EINVAL, ret); + } + EXPECT_EQ(0, close(bind_fd)); + + /* Checks bind with AF_UNSPEC and the loopback address. */ + bind_fd = socket_variant(&self->srv0); + ASSERT_LE(0, bind_fd); + ret = bind_variant(bind_fd, &self->unspec_srv0); + if (variant->prot.domain == AF_INET) { + EXPECT_EQ(-EAFNOSUPPORT, ret); + } else { + EXPECT_EQ(-EINVAL, ret) + { + TH_LOG("Wrong bind error: %s", strerror(errno)); + } + } + EXPECT_EQ(0, close(bind_fd)); +} + +TEST_F(protocol, connect_unspec) +{ + const struct landlock_ruleset_attr ruleset_attr = { + .handled_access_net = LANDLOCK_ACCESS_NET_CONNECT_TCP, + }; + const struct landlock_net_port_attr tcp_connect = { + .allowed_access = LANDLOCK_ACCESS_NET_CONNECT_TCP, + .port = self->srv0.port, + }; + int bind_fd, client_fd, status; + pid_t child; + + /* Specific connection tests. */ + bind_fd = socket_variant(&self->srv0); + ASSERT_LE(0, bind_fd); + EXPECT_EQ(0, bind_variant(bind_fd, &self->srv0)); + if (self->srv0.protocol.type == SOCK_STREAM) + EXPECT_EQ(0, listen(bind_fd, backlog)); + + child = fork(); + ASSERT_LE(0, child); + if (child == 0) { + int connect_fd, ret; + + /* Closes listening socket for the child. */ + EXPECT_EQ(0, close(bind_fd)); + + connect_fd = socket_variant(&self->srv0); + ASSERT_LE(0, connect_fd); + EXPECT_EQ(0, connect_variant(connect_fd, &self->srv0)); + + /* Tries to connect again, or set peer. */ + ret = connect_variant(connect_fd, &self->srv0); + if (self->srv0.protocol.type == SOCK_STREAM) { + EXPECT_EQ(-EISCONN, ret); + } else { + EXPECT_EQ(0, ret); + } + + if (variant->sandbox == TCP_SANDBOX) { + const int ruleset_fd = landlock_create_ruleset( + &ruleset_attr, sizeof(ruleset_attr), 0); + ASSERT_LE(0, ruleset_fd); + + /* Allows connect. */ + ASSERT_EQ(0, landlock_add_rule(ruleset_fd, + LANDLOCK_RULE_NET_PORT, + &tcp_connect, 0)); + enforce_ruleset(_metadata, ruleset_fd); + EXPECT_EQ(0, close(ruleset_fd)); + } + + /* Disconnects already connected socket, or set peer. */ + ret = connect_variant(connect_fd, &self->unspec_any0); + if (self->srv0.protocol.domain == AF_UNIX && + self->srv0.protocol.type == SOCK_STREAM) { + EXPECT_EQ(-EINVAL, ret); + } else { + EXPECT_EQ(0, ret); + } + + /* Tries to reconnect, or set peer. */ + ret = connect_variant(connect_fd, &self->srv0); + if (self->srv0.protocol.domain == AF_UNIX && + self->srv0.protocol.type == SOCK_STREAM) { + EXPECT_EQ(-EISCONN, ret); + } else { + EXPECT_EQ(0, ret); + } + + if (variant->sandbox == TCP_SANDBOX) { + const int ruleset_fd = landlock_create_ruleset( + &ruleset_attr, sizeof(ruleset_attr), 0); + ASSERT_LE(0, ruleset_fd); + + /* Denies connect. */ + enforce_ruleset(_metadata, ruleset_fd); + EXPECT_EQ(0, close(ruleset_fd)); + } + + ret = connect_variant(connect_fd, &self->unspec_any0); + if (self->srv0.protocol.domain == AF_UNIX && + self->srv0.protocol.type == SOCK_STREAM) { + EXPECT_EQ(-EINVAL, ret); + } else { + /* Always allowed to disconnect. */ + EXPECT_EQ(0, ret); + } + + EXPECT_EQ(0, close(connect_fd)); + _exit(_metadata->passed ? EXIT_SUCCESS : EXIT_FAILURE); + return; + } + + client_fd = bind_fd; + if (self->srv0.protocol.type == SOCK_STREAM) { + client_fd = accept(bind_fd, NULL, 0); + ASSERT_LE(0, client_fd); + } + + EXPECT_EQ(child, waitpid(child, &status, 0)); + EXPECT_EQ(1, WIFEXITED(status)); + EXPECT_EQ(EXIT_SUCCESS, WEXITSTATUS(status)); + + /* Closes connection, if any. */ + if (client_fd != bind_fd) + EXPECT_LE(0, close(client_fd)); + + /* Closes listening socket. */ + EXPECT_EQ(0, close(bind_fd)); +} + +FIXTURE(ipv4) +{ + struct service_fixture srv0, srv1; +}; + +FIXTURE_VARIANT(ipv4) +{ + const enum sandbox_type sandbox; + const int type; +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(ipv4, no_sandbox_with_tcp) { + /* clang-format on */ + .sandbox = NO_SANDBOX, + .type = SOCK_STREAM, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(ipv4, tcp_sandbox_with_tcp) { + /* clang-format on */ + .sandbox = TCP_SANDBOX, + .type = SOCK_STREAM, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(ipv4, no_sandbox_with_udp) { + /* clang-format on */ + .sandbox = NO_SANDBOX, + .type = SOCK_DGRAM, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(ipv4, tcp_sandbox_with_udp) { + /* clang-format on */ + .sandbox = TCP_SANDBOX, + .type = SOCK_DGRAM, +}; + +FIXTURE_SETUP(ipv4) +{ + const struct protocol_variant prot = { + .domain = AF_INET, + .type = variant->type, + }; + + disable_caps(_metadata); + + set_service(&self->srv0, prot, 0); + set_service(&self->srv1, prot, 1); + + setup_loopback(_metadata); +}; + +FIXTURE_TEARDOWN(ipv4) +{ +} + +TEST_F(ipv4, from_unix_to_inet) +{ + int unix_stream_fd, unix_dgram_fd; + + if (variant->sandbox == TCP_SANDBOX) { + const struct landlock_ruleset_attr ruleset_attr = { + .handled_access_net = LANDLOCK_ACCESS_NET_BIND_TCP | + LANDLOCK_ACCESS_NET_CONNECT_TCP, + }; + const struct landlock_net_port_attr tcp_bind_connect_p0 = { + .allowed_access = LANDLOCK_ACCESS_NET_BIND_TCP | + LANDLOCK_ACCESS_NET_CONNECT_TCP, + .port = self->srv0.port, + }; + int ruleset_fd; + + /* Denies connect and bind to check errno value. */ + ruleset_fd = landlock_create_ruleset(&ruleset_attr, + sizeof(ruleset_attr), 0); + ASSERT_LE(0, ruleset_fd); + + /* Allows connect and bind for srv0. */ + ASSERT_EQ(0, + landlock_add_rule(ruleset_fd, LANDLOCK_RULE_NET_PORT, + &tcp_bind_connect_p0, 0)); + + enforce_ruleset(_metadata, ruleset_fd); + EXPECT_EQ(0, close(ruleset_fd)); + } + + unix_stream_fd = socket(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0); + ASSERT_LE(0, unix_stream_fd); + + unix_dgram_fd = socket(AF_UNIX, SOCK_DGRAM | SOCK_CLOEXEC, 0); + ASSERT_LE(0, unix_dgram_fd); + + /* Checks unix stream bind and connect for srv0. */ + EXPECT_EQ(-EINVAL, bind_variant(unix_stream_fd, &self->srv0)); + EXPECT_EQ(-EINVAL, connect_variant(unix_stream_fd, &self->srv0)); + + /* Checks unix stream bind and connect for srv1. */ + EXPECT_EQ(-EINVAL, bind_variant(unix_stream_fd, &self->srv1)) + { + TH_LOG("Wrong bind error: %s", strerror(errno)); + } + EXPECT_EQ(-EINVAL, connect_variant(unix_stream_fd, &self->srv1)); + + /* Checks unix datagram bind and connect for srv0. */ + EXPECT_EQ(-EINVAL, bind_variant(unix_dgram_fd, &self->srv0)); + EXPECT_EQ(-EINVAL, connect_variant(unix_dgram_fd, &self->srv0)); + + /* Checks unix datagram bind and connect for srv1. */ + EXPECT_EQ(-EINVAL, bind_variant(unix_dgram_fd, &self->srv1)); + EXPECT_EQ(-EINVAL, connect_variant(unix_dgram_fd, &self->srv1)); +} + +FIXTURE(tcp_layers) +{ + struct service_fixture srv0, srv1; +}; + +FIXTURE_VARIANT(tcp_layers) +{ + const size_t num_layers; + const int domain; +}; + +FIXTURE_SETUP(tcp_layers) +{ + const struct protocol_variant prot = { + .domain = variant->domain, + .type = SOCK_STREAM, + }; + + disable_caps(_metadata); + + ASSERT_EQ(0, set_service(&self->srv0, prot, 0)); + ASSERT_EQ(0, set_service(&self->srv1, prot, 1)); + + setup_loopback(_metadata); +}; + +FIXTURE_TEARDOWN(tcp_layers) +{ +} + +/* clang-format off */ +FIXTURE_VARIANT_ADD(tcp_layers, no_sandbox_with_ipv4) { + /* clang-format on */ + .domain = AF_INET, + .num_layers = 0, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(tcp_layers, one_sandbox_with_ipv4) { + /* clang-format on */ + .domain = AF_INET, + .num_layers = 1, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(tcp_layers, two_sandboxes_with_ipv4) { + /* clang-format on */ + .domain = AF_INET, + .num_layers = 2, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(tcp_layers, three_sandboxes_with_ipv4) { + /* clang-format on */ + .domain = AF_INET, + .num_layers = 3, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(tcp_layers, no_sandbox_with_ipv6) { + /* clang-format on */ + .domain = AF_INET6, + .num_layers = 0, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(tcp_layers, one_sandbox_with_ipv6) { + /* clang-format on */ + .domain = AF_INET6, + .num_layers = 1, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(tcp_layers, two_sandboxes_with_ipv6) { + /* clang-format on */ + .domain = AF_INET6, + .num_layers = 2, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(tcp_layers, three_sandboxes_with_ipv6) { + /* clang-format on */ + .domain = AF_INET6, + .num_layers = 3, +}; + +TEST_F(tcp_layers, ruleset_overlap) +{ + const struct landlock_ruleset_attr ruleset_attr = { + .handled_access_net = LANDLOCK_ACCESS_NET_BIND_TCP | + LANDLOCK_ACCESS_NET_CONNECT_TCP, + }; + const struct landlock_net_port_attr tcp_bind = { + .allowed_access = LANDLOCK_ACCESS_NET_BIND_TCP, + .port = self->srv0.port, + }; + const struct landlock_net_port_attr tcp_bind_connect = { + .allowed_access = LANDLOCK_ACCESS_NET_BIND_TCP | + LANDLOCK_ACCESS_NET_CONNECT_TCP, + .port = self->srv0.port, + }; + + if (variant->num_layers >= 1) { + int ruleset_fd; + + ruleset_fd = landlock_create_ruleset(&ruleset_attr, + sizeof(ruleset_attr), 0); + ASSERT_LE(0, ruleset_fd); + + /* Allows bind. */ + ASSERT_EQ(0, + landlock_add_rule(ruleset_fd, LANDLOCK_RULE_NET_PORT, + &tcp_bind, 0)); + /* Also allows bind, but allows connect too. */ + ASSERT_EQ(0, + landlock_add_rule(ruleset_fd, LANDLOCK_RULE_NET_PORT, + &tcp_bind_connect, 0)); + enforce_ruleset(_metadata, ruleset_fd); + EXPECT_EQ(0, close(ruleset_fd)); + } + + if (variant->num_layers >= 2) { + int ruleset_fd; + + /* Creates another ruleset layer. */ + ruleset_fd = landlock_create_ruleset(&ruleset_attr, + sizeof(ruleset_attr), 0); + ASSERT_LE(0, ruleset_fd); + + /* Only allows bind. */ + ASSERT_EQ(0, + landlock_add_rule(ruleset_fd, LANDLOCK_RULE_NET_PORT, + &tcp_bind, 0)); + enforce_ruleset(_metadata, ruleset_fd); + EXPECT_EQ(0, close(ruleset_fd)); + } + + if (variant->num_layers >= 3) { + int ruleset_fd; + + /* Creates another ruleset layer. */ + ruleset_fd = landlock_create_ruleset(&ruleset_attr, + sizeof(ruleset_attr), 0); + ASSERT_LE(0, ruleset_fd); + + /* Try to allow bind and connect. */ + ASSERT_EQ(0, + landlock_add_rule(ruleset_fd, LANDLOCK_RULE_NET_PORT, + &tcp_bind_connect, 0)); + enforce_ruleset(_metadata, ruleset_fd); + EXPECT_EQ(0, close(ruleset_fd)); + } + + /* + * Forbids to connect to the socket because only one ruleset layer + * allows connect. + */ + test_bind_and_connect(_metadata, &self->srv0, false, + variant->num_layers >= 2); +} + +TEST_F(tcp_layers, ruleset_expand) +{ + if (variant->num_layers >= 1) { + const struct landlock_ruleset_attr ruleset_attr = { + .handled_access_net = LANDLOCK_ACCESS_NET_BIND_TCP, + }; + /* Allows bind for srv0. */ + const struct landlock_net_port_attr bind_srv0 = { + .allowed_access = LANDLOCK_ACCESS_NET_BIND_TCP, + .port = self->srv0.port, + }; + int ruleset_fd; + + ruleset_fd = landlock_create_ruleset(&ruleset_attr, + sizeof(ruleset_attr), 0); + ASSERT_LE(0, ruleset_fd); + ASSERT_EQ(0, + landlock_add_rule(ruleset_fd, LANDLOCK_RULE_NET_PORT, + &bind_srv0, 0)); + enforce_ruleset(_metadata, ruleset_fd); + EXPECT_EQ(0, close(ruleset_fd)); + } + + if (variant->num_layers >= 2) { + /* Expands network mask with connect action. */ + const struct landlock_ruleset_attr ruleset_attr = { + .handled_access_net = LANDLOCK_ACCESS_NET_BIND_TCP | + LANDLOCK_ACCESS_NET_CONNECT_TCP, + }; + /* Allows bind for srv0 and connect to srv0. */ + const struct landlock_net_port_attr tcp_bind_connect_p0 = { + .allowed_access = LANDLOCK_ACCESS_NET_BIND_TCP | + LANDLOCK_ACCESS_NET_CONNECT_TCP, + .port = self->srv0.port, + }; + /* Try to allow bind for srv1. */ + const struct landlock_net_port_attr tcp_bind_p1 = { + .allowed_access = LANDLOCK_ACCESS_NET_BIND_TCP, + .port = self->srv1.port, + }; + int ruleset_fd; + + ruleset_fd = landlock_create_ruleset(&ruleset_attr, + sizeof(ruleset_attr), 0); + ASSERT_LE(0, ruleset_fd); + ASSERT_EQ(0, + landlock_add_rule(ruleset_fd, LANDLOCK_RULE_NET_PORT, + &tcp_bind_connect_p0, 0)); + ASSERT_EQ(0, + landlock_add_rule(ruleset_fd, LANDLOCK_RULE_NET_PORT, + &tcp_bind_p1, 0)); + enforce_ruleset(_metadata, ruleset_fd); + EXPECT_EQ(0, close(ruleset_fd)); + } + + if (variant->num_layers >= 3) { + const struct landlock_ruleset_attr ruleset_attr = { + .handled_access_net = LANDLOCK_ACCESS_NET_BIND_TCP | + LANDLOCK_ACCESS_NET_CONNECT_TCP, + }; + /* Allows connect to srv0, without bind rule. */ + const struct landlock_net_port_attr tcp_bind_p0 = { + .allowed_access = LANDLOCK_ACCESS_NET_BIND_TCP, + .port = self->srv0.port, + }; + int ruleset_fd; + + ruleset_fd = landlock_create_ruleset(&ruleset_attr, + sizeof(ruleset_attr), 0); + ASSERT_LE(0, ruleset_fd); + ASSERT_EQ(0, + landlock_add_rule(ruleset_fd, LANDLOCK_RULE_NET_PORT, + &tcp_bind_p0, 0)); + enforce_ruleset(_metadata, ruleset_fd); + EXPECT_EQ(0, close(ruleset_fd)); + } + + test_bind_and_connect(_metadata, &self->srv0, false, + variant->num_layers >= 3); + + test_bind_and_connect(_metadata, &self->srv1, variant->num_layers >= 1, + variant->num_layers >= 2); +} + +/* clang-format off */ +FIXTURE(mini) {}; +/* clang-format on */ + +FIXTURE_SETUP(mini) +{ + disable_caps(_metadata); + + setup_loopback(_metadata); +}; + +FIXTURE_TEARDOWN(mini) +{ +} + +/* clang-format off */ + +#define ACCESS_LAST LANDLOCK_ACCESS_NET_CONNECT_TCP + +#define ACCESS_ALL ( \ + LANDLOCK_ACCESS_NET_BIND_TCP | \ + LANDLOCK_ACCESS_NET_CONNECT_TCP) + +/* clang-format on */ + +TEST_F(mini, network_access_rights) +{ + const struct landlock_ruleset_attr ruleset_attr = { + .handled_access_net = ACCESS_ALL, + }; + struct landlock_net_port_attr net_port = { + .port = sock_port_start, + }; + int ruleset_fd; + __u64 access; + + ruleset_fd = + landlock_create_ruleset(&ruleset_attr, sizeof(ruleset_attr), 0); + ASSERT_LE(0, ruleset_fd); + + for (access = 1; access <= ACCESS_LAST; access <<= 1) { + net_port.allowed_access = access; + EXPECT_EQ(0, + landlock_add_rule(ruleset_fd, LANDLOCK_RULE_NET_PORT, + &net_port, 0)) + { + TH_LOG("Failed to add rule with access 0x%llx: %s", + access, strerror(errno)); + } + } + EXPECT_EQ(0, close(ruleset_fd)); +} + +/* Checks invalid attribute, out of landlock network access range. */ +TEST_F(mini, unknown_access_rights) +{ + __u64 access_mask; + + for (access_mask = 1ULL << 63; access_mask != ACCESS_LAST; + access_mask >>= 1) { + const struct landlock_ruleset_attr ruleset_attr = { + .handled_access_net = access_mask, + }; + + EXPECT_EQ(-1, landlock_create_ruleset(&ruleset_attr, + sizeof(ruleset_attr), 0)); + EXPECT_EQ(EINVAL, errno); + } +} + +TEST_F(mini, inval) +{ + const struct landlock_ruleset_attr ruleset_attr = { + .handled_access_net = LANDLOCK_ACCESS_NET_BIND_TCP + }; + const struct landlock_net_port_attr tcp_bind_connect = { + .allowed_access = LANDLOCK_ACCESS_NET_BIND_TCP | + LANDLOCK_ACCESS_NET_CONNECT_TCP, + .port = sock_port_start, + }; + const struct landlock_net_port_attr tcp_denied = { + .allowed_access = 0, + .port = sock_port_start, + }; + const struct landlock_net_port_attr tcp_bind = { + .allowed_access = LANDLOCK_ACCESS_NET_BIND_TCP, + .port = sock_port_start, + }; + int ruleset_fd; + + ruleset_fd = + landlock_create_ruleset(&ruleset_attr, sizeof(ruleset_attr), 0); + ASSERT_LE(0, ruleset_fd); + + /* Checks unhandled allowed_access. */ + EXPECT_EQ(-1, landlock_add_rule(ruleset_fd, LANDLOCK_RULE_NET_PORT, + &tcp_bind_connect, 0)); + EXPECT_EQ(EINVAL, errno); + + /* Checks zero access value. */ + EXPECT_EQ(-1, landlock_add_rule(ruleset_fd, LANDLOCK_RULE_NET_PORT, + &tcp_denied, 0)); + EXPECT_EQ(ENOMSG, errno); + + /* Adds with legitimate values. */ + ASSERT_EQ(0, landlock_add_rule(ruleset_fd, LANDLOCK_RULE_NET_PORT, + &tcp_bind, 0)); +} + +TEST_F(mini, tcp_port_overflow) +{ + const struct landlock_ruleset_attr ruleset_attr = { + .handled_access_net = LANDLOCK_ACCESS_NET_BIND_TCP | + LANDLOCK_ACCESS_NET_CONNECT_TCP, + }; + const struct landlock_net_port_attr port_max_bind = { + .allowed_access = LANDLOCK_ACCESS_NET_BIND_TCP, + .port = UINT16_MAX, + }; + const struct landlock_net_port_attr port_max_connect = { + .allowed_access = LANDLOCK_ACCESS_NET_CONNECT_TCP, + .port = UINT16_MAX, + }; + const struct landlock_net_port_attr port_overflow1 = { + .allowed_access = LANDLOCK_ACCESS_NET_BIND_TCP, + .port = UINT16_MAX + 1, + }; + const struct landlock_net_port_attr port_overflow2 = { + .allowed_access = LANDLOCK_ACCESS_NET_BIND_TCP, + .port = UINT16_MAX + 2, + }; + const struct landlock_net_port_attr port_overflow3 = { + .allowed_access = LANDLOCK_ACCESS_NET_BIND_TCP, + .port = UINT32_MAX + 1UL, + }; + const struct landlock_net_port_attr port_overflow4 = { + .allowed_access = LANDLOCK_ACCESS_NET_BIND_TCP, + .port = UINT32_MAX + 2UL, + }; + const struct protocol_variant ipv4_tcp = { + .domain = AF_INET, + .type = SOCK_STREAM, + }; + struct service_fixture srv_denied, srv_max_allowed; + int ruleset_fd; + + ASSERT_EQ(0, set_service(&srv_denied, ipv4_tcp, 0)); + + /* Be careful to avoid port inconsistencies. */ + srv_max_allowed = srv_denied; + srv_max_allowed.port = port_max_bind.port; + srv_max_allowed.ipv4_addr.sin_port = htons(port_max_bind.port); + + ruleset_fd = + landlock_create_ruleset(&ruleset_attr, sizeof(ruleset_attr), 0); + ASSERT_LE(0, ruleset_fd); + + ASSERT_EQ(0, landlock_add_rule(ruleset_fd, LANDLOCK_RULE_NET_PORT, + &port_max_bind, 0)); + + EXPECT_EQ(-1, landlock_add_rule(ruleset_fd, LANDLOCK_RULE_NET_PORT, + &port_overflow1, 0)); + EXPECT_EQ(EINVAL, errno); + + EXPECT_EQ(-1, landlock_add_rule(ruleset_fd, LANDLOCK_RULE_NET_PORT, + &port_overflow2, 0)); + EXPECT_EQ(EINVAL, errno); + + EXPECT_EQ(-1, landlock_add_rule(ruleset_fd, LANDLOCK_RULE_NET_PORT, + &port_overflow3, 0)); + EXPECT_EQ(EINVAL, errno); + + /* Interleaves with invalid rule additions. */ + ASSERT_EQ(0, landlock_add_rule(ruleset_fd, LANDLOCK_RULE_NET_PORT, + &port_max_connect, 0)); + + EXPECT_EQ(-1, landlock_add_rule(ruleset_fd, LANDLOCK_RULE_NET_PORT, + &port_overflow4, 0)); + EXPECT_EQ(EINVAL, errno); + + enforce_ruleset(_metadata, ruleset_fd); + + test_bind_and_connect(_metadata, &srv_denied, true, true); + test_bind_and_connect(_metadata, &srv_max_allowed, false, false); +} + +FIXTURE(ipv4_tcp) +{ + struct service_fixture srv0, srv1; +}; + +FIXTURE_SETUP(ipv4_tcp) +{ + const struct protocol_variant ipv4_tcp = { + .domain = AF_INET, + .type = SOCK_STREAM, + }; + + disable_caps(_metadata); + + ASSERT_EQ(0, set_service(&self->srv0, ipv4_tcp, 0)); + ASSERT_EQ(0, set_service(&self->srv1, ipv4_tcp, 1)); + + setup_loopback(_metadata); +}; + +FIXTURE_TEARDOWN(ipv4_tcp) +{ +} + +TEST_F(ipv4_tcp, port_endianness) +{ + const struct landlock_ruleset_attr ruleset_attr = { + .handled_access_net = LANDLOCK_ACCESS_NET_BIND_TCP | + LANDLOCK_ACCESS_NET_CONNECT_TCP, + }; + const struct landlock_net_port_attr bind_host_endian_p0 = { + .allowed_access = LANDLOCK_ACCESS_NET_BIND_TCP, + /* Host port format. */ + .port = self->srv0.port, + }; + const struct landlock_net_port_attr connect_big_endian_p0 = { + .allowed_access = LANDLOCK_ACCESS_NET_CONNECT_TCP, + /* Big endian port format. */ + .port = htons(self->srv0.port), + }; + const struct landlock_net_port_attr bind_connect_host_endian_p1 = { + .allowed_access = LANDLOCK_ACCESS_NET_BIND_TCP | + LANDLOCK_ACCESS_NET_CONNECT_TCP, + /* Host port format. */ + .port = self->srv1.port, + }; + const unsigned int one = 1; + const char little_endian = *(const char *)&one; + int ruleset_fd; + + ruleset_fd = + landlock_create_ruleset(&ruleset_attr, sizeof(ruleset_attr), 0); + ASSERT_LE(0, ruleset_fd); + ASSERT_EQ(0, landlock_add_rule(ruleset_fd, LANDLOCK_RULE_NET_PORT, + &bind_host_endian_p0, 0)); + ASSERT_EQ(0, landlock_add_rule(ruleset_fd, LANDLOCK_RULE_NET_PORT, + &connect_big_endian_p0, 0)); + ASSERT_EQ(0, landlock_add_rule(ruleset_fd, LANDLOCK_RULE_NET_PORT, + &bind_connect_host_endian_p1, 0)); + enforce_ruleset(_metadata, ruleset_fd); + + /* No restriction for big endinan CPU. */ + test_bind_and_connect(_metadata, &self->srv0, false, little_endian); + + /* No restriction for any CPU. */ + test_bind_and_connect(_metadata, &self->srv1, false, false); +} + +TEST_F(ipv4_tcp, with_fs) +{ + const struct landlock_ruleset_attr ruleset_attr_fs_net = { + .handled_access_fs = LANDLOCK_ACCESS_FS_READ_DIR, + .handled_access_net = LANDLOCK_ACCESS_NET_BIND_TCP, + }; + struct landlock_path_beneath_attr path_beneath = { + .allowed_access = LANDLOCK_ACCESS_FS_READ_DIR, + .parent_fd = -1, + }; + struct landlock_net_port_attr tcp_bind = { + .allowed_access = LANDLOCK_ACCESS_NET_BIND_TCP, + .port = self->srv0.port, + }; + int ruleset_fd, bind_fd, dir_fd; + + /* Creates ruleset both for filesystem and network access. */ + ruleset_fd = landlock_create_ruleset(&ruleset_attr_fs_net, + sizeof(ruleset_attr_fs_net), 0); + ASSERT_LE(0, ruleset_fd); + + /* Adds a filesystem rule. */ + path_beneath.parent_fd = open("/dev", O_PATH | O_DIRECTORY | O_CLOEXEC); + ASSERT_LE(0, path_beneath.parent_fd); + ASSERT_EQ(0, landlock_add_rule(ruleset_fd, LANDLOCK_RULE_PATH_BENEATH, + &path_beneath, 0)); + EXPECT_EQ(0, close(path_beneath.parent_fd)); + + /* Adds a network rule. */ + ASSERT_EQ(0, landlock_add_rule(ruleset_fd, LANDLOCK_RULE_NET_PORT, + &tcp_bind, 0)); + + enforce_ruleset(_metadata, ruleset_fd); + EXPECT_EQ(0, close(ruleset_fd)); + + /* Tests file access. */ + dir_fd = open("/dev", O_RDONLY); + EXPECT_LE(0, dir_fd); + EXPECT_EQ(0, close(dir_fd)); + + dir_fd = open("/", O_RDONLY); + EXPECT_EQ(-1, dir_fd); + EXPECT_EQ(EACCES, errno); + + /* Tests port binding. */ + bind_fd = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0); + ASSERT_LE(0, bind_fd); + EXPECT_EQ(0, bind_variant(bind_fd, &self->srv0)); + EXPECT_EQ(0, close(bind_fd)); + + bind_fd = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0); + ASSERT_LE(0, bind_fd); + EXPECT_EQ(-EACCES, bind_variant(bind_fd, &self->srv1)); +} + +FIXTURE(port_specific) +{ + struct service_fixture srv0; +}; + +FIXTURE_VARIANT(port_specific) +{ + const enum sandbox_type sandbox; + const struct protocol_variant prot; +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(port_specific, no_sandbox_with_ipv4) { + /* clang-format on */ + .sandbox = NO_SANDBOX, + .prot = { + .domain = AF_INET, + .type = SOCK_STREAM, + }, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(port_specific, sandbox_with_ipv4) { + /* clang-format on */ + .sandbox = TCP_SANDBOX, + .prot = { + .domain = AF_INET, + .type = SOCK_STREAM, + }, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(port_specific, no_sandbox_with_ipv6) { + /* clang-format on */ + .sandbox = NO_SANDBOX, + .prot = { + .domain = AF_INET6, + .type = SOCK_STREAM, + }, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(port_specific, sandbox_with_ipv6) { + /* clang-format on */ + .sandbox = TCP_SANDBOX, + .prot = { + .domain = AF_INET6, + .type = SOCK_STREAM, + }, +}; + +FIXTURE_SETUP(port_specific) +{ + disable_caps(_metadata); + + ASSERT_EQ(0, set_service(&self->srv0, variant->prot, 0)); + + setup_loopback(_metadata); +}; + +FIXTURE_TEARDOWN(port_specific) +{ +} + +TEST_F(port_specific, bind_connect_zero) +{ + int bind_fd, connect_fd, ret; + uint16_t port; + + /* Adds a rule layer with bind and connect actions. */ + if (variant->sandbox == TCP_SANDBOX) { + const struct landlock_ruleset_attr ruleset_attr = { + .handled_access_net = LANDLOCK_ACCESS_NET_BIND_TCP | + LANDLOCK_ACCESS_NET_CONNECT_TCP + }; + const struct landlock_net_port_attr tcp_bind_connect_zero = { + .allowed_access = LANDLOCK_ACCESS_NET_BIND_TCP | + LANDLOCK_ACCESS_NET_CONNECT_TCP, + .port = 0, + }; + int ruleset_fd; + + ruleset_fd = landlock_create_ruleset(&ruleset_attr, + sizeof(ruleset_attr), 0); + ASSERT_LE(0, ruleset_fd); + + /* Checks zero port value on bind and connect actions. */ + EXPECT_EQ(0, + landlock_add_rule(ruleset_fd, LANDLOCK_RULE_NET_PORT, + &tcp_bind_connect_zero, 0)); + + enforce_ruleset(_metadata, ruleset_fd); + EXPECT_EQ(0, close(ruleset_fd)); + } + + bind_fd = socket_variant(&self->srv0); + ASSERT_LE(0, bind_fd); + + connect_fd = socket_variant(&self->srv0); + ASSERT_LE(0, connect_fd); + + /* Sets address port to 0 for both protocol families. */ + set_port(&self->srv0, 0); + /* + * Binds on port 0, which selects a random port within + * ip_local_port_range. + */ + ret = bind_variant(bind_fd, &self->srv0); + EXPECT_EQ(0, ret); + + EXPECT_EQ(0, listen(bind_fd, backlog)); + + /* Connects on port 0. */ + ret = connect_variant(connect_fd, &self->srv0); + EXPECT_EQ(-ECONNREFUSED, ret); + + /* Sets binded port for both protocol families. */ + port = get_binded_port(bind_fd, &variant->prot); + EXPECT_NE(0, port); + set_port(&self->srv0, port); + /* Connects on the binded port. */ + ret = connect_variant(connect_fd, &self->srv0); + if (is_restricted(&variant->prot, variant->sandbox)) { + /* Denied by Landlock. */ + EXPECT_EQ(-EACCES, ret); + } else { + EXPECT_EQ(0, ret); + } + + EXPECT_EQ(0, close(connect_fd)); + EXPECT_EQ(0, close(bind_fd)); +} + +TEST_F(port_specific, bind_connect_1023) +{ + int bind_fd, connect_fd, ret; + + /* Adds a rule layer with bind and connect actions. */ + if (variant->sandbox == TCP_SANDBOX) { + const struct landlock_ruleset_attr ruleset_attr = { + .handled_access_net = LANDLOCK_ACCESS_NET_BIND_TCP | + LANDLOCK_ACCESS_NET_CONNECT_TCP + }; + /* A rule with port value less than 1024. */ + const struct landlock_net_port_attr tcp_bind_connect_low_range = { + .allowed_access = LANDLOCK_ACCESS_NET_BIND_TCP | + LANDLOCK_ACCESS_NET_CONNECT_TCP, + .port = 1023, + }; + /* A rule with 1024 port. */ + const struct landlock_net_port_attr tcp_bind_connect = { + .allowed_access = LANDLOCK_ACCESS_NET_BIND_TCP | + LANDLOCK_ACCESS_NET_CONNECT_TCP, + .port = 1024, + }; + int ruleset_fd; + + ruleset_fd = landlock_create_ruleset(&ruleset_attr, + sizeof(ruleset_attr), 0); + ASSERT_LE(0, ruleset_fd); + + ASSERT_EQ(0, + landlock_add_rule(ruleset_fd, LANDLOCK_RULE_NET_PORT, + &tcp_bind_connect_low_range, 0)); + ASSERT_EQ(0, + landlock_add_rule(ruleset_fd, LANDLOCK_RULE_NET_PORT, + &tcp_bind_connect, 0)); + + enforce_ruleset(_metadata, ruleset_fd); + EXPECT_EQ(0, close(ruleset_fd)); + } + + bind_fd = socket_variant(&self->srv0); + ASSERT_LE(0, bind_fd); + + connect_fd = socket_variant(&self->srv0); + ASSERT_LE(0, connect_fd); + + /* Sets address port to 1023 for both protocol families. */ + set_port(&self->srv0, 1023); + /* Binds on port 1023. */ + ret = bind_variant(bind_fd, &self->srv0); + /* Denied by the system. */ + EXPECT_EQ(-EACCES, ret); + + /* Binds on port 1023. */ + set_cap(_metadata, CAP_NET_BIND_SERVICE); + ret = bind_variant(bind_fd, &self->srv0); + clear_cap(_metadata, CAP_NET_BIND_SERVICE); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, listen(bind_fd, backlog)); + + /* Connects on the binded port 1023. */ + ret = connect_variant(connect_fd, &self->srv0); + EXPECT_EQ(0, ret); + + EXPECT_EQ(0, close(connect_fd)); + EXPECT_EQ(0, close(bind_fd)); + + bind_fd = socket_variant(&self->srv0); + ASSERT_LE(0, bind_fd); + + connect_fd = socket_variant(&self->srv0); + ASSERT_LE(0, connect_fd); + + /* Sets address port to 1024 for both protocol families. */ + set_port(&self->srv0, 1024); + /* Binds on port 1024. */ + ret = bind_variant(bind_fd, &self->srv0); + EXPECT_EQ(0, ret); + EXPECT_EQ(0, listen(bind_fd, backlog)); + + /* Connects on the binded port 1024. */ + ret = connect_variant(connect_fd, &self->srv0); + EXPECT_EQ(0, ret); + + EXPECT_EQ(0, close(connect_fd)); + EXPECT_EQ(0, close(bind_fd)); +} + +TEST_HARNESS_MAIN From 5e990dcef12eebf683d209bac5e14591308dc216 Mon Sep 17 00:00:00 2001 From: Konstantin Meskhidze Date: Thu, 26 Oct 2023 09:47:50 +0800 Subject: [PATCH 442/475] samples/landlock: Support TCP restrictions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add TCP restrictions to the sandboxer demo. It's possible to allow a sandboxer to bind/connect to a list of specified ports restricting network actions to the rest of them. This is controlled with the new LL_TCP_BIND and LL_TCP_CONNECT environment variables. Rename ENV_PATH_TOKEN to ENV_DELIMITER. Signed-off-by: Konstantin Meskhidze Link: https://lore.kernel.org/r/20231026014751.414649-12-konstantin.meskhidze@huawei.com [mic: Extend commit message] Signed-off-by: Mickaël Salaün --- samples/landlock/sandboxer.c | 115 ++++++++++++++++++++++++++++++----- 1 file changed, 100 insertions(+), 15 deletions(-) diff --git a/samples/landlock/sandboxer.c b/samples/landlock/sandboxer.c index e2056c8b902c..08596c0ef070 100644 --- a/samples/landlock/sandboxer.c +++ b/samples/landlock/sandboxer.c @@ -8,6 +8,8 @@ */ #define _GNU_SOURCE +#define __SANE_USERSPACE_TYPES__ +#include #include #include #include @@ -51,7 +53,9 @@ static inline int landlock_restrict_self(const int ruleset_fd, #define ENV_FS_RO_NAME "LL_FS_RO" #define ENV_FS_RW_NAME "LL_FS_RW" -#define ENV_PATH_TOKEN ":" +#define ENV_TCP_BIND_NAME "LL_TCP_BIND" +#define ENV_TCP_CONNECT_NAME "LL_TCP_CONNECT" +#define ENV_DELIMITER ":" static int parse_path(char *env_path, const char ***const path_list) { @@ -60,13 +64,13 @@ static int parse_path(char *env_path, const char ***const path_list) if (env_path) { num_paths++; for (i = 0; env_path[i]; i++) { - if (env_path[i] == ENV_PATH_TOKEN[0]) + if (env_path[i] == ENV_DELIMITER[0]) num_paths++; } } *path_list = malloc(num_paths * sizeof(**path_list)); for (i = 0; i < num_paths; i++) - (*path_list)[i] = strsep(&env_path, ENV_PATH_TOKEN); + (*path_list)[i] = strsep(&env_path, ENV_DELIMITER); return num_paths; } @@ -81,8 +85,8 @@ static int parse_path(char *env_path, const char ***const path_list) /* clang-format on */ -static int populate_ruleset(const char *const env_var, const int ruleset_fd, - const __u64 allowed_access) +static int populate_ruleset_fs(const char *const env_var, const int ruleset_fd, + const __u64 allowed_access) { int num_paths, i, ret = 1; char *env_path_name; @@ -143,6 +147,39 @@ out_free_name: return ret; } +static int populate_ruleset_net(const char *const env_var, const int ruleset_fd, + const __u64 allowed_access) +{ + int ret = 1; + char *env_port_name, *strport; + struct landlock_net_port_attr net_port = { + .allowed_access = allowed_access, + .port = 0, + }; + + env_port_name = getenv(env_var); + if (!env_port_name) + return 0; + env_port_name = strdup(env_port_name); + unsetenv(env_var); + + while ((strport = strsep(&env_port_name, ENV_DELIMITER))) { + net_port.port = atoi(strport); + if (landlock_add_rule(ruleset_fd, LANDLOCK_RULE_NET_PORT, + &net_port, 0)) { + fprintf(stderr, + "Failed to update the ruleset with port \"%llu\": %s\n", + net_port.port, strerror(errno)); + goto out_free_name; + } + } + ret = 0; + +out_free_name: + free(env_port_name); + return ret; +} + /* clang-format off */ #define ACCESS_FS_ROUGHLY_READ ( \ @@ -166,39 +203,58 @@ out_free_name: /* clang-format on */ -#define LANDLOCK_ABI_LAST 3 +#define LANDLOCK_ABI_LAST 4 int main(const int argc, char *const argv[], char *const *const envp) { const char *cmd_path; char *const *cmd_argv; int ruleset_fd, abi; + char *env_port_name; __u64 access_fs_ro = ACCESS_FS_ROUGHLY_READ, access_fs_rw = ACCESS_FS_ROUGHLY_READ | ACCESS_FS_ROUGHLY_WRITE; + struct landlock_ruleset_attr ruleset_attr = { .handled_access_fs = access_fs_rw, + .handled_access_net = LANDLOCK_ACCESS_NET_BIND_TCP | + LANDLOCK_ACCESS_NET_CONNECT_TCP, }; if (argc < 2) { fprintf(stderr, - "usage: %s=\"...\" %s=\"...\" %s [args]...\n\n", - ENV_FS_RO_NAME, ENV_FS_RW_NAME, argv[0]); + "usage: %s=\"...\" %s=\"...\" %s=\"...\" %s=\"...\"%s " + " [args]...\n\n", + ENV_FS_RO_NAME, ENV_FS_RW_NAME, ENV_TCP_BIND_NAME, + ENV_TCP_CONNECT_NAME, argv[0]); fprintf(stderr, "Launch a command in a restricted environment.\n\n"); - fprintf(stderr, "Environment variables containing paths, " - "each separated by a colon:\n"); + fprintf(stderr, + "Environment variables containing paths and ports " + "each separated by a colon:\n"); fprintf(stderr, "* %s: list of paths allowed to be used in a read-only way.\n", ENV_FS_RO_NAME); fprintf(stderr, - "* %s: list of paths allowed to be used in a read-write way.\n", + "* %s: list of paths allowed to be used in a read-write way.\n\n", ENV_FS_RW_NAME); + fprintf(stderr, + "Environment variables containing ports are optional " + "and could be skipped.\n"); + fprintf(stderr, + "* %s: list of ports allowed to bind (server).\n", + ENV_TCP_BIND_NAME); + fprintf(stderr, + "* %s: list of ports allowed to connect (client).\n", + ENV_TCP_CONNECT_NAME); fprintf(stderr, "\nexample:\n" "%s=\"/bin:/lib:/usr:/proc:/etc:/dev/urandom\" " "%s=\"/dev/null:/dev/full:/dev/zero:/dev/pts:/tmp\" " + "%s=\"9418\" " + "%s=\"80:443\" " "%s bash -i\n\n", - ENV_FS_RO_NAME, ENV_FS_RW_NAME, argv[0]); + ENV_FS_RO_NAME, ENV_FS_RW_NAME, ENV_TCP_BIND_NAME, + ENV_TCP_CONNECT_NAME, argv[0]); fprintf(stderr, "This sandboxer can use Landlock features " "up to ABI version %d.\n", @@ -255,7 +311,12 @@ int main(const int argc, char *const argv[], char *const *const envp) case 2: /* Removes LANDLOCK_ACCESS_FS_TRUNCATE for ABI < 3 */ ruleset_attr.handled_access_fs &= ~LANDLOCK_ACCESS_FS_TRUNCATE; - + __attribute__((fallthrough)); + case 3: + /* Removes network support for ABI < 4 */ + ruleset_attr.handled_access_net &= + ~(LANDLOCK_ACCESS_NET_BIND_TCP | + LANDLOCK_ACCESS_NET_CONNECT_TCP); fprintf(stderr, "Hint: You should update the running kernel " "to leverage Landlock features " @@ -274,18 +335,42 @@ int main(const int argc, char *const argv[], char *const *const envp) access_fs_ro &= ruleset_attr.handled_access_fs; access_fs_rw &= ruleset_attr.handled_access_fs; + /* Removes bind access attribute if not supported by a user. */ + env_port_name = getenv(ENV_TCP_BIND_NAME); + if (!env_port_name) { + ruleset_attr.handled_access_net &= + ~LANDLOCK_ACCESS_NET_BIND_TCP; + } + /* Removes connect access attribute if not supported by a user. */ + env_port_name = getenv(ENV_TCP_CONNECT_NAME); + if (!env_port_name) { + ruleset_attr.handled_access_net &= + ~LANDLOCK_ACCESS_NET_CONNECT_TCP; + } + ruleset_fd = landlock_create_ruleset(&ruleset_attr, sizeof(ruleset_attr), 0); if (ruleset_fd < 0) { perror("Failed to create a ruleset"); return 1; } - if (populate_ruleset(ENV_FS_RO_NAME, ruleset_fd, access_fs_ro)) { + + if (populate_ruleset_fs(ENV_FS_RO_NAME, ruleset_fd, access_fs_ro)) { goto err_close_ruleset; } - if (populate_ruleset(ENV_FS_RW_NAME, ruleset_fd, access_fs_rw)) { + if (populate_ruleset_fs(ENV_FS_RW_NAME, ruleset_fd, access_fs_rw)) { goto err_close_ruleset; } + + if (populate_ruleset_net(ENV_TCP_BIND_NAME, ruleset_fd, + LANDLOCK_ACCESS_NET_BIND_TCP)) { + goto err_close_ruleset; + } + if (populate_ruleset_net(ENV_TCP_CONNECT_NAME, ruleset_fd, + LANDLOCK_ACCESS_NET_CONNECT_TCP)) { + goto err_close_ruleset; + } + if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)) { perror("Failed to restrict privileges"); goto err_close_ruleset; From 51442e8d64bcb7ac92d0cde5dccda0cfa94630fa Mon Sep 17 00:00:00 2001 From: Konstantin Meskhidze Date: Thu, 26 Oct 2023 09:47:51 +0800 Subject: [PATCH 443/475] landlock: Document network support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Describe network access rules for TCP sockets. Add network access example in the tutorial. Add kernel configuration support for network. Signed-off-by: Konstantin Meskhidze Link: https://lore.kernel.org/r/20231026014751.414649-13-konstantin.meskhidze@huawei.com [mic: Update date, and do light cosmetic changes] Signed-off-by: Mickaël Salaün --- Documentation/userspace-api/landlock.rst | 93 ++++++++++++++++++------ 1 file changed, 71 insertions(+), 22 deletions(-) diff --git a/Documentation/userspace-api/landlock.rst b/Documentation/userspace-api/landlock.rst index f6a7da21708a..2e3822677061 100644 --- a/Documentation/userspace-api/landlock.rst +++ b/Documentation/userspace-api/landlock.rst @@ -8,13 +8,13 @@ Landlock: unprivileged access control ===================================== :Author: Mickaël Salaün -:Date: October 2022 +:Date: October 2023 The goal of Landlock is to enable to restrict ambient rights (e.g. global -filesystem access) for a set of processes. Because Landlock is a stackable -LSM, it makes possible to create safe security sandboxes as new security layers -in addition to the existing system-wide access-controls. This kind of sandbox -is expected to help mitigate the security impact of bugs or +filesystem or network access) for a set of processes. Because Landlock +is a stackable LSM, it makes possible to create safe security sandboxes as new +security layers in addition to the existing system-wide access-controls. This +kind of sandbox is expected to help mitigate the security impact of bugs or unexpected/malicious behaviors in user space applications. Landlock empowers any process, including unprivileged ones, to securely restrict themselves. @@ -28,20 +28,34 @@ appropriately `. Landlock rules ============== -A Landlock rule describes an action on an object. An object is currently a -file hierarchy, and the related filesystem actions are defined with `access -rights`_. A set of rules is aggregated in a ruleset, which can then restrict +A Landlock rule describes an action on an object which the process intends to +perform. A set of rules is aggregated in a ruleset, which can then restrict the thread enforcing it, and its future children. +The two existing types of rules are: + +Filesystem rules + For these rules, the object is a file hierarchy, + and the related filesystem actions are defined with + `filesystem access rights`. + +Network rules (since ABI v4) + For these rules, the object is a TCP port, + and the related actions are defined with `network access rights`. + Defining and enforcing a security policy ---------------------------------------- -We first need to define the ruleset that will contain our rules. For this -example, the ruleset will contain rules that only allow read actions, but write -actions will be denied. The ruleset then needs to handle both of these kind of -actions. This is required for backward and forward compatibility (i.e. the -kernel and user space may not know each other's supported restrictions), hence -the need to be explicit about the denied-by-default access rights. +We first need to define the ruleset that will contain our rules. + +For this example, the ruleset will contain rules that only allow filesystem +read actions and establish a specific TCP connection. Filesystem write +actions and other TCP actions will be denied. + +The ruleset then needs to handle both these kinds of actions. This is +required for backward and forward compatibility (i.e. the kernel and user +space may not know each other's supported restrictions), hence the need +to be explicit about the denied-by-default access rights. .. code-block:: c @@ -62,6 +76,9 @@ the need to be explicit about the denied-by-default access rights. LANDLOCK_ACCESS_FS_MAKE_SYM | LANDLOCK_ACCESS_FS_REFER | LANDLOCK_ACCESS_FS_TRUNCATE, + .handled_access_net = + LANDLOCK_ACCESS_NET_BIND_TCP | + LANDLOCK_ACCESS_NET_CONNECT_TCP, }; Because we may not know on which kernel version an application will be @@ -70,9 +87,7 @@ should try to protect users as much as possible whatever the kernel they are using. To avoid binary enforcement (i.e. either all security features or none), we can leverage a dedicated Landlock command to get the current version of the Landlock ABI and adapt the handled accesses. Let's check if we should -remove the ``LANDLOCK_ACCESS_FS_REFER`` or ``LANDLOCK_ACCESS_FS_TRUNCATE`` -access rights, which are only supported starting with the second and third -version of the ABI. +remove access rights which are only supported in higher versions of the ABI. .. code-block:: c @@ -92,6 +107,12 @@ version of the ABI. case 2: /* Removes LANDLOCK_ACCESS_FS_TRUNCATE for ABI < 3 */ ruleset_attr.handled_access_fs &= ~LANDLOCK_ACCESS_FS_TRUNCATE; + __attribute__((fallthrough)); + case 3: + /* Removes network support for ABI < 4 */ + ruleset_attr.handled_access_net &= + ~(LANDLOCK_ACCESS_NET_BIND_TCP | + LANDLOCK_ACCESS_NET_CONNECT_TCP); } This enables to create an inclusive ruleset that will contain our rules. @@ -143,10 +164,23 @@ for the ruleset creation, by filtering access rights according to the Landlock ABI version. In this example, this is not required because all of the requested ``allowed_access`` rights are already available in ABI 1. -We now have a ruleset with one rule allowing read access to ``/usr`` while -denying all other handled accesses for the filesystem. The next step is to -restrict the current thread from gaining more privileges (e.g. thanks to a SUID -binary). +For network access-control, we can add a set of rules that allow to use a port +number for a specific action: HTTPS connections. + +.. code-block:: c + + struct landlock_net_port_attr net_port = { + .allowed_access = LANDLOCK_ACCESS_NET_CONNECT_TCP, + .port = 443, + }; + + err = landlock_add_rule(ruleset_fd, LANDLOCK_RULE_NET_PORT, + &net_port, 0); + +The next step is to restrict the current thread from gaining more privileges +(e.g. through a SUID binary). We now have a ruleset with the first rule +allowing read access to ``/usr`` while denying all other handled accesses for +the filesystem, and a second rule allowing HTTPS connections. .. code-block:: c @@ -355,7 +389,7 @@ Access rights ------------- .. kernel-doc:: include/uapi/linux/landlock.h - :identifiers: fs_access + :identifiers: fs_access net_access Creating a new ruleset ---------------------- @@ -374,6 +408,7 @@ Extending a ruleset .. kernel-doc:: include/uapi/linux/landlock.h :identifiers: landlock_rule_type landlock_path_beneath_attr + landlock_net_port_attr Enforcing a ruleset ------------------- @@ -451,6 +486,14 @@ always allowed when using a kernel that only supports the first or second ABI. Starting with the Landlock ABI version 3, it is now possible to securely control truncation thanks to the new ``LANDLOCK_ACCESS_FS_TRUNCATE`` access right. +Network support (ABI < 4) +------------------------- + +Starting with the Landlock ABI version 4, it is now possible to restrict TCP +bind and connect actions to only a set of allowed ports thanks to the new +``LANDLOCK_ACCESS_NET_BIND_TCP`` and ``LANDLOCK_ACCESS_NET_CONNECT_TCP`` +access rights. + .. _kernel_support: Kernel support @@ -469,6 +512,12 @@ still enable it by adding ``lsm=landlock,[...]`` to Documentation/admin-guide/kernel-parameters.rst thanks to the bootloader configuration. +To be able to explicitly allow TCP operations (e.g., adding a network rule with +``LANDLOCK_ACCESS_NET_BIND_TCP``), the kernel must support TCP +(``CONFIG_INET=y``). Otherwise, sys_landlock_add_rule() returns an +``EAFNOSUPPORT`` error, which can safely be ignored because this kind of TCP +operation is already not possible. + Questions and answers ===================== From f12f8f84509a084399444c4422661345a15cc713 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Sala=C3=BCn?= Date: Fri, 27 Oct 2023 17:46:15 +0200 Subject: [PATCH 444/475] selftests/landlock: Add tests for FS topology changes with network rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add 2 tests to the layout1 fixture: * topology_changes_with_net_only: Checks that FS topology changes are not denied by network-only restrictions. * topology_changes_with_net_and_fs: Make sure that FS topology changes are still denied with FS and network restrictions. This specifically test commit d7220364039f ("landlock: Allow FS topology changes for domains without such rule type"). Cc: Konstantin Meskhidze Link: https://lore.kernel.org/r/20231027154615.815134-1-mic@digikod.net Signed-off-by: Mickaël Salaün --- tools/testing/selftests/landlock/fs_test.c | 59 ++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/tools/testing/selftests/landlock/fs_test.c b/tools/testing/selftests/landlock/fs_test.c index 68b7a89cf65b..18e1f86a6234 100644 --- a/tools/testing/selftests/landlock/fs_test.c +++ b/tools/testing/selftests/landlock/fs_test.c @@ -1625,6 +1625,65 @@ TEST_F_FORK(layout1, move_mount) clear_cap(_metadata, CAP_SYS_ADMIN); } +TEST_F_FORK(layout1, topology_changes_with_net_only) +{ + const struct landlock_ruleset_attr ruleset_net = { + .handled_access_net = LANDLOCK_ACCESS_NET_BIND_TCP | + LANDLOCK_ACCESS_NET_CONNECT_TCP, + }; + int ruleset_fd; + + /* Add network restrictions. */ + ruleset_fd = + landlock_create_ruleset(&ruleset_net, sizeof(ruleset_net), 0); + ASSERT_LE(0, ruleset_fd); + enforce_ruleset(_metadata, ruleset_fd); + ASSERT_EQ(0, close(ruleset_fd)); + + /* Mount, remount, move_mount, umount, and pivot_root checks. */ + set_cap(_metadata, CAP_SYS_ADMIN); + ASSERT_EQ(0, mount_opt(&mnt_tmp, dir_s1d2)); + ASSERT_EQ(0, mount(NULL, dir_s1d2, NULL, MS_PRIVATE | MS_REC, NULL)); + ASSERT_EQ(0, syscall(__NR_move_mount, AT_FDCWD, dir_s1d2, AT_FDCWD, + dir_s2d2, 0)); + ASSERT_EQ(0, umount(dir_s2d2)); + ASSERT_EQ(0, syscall(__NR_pivot_root, dir_s3d2, dir_s3d3)); + ASSERT_EQ(0, chdir("/")); + clear_cap(_metadata, CAP_SYS_ADMIN); +} + +TEST_F_FORK(layout1, topology_changes_with_net_and_fs) +{ + const struct landlock_ruleset_attr ruleset_net_fs = { + .handled_access_net = LANDLOCK_ACCESS_NET_BIND_TCP | + LANDLOCK_ACCESS_NET_CONNECT_TCP, + .handled_access_fs = LANDLOCK_ACCESS_FS_EXECUTE, + }; + int ruleset_fd; + + /* Add network and filesystem restrictions. */ + ruleset_fd = landlock_create_ruleset(&ruleset_net_fs, + sizeof(ruleset_net_fs), 0); + ASSERT_LE(0, ruleset_fd); + enforce_ruleset(_metadata, ruleset_fd); + ASSERT_EQ(0, close(ruleset_fd)); + + /* Mount, remount, move_mount, umount, and pivot_root checks. */ + set_cap(_metadata, CAP_SYS_ADMIN); + ASSERT_EQ(-1, mount_opt(&mnt_tmp, dir_s1d2)); + ASSERT_EQ(EPERM, errno); + ASSERT_EQ(-1, mount(NULL, dir_s3d2, NULL, MS_PRIVATE | MS_REC, NULL)); + ASSERT_EQ(EPERM, errno); + ASSERT_EQ(-1, syscall(__NR_move_mount, AT_FDCWD, dir_s3d2, AT_FDCWD, + dir_s2d2, 0)); + ASSERT_EQ(EPERM, errno); + ASSERT_EQ(-1, umount(dir_s3d2)); + ASSERT_EQ(EPERM, errno); + ASSERT_EQ(-1, syscall(__NR_pivot_root, dir_s3d2, dir_s3d3)); + ASSERT_EQ(EPERM, errno); + clear_cap(_metadata, CAP_SYS_ADMIN); +} + TEST_F_FORK(layout1, release_inodes) { const struct rule rules[] = { From 93c65d61433bcec4070fc61b7c737c79fc25f09a Mon Sep 17 00:00:00 2001 From: Arnaldo Carvalho de Melo Date: Fri, 27 Oct 2023 10:33:30 -0300 Subject: [PATCH 445/475] perf python: Fix binding linkage due to rename and move of evsel__increase_rlimit() The changes in ("perf evsel: Rename evsel__increase_rlimit to rlimit__increase_nofile") ended up breaking the python binding that now references the rlimit__increase_nofile function, add the util/rlimit.o to the tools/perf/util/python-ext-sources to cure that. This was detected by the 'perf test python' regression test: $ perf test python 14: 'import perf' in python : FAILED! $ perf test -v python Couldn't bump rlimit(MEMLOCK), failures may take place when creating BPF maps, etc 14: 'import perf' in python : --- start --- test child forked, pid 2912462 python usage test: "echo "import sys ; sys.path.insert(0, '/tmp/build/perf-tools-next/python'); import perf" | '/usr/bin/python3' " Traceback (most recent call last): File "", line 1, in ImportError: /tmp/build/perf-tools-next/python/perf.cpython-311-x86_64-linux-gnu.so: undefined symbol: rlimit__increase_nofile test child finished with -1 ---- end ---- 'import perf' in python: FAILED! $ Fixes: e093a222d7cba1eb ("perf evsel: Rename evsel__increase_rlimit to rlimit__increase_nofile") Acked-by: Namhyung Kim Acked-by: Yang Jihong Link: https://lore.kernel.org/lkml/ZTrCS5Z3PZAmfPdV@kernel.org Signed-off-by: Arnaldo Carvalho de Melo Signed-off-by: Namhyung Kim --- tools/perf/util/python-ext-sources | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/perf/util/python-ext-sources b/tools/perf/util/python-ext-sources index 26e1c8d973ea..593b660ec75e 100644 --- a/tools/perf/util/python-ext-sources +++ b/tools/perf/util/python-ext-sources @@ -40,6 +40,7 @@ util/rwsem.c util/hashmap.c util/perf_regs.c util/fncache.c +util/rlimit.c util/perf-regs-arch/perf_regs_aarch64.c util/perf-regs-arch/perf_regs_arm.c util/perf-regs-arch/perf_regs_csky.c From 0e0f03d7fc933ffe22c67feeec4b49bdd4cbfbd1 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Fri, 27 Oct 2023 09:40:11 +0100 Subject: [PATCH 446/475] perf report: Fix spelling mistake "heirachy" -> "hierarchy" There is a spelling mistake in a ui error message. Fix it. Signed-off-by: Colin Ian King Cc: kernel-janitors@vger.kernel.org Link: https://lore.kernel.org/r/20231027084011.1167091-1-colin.i.king@gmail.com Signed-off-by: Namhyung Kim --- tools/perf/builtin-report.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/perf/builtin-report.c b/tools/perf/builtin-report.c index ca8f2331795c..9cb1da2dc0c0 100644 --- a/tools/perf/builtin-report.c +++ b/tools/perf/builtin-report.c @@ -705,7 +705,7 @@ static int report__collapse_hists(struct report *rep) */ if (perf_data__is_pipe(session->data)) { if (perf_hpp__setup_hists_formats(&perf_hpp_list, evlist) < 0) { - ui__error("Failed to setup hierachy output formats\n"); + ui__error("Failed to setup hierarchy output formats\n"); return -1; } } From ee40490dd7cdcda38ece6d081f63ecddd3fdbe25 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Fri, 27 Oct 2023 09:46:33 +0100 Subject: [PATCH 447/475] perf callchain: Fix spelling mistake "statisitcs" -> "statistics" There are a couple of spelling mistakes in perror messages. Fix them. Signed-off-by: Colin Ian King Cc: kernel-janitors@vger.kernel.org Link: https://lore.kernel.org/r/20231027084633.1167530-1-colin.i.king@gmail.com Signed-off-by: Namhyung Kim --- tools/perf/util/callchain.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/perf/util/callchain.c b/tools/perf/util/callchain.c index 99cce43ba152..8262f69118db 100644 --- a/tools/perf/util/callchain.c +++ b/tools/perf/util/callchain.c @@ -605,7 +605,7 @@ fill_node(struct callchain_node *node, struct callchain_cursor *cursor) if (!call->brtype_stat) { call->brtype_stat = zalloc(sizeof(*call->brtype_stat)); if (!call->brtype_stat) { - perror("not enough memory for the code path branch statisitcs"); + perror("not enough memory for the code path branch statistics"); free(call->brtype_stat); return -ENOMEM; } @@ -774,7 +774,7 @@ static enum match_result match_chain(struct callchain_cursor_node *node, if (!cnode->brtype_stat) { cnode->brtype_stat = zalloc(sizeof(*cnode->brtype_stat)); if (!cnode->brtype_stat) { - perror("not enough memory for the code path branch statisitcs"); + perror("not enough memory for the code path branch statistics"); return MATCH_ERROR; } } From 1768d3a0144c7aae55b9cee66dabf94946eec01e Mon Sep 17 00:00:00 2001 From: Arnaldo Carvalho de Melo Date: Fri, 27 Oct 2023 11:18:47 -0300 Subject: [PATCH 448/475] perf build: Disable BPF skeletons if clang version is < 12.0.1 While building on a wide range of distros and clang versions it was noticed that at least version 12.0.1 (noticed on Alpine 3.15 with "Alpine clang version 12.0.1") is needed to not fail with BTF generation errors such as: Debian:10 Debian clang version 11.0.1-2~deb10u1: CLANG /tmp/build/perf/util/bpf_skel/.tmp/sample_filter.bpf.o GENSKEL /tmp/build/perf/util/bpf_skel/sample_filter.skel.h libbpf: failed to find BTF for extern 'bpf_cast_to_kern_ctx' [21] section: -2 Error: failed to open BPF object file: No such file or directory make[2]: *** [Makefile.perf:1121: /tmp/build/perf/util/bpf_skel/sample_filter.skel.h] Error 254 make[2]: *** Deleting file '/tmp/build/perf/util/bpf_skel/sample_filter.skel.h' Amazon Linux 2: clang version 11.1.0 (Amazon Linux 2 11.1.0-1.amzn2.0.2) GENSKEL /tmp/build/perf/util/bpf_skel/sample_filter.skel.h libbpf: elf: skipping unrecognized data section(18) .eh_frame libbpf: elf: skipping relo section(19) .rel.eh_frame for section(18) .eh_frame libbpf: failed to find BTF for extern 'bpf_cast_to_kern_ctx' [21] section: -2 Error: failed to open BPF object file: No such file or directory make[2]: *** [/tmp/build/perf/util/bpf_skel/sample_filter.skel.h] Error 254 make[2]: *** Deleting file `/tmp/build/perf/util/bpf_skel/sample_filter.skel.h' Ubuntu 20.04: clang version 10.0.0-4ubuntu1 CLANG /tmp/build/perf/util/bpf_skel/.tmp/augmented_raw_syscalls.bpf.o GENSKEL /tmp/build/perf/util/bpf_skel/bench_uprobe.skel.h GENSKEL /tmp/build/perf/util/bpf_skel/bperf_leader.skel.h libbpf: sec '.reluprobe': corrupted symbol #27 pointing to invalid section #65522 for relo #0 GENSKEL /tmp/build/perf/util/bpf_skel/bperf_follower.skel.h Error: failed to open BPF object file: BPF object format invalid make[2]: *** [Makefile.perf:1121: /tmp/build/perf/util/bpf_skel/bench_uprobe.skel.h] Error 95 make[2]: *** Deleting file '/tmp/build/perf/util/bpf_skel/bench_uprobe.skel.h' So check if the version is at least 12.0.1 otherwise disable building BPF skels and provide a message about it, continuing the build. The message, when running on amazonlinux:2: Makefile.config:698: Warning: Disabled BPF skeletons as reliable BTF generation needs at least clang version 12.0.1 Signed-off-by: Arnaldo Carvalho de Melo Reviewed-by: Ian Rogers Link: https://lore.kernel.org/r/ZTvGx/Ou6BVnYBqi@kernel.org Signed-off-by: Namhyung Kim --- tools/perf/Makefile.config | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tools/perf/Makefile.config b/tools/perf/Makefile.config index f5ccbfc1a444..b3e6ed10f40c 100644 --- a/tools/perf/Makefile.config +++ b/tools/perf/Makefile.config @@ -693,6 +693,13 @@ ifeq ($(BUILD_BPF_SKEL),1) dummy := $(warning Warning: Disabled BPF skeletons as clang ($(CLANG)) is missing) BUILD_BPF_SKEL := 0 else + CLANG_VERSION := $(shell $(CLANG) --version | head -1 | sed 's/.*clang version \([[:digit:]]\+.[[:digit:]]\+.[[:digit:]]\+\).*/\1/g') + ifeq ($(call version-lt3,$(CLANG_VERSION),12.0.1),1) + dummy := $(warning Warning: Disabled BPF skeletons as reliable BTF generation needs at least $(CLANG) version 12.0.1) + BUILD_BPF_SKEL := 0 + endif + endif + ifeq ($(BUILD_BPF_SKEL),1) $(call feature_check,clang-bpf-co-re) ifeq ($(feature-clang-bpf-co-re), 0) dummy := $(warning Warning: Disabled BPF skeletons as clang is too old) From a28a0f6773747a8759140578f1ce23d81c6ee0d9 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Wed, 25 Oct 2023 17:31:41 -0700 Subject: [PATCH 449/475] perf vendor events intel: Update alderlake/alderlake events to v1.23 Update alderlake and alderlaken events from v1.21 to v1.23 adding the changes from: https://github.com/intel/perfmon/commit/8df4db9433a2aab59dbbac1a70281032d1af7734 https://github.com/intel/perfmon/commit/846bd247c6e04acc572ca56c992e9e65852bbe63 The tsx_cycles_per_elision metric is updated from PR: https://github.com/intel/perfmon/pull/116 Signed-off-by: Ian Rogers Reviewed-by: Kan Liang Cc: Alexandre Torgue Cc: Maxime Coquelin Cc: Edward Baker Cc: Zhengjun Xing Link: https://lore.kernel.org/r/20231026003149.3287633-1-irogers@google.com Signed-off-by: Namhyung Kim --- .../arch/x86/alderlake/adl-metrics.json | 2 +- .../arch/x86/alderlake/frontend.json | 44 +++++++++++-- .../pmu-events/arch/x86/alderlake/memory.json | 4 +- .../arch/x86/alderlake/pipeline.json | 20 +++++- .../x86/alderlake/uncore-interconnect.json | 2 + .../arch/x86/alderlaken/memory.json | 4 +- .../arch/x86/alderlaken/pipeline.json | 16 +++++ .../x86/alderlaken/uncore-interconnect.json | 66 +++++++++++++++++++ tools/perf/pmu-events/arch/x86/mapfile.csv | 4 +- 9 files changed, 147 insertions(+), 15 deletions(-) diff --git a/tools/perf/pmu-events/arch/x86/alderlake/adl-metrics.json b/tools/perf/pmu-events/arch/x86/alderlake/adl-metrics.json index 8b6bed3bc766..3388b58b8f1a 100644 --- a/tools/perf/pmu-events/arch/x86/alderlake/adl-metrics.json +++ b/tools/perf/pmu-events/arch/x86/alderlake/adl-metrics.json @@ -99,7 +99,7 @@ }, { "BriefDescription": "Number of cycles within a transaction divided by the number of elisions.", - "MetricExpr": "(cycles\\-t / el\\-start if has_event(cycles\\-t) else 0)", + "MetricExpr": "(cycles\\-t / el\\-start if has_event(el\\-start) else 0)", "MetricGroup": "transaction", "MetricName": "tsx_cycles_per_elision", "ScaleUnit": "1cycles / elision" diff --git a/tools/perf/pmu-events/arch/x86/alderlake/frontend.json b/tools/perf/pmu-events/arch/x86/alderlake/frontend.json index 81349100fe32..542ba4a81996 100644 --- a/tools/perf/pmu-events/arch/x86/alderlake/frontend.json +++ b/tools/perf/pmu-events/arch/x86/alderlake/frontend.json @@ -394,31 +394,61 @@ "Unit": "cpu_core" }, { - "BriefDescription": "Uops not delivered by IDQ when backend of the machine is not stalled", + "BriefDescription": "Uops not delivered by IDQ when backend of the machine is not stalled [This event is alias to IDQ_UOPS_NOT_DELIVERED.CORE]", "EventCode": "0x9c", - "EventName": "IDQ_UOPS_NOT_DELIVERED.CORE", - "PublicDescription": "Counts the number of uops not delivered to by the Instruction Decode Queue (IDQ) to the back-end of the pipeline when there was no back-end stalls. This event counts for one SMT thread in a given cycle.", + "EventName": "IDQ_BUBBLES.CORE", + "PublicDescription": "Counts the number of uops not delivered to by the Instruction Decode Queue (IDQ) to the back-end of the pipeline when there was no back-end stalls. This event counts for one SMT thread in a given cycle. [This event is alias to IDQ_UOPS_NOT_DELIVERED.CORE]", "SampleAfterValue": "1000003", "UMask": "0x1", "Unit": "cpu_core" }, { - "BriefDescription": "Cycles when no uops are not delivered by the IDQ when backend of the machine is not stalled", + "BriefDescription": "Cycles when no uops are not delivered by the IDQ when backend of the machine is not stalled [This event is alias to IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE]", + "CounterMask": "6", + "EventCode": "0x9c", + "EventName": "IDQ_BUBBLES.CYCLES_0_UOPS_DELIV.CORE", + "PublicDescription": "Counts the number of cycles when no uops were delivered by the Instruction Decode Queue (IDQ) to the back-end of the pipeline when there was no back-end stalls. This event counts for one SMT thread in a given cycle. [This event is alias to IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE]", + "SampleAfterValue": "1000003", + "UMask": "0x1", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Cycles when optimal number of uops was delivered to the back-end when the back-end is not stalled [This event is alias to IDQ_UOPS_NOT_DELIVERED.CYCLES_FE_WAS_OK]", + "CounterMask": "1", + "EventCode": "0x9c", + "EventName": "IDQ_BUBBLES.CYCLES_FE_WAS_OK", + "Invert": "1", + "PublicDescription": "Counts the number of cycles when the optimal number of uops were delivered by the Instruction Decode Queue (IDQ) to the back-end of the pipeline when there was no back-end stalls. This event counts for one SMT thread in a given cycle. [This event is alias to IDQ_UOPS_NOT_DELIVERED.CYCLES_FE_WAS_OK]", + "SampleAfterValue": "1000003", + "UMask": "0x1", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Uops not delivered by IDQ when backend of the machine is not stalled [This event is alias to IDQ_BUBBLES.CORE]", + "EventCode": "0x9c", + "EventName": "IDQ_UOPS_NOT_DELIVERED.CORE", + "PublicDescription": "Counts the number of uops not delivered to by the Instruction Decode Queue (IDQ) to the back-end of the pipeline when there was no back-end stalls. This event counts for one SMT thread in a given cycle. [This event is alias to IDQ_BUBBLES.CORE]", + "SampleAfterValue": "1000003", + "UMask": "0x1", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Cycles when no uops are not delivered by the IDQ when backend of the machine is not stalled [This event is alias to IDQ_BUBBLES.CYCLES_0_UOPS_DELIV.CORE]", "CounterMask": "6", "EventCode": "0x9c", "EventName": "IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE", - "PublicDescription": "Counts the number of cycles when no uops were delivered by the Instruction Decode Queue (IDQ) to the back-end of the pipeline when there was no back-end stalls. This event counts for one SMT thread in a given cycle.", + "PublicDescription": "Counts the number of cycles when no uops were delivered by the Instruction Decode Queue (IDQ) to the back-end of the pipeline when there was no back-end stalls. This event counts for one SMT thread in a given cycle. [This event is alias to IDQ_BUBBLES.CYCLES_0_UOPS_DELIV.CORE]", "SampleAfterValue": "1000003", "UMask": "0x1", "Unit": "cpu_core" }, { - "BriefDescription": "Cycles when optimal number of uops was delivered to the back-end when the back-end is not stalled", + "BriefDescription": "Cycles when optimal number of uops was delivered to the back-end when the back-end is not stalled [This event is alias to IDQ_BUBBLES.CYCLES_FE_WAS_OK]", "CounterMask": "1", "EventCode": "0x9c", "EventName": "IDQ_UOPS_NOT_DELIVERED.CYCLES_FE_WAS_OK", "Invert": "1", - "PublicDescription": "Counts the number of cycles when the optimal number of uops were delivered by the Instruction Decode Queue (IDQ) to the back-end of the pipeline when there was no back-end stalls. This event counts for one SMT thread in a given cycle.", + "PublicDescription": "Counts the number of cycles when the optimal number of uops were delivered by the Instruction Decode Queue (IDQ) to the back-end of the pipeline when there was no back-end stalls. This event counts for one SMT thread in a given cycle. [This event is alias to IDQ_BUBBLES.CYCLES_FE_WAS_OK]", "SampleAfterValue": "1000003", "UMask": "0x1", "Unit": "cpu_core" diff --git a/tools/perf/pmu-events/arch/x86/alderlake/memory.json b/tools/perf/pmu-events/arch/x86/alderlake/memory.json index 73d92d5c9f9d..23d36164433f 100644 --- a/tools/perf/pmu-events/arch/x86/alderlake/memory.json +++ b/tools/perf/pmu-events/arch/x86/alderlake/memory.json @@ -248,7 +248,7 @@ "Unit": "cpu_core" }, { - "BriefDescription": "Counts demand data reads that were not supplied by the L3 cache.", + "BriefDescription": "Counts demand data reads that were not supplied by the L3 cache. [L3_MISS_LOCAL is alias to L3_MISS]", "EventCode": "0xB7", "EventName": "OCR.DEMAND_DATA_RD.L3_MISS_LOCAL", "MSRIndex": "0x1a6,0x1a7", @@ -278,7 +278,7 @@ "Unit": "cpu_core" }, { - "BriefDescription": "Counts demand reads for ownership (RFO) and software prefetches for exclusive ownership (PREFETCHW) that were not supplied by the L3 cache.", + "BriefDescription": "Counts demand reads for ownership (RFO) and software prefetches for exclusive ownership (PREFETCHW) that were not supplied by the L3 cache. [L3_MISS_LOCAL is alias to L3_MISS]", "EventCode": "0xB7", "EventName": "OCR.DEMAND_RFO.L3_MISS_LOCAL", "MSRIndex": "0x1a6,0x1a7", diff --git a/tools/perf/pmu-events/arch/x86/alderlake/pipeline.json b/tools/perf/pmu-events/arch/x86/alderlake/pipeline.json index a92013cdf136..f9876bef16da 100644 --- a/tools/perf/pmu-events/arch/x86/alderlake/pipeline.json +++ b/tools/perf/pmu-events/arch/x86/alderlake/pipeline.json @@ -238,6 +238,15 @@ "UMask": "0x8", "Unit": "cpu_core" }, + { + "BriefDescription": "Counts the number of near taken branch instructions retired.", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.NEAR_TAKEN", + "PEBS": "1", + "SampleAfterValue": "200003", + "UMask": "0xc0", + "Unit": "cpu_atom" + }, { "BriefDescription": "Taken branch instructions retired.", "EventCode": "0xc4", @@ -411,6 +420,15 @@ "UMask": "0x7e", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of mispredicted near taken branch instructions retired.", + "EventCode": "0xc5", + "EventName": "BR_MISP_RETIRED.NEAR_TAKEN", + "PEBS": "1", + "SampleAfterValue": "200003", + "UMask": "0x80", + "Unit": "cpu_atom" + }, { "BriefDescription": "Number of near branch instructions retired that were mispredicted and taken.", "EventCode": "0xc5", @@ -842,7 +860,7 @@ "Unit": "cpu_core" }, { - "BriefDescription": "INT_MISC.UNKNOWN_BRANCH_CYCLES", + "BriefDescription": "Bubble cycles of BAClear (Unknown Branch).", "EventCode": "0xad", "EventName": "INT_MISC.UNKNOWN_BRANCH_CYCLES", "MSRIndex": "0x3F7", diff --git a/tools/perf/pmu-events/arch/x86/alderlake/uncore-interconnect.json b/tools/perf/pmu-events/arch/x86/alderlake/uncore-interconnect.json index 34fc052d00e4..8bf020a9dfa8 100644 --- a/tools/perf/pmu-events/arch/x86/alderlake/uncore-interconnect.json +++ b/tools/perf/pmu-events/arch/x86/alderlake/uncore-interconnect.json @@ -25,6 +25,7 @@ }, { "BriefDescription": "This event is deprecated. Refer to new event UNC_ARB_REQ_TRK_REQUEST.DRD", + "Deprecated": "1", "EventCode": "0x81", "EventName": "UNC_ARB_DAT_REQUESTS.RD", "PerPkg": "1", @@ -33,6 +34,7 @@ }, { "BriefDescription": "This event is deprecated. Refer to new event UNC_ARB_DAT_OCCUPANCY.ALL", + "Deprecated": "1", "EventCode": "0x85", "EventName": "UNC_ARB_IFA_OCCUPANCY.ALL", "PerPkg": "1", diff --git a/tools/perf/pmu-events/arch/x86/alderlaken/memory.json b/tools/perf/pmu-events/arch/x86/alderlaken/memory.json index 37259d38a222..863a3ba2b4b2 100644 --- a/tools/perf/pmu-events/arch/x86/alderlaken/memory.json +++ b/tools/perf/pmu-events/arch/x86/alderlaken/memory.json @@ -59,7 +59,7 @@ "UMask": "0x1" }, { - "BriefDescription": "Counts demand data reads that were not supplied by the L3 cache.", + "BriefDescription": "Counts demand data reads that were not supplied by the L3 cache. [L3_MISS_LOCAL is alias to L3_MISS]", "EventCode": "0xB7", "EventName": "OCR.DEMAND_DATA_RD.L3_MISS_LOCAL", "MSRIndex": "0x1a6,0x1a7", @@ -77,7 +77,7 @@ "UMask": "0x1" }, { - "BriefDescription": "Counts demand reads for ownership (RFO) and software prefetches for exclusive ownership (PREFETCHW) that were not supplied by the L3 cache.", + "BriefDescription": "Counts demand reads for ownership (RFO) and software prefetches for exclusive ownership (PREFETCHW) that were not supplied by the L3 cache. [L3_MISS_LOCAL is alias to L3_MISS]", "EventCode": "0xB7", "EventName": "OCR.DEMAND_RFO.L3_MISS_LOCAL", "MSRIndex": "0x1a6,0x1a7", diff --git a/tools/perf/pmu-events/arch/x86/alderlaken/pipeline.json b/tools/perf/pmu-events/arch/x86/alderlaken/pipeline.json index fa53ff11a509..3153bab527a9 100644 --- a/tools/perf/pmu-events/arch/x86/alderlaken/pipeline.json +++ b/tools/perf/pmu-events/arch/x86/alderlaken/pipeline.json @@ -90,6 +90,14 @@ "SampleAfterValue": "200003", "UMask": "0xf7" }, + { + "BriefDescription": "Counts the number of near taken branch instructions retired.", + "EventCode": "0xc4", + "EventName": "BR_INST_RETIRED.NEAR_TAKEN", + "PEBS": "1", + "SampleAfterValue": "200003", + "UMask": "0xc0" + }, { "BriefDescription": "This event is deprecated. Refer to new event BR_INST_RETIRED.INDIRECT", "Deprecated": "1", @@ -183,6 +191,14 @@ "SampleAfterValue": "200003", "UMask": "0x7e" }, + { + "BriefDescription": "Counts the number of mispredicted near taken branch instructions retired.", + "EventCode": "0xc5", + "EventName": "BR_MISP_RETIRED.NEAR_TAKEN", + "PEBS": "1", + "SampleAfterValue": "200003", + "UMask": "0x80" + }, { "BriefDescription": "This event is deprecated. Refer to new event BR_MISP_RETIRED.INDIRECT", "Deprecated": "1", diff --git a/tools/perf/pmu-events/arch/x86/alderlaken/uncore-interconnect.json b/tools/perf/pmu-events/arch/x86/alderlaken/uncore-interconnect.json index 4af695a5e755..8bf020a9dfa8 100644 --- a/tools/perf/pmu-events/arch/x86/alderlaken/uncore-interconnect.json +++ b/tools/perf/pmu-events/arch/x86/alderlaken/uncore-interconnect.json @@ -7,6 +7,56 @@ "UMask": "0x1", "Unit": "ARB" }, + { + "BriefDescription": "Each cycle counts number of any coherent request at memory controller that were issued by any core.", + "EventCode": "0x85", + "EventName": "UNC_ARB_DAT_OCCUPANCY.ALL", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "ARB" + }, + { + "BriefDescription": "Each cycle counts number of coherent reads pending on data return from memory controller that were issued by any core.", + "EventCode": "0x85", + "EventName": "UNC_ARB_DAT_OCCUPANCY.RD", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "ARB" + }, + { + "BriefDescription": "This event is deprecated. Refer to new event UNC_ARB_REQ_TRK_REQUEST.DRD", + "Deprecated": "1", + "EventCode": "0x81", + "EventName": "UNC_ARB_DAT_REQUESTS.RD", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "ARB" + }, + { + "BriefDescription": "This event is deprecated. Refer to new event UNC_ARB_DAT_OCCUPANCY.ALL", + "Deprecated": "1", + "EventCode": "0x85", + "EventName": "UNC_ARB_IFA_OCCUPANCY.ALL", + "PerPkg": "1", + "UMask": "0x1", + "Unit": "ARB" + }, + { + "BriefDescription": "Each cycle count number of 'valid' coherent Data Read entries . Such entry is defined as valid when it is allocated till deallocation. Doesn't include prefetches [This event is alias to UNC_ARB_TRK_OCCUPANCY.RD]", + "EventCode": "0x80", + "EventName": "UNC_ARB_REQ_TRK_OCCUPANCY.DRD", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "ARB" + }, + { + "BriefDescription": "Number of all coherent Data Read entries. Doesn't include prefetches [This event is alias to UNC_ARB_TRK_REQUESTS.RD]", + "EventCode": "0x81", + "EventName": "UNC_ARB_REQ_TRK_REQUEST.DRD", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "ARB" + }, { "BriefDescription": "Each cycle counts number of all outgoing valid entries in ReqTrk. Such entry is defined as valid from its allocation in ReqTrk till deallocation. Accounts for Coherent and non-coherent traffic.", "EventCode": "0x80", @@ -15,6 +65,14 @@ "UMask": "0x1", "Unit": "ARB" }, + { + "BriefDescription": "Each cycle count number of 'valid' coherent Data Read entries . Such entry is defined as valid when it is allocated till deallocation. Doesn't include prefetches [This event is alias to UNC_ARB_REQ_TRK_OCCUPANCY.DRD]", + "EventCode": "0x80", + "EventName": "UNC_ARB_TRK_OCCUPANCY.RD", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "ARB" + }, { "BriefDescription": "Counts the number of coherent and in-coherent requests initiated by IA cores, processor graphic units, or LLC.", "EventCode": "0x81", @@ -22,5 +80,13 @@ "PerPkg": "1", "UMask": "0x1", "Unit": "ARB" + }, + { + "BriefDescription": "Number of all coherent Data Read entries. Doesn't include prefetches [This event is alias to UNC_ARB_REQ_TRK_REQUEST.DRD]", + "EventCode": "0x81", + "EventName": "UNC_ARB_TRK_REQUESTS.RD", + "PerPkg": "1", + "UMask": "0x2", + "Unit": "ARB" } ] diff --git a/tools/perf/pmu-events/arch/x86/mapfile.csv b/tools/perf/pmu-events/arch/x86/mapfile.csv index 57ba7e814563..c09b81d8d5e1 100644 --- a/tools/perf/pmu-events/arch/x86/mapfile.csv +++ b/tools/perf/pmu-events/arch/x86/mapfile.csv @@ -1,6 +1,6 @@ Family-model,Version,Filename,EventType -GenuineIntel-6-(97|9A|B7|BA|BF),v1.21,alderlake,core -GenuineIntel-6-BE,v1.21,alderlaken,core +GenuineIntel-6-(97|9A|B7|BA|BF),v1.23,alderlake,core +GenuineIntel-6-BE,v1.23,alderlaken,core GenuineIntel-6-(1C|26|27|35|36),v4,bonnell,core GenuineIntel-6-(3D|47),v28,broadwell,core GenuineIntel-6-56,v11,broadwellde,core From 8a94d3bfaf45e7995ef12be1a51ec47684c7cb64 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Wed, 25 Oct 2023 17:31:42 -0700 Subject: [PATCH 450/475] perf vendor events intel: Update emeraldrapids to v1.01 Update emeraldrapids to v1.01 from v1.00 adding the changes from: https://github.com/intel/perfmon/commit/3993b600e032a9fd443ffd828aab73de7cb167e5 Signed-off-by: Ian Rogers Reviewed-by: Kan Liang Cc: Alexandre Torgue Cc: Maxime Coquelin Cc: Edward Baker Cc: Zhengjun Xing Link: https://lore.kernel.org/r/20231026003149.3287633-2-irogers@google.com Signed-off-by: Namhyung Kim --- .../arch/x86/emeraldrapids/uncore-cache.json | 36 +++++++++++++++++++ tools/perf/pmu-events/arch/x86/mapfile.csv | 2 +- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/tools/perf/pmu-events/arch/x86/emeraldrapids/uncore-cache.json b/tools/perf/pmu-events/arch/x86/emeraldrapids/uncore-cache.json index cf6fa70f37c1..bf5a511b99d1 100644 --- a/tools/perf/pmu-events/arch/x86/emeraldrapids/uncore-cache.json +++ b/tools/perf/pmu-events/arch/x86/emeraldrapids/uncore-cache.json @@ -3448,6 +3448,15 @@ "UMask": "0x10c8168201", "Unit": "CHA" }, + { + "BriefDescription": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD_CXL_EXP_LOCAL", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_DRD_CXL_EXP_LOCAL", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x20c8168201", + "Unit": "CHA" + }, { "BriefDescription": "TOR Inserts for DRds issued by IA Cores targeting DDR Mem that Missed the LLC", "EventCode": "0x35", @@ -3838,6 +3847,15 @@ "UMask": "0x10c8068201", "Unit": "CHA" }, + { + "BriefDescription": "UNC_CHA_TOR_INSERTS.IA_MISS_RFO_CXL_EXP_LOCAL", + "EventCode": "0x35", + "EventName": "UNC_CHA_TOR_INSERTS.IA_MISS_RFO_CXL_EXP_LOCAL", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x20c8068201", + "Unit": "CHA" + }, { "BriefDescription": "TOR Inserts RFO misses from local IA", "EventCode": "0x35", @@ -4877,6 +4895,15 @@ "UMask": "0x10c8168201", "Unit": "CHA" }, + { + "BriefDescription": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_CXL_EXP_LOCAL", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_CXL_EXP_LOCAL", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x20c8168201", + "Unit": "CHA" + }, { "BriefDescription": "TOR Occupancy for DRds issued by iA Cores targeting DDR Mem that Missed the LLC", "EventCode": "0x36", @@ -5267,6 +5294,15 @@ "UMask": "0x10c8068201", "Unit": "CHA" }, + { + "BriefDescription": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_RFO_CXL_EXP_LOCAL", + "EventCode": "0x36", + "EventName": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_RFO_CXL_EXP_LOCAL", + "PerPkg": "1", + "PortMask": "0x000", + "UMask": "0x20c8068201", + "Unit": "CHA" + }, { "BriefDescription": "TOR Occupancy; RFO misses from local IA", "EventCode": "0x36", diff --git a/tools/perf/pmu-events/arch/x86/mapfile.csv b/tools/perf/pmu-events/arch/x86/mapfile.csv index c09b81d8d5e1..7093561389a1 100644 --- a/tools/perf/pmu-events/arch/x86/mapfile.csv +++ b/tools/perf/pmu-events/arch/x86/mapfile.csv @@ -7,7 +7,7 @@ GenuineIntel-6-56,v11,broadwellde,core GenuineIntel-6-4F,v22,broadwellx,core GenuineIntel-6-55-[56789ABCDEF],v1.20,cascadelakex,core GenuineIntel-6-9[6C],v1.04,elkhartlake,core -GenuineIntel-6-CF,v1.00,emeraldrapids,core +GenuineIntel-6-CF,v1.01,emeraldrapids,core GenuineIntel-6-5[CF],v13,goldmont,core GenuineIntel-6-7A,v1.01,goldmontplus,core GenuineIntel-6-B6,v1.00,grandridge,core From 99a8a4c990f5a16c9971f10bc44e4894e0045d2a Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Wed, 25 Oct 2023 17:31:43 -0700 Subject: [PATCH 451/475] perf vendor events intel: Update a spelling in haswell/haswellx The spelling of "in-flight" was switched to "inflight". Signed-off-by: Ian Rogers Reviewed-by: Kan Liang Cc: Alexandre Torgue Cc: Maxime Coquelin Cc: Edward Baker Cc: Zhengjun Xing Link: https://lore.kernel.org/r/20231026003149.3287633-3-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/pmu-events/arch/x86/haswell/memory.json | 2 +- tools/perf/pmu-events/arch/x86/haswellx/memory.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/perf/pmu-events/arch/x86/haswell/memory.json b/tools/perf/pmu-events/arch/x86/haswell/memory.json index df44c28efeeb..2fc25e22a42a 100644 --- a/tools/perf/pmu-events/arch/x86/haswell/memory.json +++ b/tools/perf/pmu-events/arch/x86/haswell/memory.json @@ -62,7 +62,7 @@ "BriefDescription": "Counts the number of machine clears due to memory order conflicts.", "EventCode": "0xC3", "EventName": "MACHINE_CLEARS.MEMORY_ORDERING", - "PublicDescription": "This event counts the number of memory ordering machine clears detected. Memory ordering machine clears can result from memory address aliasing or snoops from another hardware thread or core to data in-flight in the pipeline. Machine clears can have a significant performance impact if they are happening frequently.", + "PublicDescription": "This event counts the number of memory ordering machine clears detected. Memory ordering machine clears can result from memory address aliasing or snoops from another hardware thread or core to data inflight in the pipeline. Machine clears can have a significant performance impact if they are happening frequently.", "SampleAfterValue": "100003", "UMask": "0x2" }, diff --git a/tools/perf/pmu-events/arch/x86/haswellx/memory.json b/tools/perf/pmu-events/arch/x86/haswellx/memory.json index d66e465ce41a..2d212cf59e92 100644 --- a/tools/perf/pmu-events/arch/x86/haswellx/memory.json +++ b/tools/perf/pmu-events/arch/x86/haswellx/memory.json @@ -62,7 +62,7 @@ "BriefDescription": "Counts the number of machine clears due to memory order conflicts.", "EventCode": "0xC3", "EventName": "MACHINE_CLEARS.MEMORY_ORDERING", - "PublicDescription": "This event counts the number of memory ordering machine clears detected. Memory ordering machine clears can result from memory address aliasing or snoops from another hardware thread or core to data in-flight in the pipeline. Machine clears can have a significant performance impact if they are happening frequently.", + "PublicDescription": "This event counts the number of memory ordering machine clears detected. Memory ordering machine clears can result from memory address aliasing or snoops from another hardware thread or core to data inflight in the pipeline. Machine clears can have a significant performance impact if they are happening frequently.", "SampleAfterValue": "100003", "UMask": "0x2" }, From 20e6a51f61bc061f8944b62288057098117b5dfb Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Wed, 25 Oct 2023 17:31:44 -0700 Subject: [PATCH 452/475] perf vendor events intel: Add typo fix for ivybridge FP Add a missed space. Signed-off-by: Ian Rogers Reviewed-by: Kan Liang Cc: Alexandre Torgue Cc: Maxime Coquelin Cc: Edward Baker Cc: Zhengjun Xing Link: https://lore.kernel.org/r/20231026003149.3287633-4-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/pmu-events/arch/x86/ivybridge/floating-point.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/perf/pmu-events/arch/x86/ivybridge/floating-point.json b/tools/perf/pmu-events/arch/x86/ivybridge/floating-point.json index 87c958213c7a..89c6d47cc077 100644 --- a/tools/perf/pmu-events/arch/x86/ivybridge/floating-point.json +++ b/tools/perf/pmu-events/arch/x86/ivybridge/floating-point.json @@ -73,7 +73,7 @@ "UMask": "0x20" }, { - "BriefDescription": "Number of FP Computational Uops Executed this cycle. The number of FADD, FSUB, FCOM, FMULs, integer MULsand IMULs, FDIVs, FPREMs, FSQRTS, integer DIVs, and IDIVs. This event does not distinguish an FADD used in the middle of a transcendental flow from a s", + "BriefDescription": "Number of FP Computational Uops Executed this cycle. The number of FADD, FSUB, FCOM, FMULs, integer MULs and IMULs, FDIVs, FPREMs, FSQRTS, integer DIVs, and IDIVs. This event does not distinguish an FADD used in the middle of a transcendental flow from a s", "EventCode": "0x10", "EventName": "FP_COMP_OPS_EXE.X87", "PublicDescription": "Counts number of X87 uops executed.", From f9418b524d14f20c57444f5609f5603b45fffa09 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Wed, 25 Oct 2023 17:31:45 -0700 Subject: [PATCH 453/475] perf vendor events intel: Update knightslanding events to v16 Update knightslanding from v10 to v16 adding the changes from: https://github.com/intel/perfmon/commit/6c1f169f6ed63ee1fd75ebb303d0fd06d71196f5 https://github.com/intel/perfmon/commit/b22ca587ec8b5ac20471ea2f14924f63e63afe9d https://github.com/intel/perfmon/commit/e685286f083ee81cb7dafd0cd8546c79ee433187 Signed-off-by: Ian Rogers Reviewed-by: Kan Liang Cc: Alexandre Torgue Cc: Maxime Coquelin Cc: Edward Baker Cc: Zhengjun Xing Link: https://lore.kernel.org/r/20231026003149.3287633-5-irogers@google.com Signed-off-by: Namhyung Kim --- .../arch/x86/knightslanding/cache.json | 39 ++++++++----- .../x86/knightslanding/floating-point.json | 8 +-- .../arch/x86/knightslanding/pipeline.json | 55 +++++++++++-------- .../arch/x86/knightslanding/uncore-cache.json | 26 ++++----- .../x86/knightslanding/virtual-memory.json | 2 +- tools/perf/pmu-events/arch/x86/mapfile.csv | 2 +- 6 files changed, 75 insertions(+), 57 deletions(-) diff --git a/tools/perf/pmu-events/arch/x86/knightslanding/cache.json b/tools/perf/pmu-events/arch/x86/knightslanding/cache.json index d9876cb06b08..8da3a5a7be73 100644 --- a/tools/perf/pmu-events/arch/x86/knightslanding/cache.json +++ b/tools/perf/pmu-events/arch/x86/knightslanding/cache.json @@ -6,13 +6,19 @@ "SampleAfterValue": "200003" }, { - "BriefDescription": "Counts the number of core cycles the fetch stalls because of an icache miss. This is a cumulative count of core cycles the fetch stalled for all icache misses.", + "BriefDescription": "This event counts the number of core cycles the fetch stalls because of an icache miss. This is a cumulative count of cycles the NIP stalled for all icache misses.", "EventCode": "0x86", "EventName": "FETCH_STALL.ICACHE_FILL_PENDING_CYCLES", - "PublicDescription": "This event counts the number of core cycles the fetch stalls because of an icache miss. This is a cumulative count of cycles the NIP stalled for all icache misses.", "SampleAfterValue": "200003", "UMask": "0x4" }, + { + "BriefDescription": "Counts the number of L2HWP allocated into XQ GP", + "EventCode": "0x3E", + "EventName": "L2_PREFETCHER.ALLOC_XQ", + "SampleAfterValue": "100007", + "UMask": "0x4" + }, { "BriefDescription": "Counts the number of L2 cache misses", "EventCode": "0x2E", @@ -28,7 +34,7 @@ "UMask": "0x4f" }, { - "BriefDescription": "Counts the number of MEC requests from the L2Q that reference a cache line (cacheable requests) excluding SW prefetches filling only to L2 cache and L1 evictions (automatically exlcudes L2HWP, UC, WC) that were rejected - Multiple repeated rejects should be counted multiple times", + "BriefDescription": "Counts the number of MEC requests from the L2Q that reference a cache line (cacheable requests) excluding SW prefetches filling only to L2 cache and L1 evictions (automatically excludes L2HWP, UC, WC) that were rejected - Multiple repeated rejects should be counted multiple times", "EventCode": "0x30", "EventName": "L2_REQUESTS_REJECT.ALL", "SampleAfterValue": "200003" @@ -50,11 +56,12 @@ "UMask": "0x80" }, { - "BriefDescription": "Counts the loads retired that get the data from the other core in the same tile in M state", + "BriefDescription": "Counts the loads retired that get the data from the other core in the same tile in M state (Precise Event)", "Data_LA": "1", "EventCode": "0x04", "EventName": "MEM_UOPS_RETIRED.HITM", "PEBS": "1", + "PublicDescription": "This event counts the number of load micro-ops retired that got data from another core's cache. (Precise Event).", "SampleAfterValue": "200003", "UMask": "0x20" }, @@ -67,20 +74,22 @@ "UMask": "0x1" }, { - "BriefDescription": "Counts the number of load micro-ops retired that hit in the L2", + "BriefDescription": "Counts the number of load micro-ops retired that hit in the L2 (Precise Event)", "Data_LA": "1", "EventCode": "0x04", "EventName": "MEM_UOPS_RETIRED.L2_HIT_LOADS", "PEBS": "1", + "PublicDescription": "This event counts the number of load micro-uops retired that hit in the L2 (Precise Event)", "SampleAfterValue": "200003", "UMask": "0x2" }, { - "BriefDescription": "Counts the number of load micro-ops retired that miss in the L2", + "BriefDescription": "Counts the number of load micro-ops retired that miss in the L2 (Precise Event)", "Data_LA": "1", "EventCode": "0x04", "EventName": "MEM_UOPS_RETIRED.L2_MISS_LOADS", "PEBS": "1", + "PublicDescription": "This event counts the number of load micro-ops retired that miss in the L2 (Precise Event)", "SampleAfterValue": "100007", "UMask": "0x4" }, @@ -620,6 +629,15 @@ "SampleAfterValue": "100007", "UMask": "0x1" }, + { + "BriefDescription": "Accounts for responses which miss its own tile's L2.", + "EventCode": "0xB7", + "EventName": "OFFCORE_RESPONSE.ANY_REQUEST.L2_MISS", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x18001981F8", + "SampleAfterValue": "100007", + "UMask": "0x1" + }, { "BriefDescription": "Counts any request that are outstanding, per weighted cycle, from the time of the request to when any response is received. The outstanding response should be programmed only on PMC0.", "EventCode": "0xB7", @@ -1664,15 +1682,6 @@ "SampleAfterValue": "100007", "UMask": "0x1" }, - { - "BriefDescription": "Counts L2 data RFO prefetches (includes PREFETCHW instruction) that provides no supplier details", - "EventCode": "0xB7", - "EventName": "OFFCORE_RESPONSE.PF_L2_RFO.SUPPLIER_NONE", - "MSRIndex": "0x1a6,0x1a7", - "MSRValue": "0x0000020020", - "SampleAfterValue": "100007", - "UMask": "0x1" - }, { "BriefDescription": "Counts Software Prefetches that accounts for any response", "EventCode": "0xB7", diff --git a/tools/perf/pmu-events/arch/x86/knightslanding/floating-point.json b/tools/perf/pmu-events/arch/x86/knightslanding/floating-point.json index ecc96f32f167..089aa3ef345d 100644 --- a/tools/perf/pmu-events/arch/x86/knightslanding/floating-point.json +++ b/tools/perf/pmu-events/arch/x86/knightslanding/floating-point.json @@ -8,18 +8,18 @@ "UMask": "0x4" }, { - "BriefDescription": "Counts the number of vector SSE, AVX, AVX2, AVX-512 micro-ops retired. More specifically, it counts packed SSE, AVX, AVX2, AVX-512 micro-ops (both floating point and integer) except for loads (memory-to-register mov-type micro-ops), packed byte and word multiplies.", + "BriefDescription": "Counts the number of packed SSE, AVX, AVX2, AVX-512 micro-ops (both floating point and integer) except for loads (memory-to-register mov-type micro-ops), packed byte and word multiplies.", "EventCode": "0xC2", "EventName": "UOPS_RETIRED.PACKED_SIMD", - "PublicDescription": "This event counts the number of packed vector SSE, AVX, AVX2, and AVX-512 micro-ops retired (floating point, integer and store) except for loads (memory-to-register mov-type micro-ops), packed byte and word multiplies.", + "PublicDescription": "The length of the packed operation (128bits, 256bits or 512bits) is not taken into account when updating the counter; all count the same (+1). \r\nMask (k) registers are ignored. For example: a micro-op operating with a mask that only enables one element or even zero elements will still trigger this counter (+1)\r\nThis event is defined at the micro-op level and not instruction level. Most instructions are implemented with one micro-op but not all.", "SampleAfterValue": "200003", "UMask": "0x40" }, { - "BriefDescription": "Counts the number of scalar SSE, AVX, AVX2, AVX-512 micro-ops retired. More specifically, it counts scalar SSE, AVX, AVX2, AVX-512 micro-ops except for loads (memory-to-register mov-type micro ops), division, sqrt.", + "BriefDescription": "Counts the number of scalar SSE, AVX, AVX2, AVX-512 micro-ops except for loads (memory-to-register mov-type micro ops), division, sqrt.", "EventCode": "0xC2", "EventName": "UOPS_RETIRED.SCALAR_SIMD", - "PublicDescription": "This event counts the number of scalar SSE, AVX, AVX2, AVX-512 micro-ops retired (floating point, integer and store) except for loads (memory-to-register mov-type micro ops), division, sqrt.", + "PublicDescription": "This event is defined at the micro-op level and not instruction level. Most instructions are implemented with one micro-op but not all.", "SampleAfterValue": "200003", "UMask": "0x20" } diff --git a/tools/perf/pmu-events/arch/x86/knightslanding/pipeline.json b/tools/perf/pmu-events/arch/x86/knightslanding/pipeline.json index 3dc532107ead..5b2e71750976 100644 --- a/tools/perf/pmu-events/arch/x86/knightslanding/pipeline.json +++ b/tools/perf/pmu-events/arch/x86/knightslanding/pipeline.json @@ -1,13 +1,13 @@ [ { - "BriefDescription": "Counts the number of branch instructions retired", + "BriefDescription": "Counts the number of branch instructions retired (Precise Event)", "EventCode": "0xC4", "EventName": "BR_INST_RETIRED.ALL_BRANCHES", "PEBS": "1", "SampleAfterValue": "200003" }, { - "BriefDescription": "Counts the number of near CALL branch instructions retired.", + "BriefDescription": "Counts the number of near CALL branch instructions retired. (Precise Event)", "EventCode": "0xC4", "EventName": "BR_INST_RETIRED.CALL", "PEBS": "1", @@ -15,7 +15,7 @@ "UMask": "0xf9" }, { - "BriefDescription": "Counts the number of far branch instructions retired.", + "BriefDescription": "Counts the number of far branch instructions retired. (Precise Event)", "EventCode": "0xC4", "EventName": "BR_INST_RETIRED.FAR_BRANCH", "PEBS": "1", @@ -23,7 +23,7 @@ "UMask": "0xbf" }, { - "BriefDescription": "Counts the number of near indirect CALL branch instructions retired.", + "BriefDescription": "Counts the number of near indirect CALL branch instructions retired. (Precise Event)", "EventCode": "0xC4", "EventName": "BR_INST_RETIRED.IND_CALL", "PEBS": "1", @@ -31,7 +31,7 @@ "UMask": "0xfb" }, { - "BriefDescription": "Counts the number of branch instructions retired that were conditional jumps.", + "BriefDescription": "Counts the number of branch instructions retired that were conditional jumps. (Precise Event)", "EventCode": "0xC4", "EventName": "BR_INST_RETIRED.JCC", "PEBS": "1", @@ -39,7 +39,7 @@ "UMask": "0x7e" }, { - "BriefDescription": "Counts the number of branch instructions retired that were near indirect CALL or near indirect JMP.", + "BriefDescription": "Counts the number of branch instructions retired that were near indirect CALL or near indirect JMP. (Precise Event)", "EventCode": "0xC4", "EventName": "BR_INST_RETIRED.NON_RETURN_IND", "PEBS": "1", @@ -47,7 +47,7 @@ "UMask": "0xeb" }, { - "BriefDescription": "Counts the number of near relative CALL branch instructions retired.", + "BriefDescription": "Counts the number of near relative CALL branch instructions retired. (Precise Event)", "EventCode": "0xC4", "EventName": "BR_INST_RETIRED.REL_CALL", "PEBS": "1", @@ -55,7 +55,7 @@ "UMask": "0xfd" }, { - "BriefDescription": "Counts the number of near RET branch instructions retired.", + "BriefDescription": "Counts the number of near RET branch instructions retired. (Precise Event)", "EventCode": "0xC4", "EventName": "BR_INST_RETIRED.RETURN", "PEBS": "1", @@ -63,7 +63,7 @@ "UMask": "0xf7" }, { - "BriefDescription": "Counts the number of branch instructions retired that were conditional jumps and predicted taken.", + "BriefDescription": "Counts the number of branch instructions retired that were conditional jumps and predicted taken. (Precise Event)", "EventCode": "0xC4", "EventName": "BR_INST_RETIRED.TAKEN_JCC", "PEBS": "1", @@ -71,14 +71,14 @@ "UMask": "0xfe" }, { - "BriefDescription": "Counts the number of mispredicted branch instructions retired", + "BriefDescription": "Counts the number of mispredicted branch instructions retired (Precise Event)", "EventCode": "0xC5", "EventName": "BR_MISP_RETIRED.ALL_BRANCHES", "PEBS": "1", "SampleAfterValue": "200003" }, { - "BriefDescription": "Counts the number of mispredicted near CALL branch instructions retired.", + "BriefDescription": "Counts the number of mispredicted near CALL branch instructions retired. (Precise Event)", "EventCode": "0xC5", "EventName": "BR_MISP_RETIRED.CALL", "PEBS": "1", @@ -86,7 +86,7 @@ "UMask": "0xf9" }, { - "BriefDescription": "Counts the number of mispredicted far branch instructions retired.", + "BriefDescription": "Counts the number of mispredicted far branch instructions retired. (Precise Event)", "EventCode": "0xC5", "EventName": "BR_MISP_RETIRED.FAR_BRANCH", "PEBS": "1", @@ -94,7 +94,7 @@ "UMask": "0xbf" }, { - "BriefDescription": "Counts the number of mispredicted near indirect CALL branch instructions retired.", + "BriefDescription": "Counts the number of mispredicted near indirect CALL branch instructions retired. (Precise Event)", "EventCode": "0xC5", "EventName": "BR_MISP_RETIRED.IND_CALL", "PEBS": "1", @@ -102,7 +102,7 @@ "UMask": "0xfb" }, { - "BriefDescription": "Counts the number of mispredicted branch instructions retired that were conditional jumps.", + "BriefDescription": "Counts the number of mispredicted branch instructions retired that were conditional jumps. (Precise Event)", "EventCode": "0xC5", "EventName": "BR_MISP_RETIRED.JCC", "PEBS": "1", @@ -110,7 +110,7 @@ "UMask": "0x7e" }, { - "BriefDescription": "Counts the number of mispredicted branch instructions retired that were near indirect CALL or near indirect JMP.", + "BriefDescription": "Counts the number of mispredicted branch instructions retired that were near indirect CALL or near indirect JMP. (Precise Event)", "EventCode": "0xC5", "EventName": "BR_MISP_RETIRED.NON_RETURN_IND", "PEBS": "1", @@ -118,7 +118,7 @@ "UMask": "0xeb" }, { - "BriefDescription": "Counts the number of mispredicted near relative CALL branch instructions retired.", + "BriefDescription": "Counts the number of mispredicted near relative CALL branch instructions retired. (Precise Event)", "EventCode": "0xC5", "EventName": "BR_MISP_RETIRED.REL_CALL", "PEBS": "1", @@ -126,7 +126,7 @@ "UMask": "0xfd" }, { - "BriefDescription": "Counts the number of mispredicted near RET branch instructions retired.", + "BriefDescription": "Counts the number of mispredicted near RET branch instructions retired. (Precise Event)", "EventCode": "0xC5", "EventName": "BR_MISP_RETIRED.RETURN", "PEBS": "1", @@ -134,7 +134,7 @@ "UMask": "0xf7" }, { - "BriefDescription": "Counts the number of mispredicted branch instructions retired that were conditional jumps and predicted taken.", + "BriefDescription": "Counts the number of mispredicted branch instructions retired that were conditional jumps and predicted taken. (Precise Event)", "EventCode": "0xC5", "EventName": "BR_MISP_RETIRED.TAKEN_JCC", "PEBS": "1", @@ -189,7 +189,14 @@ "SampleAfterValue": "2000003" }, { - "BriefDescription": "Counts all nukes", + "BriefDescription": "Counts the number of instructions retired (Precise Event)", + "EventCode": "0xC0", + "EventName": "INST_RETIRED.ANY_PS", + "PEBS": "2", + "SampleAfterValue": "2000003" + }, + { + "BriefDescription": "Counts all machine clears", "EventCode": "0xC3", "EventName": "MACHINE_CLEARS.ALL", "SampleAfterValue": "200003", @@ -261,20 +268,22 @@ "UMask": "0x2" }, { - "BriefDescription": "Counts the number of occurrences a retired load gets blocked because its address partially overlaps with a store", + "BriefDescription": "Counts the number of occurrences a retired load gets blocked because its address partially overlaps with a store (Precise Event)", "Data_LA": "1", "EventCode": "0x03", "EventName": "RECYCLEQ.LD_BLOCK_ST_FORWARD", "PEBS": "1", + "PublicDescription": "This event counts the number of retired loads that were prohibited from receiving forwarded data from a previous store because of address mismatch.", "SampleAfterValue": "200003", "UMask": "0x1" }, { - "BriefDescription": "Counts the number of occurrences a retired load that is a cache line split. Each split should be counted only once.", + "BriefDescription": "Counts the number of occurrences a retired load was pushed into the rehab queue because it sees a cache line split. Each split should be counted only once. (Precise Event)", "Data_LA": "1", "EventCode": "0x03", "EventName": "RECYCLEQ.LD_SPLITS", "PEBS": "1", + "PublicDescription": "This event counts the number of retired loads which was pushed into the recycled queue that experienced cache line boundary splits (Precise event). Not that each split should be counted only once.", "SampleAfterValue": "200003", "UMask": "0x8" }, @@ -286,7 +295,7 @@ "UMask": "0x10" }, { - "BriefDescription": "Counts the store micro-ops retired that were pushed in the rehad queue because the store address buffer is full", + "BriefDescription": "Counts the store micro-ops retired that were pushed in the rehab queue because the store address buffer is full", "EventCode": "0x03", "EventName": "RECYCLEQ.STA_FULL", "SampleAfterValue": "200003", @@ -301,7 +310,7 @@ "UMask": "0x4" }, { - "BriefDescription": "Counts the total number of core cycles the Alloc pipeline is stalled when any one of the reservation stations is full.", + "BriefDescription": "Counts the total number of core cycles allocation pipeline is stalled when any one of the reservation stations is full.", "EventCode": "0xCB", "EventName": "RS_FULL_STALL.ALL", "SampleAfterValue": "200003", diff --git a/tools/perf/pmu-events/arch/x86/knightslanding/uncore-cache.json b/tools/perf/pmu-events/arch/x86/knightslanding/uncore-cache.json index 1b8dcfa5461c..120e4813d82a 100644 --- a/tools/perf/pmu-events/arch/x86/knightslanding/uncore-cache.json +++ b/tools/perf/pmu-events/arch/x86/knightslanding/uncore-cache.json @@ -2558,7 +2558,7 @@ "Unit": "CHA" }, { - "BriefDescription": "REQUESTQ'' includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ)", + "BriefDescription": "REQUESTQ includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ)", "EventCode": "0x2A", "EventName": "UNC_H_INGRESS_RETRY_REQ_Q0_RETRY.AD_REQ_VN0", "PerPkg": "1", @@ -2566,7 +2566,7 @@ "Unit": "CHA" }, { - "BriefDescription": "REQUESTQ'' includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ)", + "BriefDescription": "REQUESTQ includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ)", "EventCode": "0x2A", "EventName": "UNC_H_INGRESS_RETRY_REQ_Q0_RETRY.AD_RSP_VN0", "PerPkg": "1", @@ -2574,7 +2574,7 @@ "Unit": "CHA" }, { - "BriefDescription": "REQUESTQ'' includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ)", + "BriefDescription": "REQUESTQ includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ)", "EventCode": "0x2A", "EventName": "UNC_H_INGRESS_RETRY_REQ_Q0_RETRY.AK_NON_UPI", "PerPkg": "1", @@ -2582,7 +2582,7 @@ "Unit": "CHA" }, { - "BriefDescription": "REQUESTQ'' includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ)", + "BriefDescription": "REQUESTQ includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ)", "EventCode": "0x2A", "EventName": "UNC_H_INGRESS_RETRY_REQ_Q0_RETRY.BL_NCB_VN0", "PerPkg": "1", @@ -2590,7 +2590,7 @@ "Unit": "CHA" }, { - "BriefDescription": "REQUESTQ'' includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ)", + "BriefDescription": "REQUESTQ includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ)", "EventCode": "0x2A", "EventName": "UNC_H_INGRESS_RETRY_REQ_Q0_RETRY.BL_NCS_VN0", "PerPkg": "1", @@ -2598,7 +2598,7 @@ "Unit": "CHA" }, { - "BriefDescription": "REQUESTQ'' includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ)", + "BriefDescription": "REQUESTQ includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ)", "EventCode": "0x2A", "EventName": "UNC_H_INGRESS_RETRY_REQ_Q0_RETRY.BL_RSP_VN0", "PerPkg": "1", @@ -2606,7 +2606,7 @@ "Unit": "CHA" }, { - "BriefDescription": "REQUESTQ'' includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ)", + "BriefDescription": "REQUESTQ includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ)", "EventCode": "0x2A", "EventName": "UNC_H_INGRESS_RETRY_REQ_Q0_RETRY.BL_WB_VN0", "PerPkg": "1", @@ -2614,7 +2614,7 @@ "Unit": "CHA" }, { - "BriefDescription": "REQUESTQ'' includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ)", + "BriefDescription": "REQUESTQ includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ)", "EventCode": "0x2A", "EventName": "UNC_H_INGRESS_RETRY_REQ_Q0_RETRY.IV_NON_UPI", "PerPkg": "1", @@ -2622,7 +2622,7 @@ "Unit": "CHA" }, { - "BriefDescription": "REQUESTQ'' includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ)", + "BriefDescription": "REQUESTQ includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ)", "EventCode": "0x2B", "EventName": "UNC_H_INGRESS_RETRY_REQ_Q1_RETRY.ALLOW_SNP", "PerPkg": "1", @@ -2630,7 +2630,7 @@ "Unit": "CHA" }, { - "BriefDescription": "REQUESTQ'' includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ)", + "BriefDescription": "REQUESTQ includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ)", "EventCode": "0x2B", "EventName": "UNC_H_INGRESS_RETRY_REQ_Q1_RETRY.ANY_REJECT_IRQ0", "PerPkg": "1", @@ -2638,7 +2638,7 @@ "Unit": "CHA" }, { - "BriefDescription": "REQUESTQ'' includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ)", + "BriefDescription": "REQUESTQ includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ)", "EventCode": "0x2B", "EventName": "UNC_H_INGRESS_RETRY_REQ_Q1_RETRY.PA_MATCH", "PerPkg": "1", @@ -2646,7 +2646,7 @@ "Unit": "CHA" }, { - "BriefDescription": "REQUESTQ'' includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ)", + "BriefDescription": "REQUESTQ includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ)", "EventCode": "0x2B", "EventName": "UNC_H_INGRESS_RETRY_REQ_Q1_RETRY.SF_VICTIM", "PerPkg": "1", @@ -2654,7 +2654,7 @@ "Unit": "CHA" }, { - "BriefDescription": "REQUESTQ'' includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ)", + "BriefDescription": "REQUESTQ includes: IRQ, PRQ, IPQ, RRQ, WBQ (everything except for ISMQ)", "EventCode": "0x2B", "EventName": "UNC_H_INGRESS_RETRY_REQ_Q1_RETRY.SF_WAY", "PerPkg": "1", diff --git a/tools/perf/pmu-events/arch/x86/knightslanding/virtual-memory.json b/tools/perf/pmu-events/arch/x86/knightslanding/virtual-memory.json index 99a8fa8f19cc..9be30a33b43b 100644 --- a/tools/perf/pmu-events/arch/x86/knightslanding/virtual-memory.json +++ b/tools/perf/pmu-events/arch/x86/knightslanding/virtual-memory.json @@ -1,6 +1,6 @@ [ { - "BriefDescription": "Counts the number of load micro-ops retired that cause a DTLB miss", + "BriefDescription": "Counts the number of load micro-ops retired that cause a DTLB miss (Precise Event)", "Data_LA": "1", "EventCode": "0x04", "EventName": "MEM_UOPS_RETIRED.DTLB_MISS_LOADS", diff --git a/tools/perf/pmu-events/arch/x86/mapfile.csv b/tools/perf/pmu-events/arch/x86/mapfile.csv index 7093561389a1..b25f9d90b2d7 100644 --- a/tools/perf/pmu-events/arch/x86/mapfile.csv +++ b/tools/perf/pmu-events/arch/x86/mapfile.csv @@ -19,7 +19,7 @@ GenuineIntel-6-6[AC],v1.21,icelakex,core GenuineIntel-6-3A,v24,ivybridge,core GenuineIntel-6-3E,v24,ivytown,core GenuineIntel-6-2D,v24,jaketown,core -GenuineIntel-6-(57|85),v10,knightslanding,core +GenuineIntel-6-(57|85),v16,knightslanding,core GenuineIntel-6-BD,v1.00,lunarlake,core GenuineIntel-6-A[AC],v1.04,meteorlake,core GenuineIntel-6-1[AEF],v4,nehalemep,core From 247730767c63b06a07272e8a3f06796f2df32d9c Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Wed, 25 Oct 2023 17:31:46 -0700 Subject: [PATCH 454/475] perf vendor events intel: Update meteorlake events to v1.06 Update meteorlake from v1.04 to v1.06 adding the changes from: https://github.com/intel/perfmon/commit/bc84df043091ec7c98c0629f3d074d9d7a108194 https://github.com/intel/perfmon/commit/405d3ee987d756b5b5d9a64d8a8fa77559822ecf Signed-off-by: Ian Rogers Reviewed-by: Kan Liang Cc: Alexandre Torgue Cc: Maxime Coquelin Cc: Edward Baker Cc: Zhengjun Xing Link: https://lore.kernel.org/r/20231026003149.3287633-6-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/pmu-events/arch/x86/mapfile.csv | 2 +- .../pmu-events/arch/x86/meteorlake/cache.json | 30 ++++++++ .../arch/x86/meteorlake/frontend.json | 29 ++++++-- .../arch/x86/meteorlake/memory.json | 37 ++++++++++ .../pmu-events/arch/x86/meteorlake/other.json | 40 +++++++++++ .../arch/x86/meteorlake/pipeline.json | 68 ++++++++++++++++++- .../arch/x86/meteorlake/uncore-other.json | 9 +++ 7 files changed, 209 insertions(+), 6 deletions(-) create mode 100644 tools/perf/pmu-events/arch/x86/meteorlake/uncore-other.json diff --git a/tools/perf/pmu-events/arch/x86/mapfile.csv b/tools/perf/pmu-events/arch/x86/mapfile.csv index b25f9d90b2d7..5b455739065b 100644 --- a/tools/perf/pmu-events/arch/x86/mapfile.csv +++ b/tools/perf/pmu-events/arch/x86/mapfile.csv @@ -21,7 +21,7 @@ GenuineIntel-6-3E,v24,ivytown,core GenuineIntel-6-2D,v24,jaketown,core GenuineIntel-6-(57|85),v16,knightslanding,core GenuineIntel-6-BD,v1.00,lunarlake,core -GenuineIntel-6-A[AC],v1.04,meteorlake,core +GenuineIntel-6-A[AC],v1.06,meteorlake,core GenuineIntel-6-1[AEF],v4,nehalemep,core GenuineIntel-6-2E,v4,nehalemex,core GenuineIntel-6-A7,v1.01,rocketlake,core diff --git a/tools/perf/pmu-events/arch/x86/meteorlake/cache.json b/tools/perf/pmu-events/arch/x86/meteorlake/cache.json index 1de0200b32f6..5fef87502d4b 100644 --- a/tools/perf/pmu-events/arch/x86/meteorlake/cache.json +++ b/tools/perf/pmu-events/arch/x86/meteorlake/cache.json @@ -966,6 +966,16 @@ "UMask": "0x3", "Unit": "cpu_core" }, + { + "BriefDescription": "Counts demand data reads that were supplied by the L3 cache where a snoop was sent, the snoop hit, and modified data was forwarded.", + "EventCode": "0xB7", + "EventName": "OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x10003C0001", + "SampleAfterValue": "100003", + "UMask": "0x1", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts demand data reads that resulted in a snoop hit in another cores caches, data forwarding is required as the data is modified.", "EventCode": "0x2A,0x2B", @@ -976,6 +986,16 @@ "UMask": "0x1", "Unit": "cpu_core" }, + { + "BriefDescription": "Counts demand data reads that were supplied by the L3 cache where a snoop was sent, the snoop hit, and non-modified data was forwarded.", + "EventCode": "0xB7", + "EventName": "OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HIT_WITH_FWD", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x8003C0001", + "SampleAfterValue": "100003", + "UMask": "0x1", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts demand data reads that resulted in a snoop hit in another cores caches which forwarded the unmodified data to the requesting core.", "EventCode": "0x2A,0x2B", @@ -986,6 +1006,16 @@ "UMask": "0x1", "Unit": "cpu_core" }, + { + "BriefDescription": "Counts demand reads for ownership (RFO) and software prefetches for exclusive ownership (PREFETCHW) that were supplied by the L3 cache where a snoop was sent, the snoop hit, and modified data was forwarded.", + "EventCode": "0xB7", + "EventName": "OCR.DEMAND_RFO.L3_HIT.SNOOP_HITM", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x10003C0002", + "SampleAfterValue": "100003", + "UMask": "0x1", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts demand read for ownership (RFO) requests and software prefetches for exclusive ownership (PREFETCHW) that resulted in a snoop hit in another cores caches, data forwarding is required as the data is modified.", "EventCode": "0x2A,0x2B", diff --git a/tools/perf/pmu-events/arch/x86/meteorlake/frontend.json b/tools/perf/pmu-events/arch/x86/meteorlake/frontend.json index 8264419500a5..9da8689eda81 100644 --- a/tools/perf/pmu-events/arch/x86/meteorlake/frontend.json +++ b/tools/perf/pmu-events/arch/x86/meteorlake/frontend.json @@ -460,6 +460,27 @@ "UMask": "0x1", "Unit": "cpu_core" }, + { + "BriefDescription": "Cycles when no uops are not delivered by the IDQ when backend of the machine is not stalled [This event is alias to IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE]", + "CounterMask": "6", + "EventCode": "0x9c", + "EventName": "IDQ_BUBBLES.CYCLES_0_UOPS_DELIV.CORE", + "PublicDescription": "Counts the number of cycles when no uops were delivered by the Instruction Decode Queue (IDQ) to the back-end of the pipeline when there was no back-end stalls. This event counts for one SMT thread in a given cycle. [This event is alias to IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE]", + "SampleAfterValue": "1000003", + "UMask": "0x1", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Cycles when optimal number of uops was delivered to the back-end when the back-end is not stalled [This event is alias to IDQ_UOPS_NOT_DELIVERED.CYCLES_FE_WAS_OK]", + "CounterMask": "1", + "EventCode": "0x9c", + "EventName": "IDQ_BUBBLES.CYCLES_FE_WAS_OK", + "Invert": "1", + "PublicDescription": "Counts the number of cycles when the optimal number of uops were delivered by the Instruction Decode Queue (IDQ) to the back-end of the pipeline when there was no back-end stalls. This event counts for one SMT thread in a given cycle. [This event is alias to IDQ_UOPS_NOT_DELIVERED.CYCLES_FE_WAS_OK]", + "SampleAfterValue": "1000003", + "UMask": "0x1", + "Unit": "cpu_core" + }, { "BriefDescription": "Uops not delivered by IDQ when backend of the machine is not stalled", "EventCode": "0x9c", @@ -470,22 +491,22 @@ "Unit": "cpu_core" }, { - "BriefDescription": "Cycles when no uops are not delivered by the IDQ when backend of the machine is not stalled", + "BriefDescription": "Cycles when no uops are not delivered by the IDQ when backend of the machine is not stalled [This event is alias to IDQ_BUBBLES.CYCLES_0_UOPS_DELIV.CORE]", "CounterMask": "6", "EventCode": "0x9c", "EventName": "IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE", - "PublicDescription": "Counts the number of cycles when no uops were delivered by the Instruction Decode Queue (IDQ) to the back-end of the pipeline when there was no back-end stalls. This event counts for one SMT thread in a given cycle.", + "PublicDescription": "Counts the number of cycles when no uops were delivered by the Instruction Decode Queue (IDQ) to the back-end of the pipeline when there was no back-end stalls. This event counts for one SMT thread in a given cycle. [This event is alias to IDQ_BUBBLES.CYCLES_0_UOPS_DELIV.CORE]", "SampleAfterValue": "1000003", "UMask": "0x1", "Unit": "cpu_core" }, { - "BriefDescription": "Cycles when optimal number of uops was delivered to the back-end when the back-end is not stalled", + "BriefDescription": "Cycles when optimal number of uops was delivered to the back-end when the back-end is not stalled [This event is alias to IDQ_BUBBLES.CYCLES_FE_WAS_OK]", "CounterMask": "1", "EventCode": "0x9c", "EventName": "IDQ_UOPS_NOT_DELIVERED.CYCLES_FE_WAS_OK", "Invert": "1", - "PublicDescription": "Counts the number of cycles when the optimal number of uops were delivered by the Instruction Decode Queue (IDQ) to the back-end of the pipeline when there was no back-end stalls. This event counts for one SMT thread in a given cycle.", + "PublicDescription": "Counts the number of cycles when the optimal number of uops were delivered by the Instruction Decode Queue (IDQ) to the back-end of the pipeline when there was no back-end stalls. This event counts for one SMT thread in a given cycle. [This event is alias to IDQ_BUBBLES.CYCLES_FE_WAS_OK]", "SampleAfterValue": "1000003", "UMask": "0x1", "Unit": "cpu_core" diff --git a/tools/perf/pmu-events/arch/x86/meteorlake/memory.json b/tools/perf/pmu-events/arch/x86/meteorlake/memory.json index 2605e1d0ba9f..a5b83293f157 100644 --- a/tools/perf/pmu-events/arch/x86/meteorlake/memory.json +++ b/tools/perf/pmu-events/arch/x86/meteorlake/memory.json @@ -66,6 +66,14 @@ "UMask": "0x84", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of machine clears due to memory ordering caused by a snoop from an external agent. Does not count internally generated machine clears such as those due to memory disambiguation.", + "EventCode": "0xc3", + "EventName": "MACHINE_CLEARS.MEMORY_ORDERING", + "SampleAfterValue": "20003", + "UMask": "0x2", + "Unit": "cpu_atom" + }, { "BriefDescription": "Number of machine clears due to memory ordering conflicts.", "EventCode": "0xc3", @@ -75,6 +83,15 @@ "UMask": "0x2", "Unit": "cpu_core" }, + { + "BriefDescription": "Cycles while L1 cache miss demand load is outstanding.", + "CounterMask": "2", + "EventCode": "0x47", + "EventName": "MEMORY_ACTIVITY.CYCLES_L1D_MISS", + "SampleAfterValue": "1000003", + "UMask": "0x2", + "Unit": "cpu_core" + }, { "BriefDescription": "Execution stalls while L1 cache miss demand load is outstanding.", "CounterMask": "3", @@ -279,6 +296,26 @@ "UMask": "0x4", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts demand data reads that were not supplied by the L3 cache.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_DATA_RD.L3_MISS", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x3FBFC00001", + "SampleAfterValue": "100003", + "UMask": "0x1", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Counts demand read for ownership (RFO) requests and software prefetches for exclusive ownership (PREFETCHW) that were not supplied by the L3 cache.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_RFO.L3_MISS", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x3FBFC00002", + "SampleAfterValue": "100003", + "UMask": "0x1", + "Unit": "cpu_core" + }, { "BriefDescription": "Counts demand data read requests that miss the L3 cache.", "EventCode": "0x21", diff --git a/tools/perf/pmu-events/arch/x86/meteorlake/other.json b/tools/perf/pmu-events/arch/x86/meteorlake/other.json index f4c603599df4..d55e792c0c43 100644 --- a/tools/perf/pmu-events/arch/x86/meteorlake/other.json +++ b/tools/perf/pmu-events/arch/x86/meteorlake/other.json @@ -7,6 +7,46 @@ "UMask": "0x8", "Unit": "cpu_core" }, + { + "BriefDescription": "Counts demand data reads that have any type of response.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_DATA_RD.ANY_RESPONSE", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x10001", + "SampleAfterValue": "100003", + "UMask": "0x1", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Counts demand data reads that were supplied by DRAM.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_DATA_RD.DRAM", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x184000001", + "SampleAfterValue": "100003", + "UMask": "0x1", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Counts demand read for ownership (RFO) requests and software prefetches for exclusive ownership (PREFETCHW) that have any type of response.", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_RFO.ANY_RESPONSE", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x10002", + "SampleAfterValue": "100003", + "UMask": "0x1", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Counts streaming stores that have any type of response.", + "EventCode": "0xB7", + "EventName": "OCR.STREAMING_WR.ANY_RESPONSE", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x10800", + "SampleAfterValue": "100003", + "UMask": "0x1", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts streaming stores that have any type of response.", "EventCode": "0x2A,0x2B", diff --git a/tools/perf/pmu-events/arch/x86/meteorlake/pipeline.json b/tools/perf/pmu-events/arch/x86/meteorlake/pipeline.json index 352c5efafc06..deaa7aba93f7 100644 --- a/tools/perf/pmu-events/arch/x86/meteorlake/pipeline.json +++ b/tools/perf/pmu-events/arch/x86/meteorlake/pipeline.json @@ -217,6 +217,15 @@ "UMask": "0x50", "Unit": "cpu_core" }, + { + "BriefDescription": "Counts the number of mispredicted taken JCC (Jump on Conditional Code) branch instructions retired.", + "EventCode": "0xc5", + "EventName": "BR_MISP_RETIRED.COND_TAKEN", + "PEBS": "1", + "SampleAfterValue": "200003", + "UMask": "0xfe", + "Unit": "cpu_atom" + }, { "BriefDescription": "number of branch instructions retired that were mispredicted and taken.", "EventCode": "0xc5", @@ -292,6 +301,15 @@ "UMask": "0xc0", "Unit": "cpu_core" }, + { + "BriefDescription": "Counts the number of mispredicted near taken branch instructions retired.", + "EventCode": "0xc5", + "EventName": "BR_MISP_RETIRED.NEAR_TAKEN", + "PEBS": "1", + "SampleAfterValue": "200003", + "UMask": "0x80", + "Unit": "cpu_atom" + }, { "BriefDescription": "Number of near branch instructions retired that were mispredicted and taken.", "EventCode": "0xc5", @@ -733,7 +751,7 @@ "Unit": "cpu_core" }, { - "BriefDescription": "INT_MISC.UNKNOWN_BRANCH_CYCLES", + "BriefDescription": "Bubble cycles of BAClear (Unknown Branch).", "EventCode": "0xad", "EventName": "INT_MISC.UNKNOWN_BRANCH_CYCLES", "MSRIndex": "0x3F7", @@ -1045,6 +1063,14 @@ "UMask": "0x3", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of issue slots every cycle that were not consumed by the backend due to Branch Mispredict", + "EventCode": "0x73", + "EventName": "TOPDOWN_BAD_SPECULATION.MISPREDICT", + "SampleAfterValue": "1000003", + "UMask": "0x4", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of issue slots every cycle that were not consumed by the backend due to a machine clear (nuke).", "EventCode": "0x73", @@ -1068,6 +1094,22 @@ "UMask": "0x1", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of issue slots every cycle that were not consumed by the backend due to memory reservation stall (scheduler not being able to accept another uop). This could be caused by RSV full or load/store buffer block.", + "EventCode": "0x74", + "EventName": "TOPDOWN_BE_BOUND.MEM_SCHEDULER", + "SampleAfterValue": "1000003", + "UMask": "0x2", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of issue slots every cycle that were not consumed by the backend due to IEC and FPC RAT stalls - which can be due to the FIQ and IEC reservation station stall (integer, FP and SIMD scheduler not being able to accept another uop. )", + "EventCode": "0x74", + "EventName": "TOPDOWN_BE_BOUND.NON_MEM_SCHEDULER", + "SampleAfterValue": "1000003", + "UMask": "0x8", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of issue slots every cycle that were not consumed by the backend due to mrbl stall. A 'marble' refers to a physical register file entry, also known as the physical destination (PDST).", "EventCode": "0x74", @@ -1076,6 +1118,14 @@ "UMask": "0x20", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of issue slots every cycle that were not consumed by the backend due to ROB full", + "EventCode": "0x74", + "EventName": "TOPDOWN_BE_BOUND.REORDER_BUFFER", + "SampleAfterValue": "1000003", + "UMask": "0x40", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of issue slots every cycle that were not consumed by the backend due to iq/jeu scoreboards or ms scb", "EventCode": "0x74", @@ -1156,6 +1206,14 @@ "UMask": "0x10", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of issue slots every cycle that were not delivered by the frontend that do not categorize into any other common frontend stall", + "EventCode": "0x71", + "EventName": "TOPDOWN_FE_BOUND.OTHER", + "SampleAfterValue": "1000003", + "UMask": "0x80", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of issue slots every cycle that were not delivered by the frontend due to predecode wrong", "EventCode": "0x71", @@ -1398,6 +1456,14 @@ "UMask": "0x1", "Unit": "cpu_core" }, + { + "BriefDescription": "Counts the total number of uops retired.", + "EventCode": "0xc2", + "EventName": "UOPS_RETIRED.ALL", + "PEBS": "1", + "SampleAfterValue": "2000003", + "Unit": "cpu_atom" + }, { "BriefDescription": "Cycles with retired uop(s).", "CounterMask": "1", diff --git a/tools/perf/pmu-events/arch/x86/meteorlake/uncore-other.json b/tools/perf/pmu-events/arch/x86/meteorlake/uncore-other.json new file mode 100644 index 000000000000..2af92e43b28a --- /dev/null +++ b/tools/perf/pmu-events/arch/x86/meteorlake/uncore-other.json @@ -0,0 +1,9 @@ +[ + { + "BriefDescription": "This 48-bit fixed counter counts the UCLK cycles.", + "EventCode": "0xff", + "EventName": "UNC_CLOCK.SOCKET", + "PerPkg": "1", + "Unit": "CLOCK" + } +] From b6292081615bda443842d34108adcdfe20822695 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Wed, 25 Oct 2023 17:31:47 -0700 Subject: [PATCH 455/475] perf vendor events intel: Update westmereex events to v4 Update westmereex events from v3 to v4 fixing a spelling issue. Signed-off-by: Ian Rogers Reviewed-by: Kan Liang Cc: Alexandre Torgue Cc: Maxime Coquelin Cc: Edward Baker Cc: Zhengjun Xing Link: https://lore.kernel.org/r/20231026003149.3287633-7-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/pmu-events/arch/x86/mapfile.csv | 2 +- tools/perf/pmu-events/arch/x86/westmereex/pipeline.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/perf/pmu-events/arch/x86/mapfile.csv b/tools/perf/pmu-events/arch/x86/mapfile.csv index 5b455739065b..0dee3de3187c 100644 --- a/tools/perf/pmu-events/arch/x86/mapfile.csv +++ b/tools/perf/pmu-events/arch/x86/mapfile.csv @@ -35,7 +35,7 @@ GenuineIntel-6-86,v1.21,snowridgex,core GenuineIntel-6-8[CD],v1.13,tigerlake,core GenuineIntel-6-2C,v5,westmereep-dp,core GenuineIntel-6-25,v4,westmereep-sp,core -GenuineIntel-6-2F,v3,westmereex,core +GenuineIntel-6-2F,v4,westmereex,core AuthenticAMD-23-([12][0-9A-F]|[0-9A-F]),v2,amdzen1,core AuthenticAMD-23-[[:xdigit:]]+,v1,amdzen2,core AuthenticAMD-25-([245][[:xdigit:]]|[[:xdigit:]]),v1,amdzen3,core diff --git a/tools/perf/pmu-events/arch/x86/westmereex/pipeline.json b/tools/perf/pmu-events/arch/x86/westmereex/pipeline.json index 1c61d18a4b5f..026236558d05 100644 --- a/tools/perf/pmu-events/arch/x86/westmereex/pipeline.json +++ b/tools/perf/pmu-events/arch/x86/westmereex/pipeline.json @@ -45,7 +45,7 @@ "UMask": "0x1" }, { - "BriefDescription": "Early Branch Prediciton Unit clears", + "BriefDescription": "Early Branch Prediction Unit clears", "EventCode": "0xE8", "EventName": "BPU_CLEARS.EARLY", "SampleAfterValue": "2000000", From c44c31185923637e672c10e60142e1e1106d1600 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Wed, 25 Oct 2023 17:31:48 -0700 Subject: [PATCH 456/475] perf vendor events intel: Update bonnell version number to v5 Spelling fixes were already incorporated in the Linux perf tree, update the version number to reflect this. Signed-off-by: Ian Rogers Reviewed-by: Kan Liang Cc: Alexandre Torgue Cc: Maxime Coquelin Cc: Edward Baker Cc: Zhengjun Xing Link: https://lore.kernel.org/r/20231026003149.3287633-8-irogers@google.com Signed-off-by: Namhyung Kim --- tools/perf/pmu-events/arch/x86/mapfile.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/perf/pmu-events/arch/x86/mapfile.csv b/tools/perf/pmu-events/arch/x86/mapfile.csv index 0dee3de3187c..e571683f59f3 100644 --- a/tools/perf/pmu-events/arch/x86/mapfile.csv +++ b/tools/perf/pmu-events/arch/x86/mapfile.csv @@ -1,7 +1,7 @@ Family-model,Version,Filename,EventType GenuineIntel-6-(97|9A|B7|BA|BF),v1.23,alderlake,core GenuineIntel-6-BE,v1.23,alderlaken,core -GenuineIntel-6-(1C|26|27|35|36),v4,bonnell,core +GenuineIntel-6-(1C|26|27|35|36),v5,bonnell,core GenuineIntel-6-(3D|47),v28,broadwell,core GenuineIntel-6-56,v11,broadwellde,core GenuineIntel-6-4F,v22,broadwellx,core From c43c64f8a1c68da370bf8d458ba52e24183a5264 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Wed, 25 Oct 2023 17:31:49 -0700 Subject: [PATCH 457/475] perf vendor events intel: Update tsx_cycles_per_elision metrics Update tsx_cycles_per_elision as per: https://github.com/intel/perfmon/pull/116 Prefer the el-start event rather than cycles-t for detecting whether the metric will work as HLE may be disabled. Remove the metric from sapphirerapids that has no el-start event. Signed-off-by: Ian Rogers Reviewed-by: Kan Liang Cc: Alexandre Torgue Cc: Maxime Coquelin Cc: Edward Baker Cc: Zhengjun Xing Link: https://lore.kernel.org/r/20231026003149.3287633-9-irogers@google.com Signed-off-by: Namhyung Kim --- .../perf/pmu-events/arch/x86/cascadelakex/clx-metrics.json | 2 +- tools/perf/pmu-events/arch/x86/icelake/icl-metrics.json | 2 +- tools/perf/pmu-events/arch/x86/icelakex/icx-metrics.json | 2 +- tools/perf/pmu-events/arch/x86/rocketlake/rkl-metrics.json | 2 +- .../pmu-events/arch/x86/sapphirerapids/spr-metrics.json | 7 ------- tools/perf/pmu-events/arch/x86/skylake/skl-metrics.json | 2 +- tools/perf/pmu-events/arch/x86/skylakex/skx-metrics.json | 2 +- tools/perf/pmu-events/arch/x86/tigerlake/tgl-metrics.json | 2 +- 8 files changed, 7 insertions(+), 14 deletions(-) diff --git a/tools/perf/pmu-events/arch/x86/cascadelakex/clx-metrics.json b/tools/perf/pmu-events/arch/x86/cascadelakex/clx-metrics.json index fbb111e40829..84c132af3dfa 100644 --- a/tools/perf/pmu-events/arch/x86/cascadelakex/clx-metrics.json +++ b/tools/perf/pmu-events/arch/x86/cascadelakex/clx-metrics.json @@ -1837,7 +1837,7 @@ }, { "BriefDescription": "Number of cycles within a transaction divided by the number of elisions.", - "MetricExpr": "(cycles\\-t / el\\-start if has_event(cycles\\-t) else 0)", + "MetricExpr": "(cycles\\-t / el\\-start if has_event(el\\-start) else 0)", "MetricGroup": "transaction", "MetricName": "tsx_cycles_per_elision", "ScaleUnit": "1cycles / elision" diff --git a/tools/perf/pmu-events/arch/x86/icelake/icl-metrics.json b/tools/perf/pmu-events/arch/x86/icelake/icl-metrics.json index a6eed0d9a26d..b43a6c6d8b7f 100644 --- a/tools/perf/pmu-events/arch/x86/icelake/icl-metrics.json +++ b/tools/perf/pmu-events/arch/x86/icelake/icl-metrics.json @@ -1525,7 +1525,7 @@ }, { "BriefDescription": "Number of cycles within a transaction divided by the number of elisions.", - "MetricExpr": "(cycles\\-t / el\\-start if has_event(cycles\\-t) else 0)", + "MetricExpr": "(cycles\\-t / el\\-start if has_event(el\\-start) else 0)", "MetricGroup": "transaction", "MetricName": "tsx_cycles_per_elision", "ScaleUnit": "1cycles / elision" diff --git a/tools/perf/pmu-events/arch/x86/icelakex/icx-metrics.json b/tools/perf/pmu-events/arch/x86/icelakex/icx-metrics.json index 7082ad5ba961..e98602c66707 100644 --- a/tools/perf/pmu-events/arch/x86/icelakex/icx-metrics.json +++ b/tools/perf/pmu-events/arch/x86/icelakex/icx-metrics.json @@ -1821,7 +1821,7 @@ }, { "BriefDescription": "Number of cycles within a transaction divided by the number of elisions.", - "MetricExpr": "(cycles\\-t / el\\-start if has_event(cycles\\-t) else 0)", + "MetricExpr": "(cycles\\-t / el\\-start if has_event(el\\-start) else 0)", "MetricGroup": "transaction", "MetricName": "tsx_cycles_per_elision", "ScaleUnit": "1cycles / elision" diff --git a/tools/perf/pmu-events/arch/x86/rocketlake/rkl-metrics.json b/tools/perf/pmu-events/arch/x86/rocketlake/rkl-metrics.json index a0191c8b708d..0c880e415669 100644 --- a/tools/perf/pmu-events/arch/x86/rocketlake/rkl-metrics.json +++ b/tools/perf/pmu-events/arch/x86/rocketlake/rkl-metrics.json @@ -1551,7 +1551,7 @@ }, { "BriefDescription": "Number of cycles within a transaction divided by the number of elisions.", - "MetricExpr": "(cycles\\-t / el\\-start if has_event(cycles\\-t) else 0)", + "MetricExpr": "(cycles\\-t / el\\-start if has_event(el\\-start) else 0)", "MetricGroup": "transaction", "MetricName": "tsx_cycles_per_elision", "ScaleUnit": "1cycles / elision" diff --git a/tools/perf/pmu-events/arch/x86/sapphirerapids/spr-metrics.json b/tools/perf/pmu-events/arch/x86/sapphirerapids/spr-metrics.json index 222212abd811..06c6d67cb76b 100644 --- a/tools/perf/pmu-events/arch/x86/sapphirerapids/spr-metrics.json +++ b/tools/perf/pmu-events/arch/x86/sapphirerapids/spr-metrics.json @@ -1944,13 +1944,6 @@ "MetricName": "tsx_aborted_cycles", "ScaleUnit": "100%" }, - { - "BriefDescription": "Number of cycles within a transaction divided by the number of elisions.", - "MetricExpr": "(cycles\\-t / el\\-start if has_event(cycles\\-t) else 0)", - "MetricGroup": "transaction", - "MetricName": "tsx_cycles_per_elision", - "ScaleUnit": "1cycles / elision" - }, { "BriefDescription": "Number of cycles within a transaction divided by the number of transactions.", "MetricExpr": "(cycles\\-t / tx\\-start if has_event(cycles\\-t) else 0)", diff --git a/tools/perf/pmu-events/arch/x86/skylake/skl-metrics.json b/tools/perf/pmu-events/arch/x86/skylake/skl-metrics.json index 2795a404bb58..faa615c57893 100644 --- a/tools/perf/pmu-events/arch/x86/skylake/skl-metrics.json +++ b/tools/perf/pmu-events/arch/x86/skylake/skl-metrics.json @@ -1473,7 +1473,7 @@ }, { "BriefDescription": "Number of cycles within a transaction divided by the number of elisions.", - "MetricExpr": "(cycles\\-t / el\\-start if has_event(cycles\\-t) else 0)", + "MetricExpr": "(cycles\\-t / el\\-start if has_event(el\\-start) else 0)", "MetricGroup": "transaction", "MetricName": "tsx_cycles_per_elision", "ScaleUnit": "1cycles / elision" diff --git a/tools/perf/pmu-events/arch/x86/skylakex/skx-metrics.json b/tools/perf/pmu-events/arch/x86/skylakex/skx-metrics.json index fa4209809c57..4a8f8eeb7525 100644 --- a/tools/perf/pmu-events/arch/x86/skylakex/skx-metrics.json +++ b/tools/perf/pmu-events/arch/x86/skylakex/skx-metrics.json @@ -1781,7 +1781,7 @@ }, { "BriefDescription": "Number of cycles within a transaction divided by the number of elisions.", - "MetricExpr": "(cycles\\-t / el\\-start if has_event(cycles\\-t) else 0)", + "MetricExpr": "(cycles\\-t / el\\-start if has_event(el\\-start) else 0)", "MetricGroup": "transaction", "MetricName": "tsx_cycles_per_elision", "ScaleUnit": "1cycles / elision" diff --git a/tools/perf/pmu-events/arch/x86/tigerlake/tgl-metrics.json b/tools/perf/pmu-events/arch/x86/tigerlake/tgl-metrics.json index 820d85c2933b..f11860f39c18 100644 --- a/tools/perf/pmu-events/arch/x86/tigerlake/tgl-metrics.json +++ b/tools/perf/pmu-events/arch/x86/tigerlake/tgl-metrics.json @@ -1551,7 +1551,7 @@ }, { "BriefDescription": "Number of cycles within a transaction divided by the number of elisions.", - "MetricExpr": "(cycles\\-t / el\\-start if has_event(cycles\\-t) else 0)", + "MetricExpr": "(cycles\\-t / el\\-start if has_event(el\\-start) else 0)", "MetricGroup": "transaction", "MetricName": "tsx_cycles_per_elision", "ScaleUnit": "1cycles / elision" From dcc4e5728eeaeda84878ca0018758cff1abfca21 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Fri, 27 Oct 2023 08:56:38 -0700 Subject: [PATCH 458/475] seq_buf: Introduce DECLARE_SEQ_BUF and seq_buf_str() Solve two ergonomic issues with struct seq_buf; 1) Too much boilerplate is required to initialize: struct seq_buf s; char buf[32]; seq_buf_init(s, buf, sizeof(buf)); Instead, we can build this directly on the stack. Provide DECLARE_SEQ_BUF() macro to do this: DECLARE_SEQ_BUF(s, 32); 2) %NUL termination is fragile and requires 2 steps to get a valid C String (and is a layering violation exposing the "internals" of seq_buf): seq_buf_terminate(s); do_something(s->buffer); Instead, we can just return s->buffer directly after terminating it in the refactored seq_buf_terminate(), now known as seq_buf_str(): do_something(seq_buf_str(s)); Link: https://lore.kernel.org/linux-trace-kernel/20231027155634.make.260-kees@kernel.org Link: https://lore.kernel.org/linux-trace-kernel/20231026194033.it.702-kees@kernel.org/ Cc: Yosry Ahmed Cc: "Matthew Wilcox (Oracle)" Cc: Christoph Hellwig Cc: Justin Stitt Cc: Kent Overstreet Cc: Petr Mladek Cc: Andy Shevchenko Cc: Rasmus Villemoes Cc: Sergey Senozhatsky Cc: Masami Hiramatsu Cc: Greg Kroah-Hartman Cc: Arnd Bergmann Cc: Jonathan Corbet Cc: Yun Zhou Cc: Jacob Keller Cc: Zhen Lei Signed-off-by: Kees Cook Signed-off-by: Steven Rostedt (Google) --- include/linux/seq_buf.h | 21 +++++++++++++++++---- kernel/trace/trace.c | 11 +---------- lib/seq_buf.c | 4 +--- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/include/linux/seq_buf.h b/include/linux/seq_buf.h index 8483e4b2d0d2..5fb1f12c33f9 100644 --- a/include/linux/seq_buf.h +++ b/include/linux/seq_buf.h @@ -21,9 +21,18 @@ struct seq_buf { size_t len; }; +#define DECLARE_SEQ_BUF(NAME, SIZE) \ + char __ ## NAME ## _buffer[SIZE] = ""; \ + struct seq_buf NAME = { \ + .buffer = &__ ## NAME ## _buffer, \ + .size = SIZE, \ + } + static inline void seq_buf_clear(struct seq_buf *s) { s->len = 0; + if (s->size) + s->buffer[0] = '\0'; } static inline void @@ -69,8 +78,8 @@ static inline unsigned int seq_buf_used(struct seq_buf *s) } /** - * seq_buf_terminate - Make sure buffer is nul terminated - * @s: the seq_buf descriptor to terminate. + * seq_buf_str - get %NUL-terminated C string from seq_buf + * @s: the seq_buf handle * * This makes sure that the buffer in @s is nul terminated and * safe to read as a string. @@ -81,16 +90,20 @@ static inline unsigned int seq_buf_used(struct seq_buf *s) * * After this function is called, s->buffer is safe to use * in string operations. + * + * Returns @s->buf after making sure it is terminated. */ -static inline void seq_buf_terminate(struct seq_buf *s) +static inline const char *seq_buf_str(struct seq_buf *s) { if (WARN_ON(s->size == 0)) - return; + return ""; if (seq_buf_buffer_left(s)) s->buffer[s->len] = 0; else s->buffer[s->size - 1] = 0; + + return s->buffer; } /** diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index d629065c2383..2539cfc20a97 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -3828,15 +3828,6 @@ static bool trace_safe_str(struct trace_iterator *iter, const char *str, return false; } -static const char *show_buffer(struct trace_seq *s) -{ - struct seq_buf *seq = &s->seq; - - seq_buf_terminate(seq); - - return seq->buffer; -} - static DEFINE_STATIC_KEY_FALSE(trace_no_verify); static int test_can_verify_check(const char *fmt, ...) @@ -3976,7 +3967,7 @@ void trace_check_vprintf(struct trace_iterator *iter, const char *fmt, */ if (WARN_ONCE(!trace_safe_str(iter, str, star, len), "fmt: '%s' current_buffer: '%s'", - fmt, show_buffer(&iter->seq))) { + fmt, seq_buf_str(&iter->seq.seq))) { int ret; /* Try to safely read the string */ diff --git a/lib/seq_buf.c b/lib/seq_buf.c index b7477aefff53..23518f77ea9c 100644 --- a/lib/seq_buf.c +++ b/lib/seq_buf.c @@ -109,9 +109,7 @@ void seq_buf_do_printk(struct seq_buf *s, const char *lvl) if (s->size == 0 || s->len == 0) return; - seq_buf_terminate(s); - - start = s->buffer; + start = seq_buf_str(s); while ((lf = strchr(start, '\n'))) { int len = lf - start + 1; From a0c04a3243f1db6cb47b04ba05b65af97f75a278 Mon Sep 17 00:00:00 2001 From: "Alessandro Carminati (Red Hat)" Date: Wed, 6 Sep 2023 15:57:03 +0000 Subject: [PATCH 459/475] verification/dot2k: Delete duplicate imports The presence of duplicate import lines appears to be a typo. Removing them. Link: https://lore.kernel.org/r/20230906155703.3917918-1-alessandro.carminati@gmail.com Fixes: 24bce201d798 ("tools/rv: Add dot2k") Signed-off-by: Alessandro Carminati (Red Hat) Signed-off-by: Daniel Bristot de Oliveira --- tools/verification/dot2/dot2k | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/verification/dot2/dot2k b/tools/verification/dot2/dot2k index 9dcd38abe20a..d4d7e52d549e 100644 --- a/tools/verification/dot2/dot2k +++ b/tools/verification/dot2/dot2k @@ -15,8 +15,6 @@ if __name__ == '__main__': import os import platform import sys - import sys - import argparse parser = argparse.ArgumentParser(description='transform .dot file into kernel rv monitor') parser.add_argument('-d', "--dot", dest="dot_file", required=True) From 696444a544ecd6d62c1edc89516b376cefb28929 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Thu, 27 Jul 2023 16:01:17 +0100 Subject: [PATCH 460/475] rtla: Fix uninitialized variable found Variable found is not being initialized, in the case where the desired mount is not found the variable contains garbage. Fix this by initializing it to zero. Link: https://lore.kernel.org/all/20230727150117.627730-1-colin.i.king@gmail.com/ Fixes: a957cbc02531 ("rtla: Add -C cgroup support") Signed-off-by: Colin Ian King Signed-off-by: Daniel Bristot de Oliveira --- tools/tracing/rtla/src/utils.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/tracing/rtla/src/utils.c b/tools/tracing/rtla/src/utils.c index 623a38908ed5..c769d7b3842c 100644 --- a/tools/tracing/rtla/src/utils.c +++ b/tools/tracing/rtla/src/utils.c @@ -538,7 +538,7 @@ static const int find_mount(const char *fs, char *mp, int sizeof_mp) { char mount_point[MAX_PATH]; char type[100]; - int found; + int found = 0; FILE *fp; fp = fopen("/proc/mounts", "r"); From 72fcce70faf06dd7442bf0b41b71f071d7fa29a4 Mon Sep 17 00:00:00 2001 From: Alexey Dobriyan Date: Fri, 27 Oct 2023 17:13:58 +0300 Subject: [PATCH 461/475] vsprintf: uninline simple_strntoull(), reorder arguments * uninline simple_strntoull(), gcc overinlines and this function is not performance critical * reorder arguments, so that appending INT_MAX as 4th argument generates very efficient tail call Space savings: add/remove: 1/0 grow/shrink: 0/3 up/down: 27/-179 (-152) Function old new delta simple_strntoll - 27 +27 simple_strtoull 15 10 -5 simple_strtoll 41 7 -34 vsscanf 1930 1790 -140 Signed-off-by: Alexey Dobriyan Reviewed-by: Andy Shevchenko Reviewed-by: Petr Mladek Signed-off-by: Petr Mladek Link: https://lore.kernel.org/all/82a2af6e-9b6c-4a09-89d7-ca90cc1cdad1@p183/ --- lib/vsprintf.c | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/lib/vsprintf.c b/lib/vsprintf.c index afb88b24fa74..3e3733a7084f 100644 --- a/lib/vsprintf.c +++ b/lib/vsprintf.c @@ -60,7 +60,8 @@ bool no_hash_pointers __ro_after_init; EXPORT_SYMBOL_GPL(no_hash_pointers); -static noinline unsigned long long simple_strntoull(const char *startp, size_t max_chars, char **endp, unsigned int base) +noinline +static unsigned long long simple_strntoull(const char *startp, char **endp, unsigned int base, size_t max_chars) { const char *cp; unsigned long long result = 0ULL; @@ -95,7 +96,7 @@ static noinline unsigned long long simple_strntoull(const char *startp, size_t m noinline unsigned long long simple_strtoull(const char *cp, char **endp, unsigned int base) { - return simple_strntoull(cp, INT_MAX, endp, base); + return simple_strntoull(cp, endp, base, INT_MAX); } EXPORT_SYMBOL(simple_strtoull); @@ -130,8 +131,8 @@ long simple_strtol(const char *cp, char **endp, unsigned int base) } EXPORT_SYMBOL(simple_strtol); -static long long simple_strntoll(const char *cp, size_t max_chars, char **endp, - unsigned int base) +noinline +static long long simple_strntoll(const char *cp, char **endp, unsigned int base, size_t max_chars) { /* * simple_strntoull() safely handles receiving max_chars==0 in the @@ -140,9 +141,9 @@ static long long simple_strntoll(const char *cp, size_t max_chars, char **endp, * and the content of *cp is irrelevant. */ if (*cp == '-' && max_chars > 0) - return -simple_strntoull(cp + 1, max_chars - 1, endp, base); + return -simple_strntoull(cp + 1, endp, base, max_chars - 1); - return simple_strntoull(cp, max_chars, endp, base); + return simple_strntoull(cp, endp, base, max_chars); } /** @@ -155,7 +156,7 @@ static long long simple_strntoll(const char *cp, size_t max_chars, char **endp, */ long long simple_strtoll(const char *cp, char **endp, unsigned int base) { - return simple_strntoll(cp, INT_MAX, endp, base); + return simple_strntoll(cp, endp, base, INT_MAX); } EXPORT_SYMBOL(simple_strtoll); @@ -3648,13 +3649,11 @@ int vsscanf(const char *buf, const char *fmt, va_list args) break; if (is_sign) - val.s = simple_strntoll(str, - field_width >= 0 ? field_width : INT_MAX, - &next, base); + val.s = simple_strntoll(str, &next, base, + field_width >= 0 ? field_width : INT_MAX); else - val.u = simple_strntoull(str, - field_width >= 0 ? field_width : INT_MAX, - &next, base); + val.u = simple_strntoull(str, &next, base, + field_width >= 0 ? field_width : INT_MAX); switch (qualifier) { case 'H': /* that's 'hh' in format */ From bb32500fb9b78215e4ef6ee8b4345c5f5d7eafb4 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Tue, 31 Oct 2023 12:24:53 -0400 Subject: [PATCH 462/475] tracing: Have trace_event_file have ref counters The following can crash the kernel: # cd /sys/kernel/tracing # echo 'p:sched schedule' > kprobe_events # exec 5>>events/kprobes/sched/enable # > kprobe_events # exec 5>&- The above commands: 1. Change directory to the tracefs directory 2. Create a kprobe event (doesn't matter what one) 3. Open bash file descriptor 5 on the enable file of the kprobe event 4. Delete the kprobe event (removes the files too) 5. Close the bash file descriptor 5 The above causes a crash! BUG: kernel NULL pointer dereference, address: 0000000000000028 #PF: supervisor read access in kernel mode #PF: error_code(0x0000) - not-present page PGD 0 P4D 0 Oops: 0000 [#1] PREEMPT SMP PTI CPU: 6 PID: 877 Comm: bash Not tainted 6.5.0-rc4-test-00008-g2c6b6b1029d4-dirty #186 Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 1.16.2-debian-1.16.2-1 04/01/2014 RIP: 0010:tracing_release_file_tr+0xc/0x50 What happens here is that the kprobe event creates a trace_event_file "file" descriptor that represents the file in tracefs to the event. It maintains state of the event (is it enabled for the given instance?). Opening the "enable" file gets a reference to the event "file" descriptor via the open file descriptor. When the kprobe event is deleted, the file is also deleted from the tracefs system which also frees the event "file" descriptor. But as the tracefs file is still opened by user space, it will not be totally removed until the final dput() is called on it. But this is not true with the event "file" descriptor that is already freed. If the user does a write to or simply closes the file descriptor it will reference the event "file" descriptor that was just freed, causing a use-after-free bug. To solve this, add a ref count to the event "file" descriptor as well as a new flag called "FREED". The "file" will not be freed until the last reference is released. But the FREE flag will be set when the event is removed to prevent any more modifications to that event from happening, even if there's still a reference to the event "file" descriptor. Link: https://lore.kernel.org/linux-trace-kernel/20231031000031.1e705592@gandalf.local.home/ Link: https://lore.kernel.org/linux-trace-kernel/20231031122453.7a48b923@gandalf.local.home Cc: stable@vger.kernel.org Cc: Mark Rutland Fixes: f5ca233e2e66d ("tracing: Increase trace array ref count on enable and filter files") Reported-by: Beau Belgrave Tested-by: Beau Belgrave Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- include/linux/trace_events.h | 4 ++++ kernel/trace/trace.c | 15 +++++++++++++++ kernel/trace/trace.h | 3 +++ kernel/trace/trace_events.c | 31 ++++++++++++++++++++++++++---- kernel/trace/trace_events_filter.c | 3 +++ 5 files changed, 52 insertions(+), 4 deletions(-) diff --git a/include/linux/trace_events.h b/include/linux/trace_events.h index 12207dc6722d..696f8dc4aa53 100644 --- a/include/linux/trace_events.h +++ b/include/linux/trace_events.h @@ -492,6 +492,7 @@ enum { EVENT_FILE_FL_TRIGGER_COND_BIT, EVENT_FILE_FL_PID_FILTER_BIT, EVENT_FILE_FL_WAS_ENABLED_BIT, + EVENT_FILE_FL_FREED_BIT, }; extern struct trace_event_file *trace_get_event_file(const char *instance, @@ -630,6 +631,7 @@ extern int __kprobe_event_add_fields(struct dynevent_cmd *cmd, ...); * TRIGGER_COND - When set, one or more triggers has an associated filter * PID_FILTER - When set, the event is filtered based on pid * WAS_ENABLED - Set when enabled to know to clear trace on module removal + * FREED - File descriptor is freed, all fields should be considered invalid */ enum { EVENT_FILE_FL_ENABLED = (1 << EVENT_FILE_FL_ENABLED_BIT), @@ -643,6 +645,7 @@ enum { EVENT_FILE_FL_TRIGGER_COND = (1 << EVENT_FILE_FL_TRIGGER_COND_BIT), EVENT_FILE_FL_PID_FILTER = (1 << EVENT_FILE_FL_PID_FILTER_BIT), EVENT_FILE_FL_WAS_ENABLED = (1 << EVENT_FILE_FL_WAS_ENABLED_BIT), + EVENT_FILE_FL_FREED = (1 << EVENT_FILE_FL_FREED_BIT), }; struct trace_event_file { @@ -671,6 +674,7 @@ struct trace_event_file { * caching and such. Which is mostly OK ;-) */ unsigned long flags; + atomic_t ref; /* ref count for opened files */ atomic_t sm_ref; /* soft-mode reference counter */ atomic_t tm_ref; /* trigger-mode reference counter */ }; diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 2539cfc20a97..9aebf904ff97 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -4978,6 +4978,20 @@ int tracing_open_file_tr(struct inode *inode, struct file *filp) if (ret) return ret; + mutex_lock(&event_mutex); + + /* Fail if the file is marked for removal */ + if (file->flags & EVENT_FILE_FL_FREED) { + trace_array_put(file->tr); + ret = -ENODEV; + } else { + event_file_get(file); + } + + mutex_unlock(&event_mutex); + if (ret) + return ret; + filp->private_data = inode->i_private; return 0; @@ -4988,6 +5002,7 @@ int tracing_release_file_tr(struct inode *inode, struct file *filp) struct trace_event_file *file = inode->i_private; trace_array_put(file->tr); + event_file_put(file); return 0; } diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 0e1405abf4f7..b7f4ea25a194 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -1669,6 +1669,9 @@ extern void event_trigger_unregister(struct event_command *cmd_ops, char *glob, struct event_trigger_data *trigger_data); +extern void event_file_get(struct trace_event_file *file); +extern void event_file_put(struct trace_event_file *file); + /** * struct event_trigger_ops - callbacks for trace event triggers * diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index f9e3e24d8796..f29e815ca5b2 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c @@ -990,13 +990,35 @@ static void remove_subsystem(struct trace_subsystem_dir *dir) } } +void event_file_get(struct trace_event_file *file) +{ + atomic_inc(&file->ref); +} + +void event_file_put(struct trace_event_file *file) +{ + if (WARN_ON_ONCE(!atomic_read(&file->ref))) { + if (file->flags & EVENT_FILE_FL_FREED) + kmem_cache_free(file_cachep, file); + return; + } + + if (atomic_dec_and_test(&file->ref)) { + /* Count should only go to zero when it is freed */ + if (WARN_ON_ONCE(!(file->flags & EVENT_FILE_FL_FREED))) + return; + kmem_cache_free(file_cachep, file); + } +} + static void remove_event_file_dir(struct trace_event_file *file) { eventfs_remove_dir(file->ei); list_del(&file->list); remove_subsystem(file->system); free_event_filter(file->filter); - kmem_cache_free(file_cachep, file); + file->flags |= EVENT_FILE_FL_FREED; + event_file_put(file); } /* @@ -1369,7 +1391,7 @@ event_enable_read(struct file *filp, char __user *ubuf, size_t cnt, flags = file->flags; mutex_unlock(&event_mutex); - if (!file) + if (!file || flags & EVENT_FILE_FL_FREED) return -ENODEV; if (flags & EVENT_FILE_FL_ENABLED && @@ -1403,7 +1425,7 @@ event_enable_write(struct file *filp, const char __user *ubuf, size_t cnt, ret = -ENODEV; mutex_lock(&event_mutex); file = event_file_data(filp); - if (likely(file)) { + if (likely(file && !(file->flags & EVENT_FILE_FL_FREED))) { ret = tracing_update_buffers(file->tr); if (ret < 0) { mutex_unlock(&event_mutex); @@ -1683,7 +1705,7 @@ event_filter_read(struct file *filp, char __user *ubuf, size_t cnt, mutex_lock(&event_mutex); file = event_file_data(filp); - if (file) + if (file && !(file->flags & EVENT_FILE_FL_FREED)) print_event_filter(file, s); mutex_unlock(&event_mutex); @@ -2902,6 +2924,7 @@ trace_create_new_event(struct trace_event_call *call, atomic_set(&file->tm_ref, 0); INIT_LIST_HEAD(&file->triggers); list_add(&file->list, &tr->events); + event_file_get(file); return file; } diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c index 33264e510d16..0c611b281a5b 100644 --- a/kernel/trace/trace_events_filter.c +++ b/kernel/trace/trace_events_filter.c @@ -2349,6 +2349,9 @@ int apply_event_filter(struct trace_event_file *file, char *filter_string) struct event_filter *filter = NULL; int err; + if (file->flags & EVENT_FILE_FL_FREED) + return -ENODEV; + if (!strcmp(strstrip(filter_string), "0")) { filter_disable(file); filter = event_filter(file); From 77bc4d4921bd3497678ba8e7f4e480de35692f05 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Tue, 31 Oct 2023 12:42:29 -0400 Subject: [PATCH 463/475] eventfs: Remove extra dget() in eventfs_create_events_dir() The creation of the top events directory does a dget() at the end of the creation in eventfs_create_events_dir() with a comment saying the final dput() will happen when it is removed. The problem is that a dget() is already done on the dentry when it was created with tracefs_start_creating()! The dget() now just causes a memory leak of that dentry. Remove the extra dget() as the final dput() in the deletion of the events directory actually matches the one in tracefs_start_creating(). Link: https://lore.kernel.org/linux-trace-kernel/20231031124229.4f2e3fa1@gandalf.local.home Cc: Masami Hiramatsu Cc: Mark Rutland Fixes: 5790b1fb3d672 ("eventfs: Remove eventfs_file and just use eventfs_inode") Signed-off-by: Steven Rostedt (Google) --- fs/tracefs/event_inode.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/fs/tracefs/event_inode.c b/fs/tracefs/event_inode.c index 4d2da7480e5f..5536860eb2ff 100644 --- a/fs/tracefs/event_inode.c +++ b/fs/tracefs/event_inode.c @@ -774,9 +774,6 @@ struct eventfs_inode *eventfs_create_events_dir(const char *name, struct dentry fsnotify_mkdir(dentry->d_parent->d_inode, dentry); tracefs_end_creating(dentry); - /* Will call dput when the directory is removed */ - dget(dentry); - return ei; fail: From 4f7969bcd6d33042d62e249b41b5578161e4c868 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Tue, 31 Oct 2023 15:10:33 -0400 Subject: [PATCH 464/475] tracing: Have the user copy of synthetic event address use correct context A synthetic event is created by the synthetic event interface that can read both user or kernel address memory. In reality, it reads any arbitrary memory location from within the kernel. If the address space is in USER (where CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE is set) then it uses strncpy_from_user_nofault() to copy strings otherwise it uses strncpy_from_kernel_nofault(). But since both functions use the same variable there's no annotation to what that variable is (ie. __user). This makes sparse complain. Quiet sparse by typecasting the strncpy_from_user_nofault() variable to a __user pointer. Link: https://lore.kernel.org/linux-trace-kernel/20231031151033.73c42e23@gandalf.local.home Cc: stable@vger.kernel.org Cc: Masami Hiramatsu Cc: Mark Rutland Fixes: 0934ae9977c2 ("tracing: Fix reading strings from synthetic events"); Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202311010013.fm8WTxa5-lkp@intel.com/ Signed-off-by: Steven Rostedt (Google) --- kernel/trace/trace_events_synth.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/trace/trace_events_synth.c b/kernel/trace/trace_events_synth.c index 14cb275a0bab..846e02c0fb59 100644 --- a/kernel/trace/trace_events_synth.c +++ b/kernel/trace/trace_events_synth.c @@ -452,7 +452,7 @@ static unsigned int trace_string(struct synth_trace_event *entry, #ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE if ((unsigned long)str_val < TASK_SIZE) - ret = strncpy_from_user_nofault(str_field, str_val, STR_VAR_LEN_MAX); + ret = strncpy_from_user_nofault(str_field, (const void __user *)str_val, STR_VAR_LEN_MAX); else #endif ret = strncpy_from_kernel_nofault(str_field, str_val, STR_VAR_LEN_MAX); From 9037caa09ed345b35325200f0e4acf5a94ae0a65 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Mon, 30 Oct 2023 12:15:23 -0400 Subject: [PATCH 465/475] eventfs: Fix kerneldoc of eventfs_remove_rec() The eventfs_remove_rec() had some missing parameters in the kerneldoc comment above it. Also, rephrase the description a bit more to have a bit more correct grammar. Link: https://lore.kernel.org/linux-trace-kernel/20231030121523.0b2225a7@gandalf.local.home Cc: Masami Hiramatsu Cc: Mark Rutland Fixes: 5790b1fb3d672 ("eventfs: Remove eventfs_file and just use eventfs_inode"); Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202310052216.4SgqasWo-lkp@intel.com/ Signed-off-by: Steven Rostedt (Google) --- fs/tracefs/event_inode.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fs/tracefs/event_inode.c b/fs/tracefs/event_inode.c index 5536860eb2ff..9f612a8f009d 100644 --- a/fs/tracefs/event_inode.c +++ b/fs/tracefs/event_inode.c @@ -796,9 +796,11 @@ static void free_ei(struct rcu_head *head) /** * eventfs_remove_rec - remove eventfs dir or file from list * @ei: eventfs_inode to be removed. + * @head: the list head to place the deleted @ei and children + * @level: prevent recursion from going more than 3 levels deep. * - * This function recursively remove eventfs_inode which - * contains info of file or dir. + * This function recursively removes eventfs_inodes which + * contains info of files and/or directories. */ static void eventfs_remove_rec(struct eventfs_inode *ei, struct list_head *head, int level) { From f2f496370afcbc5227d7002da28c74b91fed12ff Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Wed, 1 Nov 2023 13:25:42 -0400 Subject: [PATCH 466/475] eventfs: Remove "is_freed" union with rcu head The eventfs_inode->is_freed was a union with the rcu_head with the assumption that when it was on the srcu list the head would contain a pointer which would make "is_freed" true. But that was a wrong assumption as the rcu head is a single link list where the last element is NULL. Instead, split the nr_entries integer so that "is_freed" is one bit and the nr_entries is the next 31 bits. As there shouldn't be more than 10 (currently there's at most 5 to 7 depending on the config), this should not be a problem. Link: https://lkml.kernel.org/r/20231101172649.049758712@goodmis.org Cc: stable@vger.kernel.org Cc: Mark Rutland Cc: Andrew Morton Cc: Ajay Kaher Fixes: 63940449555e7 ("eventfs: Implement eventfs lookup, read, open functions") Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- fs/tracefs/event_inode.c | 2 ++ fs/tracefs/internal.h | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/fs/tracefs/event_inode.c b/fs/tracefs/event_inode.c index 9f612a8f009d..1ce73acf3df0 100644 --- a/fs/tracefs/event_inode.c +++ b/fs/tracefs/event_inode.c @@ -824,6 +824,8 @@ static void eventfs_remove_rec(struct eventfs_inode *ei, struct list_head *head, eventfs_remove_rec(ei_child, head, level + 1); } + ei->is_freed = 1; + list_del_rcu(&ei->list); list_add_tail(&ei->del_list, head); } diff --git a/fs/tracefs/internal.h b/fs/tracefs/internal.h index 64fde9490f52..c7d88aaa949f 100644 --- a/fs/tracefs/internal.h +++ b/fs/tracefs/internal.h @@ -23,6 +23,7 @@ struct tracefs_inode { * @d_parent: pointer to the parent's dentry * @d_children: The array of dentries to represent the files when created * @data: The private data to pass to the callbacks + * @is_freed: Flag set if the eventfs is on its way to be freed * @nr_entries: The number of items in @entries */ struct eventfs_inode { @@ -38,14 +39,13 @@ struct eventfs_inode { * Union - used for deletion * @del_list: list of eventfs_inode to delete * @rcu: eventfs_inode to delete in RCU - * @is_freed: node is freed if one of the above is set */ union { struct list_head del_list; struct rcu_head rcu; - unsigned long is_freed; }; - int nr_entries; + unsigned int is_freed:1; + unsigned int nr_entries:31; }; static inline struct tracefs_inode *get_tracefs(const struct inode *inode) From db3a397209b00d2e4e0a068608e5c546fc064b82 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Wed, 1 Nov 2023 13:25:43 -0400 Subject: [PATCH 467/475] eventfs: Have a free_ei() that just frees the eventfs_inode As the eventfs_inode is freed in two different locations, make a helper function free_ei() to make sure all the allocated fields of the eventfs_inode is freed. This requires renaming the existing free_ei() which is called by the srcu handler to free_rcu_ei() and have free_ei() just do the freeing, where free_rcu_ei() will call it. Link: https://lkml.kernel.org/r/20231101172649.265214087@goodmis.org Cc: Ajay Kaher Cc: Mark Rutland Cc: Andrew Morton Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- fs/tracefs/event_inode.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/fs/tracefs/event_inode.c b/fs/tracefs/event_inode.c index 1ce73acf3df0..dd5971855732 100644 --- a/fs/tracefs/event_inode.c +++ b/fs/tracefs/event_inode.c @@ -129,6 +129,13 @@ static struct dentry *create_dir(const char *name, struct dentry *parent) return eventfs_end_creating(dentry); } +static void free_ei(struct eventfs_inode *ei) +{ + kfree_const(ei->name); + kfree(ei->d_children); + kfree(ei); +} + /** * eventfs_set_ei_status_free - remove the dentry reference from an eventfs_inode * @ti: the tracefs_inode of the dentry @@ -168,9 +175,7 @@ void eventfs_set_ei_status_free(struct tracefs_inode *ti, struct dentry *dentry) eventfs_remove_dir(ei_child); } - kfree_const(ei->name); - kfree(ei->d_children); - kfree(ei); + free_ei(ei); return; } @@ -784,13 +789,11 @@ struct eventfs_inode *eventfs_create_events_dir(const char *name, struct dentry return ERR_PTR(-ENOMEM); } -static void free_ei(struct rcu_head *head) +static void free_rcu_ei(struct rcu_head *head) { struct eventfs_inode *ei = container_of(head, struct eventfs_inode, rcu); - kfree_const(ei->name); - kfree(ei->d_children); - kfree(ei); + free_ei(ei); } /** @@ -881,7 +884,7 @@ void eventfs_remove_dir(struct eventfs_inode *ei) for (i = 0; i < ei->nr_entries; i++) unhook_dentry(&ei->d_children[i], &dentry_list); unhook_dentry(&ei->dentry, &dentry_list); - call_srcu(&eventfs_srcu, &ei->rcu, free_ei); + call_srcu(&eventfs_srcu, &ei->rcu, free_rcu_ei); } mutex_unlock(&eventfs_mutex); From 77a06c33a22d13f3a6e31f06f6ee6bca666e6898 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Wed, 1 Nov 2023 13:25:44 -0400 Subject: [PATCH 468/475] eventfs: Test for ei->is_freed when accessing ei->dentry The eventfs_inode (ei) is protected by SRCU, but the ei->dentry is not. It is protected by the eventfs_mutex. Anytime the eventfs_mutex is released, and access to the ei->dentry needs to be done, it should first check if ei->is_freed is set under the eventfs_mutex. If it is, then the ei->dentry is invalid and must not be used. The ei->dentry must only be accessed under the eventfs_mutex and after checking if ei->is_freed is set. When the ei is being freed, it will (under the eventfs_mutex) set is_freed and at the same time move the dentry to a free list to be cleared after the eventfs_mutex is released. This means that any access to the ei->dentry must check first if ei->is_freed is set, because if it is, then the dentry is on its way to be freed. Also add comments to describe this better. Link: https://lore.kernel.org/all/CA+G9fYt6pY+tMZEOg=SoEywQOe19fGP3uR15SGowkdK+_X85Cg@mail.gmail.com/ Link: https://lore.kernel.org/all/CA+G9fYuDP3hVQ3t7FfrBAjd_WFVSurMgCepTxunSJf=MTe=6aA@mail.gmail.com/ Link: https://lkml.kernel.org/r/20231101172649.477608228@goodmis.org Cc: Ajay Kaher Cc: Mark Rutland Cc: Andrew Morton Fixes: 5790b1fb3d672 ("eventfs: Remove eventfs_file and just use eventfs_inode") Reported-by: Linux Kernel Functional Testing Reported-by: Naresh Kamboju Reported-by: Beau Belgrave Reviewed-by: Masami Hiramatsu (Google) Tested-by: Linux Kernel Functional Testing Tested-by: Naresh Kamboju Tested-by: Beau Belgrave Signed-off-by: Steven Rostedt (Google) --- fs/tracefs/event_inode.c | 45 ++++++++++++++++++++++++++++++++++------ fs/tracefs/internal.h | 3 ++- 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/fs/tracefs/event_inode.c b/fs/tracefs/event_inode.c index dd5971855732..e9625732c52d 100644 --- a/fs/tracefs/event_inode.c +++ b/fs/tracefs/event_inode.c @@ -24,7 +24,20 @@ #include #include "internal.h" +/* + * eventfs_mutex protects the eventfs_inode (ei) dentry. Any access + * to the ei->dentry must be done under this mutex and after checking + * if ei->is_freed is not set. The ei->dentry is released under the + * mutex at the same time ei->is_freed is set. If ei->is_freed is set + * then the ei->dentry is invalid. + */ static DEFINE_MUTEX(eventfs_mutex); + +/* + * The eventfs_inode (ei) itself is protected by SRCU. It is released from + * its parent's list and will have is_freed set (under eventfs_mutex). + * After the SRCU grace period is over, the ei may be freed. + */ DEFINE_STATIC_SRCU(eventfs_srcu); static struct dentry *eventfs_root_lookup(struct inode *dir, @@ -239,6 +252,10 @@ create_file_dentry(struct eventfs_inode *ei, struct dentry **e_dentry, bool invalidate = false; mutex_lock(&eventfs_mutex); + if (ei->is_freed) { + mutex_unlock(&eventfs_mutex); + return NULL; + } /* If the e_dentry already has a dentry, use it */ if (*e_dentry) { /* lookup does not need to up the ref count */ @@ -312,6 +329,8 @@ static void eventfs_post_create_dir(struct eventfs_inode *ei) struct eventfs_inode *ei_child; struct tracefs_inode *ti; + lockdep_assert_held(&eventfs_mutex); + /* srcu lock already held */ /* fill parent-child relation */ list_for_each_entry_srcu(ei_child, &ei->children, list, @@ -325,6 +344,7 @@ static void eventfs_post_create_dir(struct eventfs_inode *ei) /** * create_dir_dentry - Create a directory dentry for the eventfs_inode + * @pei: The eventfs_inode parent of ei. * @ei: The eventfs_inode to create the directory for * @parent: The dentry of the parent of this directory * @lookup: True if this is called by the lookup code @@ -332,12 +352,17 @@ static void eventfs_post_create_dir(struct eventfs_inode *ei) * This creates and attaches a directory dentry to the eventfs_inode @ei. */ static struct dentry * -create_dir_dentry(struct eventfs_inode *ei, struct dentry *parent, bool lookup) +create_dir_dentry(struct eventfs_inode *pei, struct eventfs_inode *ei, + struct dentry *parent, bool lookup) { bool invalidate = false; struct dentry *dentry = NULL; mutex_lock(&eventfs_mutex); + if (pei->is_freed || ei->is_freed) { + mutex_unlock(&eventfs_mutex); + return NULL; + } if (ei->dentry) { /* If the dentry already has a dentry, use it */ dentry = ei->dentry; @@ -440,7 +465,7 @@ static struct dentry *eventfs_root_lookup(struct inode *dir, */ mutex_lock(&eventfs_mutex); ei = READ_ONCE(ti->private); - if (ei) + if (ei && !ei->is_freed) ei_dentry = READ_ONCE(ei->dentry); mutex_unlock(&eventfs_mutex); @@ -454,7 +479,7 @@ static struct dentry *eventfs_root_lookup(struct inode *dir, if (strcmp(ei_child->name, name) != 0) continue; ret = simple_lookup(dir, dentry, flags); - create_dir_dentry(ei_child, ei_dentry, true); + create_dir_dentry(ei, ei_child, ei_dentry, true); created = true; break; } @@ -588,7 +613,7 @@ static int dcache_dir_open_wrapper(struct inode *inode, struct file *file) list_for_each_entry_srcu(ei_child, &ei->children, list, srcu_read_lock_held(&eventfs_srcu)) { - d = create_dir_dentry(ei_child, parent, false); + d = create_dir_dentry(ei, ei_child, parent, false); if (d) { ret = add_dentries(&dentries, d, cnt); if (ret < 0) @@ -705,12 +730,20 @@ struct eventfs_inode *eventfs_create_dir(const char *name, struct eventfs_inode ei->nr_entries = size; ei->data = data; INIT_LIST_HEAD(&ei->children); + INIT_LIST_HEAD(&ei->list); mutex_lock(&eventfs_mutex); - list_add_tail(&ei->list, &parent->children); - ei->d_parent = parent->dentry; + if (!parent->is_freed) { + list_add_tail(&ei->list, &parent->children); + ei->d_parent = parent->dentry; + } mutex_unlock(&eventfs_mutex); + /* Was the parent freed? */ + if (list_empty(&ei->list)) { + free_ei(ei); + ei = NULL; + } return ei; } diff --git a/fs/tracefs/internal.h b/fs/tracefs/internal.h index c7d88aaa949f..5a98e87dd3d1 100644 --- a/fs/tracefs/internal.h +++ b/fs/tracefs/internal.h @@ -24,6 +24,7 @@ struct tracefs_inode { * @d_children: The array of dentries to represent the files when created * @data: The private data to pass to the callbacks * @is_freed: Flag set if the eventfs is on its way to be freed + * Note if is_freed is set, then dentry is corrupted. * @nr_entries: The number of items in @entries */ struct eventfs_inode { @@ -31,7 +32,7 @@ struct eventfs_inode { const struct eventfs_entry *entries; const char *name; struct list_head children; - struct dentry *dentry; + struct dentry *dentry; /* Check is_freed to access */ struct dentry *d_parent; struct dentry **d_children; void *data; From 28e12c09f5aa081b2d13d1340e3610070b6c624d Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Wed, 1 Nov 2023 13:25:45 -0400 Subject: [PATCH 469/475] eventfs: Save ownership and mode Now that inodes and dentries are created on the fly, they are also reclaimed on memory pressure. Since the ownership and file mode are saved in the inode, if they are freed, any changes to the ownership and mode will be lost. To counter this, if the user changes the permissions or ownership, save them, and when creating the inodes again, restore those changes. Link: https://lkml.kernel.org/r/20231101172649.691841445@goodmis.org Cc: stable@vger.kernel.org Cc: Ajay Kaher Cc: Mark Rutland Cc: Andrew Morton Fixes: 63940449555e7 ("eventfs: Implement eventfs lookup, read, open functions") Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- fs/tracefs/event_inode.c | 148 +++++++++++++++++++++++++++++++++++---- fs/tracefs/internal.h | 16 +++++ 2 files changed, 151 insertions(+), 13 deletions(-) diff --git a/fs/tracefs/event_inode.c b/fs/tracefs/event_inode.c index e9625732c52d..93d08e552483 100644 --- a/fs/tracefs/event_inode.c +++ b/fs/tracefs/event_inode.c @@ -40,6 +40,15 @@ static DEFINE_MUTEX(eventfs_mutex); */ DEFINE_STATIC_SRCU(eventfs_srcu); +/* Mode is unsigned short, use the upper bits for flags */ +enum { + EVENTFS_SAVE_MODE = BIT(16), + EVENTFS_SAVE_UID = BIT(17), + EVENTFS_SAVE_GID = BIT(18), +}; + +#define EVENTFS_MODE_MASK (EVENTFS_SAVE_MODE - 1) + static struct dentry *eventfs_root_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags); @@ -47,8 +56,89 @@ static int dcache_dir_open_wrapper(struct inode *inode, struct file *file); static int dcache_readdir_wrapper(struct file *file, struct dir_context *ctx); static int eventfs_release(struct inode *inode, struct file *file); +static void update_attr(struct eventfs_attr *attr, struct iattr *iattr) +{ + unsigned int ia_valid = iattr->ia_valid; + + if (ia_valid & ATTR_MODE) { + attr->mode = (attr->mode & ~EVENTFS_MODE_MASK) | + (iattr->ia_mode & EVENTFS_MODE_MASK) | + EVENTFS_SAVE_MODE; + } + if (ia_valid & ATTR_UID) { + attr->mode |= EVENTFS_SAVE_UID; + attr->uid = iattr->ia_uid; + } + if (ia_valid & ATTR_GID) { + attr->mode |= EVENTFS_SAVE_GID; + attr->gid = iattr->ia_gid; + } +} + +static int eventfs_set_attr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *iattr) +{ + const struct eventfs_entry *entry; + struct eventfs_inode *ei; + const char *name; + int ret; + + mutex_lock(&eventfs_mutex); + ei = dentry->d_fsdata; + /* The LSB is set when the eventfs_inode is being freed */ + if (((unsigned long)ei & 1UL) || ei->is_freed) { + /* Do not allow changes if the event is about to be removed. */ + mutex_unlock(&eventfs_mutex); + return -ENODEV; + } + + /* Preallocate the children mode array if necessary */ + if (!(dentry->d_inode->i_mode & S_IFDIR)) { + if (!ei->entry_attrs) { + ei->entry_attrs = kzalloc(sizeof(*ei->entry_attrs) * ei->nr_entries, + GFP_KERNEL); + if (!ei->entry_attrs) { + ret = -ENOMEM; + goto out; + } + } + } + + ret = simple_setattr(idmap, dentry, iattr); + if (ret < 0) + goto out; + + /* + * If this is a dir, then update the ei cache, only the file + * mode is saved in the ei->m_children, and the ownership is + * determined by the parent directory. + */ + if (dentry->d_inode->i_mode & S_IFDIR) { + update_attr(&ei->attr, iattr); + + } else { + name = dentry->d_name.name; + + for (int i = 0; i < ei->nr_entries; i++) { + entry = &ei->entries[i]; + if (strcmp(name, entry->name) == 0) { + update_attr(&ei->entry_attrs[i], iattr); + break; + } + } + } + out: + mutex_unlock(&eventfs_mutex); + return ret; +} + static const struct inode_operations eventfs_root_dir_inode_operations = { .lookup = eventfs_root_lookup, + .setattr = eventfs_set_attr, +}; + +static const struct inode_operations eventfs_file_inode_operations = { + .setattr = eventfs_set_attr, }; static const struct file_operations eventfs_file_operations = { @@ -59,10 +149,30 @@ static const struct file_operations eventfs_file_operations = { .release = eventfs_release, }; +static void update_inode_attr(struct inode *inode, struct eventfs_attr *attr, umode_t mode) +{ + if (!attr) { + inode->i_mode = mode; + return; + } + + if (attr->mode & EVENTFS_SAVE_MODE) + inode->i_mode = attr->mode & EVENTFS_MODE_MASK; + else + inode->i_mode = mode; + + if (attr->mode & EVENTFS_SAVE_UID) + inode->i_uid = attr->uid; + + if (attr->mode & EVENTFS_SAVE_GID) + inode->i_gid = attr->gid; +} + /** * create_file - create a file in the tracefs filesystem * @name: the name of the file to create. * @mode: the permission that the file should have. + * @attr: saved attributes changed by user * @parent: parent dentry for this file. * @data: something that the caller will want to get to later on. * @fop: struct file_operations that should be used for this file. @@ -72,6 +182,7 @@ static const struct file_operations eventfs_file_operations = { * call. */ static struct dentry *create_file(const char *name, umode_t mode, + struct eventfs_attr *attr, struct dentry *parent, void *data, const struct file_operations *fop) { @@ -95,7 +206,10 @@ static struct dentry *create_file(const char *name, umode_t mode, if (unlikely(!inode)) return eventfs_failed_creating(dentry); - inode->i_mode = mode; + /* If the user updated the directory's attributes, use them */ + update_inode_attr(inode, attr, mode); + + inode->i_op = &eventfs_file_inode_operations; inode->i_fop = fop; inode->i_private = data; @@ -108,19 +222,19 @@ static struct dentry *create_file(const char *name, umode_t mode, /** * create_dir - create a dir in the tracefs filesystem - * @name: the name of the file to create. + * @ei: the eventfs_inode that represents the directory to create * @parent: parent dentry for this file. * * This function will create a dentry for a directory represented by * a eventfs_inode. */ -static struct dentry *create_dir(const char *name, struct dentry *parent) +static struct dentry *create_dir(struct eventfs_inode *ei, struct dentry *parent) { struct tracefs_inode *ti; struct dentry *dentry; struct inode *inode; - dentry = eventfs_start_creating(name, parent); + dentry = eventfs_start_creating(ei->name, parent); if (IS_ERR(dentry)) return dentry; @@ -128,7 +242,9 @@ static struct dentry *create_dir(const char *name, struct dentry *parent) if (unlikely(!inode)) return eventfs_failed_creating(dentry); - inode->i_mode = S_IFDIR | S_IRWXU | S_IRUGO | S_IXUGO; + /* If the user updated the directory's attributes, use them */ + update_inode_attr(inode, &ei->attr, S_IFDIR | S_IRWXU | S_IRUGO | S_IXUGO); + inode->i_op = &eventfs_root_dir_inode_operations; inode->i_fop = &eventfs_file_operations; @@ -146,6 +262,7 @@ static void free_ei(struct eventfs_inode *ei) { kfree_const(ei->name); kfree(ei->d_children); + kfree(ei->entry_attrs); kfree(ei); } @@ -231,7 +348,7 @@ void eventfs_set_ei_status_free(struct tracefs_inode *ti, struct dentry *dentry) /** * create_file_dentry - create a dentry for a file of an eventfs_inode * @ei: the eventfs_inode that the file will be created under - * @e_dentry: a pointer to the d_children[] of the @ei + * @idx: the index into the d_children[] of the @ei * @parent: The parent dentry of the created file. * @name: The name of the file to create * @mode: The mode of the file. @@ -244,10 +361,12 @@ void eventfs_set_ei_status_free(struct tracefs_inode *ti, struct dentry *dentry) * just do a dget() on it and return. Otherwise create the dentry and attach it. */ static struct dentry * -create_file_dentry(struct eventfs_inode *ei, struct dentry **e_dentry, +create_file_dentry(struct eventfs_inode *ei, int idx, struct dentry *parent, const char *name, umode_t mode, void *data, const struct file_operations *fops, bool lookup) { + struct eventfs_attr *attr = NULL; + struct dentry **e_dentry = &ei->d_children[idx]; struct dentry *dentry; bool invalidate = false; @@ -264,13 +383,18 @@ create_file_dentry(struct eventfs_inode *ei, struct dentry **e_dentry, mutex_unlock(&eventfs_mutex); return *e_dentry; } + + /* ei->entry_attrs are protected by SRCU */ + if (ei->entry_attrs) + attr = &ei->entry_attrs[idx]; + mutex_unlock(&eventfs_mutex); /* The lookup already has the parent->d_inode locked */ if (!lookup) inode_lock(parent->d_inode); - dentry = create_file(name, mode, parent, data, fops); + dentry = create_file(name, mode, attr, parent, data, fops); if (!lookup) inode_unlock(parent->d_inode); @@ -378,7 +502,7 @@ create_dir_dentry(struct eventfs_inode *pei, struct eventfs_inode *ei, if (!lookup) inode_lock(parent->d_inode); - dentry = create_dir(ei->name, parent); + dentry = create_dir(ei, parent); if (!lookup) inode_unlock(parent->d_inode); @@ -495,8 +619,7 @@ static struct dentry *eventfs_root_lookup(struct inode *dir, if (r <= 0) continue; ret = simple_lookup(dir, dentry, flags); - create_file_dentry(ei, &ei->d_children[i], - ei_dentry, name, mode, cdata, + create_file_dentry(ei, i, ei_dentry, name, mode, cdata, fops, true); break; } @@ -629,8 +752,7 @@ static int dcache_dir_open_wrapper(struct inode *inode, struct file *file) r = entry->callback(name, &mode, &cdata, &fops); if (r <= 0) continue; - d = create_file_dentry(ei, &ei->d_children[i], - parent, name, mode, cdata, fops, false); + d = create_file_dentry(ei, i, parent, name, mode, cdata, fops, false); if (d) { ret = add_dentries(&dentries, d, cnt); if (ret < 0) diff --git a/fs/tracefs/internal.h b/fs/tracefs/internal.h index 5a98e87dd3d1..5f60bcd69289 100644 --- a/fs/tracefs/internal.h +++ b/fs/tracefs/internal.h @@ -13,6 +13,18 @@ struct tracefs_inode { struct inode vfs_inode; }; +/* + * struct eventfs_attr - cache the mode and ownership of a eventfs entry + * @mode: saved mode plus flags of what is saved + * @uid: saved uid if changed + * @gid: saved gid if changed + */ +struct eventfs_attr { + int mode; + kuid_t uid; + kgid_t gid; +}; + /* * struct eventfs_inode - hold the properties of the eventfs directories. * @list: link list into the parent directory @@ -22,6 +34,8 @@ struct tracefs_inode { * @dentry: the dentry of the directory * @d_parent: pointer to the parent's dentry * @d_children: The array of dentries to represent the files when created + * @entry_attrs: Saved mode and ownership of the @d_children + * @attr: Saved mode and ownership of eventfs_inode itself * @data: The private data to pass to the callbacks * @is_freed: Flag set if the eventfs is on its way to be freed * Note if is_freed is set, then dentry is corrupted. @@ -35,6 +49,8 @@ struct eventfs_inode { struct dentry *dentry; /* Check is_freed to access */ struct dentry *d_parent; struct dentry **d_children; + struct eventfs_attr *entry_attrs; + struct eventfs_attr attr; void *data; /* * Union - used for deletion From 44365329f8219fc379097c2c9a75ff53f123764f Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Wed, 1 Nov 2023 13:25:46 -0400 Subject: [PATCH 470/475] eventfs: Hold eventfs_mutex when calling callback functions The callback function that is used to create inodes and dentries is not protected by anything and the data that is passed to it could become stale. After eventfs_remove_dir() is called by the tracing system, it is free to remove the events that are associated to that directory. Unfortunately, that means the callbacks must not be called after that. CPU0 CPU1 ---- ---- eventfs_root_lookup() { eventfs_remove_dir() { mutex_lock(&event_mutex); ei->is_freed = set; mutex_unlock(&event_mutex); } kfree(event_call); for (...) { entry = &ei->entries[i]; r = entry->callback() { call = data; // call == event_call above if (call->flags ...) [ USE AFTER FREE BUG ] The safest way to protect this is to wrap the callback with: mutex_lock(&eventfs_mutex); if (!ei->is_freed) r = entry->callback(); else r = -1; mutex_unlock(&eventfs_mutex); This will make sure that the callback will not be called after it is freed. But now it needs to be known that the callback is called while holding internal eventfs locks, and that it must not call back into the eventfs / tracefs system. There's no reason it should anyway, but document that as well. Link: https://lore.kernel.org/all/CA+G9fYu9GOEbD=rR5eMR-=HJ8H6rMsbzDC2ZY5=Y50WpWAE7_Q@mail.gmail.com/ Link: https://lkml.kernel.org/r/20231101172649.906696613@goodmis.org Cc: Ajay Kaher Cc: Mark Rutland Cc: Andrew Morton Fixes: 5790b1fb3d672 ("eventfs: Remove eventfs_file and just use eventfs_inode") Reported-by: Linux Kernel Functional Testing Reported-by: Naresh Kamboju Tested-by: Linux Kernel Functional Testing Tested-by: Naresh Kamboju Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- fs/tracefs/event_inode.c | 22 ++++++++++++++++++-- include/linux/tracefs.h | 43 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 2 deletions(-) diff --git a/fs/tracefs/event_inode.c b/fs/tracefs/event_inode.c index 93d08e552483..8ac9abf7a3d5 100644 --- a/fs/tracefs/event_inode.c +++ b/fs/tracefs/event_inode.c @@ -615,7 +615,13 @@ static struct dentry *eventfs_root_lookup(struct inode *dir, entry = &ei->entries[i]; if (strcmp(name, entry->name) == 0) { void *cdata = data; - r = entry->callback(name, &mode, &cdata, &fops); + mutex_lock(&eventfs_mutex); + /* If ei->is_freed, then the event itself may be too */ + if (!ei->is_freed) + r = entry->callback(name, &mode, &cdata, &fops); + else + r = -1; + mutex_unlock(&eventfs_mutex); if (r <= 0) continue; ret = simple_lookup(dir, dentry, flags); @@ -749,7 +755,13 @@ static int dcache_dir_open_wrapper(struct inode *inode, struct file *file) void *cdata = data; entry = &ei->entries[i]; name = entry->name; - r = entry->callback(name, &mode, &cdata, &fops); + mutex_lock(&eventfs_mutex); + /* If ei->is_freed, then the event itself may be too */ + if (!ei->is_freed) + r = entry->callback(name, &mode, &cdata, &fops); + else + r = -1; + mutex_unlock(&eventfs_mutex); if (r <= 0) continue; d = create_file_dentry(ei, i, parent, name, mode, cdata, fops, false); @@ -819,6 +831,10 @@ static int dcache_readdir_wrapper(struct file *file, struct dir_context *ctx) * data = A pointer to @data, and the callback may replace it, which will * cause the file created to pass the new data to the open() call. * fops = the fops to use for the created file. + * + * NB. @callback is called while holding internal locks of the eventfs + * system. The callback must not call any code that might also call into + * the tracefs or eventfs system or it will risk creating a deadlock. */ struct eventfs_inode *eventfs_create_dir(const char *name, struct eventfs_inode *parent, const struct eventfs_entry *entries, @@ -878,6 +894,8 @@ struct eventfs_inode *eventfs_create_dir(const char *name, struct eventfs_inode * @data: The default data to pass to the files (an entry may override it). * * This function creates the top of the trace event directory. + * + * See eventfs_create_dir() for use of @entries. */ struct eventfs_inode *eventfs_create_events_dir(const char *name, struct dentry *parent, const struct eventfs_entry *entries, diff --git a/include/linux/tracefs.h b/include/linux/tracefs.h index 13359b1a35d1..7a5fe17b6bf9 100644 --- a/include/linux/tracefs.h +++ b/include/linux/tracefs.h @@ -23,9 +23,52 @@ struct file_operations; struct eventfs_file; +/** + * eventfs_callback - A callback function to create dynamic files in eventfs + * @name: The name of the file that is to be created + * @mode: return the file mode for the file (RW access, etc) + * @data: data to pass to the created file ops + * @fops: the file operations of the created file + * + * The evetnfs files are dynamically created. The struct eventfs_entry array + * is passed to eventfs_create_dir() or eventfs_create_events_dir() that will + * be used to create the files within those directories. When a lookup + * or access to a file within the directory is made, the struct eventfs_entry + * array is used to find a callback() with the matching name that is being + * referenced (for lookups, the entire array is iterated and each callback + * will be called). + * + * The callback will be called with @name for the name of the file to create. + * The callback can return less than 1 to indicate that no file should be + * created. + * + * If a file is to be created, then @mode should be populated with the file + * mode (permissions) for which the file is created for. This would be + * used to set the created inode i_mode field. + * + * The @data should be set to the data passed to the other file operations + * (read, write, etc). Note, @data will also point to the data passed in + * to eventfs_create_dir() or eventfs_create_events_dir(), but the callback + * can replace the data if it chooses to. Otherwise, the original data + * will be used for the file operation functions. + * + * The @fops should be set to the file operations that will be used to create + * the inode. + * + * NB. This callback is called while holding internal locks of the eventfs + * system. The callback must not call any code that might also call into + * the tracefs or eventfs system or it will risk creating a deadlock. + */ typedef int (*eventfs_callback)(const char *name, umode_t *mode, void **data, const struct file_operations **fops); +/** + * struct eventfs_entry - dynamically created eventfs file call back handler + * @name: Then name of the dynamic file in an eventfs directory + * @callback: The callback to get the fops of the file when it is created + * + * See evenfs_callback() typedef for how to set up @callback. + */ struct eventfs_entry { const char *name; eventfs_callback callback; From 020010fbfa202aa528a52743eba4ab0da3400a4e Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Wed, 1 Nov 2023 13:25:47 -0400 Subject: [PATCH 471/475] eventfs: Delete eventfs_inode when the last dentry is freed There exists a race between holding a reference of an eventfs_inode dentry and the freeing of the eventfs_inode. If user space has a dentry held long enough, it may still be able to access the dentry's eventfs_inode after it has been freed. To prevent this, have he eventfs_inode freed via the last dput() (or via RCU if the eventfs_inode does not have a dentry). This means reintroducing the eventfs_inode del_list field at a temporary place to put the eventfs_inode. It needs to mark it as freed (via the list) but also must invalidate the dentry immediately as the return from eventfs_remove_dir() expects that they are. But the dentry invalidation must not be called under the eventfs_mutex, so it must be done after the eventfs_inode is marked as free (put on a deletion list). Link: https://lkml.kernel.org/r/20231101172650.123479767@goodmis.org Cc: stable@vger.kernel.org Cc: Masami Hiramatsu Cc: Mark Rutland Cc: Andrew Morton Cc: Ajay Kaher Fixes: 5bdcd5f5331a2 ("eventfs: Implement removal of meta data from eventfs") Signed-off-by: Steven Rostedt (Google) --- fs/tracefs/event_inode.c | 148 ++++++++++++++++++--------------------- fs/tracefs/internal.h | 2 + 2 files changed, 70 insertions(+), 80 deletions(-) diff --git a/fs/tracefs/event_inode.c b/fs/tracefs/event_inode.c index 8ac9abf7a3d5..0a04ae0ca8c8 100644 --- a/fs/tracefs/event_inode.c +++ b/fs/tracefs/event_inode.c @@ -85,8 +85,7 @@ static int eventfs_set_attr(struct mnt_idmap *idmap, struct dentry *dentry, mutex_lock(&eventfs_mutex); ei = dentry->d_fsdata; - /* The LSB is set when the eventfs_inode is being freed */ - if (((unsigned long)ei & 1UL) || ei->is_freed) { + if (ei->is_freed) { /* Do not allow changes if the event is about to be removed. */ mutex_unlock(&eventfs_mutex); return -ENODEV; @@ -276,35 +275,17 @@ static void free_ei(struct eventfs_inode *ei) void eventfs_set_ei_status_free(struct tracefs_inode *ti, struct dentry *dentry) { struct tracefs_inode *ti_parent; - struct eventfs_inode *ei_child, *tmp; struct eventfs_inode *ei; int i; /* The top level events directory may be freed by this */ if (unlikely(ti->flags & TRACEFS_EVENT_TOP_INODE)) { - LIST_HEAD(ef_del_list); - mutex_lock(&eventfs_mutex); - ei = ti->private; - - /* Record all the top level files */ - list_for_each_entry_srcu(ei_child, &ei->children, list, - lockdep_is_held(&eventfs_mutex)) { - list_add_tail(&ei_child->del_list, &ef_del_list); - } - /* Nothing should access this, but just in case! */ ti->private = NULL; - mutex_unlock(&eventfs_mutex); - /* Now safely free the top level files and their children */ - list_for_each_entry_safe(ei_child, tmp, &ef_del_list, del_list) { - list_del(&ei_child->del_list); - eventfs_remove_dir(ei_child); - } - free_ei(ei); return; } @@ -319,14 +300,6 @@ void eventfs_set_ei_status_free(struct tracefs_inode *ti, struct dentry *dentry) if (!ei) goto out; - /* - * If ei was freed, then the LSB bit is set for d_fsdata. - * But this should not happen, as it should still have a - * ref count that prevents it. Warn in case it does. - */ - if (WARN_ON_ONCE((unsigned long)ei & 1)) - goto out; - /* This could belong to one of the files of the ei */ if (ei->dentry != dentry) { for (i = 0; i < ei->nr_entries; i++) { @@ -336,6 +309,8 @@ void eventfs_set_ei_status_free(struct tracefs_inode *ti, struct dentry *dentry) if (WARN_ON_ONCE(i == ei->nr_entries)) goto out; ei->d_children[i] = NULL; + } else if (ei->is_freed) { + free_ei(ei); } else { ei->dentry = NULL; } @@ -962,13 +937,65 @@ struct eventfs_inode *eventfs_create_events_dir(const char *name, struct dentry return ERR_PTR(-ENOMEM); } +static LLIST_HEAD(free_list); + +static void eventfs_workfn(struct work_struct *work) +{ + struct eventfs_inode *ei, *tmp; + struct llist_node *llnode; + + llnode = llist_del_all(&free_list); + llist_for_each_entry_safe(ei, tmp, llnode, llist) { + /* This dput() matches the dget() from unhook_dentry() */ + for (int i = 0; i < ei->nr_entries; i++) { + if (ei->d_children[i]) + dput(ei->d_children[i]); + } + /* This should only get here if it had a dentry */ + if (!WARN_ON_ONCE(!ei->dentry)) + dput(ei->dentry); + } +} + +static DECLARE_WORK(eventfs_work, eventfs_workfn); + static void free_rcu_ei(struct rcu_head *head) { struct eventfs_inode *ei = container_of(head, struct eventfs_inode, rcu); + if (ei->dentry) { + /* Do not free the ei until all references of dentry are gone */ + if (llist_add(&ei->llist, &free_list)) + queue_work(system_unbound_wq, &eventfs_work); + return; + } + + /* If the ei doesn't have a dentry, neither should its children */ + for (int i = 0; i < ei->nr_entries; i++) { + WARN_ON_ONCE(ei->d_children[i]); + } + free_ei(ei); } +static void unhook_dentry(struct dentry *dentry) +{ + if (!dentry) + return; + + /* Keep the dentry from being freed yet (see eventfs_workfn()) */ + dget(dentry); + + dentry->d_fsdata = NULL; + d_invalidate(dentry); + mutex_lock(&eventfs_mutex); + /* dentry should now have at least a single reference */ + WARN_ONCE((int)d_count(dentry) < 1, + "dentry %px (%s) less than one reference (%d) after invalidate\n", + dentry, dentry->d_name.name, d_count(dentry)); + mutex_unlock(&eventfs_mutex); +} + /** * eventfs_remove_rec - remove eventfs dir or file from list * @ei: eventfs_inode to be removed. @@ -1006,33 +1033,6 @@ static void eventfs_remove_rec(struct eventfs_inode *ei, struct list_head *head, list_add_tail(&ei->del_list, head); } -static void unhook_dentry(struct dentry **dentry, struct dentry **list) -{ - if (*dentry) { - unsigned long ptr = (unsigned long)*list; - - /* Keep the dentry from being freed yet */ - dget(*dentry); - - /* - * Paranoid: The dget() above should prevent the dentry - * from being freed and calling eventfs_set_ei_status_free(). - * But just in case, set the link list LSB pointer to 1 - * and have eventfs_set_ei_status_free() check that to - * make sure that if it does happen, it will not think - * the d_fsdata is an eventfs_inode. - * - * For this to work, no eventfs_inode should be allocated - * on a odd space, as the ef should always be allocated - * to be at least word aligned. Check for that too. - */ - WARN_ON_ONCE(ptr & 1); - - (*dentry)->d_fsdata = (void *)(ptr | 1); - *list = *dentry; - *dentry = NULL; - } -} /** * eventfs_remove_dir - remove eventfs dir or file from list * @ei: eventfs_inode to be removed. @@ -1043,39 +1043,27 @@ void eventfs_remove_dir(struct eventfs_inode *ei) { struct eventfs_inode *tmp; LIST_HEAD(ei_del_list); - struct dentry *dentry_list = NULL; - struct dentry *dentry; - int i; if (!ei) return; + /* + * Move the deleted eventfs_inodes onto the ei_del_list + * which will also set the is_freed value. Note, this has to be + * done under the eventfs_mutex, but the deletions of + * the dentries must be done outside the eventfs_mutex. + * Hence moving them to this temporary list. + */ mutex_lock(&eventfs_mutex); eventfs_remove_rec(ei, &ei_del_list, 0); - - list_for_each_entry_safe(ei, tmp, &ei_del_list, del_list) { - for (i = 0; i < ei->nr_entries; i++) - unhook_dentry(&ei->d_children[i], &dentry_list); - unhook_dentry(&ei->dentry, &dentry_list); - call_srcu(&eventfs_srcu, &ei->rcu, free_rcu_ei); - } mutex_unlock(&eventfs_mutex); - while (dentry_list) { - unsigned long ptr; - - dentry = dentry_list; - ptr = (unsigned long)dentry->d_fsdata & ~1UL; - dentry_list = (struct dentry *)ptr; - dentry->d_fsdata = NULL; - d_invalidate(dentry); - mutex_lock(&eventfs_mutex); - /* dentry should now have at least a single reference */ - WARN_ONCE((int)d_count(dentry) < 1, - "dentry %px (%s) less than one reference (%d) after invalidate\n", - dentry, dentry->d_name.name, d_count(dentry)); - mutex_unlock(&eventfs_mutex); - dput(dentry); + list_for_each_entry_safe(ei, tmp, &ei_del_list, del_list) { + for (int i = 0; i < ei->nr_entries; i++) + unhook_dentry(ei->d_children[i]); + unhook_dentry(ei->dentry); + list_del(&ei->del_list); + call_srcu(&eventfs_srcu, &ei->rcu, free_rcu_ei); } } diff --git a/fs/tracefs/internal.h b/fs/tracefs/internal.h index 5f60bcd69289..06a1f220b901 100644 --- a/fs/tracefs/internal.h +++ b/fs/tracefs/internal.h @@ -54,10 +54,12 @@ struct eventfs_inode { void *data; /* * Union - used for deletion + * @llist: for calling dput() if needed after RCU * @del_list: list of eventfs_inode to delete * @rcu: eventfs_inode to delete in RCU */ union { + struct llist_node llist; struct list_head del_list; struct rcu_head rcu; }; From 62d65cac119d08d39f751b4e3e2063ed996edc05 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Wed, 1 Nov 2023 13:25:48 -0400 Subject: [PATCH 472/475] eventfs: Remove special processing of dput() of events directory The top level events directory is no longer special with regards to how it should be delete. Remove the extra processing for it in eventfs_set_ei_status_free(). Link: https://lkml.kernel.org/r/20231101172650.340876747@goodmis.org Cc: Ajay Kaher Cc: Mark Rutland Cc: Andrew Morton Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- fs/tracefs/event_inode.c | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/fs/tracefs/event_inode.c b/fs/tracefs/event_inode.c index 0a04ae0ca8c8..0087a3f455f1 100644 --- a/fs/tracefs/event_inode.c +++ b/fs/tracefs/event_inode.c @@ -274,28 +274,11 @@ static void free_ei(struct eventfs_inode *ei) */ void eventfs_set_ei_status_free(struct tracefs_inode *ti, struct dentry *dentry) { - struct tracefs_inode *ti_parent; struct eventfs_inode *ei; int i; - /* The top level events directory may be freed by this */ - if (unlikely(ti->flags & TRACEFS_EVENT_TOP_INODE)) { - mutex_lock(&eventfs_mutex); - ei = ti->private; - /* Nothing should access this, but just in case! */ - ti->private = NULL; - mutex_unlock(&eventfs_mutex); - - free_ei(ei); - return; - } - mutex_lock(&eventfs_mutex); - ti_parent = get_tracefs(dentry->d_parent->d_inode); - if (!ti_parent || !(ti_parent->flags & TRACEFS_EVENT_INODE)) - goto out; - ei = dentry->d_fsdata; if (!ei) goto out; @@ -920,6 +903,8 @@ struct eventfs_inode *eventfs_create_events_dir(const char *name, struct dentry inode->i_op = &eventfs_root_dir_inode_operations; inode->i_fop = &eventfs_file_operations; + dentry->d_fsdata = ei; + /* directory inodes start off with i_nlink == 2 (for "." entry) */ inc_nlink(inode); d_instantiate(dentry, inode); From 407c6726ca71b33330d2d6345d9ea7ebc02575e9 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Wed, 1 Nov 2023 13:25:49 -0400 Subject: [PATCH 473/475] eventfs: Use simple_recursive_removal() to clean up dentries Looking at how dentry is removed via the tracefs system, I found that eventfs does not do everything that it did under tracefs. The tracefs removal of a dentry calls simple_recursive_removal() that does a lot more than a simple d_invalidate(). As it should be a requirement that any eventfs_inode that has a dentry, so does its parent. When removing a eventfs_inode, if it has a dentry, a call to simple_recursive_removal() on that dentry should clean up all the dentries underneath it. Add WARN_ON_ONCE() to check for the parent having a dentry if any children do. Link: https://lore.kernel.org/all/20231101022553.GE1957730@ZenIV/ Link: https://lkml.kernel.org/r/20231101172650.552471568@goodmis.org Cc: stable@vger.kernel.org Cc: Masami Hiramatsu Cc: Mark Rutland Cc: Andrew Morton Cc: Al Viro Fixes: 5bdcd5f5331a2 ("eventfs: Implement removal of meta data from eventfs") Signed-off-by: Steven Rostedt (Google) --- fs/tracefs/event_inode.c | 77 +++++++++++++++++++++++----------------- fs/tracefs/internal.h | 2 -- 2 files changed, 44 insertions(+), 35 deletions(-) diff --git a/fs/tracefs/event_inode.c b/fs/tracefs/event_inode.c index 0087a3f455f1..f8a594a50ae6 100644 --- a/fs/tracefs/event_inode.c +++ b/fs/tracefs/event_inode.c @@ -967,30 +967,29 @@ static void unhook_dentry(struct dentry *dentry) { if (!dentry) return; - - /* Keep the dentry from being freed yet (see eventfs_workfn()) */ + /* + * Need to add a reference to the dentry that is expected by + * simple_recursive_removal(), which will include a dput(). + */ dget(dentry); - dentry->d_fsdata = NULL; - d_invalidate(dentry); - mutex_lock(&eventfs_mutex); - /* dentry should now have at least a single reference */ - WARN_ONCE((int)d_count(dentry) < 1, - "dentry %px (%s) less than one reference (%d) after invalidate\n", - dentry, dentry->d_name.name, d_count(dentry)); - mutex_unlock(&eventfs_mutex); + /* + * Also add a reference for the dput() in eventfs_workfn(). + * That is required as that dput() will free the ei after + * the SRCU grace period is over. + */ + dget(dentry); } /** * eventfs_remove_rec - remove eventfs dir or file from list * @ei: eventfs_inode to be removed. - * @head: the list head to place the deleted @ei and children * @level: prevent recursion from going more than 3 levels deep. * * This function recursively removes eventfs_inodes which * contains info of files and/or directories. */ -static void eventfs_remove_rec(struct eventfs_inode *ei, struct list_head *head, int level) +static void eventfs_remove_rec(struct eventfs_inode *ei, int level) { struct eventfs_inode *ei_child; @@ -1009,13 +1008,26 @@ static void eventfs_remove_rec(struct eventfs_inode *ei, struct list_head *head, /* search for nested folders or files */ list_for_each_entry_srcu(ei_child, &ei->children, list, lockdep_is_held(&eventfs_mutex)) { - eventfs_remove_rec(ei_child, head, level + 1); + /* Children only have dentry if parent does */ + WARN_ON_ONCE(ei_child->dentry && !ei->dentry); + eventfs_remove_rec(ei_child, level + 1); } + ei->is_freed = 1; + for (int i = 0; i < ei->nr_entries; i++) { + if (ei->d_children[i]) { + /* Children only have dentry if parent does */ + WARN_ON_ONCE(!ei->dentry); + unhook_dentry(ei->d_children[i]); + } + } + + unhook_dentry(ei->dentry); + list_del_rcu(&ei->list); - list_add_tail(&ei->del_list, head); + call_srcu(&eventfs_srcu, &ei->rcu, free_rcu_ei); } /** @@ -1026,30 +1038,22 @@ static void eventfs_remove_rec(struct eventfs_inode *ei, struct list_head *head, */ void eventfs_remove_dir(struct eventfs_inode *ei) { - struct eventfs_inode *tmp; - LIST_HEAD(ei_del_list); + struct dentry *dentry; if (!ei) return; - /* - * Move the deleted eventfs_inodes onto the ei_del_list - * which will also set the is_freed value. Note, this has to be - * done under the eventfs_mutex, but the deletions of - * the dentries must be done outside the eventfs_mutex. - * Hence moving them to this temporary list. - */ mutex_lock(&eventfs_mutex); - eventfs_remove_rec(ei, &ei_del_list, 0); + dentry = ei->dentry; + eventfs_remove_rec(ei, 0); mutex_unlock(&eventfs_mutex); - list_for_each_entry_safe(ei, tmp, &ei_del_list, del_list) { - for (int i = 0; i < ei->nr_entries; i++) - unhook_dentry(ei->d_children[i]); - unhook_dentry(ei->dentry); - list_del(&ei->del_list); - call_srcu(&eventfs_srcu, &ei->rcu, free_rcu_ei); - } + /* + * If any of the ei children has a dentry, then the ei itself + * must have a dentry. + */ + if (dentry) + simple_recursive_removal(dentry, NULL); } /** @@ -1060,10 +1064,17 @@ void eventfs_remove_dir(struct eventfs_inode *ei) */ void eventfs_remove_events_dir(struct eventfs_inode *ei) { - struct dentry *dentry = ei->dentry; + struct dentry *dentry; + dentry = ei->dentry; eventfs_remove_dir(ei); - /* Matches the dget() from eventfs_create_events_dir() */ + /* + * Matches the dget() done by tracefs_start_creating() + * in eventfs_create_events_dir() when it the dentry was + * created. In other words, it's a normal dentry that + * sticks around while the other ei->dentry are created + * and destroyed dynamically. + */ dput(dentry); } diff --git a/fs/tracefs/internal.h b/fs/tracefs/internal.h index 06a1f220b901..ccee18ca66c7 100644 --- a/fs/tracefs/internal.h +++ b/fs/tracefs/internal.h @@ -55,12 +55,10 @@ struct eventfs_inode { /* * Union - used for deletion * @llist: for calling dput() if needed after RCU - * @del_list: list of eventfs_inode to delete * @rcu: eventfs_inode to delete in RCU */ union { struct llist_node llist; - struct list_head del_list; struct rcu_head rcu; }; unsigned int is_freed:1; From 685b38c7650a0f461f56761cf61936b453d5bb24 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Wed, 1 Nov 2023 18:59:05 +0100 Subject: [PATCH 474/475] seq_buf: Export seq_buf_putc() Mark seq_buf_putc() which is part of the seq_buf API to be exported to kernel loadable GPL modules. Link: https://lkml.kernel.org/r/5c9a5ed97ac37dbdcd9c1e7bcbdec9ac166e79be.1698861216.git.christophe.jaillet@wanadoo.fr Signed-off-by: Christophe JAILLET Signed-off-by: Steven Rostedt (Google) --- lib/seq_buf.c | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/seq_buf.c b/lib/seq_buf.c index 23518f77ea9c..fb99168c3309 100644 --- a/lib/seq_buf.c +++ b/lib/seq_buf.c @@ -208,6 +208,7 @@ int seq_buf_putc(struct seq_buf *s, unsigned char c) seq_buf_set_overflow(s); return -1; } +EXPORT_SYMBOL_GPL(seq_buf_putc); /** * seq_buf_putmem - write raw data into the sequenc buffer From 70a9affa930c7aeba27893c7d402ef1294f43aa2 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Wed, 1 Nov 2023 18:59:06 +0100 Subject: [PATCH 475/475] seq_buf: Export seq_buf_puts() Mark seq_buf_puts() which is part of the seq_buf API to be exported to kernel loadable GPL modules. Link: https://lkml.kernel.org/r/b9e3737f66ec2450221b492048ce0d9c65c84953.1698861216.git.christophe.jaillet@wanadoo.fr Signed-off-by: Christophe JAILLET Signed-off-by: Steven Rostedt (Google) --- lib/seq_buf.c | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/seq_buf.c b/lib/seq_buf.c index fb99168c3309..010c730ca7fc 100644 --- a/lib/seq_buf.c +++ b/lib/seq_buf.c @@ -187,6 +187,7 @@ int seq_buf_puts(struct seq_buf *s, const char *str) seq_buf_set_overflow(s); return -1; } +EXPORT_SYMBOL_GPL(seq_buf_puts); /** * seq_buf_putc - sequence printing of simple character