| 1 |
From foo@baz Tue Mar 8 08:47:19 PM CET 2022 |
| 2 |
From: James Morse <james.morse@arm.com> |
| 3 |
Date: Tue, 23 Nov 2021 18:29:25 +0000 |
| 4 |
Subject: arm64: Add percpu vectors for EL1 |
| 5 |
|
| 6 |
From: James Morse <james.morse@arm.com> |
| 7 |
|
| 8 |
commit bd09128d16fac3c34b80bd6a29088ac632e8ce09 upstream. |
| 9 |
|
| 10 |
The Spectre-BHB workaround adds a firmware call to the vectors. This |
| 11 |
is needed on some CPUs, but not others. To avoid the unaffected CPU in |
| 12 |
a big/little pair from making the firmware call, create per cpu vectors. |
| 13 |
|
| 14 |
The per-cpu vectors only apply when returning from EL0. |
| 15 |
|
| 16 |
Systems using KPTI can use the canonical 'full-fat' vectors directly at |
| 17 |
EL1, the trampoline exit code will switch to this_cpu_vector on exit to |
| 18 |
EL0. Systems not using KPTI should always use this_cpu_vector. |
| 19 |
|
| 20 |
this_cpu_vector will point at a vector in tramp_vecs or |
| 21 |
__bp_harden_el1_vectors, depending on whether KPTI is in use. |
| 22 |
|
| 23 |
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com> |
| 24 |
Signed-off-by: James Morse <james.morse@arm.com> |
| 25 |
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org> |
| 26 |
--- |
| 27 |
arch/arm64/include/asm/vectors.h | 29 ++++++++++++++++++++++++++++- |
| 28 |
arch/arm64/kernel/cpufeature.c | 11 +++++++++++ |
| 29 |
arch/arm64/kernel/entry.S | 12 ++++++------ |
| 30 |
arch/arm64/kvm/hyp/vhe/switch.c | 9 +++++++-- |
| 31 |
4 files changed, 52 insertions(+), 9 deletions(-) |
| 32 |
|
| 33 |
--- a/arch/arm64/include/asm/vectors.h |
| 34 |
+++ b/arch/arm64/include/asm/vectors.h |
| 35 |
@@ -5,6 +5,15 @@ |
| 36 |
#ifndef __ASM_VECTORS_H |
| 37 |
#define __ASM_VECTORS_H |
| 38 |
|
| 39 |
+#include <linux/bug.h> |
| 40 |
+#include <linux/percpu.h> |
| 41 |
+ |
| 42 |
+#include <asm/fixmap.h> |
| 43 |
+ |
| 44 |
+extern char vectors[]; |
| 45 |
+extern char tramp_vectors[]; |
| 46 |
+extern char __bp_harden_el1_vectors[]; |
| 47 |
+ |
| 48 |
/* |
| 49 |
* Note: the order of this enum corresponds to two arrays in entry.S: |
| 50 |
* tramp_vecs and __bp_harden_el1_vectors. By default the canonical |
| 51 |
@@ -29,6 +38,24 @@ enum arm64_bp_harden_el1_vectors { |
| 52 |
* Remap the kernel before branching to the canonical vectors. |
| 53 |
*/ |
| 54 |
EL1_VECTOR_KPTI, |
| 55 |
-+}; |
| 56 |
+}; |
| 57 |
+ |
| 58 |
+/* The vectors to use on return from EL0. e.g. to remap the kernel */ |
| 59 |
+DECLARE_PER_CPU_READ_MOSTLY(const char *, this_cpu_vector); |
| 60 |
+ |
| 61 |
+#ifndef CONFIG_UNMAP_KERNEL_AT_EL0 |
| 62 |
+#define TRAMP_VALIAS 0 |
| 63 |
+#endif |
| 64 |
+ |
| 65 |
+static inline const char * |
| 66 |
+arm64_get_bp_hardening_vector(enum arm64_bp_harden_el1_vectors slot) |
| 67 |
+{ |
| 68 |
+ if (arm64_kernel_unmapped_at_el0()) |
| 69 |
+ return (char *)TRAMP_VALIAS + SZ_2K * slot; |
| 70 |
+ |
| 71 |
+ WARN_ON_ONCE(slot == EL1_VECTOR_KPTI); |
| 72 |
+ |
| 73 |
+ return __bp_harden_el1_vectors + SZ_2K * slot; |
| 74 |
+} |
| 75 |
|
| 76 |
#endif /* __ASM_VECTORS_H */ |
| 77 |
--- a/arch/arm64/kernel/cpufeature.c |
| 78 |
+++ b/arch/arm64/kernel/cpufeature.c |
| 79 |
@@ -73,6 +73,8 @@ |
| 80 |
#include <linux/mm.h> |
| 81 |
#include <linux/cpu.h> |
| 82 |
#include <linux/kasan.h> |
| 83 |
+#include <linux/percpu.h> |
| 84 |
+ |
| 85 |
#include <asm/cpu.h> |
| 86 |
#include <asm/cpufeature.h> |
| 87 |
#include <asm/cpu_ops.h> |
| 88 |
@@ -85,6 +87,7 @@ |
| 89 |
#include <asm/smp.h> |
| 90 |
#include <asm/sysreg.h> |
| 91 |
#include <asm/traps.h> |
| 92 |
+#include <asm/vectors.h> |
| 93 |
#include <asm/virt.h> |
| 94 |
|
| 95 |
/* Kernel representation of AT_HWCAP and AT_HWCAP2 */ |
| 96 |
@@ -110,6 +113,8 @@ DECLARE_BITMAP(boot_capabilities, ARM64_ |
| 97 |
bool arm64_use_ng_mappings = false; |
| 98 |
EXPORT_SYMBOL(arm64_use_ng_mappings); |
| 99 |
|
| 100 |
+DEFINE_PER_CPU_READ_MOSTLY(const char *, this_cpu_vector) = vectors; |
| 101 |
+ |
| 102 |
/* |
| 103 |
* Permit PER_LINUX32 and execve() of 32-bit binaries even if not all CPUs |
| 104 |
* support it? |
| 105 |
@@ -1590,6 +1595,12 @@ kpti_install_ng_mappings(const struct ar |
| 106 |
|
| 107 |
int cpu = smp_processor_id(); |
| 108 |
|
| 109 |
+ if (__this_cpu_read(this_cpu_vector) == vectors) { |
| 110 |
+ const char *v = arm64_get_bp_hardening_vector(EL1_VECTOR_KPTI); |
| 111 |
+ |
| 112 |
+ __this_cpu_write(this_cpu_vector, v); |
| 113 |
+ } |
| 114 |
+ |
| 115 |
/* |
| 116 |
* We don't need to rewrite the page-tables if either we've done |
| 117 |
* it already or we have KASLR enabled and therefore have not |
| 118 |
--- a/arch/arm64/kernel/entry.S |
| 119 |
+++ b/arch/arm64/kernel/entry.S |
| 120 |
@@ -38,7 +38,6 @@ |
| 121 |
.macro kernel_ventry, el:req, ht:req, regsize:req, label:req |
| 122 |
.align 7 |
| 123 |
.Lventry_start\@: |
| 124 |
-#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 |
| 125 |
.if \el == 0 |
| 126 |
/* |
| 127 |
* This must be the first instruction of the EL0 vector entries. It is |
| 128 |
@@ -53,7 +52,6 @@ |
| 129 |
.endif |
| 130 |
.Lskip_tramp_vectors_cleanup\@: |
| 131 |
.endif |
| 132 |
-#endif |
| 133 |
|
| 134 |
sub sp, sp, #PT_REGS_SIZE |
| 135 |
#ifdef CONFIG_VMAP_STACK |
| 136 |
@@ -712,10 +710,10 @@ alternative_else_nop_endif |
| 137 |
.endm |
| 138 |
|
| 139 |
.macro tramp_exit, regsize = 64 |
| 140 |
- adr x30, tramp_vectors |
| 141 |
-#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY |
| 142 |
- add x30, x30, SZ_4K |
| 143 |
-#endif |
| 144 |
+ tramp_data_read_var x30, this_cpu_vector |
| 145 |
+ get_this_cpu_offset x29 |
| 146 |
+ ldr x30, [x30, x29] |
| 147 |
+ |
| 148 |
msr vbar_el1, x30 |
| 149 |
ldr lr, [sp, #S_LR] |
| 150 |
tramp_unmap_kernel x29 |
| 151 |
@@ -775,6 +773,8 @@ __entry_tramp_data_vectors: |
| 152 |
__entry_tramp_data___sdei_asm_handler: |
| 153 |
.quad __sdei_asm_handler |
| 154 |
#endif /* CONFIG_ARM_SDE_INTERFACE */ |
| 155 |
+__entry_tramp_data_this_cpu_vector: |
| 156 |
+ .quad this_cpu_vector |
| 157 |
SYM_DATA_END(__entry_tramp_data_start) |
| 158 |
.popsection // .rodata |
| 159 |
#endif /* CONFIG_RANDOMIZE_BASE */ |
| 160 |
--- a/arch/arm64/kvm/hyp/vhe/switch.c |
| 161 |
+++ b/arch/arm64/kvm/hyp/vhe/switch.c |
| 162 |
@@ -10,6 +10,7 @@ |
| 163 |
#include <linux/kvm_host.h> |
| 164 |
#include <linux/types.h> |
| 165 |
#include <linux/jump_label.h> |
| 166 |
+#include <linux/percpu.h> |
| 167 |
#include <uapi/linux/psci.h> |
| 168 |
|
| 169 |
#include <kvm/arm_psci.h> |
| 170 |
@@ -25,6 +26,7 @@ |
| 171 |
#include <asm/debug-monitors.h> |
| 172 |
#include <asm/processor.h> |
| 173 |
#include <asm/thread_info.h> |
| 174 |
+#include <asm/vectors.h> |
| 175 |
|
| 176 |
/* VHE specific context */ |
| 177 |
DEFINE_PER_CPU(struct kvm_host_data, kvm_host_data); |
| 178 |
@@ -68,7 +70,7 @@ NOKPROBE_SYMBOL(__activate_traps); |
| 179 |
|
| 180 |
static void __deactivate_traps(struct kvm_vcpu *vcpu) |
| 181 |
{ |
| 182 |
- extern char vectors[]; /* kernel exception vectors */ |
| 183 |
+ const char *host_vectors = vectors; |
| 184 |
|
| 185 |
___deactivate_traps(vcpu); |
| 186 |
|
| 187 |
@@ -82,7 +84,10 @@ static void __deactivate_traps(struct kv |
| 188 |
asm(ALTERNATIVE("nop", "isb", ARM64_WORKAROUND_SPECULATIVE_AT)); |
| 189 |
|
| 190 |
write_sysreg(CPACR_EL1_DEFAULT, cpacr_el1); |
| 191 |
- write_sysreg(vectors, vbar_el1); |
| 192 |
+ |
| 193 |
+ if (!arm64_kernel_unmapped_at_el0()) |
| 194 |
+ host_vectors = __this_cpu_read(this_cpu_vector); |
| 195 |
+ write_sysreg(host_vectors, vbar_el1); |
| 196 |
} |
| 197 |
NOKPROBE_SYMBOL(__deactivate_traps); |
| 198 |
|
Parent Directory
| 
Revision Log