[gentoo-commits] proj/linux-patches:4.14 commit in: /

[Mike Pagano]

commit:     5e8982d6c61e5a378861f06965c7fb3c651f1902
Author:     Mike Pagano <mpagano <AT> gentoo <DOT> org>
AuthorDate: Sat Apr  2 16:32:30 2022 +0000
Commit:     Mike Pagano <mpagano <AT> gentoo <DOT> org>
CommitDate: Sat Apr  2 16:32:30 2022 +0000
URL:        https://gitweb.gentoo.org/proj/linux-patches.git/commit/?id=5e8982d6

Linux patch 4.14.275

Signed-off-by: Mike Pagano <mpagano <AT> gentoo.org>

 0000_README               |    4 +
 1274_linux-4.14.275.patch | 1544 +++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 1548 insertions(+)

diff --git a/0000_README b/0000_README
index 1469ce24..572ede3f 100644
--- a/0000_README
+++ b/0000_README
@@ -1143,6 +1143,10 @@ Patch:  1273_linux-4.14.274.patch
 From:   https://www.kernel.org
 Desc:   Linux 4.14.274
 
+Patch:  1274_linux-4.14.275.patch
+From:   https://www.kernel.org
+Desc:   Linux 4.14.275
+
 Patch:  1500_XATTR_USER_PREFIX.patch
 From:   https://bugs.gentoo.org/show_bug.cgi?id=470644
 Desc:   Support for namespace user.pax.* on tmpfs.

diff --git a/1274_linux-4.14.275.patch b/1274_linux-4.14.275.patch
new file mode 100644
index 00000000..32419e47
--- /dev/null
+++ b/1274_linux-4.14.275.patch
@@ -0,0 +1,1544 @@
+diff --git a/Documentation/arm64/silicon-errata.txt b/Documentation/arm64/silicon-errata.txt
+index e4fe6adc372bf..42f5672e89179 100644
+--- a/Documentation/arm64/silicon-errata.txt
++++ b/Documentation/arm64/silicon-errata.txt
+@@ -56,6 +56,7 @@ stable kernels.
+ | ARM            | Cortex-A72      | #853709         | N/A                         |
+ | ARM            | Cortex-A73      | #858921         | ARM64_ERRATUM_858921        |
+ | ARM            | Cortex-A55      | #1024718        | ARM64_ERRATUM_1024718       |
++| ARM            | Cortex-A76      | #1188873        | ARM64_ERRATUM_1188873       |
+ | ARM            | MMU-500         | #841119,#826419 | N/A                         |
+ |                |                 |                 |                             |
+ | Cavium         | ThunderX ITS    | #22375, #24313  | CAVIUM_ERRATUM_22375        |
+diff --git a/Makefile b/Makefile
+index a06abc38f35db..cad522127bb90 100644
+--- a/Makefile
++++ b/Makefile
+@@ -1,7 +1,7 @@
+ # SPDX-License-Identifier: GPL-2.0
+ VERSION = 4
+ PATCHLEVEL = 14
+-SUBLEVEL = 274
++SUBLEVEL = 275
+ EXTRAVERSION =
+ NAME = Petit Gorille
+ 
+diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h
+index b602326399845..dbd9615b428c3 100644
+--- a/arch/arm/include/asm/kvm_host.h
++++ b/arch/arm/include/asm/kvm_host.h
+@@ -26,6 +26,7 @@
+ #include <asm/kvm_asm.h>
+ #include <asm/kvm_mmio.h>
+ #include <asm/fpstate.h>
++#include <asm/spectre.h>
+ #include <kvm/arm_arch_timer.h>
+ 
+ #define __KVM_HAVE_ARCH_INTC_INITIALIZED
+@@ -324,4 +325,9 @@ static inline int kvm_arm_have_ssbd(void)
+ 	return KVM_SSBD_UNKNOWN;
+ }
+ 
++static inline int kvm_arm_get_spectre_bhb_state(void)
++{
++	/* 32bit guests don't need firmware for this */
++	return SPECTRE_VULNERABLE; /* aka SMCCC_RET_NOT_SUPPORTED */
++}
+ #endif /* __ARM_KVM_HOST_H__ */
+diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
+index e76f74874a420..7605d2f00d553 100644
+--- a/arch/arm64/Kconfig
++++ b/arch/arm64/Kconfig
+@@ -458,6 +458,20 @@ config ARM64_ERRATUM_1024718
+ 
+ 	  If unsure, say Y.
+ 
++config ARM64_ERRATUM_1188873
++	bool "Cortex-A76: MRC read following MRRC read of specific Generic Timer in AArch32 might give incorrect result"
++	default y
++	depends on COMPAT
++	select ARM_ARCH_TIMER_OOL_WORKAROUND
++	help
++	  This option adds work arounds for ARM Cortex-A76 erratum 1188873
++
++	  Affected Cortex-A76 cores (r0p0, r1p0, r2p0) could cause
++	  register corruption when accessing the timer registers from
++	  AArch32 userspace.
++
++	  If unsure, say Y.
++
+ config CAVIUM_ERRATUM_22375
+ 	bool "Cavium erratum 22375, 24313"
+ 	default y
+@@ -858,6 +872,16 @@ config ARM64_SSBD
+ 
+ 	  If unsure, say Y.
+ 
++config MITIGATE_SPECTRE_BRANCH_HISTORY
++	bool "Mitigate Spectre style attacks against branch history" if EXPERT
++	default y
++	depends on HARDEN_BRANCH_PREDICTOR || !KVM
++	help
++	  Speculation attacks against some high-performance processors can
++	  make use of branch history to influence future speculation.
++	  When taking an exception from user-space, a sequence of branches
++	  or a firmware call overwrites the branch history.
++
+ menuconfig ARMV8_DEPRECATED
+ 	bool "Emulate deprecated/obsolete ARMv8 instructions"
+ 	depends on COMPAT
+diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
+index 02d73d83f0deb..6b38f3b3095a3 100644
+--- a/arch/arm64/include/asm/assembler.h
++++ b/arch/arm64/include/asm/assembler.h
+@@ -103,6 +103,13 @@
+ 	hint	#20
+ 	.endm
+ 
++/*
++ * Clear Branch History instruction
++ */
++	.macro clearbhb
++	hint	#22
++	.endm
++
+ /*
+  * Sanitise a 64-bit bounded index wrt speculation, returning zero if out
+  * of bounds.
+@@ -549,4 +556,31 @@ alternative_endif
+ .Ldone\@:
+ 	.endm
+ 
++	.macro __mitigate_spectre_bhb_loop      tmp
++#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
++alternative_cb  spectre_bhb_patch_loop_iter
++	mov	\tmp, #32		// Patched to correct the immediate
++alternative_cb_end
++.Lspectre_bhb_loop\@:
++	b	. + 4
++	subs	\tmp, \tmp, #1
++	b.ne	.Lspectre_bhb_loop\@
++	dsb	nsh
++	isb
++#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
++	.endm
++
++	/* Save/restores x0-x3 to the stack */
++	.macro __mitigate_spectre_bhb_fw
++#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
++	stp	x0, x1, [sp, #-16]!
++	stp	x2, x3, [sp, #-16]!
++	mov	w0, #ARM_SMCCC_ARCH_WORKAROUND_3
++alternative_cb	arm64_update_smccc_conduit
++	nop					// Patched to SMC/HVC #0
++alternative_cb_end
++	ldp	x2, x3, [sp], #16
++	ldp	x0, x1, [sp], #16
++#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
++	.endm
+ #endif	/* __ASM_ASSEMBLER_H */
+diff --git a/arch/arm64/include/asm/cpu.h b/arch/arm64/include/asm/cpu.h
+index 889226b4c6e1c..c7f17e663e729 100644
+--- a/arch/arm64/include/asm/cpu.h
++++ b/arch/arm64/include/asm/cpu.h
+@@ -36,6 +36,7 @@ struct cpuinfo_arm64 {
+ 	u64		reg_id_aa64dfr1;
+ 	u64		reg_id_aa64isar0;
+ 	u64		reg_id_aa64isar1;
++	u64		reg_id_aa64isar2;
+ 	u64		reg_id_aa64mmfr0;
+ 	u64		reg_id_aa64mmfr1;
+ 	u64		reg_id_aa64mmfr2;
+diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h
+index 2f8bd0388905d..20ca422eb094a 100644
+--- a/arch/arm64/include/asm/cpucaps.h
++++ b/arch/arm64/include/asm/cpucaps.h
+@@ -45,7 +45,9 @@
+ #define ARM64_SSBD				25
+ #define ARM64_MISMATCHED_CACHE_TYPE		26
+ #define ARM64_SSBS				27
++#define ARM64_WORKAROUND_1188873		28
++#define ARM64_SPECTRE_BHB			29
+ 
+-#define ARM64_NCAPS				28
++#define ARM64_NCAPS				30
+ 
+ #endif /* __ASM_CPUCAPS_H */
+diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h
+index 166f81b7afee6..3e9d042d1b1e7 100644
+--- a/arch/arm64/include/asm/cpufeature.h
++++ b/arch/arm64/include/asm/cpufeature.h
+@@ -456,6 +456,34 @@ static inline bool cpu_supports_mixed_endian_el0(void)
+ 	return id_aa64mmfr0_mixed_endian_el0(read_cpuid(ID_AA64MMFR0_EL1));
+ }
+ 
++static inline bool supports_csv2p3(int scope)
++{
++	u64 pfr0;
++	u8 csv2_val;
++
++	if (scope == SCOPE_LOCAL_CPU)
++		pfr0 = read_sysreg_s(SYS_ID_AA64PFR0_EL1);
++	else
++		pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
++
++	csv2_val = cpuid_feature_extract_unsigned_field(pfr0,
++							ID_AA64PFR0_CSV2_SHIFT);
++	return csv2_val == 3;
++}
++
++static inline bool supports_clearbhb(int scope)
++{
++	u64 isar2;
++
++	if (scope == SCOPE_LOCAL_CPU)
++		isar2 = read_sysreg_s(SYS_ID_AA64ISAR2_EL1);
++	else
++		isar2 = read_sanitised_ftr_reg(SYS_ID_AA64ISAR2_EL1);
++
++	return cpuid_feature_extract_unsigned_field(isar2,
++						    ID_AA64ISAR2_CLEARBHB_SHIFT);
++}
++
+ static inline bool system_supports_32bit_el0(void)
+ {
+ 	return cpus_have_const_cap(ARM64_HAS_32BIT_EL0);
+@@ -495,6 +523,17 @@ static inline int arm64_get_ssbd_state(void)
+ 
+ void arm64_set_ssbd_mitigation(bool state);
+ 
++/* Watch out, ordering is important here. */
++enum mitigation_state {
++	SPECTRE_UNAFFECTED,
++	SPECTRE_MITIGATED,
++	SPECTRE_VULNERABLE,
++};
++
++enum mitigation_state arm64_get_spectre_bhb_state(void);
++bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry, int scope);
++u8 spectre_bhb_loop_affected(int scope);
++void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *__unused);
+ #endif /* __ASSEMBLY__ */
+ 
+ #endif
+diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h
+index b23456035eac5..401088d9cd827 100644
+--- a/arch/arm64/include/asm/cputype.h
++++ b/arch/arm64/include/asm/cputype.h
+@@ -87,6 +87,16 @@
+ #define ARM_CPU_PART_CORTEX_A75		0xD0A
+ #define ARM_CPU_PART_CORTEX_A35		0xD04
+ #define ARM_CPU_PART_CORTEX_A55		0xD05
++#define ARM_CPU_PART_CORTEX_A76		0xD0B
++#define ARM_CPU_PART_NEOVERSE_N1	0xD0C
++#define ARM_CPU_PART_CORTEX_A77		0xD0D
++#define ARM_CPU_PART_NEOVERSE_V1	0xD40
++#define ARM_CPU_PART_CORTEX_A78		0xD41
++#define ARM_CPU_PART_CORTEX_X1		0xD44
++#define ARM_CPU_PART_CORTEX_A710	0xD47
++#define ARM_CPU_PART_CORTEX_X2		0xD48
++#define ARM_CPU_PART_NEOVERSE_N2	0xD49
++#define ARM_CPU_PART_CORTEX_A78C	0xD4B
+ 
+ #define APM_CPU_PART_POTENZA		0x000
+ 
+@@ -112,6 +122,16 @@
+ #define MIDR_CORTEX_A75 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A75)
+ #define MIDR_CORTEX_A35 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A35)
+ #define MIDR_CORTEX_A55 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A55)
++#define MIDR_CORTEX_A76	MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A76)
++#define MIDR_NEOVERSE_N1 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N1)
++#define MIDR_CORTEX_A77	MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A77)
++#define MIDR_NEOVERSE_V1	MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_V1)
++#define MIDR_CORTEX_A78	MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A78)
++#define MIDR_CORTEX_X1	MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X1)
++#define MIDR_CORTEX_A710 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A710)
++#define MIDR_CORTEX_X2 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X2)
++#define MIDR_NEOVERSE_N2 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N2)
++#define MIDR_CORTEX_A78C	MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A78C)
+ #define MIDR_THUNDERX	MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX)
+ #define MIDR_THUNDERX_81XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_81XX)
+ #define MIDR_THUNDERX_83XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_83XX)
+diff --git a/arch/arm64/include/asm/fixmap.h b/arch/arm64/include/asm/fixmap.h
+index ec1e6d6fa14cc..3c962ef081f84 100644
+--- a/arch/arm64/include/asm/fixmap.h
++++ b/arch/arm64/include/asm/fixmap.h
+@@ -59,9 +59,11 @@ enum fixed_addresses {
+ #endif /* CONFIG_ACPI_APEI_GHES */
+ 
+ #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
++	FIX_ENTRY_TRAMP_TEXT3,
++	FIX_ENTRY_TRAMP_TEXT2,
++	FIX_ENTRY_TRAMP_TEXT1,
+ 	FIX_ENTRY_TRAMP_DATA,
+-	FIX_ENTRY_TRAMP_TEXT,
+-#define TRAMP_VALIAS		(__fix_to_virt(FIX_ENTRY_TRAMP_TEXT))
++#define TRAMP_VALIAS		(__fix_to_virt(FIX_ENTRY_TRAMP_TEXT1))
+ #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
+ 	__end_of_permanent_fixed_addresses,
+ 
+diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
+index 8d94404829f0f..be82119ed24a9 100644
+--- a/arch/arm64/include/asm/kvm_host.h
++++ b/arch/arm64/include/asm/kvm_host.h
+@@ -450,4 +450,9 @@ static inline int kvm_arm_have_ssbd(void)
+ 	}
+ }
+ 
++static inline enum mitigation_state kvm_arm_get_spectre_bhb_state(void)
++{
++	return arm64_get_spectre_bhb_state();
++}
++
+ #endif /* __ARM64_KVM_HOST_H__ */
+diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
+index 47ba6a57dc457..04c7c4596240a 100644
+--- a/arch/arm64/include/asm/kvm_mmu.h
++++ b/arch/arm64/include/asm/kvm_mmu.h
+@@ -358,7 +358,7 @@ static inline void *kvm_get_hyp_vector(void)
+ 	struct bp_hardening_data *data = arm64_get_bp_hardening_data();
+ 	void *vect = kvm_ksym_ref(__kvm_hyp_vector);
+ 
+-	if (data->fn) {
++	if (data->template_start) {
+ 		vect = __bp_harden_hyp_vecs_start +
+ 		       data->hyp_vectors_slot * SZ_2K;
+ 
+diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h
+index 6dd83d75b82ab..5a77dc775cc3c 100644
+--- a/arch/arm64/include/asm/mmu.h
++++ b/arch/arm64/include/asm/mmu.h
+@@ -35,7 +35,7 @@ typedef struct {
+  */
+ #define ASID(mm)	((mm)->context.id.counter & 0xffff)
+ 
+-static inline bool arm64_kernel_unmapped_at_el0(void)
++static __always_inline bool arm64_kernel_unmapped_at_el0(void)
+ {
+ 	return IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0) &&
+ 	       cpus_have_const_cap(ARM64_UNMAP_KERNEL_AT_EL0);
+@@ -46,6 +46,12 @@ typedef void (*bp_hardening_cb_t)(void);
+ struct bp_hardening_data {
+ 	int			hyp_vectors_slot;
+ 	bp_hardening_cb_t	fn;
++
++	/*
++	 * template_start is only used by the BHB mitigation to identify the
++	 * hyp_vectors_slot sequence.
++	 */
++	const char *template_start;
+ };
+ 
+ #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
+diff --git a/arch/arm64/include/asm/sections.h b/arch/arm64/include/asm/sections.h
+index 941267caa39c2..8d3f1eab58e04 100644
+--- a/arch/arm64/include/asm/sections.h
++++ b/arch/arm64/include/asm/sections.h
+@@ -28,5 +28,11 @@ extern char __initdata_begin[], __initdata_end[];
+ extern char __inittext_begin[], __inittext_end[];
+ extern char __irqentry_text_start[], __irqentry_text_end[];
+ extern char __mmuoff_data_start[], __mmuoff_data_end[];
++extern char __entry_tramp_text_start[], __entry_tramp_text_end[];
++
++static inline size_t entry_tramp_text_size(void)
++{
++	return __entry_tramp_text_end - __entry_tramp_text_start;
++}
+ 
+ #endif /* __ASM_SECTIONS_H */
+diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
+index 2564dd429ab68..3bbf0dc5ecad0 100644
+--- a/arch/arm64/include/asm/sysreg.h
++++ b/arch/arm64/include/asm/sysreg.h
+@@ -157,6 +157,7 @@
+ 
+ #define SYS_ID_AA64ISAR0_EL1		sys_reg(3, 0, 0, 6, 0)
+ #define SYS_ID_AA64ISAR1_EL1		sys_reg(3, 0, 0, 6, 1)
++#define SYS_ID_AA64ISAR2_EL1		sys_reg(3, 0, 0, 6, 2)
+ 
+ #define SYS_ID_AA64MMFR0_EL1		sys_reg(3, 0, 0, 7, 0)
+ #define SYS_ID_AA64MMFR1_EL1		sys_reg(3, 0, 0, 7, 1)
+@@ -403,6 +404,9 @@
+ #define ID_AA64ISAR1_JSCVT_SHIFT	12
+ #define ID_AA64ISAR1_DPB_SHIFT		0
+ 
++/* id_aa64isar2 */
++#define ID_AA64ISAR2_CLEARBHB_SHIFT	28
++
+ /* id_aa64pfr0 */
+ #define ID_AA64PFR0_CSV3_SHIFT		60
+ #define ID_AA64PFR0_CSV2_SHIFT		56
+@@ -448,6 +452,7 @@
+ #define ID_AA64MMFR0_TGRAN16_SUPPORTED	0x1
+ 
+ /* id_aa64mmfr1 */
++#define ID_AA64MMFR1_ECBHB_SHIFT	60
+ #define ID_AA64MMFR1_PAN_SHIFT		20
+ #define ID_AA64MMFR1_LOR_SHIFT		16
+ #define ID_AA64MMFR1_HPD_SHIFT		12
+diff --git a/arch/arm64/include/asm/vectors.h b/arch/arm64/include/asm/vectors.h
+new file mode 100644
+index 0000000000000..695583b9a145b
+--- /dev/null
++++ b/arch/arm64/include/asm/vectors.h
+@@ -0,0 +1,74 @@
++/* SPDX-License-Identifier: GPL-2.0-only */
++/*
++ * Copyright (C) 2022 ARM Ltd.
++ */
++#ifndef __ASM_VECTORS_H
++#define __ASM_VECTORS_H
++
++#include <linux/bug.h>
++#include <linux/percpu.h>
++
++#include <asm/fixmap.h>
++#include <asm/mmu.h>
++
++extern char vectors[];
++extern char tramp_vectors[];
++extern char __bp_harden_el1_vectors[];
++
++/*
++ * Note: the order of this enum corresponds to two arrays in entry.S:
++ * tramp_vecs and __bp_harden_el1_vectors. By default the canonical
++ * 'full fat' vectors are used directly.
++ */
++enum arm64_bp_harden_el1_vectors {
++#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
++	/*
++	 * Perform the BHB loop mitigation, before branching to the canonical
++	 * vectors.
++	 */
++	EL1_VECTOR_BHB_LOOP,
++
++	/*
++	 * Make the SMC call for firmware mitigation, before branching to the
++	 * canonical vectors.
++	 */
++	EL1_VECTOR_BHB_FW,
++
++	/*
++	 * Use the ClearBHB instruction, before branching to the canonical
++	 * vectors.
++	 */
++	EL1_VECTOR_BHB_CLEAR_INSN,
++#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
++
++	/*
++	 * Remap the kernel before branching to the canonical vectors.
++	 */
++	EL1_VECTOR_KPTI,
++};
++
++#ifndef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
++#define EL1_VECTOR_BHB_LOOP		-1
++#define EL1_VECTOR_BHB_FW		-1
++#define EL1_VECTOR_BHB_CLEAR_INSN	-1
++#endif /* !CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
++
++/* The vectors to use on return from EL0. e.g. to remap the kernel */
++DECLARE_PER_CPU_READ_MOSTLY(const char *, this_cpu_vector);
++
++#ifndef CONFIG_UNMAP_KERNEL_AT_EL0
++#define TRAMP_VALIAS	0
++#endif
++
++static inline const char *
++arm64_get_bp_hardening_vector(enum arm64_bp_harden_el1_vectors slot)
++{
++	if (arm64_kernel_unmapped_at_el0())
++		return (char *)TRAMP_VALIAS + SZ_2K * slot;
++
++	WARN_ON_ONCE(slot == EL1_VECTOR_KPTI);
++
++	return __bp_harden_el1_vectors + SZ_2K * slot;
++}
++
++#endif /* __ASM_VECTORS_H */
+diff --git a/arch/arm64/kernel/bpi.S b/arch/arm64/kernel/bpi.S
+index 4cae34e5a24e2..bd6ef8750f440 100644
+--- a/arch/arm64/kernel/bpi.S
++++ b/arch/arm64/kernel/bpi.S
+@@ -66,3 +66,58 @@ ENTRY(__smccc_workaround_1_smc_start)
+ 	ldp	x0, x1, [sp, #(8 * 2)]
+ 	add	sp, sp, #(8 * 4)
+ ENTRY(__smccc_workaround_1_smc_end)
++
++ENTRY(__smccc_workaround_3_smc_start)
++	sub     sp, sp, #(8 * 4)
++	stp     x2, x3, [sp, #(8 * 0)]
++	stp     x0, x1, [sp, #(8 * 2)]
++	mov     w0, #ARM_SMCCC_ARCH_WORKAROUND_3
++	smc     #0
++	ldp     x2, x3, [sp, #(8 * 0)]
++	ldp     x0, x1, [sp, #(8 * 2)]
++	add     sp, sp, #(8 * 4)
++ENTRY(__smccc_workaround_3_smc_end)
++
++ENTRY(__spectre_bhb_loop_k8_start)
++	sub     sp, sp, #(8 * 2)
++	stp     x0, x1, [sp, #(8 * 0)]
++	mov     x0, #8
++2:	b       . + 4
++	subs    x0, x0, #1
++	b.ne    2b
++	dsb     nsh
++	isb
++	ldp     x0, x1, [sp, #(8 * 0)]
++	add     sp, sp, #(8 * 2)
++ENTRY(__spectre_bhb_loop_k8_end)
++
++ENTRY(__spectre_bhb_loop_k24_start)
++	sub     sp, sp, #(8 * 2)
++	stp     x0, x1, [sp, #(8 * 0)]
++	mov     x0, #24
++2:	b       . + 4
++	subs    x0, x0, #1
++	b.ne    2b
++	dsb     nsh
++	isb
++	ldp     x0, x1, [sp, #(8 * 0)]
++	add     sp, sp, #(8 * 2)
++ENTRY(__spectre_bhb_loop_k24_end)
++
++ENTRY(__spectre_bhb_loop_k32_start)
++	sub     sp, sp, #(8 * 2)
++	stp     x0, x1, [sp, #(8 * 0)]
++	mov     x0, #32
++2:	b       . + 4
++	subs    x0, x0, #1
++	b.ne    2b
++	dsb     nsh
++	isb
++	ldp     x0, x1, [sp, #(8 * 0)]
++	add     sp, sp, #(8 * 2)
++ENTRY(__spectre_bhb_loop_k32_end)
++
++ENTRY(__spectre_bhb_clearbhb_start)
++	hint	#22	/* aka clearbhb */
++	isb
++ENTRY(__spectre_bhb_clearbhb_end)
+diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c
+index 7d15f4cb63937..ed627d44746ad 100644
+--- a/arch/arm64/kernel/cpu_errata.c
++++ b/arch/arm64/kernel/cpu_errata.c
+@@ -23,6 +23,7 @@
+ #include <asm/cpu.h>
+ #include <asm/cputype.h>
+ #include <asm/cpufeature.h>
++#include <asm/vectors.h>
+ 
+ static bool __maybe_unused
+ is_affected_midr_range(const struct arm64_cpu_capabilities *entry, int scope)
+@@ -85,6 +86,16 @@ DEFINE_PER_CPU_READ_MOSTLY(struct bp_hardening_data, bp_hardening_data);
+ #ifdef CONFIG_KVM
+ extern char __smccc_workaround_1_smc_start[];
+ extern char __smccc_workaround_1_smc_end[];
++extern char __smccc_workaround_3_smc_start[];
++extern char __smccc_workaround_3_smc_end[];
++extern char __spectre_bhb_loop_k8_start[];
++extern char __spectre_bhb_loop_k8_end[];
++extern char __spectre_bhb_loop_k24_start[];
++extern char __spectre_bhb_loop_k24_end[];
++extern char __spectre_bhb_loop_k32_start[];
++extern char __spectre_bhb_loop_k32_end[];
++extern char __spectre_bhb_clearbhb_start[];
++extern char __spectre_bhb_clearbhb_end[];
+ 
+ static void __copy_hyp_vect_bpi(int slot, const char *hyp_vecs_start,
+ 				const char *hyp_vecs_end)
+@@ -98,12 +109,14 @@ static void __copy_hyp_vect_bpi(int slot, const char *hyp_vecs_start,
+ 	flush_icache_range((uintptr_t)dst, (uintptr_t)dst + SZ_2K);
+ }
+ 
++static DEFINE_SPINLOCK(bp_lock);
++static int last_slot = -1;
++
+ static void install_bp_hardening_cb(bp_hardening_cb_t fn,
+ 				    const char *hyp_vecs_start,
+ 				    const char *hyp_vecs_end)
+ {
+-	static int last_slot = -1;
+-	static DEFINE_SPINLOCK(bp_lock);
++
+ 	int cpu, slot = -1;
+ 
+ 	spin_lock(&bp_lock);
+@@ -124,6 +137,7 @@ static void install_bp_hardening_cb(bp_hardening_cb_t fn,
+ 
+ 	__this_cpu_write(bp_hardening_data.hyp_vectors_slot, slot);
+ 	__this_cpu_write(bp_hardening_data.fn, fn);
++	__this_cpu_write(bp_hardening_data.template_start, hyp_vecs_start);
+ 	spin_unlock(&bp_lock);
+ }
+ #else
+@@ -712,6 +726,21 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
+ 		.matches = has_ssbd_mitigation,
+ 		.midr_range_list = arm64_ssb_cpus,
+ 	},
++#ifdef CONFIG_ARM64_ERRATUM_1188873
++	{
++		/* Cortex-A76 r0p0 to r2p0 */
++		.desc = "ARM erratum 1188873",
++		.capability = ARM64_WORKAROUND_1188873,
++		ERRATA_MIDR_RANGE(MIDR_CORTEX_A76, 0, 0, 2, 0),
++	},
++#endif
++	{
++		.desc = "Spectre-BHB",
++		.capability = ARM64_SPECTRE_BHB,
++		.type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
++		.matches = is_spectre_bhb_affected,
++		.cpu_enable = spectre_bhb_enable_mitigation,
++	},
+ 	{
+ 	}
+ };
+@@ -722,14 +751,39 @@ ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr,
+ 	return sprintf(buf, "Mitigation: __user pointer sanitization\n");
+ }
+ 
++static const char *get_bhb_affected_string(enum mitigation_state bhb_state)
++{
++	switch (bhb_state) {
++	case SPECTRE_UNAFFECTED:
++		return "";
++	default:
++	case SPECTRE_VULNERABLE:
++		return ", but not BHB";
++	case SPECTRE_MITIGATED:
++		return ", BHB";
++	}
++}
++
+ ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr,
+ 		char *buf)
+ {
+-	if (__spectrev2_safe)
+-		return sprintf(buf, "Not affected\n");
++	enum mitigation_state bhb_state = arm64_get_spectre_bhb_state();
++	const char *bhb_str = get_bhb_affected_string(bhb_state);
++	const char *v2_str = "Branch predictor hardening";
++
++	if (__spectrev2_safe) {
++		if (bhb_state == SPECTRE_UNAFFECTED)
++			return sprintf(buf, "Not affected\n");
++
++		/*
++		 * Platforms affected by Spectre-BHB can't report
++		 * "Not affected" for Spectre-v2.
++		 */
++		v2_str = "CSV2";
++	}
+ 
+ 	if (__hardenbp_enab)
+-		return sprintf(buf, "Mitigation: Branch predictor hardening\n");
++		return sprintf(buf, "Mitigation: %s%s\n", v2_str, bhb_str);
+ 
+ 	return sprintf(buf, "Vulnerable\n");
+ }
+@@ -750,3 +804,334 @@ ssize_t cpu_show_spec_store_bypass(struct device *dev,
+ 
+ 	return sprintf(buf, "Vulnerable\n");
+ }
++
++/*
++ * We try to ensure that the mitigation state can never change as the result of
++ * onlining a late CPU.
++ */
++static void update_mitigation_state(enum mitigation_state *oldp,
++				    enum mitigation_state new)
++{
++	enum mitigation_state state;
++
++	do {
++		state = READ_ONCE(*oldp);
++		if (new <= state)
++			break;
++	} while (cmpxchg_relaxed(oldp, state, new) != state);
++}
++
++/*
++ * Spectre BHB.
++ *
++ * A CPU is either:
++ * - Mitigated by a branchy loop a CPU specific number of times, and listed
++ *   in our "loop mitigated list".
++ * - Mitigated in software by the firmware Spectre v2 call.
++ * - Has the ClearBHB instruction to perform the mitigation.
++ * - Has the 'Exception Clears Branch History Buffer' (ECBHB) feature, so no
++ *   software mitigation in the vectors is needed.
++ * - Has CSV2.3, so is unaffected.
++ */
++static enum mitigation_state spectre_bhb_state;
++
++enum mitigation_state arm64_get_spectre_bhb_state(void)
++{
++	return spectre_bhb_state;
++}
++
++/*
++ * This must be called with SCOPE_LOCAL_CPU for each type of CPU, before any
++ * SCOPE_SYSTEM call will give the right answer.
++ */
++u8 spectre_bhb_loop_affected(int scope)
++{
++	u8 k = 0;
++	static u8 max_bhb_k;
++
++	if (scope == SCOPE_LOCAL_CPU) {
++		static const struct midr_range spectre_bhb_k32_list[] = {
++			MIDR_ALL_VERSIONS(MIDR_CORTEX_A78),
++			MIDR_ALL_VERSIONS(MIDR_CORTEX_A78C),
++			MIDR_ALL_VERSIONS(MIDR_CORTEX_X1),
++			MIDR_ALL_VERSIONS(MIDR_CORTEX_A710),
++			MIDR_ALL_VERSIONS(MIDR_CORTEX_X2),
++			MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2),
++			MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V1),
++			{},
++		};
++		static const struct midr_range spectre_bhb_k24_list[] = {
++			MIDR_ALL_VERSIONS(MIDR_CORTEX_A77),
++			MIDR_ALL_VERSIONS(MIDR_CORTEX_A76),
++			MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1),
++			{},
++		};
++		static const struct midr_range spectre_bhb_k8_list[] = {
++			MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
++			MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
++			{},
++		};
++
++		if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k32_list))
++			k = 32;
++		else if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k24_list))
++			k = 24;
++		else if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k8_list))
++			k =  8;
++
++		max_bhb_k = max(max_bhb_k, k);
++	} else {
++		k = max_bhb_k;
++	}
++
++	return k;
++}
++
++static enum mitigation_state spectre_bhb_get_cpu_fw_mitigation_state(void)
++{
++	int ret;
++	struct arm_smccc_res res;
++
++	if (psci_ops.smccc_version == SMCCC_VERSION_1_0)
++		return SPECTRE_VULNERABLE;
++
++	switch (psci_ops.conduit) {
++	case PSCI_CONDUIT_HVC:
++		arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
++				  ARM_SMCCC_ARCH_WORKAROUND_3, &res);
++		break;
++
++	case PSCI_CONDUIT_SMC:
++		arm_smccc_1_1_smc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
++				  ARM_SMCCC_ARCH_WORKAROUND_3, &res);
++		break;
++
++	default:
++		return SPECTRE_VULNERABLE;
++	}
++
++	ret = res.a0;
++	switch (ret) {
++	case SMCCC_RET_SUCCESS:
++		return SPECTRE_MITIGATED;
++	case SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED:
++		return SPECTRE_UNAFFECTED;
++	default:
++	case SMCCC_RET_NOT_SUPPORTED:
++		return SPECTRE_VULNERABLE;
++	}
++}
++
++static bool is_spectre_bhb_fw_affected(int scope)
++{
++	static bool system_affected;
++	enum mitigation_state fw_state;
++	bool has_smccc = (psci_ops.smccc_version >= SMCCC_VERSION_1_1);
++	static const struct midr_range spectre_bhb_firmware_mitigated_list[] = {
++		MIDR_ALL_VERSIONS(MIDR_CORTEX_A73),
++		MIDR_ALL_VERSIONS(MIDR_CORTEX_A75),
++		{},
++	};
++	bool cpu_in_list = is_midr_in_range_list(read_cpuid_id(),
++					 spectre_bhb_firmware_mitigated_list);
++
++	if (scope != SCOPE_LOCAL_CPU)
++		return system_affected;
++
++	fw_state = spectre_bhb_get_cpu_fw_mitigation_state();
++	if (cpu_in_list || (has_smccc && fw_state == SPECTRE_MITIGATED)) {
++		system_affected = true;
++		return true;
++	}
++
++	return false;
++}
++
++static bool supports_ecbhb(int scope)
++{
++	u64 mmfr1;
++
++	if (scope == SCOPE_LOCAL_CPU)
++		mmfr1 = read_sysreg_s(SYS_ID_AA64MMFR1_EL1);
++	else
++		mmfr1 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
++
++	return cpuid_feature_extract_unsigned_field(mmfr1,
++						    ID_AA64MMFR1_ECBHB_SHIFT);
++}
++
++bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry,
++			     int scope)
++{
++	WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
++
++	if (supports_csv2p3(scope))
++		return false;
++
++	if (supports_clearbhb(scope))
++		return true;
++
++	if (spectre_bhb_loop_affected(scope))
++		return true;
++
++	if (is_spectre_bhb_fw_affected(scope))
++		return true;
++
++	return false;
++}
++
++static void this_cpu_set_vectors(enum arm64_bp_harden_el1_vectors slot)
++{
++	const char *v = arm64_get_bp_hardening_vector(slot);
++
++	if (slot < 0)
++		return;
++
++	__this_cpu_write(this_cpu_vector, v);
++
++	/*
++	 * When KPTI is in use, the vectors are switched when exiting to
++	 * user-space.
++	 */
++	if (arm64_kernel_unmapped_at_el0())
++		return;
++
++	write_sysreg(v, vbar_el1);
++	isb();
++}
++
++#ifdef CONFIG_KVM
++static const char *kvm_bhb_get_vecs_end(const char *start)
++{
++	if (start == __smccc_workaround_3_smc_start)
++		return __smccc_workaround_3_smc_end;
++	else if (start == __spectre_bhb_loop_k8_start)
++		return __spectre_bhb_loop_k8_end;
++	else if (start == __spectre_bhb_loop_k24_start)
++		return __spectre_bhb_loop_k24_end;
++	else if (start == __spectre_bhb_loop_k32_start)
++		return __spectre_bhb_loop_k32_end;
++	else if (start == __spectre_bhb_clearbhb_start)
++		return __spectre_bhb_clearbhb_end;
++
++	return NULL;
++}
++
++static void kvm_setup_bhb_slot(const char *hyp_vecs_start)
++{
++	int cpu, slot = -1;
++	const char *hyp_vecs_end;
++
++	if (!IS_ENABLED(CONFIG_KVM) || !is_hyp_mode_available())
++		return;
++
++	hyp_vecs_end = kvm_bhb_get_vecs_end(hyp_vecs_start);
++	if (WARN_ON_ONCE(!hyp_vecs_start || !hyp_vecs_end))
++		return;
++
++	spin_lock(&bp_lock);
++	for_each_possible_cpu(cpu) {
++		if (per_cpu(bp_hardening_data.template_start, cpu) == hyp_vecs_start) {
++			slot = per_cpu(bp_hardening_data.hyp_vectors_slot, cpu);
++			break;
++		}
++	}
++
++	if (slot == -1) {
++		last_slot++;
++		BUG_ON(((__bp_harden_hyp_vecs_end - __bp_harden_hyp_vecs_start)
++			/ SZ_2K) <= last_slot);
++		slot = last_slot;
++		__copy_hyp_vect_bpi(slot, hyp_vecs_start, hyp_vecs_end);
++	}
++
++	__this_cpu_write(bp_hardening_data.hyp_vectors_slot, slot);
++	__this_cpu_write(bp_hardening_data.template_start, hyp_vecs_start);
++	spin_unlock(&bp_lock);
++}
++#else
++#define __smccc_workaround_3_smc_start NULL
++#define __spectre_bhb_loop_k8_start NULL
++#define __spectre_bhb_loop_k24_start NULL
++#define __spectre_bhb_loop_k32_start NULL
++#define __spectre_bhb_clearbhb_start NULL
++
++static void kvm_setup_bhb_slot(const char *hyp_vecs_start) { };
++#endif
++
++void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *entry)
++{
++	enum mitigation_state fw_state, state = SPECTRE_VULNERABLE;
++
++	if (!is_spectre_bhb_affected(entry, SCOPE_LOCAL_CPU))
++		return;
++
++	if (!__spectrev2_safe &&  !__hardenbp_enab) {
++		/* No point mitigating Spectre-BHB alone. */
++	} else if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY)) {
++		pr_info_once("spectre-bhb mitigation disabled by compile time option\n");
++	} else if (cpu_mitigations_off()) {
++		pr_info_once("spectre-bhb mitigation disabled by command line option\n");
++	} else if (supports_ecbhb(SCOPE_LOCAL_CPU)) {
++		state = SPECTRE_MITIGATED;
++	} else if (supports_clearbhb(SCOPE_LOCAL_CPU)) {
++		kvm_setup_bhb_slot(__spectre_bhb_clearbhb_start);
++		this_cpu_set_vectors(EL1_VECTOR_BHB_CLEAR_INSN);
++
++		state = SPECTRE_MITIGATED;
++	} else if (spectre_bhb_loop_affected(SCOPE_LOCAL_CPU)) {
++		switch (spectre_bhb_loop_affected(SCOPE_SYSTEM)) {
++		case 8:
++			kvm_setup_bhb_slot(__spectre_bhb_loop_k8_start);
++			break;
++		case 24:
++			kvm_setup_bhb_slot(__spectre_bhb_loop_k24_start);
++			break;
++		case 32:
++			kvm_setup_bhb_slot(__spectre_bhb_loop_k32_start);
++			break;
++		default:
++			WARN_ON_ONCE(1);
++		}
++		this_cpu_set_vectors(EL1_VECTOR_BHB_LOOP);
++
++		state = SPECTRE_MITIGATED;
++	} else if (is_spectre_bhb_fw_affected(SCOPE_LOCAL_CPU)) {
++		fw_state = spectre_bhb_get_cpu_fw_mitigation_state();
++		if (fw_state == SPECTRE_MITIGATED) {
++			kvm_setup_bhb_slot(__smccc_workaround_3_smc_start);
++			this_cpu_set_vectors(EL1_VECTOR_BHB_FW);
++
++			/*
++			 * With WA3 in the vectors, the WA1 calls can be
++			 * removed.
++			 */
++			__this_cpu_write(bp_hardening_data.fn, NULL);
++
++			state = SPECTRE_MITIGATED;
++		}
++	}
++
++	update_mitigation_state(&spectre_bhb_state, state);
++}
++
++/* Patched to correct the immediate */
++void __init spectre_bhb_patch_loop_iter(struct alt_instr *alt,
++					__le32 *origptr, __le32 *updptr, int nr_inst)
++{
++	u8 rd;
++	u32 insn;
++	u16 loop_count = spectre_bhb_loop_affected(SCOPE_SYSTEM);
++
++	BUG_ON(nr_inst != 1); /* MOV -> MOV */
++
++	if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY))
++		return;
++
++	insn = le32_to_cpu(*origptr);
++	rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, insn);
++	insn = aarch64_insn_gen_movewide(rd, loop_count, 0,
++					 AARCH64_INSN_VARIANT_64BIT,
++					 AARCH64_INSN_MOVEWIDE_ZERO);
++	*updptr++ = cpu_to_le32(insn);
++}
+diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
+index 1481e18aa5ca0..b6922f33d306b 100644
+--- a/arch/arm64/kernel/cpufeature.c
++++ b/arch/arm64/kernel/cpufeature.c
+@@ -20,11 +20,13 @@
+ 
+ #include <linux/bsearch.h>
+ #include <linux/cpumask.h>
++#include <linux/percpu.h>
+ #include <linux/sort.h>
+ #include <linux/stop_machine.h>
+ #include <linux/types.h>
+ #include <linux/mm.h>
+ #include <linux/cpu.h>
++
+ #include <asm/cpu.h>
+ #include <asm/cpufeature.h>
+ #include <asm/cpu_ops.h>
+@@ -32,6 +34,7 @@
+ #include <asm/processor.h>
+ #include <asm/sysreg.h>
+ #include <asm/traps.h>
++#include <asm/vectors.h>
+ #include <asm/virt.h>
+ 
+ unsigned long elf_hwcap __read_mostly;
+@@ -50,6 +53,8 @@ unsigned int compat_elf_hwcap2 __read_mostly;
+ DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS);
+ EXPORT_SYMBOL(cpu_hwcaps);
+ 
++DEFINE_PER_CPU_READ_MOSTLY(const char *, this_cpu_vector) = vectors;
++
+ static int dump_cpu_hwcaps(struct notifier_block *self, unsigned long v, void *p)
+ {
+ 	/* file-wide pr_fmt adds "CPU features: " prefix */
+@@ -129,6 +134,11 @@ static const struct arm64_ftr_bits ftr_id_aa64isar1[] = {
+ 	ARM64_FTR_END,
+ };
+ 
++static const struct arm64_ftr_bits ftr_id_aa64isar2[] = {
++	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_HIGHER_SAFE, ID_AA64ISAR2_CLEARBHB_SHIFT, 4, 0),
++	ARM64_FTR_END,
++};
++
+ static const struct arm64_ftr_bits ftr_id_aa64pfr0[] = {
+ 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_CSV3_SHIFT, 4, 0),
+ 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_CSV2_SHIFT, 4, 0),
+@@ -356,6 +366,7 @@ static const struct __ftr_reg_entry {
+ 	/* Op1 = 0, CRn = 0, CRm = 6 */
+ 	ARM64_FTR_REG(SYS_ID_AA64ISAR0_EL1, ftr_id_aa64isar0),
+ 	ARM64_FTR_REG(SYS_ID_AA64ISAR1_EL1, ftr_id_aa64isar1),
++	ARM64_FTR_REG(SYS_ID_AA64ISAR2_EL1, ftr_id_aa64isar2),
+ 
+ 	/* Op1 = 0, CRn = 0, CRm = 7 */
+ 	ARM64_FTR_REG(SYS_ID_AA64MMFR0_EL1, ftr_id_aa64mmfr0),
+@@ -501,6 +512,7 @@ void __init init_cpu_features(struct cpuinfo_arm64 *info)
+ 	init_cpu_ftr_reg(SYS_ID_AA64DFR1_EL1, info->reg_id_aa64dfr1);
+ 	init_cpu_ftr_reg(SYS_ID_AA64ISAR0_EL1, info->reg_id_aa64isar0);
+ 	init_cpu_ftr_reg(SYS_ID_AA64ISAR1_EL1, info->reg_id_aa64isar1);
++	init_cpu_ftr_reg(SYS_ID_AA64ISAR2_EL1, info->reg_id_aa64isar2);
+ 	init_cpu_ftr_reg(SYS_ID_AA64MMFR0_EL1, info->reg_id_aa64mmfr0);
+ 	init_cpu_ftr_reg(SYS_ID_AA64MMFR1_EL1, info->reg_id_aa64mmfr1);
+ 	init_cpu_ftr_reg(SYS_ID_AA64MMFR2_EL1, info->reg_id_aa64mmfr2);
+@@ -612,6 +624,8 @@ void update_cpu_features(int cpu,
+ 				      info->reg_id_aa64isar0, boot->reg_id_aa64isar0);
+ 	taint |= check_update_ftr_reg(SYS_ID_AA64ISAR1_EL1, cpu,
+ 				      info->reg_id_aa64isar1, boot->reg_id_aa64isar1);
++	taint |= check_update_ftr_reg(SYS_ID_AA64ISAR2_EL1, cpu,
++				      info->reg_id_aa64isar2, boot->reg_id_aa64isar2);
+ 
+ 	/*
+ 	 * Differing PARange support is fine as long as all peripherals and
+@@ -732,6 +746,7 @@ static u64 __read_sysreg_by_encoding(u32 sys_id)
+ 	read_sysreg_case(SYS_ID_AA64MMFR2_EL1);
+ 	read_sysreg_case(SYS_ID_AA64ISAR0_EL1);
+ 	read_sysreg_case(SYS_ID_AA64ISAR1_EL1);
++	read_sysreg_case(SYS_ID_AA64ISAR2_EL1);
+ 
+ 	read_sysreg_case(SYS_CNTFRQ_EL0);
+ 	read_sysreg_case(SYS_CTR_EL0);
+@@ -892,6 +907,12 @@ kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused)
+ 	static bool kpti_applied = false;
+ 	int cpu = smp_processor_id();
+ 
++	if (__this_cpu_read(this_cpu_vector) == vectors) {
++		const char *v = arm64_get_bp_hardening_vector(EL1_VECTOR_KPTI);
++
++		__this_cpu_write(this_cpu_vector, v);
++	}
++
+ 	if (kpti_applied)
+ 		return;
+ 
+diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c
+index 9ff64e04e63d6..6b7db546efda5 100644
+--- a/arch/arm64/kernel/cpuinfo.c
++++ b/arch/arm64/kernel/cpuinfo.c
+@@ -333,6 +333,7 @@ static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info)
+ 	info->reg_id_aa64dfr1 = read_cpuid(ID_AA64DFR1_EL1);
+ 	info->reg_id_aa64isar0 = read_cpuid(ID_AA64ISAR0_EL1);
+ 	info->reg_id_aa64isar1 = read_cpuid(ID_AA64ISAR1_EL1);
++	info->reg_id_aa64isar2 = read_cpuid(ID_AA64ISAR2_EL1);
+ 	info->reg_id_aa64mmfr0 = read_cpuid(ID_AA64MMFR0_EL1);
+ 	info->reg_id_aa64mmfr1 = read_cpuid(ID_AA64MMFR1_EL1);
+ 	info->reg_id_aa64mmfr2 = read_cpuid(ID_AA64MMFR2_EL1);
+diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
+index c1ffa95c0ad24..f526148d14bd8 100644
+--- a/arch/arm64/kernel/entry.S
++++ b/arch/arm64/kernel/entry.S
+@@ -74,18 +74,21 @@
+ 
+ 	.macro kernel_ventry, el, label, regsize = 64
+ 	.align 7
+-#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+-alternative_if ARM64_UNMAP_KERNEL_AT_EL0
++.Lventry_start\@:
+ 	.if	\el == 0
++	/*
++	 * This must be the first instruction of the EL0 vector entries. It is
++	 * skipped by the trampoline vectors, to trigger the cleanup.
++	 */
++	b	.Lskip_tramp_vectors_cleanup\@
+ 	.if	\regsize == 64
+ 	mrs	x30, tpidrro_el0
+ 	msr	tpidrro_el0, xzr
+ 	.else
+ 	mov	x30, xzr
+ 	.endif
++.Lskip_tramp_vectors_cleanup\@:
+ 	.endif
+-alternative_else_nop_endif
+-#endif
+ 
+ 	sub	sp, sp, #S_FRAME_SIZE
+ #ifdef CONFIG_VMAP_STACK
+@@ -131,11 +134,15 @@ alternative_else_nop_endif
+ 	mrs	x0, tpidrro_el0
+ #endif
+ 	b	el\()\el\()_\label
++.org .Lventry_start\@ + 128	// Did we overflow the ventry slot?
+ 	.endm
+ 
+-	.macro tramp_alias, dst, sym
++	.macro tramp_alias, dst, sym, tmp
+ 	mov_q	\dst, TRAMP_VALIAS
+-	add	\dst, \dst, #(\sym - .entry.tramp.text)
++	adr_l	\tmp, \sym
++	add	\dst, \dst, \tmp
++	adr_l	\tmp, .entry.tramp.text
++	sub	\dst, \dst, \tmp
+ 	.endm
+ 
+ 	// This macro corrupts x0-x3. It is the caller's duty
+@@ -350,21 +357,25 @@ alternative_else_nop_endif
+ 	ldp	x24, x25, [sp, #16 * 12]
+ 	ldp	x26, x27, [sp, #16 * 13]
+ 	ldp	x28, x29, [sp, #16 * 14]
+-	ldr	lr, [sp, #S_LR]
+-	add	sp, sp, #S_FRAME_SIZE		// restore sp
+ 
+ 	.if	\el == 0
+-alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0
++alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0
++	ldr	lr, [sp, #S_LR]
++	add	sp, sp, #S_FRAME_SIZE		// restore sp
++	eret
++alternative_else_nop_endif
+ #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+ 	bne	4f
+-	msr	far_el1, x30
+-	tramp_alias	x30, tramp_exit_native
++	msr	far_el1, x29
++	tramp_alias	x30, tramp_exit_native, x29
+ 	br	x30
+ 4:
+-	tramp_alias	x30, tramp_exit_compat
++	tramp_alias	x30, tramp_exit_compat, x29
+ 	br	x30
+ #endif
+ 	.else
++	ldr	lr, [sp, #S_LR]
++	add	sp, sp, #S_FRAME_SIZE		// restore sp
+ 	eret
+ 	.endif
+ 	.endm
+@@ -972,12 +983,7 @@ __ni_sys_trace:
+ 
+ 	.popsection				// .entry.text
+ 
+-#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+-/*
+- * Exception vectors trampoline.
+- */
+-	.pushsection ".entry.tramp.text", "ax"
+-
++	// Move from tramp_pg_dir to swapper_pg_dir
+ 	.macro tramp_map_kernel, tmp
+ 	mrs	\tmp, ttbr1_el1
+ 	sub	\tmp, \tmp, #(SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE)
+@@ -1009,12 +1015,47 @@ alternative_else_nop_endif
+ 	 */
+ 	.endm
+ 
+-	.macro tramp_ventry, regsize = 64
++	.macro tramp_data_page	dst
++	adr_l	\dst, .entry.tramp.text
++	sub	\dst, \dst, PAGE_SIZE
++	.endm
++
++	.macro tramp_data_read_var	dst, var
++#ifdef CONFIG_RANDOMIZE_BASE
++	tramp_data_page		\dst
++	add	\dst, \dst, #:lo12:__entry_tramp_data_\var
++	ldr	\dst, [\dst]
++#else
++	ldr	\dst, =\var
++#endif
++	.endm
++
++#define BHB_MITIGATION_NONE	0
++#define BHB_MITIGATION_LOOP	1
++#define BHB_MITIGATION_FW	2
++#define BHB_MITIGATION_INSN	3
++
++	.macro tramp_ventry, vector_start, regsize, kpti, bhb
+ 	.align	7
+ 1:
+ 	.if	\regsize == 64
+ 	msr	tpidrro_el0, x30	// Restored in kernel_ventry
+ 	.endif
++
++	.if	\bhb == BHB_MITIGATION_LOOP
++	/*
++	 * This sequence must appear before the first indirect branch. i.e. the
++	 * ret out of tramp_ventry. It appears here because x30 is free.
++	 */
++	__mitigate_spectre_bhb_loop	x30
++	.endif // \bhb == BHB_MITIGATION_LOOP
++
++	.if	\bhb == BHB_MITIGATION_INSN
++	clearbhb
++	isb
++	.endif // \bhb == BHB_MITIGATION_INSN
++
++	.if	\kpti == 1
+ 	/*
+ 	 * Defend against branch aliasing attacks by pushing a dummy
+ 	 * entry onto the return stack and using a RET instruction to
+@@ -1024,43 +1065,75 @@ alternative_else_nop_endif
+ 	b	.
+ 2:
+ 	tramp_map_kernel	x30
+-#ifdef CONFIG_RANDOMIZE_BASE
+-	adr	x30, tramp_vectors + PAGE_SIZE
+ alternative_insn isb, nop, ARM64_WORKAROUND_QCOM_FALKOR_E1003
+-	ldr	x30, [x30]
+-#else
+-	ldr	x30, =vectors
+-#endif
+-	prfm	plil1strm, [x30, #(1b - tramp_vectors)]
++	tramp_data_read_var	x30, vectors
++	prfm	plil1strm, [x30, #(1b - \vector_start)]
+ 	msr	vbar_el1, x30
+-	add	x30, x30, #(1b - tramp_vectors)
+ 	isb
++	.else
++	ldr	x30, =vectors
++	.endif // \kpti == 1
++
++	.if	\bhb == BHB_MITIGATION_FW
++	/*
++	 * The firmware sequence must appear before the first indirect branch.
++	 * i.e. the ret out of tramp_ventry. But it also needs the stack to be
++	 * mapped to save/restore the registers the SMC clobbers.
++	 */
++	__mitigate_spectre_bhb_fw
++	.endif // \bhb == BHB_MITIGATION_FW
++
++	add	x30, x30, #(1b - \vector_start + 4)
+ 	ret
++.org 1b + 128	// Did we overflow the ventry slot?
+ 	.endm
+ 
+ 	.macro tramp_exit, regsize = 64
+-	adr	x30, tramp_vectors
++	tramp_data_read_var	x30, this_cpu_vector
++alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
++	mrs	x29, tpidr_el1
++alternative_else
++	mrs	x29, tpidr_el2
++alternative_endif
++	ldr	x30, [x30, x29]
++
+ 	msr	vbar_el1, x30
+-	tramp_unmap_kernel	x30
++	ldr	lr, [sp, #S_LR]
++	tramp_unmap_kernel	x29
+ 	.if	\regsize == 64
+-	mrs	x30, far_el1
++	mrs	x29, far_el1
+ 	.endif
++	add	sp, sp, #S_FRAME_SIZE		// restore sp
+ 	eret
+ 	.endm
+ 
+-	.align	11
+-ENTRY(tramp_vectors)
++	.macro	generate_tramp_vector,	kpti, bhb
++.Lvector_start\@:
+ 	.space	0x400
+ 
+-	tramp_ventry
+-	tramp_ventry
+-	tramp_ventry
+-	tramp_ventry
++	.rept	4
++	tramp_ventry	.Lvector_start\@, 64, \kpti, \bhb
++	.endr
++	.rept	4
++	tramp_ventry	.Lvector_start\@, 32, \kpti, \bhb
++	.endr
++	.endm
+ 
+-	tramp_ventry	32
+-	tramp_ventry	32
+-	tramp_ventry	32
+-	tramp_ventry	32
++#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
++/*
++ * Exception vectors trampoline.
++ * The order must match __bp_harden_el1_vectors and the
++ * arm64_bp_harden_el1_vectors enum.
++ */
++	.pushsection ".entry.tramp.text", "ax"
++	.align	11
++ENTRY(tramp_vectors)
++#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
++	generate_tramp_vector	kpti=1, bhb=BHB_MITIGATION_LOOP
++	generate_tramp_vector	kpti=1, bhb=BHB_MITIGATION_FW
++	generate_tramp_vector	kpti=1, bhb=BHB_MITIGATION_INSN
++#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
++	generate_tramp_vector	kpti=1, bhb=BHB_MITIGATION_NONE
+ END(tramp_vectors)
+ 
+ ENTRY(tramp_exit_native)
+@@ -1078,11 +1151,54 @@ END(tramp_exit_compat)
+ 	.align PAGE_SHIFT
+ 	.globl	__entry_tramp_data_start
+ __entry_tramp_data_start:
++__entry_tramp_data_vectors:
+ 	.quad	vectors
++#ifdef CONFIG_ARM_SDE_INTERFACE
++__entry_tramp_data___sdei_asm_trampoline_next_handler:
++	.quad	__sdei_asm_handler
++#endif /* CONFIG_ARM_SDE_INTERFACE */
++__entry_tramp_data_this_cpu_vector:
++	.quad	this_cpu_vector
+ 	.popsection				// .rodata
+ #endif /* CONFIG_RANDOMIZE_BASE */
+ #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
+ 
++/*
++ * Exception vectors for spectre mitigations on entry from EL1 when
++ * kpti is not in use.
++ */
++	.macro generate_el1_vector, bhb
++.Lvector_start\@:
++	kernel_ventry	1, sync_invalid			// Synchronous EL1t
++	kernel_ventry	1, irq_invalid			// IRQ EL1t
++	kernel_ventry	1, fiq_invalid			// FIQ EL1t
++	kernel_ventry	1, error_invalid		// Error EL1t
++
++	kernel_ventry	1, sync				// Synchronous EL1h
++	kernel_ventry	1, irq				// IRQ EL1h
++	kernel_ventry	1, fiq_invalid			// FIQ EL1h
++	kernel_ventry	1, error_invalid		// Error EL1h
++
++	.rept	4
++	tramp_ventry	.Lvector_start\@, 64, 0, \bhb
++	.endr
++	.rept 4
++	tramp_ventry	.Lvector_start\@, 32, 0, \bhb
++	.endr
++	.endm
++
++/* The order must match tramp_vecs and the arm64_bp_harden_el1_vectors enum. */
++	.pushsection ".entry.text", "ax"
++	.align	11
++ENTRY(__bp_harden_el1_vectors)
++#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
++	generate_el1_vector	bhb=BHB_MITIGATION_LOOP
++	generate_el1_vector	bhb=BHB_MITIGATION_FW
++	generate_el1_vector	bhb=BHB_MITIGATION_INSN
++#endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
++END(__bp_harden_el1_vectors)
++	.popsection
++
+ /*
+  * Special system call wrappers.
+  */
+diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
+index 4c11d3e64aef4..6543c58f26ec5 100644
+--- a/arch/arm64/kernel/vmlinux.lds.S
++++ b/arch/arm64/kernel/vmlinux.lds.S
+@@ -258,7 +258,7 @@ ASSERT(__hibernate_exit_text_end - (__hibernate_exit_text_start & ~(SZ_4K - 1))
+ 	<= SZ_4K, "Hibernate exit text too big or misaligned")
+ #endif
+ #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+-ASSERT((__entry_tramp_text_end - __entry_tramp_text_start) == PAGE_SIZE,
++ASSERT((__entry_tramp_text_end - __entry_tramp_text_start) <= 3*PAGE_SIZE,
+ 	"Entry trampoline text too big")
+ #endif
+ /*
+diff --git a/arch/arm64/kvm/hyp/hyp-entry.S b/arch/arm64/kvm/hyp/hyp-entry.S
+index 5e041eabdd03e..8086294aedea7 100644
+--- a/arch/arm64/kvm/hyp/hyp-entry.S
++++ b/arch/arm64/kvm/hyp/hyp-entry.S
+@@ -135,6 +135,10 @@ el1_hvc_guest:
+ 	/* ARM_SMCCC_ARCH_WORKAROUND_2 handling */
+ 	eor	w1, w1, #(ARM_SMCCC_ARCH_WORKAROUND_1 ^ \
+ 			  ARM_SMCCC_ARCH_WORKAROUND_2)
++	cbz	w1, wa_epilogue
++
++	eor	w1, w1, #(ARM_SMCCC_ARCH_WORKAROUND_2 ^ \
++			  ARM_SMCCC_ARCH_WORKAROUND_3)
+ 	cbnz	w1, el1_trap
+ 
+ #ifdef CONFIG_ARM64_SSBD
+diff --git a/arch/arm64/kvm/hyp/switch.c b/arch/arm64/kvm/hyp/switch.c
+index 99ae75a43985c..0f05f402e04a1 100644
+--- a/arch/arm64/kvm/hyp/switch.c
++++ b/arch/arm64/kvm/hyp/switch.c
+@@ -27,6 +27,7 @@
+ #include <asm/kvm_emulate.h>
+ #include <asm/kvm_hyp.h>
+ #include <asm/fpsimd.h>
++#include <asm/vectors.h>
+ 
+ extern struct exception_table_entry __start___kvm_ex_table;
+ extern struct exception_table_entry __stop___kvm_ex_table;
+@@ -110,17 +111,21 @@ static void __hyp_text __activate_traps(struct kvm_vcpu *vcpu)
+ 
+ static void __hyp_text __deactivate_traps_vhe(void)
+ {
+-	extern char vectors[];	/* kernel exception vectors */
++	const char *host_vectors = vectors;
+ 	u64 mdcr_el2 = read_sysreg(mdcr_el2);
+ 
+ 	mdcr_el2 &= MDCR_EL2_HPMN_MASK |
+ 		    MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT |
+ 		    MDCR_EL2_TPMS;
+ 
++
+ 	write_sysreg(mdcr_el2, mdcr_el2);
+ 	write_sysreg(HCR_HOST_VHE_FLAGS, hcr_el2);
+ 	write_sysreg(CPACR_EL1_FPEN, cpacr_el1);
+-	write_sysreg(vectors, vbar_el1);
++
++	if (!arm64_kernel_unmapped_at_el0())
++		host_vectors = __this_cpu_read(this_cpu_vector);
++	write_sysreg(host_vectors, vbar_el1);
+ }
+ 
+ static void __hyp_text __deactivate_traps_nvhe(void)
+diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
+index e02a6326c8003..4d472907194dd 100644
+--- a/arch/arm64/mm/mmu.c
++++ b/arch/arm64/mm/mmu.c
+@@ -532,6 +532,7 @@ early_param("rodata", parse_rodata);
+ #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+ static int __init map_entry_trampoline(void)
+ {
++	int i;
+ 	extern char __entry_tramp_text_start[];
+ 
+ 	pgprot_t prot = rodata_enabled ? PAGE_KERNEL_ROX : PAGE_KERNEL_EXEC;
+@@ -542,11 +543,15 @@ static int __init map_entry_trampoline(void)
+ 
+ 	/* Map only the text into the trampoline page table */
+ 	memset(tramp_pg_dir, 0, PGD_SIZE);
+-	__create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS, PAGE_SIZE,
+-			     prot, pgd_pgtable_alloc, 0);
++	__create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS,
++			     entry_tramp_text_size(), prot, pgd_pgtable_alloc,
++			     0);
+ 
+ 	/* Map both the text and data into the kernel page table */
+-	__set_fixmap(FIX_ENTRY_TRAMP_TEXT, pa_start, prot);
++	for (i = 0; i < DIV_ROUND_UP(entry_tramp_text_size(), PAGE_SIZE); i++)
++		__set_fixmap(FIX_ENTRY_TRAMP_TEXT1 - i,
++			     pa_start + i * PAGE_SIZE, prot);
++
+ 	if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) {
+ 		extern char __entry_tramp_data_start[];
+ 
+diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c
+index 2c5913057b87b..439a4d0058129 100644
+--- a/drivers/clocksource/arm_arch_timer.c
++++ b/drivers/clocksource/arm_arch_timer.c
+@@ -298,6 +298,13 @@ static u64 notrace arm64_858921_read_cntvct_el0(void)
+ }
+ #endif
+ 
++#ifdef CONFIG_ARM64_ERRATUM_1188873
++static u64 notrace arm64_1188873_read_cntvct_el0(void)
++{
++	return read_sysreg(cntvct_el0);
++}
++#endif
++
+ #ifdef CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND
+ DEFINE_PER_CPU(const struct arch_timer_erratum_workaround *,
+ 	       timer_unstable_counter_workaround);
+@@ -381,6 +388,14 @@ static const struct arch_timer_erratum_workaround ool_workarounds[] = {
+ 		.read_cntvct_el0 = arm64_858921_read_cntvct_el0,
+ 	},
+ #endif
++#ifdef CONFIG_ARM64_ERRATUM_1188873
++	{
++		.match_type = ate_match_local_cap_id,
++		.id = (void *)ARM64_WORKAROUND_1188873,
++		.desc = "ARM erratum 1188873",
++		.read_cntvct_el0 = arm64_1188873_read_cntvct_el0,
++	},
++#endif
+ };
+ 
+ typedef bool (*ate_match_fn_t)(const struct arch_timer_erratum_workaround *,
+diff --git a/include/linux/arm-smccc.h b/include/linux/arm-smccc.h
+index 6366b04c7d5f4..0402668914147 100644
+--- a/include/linux/arm-smccc.h
++++ b/include/linux/arm-smccc.h
+@@ -85,6 +85,13 @@
+ 			   ARM_SMCCC_SMC_32,				\
+ 			   0, 0x7fff)
+ 
++#define ARM_SMCCC_ARCH_WORKAROUND_3					\
++	ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL,				\
++			   ARM_SMCCC_SMC_32,				\
++			   0, 0x3fff)
++
++#define SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED	1
++
+ #ifndef __ASSEMBLY__
+ 
+ #include <linux/linkage.h>
+diff --git a/virt/kvm/arm/psci.c b/virt/kvm/arm/psci.c
+index c95ab4c5a4751..129b755824e12 100644
+--- a/virt/kvm/arm/psci.c
++++ b/virt/kvm/arm/psci.c
+@@ -433,6 +433,18 @@ int kvm_hvc_call_handler(struct kvm_vcpu *vcpu)
+ 				break;
+ 			}
+ 			break;
++		case ARM_SMCCC_ARCH_WORKAROUND_3:
++			switch (kvm_arm_get_spectre_bhb_state()) {
++			case SPECTRE_VULNERABLE:
++				break;
++			case SPECTRE_MITIGATED:
++				val = SMCCC_RET_SUCCESS;
++				break;
++			case SPECTRE_UNAFFECTED:
++				val = SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED;
++				break;
++			}
++			break;
+ 		}
+ 		break;
+ 	default: