diff --git a/Documentation/x86/mds.rst b/Documentation/x86/mds.rst index 5d4330be200f..e801df0bb3a8 100644 --- a/Documentation/x86/mds.rst +++ b/Documentation/x86/mds.rst @@ -95,6 +95,9 @@ The kernel provides a function to invoke the buffer clearing: mds_clear_cpu_buffers() +Also macro CLEAR_CPU_BUFFERS can be used in ASM late in exit-to-user path. +Other than CFLAGS.ZF, this macro doesn't clobber any registers. + The mitigation is invoked on kernel/userspace, hypervisor/guest and C-state (idle) transitions. @@ -138,17 +141,30 @@ Mitigation points When transitioning from kernel to user space the CPU buffers are flushed on affected CPUs when the mitigation is not disabled on the kernel - command line. The migitation is enabled through the static key - mds_user_clear. + command line. The mitigation is enabled through the feature flag + X86_FEATURE_CLEAR_CPU_BUF. - The mitigation is invoked in prepare_exit_to_usermode() which covers - all but one of the kernel to user space transitions. The exception - is when we return from a Non Maskable Interrupt (NMI), which is - handled directly in do_nmi(). + The mitigation is invoked just before transitioning to userspace after + user registers are restored. This is done to minimize the window in + which kernel data could be accessed after VERW e.g. via an NMI after + VERW. - (The reason that NMI is special is that prepare_exit_to_usermode() can - enable IRQs. In NMI context, NMIs are blocked, and we don't want to - enable IRQs with NMIs blocked.) + **Corner case not handled** + Interrupts returning to kernel don't clear CPUs buffers since the + exit-to-user path is expected to do that anyways. But, there could be + a case when an NMI is generated in kernel after the exit-to-user path + has cleared the buffers. This case is not handled and NMI returning to + kernel don't clear CPU buffers because: + + 1. It is rare to get an NMI after VERW, but before returning to userspace. + 2. For an unprivileged user, there is no known way to make that NMI + less rare or target it. + 3. It would take a large number of these precisely-timed NMIs to mount + an actual attack. There's presumably not enough bandwidth. + 4. The NMI in question occurs after a VERW, i.e. when user state is + restored and most interesting data is already scrubbed. Whats left + is only the data that NMI touches, and that may or may not be of + any interest. 2. C-State transition diff --git a/Makefile b/Makefile index 3b78f20a82aa..87c11c7c237a 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 VERSION = 6 PATCHLEVEL = 1 -SUBLEVEL = 80 +SUBLEVEL = 81 EXTRAVERSION =-valve17-chos2 NAME = Curry Ramen diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S index e309e7156038..ee5def1060c8 100644 --- a/arch/x86/entry/entry_32.S +++ b/arch/x86/entry/entry_32.S @@ -912,6 +912,7 @@ SYM_FUNC_START(entry_SYSENTER_32) BUG_IF_WRONG_CR3 no_user_check=1 popfl popl %eax + CLEAR_CPU_BUFFERS /* * Return back to the vDSO, which will pop ecx and edx. @@ -981,6 +982,7 @@ restore_all_switch_stack: /* Restore user state */ RESTORE_REGS pop=4 # skip orig_eax/error_code + CLEAR_CPU_BUFFERS .Lirq_return: /* * ARCH_HAS_MEMBARRIER_SYNC_CORE rely on IRET core serialization @@ -1173,6 +1175,7 @@ SYM_CODE_START(asm_exc_nmi) /* Not on SYSENTER stack. */ call exc_nmi + CLEAR_CPU_BUFFERS jmp .Lnmi_return .Lnmi_from_sysenter_stack: diff --git a/arch/x86/include/asm/entry-common.h b/arch/x86/include/asm/entry-common.h index 11203a9fe0a8..ffe72790ceaf 100644 --- a/arch/x86/include/asm/entry-common.h +++ b/arch/x86/include/asm/entry-common.h @@ -91,7 +91,6 @@ static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs, static __always_inline void arch_exit_to_user_mode(void) { - mds_user_clear_cpu_buffers(); amd_clear_divider(); } #define arch_exit_to_user_mode arch_exit_to_user_mode diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h index 2c66b2081f87..8f6f17a8617b 100644 --- a/arch/x86/include/asm/nospec-branch.h +++ b/arch/x86/include/asm/nospec-branch.h @@ -381,7 +381,6 @@ DECLARE_STATIC_KEY_FALSE(switch_to_cond_stibp); DECLARE_STATIC_KEY_FALSE(switch_mm_cond_ibpb); DECLARE_STATIC_KEY_FALSE(switch_mm_always_ibpb); -DECLARE_STATIC_KEY_FALSE(mds_user_clear); DECLARE_STATIC_KEY_FALSE(mds_idle_clear); DECLARE_STATIC_KEY_FALSE(switch_mm_cond_l1d_flush); @@ -415,17 +414,6 @@ static __always_inline void mds_clear_cpu_buffers(void) asm volatile("verw %[ds]" : : [ds] "m" (ds) : "cc"); } -/** - * mds_user_clear_cpu_buffers - Mitigation for MDS and TAA vulnerability - * - * Clear CPU buffers if the corresponding static key is enabled - */ -static __always_inline void mds_user_clear_cpu_buffers(void) -{ - if (static_branch_likely(&mds_user_clear)) - mds_clear_cpu_buffers(); -} - /** * mds_idle_clear_cpu_buffers - Mitigation for MDS vulnerability * diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 13dffc43ded0..d1895930e6eb 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -110,9 +110,6 @@ DEFINE_STATIC_KEY_FALSE(switch_mm_cond_ibpb); /* Control unconditional IBPB in switch_mm() */ DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb); -/* Control MDS CPU buffer clear before returning to user space */ -DEFINE_STATIC_KEY_FALSE(mds_user_clear); -EXPORT_SYMBOL_GPL(mds_user_clear); /* Control MDS CPU buffer clear before idling (halt, mwait) */ DEFINE_STATIC_KEY_FALSE(mds_idle_clear); EXPORT_SYMBOL_GPL(mds_idle_clear); @@ -251,7 +248,7 @@ static void __init mds_select_mitigation(void) if (!boot_cpu_has(X86_FEATURE_MD_CLEAR)) mds_mitigation = MDS_MITIGATION_VMWERV; - static_branch_enable(&mds_user_clear); + setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); if (!boot_cpu_has(X86_BUG_MSBDS_ONLY) && (mds_nosmt || cpu_mitigations_auto_nosmt())) @@ -355,7 +352,7 @@ static void __init taa_select_mitigation(void) * For guests that can't determine whether the correct microcode is * present on host, enable the mitigation for UCODE_NEEDED as well. */ - static_branch_enable(&mds_user_clear); + setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); if (taa_nosmt || cpu_mitigations_auto_nosmt()) cpu_smt_disable(false); @@ -423,7 +420,7 @@ static void __init mmio_select_mitigation(void) */ if (boot_cpu_has_bug(X86_BUG_MDS) || (boot_cpu_has_bug(X86_BUG_TAA) && boot_cpu_has(X86_FEATURE_RTM))) - static_branch_enable(&mds_user_clear); + setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF); else static_branch_enable(&mmio_stale_data_clear); @@ -483,12 +480,12 @@ static void __init md_clear_update_mitigation(void) if (cpu_mitigations_off()) return; - if (!static_key_enabled(&mds_user_clear)) + if (!boot_cpu_has(X86_FEATURE_CLEAR_CPU_BUF)) goto out; /* - * mds_user_clear is now enabled. Update MDS, TAA and MMIO Stale Data - * mitigation, if necessary. + * X86_FEATURE_CLEAR_CPU_BUF is now enabled. Update MDS, TAA and MMIO + * Stale Data mitigation, if necessary. */ if (mds_mitigation == MDS_MITIGATION_OFF && boot_cpu_has_bug(X86_BUG_MDS)) { diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c index cec0bfa3bc04..ed6cce6c3950 100644 --- a/arch/x86/kernel/nmi.c +++ b/arch/x86/kernel/nmi.c @@ -522,9 +522,6 @@ DEFINE_IDTENTRY_RAW(exc_nmi) write_cr2(this_cpu_read(nmi_cr2)); if (this_cpu_dec_return(nmi_state)) goto nmi_restart; - - if (user_mode(regs)) - mds_user_clear_cpu_buffers(); } #if defined(CONFIG_X86_64) && IS_ENABLED(CONFIG_KVM_INTEL) diff --git a/arch/x86/kvm/vmx/run_flags.h b/arch/x86/kvm/vmx/run_flags.h index edc3f16cc189..6a9bfdfbb6e5 100644 --- a/arch/x86/kvm/vmx/run_flags.h +++ b/arch/x86/kvm/vmx/run_flags.h @@ -2,7 +2,10 @@ #ifndef __KVM_X86_VMX_RUN_FLAGS_H #define __KVM_X86_VMX_RUN_FLAGS_H -#define VMX_RUN_VMRESUME (1 << 0) -#define VMX_RUN_SAVE_SPEC_CTRL (1 << 1) +#define VMX_RUN_VMRESUME_SHIFT 0 +#define VMX_RUN_SAVE_SPEC_CTRL_SHIFT 1 + +#define VMX_RUN_VMRESUME BIT(VMX_RUN_VMRESUME_SHIFT) +#define VMX_RUN_SAVE_SPEC_CTRL BIT(VMX_RUN_SAVE_SPEC_CTRL_SHIFT) #endif /* __KVM_X86_VMX_RUN_FLAGS_H */ diff --git a/arch/x86/kvm/vmx/vmenter.S b/arch/x86/kvm/vmx/vmenter.S index 0b5db4de4d09..0b2cad66dee1 100644 --- a/arch/x86/kvm/vmx/vmenter.S +++ b/arch/x86/kvm/vmx/vmenter.S @@ -106,7 +106,7 @@ SYM_FUNC_START(__vmx_vcpu_run) mov (%_ASM_SP), %_ASM_AX /* Check if vmlaunch or vmresume is needed */ - testb $VMX_RUN_VMRESUME, %bl + bt $VMX_RUN_VMRESUME_SHIFT, %bx /* Load guest registers. Don't clobber flags. */ mov VCPU_RCX(%_ASM_AX), %_ASM_CX @@ -128,8 +128,11 @@ SYM_FUNC_START(__vmx_vcpu_run) /* Load guest RAX. This kills the @regs pointer! */ mov VCPU_RAX(%_ASM_AX), %_ASM_AX - /* Check EFLAGS.ZF from 'testb' above */ - jz .Lvmlaunch + /* Clobbers EFLAGS.ZF */ + CLEAR_CPU_BUFFERS + + /* Check EFLAGS.CF from the VMX_RUN_VMRESUME bit test above. */ + jnc .Lvmlaunch /* * After a successful VMRESUME/VMLAUNCH, control flow "magically" diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 57c1374fdfd4..5c1590855ffc 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -407,7 +407,8 @@ static __always_inline void vmx_enable_fb_clear(struct vcpu_vmx *vmx) static void vmx_update_fb_clear_dis(struct kvm_vcpu *vcpu, struct vcpu_vmx *vmx) { - vmx->disable_fb_clear = vmx_fb_clear_ctrl_available; + vmx->disable_fb_clear = !cpu_feature_enabled(X86_FEATURE_CLEAR_CPU_BUF) && + vmx_fb_clear_ctrl_available; /* * If guest will not execute VERW, there is no need to set FB_CLEAR_DIS @@ -7120,11 +7121,14 @@ static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu, { guest_state_enter_irqoff(); - /* L1D Flush includes CPU buffer clear to mitigate MDS */ + /* + * L1D Flush includes CPU buffer clear to mitigate MDS, but VERW + * mitigation for MDS is done late in VMentry and is still + * executed in spite of L1D Flush. This is because an extra VERW + * should not matter much after the big hammer L1D Flush. + */ if (static_branch_unlikely(&vmx_l1d_should_flush)) vmx_l1d_flush(vcpu); - else if (static_branch_unlikely(&mds_user_clear)) - mds_clear_cpu_buffers(); else if (static_branch_unlikely(&mmio_stale_data_clear) && kvm_arch_has_assigned_device(vcpu->kvm)) mds_clear_cpu_buffers(); diff --git a/drivers/firmware/efi/libstub/x86-stub.c b/drivers/firmware/efi/libstub/x86-stub.c index a0757a37b482..784e1b2ae5cc 100644 --- a/drivers/firmware/efi/libstub/x86-stub.c +++ b/drivers/firmware/efi/libstub/x86-stub.c @@ -212,8 +212,8 @@ static void retrieve_apple_device_properties(struct boot_params *boot_params) } } -void efi_adjust_memory_range_protection(unsigned long start, - unsigned long size) +efi_status_t efi_adjust_memory_range_protection(unsigned long start, + unsigned long size) { efi_status_t status; efi_gcd_memory_space_desc_t desc; @@ -225,13 +225,17 @@ void efi_adjust_memory_range_protection(unsigned long start, rounded_end = roundup(start + size, EFI_PAGE_SIZE); if (memattr != NULL) { - efi_call_proto(memattr, clear_memory_attributes, rounded_start, - rounded_end - rounded_start, EFI_MEMORY_XP); - return; + status = efi_call_proto(memattr, clear_memory_attributes, + rounded_start, + rounded_end - rounded_start, + EFI_MEMORY_XP); + if (status != EFI_SUCCESS) + efi_warn("Failed to clear EFI_MEMORY_XP attribute\n"); + return status; } if (efi_dxe_table == NULL) - return; + return EFI_SUCCESS; /* * Don't modify memory region attributes, they are @@ -244,7 +248,7 @@ void efi_adjust_memory_range_protection(unsigned long start, status = efi_dxe_call(get_memory_space_descriptor, start, &desc); if (status != EFI_SUCCESS) - return; + break; next = desc.base_address + desc.length; @@ -269,8 +273,10 @@ void efi_adjust_memory_range_protection(unsigned long start, unprotect_start, unprotect_start + unprotect_size, status); + break; } } + return EFI_SUCCESS; } static efi_char16_t *efistub_fw_vendor(void) @@ -800,9 +806,7 @@ static efi_status_t efi_decompress_kernel(unsigned long *kernel_entry) *kernel_entry = addr + entry; - efi_adjust_memory_range_protection(addr, kernel_total_size); - - return EFI_SUCCESS; + return efi_adjust_memory_range_protection(addr, kernel_total_size); } static void __noreturn enter_kernel(unsigned long kernel_addr, diff --git a/drivers/firmware/efi/libstub/x86-stub.h b/drivers/firmware/efi/libstub/x86-stub.h index 37c5a36b9d8c..1c20e99a6494 100644 --- a/drivers/firmware/efi/libstub/x86-stub.h +++ b/drivers/firmware/efi/libstub/x86-stub.h @@ -5,8 +5,8 @@ extern void trampoline_32bit_src(void *, bool); extern const u16 trampoline_ljmp_imm_offset; -void efi_adjust_memory_range_protection(unsigned long start, - unsigned long size); +efi_status_t efi_adjust_memory_range_protection(unsigned long start, + unsigned long size); #ifdef CONFIG_X86_64 efi_status_t efi_setup_5level_paging(void); diff --git a/drivers/xen/events/events_base.c b/drivers/xen/events/events_base.c index 00f8e349921d..96b96516c980 100644 --- a/drivers/xen/events/events_base.c +++ b/drivers/xen/events/events_base.c @@ -937,8 +937,8 @@ static void shutdown_pirq(struct irq_data *data) return; do_mask(info, EVT_MASK_REASON_EXPLICIT); - xen_evtchn_close(evtchn); xen_irq_info_cleanup(info); + xen_evtchn_close(evtchn); } static void enable_pirq(struct irq_data *data) @@ -982,8 +982,6 @@ static void __unbind_from_irq(unsigned int irq) unsigned int cpu = cpu_from_irq(irq); struct xenbus_device *dev; - xen_evtchn_close(evtchn); - switch (type_from_irq(irq)) { case IRQT_VIRQ: per_cpu(virq_to_irq, cpu)[virq_from_irq(irq)] = -1; @@ -1001,6 +999,7 @@ static void __unbind_from_irq(unsigned int irq) } xen_irq_info_cleanup(info); + xen_evtchn_close(evtchn); } xen_free_irq(irq); diff --git a/include/net/ipv6_stubs.h b/include/net/ipv6_stubs.h index c48186bf4737..21da31e1dff5 100644 --- a/include/net/ipv6_stubs.h +++ b/include/net/ipv6_stubs.h @@ -85,6 +85,11 @@ struct ipv6_bpf_stub { sockptr_t optval, unsigned int optlen); int (*ipv6_getsockopt)(struct sock *sk, int level, int optname, sockptr_t optval, sockptr_t optlen); + int (*ipv6_dev_get_saddr)(struct net *net, + const struct net_device *dst_dev, + const struct in6_addr *daddr, + unsigned int prefs, + struct in6_addr *saddr); }; extern const struct ipv6_bpf_stub *ipv6_bpf_stub __read_mostly; diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 201dc77ebbd7..d5d2183730b9 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -3109,6 +3109,10 @@ union bpf_attr { * **BPF_FIB_LOOKUP_DIRECT** * Do a direct table lookup vs full lookup using FIB * rules. + * **BPF_FIB_LOOKUP_TBID** + * Used with BPF_FIB_LOOKUP_DIRECT. + * Use the routing table ID present in *params*->tbid + * for the fib lookup. * **BPF_FIB_LOOKUP_OUTPUT** * Perform lookup from an egress perspective (default is * ingress). @@ -3117,6 +3121,11 @@ union bpf_attr { * and *params*->smac will not be set as output. A common * use case is to call **bpf_redirect_neigh**\ () after * doing **bpf_fib_lookup**\ (). + * **BPF_FIB_LOOKUP_SRC** + * Derive and set source IP addr in *params*->ipv{4,6}_src + * for the nexthop. If the src addr cannot be derived, + * **BPF_FIB_LKUP_RET_NO_SRC_ADDR** is returned. In this + * case, *params*->dmac and *params*->smac are not set either. * * *ctx* is either **struct xdp_md** for XDP programs or * **struct sk_buff** tc cls_act programs. @@ -6687,6 +6696,8 @@ enum { BPF_FIB_LOOKUP_DIRECT = (1U << 0), BPF_FIB_LOOKUP_OUTPUT = (1U << 1), BPF_FIB_LOOKUP_SKIP_NEIGH = (1U << 2), + BPF_FIB_LOOKUP_TBID = (1U << 3), + BPF_FIB_LOOKUP_SRC = (1U << 4), }; enum { @@ -6699,6 +6710,7 @@ enum { BPF_FIB_LKUP_RET_UNSUPP_LWT, /* fwd requires encapsulation */ BPF_FIB_LKUP_RET_NO_NEIGH, /* no neighbor entry for nh */ BPF_FIB_LKUP_RET_FRAG_NEEDED, /* fragmentation required to fwd */ + BPF_FIB_LKUP_RET_NO_SRC_ADDR, /* failed to derive IP src addr */ }; struct bpf_fib_lookup { @@ -6733,6 +6745,9 @@ struct bpf_fib_lookup { __u32 rt_metric; }; + /* input: source address to consider for lookup + * output: source address result from lookup + */ union { __be32 ipv4_src; __u32 ipv6_src[4]; /* in6_addr; network order */ @@ -6747,9 +6762,19 @@ struct bpf_fib_lookup { __u32 ipv6_dst[4]; /* in6_addr; network order */ }; - /* output */ - __be16 h_vlan_proto; - __be16 h_vlan_TCI; + union { + struct { + /* output */ + __be16 h_vlan_proto; + __be16 h_vlan_TCI; + }; + /* input: when accompanied with the + * 'BPF_FIB_LOOKUP_DIRECT | BPF_FIB_LOOKUP_TBID` flags, a + * specific routing table to use for the fib lookup. + */ + __u32 tbid; + }; + __u8 smac[6]; /* ETH_ALEN */ __u8 dmac[6]; /* ETH_ALEN */ }; diff --git a/net/core/filter.c b/net/core/filter.c index 3a6110ea4009..cb7c4651eaec 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -5752,6 +5752,12 @@ static int bpf_ipv4_fib_lookup(struct net *net, struct bpf_fib_lookup *params, u32 tbid = l3mdev_fib_table_rcu(dev) ? : RT_TABLE_MAIN; struct fib_table *tb; + if (flags & BPF_FIB_LOOKUP_TBID) { + tbid = params->tbid; + /* zero out for vlan output */ + params->tbid = 0; + } + tb = fib_get_table(net, tbid); if (unlikely(!tb)) return BPF_FIB_LKUP_RET_NOT_FWDED; @@ -5803,6 +5809,9 @@ static int bpf_ipv4_fib_lookup(struct net *net, struct bpf_fib_lookup *params, params->rt_metric = res.fi->fib_priority; params->ifindex = dev->ifindex; + if (flags & BPF_FIB_LOOKUP_SRC) + params->ipv4_src = fib_result_prefsrc(net, &res); + /* xdp and cls_bpf programs are run in RCU-bh so * rcu_read_lock_bh is not needed here */ @@ -5885,6 +5894,12 @@ static int bpf_ipv6_fib_lookup(struct net *net, struct bpf_fib_lookup *params, u32 tbid = l3mdev_fib_table_rcu(dev) ? : RT_TABLE_MAIN; struct fib6_table *tb; + if (flags & BPF_FIB_LOOKUP_TBID) { + tbid = params->tbid; + /* zero out for vlan output */ + params->tbid = 0; + } + tb = ipv6_stub->fib6_get_table(net, tbid); if (unlikely(!tb)) return BPF_FIB_LKUP_RET_NOT_FWDED; @@ -5939,6 +5954,18 @@ static int bpf_ipv6_fib_lookup(struct net *net, struct bpf_fib_lookup *params, params->rt_metric = res.f6i->fib6_metric; params->ifindex = dev->ifindex; + if (flags & BPF_FIB_LOOKUP_SRC) { + if (res.f6i->fib6_prefsrc.plen) { + *src = res.f6i->fib6_prefsrc.addr; + } else { + err = ipv6_bpf_stub->ipv6_dev_get_saddr(net, dev, + &fl6.daddr, 0, + src); + if (err) + return BPF_FIB_LKUP_RET_NO_SRC_ADDR; + } + } + if (flags & BPF_FIB_LOOKUP_SKIP_NEIGH) goto set_fwd_params; @@ -5957,7 +5984,8 @@ static int bpf_ipv6_fib_lookup(struct net *net, struct bpf_fib_lookup *params, #endif #define BPF_FIB_LOOKUP_MASK (BPF_FIB_LOOKUP_DIRECT | BPF_FIB_LOOKUP_OUTPUT | \ - BPF_FIB_LOOKUP_SKIP_NEIGH) + BPF_FIB_LOOKUP_SKIP_NEIGH | BPF_FIB_LOOKUP_TBID | \ + BPF_FIB_LOOKUP_SRC) BPF_CALL_4(bpf_xdp_fib_lookup, struct xdp_buff *, ctx, struct bpf_fib_lookup *, params, int, plen, u32, flags) diff --git a/net/ipv6/af_inet6.c b/net/ipv6/af_inet6.c index 0b42eb8c55aa..62247621cea5 100644 --- a/net/ipv6/af_inet6.c +++ b/net/ipv6/af_inet6.c @@ -1077,6 +1077,7 @@ static const struct ipv6_bpf_stub ipv6_bpf_stub_impl = { .udp6_lib_lookup = __udp6_lib_lookup, .ipv6_setsockopt = do_ipv6_setsockopt, .ipv6_getsockopt = do_ipv6_getsockopt, + .ipv6_dev_get_saddr = ipv6_dev_get_saddr, }; static int __init inet6_init(void) diff --git a/processed_commits.txt b/processed_commits.txt index c1468dfcc5c8..b6a9a00261dd 100644 --- a/processed_commits.txt +++ b/processed_commits.txt @@ -841,3 +841,219 @@ a3eb3a74aa8c94e6c8130b55f3b031f29162868c ad5b847499287227525a35a0c463091fc7ccd252 3de7eedf00702831851c1046aaa64c575ee2fc90 56f768c4e358581a166924b8229ca200f73bbfdd +b7be6c737a179a76901c872f6b4c1d00552d9a1b +ae5f10ed9539878f1128f3fa129f104ba97ffc86 +7d34b1078665e171f4883b8675e52d17ebfc5c64 +cf33e6ca12d814e1be2263cb76960d0019d7fb94 +b73dd5f9997279715cd450ee8ca599aaff2eabb9 +2a3d40b4025fcfe51b04924979f1653993b17669 +a76072bc73c77cbdc6c77e5893376939894e6f73 +00459ae532d6f1e7c720b5a331f40f72cf158dca +174ac6b53a20cc7f466eead68ccee55ab633e5a1 +5dbedec7e5cf668caa0d76e02915eef16d22e97f +e30f82597bf64ad32f3b9718bb12791bf3926f3d +507eeaad4d32174640440f225a30112d8cccd374 +0cea0c330a11461d0fbad5347a5d68d499db56fd +49e734926a4b07308d98dc9d3c8f05eb77f1da00 +ed9fdc82cafbcf8a46b55d315219bf9464621bca +f8faa536370ec9db460bac96460e16801f62325e +8745f3592ee4a7b49ede16ddd3f12a41ecaa23c9 +e89c84422f35ce9fcb0fe9e3f3f60506586a7bae +39c6312009574ca73865354133ca222e7753a71b +976126f2def45f4075f18372bc4e97bb5da3757a +0d04e45c65f0785e558b93d2631d58680f263e10 +b3152afc0eb864f7c6ecad134a15b577ef7aec77 +329fc4d3f73d865b25f2ee4eafafb040ace37ad5 +e5f488993bc1893b84d93e9915155fab66a070d2 +afbf1a5cef46427241e76704991cc83c9b1a463b +87632bc9ecff5ded93433bc0fca428019bdd1cfe +65a389ef979b5ca96bc08aa165d6710fe8f1e890 +0b27bf4c494d61e5663baa34c3edd7ccebf0ea44 +0ac219c4c3ab253f3981f346903458d20bacab32 +ab63de24ebea36fe73ac7121738595d704b66d96 +a3c8fa54e904b0ddb52a08cc2d8ac239054f61fd +29360fd3288f3978ccde2f8f7eba22282c4a08a3 +e85b3c15398f6fa1f3941be8acbef79ae114744d +7985d73961bbb4e726c1be7b9cd26becc7be8325 +1b0998fdd85776775d975d0024bca227597e836a +c41548fede3d4b0305be2237ba7dbf657e9ff30b +548ab66730848c8ed105e1d7caf9f4e3f68cdc94 +d77ab053fb2f97ff366118d4dbffd8fe48168541 +1b4223e807fa17bc53062e922e4e7266450e304f +aa5897232682c27ff731b083b40c879b0eb2c994 +17ccd9798fe0beda3db212cfa3ebe373f605cbd6 +cad078914b628737fa0946de02169e80fba721cf +45085686b9559bfbe3a4f41d3d695a520668f5e1 +926405765f25809602c52e037827d0d5a9f62692 +0b056a52b3adfe5fedf20cd64addbe4e1d226c95 +30a5e812f78e3d1cced90e1ed750bf027599205f +2dc94c160ef0292d7da7ea2d4c3087c852c97fcc +7b410226d9eff7f64857a75d65e149440bff2b2f +eb7b5777d3c7f5dbbb0736f638068f50006d81b0 +e5383662fd02ad9516ae2c27f85cd56296372ba9 +29059d0f3bc21f76db5a375a70a449ba86f3d6cc +940963613275a39fd693fb1969c1c6fbc0798a21 +fc47ed389a884ee4a5b01f62ecae8137b41f63d7 +67ffc334b92a96e65b627b6b3349c25946ff69f6 +92b8a3273f3812ba441d2a842d602ff7d33362b3 +ddf6ee3df30b694ac0a66b243245e5b89b6162e2 +b8afc22a1160121d108d7ea8496f133804d69b93 +2b1414d5e94e477edff1d2c79030f1d742625ea0 +f2261eb994aa5757c1da046b78e3229a3ece0ad9 +a0222b48175709b8a66e5f373d17a10ca5659cc8 +7d4121b40149aed0698c7b82384c5c069da91836 +40f0f326cfe6847faaa409f4883b94fcdda468ab +08562ca971ff6d4d30ef7eb3fe932f8bf9dcd841 +ddc547dd05a46720866c32022300f7376c40119f +cefe18e9ec84f8fe3e198ccebb815cc996eb9797 +7f8644b6a86d45c9f8240734b161896a09069fe5 +d36b9a1b4e5214abaf864afde5617b021b5cb588 +2f91a96b892fab2f2543b4a55740c5bee36b1a6b +058ed71e0f7aa3b6694ca357e23d084e5d3f2470 +8310080799b40fd9f2a8b808c657269678c149af +8f626221e5fa89134515d358e7d614609b612a5c +3bfe04c1273d30b866f4c7c238331ed3b08e5824 +8cec41a35065dcfcca5a2337f4edd56dadd1425c +4cbbc2f0dbe22498e290997c52f088413d6b9ad5 +fd3289ab8ed1f8a2f6e3593adf39bb610fbc17a5 +59ed284c7bff4da0f6cafd05ca15de1c0ae1d087 +abd32d7f5c0294c1b2454c5a3b13b18446bac627 +930e826962d9f01dcd2220176134427358d112f2 +c34adc20b91a8e55e048b18d63f4f4ae003ecf8f +f590040ce2b712177306b03c2a63b16f7d48d3c8 +444d70889d199b7f74eec45f14768a83c0b04d73 +2e443ed55fe3ffb08327b331a9f45e9382413c94 +8dafc066c54669384ce01b4bbdfe9708a085afb9 +237ecf1afe6c22534fa43abdf2bf0b0f52de0aaa +034e2d70b5c7f578200ad09955aeb2aa65d1164a +300111cd9042d133d1edd0255f50556211125ce9 +474d521da890b3e3585335fb80a6044cb2553d99 +70af82bb9c897faa25a44e4181f36c60312b71ef +bc9f87a41d185d7678c5742a4f5952df04bf2375 +c65c475560851291ad64272d4b85b55e70c4adbb +4974d928d5e3909bd8cfe4b0bca2509636a8ebf2 +76109a226a39aea5d621b9b0af04ba23fc9cf7de +249d6ca4ff0022a4b51a8eb9fac6d7bff2c94d1b +396a4120011d8d574eb57793efb0eec5f271a2c8 +c9fa51d4c434fa7bdafd0c7a9e19cf9023787fd4 +65742f4bb1f919caa564b8a20b15b8cdd6eca2ef +e6e04845c2e8af9fef7d58439e9f62a3ed93f33b +e64148635509bf13eea851986f5a0b150e5bd066 +fbccc5eb1652b6c4ff446f34eb5a18869b7f4f3b +53e3f2ee8a0ce7fb33488325a74b678ddf74632a +fb7be5e5ec265a47f6763b6d772873da78bb09d0 +84a3c10a0c79ede027b030f61a89b6ab7cf98226 +03ad085eb14db2ddc4de5d9474426d258dc53954 +a8722cece375838f7067aa929d89a819f7d1ae96 +d93fd40c62397326046902a2c5cb75af50882a85 +f27d319df055629480b84b9288a502337b6f2a2e +88067197e97af3fcb104dd86030f788ec1b32fdb +2d9b3e1ae1bed1f20621d5cc95e74746a4afbe7d +e7945d93fece3ae43d2ed47d5ef4e254c8a3b712 +bad6e66d0701d88a1b7018ca0334b551fb71d74a +3bad8dc0ae8db10540290c69bbb3a3f8e6e5aff4 +d8950e8e20e006c8cbc4cc1ff81c35921053a8a2 +c577208f81c9ddbc5ab1418bfe810680a671fa84 +469b84516cc456fdf003dc99d9a9b0e7eab27c24 +beeeb4655db99ed0eb70f6756518fd202fd12e1a +ef12d049fa7b429a0f1842307e921da30ff2e97b +88035744b91a187bcc23253a73ef3b1ccc08a2f9 +530a4271b7ba5776b7f5a67015ae63a3ba3d2348 +29134968f72da9337dff949bba0bdb0c5134ba0a +2e47116315a08bd5fa451bbeb66cb14ffc3f0de1 +801873f1750aa1cc42e290d8a818e340fd7d0987 +e840ae3dc277f7f4ae38f600e7f5da7f169b8d7c +0912dce9ed4e8a6442fb39627cd37ca5a25beec5 +cac22c9a5e661a000e734af797641375fa181dbc +71c43b714fd688ff5ee6d906e5cc38e6a8f2836f +a8901f331b8b7f95a7315d033a22bc84c8365f35 +7bc9533e077e2553264b447189d13f83c47770a0 +4f3077c3eae7e68e2c0ba6d1bd3f5afeb61eb269 +51a0710218cea5c7d5528b92ba19e964423c7f5a +2cca5f519e3a967f4b5b72e69758521401f021eb +99a20f58913a4093c73817f5b364f0cf050a6d75 +640f27fc2e7bd69d511675c0c62a90bd9ed977cb +6083b4c5908e0e6d1b578af04103f64c257ffb82 +1523291591de054393ff4d732f18abd222ff5949 +364d7745974f20ed940918e3129d10c271638153 +e2fa53a04cc722aaaedbd91cd414d170067fb09d +df3dec320b7c14780484e824f3ec9c213e4996e1 +463b51e90c576cd63269f8420c0a0b09152092e5 +5c4feadb0011983bbc4587bc61056c7b379d9969 +bf0ca988e250af95824c121873b2f76fccfc91df +04dd4403ff3721ad0bff925116fada773ed6ae69 +831e9e63cc3b90f62d82df854cda8232408526a9 +d03a9855cbe6f41b2928c4df2e33e05f32a8e7fa +0a49efb94888b6381d9c43fda17115ffda40a039 +bfef0cfab41cb4894bc5cf8b93e76327ac04b9b9 +850333a25aab582118d9fa405af00caae32faa62 +f82865e2a026b6d491377e64ad18326a413e6421 +137d20da8ea0daa9e0a2787acc4b66261e8796df +e62d8c1281662a0cff23df2948162c1fe705d613 +519a80ea5a1770f1bb7d0627f4670ca1c1767f80 +9fbef7dcd8aa552d5a7e6867eec570e89d7d1631 +fae3f8b554fae8631a954e2b205ad84c531ba71b +255ac53d78d562fe27b486360699dcbeb0bfacf8 +6ee5c4e269a9136da48df4126c4dde9b899d35cf +c8d8876aae34f2609d3ea815106024645fa85112 +5aa0c564c017a008b3d971a6228cfae171695f57 +0d4150f5eb20b2f14153474af7ca3a26814850a8 +49e8d9f465006ba7197cbcc6d297528b72a2f196 +5a1f61516f802d95959944e7529d23dfa6868031 +6b12589f610ae5ca924573315b4cf3afd593cb09 +1f76cb66ff2257675666172aba42b4e661809a20 +c66f9f22e6e555edd575d471c6a309466651a2f5 +e017486dadf9ebb890bdd654b67014a9aeaa41c1 +8b7be6ef588e0df6036e99f0f637fdac641da396 +12e63680a76cad3bf505669b753560582ccadfcb +8973a8f9b72dbafbd1083c220d975a0e7ec871d0 +ccbf6efab8d37e3af007e83d7e7797f0ab2f3064 +0920deeec6dd2e8d142a688a81744702895d46c6 +371e1c1b326b5de0a12204f217709e2f626c7fe7 +4481d72a4b63eb190e71e381050aa2959226e13b +f30f07ba5789ef5c68c2352b996d8a98fefca8a2 +f28dae54632c5ea45f32ddc6fba494f5efc15007 +7b2b8a6c75f0c0175f626d61a74e4f7f75d38df4 +eb73733124305ce47d86d74fc3610ea7a4e55260 +5c6c2fb3c12f7d7bb7f04259878ac965a8ea2d2d +ce606d5334c2abd772bac18c5ee83f3dd82f2a11 +c479755cb80a85bbd7569fa7a7e133a66f792a31 +f3ea5ec83d1a827f074b2b660749817e0bf2b23e +56587affe21c5cd806523a89efd8da5b49872a72 +e58f2862e9fe500b073d20f94e73abc52fb70634 +33d064aecd89846d5cf284ab75eeb9098b5ff49e +f0acafd6f79fa6068b7fc4af7980ac9bbd14f1d1 +1f3fd81bff03355c3acc8558c3c4da2f2d4e1d18 +34378d7ad273ff859c1ed9ab77bb71e55f652b06 +8ff6d88c0443acdd4199aacb69f1dd4a24120e8e +476a48cd37c948b160cc3d5ff5b4d2e711f1ca36 +350265a753d8b39e2bb11660f2109c8dd5306b45 +5a664585a71c3af82a64aa9b38cadfa02f11c841 +77330c123d7c443936585f25b31d3979876ba1d0 +fff7614f576f802fb0f4ff169cb251c180ce377e +2dfaeac3f38e4e550d215204eedd97a061fdc118 +1b54062576792b41f0acb8d562deea7c4c718c33 +86c909d2275b91fb34be07b081c7343a0c2351f2 +8f05493706ff8296d26b449db295b1dbb1de31dd +3a396c409a39ce701533f3f55f3db0ab700aaeae +2402392bed4e440e05442fb1de4ef97536ff5a96 +c4c795b21dd23d9514ae1c6646c3fb2c78b5be60 +2c96f66cd0cca5695ec326398f98b58f545ac087 +7eb95e0af5c9c2e6fad50356eaf32d216d0e7bc3 +a3d369aeb332bc7a29ba1facb9a3d3d8ba8d2568 +17acece41de3dafb63018fecbf54d288366901eb +c6ff5fb6b157cf4101889c1f3e169eb6897e8f50 +0e351d1aa2e4c1a7a4cb2a5753b86db89796d3c8 +19ec82b3cad1abef2a929262b8c1528f4e0c192d +559035e04e442a0c7fd58d5fe00308b0d99e2318 +29d3e02fb448b50ffd5d83156de9680daf16f47a +22444d079b4ccc608b9bac3e591cd88629c73df7 +2e3087505ddb8ba2d3d4c81306cca11e868fcdb9 +07946d956b55703102d5eb1518888f0d0ac87e14 +edfaad334a11d4fba21cbd860ba9a61213f4bd0b +da67116b74e6aa9c531de386e1d99f2e460d1cc4 +5fafd8254add75d8337df44ba8536e407ffe8928 +2d7ebcb5d878b4311db56eeaf7bdd76dbe9b9a13 +8866334e35102d054160a86750b7db9203f721f9 +585a344af6bcac222608a158fc2830ff02712af5 +61adba85cc40287232a539e607164f273260e0fe diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 201dc77ebbd7..d5d2183730b9 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -3109,6 +3109,10 @@ union bpf_attr { * **BPF_FIB_LOOKUP_DIRECT** * Do a direct table lookup vs full lookup using FIB * rules. + * **BPF_FIB_LOOKUP_TBID** + * Used with BPF_FIB_LOOKUP_DIRECT. + * Use the routing table ID present in *params*->tbid + * for the fib lookup. * **BPF_FIB_LOOKUP_OUTPUT** * Perform lookup from an egress perspective (default is * ingress). @@ -3117,6 +3121,11 @@ union bpf_attr { * and *params*->smac will not be set as output. A common * use case is to call **bpf_redirect_neigh**\ () after * doing **bpf_fib_lookup**\ (). + * **BPF_FIB_LOOKUP_SRC** + * Derive and set source IP addr in *params*->ipv{4,6}_src + * for the nexthop. If the src addr cannot be derived, + * **BPF_FIB_LKUP_RET_NO_SRC_ADDR** is returned. In this + * case, *params*->dmac and *params*->smac are not set either. * * *ctx* is either **struct xdp_md** for XDP programs or * **struct sk_buff** tc cls_act programs. @@ -6687,6 +6696,8 @@ enum { BPF_FIB_LOOKUP_DIRECT = (1U << 0), BPF_FIB_LOOKUP_OUTPUT = (1U << 1), BPF_FIB_LOOKUP_SKIP_NEIGH = (1U << 2), + BPF_FIB_LOOKUP_TBID = (1U << 3), + BPF_FIB_LOOKUP_SRC = (1U << 4), }; enum { @@ -6699,6 +6710,7 @@ enum { BPF_FIB_LKUP_RET_UNSUPP_LWT, /* fwd requires encapsulation */ BPF_FIB_LKUP_RET_NO_NEIGH, /* no neighbor entry for nh */ BPF_FIB_LKUP_RET_FRAG_NEEDED, /* fragmentation required to fwd */ + BPF_FIB_LKUP_RET_NO_SRC_ADDR, /* failed to derive IP src addr */ }; struct bpf_fib_lookup { @@ -6733,6 +6745,9 @@ struct bpf_fib_lookup { __u32 rt_metric; }; + /* input: source address to consider for lookup + * output: source address result from lookup + */ union { __be32 ipv4_src; __u32 ipv6_src[4]; /* in6_addr; network order */ @@ -6747,9 +6762,19 @@ struct bpf_fib_lookup { __u32 ipv6_dst[4]; /* in6_addr; network order */ }; - /* output */ - __be16 h_vlan_proto; - __be16 h_vlan_TCI; + union { + struct { + /* output */ + __be16 h_vlan_proto; + __be16 h_vlan_TCI; + }; + /* input: when accompanied with the + * 'BPF_FIB_LOOKUP_DIRECT | BPF_FIB_LOOKUP_TBID` flags, a + * specific routing table to use for the fib lookup. + */ + __u32 tbid; + }; + __u8 smac[6]; /* ETH_ALEN */ __u8 dmac[6]; /* ETH_ALEN */ };