diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs index 5b9f3231fc744..1e02404d6a791 100644 --- a/compiler/rustc_ast/src/ast.rs +++ b/compiler/rustc_ast/src/ast.rs @@ -1624,6 +1624,7 @@ impl Expr { | ExprKind::While(..) | ExprKind::Yield(YieldKind::Postfix(..)) | ExprKind::DirectConstArg(..) + | ExprKind::BtfFieldInfo(..) | ExprKind::Err(_) | ExprKind::Dummy => prefix_attrs_precedence(&self.attrs), } @@ -1923,6 +1924,9 @@ pub enum ExprKind { /// An mGCA `direct_const_arg!()` expression. DirectConstArg(Box), + /// A BTF field metadata query. + BtfFieldInfo(BtfRelocKind, Box, ThinVec), + /// Placeholder for an expression that wasn't syntactically well formed in some way. Err(ErrorGuaranteed), @@ -2195,6 +2199,44 @@ impl YieldKind { } } +/// The kind of [BPF Type Format (BTF)][btf] relocation. +/// +/// [BTF][btf] is the type metadata format used by the Linux kernel and eBPF +/// tooling for relocations: the compiled program records which field or array +/// element it intended to access, and the loader rewrites the bytecode to +/// match the layout of the kernel it is about to run on. +/// +/// The following variants are a subset of the relocation kinds defined by +/// Linux's [`bpf_core_relo_kind`]. +/// +/// [btf]: https://docs.kernel.org/bpf/btf.html +/// [`bpf_core_relo_kind`]: https://docs.kernel.org/bpf/llvm_reloc.html#relocation-kinds +#[derive(Clone, Copy, Encodable, Decodable, Debug, Eq, PartialEq, StableHash, Walkable)] +pub enum BtfRelocKind { + /// Offset of the field. + ByteOffset, + /// Size of the field. + ByteSize, + /// Whether the field exists. + Exists, +} + +impl BtfRelocKind { + pub fn as_str(&self) -> &'static str { + match self { + Self::ByteOffset => "btf_field_byte_offset", + Self::ByteSize => "btf_field_byte_size", + Self::Exists => "btf_field_exists", + } + } +} + +impl fmt::Display for BtfRelocKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.as_str().fmt(f) + } +} + /// A literal in a meta item. #[derive(Clone, Copy, Encodable, Decodable, Debug, StableHash)] pub struct MetaItemLit { diff --git a/compiler/rustc_ast/src/util/classify.rs b/compiler/rustc_ast/src/util/classify.rs index e799f73ff544f..4765fa0bb53c8 100644 --- a/compiler/rustc_ast/src/util/classify.rs +++ b/compiler/rustc_ast/src/util/classify.rs @@ -159,6 +159,7 @@ pub fn leading_labeled_expr(mut expr: &ast::Expr) -> bool { | Yield(..) | UnsafeBinderCast(..) | DirectConstArg(..) + | BtfFieldInfo(..) | Err(..) | Dummy => return false, } @@ -218,7 +219,7 @@ pub fn expr_trailing_brace(mut expr: &ast::Expr) -> Option> { break (mac.args.delim == Delimiter::Brace).then_some(TrailingBrace::MacCall(mac)); } - InlineAsm(_) | OffsetOf(_, _) | IncludedBytes(_) | FormatArgs(_) => { + InlineAsm(_) | OffsetOf(_, _) | IncludedBytes(_) | FormatArgs(_) | BtfFieldInfo(..) => { // These should have been denied pre-expansion. break None; } diff --git a/compiler/rustc_ast/src/visit.rs b/compiler/rustc_ast/src/visit.rs index 4270ac0656deb..e17d8e6cb2539 100644 --- a/compiler/rustc_ast/src/visit.rs +++ b/compiler/rustc_ast/src/visit.rs @@ -502,6 +502,7 @@ macro_rules! common_visitor_and_walkers { BoundAsyncness, BoundConstness, BoundPolarity, + BtfRelocKind, ByRef, Closure, Const, @@ -1114,6 +1115,8 @@ macro_rules! common_visitor_and_walkers { visit_visitable!(vis, kind, expr, ty), ExprKind::DirectConstArg(expr) => visit_visitable!(vis, expr), + ExprKind::BtfFieldInfo(kind, container, fields) => + visit_visitable!(vis, kind, container, fields), ExprKind::Err(_guar) => {} ExprKind::Dummy => {} } diff --git a/compiler/rustc_ast_lowering/src/expr.rs b/compiler/rustc_ast_lowering/src/expr.rs index 1c1b9a247f7a2..0989bf611d541 100644 --- a/compiler/rustc_ast_lowering/src/expr.rs +++ b/compiler/rustc_ast_lowering/src/expr.rs @@ -548,6 +548,15 @@ impl<'hir> LoweringContext<'_, 'hir> { let e = self.emit_bad_direct_const_arg(e.span, expr, "expression"); hir::ExprKind::Err(e) } + + ExprKind::BtfFieldInfo(kind, container, fields) => hir::ExprKind::BtfFieldInfo( + *kind, + self.lower_ty_alloc( + container, + ImplTraitContext::Disallowed(ImplTraitPosition::BtfFieldInfo), + ), + self.arena.alloc_from_iter(fields.iter().map(|&ident| self.lower_ident(ident))), + ), }; hir::Expr { hir_id: expr_hir_id, kind, span } diff --git a/compiler/rustc_ast_lowering/src/lib.rs b/compiler/rustc_ast_lowering/src/lib.rs index 19c37f4a76065..eec34c26a7b3e 100644 --- a/compiler/rustc_ast_lowering/src/lib.rs +++ b/compiler/rustc_ast_lowering/src/lib.rs @@ -528,6 +528,7 @@ enum ImplTraitPosition { Cast, ImplSelf, OffsetOf, + BtfFieldInfo, } impl std::fmt::Display for ImplTraitPosition { @@ -554,6 +555,7 @@ impl std::fmt::Display for ImplTraitPosition { ImplTraitPosition::Cast => "cast expression types", ImplTraitPosition::ImplSelf => "impl headers", ImplTraitPosition::OffsetOf => "`offset_of!` parameters", + ImplTraitPosition::BtfFieldInfo => "BTF field info query parameters", }; write!(f, "{name}") diff --git a/compiler/rustc_ast_pretty/src/pprust/state/expr.rs b/compiler/rustc_ast_pretty/src/pprust/state/expr.rs index b6c22e7da9cb1..cfd1b80d5521b 100644 --- a/compiler/rustc_ast_pretty/src/pprust/state/expr.rs +++ b/compiler/rustc_ast_pretty/src/pprust/state/expr.rs @@ -889,6 +889,24 @@ impl<'a> State<'a> { self.print_expr(expr, FixupContext::default()); self.pclose() } + ast::ExprKind::BtfFieldInfo(kind, container, fields) => { + self.word("builtin # "); + self.word(kind.as_str()); + self.popen(); + let ib = self.ibox(0); + self.print_type(container); + self.word(","); + self.space(); + if let Some((&first, rest)) = fields.split_first() { + self.print_ident(first); + for &field in rest { + self.word("."); + self.print_ident(field); + } + } + self.end(ib); + self.pclose(); + } } self.ann.post(self, AnnNode::Expr(expr)); diff --git a/compiler/rustc_attr_ir/src/data_structures.rs b/compiler/rustc_attr_ir/src/data_structures.rs index d722d515582dc..72bcbe7508ecd 100644 --- a/compiler/rustc_attr_ir/src/data_structures.rs +++ b/compiler/rustc_attr_ir/src/data_structures.rs @@ -791,6 +791,9 @@ pub enum AttributeKind { /// Represents `#[automatically_derived]` AutomaticallyDerived, + /// Represents `#[btf_relocatable]`. + BtfRelocatable(Span), + /// Represents the trace attribute of `#[cfg_attr]` CfgAttrTrace(ThinVec<(CfgEntry, Span)>), diff --git a/compiler/rustc_attr_ir/src/encode_cross_crate.rs b/compiler/rustc_attr_ir/src/encode_cross_crate.rs index 6f05f763f2ada..5187f541753a4 100644 --- a/compiler/rustc_attr_ir/src/encode_cross_crate.rs +++ b/compiler/rustc_attr_ir/src/encode_cross_crate.rs @@ -22,6 +22,7 @@ impl AttributeKind { AllowInternalUnstable(..) => Yes, AlwaysGca => Yes, AutomaticallyDerived => Yes, + BtfRelocatable(..) => Yes, CfgAttrTrace(..) => Yes, CfgTrace(..) => Yes, CfiEncoding { .. } => Yes, diff --git a/compiler/rustc_attr_ir/src/lang_items.rs b/compiler/rustc_attr_ir/src/lang_items.rs index c1ad05dc8e4a8..ddc3fa0034a98 100644 --- a/compiler/rustc_attr_ir/src/lang_items.rs +++ b/compiler/rustc_attr_ir/src/lang_items.rs @@ -468,6 +468,12 @@ language_item_table! { // Experimental lang item for `Reflection and comptime`(https://goals.rust-lang.org/2025h2/reflection-and-comptime.html) FnPtr, sym::FnPtr, fn_ptr, Target::Struct, GenericRequirement::None; + + // Experimental lang items for BTF CO-RE relocations. + BtfPreserveAccessIndex, sym::btf_preserve_access_index, btf_preserve_access_index, Target::Fn, GenericRequirement::Exact(1); + BtfPreserveFieldByteOffset, sym::btf_preserve_field_byte_offset, btf_preserve_field_byte_offset, Target::Fn, GenericRequirement::Exact(0); + BtfPreserveFieldByteSize, sym::btf_preserve_field_byte_size, btf_preserve_field_byte_size, Target::Fn, GenericRequirement::Exact(0); + BtfPreserveFieldExists, sym::btf_preserve_field_exists, btf_preserve_field_exists, Target::Fn, GenericRequirement::Exact(0); } /// The requirement imposed on the generics of a lang item diff --git a/compiler/rustc_attr_parsing/src/attributes/btf_relocatable.rs b/compiler/rustc_attr_parsing/src/attributes/btf_relocatable.rs new file mode 100644 index 0000000000000..e669dac5392a6 --- /dev/null +++ b/compiler/rustc_attr_parsing/src/attributes/btf_relocatable.rs @@ -0,0 +1,22 @@ +use rustc_feature::AttributeStability; +use rustc_target::spec::Arch; + +use super::prelude::*; +use crate::diagnostics::BtfRelocatableOnNonBpfArch; + +pub(crate) struct BtfRelocatableParser; + +impl NoArgsAttributeParser for BtfRelocatableParser { + const PATH: &[Symbol] = &[sym::btf_relocatable]; + const ALLOWED_TARGETS: AllowedTargets<'_> = + AllowedTargets::AllowList(&[Allow(Target::Struct), Allow(Target::Union)]); + const STABILITY: AttributeStability = unstable!(btf_relocations); + const CREATE: fn(Span) -> AttributeKind = AttributeKind::BtfRelocatable; + + fn finalize_check(cx: &FinalizeCheckContext<'_, '_>, attr_span: Span) { + // `#[btf_relocatable]` may be only applied on BPF architecture. + if cx.shared.cx.sess().target.arch != Arch::Bpf { + cx.shared.cx.dcx().emit_err(BtfRelocatableOnNonBpfArch { span: attr_span }); + } + } +} diff --git a/compiler/rustc_attr_parsing/src/attributes/mod.rs b/compiler/rustc_attr_parsing/src/attributes/mod.rs index 242b4a73b06a6..05fa44b008adc 100644 --- a/compiler/rustc_attr_parsing/src/attributes/mod.rs +++ b/compiler/rustc_attr_parsing/src/attributes/mod.rs @@ -38,6 +38,7 @@ mod prelude; pub(crate) mod allow_unstable; pub(crate) mod autodiff; pub(crate) mod body; +pub(crate) mod btf_relocatable; pub(crate) mod cfg; pub(crate) mod cfg_select; pub(crate) mod cfi_encoding; diff --git a/compiler/rustc_attr_parsing/src/context.rs b/compiler/rustc_attr_parsing/src/context.rs index f936f5aab8265..662be89fe5529 100644 --- a/compiler/rustc_attr_parsing/src/context.rs +++ b/compiler/rustc_attr_parsing/src/context.rs @@ -24,6 +24,7 @@ use rustc_span::{ErrorGuaranteed, Ident, Span, Symbol}; use crate::attributes::allow_unstable::*; use crate::attributes::autodiff::*; use crate::attributes::body::*; +use crate::attributes::btf_relocatable::*; use crate::attributes::cfi_encoding::*; use crate::attributes::codegen_attrs::*; use crate::attributes::confusables::*; @@ -261,6 +262,7 @@ attribute_parsers!( Single>, Single>, Single>, + Single>, Single>, Single>, Single>, diff --git a/compiler/rustc_attr_parsing/src/diagnostics.rs b/compiler/rustc_attr_parsing/src/diagnostics.rs index e5f49690f71dc..d2c310da5ad03 100644 --- a/compiler/rustc_attr_parsing/src/diagnostics.rs +++ b/compiler/rustc_attr_parsing/src/diagnostics.rs @@ -2056,3 +2056,10 @@ pub(crate) struct UnusedDuplicate { )] pub warning: bool, } + +#[derive(Diagnostic)] +#[diag("the `btf_relocatable` attribute can only be used on BPF architecture")] +pub(crate) struct BtfRelocatableOnNonBpfArch { + #[primary_span] + pub span: Span, +} diff --git a/compiler/rustc_builtin_macros/src/assert/context.rs b/compiler/rustc_builtin_macros/src/assert/context.rs index ef39ca049b811..98a58cb4d58cf 100644 --- a/compiler/rustc_builtin_macros/src/assert/context.rs +++ b/compiler/rustc_builtin_macros/src/assert/context.rs @@ -322,7 +322,8 @@ impl<'cx, 'a> Context<'cx, 'a> { | ExprKind::Become(_) | ExprKind::Yield(_) | ExprKind::DirectConstArg(_) - | ExprKind::UnsafeBinderCast(..) => {} + | ExprKind::UnsafeBinderCast(..) + | ExprKind::BtfFieldInfo(..) => {} } } diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 9d4602e49968d..e2a79af67f726 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -7,7 +7,7 @@ pub(crate) mod autodiff; pub(crate) mod gpu_offload; use libc::{c_char, c_uint}; -use rustc_abi::{self as abi, Align, CanonAbi, Size, WrappingRange}; +use rustc_abi::{self as abi, Align, CanonAbi, FieldIdx, Size, VariantIdx, WrappingRange}; use rustc_codegen_ssa::MemFlags; use rustc_codegen_ssa::common::{IntPredicate, RealPredicate, SynchronizationScope, TypeKind}; use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; @@ -24,7 +24,7 @@ use rustc_middle::ty::layout::{ use rustc_middle::ty::{self, Instance, Ty, TyCtxt}; use rustc_sanitizers::{cfi, kcfi}; use rustc_session::config::OptLevel; -use rustc_span::Span; +use rustc_span::{Span, bug}; use rustc_target::callconv::{FnAbi, PassMode}; use rustc_target::spec::{Arch, HasTargetSpec, SanitizerSet, Target}; use smallvec::SmallVec; @@ -34,6 +34,7 @@ use crate::abi::FnAbiLlvmExt; use crate::attributes; use crate::common::Funclet; use crate::context::{CodegenCx, FullCx, GenericCx, SCx}; +use crate::debuginfo::metadata::type_di_node; use crate::llvm::{ self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, FromGeneric, GEPNoWrapFlags, Metadata, TRUE, ToLlvmBool, Type, Value, @@ -286,6 +287,22 @@ macro_rules! set_math_builder_methods { } } +// Kinds of BPF Type Format (BTF) CO-RE relocations +// (https://docs.kernel.org/bpf/llvm_reloc.html#btf-co-re-relocations), +// defined by: +// +// * Linux kernel: +// https://elixir.bootlin.com/linux/v7.2.5/source/include/uapi/linux/bpf.h#L7616 +// * LLVM: +// https://github.com/llvm/llvm-project/blob/llvmorg-23.1.1/llvm/include/llvm/DebugInfo/BTF/BTF.h#L281 + +/// Field byte offset BTF CO-RE relocation. +const BPF_CORE_FIELD_BYTE_OFFSET: u64 = 0; +/// Field size (in bytes) BTF CO-RE relocation. +const BPF_CORE_FIELD_BYTE_SIZE: u64 = 1; +/// Field existence in target kernel BTF CO-RE relocation. +const BPF_CORE_FIELD_EXISTS: u64 = 2; + impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { type CodegenCx = CodegenCx<'ll, 'tcx>; @@ -1582,6 +1599,100 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let cold_inline = llvm::AttributeKind::Cold.create_attr(self.llcx); attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[cold_inline]); } + + // Experimental intrinsics for BPF Type Format (BTF) CO-RE relocations: + // + // https://docs.kernel.org/bpf/llvm_reloc.html#btf-co-re-relocations + fn btf_preserve_access_index( + &mut self, + base: &'ll Value, + container_ty: Ty<'tcx>, + variant: VariantIdx, + field: FieldIdx, + ) -> &'ll Value { + fn llvm_struct_field_index<'ll, 'tcx>( + bx: &Builder<'_, 'll, 'tcx>, + layout: TyAndLayout<'tcx>, + field_index: usize, + ) -> usize { + let mut llvm_index = 0; + let mut offset = Size::ZERO; + + for i in layout.fields.index_by_increasing_offset() { + let target_offset = layout.fields.offset(i as usize); + if target_offset != offset { + llvm_index += 1; + } + if i as usize == field_index { + return llvm_index; + } + + let field = layout.field(bx.cx(), i); + llvm_index += 1; + offset = target_offset + field.size; + } + + bug!("field index {field_index} not found in layout {layout:#?}") + } + + let layout_cx = ty::layout::LayoutCx::new(self.tcx, self.typing_env()); + let layout = self.layout_of(container_ty).for_variant(&layout_cx, variant); + match container_ty.kind() { + ty::Adt(adt, _) if adt.is_union() => { + let dbg_info: &'ll Metadata = type_di_node(self.cx, container_ty); + unsafe { + llvm::LLVMRustBuildPreserveUnionAccessIndex( + self.llbuilder, + base, + field.index() as c_uint, + Some(dbg_info), + ) + } + } + ty::Adt(..) | ty::Tuple(..) => { + let llvm_index = llvm_struct_field_index(self, layout, field.index()); + let dbg_info: &'ll Metadata = type_di_node(self.cx, container_ty); + unsafe { + llvm::LLVMRustBuildPreserveStructAccessIndex( + self.llbuilder, + self.cx().backend_type(layout), + base, + llvm_index as c_uint, + field.index() as c_uint, + Some(dbg_info), + ) + } + } + _ => bug!("BTF field info query has unsupported container type: {container_ty:?}"), + } + } + + fn btf_preserve_field_byte_offset(&mut self, field: &'ll Value) -> &'ll Value { + let offset = self.call_intrinsic( + "llvm.bpf.preserve.field.info", + &[self.val_ty(field)], + &[field, self.const_u64(BPF_CORE_FIELD_BYTE_OFFSET)], + ); + self.intcast(offset, self.type_isize(), false) + } + + fn btf_preserve_field_byte_size(&mut self, field: &'ll Value) -> &'ll Value { + let byte_size = self.call_intrinsic( + "llvm.bpf.preserve.field.info", + &[self.val_ty(field)], + &[field, self.const_u64(BPF_CORE_FIELD_BYTE_SIZE)], + ); + self.intcast(byte_size, self.type_isize(), false) + } + + fn btf_preserve_field_exists(&mut self, field: &'ll Value) -> &'ll Value { + let exists = self.call_intrinsic( + "llvm.bpf.preserve.field.info", + &[self.val_ty(field)], + &[field, self.const_u64(BPF_CORE_FIELD_EXISTS)], + ); + self.icmp(IntPredicate::IntNE, exists, self.const_i32(0)) + } } impl<'ll> StaticBuilderMethods for Builder<'_, 'll, '_> { diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index d1cdf7bada0b1..35679a385e523 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -1711,6 +1711,22 @@ unsafe extern "C" { NumBundles: c_uint, Name: *const c_char, ) -> &'a Value; + + // BTF relocations + pub(crate) fn LLVMRustBuildPreserveUnionAccessIndex<'a>( + B: &Builder<'a>, + Base: &'a Value, + FieldIndex: c_uint, + DbgInfo: Option<&'a Metadata>, + ) -> &'a Value; + pub(crate) fn LLVMRustBuildPreserveStructAccessIndex<'a>( + B: &Builder<'a>, + ElTy: &'a Type, + Base: &'a Value, + Index: c_uint, + FieldIndex: c_uint, + DbgInfo: Option<&'a Metadata>, + ) -> &'a Value; } // FFI bindings for `DIBuilder` functions in the LLVM-C API. diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs index 22ab228e15b5f..16a8279246194 100644 --- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs +++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs @@ -1,4 +1,4 @@ -use rustc_abi::{Align, FieldIdx, WrappingRange}; +use rustc_abi::{Align, FieldIdx, VariantIdx, WrappingRange}; use rustc_middle::mir::SourceInfo; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_session::config::OptLevel; @@ -173,6 +173,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let (_, llalign) = size_of_val::size_and_align_of_dst(bx, tp_ty, meta, span); OperandValue::Immediate(llalign) } + sym::vtable_size | sym::vtable_align => { let vtable = args[0].immediate(); let idx = match name { @@ -597,6 +598,37 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { OperandValue::ZeroSized } + // Experimental intrinsics for BPF Type Format (BTF) CO-RE relocations: + // + // https://docs.kernel.org/bpf/llvm_reloc.html#btf-co-re-relocations + sym::btf_preserve_access_index => { + let base = args[0].immediate(); + let Some(variant) = bx.const_to_opt_uint(args[1].immediate()) else { + span_bug!(span, "BTF variant index is not a constant") + }; + let Some(field) = bx.const_to_opt_uint(args[2].immediate()) else { + span_bug!(span, "BTF field index is not a constant") + }; + OperandValue::Immediate(bx.btf_preserve_access_index( + base, + fn_args.type_at(0), + VariantIdx::from_u32(variant as u32), + FieldIdx::from_u32(field as u32), + )) + } + sym::btf_preserve_field_byte_offset => { + let field = args[0].immediate(); + OperandValue::Immediate(bx.btf_preserve_field_byte_offset(field)) + } + sym::btf_preserve_field_byte_size => { + let field = args[0].immediate(); + OperandValue::Immediate(bx.btf_preserve_field_byte_size(field)) + } + sym::btf_preserve_field_exists => { + let field = args[0].immediate(); + OperandValue::Immediate(bx.btf_preserve_field_exists(field)) + } + _ => { // Need to use backend-specific things in the implementation. let result = diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs index b7b694922bcfa..a7debcd8cdb72 100644 --- a/compiler/rustc_codegen_ssa/src/traits/builder.rs +++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs @@ -372,6 +372,28 @@ pub trait BuilderMethods<'a, 'tcx>: self.inbounds_gep(self.cx().type_i8(), ptr, &[offset]) } + // Experimental intrinsics for BPF Type Format (BTF) CO-RE relocations: + // + // https://docs.kernel.org/bpf/llvm_reloc.html#btf-co-re-relocations + fn btf_preserve_access_index( + &mut self, + _base: Self::Value, + _container_ty: Ty<'tcx>, + _variant: rustc_abi::VariantIdx, + _field: rustc_abi::FieldIdx, + ) -> Self::Value { + self.tcx().dcx().fatal("the selected codegen backend does not support BTF relocations") + } + fn btf_preserve_field_byte_offset(&mut self, _field: Self::Value) -> Self::Value { + self.tcx().dcx().fatal("the selected codegen backend does not support BTF relocations") + } + fn btf_preserve_field_byte_size(&mut self, _field: Self::Value) -> Self::Value { + self.tcx().dcx().fatal("the selected codegen backend does not support BTF relocations") + } + fn btf_preserve_field_exists(&mut self, _field: Self::Value) -> Self::Value { + self.tcx().dcx().fatal("the selected codegen backend does not support BTF relocations") + } + fn trunc(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value; /// Produces the same value as [`Self::trunc`] (and defaults to that), /// but is UB unless the *zero*-extending the result can reproduce `val`. diff --git a/compiler/rustc_feature/src/builtin_attrs.rs b/compiler/rustc_feature/src/builtin_attrs.rs index bb35a3281ccdc..1e2de980c6a80 100644 --- a/compiler/rustc_feature/src/builtin_attrs.rs +++ b/compiler/rustc_feature/src/builtin_attrs.rs @@ -182,6 +182,9 @@ pub static BUILTIN_ATTRIBUTES: &[Symbol] = &[ // RFC 2412 sym::optimize, + // BTF CO-RE relocation support. + sym::btf_relocatable, + sym::ffi_pure, sym::ffi_const, sym::register_attribute_tool, diff --git a/compiler/rustc_feature/src/unstable.rs b/compiler/rustc_feature/src/unstable.rs index a7138a88ee399..6882f1beb63d7 100644 --- a/compiler/rustc_feature/src/unstable.rs +++ b/compiler/rustc_feature/src/unstable.rs @@ -399,6 +399,10 @@ declare_features! ( (unstable, avx10_target_feature, "1.88.0", Some(138843)), /// Target features on bpf. (unstable, bpf_target_feature, "1.54.0", Some(150247)), + // no-tracking-issue-start + /// Allows BTF CO-RE field relocation queries. + (unstable, btf_relocations, "CURRENT_RUSTC_VERSION", Some(160616)), + // no-tracking-issue-end /// Allows defining c-variadic functions on targets where this feature has not yet /// undergone sufficient testing for stabilization. (unstable, c_variadic_experimental_arch, "1.97.0", Some(155973)), diff --git a/compiler/rustc_hir/src/hir.rs b/compiler/rustc_hir/src/hir.rs index ee79680d7d1e9..4282fa7c48578 100644 --- a/compiler/rustc_hir/src/hir.rs +++ b/compiler/rustc_hir/src/hir.rs @@ -6,8 +6,8 @@ use std::ops::Not; use rustc_abi::ExternAbi; use rustc_ast::util::parser::ExprPrecedence; use rustc_ast::{ - self as ast, FloatTy, InlineAsmOptions, InlineAsmTemplatePiece, IntTy, Label, LitIntType, - LitKind, TraitObjectSyntax, UintTy, UnsafeBinderCastKind, + self as ast, BtfRelocKind, FloatTy, InlineAsmOptions, InlineAsmTemplatePiece, IntTy, Label, + LitIntType, LitKind, TraitObjectSyntax, UintTy, UnsafeBinderCastKind, }; pub use rustc_ast::{ AssignOp, AssignOpKind, AttrId, AttrStyle, BinOp, BinOpKind, BindingMode, BorrowKind, @@ -2267,6 +2267,7 @@ impl Expr<'_> { | ExprKind::Type(..) | ExprKind::UnsafeBinderCast(..) | ExprKind::Use(..) + | ExprKind::BtfFieldInfo(..) | ExprKind::Err(_) => prefix_attrs_precedence(), ExprKind::DropTemps(expr, ..) => expr.precedence(has_attr), @@ -2340,7 +2341,8 @@ impl Expr<'_> { | ExprKind::Binary(..) | ExprKind::Yield(..) | ExprKind::Cast(..) - | ExprKind::DropTemps(..) => false, + | ExprKind::DropTemps(..) + | ExprKind::BtfFieldInfo(..) => false, } } @@ -2393,9 +2395,11 @@ impl Expr<'_> { pub fn can_have_side_effects(&self) -> bool { match self.peel_drop_temps().kind { - ExprKind::Path(_) | ExprKind::Lit(_) | ExprKind::OffsetOf(..) | ExprKind::Use(..) => { - false - } + ExprKind::Path(_) + | ExprKind::Lit(_) + | ExprKind::OffsetOf(..) + | ExprKind::Use(..) + | ExprKind::BtfFieldInfo(..) => false, ExprKind::Type(base, _) | ExprKind::Unary(_, base) | ExprKind::Field(base, _) @@ -2696,6 +2700,11 @@ pub enum ExprKind<'hir> { /// e.g. `unsafe<'a> &'a i32` <=> `&i32`. UnsafeBinderCast(UnsafeBinderCastKind, &'hir Expr<'hir>, Option<&'hir Ty<'hir>>), + /// [BPF Type Format (BTF) CO-RE relocation][btf-relocation]. + /// + /// [btf-relocation]: https://docs.kernel.org/bpf/llvm_reloc.html#btf-co-re-relocations + BtfFieldInfo(BtfRelocKind, &'hir Ty<'hir>, &'hir [Ident]), + /// A placeholder for an expression that wasn't syntactically well formed in some way. Err(rustc_span::ErrorGuaranteed), } diff --git a/compiler/rustc_hir/src/intravisit.rs b/compiler/rustc_hir/src/intravisit.rs index 9cd4b5d7d001f..559ea5e73b5d3 100644 --- a/compiler/rustc_hir/src/intravisit.rs +++ b/compiler/rustc_hir/src/intravisit.rs @@ -949,7 +949,8 @@ pub fn walk_expr<'v, V: Visitor<'v>>(visitor: &mut V, expression: &'v Expr<'v>) ExprKind::InlineAsm(ref asm) => { try_visit!(visitor.visit_inline_asm(asm, *hir_id)); } - ExprKind::OffsetOf(ref container, ref fields) => { + ExprKind::OffsetOf(ref container, ref fields) + | ExprKind::BtfFieldInfo(_, ref container, ref fields) => { try_visit!(visitor.visit_ty_unambig(container)); walk_list!(visitor, visit_ident, fields.iter().copied()); } diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs index cca93e8aef0ec..34192c81f9a3d 100644 --- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs +++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs @@ -79,6 +79,10 @@ fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) -> hi | sym::black_box | sym::breakpoint | sym::bswap + | sym::btf_preserve_access_index + | sym::btf_preserve_field_byte_offset + | sym::btf_preserve_field_byte_size + | sym::btf_preserve_field_exists | sym::caller_location | sym::carrying_mul_add | sym::carryless_mul @@ -811,6 +815,25 @@ pub(crate) fn check_intrinsic_type( sym::return_address => (0, 0, vec![], Ty::new_imm_ptr(tcx, tcx.types.unit)), + // Experimental intrinsics for BPF Type Format (BTF) CO-RE relocations: + // + // https://docs.kernel.org/bpf/llvm_reloc.html#btf-co-re-relocations + sym::btf_preserve_access_index => ( + 1, + 0, + vec![Ty::new_imm_ptr(tcx, tcx.types.unit), tcx.types.u32, tcx.types.u32], + Ty::new_imm_ptr(tcx, tcx.types.unit), + ), + sym::btf_preserve_field_byte_offset => { + (0, 0, vec![Ty::new_imm_ptr(tcx, tcx.types.unit)], tcx.types.usize) + } + sym::btf_preserve_field_byte_size => { + (0, 0, vec![Ty::new_imm_ptr(tcx, tcx.types.unit)], tcx.types.usize) + } + sym::btf_preserve_field_exists => { + (0, 0, vec![Ty::new_imm_ptr(tcx, tcx.types.unit)], tcx.types.bool) + } + other => { tcx.dcx().emit_err(UnrecognizedIntrinsicFunction { span, name: other }); return; diff --git a/compiler/rustc_hir_pretty/src/lib.rs b/compiler/rustc_hir_pretty/src/lib.rs index 6d1ae563a9fa2..8a02ce03bbff8 100644 --- a/compiler/rustc_hir_pretty/src/lib.rs +++ b/compiler/rustc_hir_pretty/src/lib.rs @@ -1786,6 +1786,23 @@ impl<'a> State<'a> { self.word_space("yield"); self.print_expr_cond_paren(expr, self.precedence(expr) < ExprPrecedence::Jump); } + hir::ExprKind::BtfFieldInfo(kind, container, fields) => { + self.word(format!("{}!(", kind.as_str())); + self.print_type(container); + self.word(","); + self.space(); + + if let Some((&first, rest)) = fields.split_first() { + self.print_ident(first); + + for &field in rest { + self.word("."); + self.print_ident(field); + } + } + + self.word(")"); + } hir::ExprKind::Err(_) => { self.popen(); self.word("/*ERROR*/"); diff --git a/compiler/rustc_hir_typeck/src/expr.rs b/compiler/rustc_hir_typeck/src/expr.rs index a8898acf3a415..152b4587a8586 100644 --- a/compiler/rustc_hir_typeck/src/expr.rs +++ b/compiler/rustc_hir_typeck/src/expr.rs @@ -5,8 +5,9 @@ //! //! See [`rustc_hir_analysis::check`] for more context on type checking in general. -use rustc_abi::{FIRST_VARIANT, FieldIdx}; +use rustc_abi::{FIRST_VARIANT, FieldIdx, VariantIdx}; use rustc_ast as ast; +use rustc_ast::BtfRelocKind; use rustc_ast::util::parser::ExprPrecedence; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::thin_vec::ThinVec; @@ -28,10 +29,12 @@ use rustc_infer::traits::query::NoSolution; use rustc_middle::ty::adjustment::{Adjust, Adjustment, AllowTwoPhase}; use rustc_middle::ty::error::{ExpectedFound, TypeError}; use rustc_middle::ty::{self, AdtKind, GenericArgsRef, Ty, TypeVisitableExt, Unnormalized}; +use rustc_session::config::DebugInfo; use rustc_session::diagnostics::feature_err; use rustc_span::edit_distance::find_best_match_for_name; use rustc_span::hygiene::DesugaringKind; use rustc_span::{Ident, Span, Spanned, Symbol, bug, kw, span_bug, sym}; +use rustc_target::spec::Arch; use rustc_trait_selection::infer::InferCtxtExt; use rustc_trait_selection::traits::{self, ObligationCauseCode, ObligationCtxt}; use tracing::{debug, instrument, trace}; @@ -398,6 +401,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { ExprKind::UnsafeBinderCast(kind, inner_expr, ty) => { self.check_expr_unsafe_binder_cast(expr.span, kind, inner_expr, ty, expected) } + ExprKind::BtfFieldInfo(kind, container, fields) => { + self.check_expr_btf_field_info(kind, container, fields, expr) + } ExprKind::Err(guar) => Ty::new_error(tcx, guar), } } @@ -2797,6 +2803,18 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { ); if let Some((idx, field)) = self.find_adt_field(*base_def, ident) { + if find_attr!(self.tcx, base_def.did(), BtfRelocatable(..)) { + let mut err = self.dcx().struct_span_err( + expr.span, + "cannot access fields of a `btf_relocatable` type directly", + ); + err.span_label( + ident.span, + "direct field access is forbidden for BTF-relocatable types", + ); + return Ty::new_error(self.tcx, err.emit()); + } + self.write_field_index(expr.hir_id, idx); let adjustments = self.adjust_steps(&autoderef); @@ -3820,6 +3838,50 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { fields: &[Ident], expr: &'tcx hir::Expr<'tcx>, ) -> Ty<'tcx> { + let field_indices = self.resolve_field_path(container, fields, false, expr); + self.typeck_results.borrow_mut().offset_of_data_mut().insert(expr.hir_id, field_indices); + self.tcx.types.usize + } + + fn check_expr_btf_field_info( + &self, + kind: BtfRelocKind, + container: &'tcx hir::Ty<'tcx>, + fields: &[Ident], + expr: &'tcx hir::Expr<'tcx>, + ) -> Ty<'tcx> { + if self.tcx.sess.target.arch != Arch::Bpf { + self.dcx() + .struct_span_err( + expr.span, + "BTF field relocation queries are only supported for BPF targets", + ) + .emit(); + } else if self.tcx.sess.opts.debuginfo == DebugInfo::None { + let mut err = self + .dcx() + .struct_span_err(expr.span, "BTF field relocation queries require debug info"); + err.help("compile with `-C debuginfo=2`"); + err.emit(); + } + let field_indices = self.resolve_field_path(container, fields, true, expr); + self.typeck_results + .borrow_mut() + .btf_field_info_data_mut() + .insert(expr.hir_id, field_indices); + match kind { + BtfRelocKind::Exists => self.tcx.types.bool, + BtfRelocKind::ByteOffset | BtfRelocKind::ByteSize => self.tcx.types.usize, + } + } + + fn resolve_field_path( + &self, + container: &'tcx hir::Ty<'tcx>, + fields: &[Ident], + allow_btf_relocatable: bool, + expr: &'tcx hir::Expr<'tcx>, + ) -> Vec<(Ty<'tcx>, VariantIdx, FieldIdx)> { let mut current_container = self.lower_ty(container).normalized; let mut field_indices = Vec::with_capacity(fields.len()); let mut fields = fields.into_iter(); @@ -3916,6 +3978,18 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { continue; } ty::Adt(container_def, args) => { + if find_attr!(self.tcx, container_def.did(), BtfRelocatable(..)) + && !allow_btf_relocatable + { + let mut err = self.dcx().struct_span_err( + expr.span, + "cannot use `offset_of!` with a `btf_relocatable` type", + ); + err.span_label(field.span, "this field requires BTF relocation"); + err.emit(); + break; + } + let (ident, def_scope) = self.tcx.adjust_ident_and_get_scope( field, container_def.did(), @@ -3983,8 +4057,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { break; } - self.typeck_results.borrow_mut().offset_of_data_mut().insert(expr.hir_id, field_indices); - - self.tcx.types.usize + field_indices } } diff --git a/compiler/rustc_hir_typeck/src/expr_use_visitor.rs b/compiler/rustc_hir_typeck/src/expr_use_visitor.rs index 0c49ec002671d..410c80c20ab09 100644 --- a/compiler/rustc_hir_typeck/src/expr_use_visitor.rs +++ b/compiler/rustc_hir_typeck/src/expr_use_visitor.rs @@ -506,6 +506,7 @@ impl<'tcx, Cx: TypeInformationCtxt<'tcx>, D: Delegate<'tcx>> ExprUseVisitor<'tcx | hir::ExprKind::Lit(..) | hir::ExprKind::ConstBlock(..) | hir::ExprKind::OffsetOf(..) + | hir::ExprKind::BtfFieldInfo(..) | hir::ExprKind::Err(_) => {} hir::ExprKind::Loop(blk, ..) => { @@ -1386,6 +1387,7 @@ impl<'tcx, Cx: TypeInformationCtxt<'tcx>, D: Delegate<'tcx>> ExprUseVisitor<'tcx | hir::ExprKind::InlineAsm(..) | hir::ExprKind::OffsetOf(..) | hir::ExprKind::UnsafeBinderCast(UnsafeBinderCastKind::Wrap, ..) + | hir::ExprKind::BtfFieldInfo(..) | hir::ExprKind::Err(_) => Ok(self.cat_rvalue(expr.hir_id, expr_ty)), } } diff --git a/compiler/rustc_hir_typeck/src/naked_functions.rs b/compiler/rustc_hir_typeck/src/naked_functions.rs index e241cd0797cc9..0dbfa24d2a24d 100644 --- a/compiler/rustc_hir_typeck/src/naked_functions.rs +++ b/compiler/rustc_hir_typeck/src/naked_functions.rs @@ -159,7 +159,8 @@ impl CheckInlineAssembly { | ExprKind::Become(..) | ExprKind::Struct(..) | ExprKind::Repeat(..) - | ExprKind::Yield(..) => { + | ExprKind::Yield(..) + | ExprKind::BtfFieldInfo(..) => { self.items.push((ItemKind::NonAsm, span)); } diff --git a/compiler/rustc_hir_typeck/src/writeback.rs b/compiler/rustc_hir_typeck/src/writeback.rs index f4fe38924351a..f7643f236fc70 100644 --- a/compiler/rustc_hir_typeck/src/writeback.rs +++ b/compiler/rustc_hir_typeck/src/writeback.rs @@ -78,6 +78,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { wbcx.visit_transmutes(); wbcx.visit_offloads(); wbcx.visit_offset_of_container_types(); + wbcx.visit_btf_field_info_container_types(); wbcx.visit_potentially_region_dependent_goals(); let used_trait_imports = @@ -803,6 +804,22 @@ impl<'cx, 'tcx> WritebackCx<'cx, 'tcx> { } } + fn visit_btf_field_info_container_types(&mut self) { + let fcx_typeck_results = self.fcx.typeck_results.borrow(); + assert_eq!(fcx_typeck_results.hir_owner, self.typeck_results.hir_owner); + let common_hir_owner = fcx_typeck_results.hir_owner; + + for (local_id, indices) in fcx_typeck_results.btf_field_info_data().items_in_stable_order() + { + let hir_id = HirId { owner: common_hir_owner, local_id }; + let indices = indices + .iter() + .map(|&(ty, variant, field)| (self.resolve(ty, &hir_id), variant, field)) + .collect(); + self.typeck_results.btf_field_info_data_mut().insert(hir_id, indices); + } + } + fn visit_potentially_region_dependent_goals(&mut self) { let obligations = self.fcx.take_hir_typeck_potentially_region_dependent_goals(); if self.fcx.tainted_by_errors().is_none() { diff --git a/compiler/rustc_lint/src/dangling.rs b/compiler/rustc_lint/src/dangling.rs index c272dd4496984..8b6e4e383a931 100644 --- a/compiler/rustc_lint/src/dangling.rs +++ b/compiler/rustc_lint/src/dangling.rs @@ -319,7 +319,10 @@ fn is_temporary_rvalue(expr: &Expr<'_>) -> bool { ExprKind::Assign(..) | ExprKind::AssignOp(..) | ExprKind::Yield(..) => false, // Compiler-magic macros - ExprKind::AddrOf(..) | ExprKind::OffsetOf(..) | ExprKind::InlineAsm(..) => false, + ExprKind::AddrOf(..) + | ExprKind::OffsetOf(..) + | ExprKind::InlineAsm(..) + | ExprKind::BtfFieldInfo(..) => false, // We are not interested in these ExprKind::Cast(..) diff --git a/compiler/rustc_lint/src/unused.rs b/compiler/rustc_lint/src/unused.rs index 17c078615c411..da06a29cc9363 100644 --- a/compiler/rustc_lint/src/unused.rs +++ b/compiler/rustc_lint/src/unused.rs @@ -196,7 +196,8 @@ trait UnusedDelimLint { | Yield(_) | Yeet(_) | Paren(_) - | Become(_) => true, + | Become(_) + | BtfFieldInfo(..) => true, Call(..) | MethodCall(_) | Let(..) | Field(..) | MacCall(_) | FormatArgs(_) => false, // `direct_const_arg!()` is invalid in function/method argument position. DirectConstArg(_) => false, diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index 161b5bdb952d3..81ab98709ce91 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -1784,6 +1784,22 @@ extern "C" LLVMValueRef LLVMRustConstPtrAuth(LLVMValueRef Ptr, uint32_t Key, #endif } +extern "C" LLVMValueRef +LLVMRustBuildPreserveUnionAccessIndex(LLVMBuilderRef B, LLVMValueRef Base, + unsigned FieldIndex, + LLVMMetadataRef DbgInfo) { + return wrap(unwrap(B)->CreatePreserveUnionAccessIndex( + unwrap(Base), FieldIndex, unwrapDI(DbgInfo))); +} + +extern "C" LLVMValueRef LLVMRustBuildPreserveStructAccessIndex( + LLVMBuilderRef B, LLVMTypeRef ElTy, LLVMValueRef Base, unsigned Index, + unsigned FieldIndex, LLVMMetadataRef DbgInfo) { + return wrap(unwrap(B)->CreatePreserveStructAccessIndex( + unwrap(ElTy), unwrap(Base), Index, FieldIndex, + unwrapDI(DbgInfo))); +} + // Statically assert that the fixed metadata kind IDs declared in // `metadata_kind.rs` match the ones actually used by LLVM. #define FIXED_MD_KIND(VARIANT, VALUE) \ diff --git a/compiler/rustc_middle/src/hir/mod.rs b/compiler/rustc_middle/src/hir/mod.rs index eae84fde7e305..d920b83004eb9 100644 --- a/compiler/rustc_middle/src/hir/mod.rs +++ b/compiler/rustc_middle/src/hir/mod.rs @@ -297,6 +297,7 @@ impl<'tcx> TyCtxt<'tcx> { | ExprKind::Path(_) | ExprKind::Continue(_) | ExprKind::OffsetOf(_, _) + | ExprKind::BtfFieldInfo(..) | ExprKind::Err(_) => unreachable!("no sub-expr expected for {:?}", expr.kind), } } diff --git a/compiler/rustc_middle/src/mir/visit.rs b/compiler/rustc_middle/src/mir/visit.rs index a6d15ac6458b8..4c58e00181b7e 100644 --- a/compiler/rustc_middle/src/mir/visit.rs +++ b/compiler/rustc_middle/src/mir/visit.rs @@ -809,7 +809,6 @@ macro_rules! make_mir_visitor { self.visit_ty($(& $mutability)? *ty, TyContext::Location(location)); } - } } diff --git a/compiler/rustc_middle/src/ty/typeck_results.rs b/compiler/rustc_middle/src/ty/typeck_results.rs index 509ff98741731..f2d330e08371b 100644 --- a/compiler/rustc_middle/src/ty/typeck_results.rs +++ b/compiler/rustc_middle/src/ty/typeck_results.rs @@ -228,6 +228,9 @@ pub struct TypeckResults<'tcx> { /// Container types and field indices of `offset_of!` expressions offset_of_data: ItemLocalMap, VariantIdx, FieldIdx)>>, + + /// Container types and field indices of BTF field info expressions. + btf_field_info_data: ItemLocalMap, VariantIdx, FieldIdx)>>, } impl<'tcx> TypeckResults<'tcx> { @@ -261,6 +264,7 @@ impl<'tcx> TypeckResults<'tcx> { transmutes_to_check: Default::default(), offloads_to_check: Default::default(), offset_of_data: Default::default(), + btf_field_info_data: Default::default(), } } @@ -596,6 +600,18 @@ impl<'tcx> TypeckResults<'tcx> { ) -> LocalTableInContextMut<'_, Vec<(Ty<'tcx>, VariantIdx, FieldIdx)>> { LocalTableInContextMut { hir_owner: self.hir_owner, data: &mut self.offset_of_data } } + + pub fn btf_field_info_data( + &self, + ) -> LocalTableInContext<'_, Vec<(Ty<'tcx>, VariantIdx, FieldIdx)>> { + LocalTableInContext { hir_owner: self.hir_owner, data: &self.btf_field_info_data } + } + + pub fn btf_field_info_data_mut( + &mut self, + ) -> LocalTableInContextMut<'_, Vec<(Ty<'tcx>, VariantIdx, FieldIdx)>> { + LocalTableInContextMut { hir_owner: self.hir_owner, data: &mut self.btf_field_info_data } + } } /// A resolved splatted function call. diff --git a/compiler/rustc_mir_build/src/thir/cx/expr.rs b/compiler/rustc_mir_build/src/thir/cx/expr.rs index c2d1739d37187..b3c2a7b305f5b 100644 --- a/compiler/rustc_mir_build/src/thir/cx/expr.rs +++ b/compiler/rustc_mir_build/src/thir/cx/expr.rs @@ -1,6 +1,6 @@ use itertools::Itertools; use rustc_abi::{FIRST_VARIANT, FieldIdx, Size, VariantIdx}; -use rustc_ast::UnsafeBinderCastKind; +use rustc_ast::{BtfRelocKind, UnsafeBinderCastKind}; use rustc_data_structures::thin_vec::ThinVec; use rustc_hir as hir; use rustc_hir::attrs::lang_items::LangItem; @@ -1183,6 +1183,91 @@ impl<'tcx> ThirBuildCx<'tcx> { hir::ExprKind::DropTemps(source) => { ExprKind::ValueExpr { source: self.mirror_expr(source) } } + + // Experimental expression for BPF Type Format (BTF) CO-RE + // relocations: + // + // https://docs.kernel.org/bpf/llvm_reloc.html#btf-co-re-relocations + hir::ExprKind::BtfFieldInfo(kind, _, _) => { + let mk_usize_kind = |val: u64| ExprKind::NonHirLiteral { + lit: ScalarInt::try_from_target_usize(val, tcx).unwrap(), + user_ty: None, + }; + + let indices = self.typeck_results.btf_field_info_data().get(expr.hir_id).unwrap(); + if indices.is_empty() { + return match kind { + BtfRelocKind::ByteOffset | BtfRelocKind::ByteSize => { + mk_expr(mk_usize_kind(0), tcx.types.usize) + } + BtfRelocKind::Exists => mk_expr( + ExprKind::NonHirLiteral { lit: false.into(), user_ty: None }, + tcx.types.bool, + ), + }; + } + + // Traverse all the components of the requested field path and + // emit `preserve_access_index` intrinsics for them. + let preserve_access_index = + tcx.require_lang_item(LangItem::BtfPreserveAccessIndex, expr.span); + let unit_ptr_ty = Ty::new_imm_ptr(tcx, tcx.types.unit); + let mk_u32_kind = |value: u32| ExprKind::NonHirLiteral { + lit: ScalarInt::try_from_uint(value, Size::from_bits(32)).unwrap(), + user_ty: None, + }; + let zero = self.thir.exprs.push(mk_expr(mk_usize_kind(0), tcx.types.usize)); + // The base for the field access. Initially it's null, then + // for each subsequent component it's the result of the + // previous intrinsic call. It's an opaque input to the + // intrinsic and it's never dereferenced. + let mut base_ptr = + self.thir.exprs.push(mk_expr(ExprKind::Cast { source: zero }, unit_ptr_ty)); + for &(container_ty, variant, field) in indices { + let fun_ty = tcx + .type_of(preserve_access_index) + .instantiate(tcx, &[container_ty.into()]) + .skip_norm_wip(); + let fun = self + .thir + .exprs + .push(mk_expr(ExprKind::ZstLiteral { user_ty: None }, fun_ty)); + let variant = + self.thir.exprs.push(mk_expr(mk_u32_kind(variant.as_u32()), tcx.types.u32)); + let field = + self.thir.exprs.push(mk_expr(mk_u32_kind(field.as_u32()), tcx.types.u32)); + base_ptr = self.thir.exprs.push(mk_expr( + ExprKind::Call { + ty: fun_ty, + fun, + args: Box::new([base_ptr, variant, field]), + from_hir_call: false, + fn_span: expr.span, + }, + unit_ptr_ty, + )); + } + + // Emit an appropriate relocation intrinsic call for the last + // component of the field path. + let field_intrinsic = match kind { + BtfRelocKind::ByteOffset => LangItem::BtfPreserveFieldByteOffset, + BtfRelocKind::ByteSize => LangItem::BtfPreserveFieldByteSize, + BtfRelocKind::Exists => LangItem::BtfPreserveFieldExists, + }; + let field_intrinsic = tcx.require_lang_item(field_intrinsic, expr.span); + let fun_ty = tcx.type_of(field_intrinsic).instantiate_identity().skip_norm_wip(); + let fun = + self.thir.exprs.push(mk_expr(ExprKind::ZstLiteral { user_ty: None }, fun_ty)); + ExprKind::Call { + ty: fun_ty, + fun, + args: Box::new([base_ptr]), + from_hir_call: false, + fn_span: expr.span, + } + } + hir::ExprKind::Array(fields) => ExprKind::Array { fields: self.mirror_exprs(fields) }, hir::ExprKind::Tup(fields) => ExprKind::Tuple { fields: self.mirror_exprs(fields) }, diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs index 8238a6518e41d..b85dc7d1d0e87 100644 --- a/compiler/rustc_parse/src/parser/expr.rs +++ b/compiler/rustc_parse/src/parser/expr.rs @@ -13,9 +13,9 @@ use rustc_ast::util::parser::{AssocOp, ExprPrecedence, Fixity, prec_let_scrutine use rustc_ast::visit::{Visitor, walk_expr}; use rustc_ast::{ self as ast, AnonConst, Arm, AssignOp, AssignOpKind, AttrStyle, AttrVec, BinOp, BinOpKind, - BlockCheckMode, CaptureBy, ClosureBinder, CoroutineKind, DUMMY_NODE_ID, Expr, ExprField, - ExprKind, FnDecl, FnRetTy, ForLoop, Guard, Label, MacCall, MetaItemLit, Movability, Param, - RangeLimits, StmtKind, Ty, TyKind, UnOp, UnsafeBinderCastKind, YieldKind, + BlockCheckMode, BtfRelocKind, CaptureBy, ClosureBinder, CoroutineKind, DUMMY_NODE_ID, Expr, + ExprField, ExprKind, FnDecl, FnRetTy, ForLoop, Guard, Label, MacCall, MetaItemLit, Movability, + Param, RangeLimits, StmtKind, Ty, TyKind, UnOp, UnsafeBinderCastKind, YieldKind, }; use rustc_ast_pretty::pprust; use rustc_errors::{Applicability, Diag, PResult, StashKey, Subdiagnostic}; @@ -2013,6 +2013,21 @@ impl<'a> Parser<'a> { sym::unwrap_binder => { Some(this.parse_expr_unsafe_binder_cast(lo, UnsafeBinderCastKind::Unwrap)?) } + sym::btf_field_byte_offset => Some(this.parse_expr_btf_field_info( + lo, + ident.as_str(), + BtfRelocKind::ByteOffset, + )?), + sym::btf_field_byte_size => Some(this.parse_expr_btf_field_info( + lo, + ident.as_str(), + BtfRelocKind::ByteSize, + )?), + sym::btf_field_exists => Some(this.parse_expr_btf_field_info( + lo, + ident.as_str(), + BtfRelocKind::Exists, + )?), _ => None, }) }) @@ -2052,6 +2067,31 @@ impl<'a> Parser<'a> { /// Built-in macro for `offset_of!` expressions. pub(crate) fn parse_expr_offset_of(&mut self, lo: Span) -> PResult<'a, Box> { + let (container, fields) = self.parse_ty_and_field_path("offset_of", "field and variant")?; + let span = lo.to(self.token.span); + Ok(self.mk_expr(span, ExprKind::OffsetOf(container, fields))) + } + + /// Built-in macro for + /// [BPF Type Format (BTF) CO-RE relocations][btf-relocations]. + /// + /// [btf-relocations]: https://docs.kernel.org/bpf/llvm_reloc.html#btf-co-re-relocations + pub(crate) fn parse_expr_btf_field_info( + &mut self, + lo: Span, + name: &str, + kind: BtfRelocKind, + ) -> PResult<'a, Box> { + let (container, fields) = self.parse_ty_and_field_path(name, "field")?; + let span = lo.to(self.token.span); + Ok(self.mk_expr(span, ExprKind::BtfFieldInfo(kind, container, fields))) + } + + fn parse_ty_and_field_path( + &mut self, + name: &str, + path_element_description: &str, + ) -> PResult<'a, (Box, ThinVec)> { let container = self.parse_ty()?; self.expect(exp!(Comma))?; @@ -2060,9 +2100,9 @@ impl<'a> Parser<'a> { if let Err(mut e) = self.expect_one_of(&[], &[exp!(CloseParen)]) { if trailing_comma { - e.note("unexpected third argument to offset_of"); + e.note(format!("unexpected third argument to {name}")); } else { - e.note("offset_of expects dot-separated field and variant names"); + e.note(format!("{name} expects dot-separated {path_element_description} names")); } e.emit(); } @@ -2074,8 +2114,7 @@ impl<'a> Parser<'a> { } } - let span = lo.to(self.token.span); - Ok(self.mk_expr(span, ExprKind::OffsetOf(container, fields))) + Ok((container, fields)) } /// Built-in macro for type ascription expressions. @@ -4489,6 +4528,7 @@ impl MutVisitor for CondChecker<'_> { | ExprKind::FormatArgs(_) | ExprKind::Err(_) | ExprKind::DirectConstArg(_) + | ExprKind::BtfFieldInfo(..) | ExprKind::Dummy => { // These would forbid any let expressions they contain already. } diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs index 79987b86385ef..8f17e1c3add89 100644 --- a/compiler/rustc_passes/src/check_attr.rs +++ b/compiler/rustc_passes/src/check_attr.rs @@ -242,6 +242,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> { AttributeKind::AllowInternalUnstable(..) => (), AttributeKind::AlwaysGca => (), AttributeKind::AutomaticallyDerived => (), + AttributeKind::BtfRelocatable(..) => (), AttributeKind::CfgAttrTrace(..) => (), AttributeKind::CfgTrace(..) => (), AttributeKind::CfiEncoding { .. } => (), diff --git a/compiler/rustc_passes/src/input_stats.rs b/compiler/rustc_passes/src/input_stats.rs index f2a7cb6e46fca..172c2aece485a 100644 --- a/compiler/rustc_passes/src/input_stats.rs +++ b/compiler/rustc_passes/src/input_stats.rs @@ -380,6 +380,7 @@ impl<'v> hir_visit::Visitor<'v> for StatCollector<'v> { Repeat, Yield, UnsafeBinderCast, + BtfFieldInfo, Err ] ); @@ -661,7 +662,7 @@ impl<'v> ast_visit::Visitor<'v> for StatCollector<'v> { If, While, ForLoop, Loop, Match, Closure, Block, Await, Move, Use, TryBlock, Assign, AssignOp, Field, Index, Range, Underscore, Path, AddrOf, Break, Continue, Ret, InlineAsm, FormatArgs, OffsetOf, MacCall, Struct, Repeat, Paren, Try, Yield, Yeet, - Become, IncludedBytes, Gen, UnsafeBinderCast, Err, Dummy, DirectConstArg + Become, IncludedBytes, Gen, UnsafeBinderCast, Err, Dummy, DirectConstArg, BtfFieldInfo ] ); ast_visit::walk_expr(self, e) diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 39d9d2b7b05ce..a4a6d9faa85a1 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -574,6 +574,15 @@ symbols! { breg, bridge, bswap, + btf_field_byte_offset, + btf_field_byte_size, + btf_field_exists, + btf_preserve_access_index, + btf_preserve_field_byte_offset, + btf_preserve_field_byte_size, + btf_preserve_field_exists, + btf_relocatable, + btf_relocations, built, builtin_syntax, bundle, diff --git a/compiler/rustc_ty_utils/src/consts.rs b/compiler/rustc_ty_utils/src/consts.rs index 6db234fd886ca..b8ba9016f7828 100644 --- a/compiler/rustc_ty_utils/src/consts.rs +++ b/compiler/rustc_ty_utils/src/consts.rs @@ -208,7 +208,6 @@ fn recurse_build<'tcx>( ExprKind::InlineAsm { .. } => { error(GenericConstantTooComplexSub::InlineAsmNotSupported(node.span))? } - // we dont permit let stmts so `VarRef` and `UpvarRef` cant happen ExprKind::VarRef { .. } | ExprKind::UpvarRef { .. } diff --git a/src/tools/clippy/clippy_lints/src/loops/never_loop.rs b/src/tools/clippy/clippy_lints/src/loops/never_loop.rs index 43fa115f7e749..159d0bf17f0e0 100644 --- a/src/tools/clippy/clippy_lints/src/loops/never_loop.rs +++ b/src/tools/clippy/clippy_lints/src/loops/never_loop.rs @@ -451,6 +451,7 @@ fn never_loop_expr<'tcx>( | ExprKind::Path(_) | ExprKind::ConstBlock(_) | ExprKind::Lit(_) + | ExprKind::BtfFieldInfo(..) | ExprKind::Err(_) => NeverLoopResult::Normal, }; @@ -547,6 +548,7 @@ fn find_non_obvious_spans<'tcx>(cx: &LateContext<'tcx>, e: &'tcx Expr<'tcx>) -> | ExprKind::Repeat(..) | ExprKind::Yield(..) | ExprKind::UnsafeBinderCast(..) + | ExprKind::BtfFieldInfo(..) | ExprKind::Err(..) => { spans.push(expr.span); return ControlFlow::Continue(Descend::No); diff --git a/src/tools/clippy/clippy_lints/src/utils/author.rs b/src/tools/clippy/clippy_lints/src/utils/author.rs index a32a641c34ee5..a98b0d8f9a936 100644 --- a/src/tools/clippy/clippy_lints/src/utils/author.rs +++ b/src/tools/clippy/clippy_lints/src/utils/author.rs @@ -716,6 +716,10 @@ impl<'a, 'tcx> PrintVisitor<'a, 'tcx> { ExprKind::UnsafeBinderCast(..) => { unimplemented!("unsafe binders are not implemented yet"); }, + ExprKind::BtfFieldInfo(kind, container, ref fields) => { + bind!(self, container, fields); + kind!("BtfFieldInfo({kind}, {container}, {fields})"); + }, } } diff --git a/src/tools/clippy/clippy_utils/src/eager_or_lazy.rs b/src/tools/clippy/clippy_utils/src/eager_or_lazy.rs index 59fe5f4964822..71d4b15126af1 100644 --- a/src/tools/clippy/clippy_utils/src/eager_or_lazy.rs +++ b/src/tools/clippy/clippy_utils/src/eager_or_lazy.rs @@ -301,7 +301,8 @@ fn expr_eagerness<'tcx>(cx: &LateContext<'tcx>, e: &'tcx Expr<'_>) -> EagernessS | ExprKind::Repeat(..) | ExprKind::Block(Block { stmts: [], .. }, _) | ExprKind::OffsetOf(..) - | ExprKind::UnsafeBinderCast(..) => (), + | ExprKind::UnsafeBinderCast(..) + | ExprKind::BtfFieldInfo(..) => (), // Assignment might be to a local defined earlier, so don't eagerly evaluate. // Blocks with multiple statements might be expensive, so don't eagerly evaluate. diff --git a/src/tools/clippy/clippy_utils/src/hir_utils.rs b/src/tools/clippy/clippy_utils/src/hir_utils.rs index cf7777d037d9b..598cc4158aa69 100644 --- a/src/tools/clippy/clippy_utils/src/hir_utils.rs +++ b/src/tools/clippy/clippy_utils/src/hir_utils.rs @@ -619,6 +619,11 @@ impl HirEqInterExpr<'_, '_, '_> { (ExprKind::Type(le, lt), ExprKind::Type(re, rt)) => self.eq_expr(le, re) && self.eq_ty(lt, rt), (ExprKind::Unary(l_op, le), ExprKind::Unary(r_op, re)) => l_op == r_op && self.eq_expr(le, re), (ExprKind::Yield(le, _), ExprKind::Yield(re, _)) => return self.eq_expr(le, re), + (ExprKind::BtfFieldInfo(l_kind, l_container, l_fields), ExprKind::BtfFieldInfo(r_kind, r_container, r_fields)) => { + l_kind == r_kind + && self.eq_ty(l_container, r_container) + && over(l_fields, r_fields, |l, r| l.name == r.name) + }, ( // Else branches for branches above, grouped as per `match_same_arms`. | ExprKind::AddrOf(..) @@ -653,6 +658,7 @@ impl HirEqInterExpr<'_, '_, '_> { | ExprKind::Unary(..) | ExprKind::Yield(..) | ExprKind::UnsafeBinderCast(..) + | ExprKind::BtfFieldInfo(..) // --- Special cases that do not have a positive branch. @@ -1400,6 +1406,13 @@ impl<'a, 'tcx> SpanlessHash<'a, 'tcx> { self.hash_ty(ty); } }, + ExprKind::BtfFieldInfo(kind, container, fields) => { + mem::discriminant(kind).hash(&mut self.s); + self.hash_ty(container); + for field in *fields { + self.hash_name(field.name); + } + } ExprKind::Err(_) => {}, } } diff --git a/src/tools/clippy/clippy_utils/src/sugg.rs b/src/tools/clippy/clippy_utils/src/sugg.rs index 702bbd0be46dc..b525af2ecd8b9 100644 --- a/src/tools/clippy/clippy_utils/src/sugg.rs +++ b/src/tools/clippy/clippy_utils/src/sugg.rs @@ -174,7 +174,8 @@ impl<'a> Sugg<'a> { | ExprKind::UnsafeBinderCast(..) | ExprKind::Match(_, _, MatchSource::AwaitDesugar | MatchSource::TryDesugar(_) | MatchSource::FormatArgs - ) => Sugg::NonParen(get_snippet(expr.span)), + ) + | ExprKind::BtfFieldInfo(..) => Sugg::NonParen(get_snippet(expr.span)), ExprKind::DropTemps(inner) => Self::hir_from_snippet(cx, inner, get_snippet), ExprKind::Assign(lhs, rhs, _) => { Sugg::BinOp(AssocOp::Assign, get_snippet(lhs.span), get_snippet(rhs.span)) @@ -251,7 +252,8 @@ impl<'a> Sugg<'a> { | ast::ExprKind::DirectConstArg(..) | ast::ExprKind::Err(_) | ast::ExprKind::Dummy - | ast::ExprKind::UnsafeBinderCast(..) => Sugg::NonParen(snippet(expr.span)), + | ast::ExprKind::UnsafeBinderCast(..) + | ast::ExprKind::BtfFieldInfo(..) => Sugg::NonParen(snippet(expr.span)), ast::ExprKind::Range(ref lhs, ref rhs, limits) => Sugg::BinOp( AssocOp::Range(limits), lhs.as_ref().map_or("".into(), |lhs| snippet(lhs.span)), diff --git a/src/tools/clippy/clippy_utils/src/visitors.rs b/src/tools/clippy/clippy_utils/src/visitors.rs index c5a101caebc21..92fa7bdd9a92f 100644 --- a/src/tools/clippy/clippy_utils/src/visitors.rs +++ b/src/tools/clippy/clippy_utils/src/visitors.rs @@ -775,6 +775,7 @@ pub fn for_each_unconsumed_temporary<'tcx, B>( | ExprKind::Continue(_) | ExprKind::InlineAsm(_) | ExprKind::OffsetOf(..) + | ExprKind::BtfFieldInfo(..) | ExprKind::Err(_) => (), } ControlFlow::Continue(()) diff --git a/src/tools/rustfmt/src/expr.rs b/src/tools/rustfmt/src/expr.rs index cef0bd97d578c..421a6fe63b7c3 100644 --- a/src/tools/rustfmt/src/expr.rs +++ b/src/tools/rustfmt/src/expr.rs @@ -423,7 +423,8 @@ pub(crate) fn format_expr( | ast::ExprKind::IncludedBytes(..) | ast::ExprKind::OffsetOf(..) | ast::ExprKind::UnsafeBinderCast(..) - | ast::ExprKind::DirectConstArg(..) => { + | ast::ExprKind::DirectConstArg(..) + | ast::ExprKind::BtfFieldInfo(..) => { // These don't normally occur in the AST because macros aren't expanded. However, // rustfmt tries to parse macro arguments when formatting macros, so it's not totally // impossible for rustfmt to come across one of these nodes when formatting a file. diff --git a/src/tools/rustfmt/src/utils.rs b/src/tools/rustfmt/src/utils.rs index 2936025d2f92c..7f1af83c25d38 100644 --- a/src/tools/rustfmt/src/utils.rs +++ b/src/tools/rustfmt/src/utils.rs @@ -582,7 +582,8 @@ pub(crate) fn is_block_expr(context: &RewriteContext<'_>, expr: &ast::Expr, repr | ast::ExprKind::Use(..) | ast::ExprKind::Type(..) | ast::ExprKind::Yield(..) - | ast::ExprKind::Underscore => false, + | ast::ExprKind::Underscore + | ast::ExprKind::BtfFieldInfo(..) => false, } } diff --git a/tests/auxiliary/minicore.rs b/tests/auxiliary/minicore.rs index 2b7eb0b9afbcf..72a7dd6ab74a3 100644 --- a/tests/auxiliary/minicore.rs +++ b/tests/auxiliary/minicore.rs @@ -19,6 +19,7 @@ // ignore-tidy-file-linelength #![feature( + allow_internal_unstable, no_core, intrinsics, lang_items, @@ -357,6 +358,30 @@ trait Drop { #[rustc_intrinsic] pub const unsafe fn copy_nonoverlapping(src: *const T, dst: *mut T, count: usize); +#[rustc_nounwind] +#[rustc_intrinsic] +#[lang = "btf_preserve_access_index"] +pub fn btf_preserve_access_index( + base: *const (), + variant: u32, + field: u32, +) -> *const (); + +#[rustc_nounwind] +#[rustc_intrinsic] +#[lang = "btf_preserve_field_byte_offset"] +pub fn btf_preserve_field_byte_offset(field: *const ()) -> usize; + +#[rustc_nounwind] +#[rustc_intrinsic] +#[lang = "btf_preserve_field_byte_size"] +pub fn btf_preserve_field_byte_size(field: *const ()) -> usize; + +#[rustc_nounwind] +#[rustc_intrinsic] +#[lang = "btf_preserve_field_exists"] +pub fn btf_preserve_field_exists(field: *const ()) -> bool; + pub mod mem { #[rustc_nounwind] #[rustc_intrinsic] @@ -368,6 +393,11 @@ pub mod mem { #[rustc_nounwind] #[rustc_intrinsic] pub const fn align_of() -> usize; + + #[allow_internal_unstable(builtin_syntax)] + pub macro offset_of($Container:ty, $($fields:expr)+ $(,)?) { + const { builtin # offset_of($Container, $($fields)+) } + } } pub mod ptr { diff --git a/tests/codegen-llvm/btf-relocations.rs b/tests/codegen-llvm/btf-relocations.rs new file mode 100644 index 0000000000000..063d3093e6a36 --- /dev/null +++ b/tests/codegen-llvm/btf-relocations.rs @@ -0,0 +1,178 @@ +//@ add-minicore +//@ needs-llvm-components: bpf +//@ compile-flags: --target bpfel-unknown-none -Cdebuginfo=2 + +#![feature(allow_internal_unstable, btf_relocations, decl_macro, no_core)] +#![no_core] +#![no_std] +#![no_main] + +extern crate minicore; +use minicore::*; + +#[allow_internal_unstable(builtin_syntax)] +pub macro field_byte_offset($Container:ty, $($fields:expr)+ $(,)?) {{ + builtin # btf_field_byte_offset($Container, $($fields)+) +}} + +#[allow_internal_unstable(builtin_syntax)] +pub macro field_byte_size($Container:ty, $($fields:expr)+ $(,)?) {{ + builtin # btf_field_byte_size($Container, $($fields)+) +}} + +#[allow_internal_unstable(builtin_syntax)] +pub macro field_exists($Container:ty, $($fields:expr)+ $(,)?) {{ + builtin # btf_field_exists($Container, $($fields)+) +}} + +#[btf_relocatable] +#[repr(C)] +pub struct Inner { + pub x: u32, + pub y: u64, +} + +#[btf_relocatable] +#[repr(C)] +pub union Payload { + pub word: u64, + pub half: u32, +} + +#[btf_relocatable] +#[repr(C)] +pub struct Outer { + pub pad: u32, + pub inner: Inner, + pub payload: Payload, +} + +// BTF CO-RE relocations are represented in the following format in LLVM IR: +// +// llvm.::$ +// +// For example: +// +// llvm.Outer:0:8$0:1 +// │ │ │ └─ access path: 0:1 (base 0, field 1 `inner`), Outer.inner +// │ │ └──── compile-time value: 8 +// │ └────── relocation kind: 0 (FIELD_BYTE_OFFSET) +// └──────────── root type: Outer +// +// llvm.Outer:1:8$0:1:1 +// │ │ │ └─ access path: 0:1:1 (base 0, field 1 `inner`, field 1 `y`), Outer.inner.y +// │ │ └──── compile-time value: 8 +// │ └────── relocation kind: 1(FIELD_BYTE_SIZE) +// └──────────── root type: Outer +// +// The relocation types are standardized across Linux kernel[0] and LLVM[1]. +// Rust currently supports: +// +// * FIELD_BYTE_OFFSET: 0 +// * FIELD_BYTE_SIZE: 1 +// * FIELD_EXISTS: 2 +// +// [0] https://elixir.bootlin.com/linux/v7.2.5/source/include/uapi/linux/bpf.h#L7616 +// [1] https://github.com/llvm/llvm-project/blob/llvmorg-23.1.1/llvm/include/llvm/DebugInfo/BTF/BTF.h#L281 +// +// CHECK-DAG: @"llvm.Outer:[[BTF_FIELD_BYTE_OFFSET:0]]:[[INNER_OFFSET:8]]$[[INNER_PATH:0:1]]" = external global i32, !llvm.preserve.access.index !0 #0 +// CHECK-DAG: @"llvm.Outer:[[BTF_FIELD_BYTE_SIZE:1]]:[[INNER_SIZE:16]]$[[INNER_PATH]]" = external global i32, !llvm.preserve.access.index !0 #0 +// CHECK-DAG: @"llvm.Outer:[[BTF_FIELD_EXISTS:2]]:[[INNER_EXISTS:1]]$[[INNER_PATH]]" = external global i32, !llvm.preserve.access.index !0 #0 +// +// CHECK-DAG: @"llvm.Outer:[[BTF_FIELD_BYTE_OFFSET]]:[[INNER_Y_OFFSET:16]]$[[INNER_Y_PATH:0:1:1]]" = external global i32, !llvm.preserve.access.index !0 #0 +// CHECK-DAG: @"llvm.Outer:[[BTF_FIELD_BYTE_SIZE]]:[[INNER_Y_SIZE:8]]$[[INNER_Y_PATH]]" = external global i32, !llvm.preserve.access.index !0 #0 +// CHECK-DAG: @"llvm.Outer:[[BTF_FIELD_EXISTS]]:[[INNER_Y_EXISTS:1]]$[[INNER_Y_PATH]]" = external global i32, !llvm.preserve.access.index !0 #0 +// +// CHECK-DAG: @"llvm.Outer:[[BTF_FIELD_BYTE_OFFSET]]:[[PAYLOAD_HALF_OFFSET:24]]$[[PAYLOAD_HALF_PATH:0:2:1]]" = external global i32, !llvm.preserve.access.index !0 #0 +// CHECK-DAG: @"llvm.Outer:[[BTF_FIELD_BYTE_SIZE]]:[[PAYLOAD_HALF_SIZE:4]]$[[PAYLOAD_HALF_PATH]]" = external global i32, !llvm.preserve.access.index !0 #0 +// CHECK-DAG: @"llvm.Outer:[[BTF_FIELD_EXISTS]]:[[PAYLOAD_HALF_EXISTS:1]]$[[PAYLOAD_HALF_PATH]]" = external global i32, !llvm.preserve.access.index !0 #0 + +// CHECK-LABEL: define{{.*}} @field_offset( +#[unsafe(no_mangle)] +pub fn field_offset() -> usize { + // CHECK: [[A:%.*]] = load i32, ptr @"llvm.Outer:[[BTF_FIELD_BYTE_OFFSET]]:[[INNER_OFFSET]]$[[INNER_PATH]]", align 4 + // CHECK-NEXT: [[B:%.*]] = tail call i32 @llvm.bpf.passthrough.i32.i32(i32 {{[0-9]+}}, i32 [[A]]) + // CHECK-NEXT: [[C:%.*]] = zext i32 [[B]] to i64 + // CHECK-NEXT: ret i64 [[C]] + field_byte_offset!(Outer, inner) +} + +// CHECK-LABEL: define{{.*}} @field_size( +#[unsafe(no_mangle)] +pub fn field_size() -> usize { + // CHECK: [[A:%.*]] = load i32, ptr @"llvm.Outer:[[BTF_FIELD_BYTE_SIZE]]:[[INNER_SIZE]]$[[INNER_PATH]]", align 4 + // CHECK-NEXT: [[B:%.*]] = tail call i32 @llvm.bpf.passthrough.i32.i32(i32 {{[0-9]+}}, i32 [[A]]) + // CHECK-NEXT: [[C:%.*]] = zext i32 [[B]] to i64 + // CHECK-NEXT: ret i64 [[C]] + field_byte_size!(Outer, inner) +} + +// CHECK-LABEL: define{{.*}} @field_exists( +#[unsafe(no_mangle)] +pub fn field_exists() -> bool { + // CHECK: [[A:%.*]] = load i32, ptr @"llvm.Outer:[[BTF_FIELD_EXISTS]]:[[INNER_EXISTS]]$[[INNER_PATH]]", align 4 + // CHECK-NEXT: [[B:%.*]] = tail call i32 @llvm.bpf.passthrough.i32.i32(i32 {{[0-9]+}}, i32 [[A]]) + // CHECK-NEXT: [[C:%.*]] = icmp ne i32 [[B]], 0 + // CHECK-NEXT: ret i1 [[C]] + field_exists!(Outer, inner) +} + +// CHECK-LABEL: define{{.*}} @nested_field_offset( +#[unsafe(no_mangle)] +pub fn nested_field_offset() -> usize { + // CHECK: [[A:%.*]] = load i32, ptr @"llvm.Outer:[[BTF_FIELD_BYTE_OFFSET]]:[[INNER_Y_OFFSET]]$[[INNER_Y_PATH]]", align 4 + // CHECK-NEXT: [[B:%.*]] = tail call i32 @llvm.bpf.passthrough.i32.i32(i32 {{[0-9]+}}, i32 [[A]]) + // CHECK-NEXT: [[C:%.*]] = zext i32 %1 to i64 + // CHECK-NEXT: ret i64 [[C]] + field_byte_offset!(Outer, inner.y) +} + +// CHECK-LABEL: define{{.*}} @nested_field_size( +#[unsafe(no_mangle)] +pub fn nested_field_size() -> usize { + // CHECK: [[A:%.*]] = load i32, ptr @"llvm.Outer:[[BTF_FIELD_BYTE_SIZE]]:[[INNER_Y_SIZE]]$[[INNER_Y_PATH]]", align 4 + // CHECK-NEXT: [[B:%.*]] = tail call i32 @llvm.bpf.passthrough.i32.i32(i32 {{[0-9]+}}, i32 [[A]]) + // CHECK-NEXT: [[C:%.*]] = zext i32 [[B]] to i64 + // CHECK-NEXT: ret i64 [[C]] + field_byte_size!(Outer, inner.y) +} + +// CHECK-LABEL: define{{.*}} @nested_field_exists( +#[unsafe(no_mangle)] +pub fn nested_field_exists() -> bool { + // CHECK: [[A:%.*]] = load i32, ptr @"llvm.Outer:[[BTF_FIELD_EXISTS]]:[[INNER_Y_EXISTS]]$[[INNER_Y_PATH]]", align 4 + // CHECK-NEXT: [[B:%.*]] = tail call i32 @llvm.bpf.passthrough.i32.i32(i32 {{[0-9]+}}, i32 [[A]]) + // CHECK-NEXT: [[C:%.*]] = icmp ne i32 [[B]], 0 + // CHECK-NEXT: ret i1 [[C]] + field_exists!(Outer, inner.y) +} + +// CHECK-LABEL: define{{.*}} @union_field_offset( +#[unsafe(no_mangle)] +pub fn union_field_offset() -> usize { + // CHECK: [[A:%.*]] = load i32, ptr @"llvm.Outer:[[BTF_FIELD_BYTE_OFFSET]]:[[PAYLOAD_HALF_OFFSET]]$[[PAYLOAD_HALF_PATH]]", align 4 + // CHECK-NEXT: [[B:%.*]] = tail call i32 @llvm.bpf.passthrough.i32.i32(i32 {{[0-9]+}}, i32 [[A]]) + // CHECK-NEXT: [[C:%.*]] = zext i32 [[B]] to i64 + // CHECK-NEXT: ret i64 [[C]] + field_byte_offset!(Outer, payload.half) +} + +// CHECK-LABEL: define{{.*}} @union_field_size( +#[unsafe(no_mangle)] +pub fn union_field_size() -> usize { + // CHECK: [[A:%.*]] = load i32, ptr @"llvm.Outer:[[BTF_FIELD_BYTE_SIZE]]:[[PAYLOAD_HALF_SIZE]]$[[PAYLOAD_HALF_PATH]]", align 4 + // CHECK-NEXT: [[B:%.*]] = tail call i32 @llvm.bpf.passthrough.i32.i32(i32 {{[0-9]+}}, i32 [[A]]) + // CHECK-NEXT: [[C:%.*]] = zext i32 [[B]] to i64 + // CHECK-NEXT: ret i64 [[C]] + field_byte_size!(Outer, payload.half) +} + +// CHECK-LABEL: define{{.*}} @union_field_exists( +#[unsafe(no_mangle)] +pub fn union_field_exists() -> bool { + // CHECK: [[A:%.*]] = load i32, ptr @"llvm.Outer:[[BTF_FIELD_EXISTS]]:[[PAYLOAD_HALF_EXISTS]]$[[PAYLOAD_HALF_PATH]]", align 4 + // CHECK-NEXT: [[B:%.*]] = tail call i32 @llvm.bpf.passthrough.i32.i32(i32 {{[0-9]+}}, i32 [[A]]) + // CHECK-NEXT: [[C:%.*]] = icmp ne i32 [[B]], 0 + // CHECK-NEXT: ret i1 [[C]] + field_exists!(Outer, payload.half) +} diff --git a/tests/ui/README.md b/tests/ui/README.md index 8df2769996f41..c807109931d00 100644 --- a/tests/ui/README.md +++ b/tests/ui/README.md @@ -185,6 +185,13 @@ See: - [`std::box::Boxed`](https://doc.rust-lang.org/std/boxed/struct.Box.html) +## `tests/ui/btf-relocations/`: BTF relocations + +Tests for [Compile Once, Run Everywhere (CO-RE)][co-re] relocations based on the [BPF Type Format (BTF)][btf]. + +[co-re]: https://nakryiko.com/posts/bpf-portability-and-co-re/ +[btf]: https://docs.kernel.org/bpf/btf.html + ## `tests/ui/builtin-superkinds/`: Built-in Trait Hierarchy Tests Tests for built-in trait hierarchy (Send, Sync, Sized, etc.) and their supertrait relationships. E.g. auto traits and marker trait constraints. diff --git a/tests/ui/btf-relocations/attribute-arch-check.rs b/tests/ui/btf-relocations/attribute-arch-check.rs new file mode 100644 index 0000000000000..4df06da0e8095 --- /dev/null +++ b/tests/ui/btf-relocations/attribute-arch-check.rs @@ -0,0 +1,48 @@ +//@ add-minicore +//@ needs-llvm-components: x86 +//@ compile-flags: --target x86_64-unknown-none + +#![feature(btf_relocations)] +#![feature(no_core)] +#![no_core] + +extern crate minicore; + +#[btf_relocatable] +//~^ ERROR the `btf_relocatable` attribute can only be used on BPF architecture +struct ValidStructInner { + field: u32, +} + +#[btf_relocatable] +//~^ ERROR the `btf_relocatable` attribute can only be used on BPF architecture +struct ValidStruct { + field: u32, + inner: ValidStructInner, +} + +#[btf_relocatable] +//~^ ERROR the `btf_relocatable` attribute can only be used on BPF architecture +union ValidUnion { + word: u64, + half: u32, +} + +#[btf_relocatable] +//~^ ERROR the `btf_relocatable` attribute can only be used on BPF architecture +//~| ERROR the `btf_relocatable` attribute cannot be used on enums +enum InvalidEnum { + A, +} + +#[btf_relocatable] +//~^ ERROR the `btf_relocatable` attribute can only be used on BPF architecture +//~| ERROR the `btf_relocatable` attribute cannot be used on functions +fn invalid_function() {} + +#[btf_relocatable] +//~^ ERROR the `btf_relocatable` attribute can only be used on BPF architecture +//~| ERROR the `btf_relocatable` attribute cannot be used on traits +trait InvalidTrait {} + +fn main() {} diff --git a/tests/ui/btf-relocations/attribute-arch-check.stderr b/tests/ui/btf-relocations/attribute-arch-check.stderr new file mode 100644 index 0000000000000..c815cac8a1397 --- /dev/null +++ b/tests/ui/btf-relocations/attribute-arch-check.stderr @@ -0,0 +1,62 @@ +error: the `btf_relocatable` attribute can only be used on BPF architecture + --> $DIR/attribute-arch-check.rs:11:1 + | +LL | #[btf_relocatable] + | ^^^^^^^^^^^^^^^^^^ + +error: the `btf_relocatable` attribute can only be used on BPF architecture + --> $DIR/attribute-arch-check.rs:17:1 + | +LL | #[btf_relocatable] + | ^^^^^^^^^^^^^^^^^^ + +error: the `btf_relocatable` attribute can only be used on BPF architecture + --> $DIR/attribute-arch-check.rs:24:1 + | +LL | #[btf_relocatable] + | ^^^^^^^^^^^^^^^^^^ + +error: the `btf_relocatable` attribute cannot be used on enums + --> $DIR/attribute-arch-check.rs:31:3 + | +LL | #[btf_relocatable] + | ^^^^^^^^^^^^^^^ + | + = help: the `btf_relocatable` attribute can be applied to structs and unions + +error: the `btf_relocatable` attribute can only be used on BPF architecture + --> $DIR/attribute-arch-check.rs:31:1 + | +LL | #[btf_relocatable] + | ^^^^^^^^^^^^^^^^^^ + +error: the `btf_relocatable` attribute cannot be used on functions + --> $DIR/attribute-arch-check.rs:38:3 + | +LL | #[btf_relocatable] + | ^^^^^^^^^^^^^^^ + | + = help: the `btf_relocatable` attribute can only be applied to data types + +error: the `btf_relocatable` attribute can only be used on BPF architecture + --> $DIR/attribute-arch-check.rs:38:1 + | +LL | #[btf_relocatable] + | ^^^^^^^^^^^^^^^^^^ + +error: the `btf_relocatable` attribute cannot be used on traits + --> $DIR/attribute-arch-check.rs:43:3 + | +LL | #[btf_relocatable] + | ^^^^^^^^^^^^^^^ + | + = help: the `btf_relocatable` attribute can only be applied to data types + +error: the `btf_relocatable` attribute can only be used on BPF architecture + --> $DIR/attribute-arch-check.rs:43:1 + | +LL | #[btf_relocatable] + | ^^^^^^^^^^^^^^^^^^ + +error: aborting due to 9 previous errors + diff --git a/tests/ui/btf-relocations/attribute.rs b/tests/ui/btf-relocations/attribute.rs new file mode 100644 index 0000000000000..157ff8a7fa4b0 --- /dev/null +++ b/tests/ui/btf-relocations/attribute.rs @@ -0,0 +1,42 @@ +//@ add-minicore +//@ needs-llvm-components: bpf +//@ compile-flags: --target bpfel-unknown-none + +#![feature(btf_relocations)] +#![feature(no_core)] +#![no_core] + +extern crate minicore; + +#[btf_relocatable] +struct ValidStructInner { + field: u32, +} + +#[btf_relocatable] +struct ValidStruct { + field: u32, + inner: ValidStructInner, +} + +#[btf_relocatable] +union ValidUnion { + word: u64, + half: u32, +} + +#[btf_relocatable] +//~^ ERROR the `btf_relocatable` attribute cannot be used on enums +enum InvalidEnum { + A, +} + +#[btf_relocatable] +//~^ ERROR the `btf_relocatable` attribute cannot be used on functions +fn invalid_function() {} + +#[btf_relocatable] +//~^ ERROR the `btf_relocatable` attribute cannot be used on traits +trait InvalidTrait {} + +fn main() {} diff --git a/tests/ui/btf-relocations/attribute.stderr b/tests/ui/btf-relocations/attribute.stderr new file mode 100644 index 0000000000000..6e0e411594073 --- /dev/null +++ b/tests/ui/btf-relocations/attribute.stderr @@ -0,0 +1,26 @@ +error: the `btf_relocatable` attribute cannot be used on enums + --> $DIR/attribute.rs:28:3 + | +LL | #[btf_relocatable] + | ^^^^^^^^^^^^^^^ + | + = help: the `btf_relocatable` attribute can be applied to structs and unions + +error: the `btf_relocatable` attribute cannot be used on functions + --> $DIR/attribute.rs:34:3 + | +LL | #[btf_relocatable] + | ^^^^^^^^^^^^^^^ + | + = help: the `btf_relocatable` attribute can only be applied to data types + +error: the `btf_relocatable` attribute cannot be used on traits + --> $DIR/attribute.rs:38:3 + | +LL | #[btf_relocatable] + | ^^^^^^^^^^^^^^^ + | + = help: the `btf_relocatable` attribute can only be applied to data types + +error: aborting due to 3 previous errors + diff --git a/tests/ui/btf-relocations/field-access.rs b/tests/ui/btf-relocations/field-access.rs new file mode 100644 index 0000000000000..4e9d9087efa7c --- /dev/null +++ b/tests/ui/btf-relocations/field-access.rs @@ -0,0 +1,44 @@ +//@ add-minicore +//@ needs-llvm-components: bpf +//@ compile-flags: --target bpfel-unknown-none + +#![feature(btf_relocations)] +#![feature(no_core)] +#![no_core] + +extern crate minicore; +use minicore::*; + +#[btf_relocatable] +#[repr(C)] +struct Inner { + value: u32, +} + +#[btf_relocatable] +#[repr(C)] +struct Outer { + inner: Inner, +} + +fn direct(inner: &Inner) -> u32 { + inner.value + //~^ ERROR cannot access fields of a `btf_relocatable` type directly +} + +fn nested(outer: &Outer) -> u32 { + outer.inner.value + //~^ ERROR cannot access fields of a `btf_relocatable` type directly +} + +fn offset() -> usize { + mem::offset_of!(Inner, value) + //~^ ERROR cannot use `offset_of!` with a `btf_relocatable` type +} + +fn nested_offset() -> usize { + mem::offset_of!(Outer, inner.value) + //~^ ERROR cannot use `offset_of!` with a `btf_relocatable` type +} + +fn main() {} diff --git a/tests/ui/btf-relocations/field-access.stderr b/tests/ui/btf-relocations/field-access.stderr new file mode 100644 index 0000000000000..a00088e96a74e --- /dev/null +++ b/tests/ui/btf-relocations/field-access.stderr @@ -0,0 +1,38 @@ +error: cannot access fields of a `btf_relocatable` type directly + --> $DIR/field-access.rs:25:5 + | +LL | inner.value + | ^^^^^^----- + | | + | direct field access is forbidden for BTF-relocatable types + +error: cannot access fields of a `btf_relocatable` type directly + --> $DIR/field-access.rs:30:5 + | +LL | outer.inner.value + | ^^^^^^----- + | | + | direct field access is forbidden for BTF-relocatable types + +error: cannot use `offset_of!` with a `btf_relocatable` type + --> $DIR/field-access.rs:35:5 + | +LL | mem::offset_of!(Inner, value) + | ^^^^^^^^^^^^^^^^^^^^^^^-----^ + | | + | this field requires BTF relocation + | + = note: this error originates in the macro `mem::offset_of` (in Nightly builds, run with -Z macro-backtrace for more info) + +error: cannot use `offset_of!` with a `btf_relocatable` type + --> $DIR/field-access.rs:40:5 + | +LL | mem::offset_of!(Outer, inner.value) + | ^^^^^^^^^^^^^^^^^^^^^^^-----^^^^^^^ + | | + | this field requires BTF relocation + | + = note: this error originates in the macro `mem::offset_of` (in Nightly builds, run with -Z macro-backtrace for more info) + +error: aborting due to 4 previous errors + diff --git a/tests/ui/feature-gates/feature-gate-btf-relocations.rs b/tests/ui/feature-gates/feature-gate-btf-relocations.rs new file mode 100644 index 0000000000000..f61fb8a07ea1c --- /dev/null +++ b/tests/ui/feature-gates/feature-gate-btf-relocations.rs @@ -0,0 +1,16 @@ +//@ add-minicore +//@ needs-llvm-components: bpf +//@ compile-flags: --target bpfel-unknown-none + +#![feature(no_core)] +#![no_core] + +extern crate minicore; + +#[btf_relocatable] +//~^ ERROR the `btf_relocatable` attribute is an experimental feature +struct KernelType { + field: u32, +} + +fn main() {} diff --git a/tests/ui/feature-gates/feature-gate-btf-relocations.stderr b/tests/ui/feature-gates/feature-gate-btf-relocations.stderr new file mode 100644 index 0000000000000..b6c41284596de --- /dev/null +++ b/tests/ui/feature-gates/feature-gate-btf-relocations.stderr @@ -0,0 +1,13 @@ +error[E0658]: the `btf_relocatable` attribute is an experimental feature + --> $DIR/feature-gate-btf-relocations.rs:10:3 + | +LL | #[btf_relocatable] + | ^^^^^^^^^^^^^^^ + | + = note: see issue #160616 for more information + = help: add `#![feature(btf_relocations)]` to the crate attributes to enable + = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date + +error: aborting due to 1 previous error + +For more information about this error, try `rustc --explain E0658`. diff --git a/tests/ui/macros/stringify.rs b/tests/ui/macros/stringify.rs index 242a91cab7e77..7ef057d3baf97 100644 --- a/tests/ui/macros/stringify.rs +++ b/tests/ui/macros/stringify.rs @@ -337,6 +337,8 @@ fn test_expr() { // ExprKind::FormatArgs: untestable because this test works pre-expansion. + // ExprKind::BtfFieldInfo: untestable because this test works pre-expansion. + // ExprKind::Err: untestable. // Ones involving attributes.