From 19a9a0871a62e9f84421957d30c034643e91c6aa Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Fri, 9 Oct 2026 13:53:08 +1100 Subject: [PATCH 1/6] Instantiate a response in pieces In `instantiate_and_apply_query_response`, instantiate the two instantiatable parts of the response separately. This requires adding a new method `instantiate_value` to `SolverDelegate` (r-a will need updating separately) and making a function of the same name in `rustc_infer` public. Do a similar thing for `State` in `instantiate_canonical_state`. This is equivalent behaviour that will enable further changes in subsequent commits. --- compiler/rustc_infer/src/infer/canonical/instantiate.rs | 2 +- compiler/rustc_infer/src/infer/canonical/mod.rs | 2 +- compiler/rustc_next_trait_solver/src/canonical/mod.rs | 9 ++++++--- compiler/rustc_next_trait_solver/src/delegate.rs | 4 ++++ compiler/rustc_trait_selection/src/solve/delegate.rs | 9 ++++++++- 5 files changed, 20 insertions(+), 6 deletions(-) diff --git a/compiler/rustc_infer/src/infer/canonical/instantiate.rs b/compiler/rustc_infer/src/infer/canonical/instantiate.rs index 2bc7e269efc9d..58d3dbd72ed06 100644 --- a/compiler/rustc_infer/src/infer/canonical/instantiate.rs +++ b/compiler/rustc_infer/src/infer/canonical/instantiate.rs @@ -52,7 +52,7 @@ impl<'tcx, V> Canonical<'tcx, V> { /// Instantiate the values from `var_values` into `value`. `var_values` /// must be values for the set of canonical variables that appear in /// `value`. -pub(super) fn instantiate_value<'tcx, T>( +pub fn instantiate_value<'tcx, T>( tcx: TyCtxt<'tcx>, var_values: &CanonicalVarValues<'tcx>, value: T, diff --git a/compiler/rustc_infer/src/infer/canonical/mod.rs b/compiler/rustc_infer/src/infer/canonical/mod.rs index 55c78036c6d17..626d3f1b62fcf 100644 --- a/compiler/rustc_infer/src/infer/canonical/mod.rs +++ b/compiler/rustc_infer/src/infer/canonical/mod.rs @@ -21,7 +21,7 @@ //! //! [c]: https://rust-lang.github.io/chalk/book/canonical_queries/canonicalization.html -pub use instantiate::CanonicalExt; +pub use instantiate::{CanonicalExt, instantiate_value}; use rustc_index::IndexVec; pub use rustc_middle::infer::canonical::*; use rustc_middle::ty::{self, GenericArg, Ty, TyCtxt, TypeFoldable}; diff --git a/compiler/rustc_next_trait_solver/src/canonical/mod.rs b/compiler/rustc_next_trait_solver/src/canonical/mod.rs index 45587d717b418..4074a8d30ccfe 100644 --- a/compiler/rustc_next_trait_solver/src/canonical/mod.rs +++ b/compiler/rustc_next_trait_solver/src/canonical/mod.rs @@ -111,8 +111,9 @@ where let instantiation = compute_query_response_instantiation_values(delegate, &original_values, &response, span); - let Response { var_values, external_constraints, certainty } = - delegate.instantiate_canonical(response, instantiation); + let Response { var_values, external_constraints, certainty } = response.value; + let external_constraints = delegate.instantiate_value(external_constraints, instantiation); + let var_values = delegate.instantiate_value(var_values, instantiation); unify_query_var_values(delegate, &original_values, var_values, span); @@ -608,7 +609,9 @@ where prev_universe, ); - let inspect::State { var_values, data } = delegate.instantiate_canonical(state, instantiation); + let inspect::State { var_values, data } = state.value; + let data = delegate.instantiate_value(data, instantiation); + let var_values = delegate.instantiate_value(var_values, instantiation); unify_query_var_values(delegate, orig_values, var_values, span); data diff --git a/compiler/rustc_next_trait_solver/src/delegate.rs b/compiler/rustc_next_trait_solver/src/delegate.rs index 630e1dd68e041..437d4990c809f 100644 --- a/compiler/rustc_next_trait_solver/src/delegate.rs +++ b/compiler/rustc_next_trait_solver/src/delegate.rs @@ -79,6 +79,10 @@ pub trait SolverDelegate: Deref + Sized { &self, ) -> Vec<(ty::RegionConstraint, VisibleForLeakCheck)>; + fn instantiate_value(&self, value: V, values: ty::CanonicalVarValues) -> V + where + V: TypeFoldable; + fn instantiate_canonical( &self, canonical: ty::Canonical, diff --git a/compiler/rustc_trait_selection/src/solve/delegate.rs b/compiler/rustc_trait_selection/src/solve/delegate.rs index 2335868a17b40..4f860756d321f 100644 --- a/compiler/rustc_trait_selection/src/solve/delegate.rs +++ b/compiler/rustc_trait_selection/src/solve/delegate.rs @@ -11,7 +11,7 @@ use rustc_hir::def_id::{CRATE_DEF_ID, DefId, LOCAL_CRATE}; use rustc_infer::infer::canonical::query_response::make_query_region_constraints; use rustc_infer::infer::canonical::{ Canonical, CanonicalExt as _, CanonicalQueryInput, CanonicalVarKind, CanonicalVarValues, - QueryRegionConstraint, + QueryRegionConstraint, instantiate_value, }; use rustc_infer::infer::{InferCtxt, RegionVariableOrigin, SubregionOrigin, TyCtxtInferExt}; use rustc_infer::traits::solve::{ @@ -406,6 +406,13 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate< constraints } + fn instantiate_value(&self, value: V, values: CanonicalVarValues<'tcx>) -> V + where + V: TypeFoldable>, + { + instantiate_value(self.tcx, &values, value) + } + fn instantiate_canonical( &self, canonical: Canonical<'tcx, V>, From fdbbf3fd94d59ae7ca696fe6c46293d48f982dcc Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Fri, 9 Oct 2026 15:51:37 +1100 Subject: [PATCH 2/6] Optimize `unify_query_var_values` Both `unify_query_var_values` call sites are preceded by a call to `instantiate_value` on the `var_values`. If instantiation changes any of the elements we end up interning an entirely new list, through `fold_list`. This commit changes things so that `unify_query_var_values` does instantiation on individual elements as it iterates, and avoids having to intern an entire list. This changes the caching behaviour in some cases, but overall it's a perf win. --- compiler/rustc_next_trait_solver/src/canonical/mod.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/compiler/rustc_next_trait_solver/src/canonical/mod.rs b/compiler/rustc_next_trait_solver/src/canonical/mod.rs index 4074a8d30ccfe..1d9da129b75bb 100644 --- a/compiler/rustc_next_trait_solver/src/canonical/mod.rs +++ b/compiler/rustc_next_trait_solver/src/canonical/mod.rs @@ -113,9 +113,8 @@ where let Response { var_values, external_constraints, certainty } = response.value; let external_constraints = delegate.instantiate_value(external_constraints, instantiation); - let var_values = delegate.instantiate_value(var_values, instantiation); - unify_query_var_values(delegate, &original_values, var_values, span); + unify_query_var_values(delegate, &original_values, var_values, instantiation, span); let ExternalConstraintsData { region_constraints, opaque_types, normalization_nested_goals } = &*external_constraints; @@ -492,6 +491,7 @@ fn unify_query_var_values( delegate: &D, original_values: &[I::GenericArg], var_values: I::GenericArgs, + instantiation: CanonicalVarValues, span: I::Span, ) where D: SolverDelegate, @@ -500,6 +500,7 @@ fn unify_query_var_values( assert_eq!(original_values.len(), var_values.len()); for (&orig, response) in iter::zip(original_values, var_values.iter()) { + let response = delegate.instantiate_value(response, instantiation); if orig == response { continue; } @@ -611,9 +612,8 @@ where let inspect::State { var_values, data } = state.value; let data = delegate.instantiate_value(data, instantiation); - let var_values = delegate.instantiate_value(var_values, instantiation); - unify_query_var_values(delegate, orig_values, var_values, span); + unify_query_var_values(delegate, orig_values, var_values, instantiation, span); data } From 75cca6d6bdea68d6b494b79742d24403c6585ced Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Fri, 9 Oct 2026 17:47:19 +1100 Subject: [PATCH 3/6] Special-case canonical bound vars in `unify_query_var_values` This avoids a lot of calls to `instantiate_value`. --- .../src/canonical/mod.rs | 25 ++++++++++++++++++- compiler/rustc_type_ir/src/canonical.rs | 9 +++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/compiler/rustc_next_trait_solver/src/canonical/mod.rs b/compiler/rustc_next_trait_solver/src/canonical/mod.rs index 1d9da129b75bb..39c74a2643987 100644 --- a/compiler/rustc_next_trait_solver/src/canonical/mod.rs +++ b/compiler/rustc_next_trait_solver/src/canonical/mod.rs @@ -497,10 +497,33 @@ fn unify_query_var_values( D: SolverDelegate, I: Interner, { + // Canonical bound vars are very common here. `CanonicalInstantiator::fold_{ty,region,const}`'s + // handling of them is simple, and we replicate that handling here to avoid the cost of the + // surrounding machinery. + fn canonical_bound_var(arg: I::GenericArg) -> Option { + match arg.kind() { + ty::GenericArgKind::Type(t) => match t.kind() { + ty::Bound(ty::BoundVarIndexKind::Canonical, bt) => Some(bt.var()), + _ => None, + }, + ty::GenericArgKind::Lifetime(r) => match r.kind() { + ty::ReBound(ty::BoundVarIndexKind::Canonical, br) => Some(br.var()), + _ => None, + }, + ty::GenericArgKind::Const(c) => match c.kind() { + ty::ConstKind::Bound(ty::BoundVarIndexKind::Canonical, bc) => Some(bc.var()), + _ => None, + }, + } + } + assert_eq!(original_values.len(), var_values.len()); for (&orig, response) in iter::zip(original_values, var_values.iter()) { - let response = delegate.instantiate_value(response, instantiation); + let response = match canonical_bound_var::(response) { + Some(var) => instantiation[var], + None => delegate.instantiate_value(response, instantiation), + }; if orig == response { continue; } diff --git a/compiler/rustc_type_ir/src/canonical.rs b/compiler/rustc_type_ir/src/canonical.rs index fe3d68b0d5593..ef6861d83f627 100644 --- a/compiler/rustc_type_ir/src/canonical.rs +++ b/compiler/rustc_type_ir/src/canonical.rs @@ -1,4 +1,5 @@ use std::fmt; +use std::ops::Index; use arrayvec::ArrayVec; use derive_where::derive_where; @@ -267,6 +268,14 @@ impl CanonicalVarValues { } } +impl Index for CanonicalVarValues { + type Output = I::GenericArg; + + fn index(&self, value: ty::BoundVar) -> &I::GenericArg { + &self.var_values.as_slice()[value.as_usize()] + } +} + #[derive_where(Default; I: Interner)] pub struct CanonicalParamEnvCache( pub HashMap>, From 52d3937bf8fabd9229892a2936fa1ced4001542f Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 12 Oct 2026 14:32:47 +1100 Subject: [PATCH 4/6] Introduce `IsIdentity` type `Response` has `is_identity` and `is_identity_modulo_regions` methods. These are reasonably hot and traverse the response's `var_values` in very similar ways. This commit merges these two methods into a single method that returns a new three-value enum `IsIdentity`. It also pre-computes that `IsIdentity` for each response and stores it in the new `Response::var_values_is_identity` field. This is a perf win because it avoids repeated traversals. --- .../src/canonical/mod.rs | 7 ++- .../src/solve/eval_ctxt/mod.rs | 12 ++-- .../rustc_next_trait_solver/src/solve/mod.rs | 7 ++- .../src/solve/trait_goals.rs | 8 +-- compiler/rustc_type_ir/src/macros.rs | 1 + compiler/rustc_type_ir/src/solve/mod.rs | 57 ++++++++++--------- 6 files changed, 51 insertions(+), 41 deletions(-) diff --git a/compiler/rustc_next_trait_solver/src/canonical/mod.rs b/compiler/rustc_next_trait_solver/src/canonical/mod.rs index 39c74a2643987..bfb1255424c38 100644 --- a/compiler/rustc_next_trait_solver/src/canonical/mod.rs +++ b/compiler/rustc_next_trait_solver/src/canonical/mod.rs @@ -27,7 +27,7 @@ use tracing::instrument; use crate::delegate::SolverDelegate; use crate::solve::{ CanonicalResponse, Certainty, ExternalConstraintsData, ExternalRegionConstraints, Goal, - NestedNormalizationGoals, QueryInput, Response, VisibleForLeakCheck, inspect, + IsIdentity, NestedNormalizationGoals, QueryInput, Response, VisibleForLeakCheck, inspect, }; pub mod canonicalizer; @@ -111,7 +111,7 @@ where let instantiation = compute_query_response_instantiation_values(delegate, &original_values, &response, span); - let Response { var_values, external_constraints, certainty } = response.value; + let Response { certainty, var_values, var_values_is_identity: _, external_constraints } = response.value; let external_constraints = delegate.instantiate_value(external_constraints, instantiation); unify_query_var_values(delegate, &original_values, var_values, instantiation, span); @@ -670,11 +670,12 @@ pub fn response_no_constraints_raw( max_universe, var_kinds, value: Response { + certainty, var_values, + var_values_is_identity: IsIdentity::Yes, // FIXME: maybe we should store the "no response" version in cx, like // we do for cx.types and stuff. external_constraints: cx.mk_external_constraints(ExternalConstraintsData::new(cx)), - certainty, }, } } diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs index 3b372876f574b..4f96f84da2e9f 100644 --- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs @@ -43,9 +43,9 @@ use crate::solve::search_graph::SearchGraph; use crate::solve::ty::may_use_unstable_feature; use crate::solve::{ CanonicalResponse, Certainty, ExternalConstraintsData, FIXPOINT_STEP_LIMIT, Goal, - GoalEvaluation, GoalSource, GoalStalledOn, GoalStalledOnOpaques, HasChanged, MaybeCause, - NestedNormalizationGoals, NoSolution, QueryInput, QueryResult, Response, SucceededInErased, - VisibleForLeakCheck, inspect, + GoalEvaluation, GoalSource, GoalStalledOn, GoalStalledOnOpaques, HasChanged, IsIdentity, + MaybeCause, NestedNormalizationGoals, NoSolution, QueryInput, QueryResult, Response, + SucceededInErased, VisibleForLeakCheck, inspect, }; pub mod fast_path; @@ -1711,15 +1711,17 @@ where filter_irrelevant_region_constraints(self.delegate, var_values, &mut external_constraints); - let canonical = canonicalize_response( + let mut canonical = canonicalize_response( self.delegate, self.max_input_universe, Response { - var_values, certainty, + var_values, + var_values_is_identity: IsIdentity::No, // updated below, after canonicalization external_constraints: self.cx().mk_external_constraints(external_constraints), }, ); + canonical.value.update_var_values_is_identity(); Ok(canonical) } diff --git a/compiler/rustc_next_trait_solver/src/solve/mod.rs b/compiler/rustc_next_trait_solver/src/solve/mod.rs index c9da77eced6d3..255ed762d8a10 100644 --- a/compiler/rustc_next_trait_solver/src/solve/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/mod.rs @@ -65,7 +65,7 @@ fn has_no_inference_or_external_constraints( ref opaque_types, ref normalization_nested_goals, } = *response.value.external_constraints; - response.value.is_identity() + response.value.var_values_is_identity == IsIdentity::Yes && region_constraints.is_empty() && opaque_types.is_empty() && normalization_nested_goals.is_empty() @@ -77,7 +77,7 @@ fn has_only_region_constraints(response: ty::Canonical= IsIdentity::ModuloRegions && opaque_types.is_empty() && normalization_nested_goals.is_empty() } @@ -93,8 +93,9 @@ fn equal_response_modulo_region_constraints( var_kinds: a_var_kinds, value: Response { - var_values: a_var_values, certainty: a_certainty, + var_values: a_var_values, + var_values_is_identity: _, // ignore because it's purely derived from `var_values` external_constraints: a_external_constraints, }, } = a; diff --git a/compiler/rustc_next_trait_solver/src/solve/trait_goals.rs b/compiler/rustc_next_trait_solver/src/solve/trait_goals.rs index 1d3008da399dc..ecf7448b912e5 100644 --- a/compiler/rustc_next_trait_solver/src/solve/trait_goals.rs +++ b/compiler/rustc_next_trait_solver/src/solve/trait_goals.rs @@ -5,9 +5,9 @@ use rustc_type_ir::fast_reject::DeepRejectCtxt; use rustc_type_ir::inherent::*; use rustc_type_ir::lang_items::SolverTraitLangItem; use rustc_type_ir::solve::{ - AliasBoundKind, CandidatePreferenceMode, CanonicalResponse, ExternalConstraintsData, MaybeInfo, - NoSolutionOrRerunNonErased, OpaqueTypesJank, QueryResultOrRerunNonErased, RerunNonErased, - RerunReason, RerunResultExt, SizedTraitKind, + AliasBoundKind, CandidatePreferenceMode, CanonicalResponse, ExternalConstraintsData, + IsIdentity, MaybeInfo, NoSolutionOrRerunNonErased, OpaqueTypesJank, + QueryResultOrRerunNonErased, RerunNonErased, RerunReason, RerunResultExt, SizedTraitKind, }; use rustc_type_ir::{ self as ty, ClausePolarity, ExistentialPredicate, FieldInfo, Interner, MayBeErased, Movability, @@ -1332,7 +1332,7 @@ where // Proving an auto trait for the hidden type must not constrain inference // variables, as that would leak the hidden type itself. - if !candidate.result.value.is_identity_modulo_regions() { + if candidate.result.value.var_values_is_identity == IsIdentity::No { return self.forced_ambiguity(MaybeInfo::AMBIGUOUS); } diff --git a/compiler/rustc_type_ir/src/macros.rs b/compiler/rustc_type_ir/src/macros.rs index 096f43f81e608..6559d28fad5d3 100644 --- a/compiler/rustc_type_ir/src/macros.rs +++ b/compiler/rustc_type_ir/src/macros.rs @@ -78,6 +78,7 @@ TrivialTypeTraversalImpls! { crate::solve::BuiltinImplSource, crate::solve::Certainty, crate::solve::GoalSource, + crate::solve::IsIdentity, crate::solve::VisibleForLeakCheck, rustc_abi::ExternAbi, rustc_ast_ir::Mutability, diff --git a/compiler/rustc_type_ir/src/solve/mod.rs b/compiler/rustc_type_ir/src/solve/mod.rs index 55fb628e1ee3e..53afc311cc623 100644 --- a/compiler/rustc_type_ir/src/solve/mod.rs +++ b/compiler/rustc_type_ir/src/solve/mod.rs @@ -582,63 +582,68 @@ pub enum FetchEligibleAssocItemResponse { NotFoundBecauseErased, } +/// The variant order here matters for the `PartialOrd`/`Ord` impls. +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr(feature = "nightly", derive(StableHash_NoContext))] +pub enum IsIdentity { + No, + ModuloRegions, + Yes, +} + #[derive_where(Clone, Copy, Hash, PartialEq, Debug; I: Interner)] #[derive(TypeVisitable_Generic, GenericTypeVisitable, TypeFoldable_Generic)] #[cfg_attr(feature = "nightly", derive(StableHash_NoContext))] pub struct Response { pub certainty: Certainty, pub var_values: I::GenericArgs, + /// Is `var_values` an identity? + pub var_values_is_identity: IsIdentity, /// Additional constraints returned by this query. pub external_constraints: I::ExternalConstraints, } impl Response { - pub fn is_identity(&self) -> bool { - self.var_values.iter().enumerate().all(|(bv, arg)| match arg.kind() { - ty::GenericArgKind::Lifetime(r) => { - matches!(r.kind(), ty::ReBound(ty::BoundVarIndexKind::Canonical, br) if br.var().as_usize() == bv) - } - ty::GenericArgKind::Type(ty) => { - matches!(ty.kind(), ty::Bound(ty::BoundVarIndexKind::Canonical, bt) if bt.var().as_usize() == bv) - } - ty::GenericArgKind::Const(ct) => { - matches!(ct.kind(), ty::ConstKind::Bound(ty::BoundVarIndexKind::Canonical, bc) if bc.var().as_usize() == bv) - } - }) - } + pub fn update_var_values_is_identity(&mut self) { + // We start optimistic and downgrade if necessary along the way. + self.var_values_is_identity = IsIdentity::Yes; - pub fn is_identity_modulo_regions(&self) -> bool { - let mut var = ty::BoundVar::ZERO; + let mut bv = ty::BoundVar::ZERO; for arg in self.var_values.iter() { match arg.kind() { ty::GenericArgKind::Lifetime(r) => { - if matches!(r.kind(), ty::ReBound(ty::BoundVarIndexKind::Canonical, br) if var == br.var()) + if let ty::ReBound(ty::BoundVarIndexKind::Canonical, br) = r.kind() + && bv == br.var() { - var = var + 1; + bv = bv + 1; } else { - // It's ok if this region var isn't an identity variable + // If this region var isn't an identity variable we can't be + // `IsIdentity::Yes`, but we can still be `IsIdentity::ModuloRegions`. + self.var_values_is_identity = IsIdentity::ModuloRegions; } } ty::GenericArgKind::Type(ty) => { - if matches!(ty.kind(), ty::Bound(ty::BoundVarIndexKind::Canonical, bt) if var == bt.var()) + if let ty::Bound(ty::BoundVarIndexKind::Canonical, bt) = ty.kind() + && bv == bt.var() { - var = var + 1; + bv = bv + 1; } else { - return false; + self.var_values_is_identity = IsIdentity::No; + break; } } ty::GenericArgKind::Const(ct) => { - if matches!(ct.kind(), ty::ConstKind::Bound(ty::BoundVarIndexKind::Canonical, bc) if var == bc.var()) + if let ty::ConstKind::Bound(ty::BoundVarIndexKind::Canonical, bc) = ct.kind() + && bv == bc.var() { - var = var + 1; + bv = bv + 1; } else { - return false; + self.var_values_is_identity = IsIdentity::No; + break; } } } } - - true } } From 211d8e5834ee86c2c2d149805edd241f86dd734a Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 12 Oct 2026 16:07:19 +1100 Subject: [PATCH 5/6] Add an early return to `instantiate_and_apply_query_response` If the response is an identity, and there are no external constraints, and the response var kinds match the canonical goal var kinds, this function is a no-op. --- .../src/canonical/mod.rs | 17 ++++++++++++++--- .../src/solve/eval_ctxt/mod.rs | 2 ++ .../rustc_next_trait_solver/src/solve/mod.rs | 2 +- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/compiler/rustc_next_trait_solver/src/canonical/mod.rs b/compiler/rustc_next_trait_solver/src/canonical/mod.rs index bfb1255424c38..82f9331a15cff 100644 --- a/compiler/rustc_next_trait_solver/src/canonical/mod.rs +++ b/compiler/rustc_next_trait_solver/src/canonical/mod.rs @@ -27,7 +27,8 @@ use tracing::instrument; use crate::delegate::SolverDelegate; use crate::solve::{ CanonicalResponse, Certainty, ExternalConstraintsData, ExternalRegionConstraints, Goal, - IsIdentity, NestedNormalizationGoals, QueryInput, Response, VisibleForLeakCheck, inspect, + IsIdentity, NestedNormalizationGoals, QueryInput, Response, VisibleForLeakCheck, + has_no_inference_or_external_constraints, inspect, }; pub mod canonicalizer; @@ -93,7 +94,7 @@ where /// After calling a canonical query, we apply the constraints returned /// by the query using this function. /// -/// This happens in three steps: +/// If a quick early-return check fails, this happens in three steps: /// - we instantiate the bound variables of the query response /// - we unify the `var_values` of the response with the `original_values` /// - we apply the `external_constraints` returned by the query, returning @@ -102,16 +103,26 @@ pub(super) fn instantiate_and_apply_query_response( delegate: &D, original_values: &[I::GenericArg], response: CanonicalResponse, + canonical_goal_var_kinds: I::CanonicalVarKinds, span: I::Span, ) -> (NestedNormalizationGoals, Certainty) where D: SolverDelegate, I: Interner, { + let Response { certainty, var_values, external_constraints, .. } = response.value; + + // If the response is an identity, and there are no external constraints, and the response var + // kinds match the canonical goal var kinds, this function is a no-op and we can return early. + if has_no_inference_or_external_constraints(response) + && response.var_kinds == canonical_goal_var_kinds + { + return (NestedNormalizationGoals::empty(), certainty); + } + let instantiation = compute_query_response_instantiation_values(delegate, &original_values, &response, span); - let Response { certainty, var_values, var_values_is_identity: _, external_constraints } = response.value; let external_constraints = delegate.instantiate_value(external_constraints, instantiation); unify_query_var_values(delegate, &original_values, var_values, instantiation, span); diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs index 4f96f84da2e9f..bb40dd8d00dc6 100644 --- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs @@ -883,6 +883,7 @@ where self.delegate, &orig_values, response, + canonical_goal.canonical.var_kinds, self.origin_span, ); @@ -2064,6 +2065,7 @@ pub(super) fn evaluate_root_goal_for_proof_tree, delegate, &proof_tree.orig_values, response, + canonical_goal.canonical.var_kinds, origin_span, ); diff --git a/compiler/rustc_next_trait_solver/src/solve/mod.rs b/compiler/rustc_next_trait_solver/src/solve/mod.rs index 255ed762d8a10..b1d30e4ad38ce 100644 --- a/compiler/rustc_next_trait_solver/src/solve/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/mod.rs @@ -57,7 +57,7 @@ pub enum HasChanged { // FIXME(trait-system-refactor-initiative#117): we don't detect whether a response // ended up pulling down any universes. -fn has_no_inference_or_external_constraints( +pub(crate) fn has_no_inference_or_external_constraints( response: ty::Canonical>, ) -> bool { let ExternalConstraintsData { From 1d54383e4ac2bcd14880562abf0a8b63e925dd38 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 14 Oct 2026 15:28:22 +1100 Subject: [PATCH 6/6] Introduce `Canonicalized` and other clarifications It provides a more principled and type-safe way to extract and process parts of a `Canonical` value. --- compiler/rustc_middle/src/infer/canonical.rs | 1 + .../src/canonical/mod.rs | 86 +++++++++++++------ .../rustc_next_trait_solver/src/delegate.rs | 6 +- .../src/solve/delegate.rs | 6 +- compiler/rustc_type_ir/src/canonical.rs | 29 +++++++ 5 files changed, 96 insertions(+), 32 deletions(-) diff --git a/compiler/rustc_middle/src/infer/canonical.rs b/compiler/rustc_middle/src/infer/canonical.rs index 291a7b8e59f4f..b25848e441b70 100644 --- a/compiler/rustc_middle/src/infer/canonical.rs +++ b/compiler/rustc_middle/src/infer/canonical.rs @@ -34,6 +34,7 @@ use crate::ty::{self, GenericArg, GenericArgsRef, List, Ty, TyCtxt, TypeFlags, T pub type CanonicalQueryInput<'tcx, V> = ir::CanonicalQueryInput, V>; pub type Canonical<'tcx, V> = ir::Canonical, V>; +pub type Canonicalized = ir::Canonicalized; pub type CanonicalVarKind<'tcx> = ir::CanonicalVarKind>; pub type CanonicalVarValues<'tcx> = ir::CanonicalVarValues>; pub type CanonicalVarKinds<'tcx> = &'tcx List>; diff --git a/compiler/rustc_next_trait_solver/src/canonical/mod.rs b/compiler/rustc_next_trait_solver/src/canonical/mod.rs index 82f9331a15cff..d59a90e72c2c9 100644 --- a/compiler/rustc_next_trait_solver/src/canonical/mod.rs +++ b/compiler/rustc_next_trait_solver/src/canonical/mod.rs @@ -18,8 +18,8 @@ use rustc_type_ir::relate::{ self, Relate, RelateResult, TypeRelation, VarianceDiagInfo, relate_args_invariantly, }; use rustc_type_ir::{ - self as ty, Canonical, CanonicalVarKind, CanonicalVarValues, Const, InferCtxtLike, Interner, - Region, TypeFoldable, TypingMode, TypingModeEqWrapper, + self as ty, Canonical, CanonicalVarKind, CanonicalVarValues, Canonicalized, Const, + InferCtxtLike, Interner, Region, TypeFoldable, TypingMode, TypingModeEqWrapper, }; use thin_vec::ThinVec; use tracing::instrument; @@ -110,7 +110,7 @@ where D: SolverDelegate, I: Interner, { - let Response { certainty, var_values, external_constraints, .. } = response.value; + let certainty = response.value.certainty; // If the response is an identity, and there are no external constraints, and the response var // kinds match the canonical goal var kinds, this function is a no-op and we can return early. @@ -123,9 +123,19 @@ where let instantiation = compute_query_response_instantiation_values(delegate, &original_values, &response, span); - let external_constraints = delegate.instantiate_value(external_constraints, instantiation); - - unify_query_var_values(delegate, &original_values, var_values, instantiation, span); + // We could call `instantiate_canonical` on `response`, but it's more efficient to instantiate + // the fields within `response` separately. That way we can instantiate and unify `var_values` + // in a single step, rather than instantiating first (which creates an intermediate list) and + // then unifying afterwards. + let external_constraints = + delegate.instantiate_value(response.project(|r| r.external_constraints), instantiation); + instantiate_and_unify_query_var_values( + delegate, + &original_values, + response.project(|r| r.var_values), + instantiation, + span, + ); let ExternalConstraintsData { region_constraints, opaque_types, normalization_nested_goals } = &*external_constraints; @@ -485,7 +495,8 @@ where } } -/// Unify the `original_values` with the `var_values` returned by the canonical query.. +/// Instantiate `var_values` as returned by the canonical query and also unify it with +/// `original_values`. /// /// This assumes that this unification will always succeed. This is the case when /// applying a query response right away. However, calling a canonical query, doing any @@ -498,21 +509,43 @@ where /// from the solver we assume that the solver correctly handled aliases and therefore /// always relate them structurally here. #[instrument(level = "trace", skip(delegate))] -fn unify_query_var_values( +fn instantiate_and_unify_query_var_values( delegate: &D, original_values: &[I::GenericArg], - var_values: I::GenericArgs, + var_values: Canonicalized, instantiation: CanonicalVarValues, span: I::Span, ) where D: SolverDelegate, I: Interner, +{ + assert_eq!(original_values.len(), var_values.len()); + + for (&orig, response) in iter::zip(original_values, var_values.iter()) { + let response = instantiate_generic_arg(delegate, response, instantiation); + if orig == response { + continue; + } + + let mut must_eq = ResponseRelating::new(&**delegate, span); + must_eq.relate(orig, response).unwrap(); + } +} + +fn instantiate_generic_arg( + delegate: &D, + arg: Canonicalized, + instantiation: CanonicalVarValues, +) -> I::GenericArg +where + D: SolverDelegate, + I: Interner, { // Canonical bound vars are very common here. `CanonicalInstantiator::fold_{ty,region,const}`'s // handling of them is simple, and we replicate that handling here to avoid the cost of the // surrounding machinery. - fn canonical_bound_var(arg: I::GenericArg) -> Option { - match arg.kind() { + fn canonical_bound_var(arg: Canonicalized) -> Option { + match arg.into_uninstantiated().kind() { ty::GenericArgKind::Type(t) => match t.kind() { ty::Bound(ty::BoundVarIndexKind::Canonical, bt) => Some(bt.var()), _ => None, @@ -528,19 +561,9 @@ fn unify_query_var_values( } } - assert_eq!(original_values.len(), var_values.len()); - - for (&orig, response) in iter::zip(original_values, var_values.iter()) { - let response = match canonical_bound_var::(response) { - Some(var) => instantiation[var], - None => delegate.instantiate_value(response, instantiation), - }; - if orig == response { - continue; - } - - let mut must_eq = ResponseRelating::new(&**delegate, span); - must_eq.relate(orig, response).unwrap(); + match canonical_bound_var::(arg) { + Some(var) => instantiation[var], + None => delegate.instantiate_value(arg, instantiation), } } @@ -622,7 +645,7 @@ pub fn instantiate_canonical_state( where D: SolverDelegate, I: Interner, - T: TypeFoldable, + T: TypeFoldable + Copy, { // In case any fresh inference variables have been created between `state` // and the previous instantiation, extend `orig_values` for it. @@ -644,10 +667,17 @@ where prev_universe, ); - let inspect::State { var_values, data } = state.value; - let data = delegate.instantiate_value(data, instantiation); + // We instantiate the fields within `state` separately. See the corresponding code in + // `instantiate_and_apply_query_response` for an explanation why. + let data = delegate.instantiate_value(state.project(|r| r.data), instantiation); + instantiate_and_unify_query_var_values( + delegate, + orig_values, + state.project(|r| r.var_values), + instantiation, + span, + ); - unify_query_var_values(delegate, orig_values, var_values, instantiation, span); data } diff --git a/compiler/rustc_next_trait_solver/src/delegate.rs b/compiler/rustc_next_trait_solver/src/delegate.rs index 437d4990c809f..7c4470efa082b 100644 --- a/compiler/rustc_next_trait_solver/src/delegate.rs +++ b/compiler/rustc_next_trait_solver/src/delegate.rs @@ -79,7 +79,11 @@ pub trait SolverDelegate: Deref + Sized { &self, ) -> Vec<(ty::RegionConstraint, VisibleForLeakCheck)>; - fn instantiate_value(&self, value: V, values: ty::CanonicalVarValues) -> V + fn instantiate_value( + &self, + value: ty::Canonicalized, + values: ty::CanonicalVarValues, + ) -> V where V: TypeFoldable; diff --git a/compiler/rustc_trait_selection/src/solve/delegate.rs b/compiler/rustc_trait_selection/src/solve/delegate.rs index 4f860756d321f..70b7ab1e71afb 100644 --- a/compiler/rustc_trait_selection/src/solve/delegate.rs +++ b/compiler/rustc_trait_selection/src/solve/delegate.rs @@ -11,7 +11,7 @@ use rustc_hir::def_id::{CRATE_DEF_ID, DefId, LOCAL_CRATE}; use rustc_infer::infer::canonical::query_response::make_query_region_constraints; use rustc_infer::infer::canonical::{ Canonical, CanonicalExt as _, CanonicalQueryInput, CanonicalVarKind, CanonicalVarValues, - QueryRegionConstraint, instantiate_value, + Canonicalized, QueryRegionConstraint, instantiate_value, }; use rustc_infer::infer::{InferCtxt, RegionVariableOrigin, SubregionOrigin, TyCtxtInferExt}; use rustc_infer::traits::solve::{ @@ -406,11 +406,11 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate< constraints } - fn instantiate_value(&self, value: V, values: CanonicalVarValues<'tcx>) -> V + fn instantiate_value(&self, value: Canonicalized, values: CanonicalVarValues<'tcx>) -> V where V: TypeFoldable>, { - instantiate_value(self.tcx, &values, value) + instantiate_value(self.tcx, &values, value.into_uninstantiated()) } fn instantiate_canonical( diff --git a/compiler/rustc_type_ir/src/canonical.rs b/compiler/rustc_type_ir/src/canonical.rs index ef6861d83f627..6cdfe02ae97c0 100644 --- a/compiler/rustc_type_ir/src/canonical.rs +++ b/compiler/rustc_type_ir/src/canonical.rs @@ -72,6 +72,10 @@ impl Canonical { let Canonical { max_universe, var_kinds, value } = self; Canonical { max_universe, var_kinds, value: map_op(value) } } + + pub fn project(self, f: impl FnOnce(V) -> W) -> Canonicalized { + Canonicalized(f(self.value)) + } } impl fmt::Display for Canonical { @@ -84,6 +88,31 @@ impl fmt::Display for Canonical { } } +/// A projection of a `Canonical`, as produced by `Canonical::project`. Useful as a marker when +/// dealing with fields within a `Canonical`, because it indicates that `T` may contain +/// canonical bound variables that must be instantiated before use. +/// +/// The field is private and the contents can only be accessed directly with `into_uninstantiated`, +/// which is easy to spot. +#[derive(Copy, Clone, Debug)] +pub struct Canonicalized(T); + +impl Canonicalized { + pub fn into_uninstantiated(self) -> T { + self.0 + } +} + +impl Canonicalized { + pub fn len(self) -> usize { + self.0.len() + } + + pub fn iter(self) -> impl Iterator> { + self.0.iter().map(|x| Canonicalized(x)) + } +} + /// Information about a canonical variable that is included with the /// canonical value. This is sufficient information for code to create /// a copy of the canonical value in some other inference context,