Skip to content

Make more float intrinsics generic #162395

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
N1ark wants to merge 8 commits into
base: main
Choose a base branch
from
Open

Make more float intrinsics generic #162395

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
18d2205
cranelift: use fallbacks for abs f16, f128
N1ark Sep 6, 2026
da81486
Make rounding intrinsics generic
N1ark Sep 6, 2026
c9d0c7a
Make sqrt generic
N1ark Sep 6, 2026
605acfb
Make powf generic
N1ark Sep 6, 2026
66614fa
Make `fma` and `fmuladd` generic
N1ark Sep 6, 2026
5f1bc1c
Make `powi` generic
N1ark Sep 7, 2026
33503c2
Add test for `rustc_do_not_const_check` intrinsics
N1ark Sep 7, 2026
8d81bf9
Add a fallback to `powi`
N1ark Sep 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 0 additions & 14 deletions compiler/rustc_codegen_cranelift/src/codegen_f16_f128.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,20 +160,6 @@ pub(crate) fn neg_f128(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value {
fx.bcx.ins().bitcast(types::F128, MemFlagsData::new(), bits)
}

pub(crate) fn abs_f16(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value {
let bits = fx.bcx.ins().bitcast(types::I16, MemFlagsData::new(), value);
let bits = fx.bcx.ins().band_imm_u(bits, 0x7fff);
fx.bcx.ins().bitcast(types::F16, MemFlagsData::new(), bits)
}

pub(crate) fn abs_f128(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value {
let bits = fx.bcx.ins().bitcast(types::I128, MemFlagsData::new(), value);
let (low, high) = fx.bcx.ins().isplit(bits);
let high = fx.bcx.ins().band_imm_u(high, 0x7fff_ffff_ffff_ffff_u64 as i64);
let bits = fx.bcx.ins().iconcat(low, high);
fx.bcx.ins().bitcast(types::F128, MemFlagsData::new(), bits)
}

pub(crate) fn codegen_cast(
fx: &mut FunctionCx<'_, '_, '_>,
from: Value,
Expand Down
4 changes: 4 additions & 0 deletions compiler/rustc_codegen_cranelift/src/compiler_builtins.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,10 @@ builtin_functions! {
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
fn rintf128(f: f128) -> f128;
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
fn roundf16(f: f16) -> f16;
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
fn roundf128(f: f128) -> f128;
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
fn sqrtf16(f: f16) -> f16;
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
fn sqrtf128(f: f128) -> f128;
Expand Down
211 changes: 120 additions & 91 deletions compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -322,63 +322,11 @@ fn codegen_float_intrinsic_call<'tcx>(
ret: CPlace<'tcx>,
) -> bool {
let (name, arg_count, ty, clif_ty) = match intrinsic {
sym::sqrtf16 => return false, // has a fallback via f32
sym::sqrtf32 => ("sqrtf", 1, fx.tcx.types.f32, types::F32),
sym::sqrtf64 => ("sqrt", 1, fx.tcx.types.f64, types::F64),
sym::sqrtf128 => ("sqrtf128", 1, fx.tcx.types.f128, types::F128),

sym::powif16 => ("__powisf2", 2, fx.tcx.types.f16, types::F16), // compiler-builtins
sym::powif32 => ("__powisf2", 2, fx.tcx.types.f32, types::F32), // compiler-builtins
sym::powif64 => ("__powidf2", 2, fx.tcx.types.f64, types::F64), // compiler-builtins
sym::powif128 => ("__powitf2", 2, fx.tcx.types.f128, types::F128), // compiler-builtins

sym::powf16 => return false, // has a fallback via f32
sym::powf32 => ("powf", 2, fx.tcx.types.f32, types::F32),
sym::powf64 => ("pow", 2, fx.tcx.types.f64, types::F64),
sym::powf128 => ("powf128", 2, fx.tcx.types.f128, types::F128),

sym::fmaf16 => return false, // has a fallback via f64
sym::fmaf32 => ("fmaf", 3, fx.tcx.types.f32, types::F32),
sym::fmaf64 => ("fma", 3, fx.tcx.types.f64, types::F64),
sym::fmaf128 => ("fmaf128", 3, fx.tcx.types.f128, types::F128),

// FIXME: calling `fma` from libc without FMA target feature uses expensive sofware
// emulation, use cranelift intrinsic analogous to llvm.fmuladd.*.
sym::fmuladdf16 => return false, // has a fallback
sym::fmuladdf32 => ("fmaf", 3, fx.tcx.types.f32, types::F32),
sym::fmuladdf64 => ("fma", 3, fx.tcx.types.f64, types::F64),
sym::fmuladdf128 => return false, // has a fallback

sym::copysignf16 => return false, // has a fallback
sym::copysignf32 => ("copysignf", 2, fx.tcx.types.f32, types::F32),
sym::copysignf64 => ("copysign", 2, fx.tcx.types.f64, types::F64),
sym::copysignf128 => return false, // has a fallback

sym::floorf16 => return false, // has a fallback via f32
sym::floorf32 => ("floorf", 1, fx.tcx.types.f32, types::F32),
sym::floorf64 => ("floor", 1, fx.tcx.types.f64, types::F64),
sym::floorf128 => ("floorf128", 1, fx.tcx.types.f128, types::F128),

sym::ceilf16 => return false, // has a fallback via f32
sym::ceilf32 => ("ceilf", 1, fx.tcx.types.f32, types::F32),
sym::ceilf64 => ("ceil", 1, fx.tcx.types.f64, types::F64),
sym::ceilf128 => ("ceilf128", 1, fx.tcx.types.f128, types::F128),

sym::truncf16 => return false, // has a fallback via f32
sym::truncf32 => ("truncf", 1, fx.tcx.types.f32, types::F32),
sym::truncf64 => ("trunc", 1, fx.tcx.types.f64, types::F64),
sym::truncf128 => ("truncf128", 1, fx.tcx.types.f128, types::F128),

sym::round_ties_even_f16 => return false, // has a fallback via f32
sym::round_ties_even_f32 => ("rintf", 1, fx.tcx.types.f32, types::F32),
sym::round_ties_even_f64 => ("rint", 1, fx.tcx.types.f64, types::F64),
sym::round_ties_even_f128 => ("rintf128", 1, fx.tcx.types.f128, types::F128),

sym::roundf16 => return false, // has a fallback via f32
sym::roundf32 => ("roundf", 1, fx.tcx.types.f32, types::F32),
sym::roundf64 => ("round", 1, fx.tcx.types.f64, types::F64),
sym::roundf128 => ("roundf128", 1, fx.tcx.types.f128, types::F128),

_ => return false,
};

Expand Down Expand Up @@ -414,35 +362,8 @@ fn codegen_float_intrinsic_call<'tcx>(
// FIXME(bytecodealliance/wasmtime#8312): Use native Cranelift operations
// for `f16` and `f128` once the lowerings have been implemented in Cranelift.
let val = match intrinsic {
sym::fmaf32 | sym::fmaf64 | sym::fmuladdf32 | sym::fmuladdf64 => {
fx.bcx.ins().fma(args[0], args[1], args[2])
}
sym::copysignf32 | sym::copysignf64 => fx.bcx.ins().fcopysign(args[0], args[1]),
sym::floorf32 | sym::floorf64 => fx.bcx.ins().floor(args[0]),
sym::ceilf32 | sym::ceilf64 => fx.bcx.ins().ceil(args[0]),
sym::truncf32 | sym::truncf64 => fx.bcx.ins().trunc(args[0]),
sym::round_ties_even_f32 | sym::round_ties_even_f64 => fx.bcx.ins().nearest(args[0]),
sym::sqrtf32 | sym::sqrtf64 => fx.bcx.ins().sqrt(args[0]),

// These intrinsics aren't supported natively by Cranelift.
// Lower them to a libcall.
sym::powif16 | sym::powif32 | sym::powif64 | sym::powif128 => {
let temp;
let (clif_ty, args) = if intrinsic == sym::powif16 {
temp = [codegen_f16_f128::f16_to_f32(fx, args[0]), args[1]];
(types::F32, temp.as_slice())
} else {
(clif_ty, args)
};
let input_tys: Vec<_> =
vec![AbiParam::new(clif_ty), lib_call_arg_param(fx.tcx, types::I32, true)];
let ret_val = fx.lib_call(name, input_tys, vec![AbiParam::new(clif_ty)], args)[0];
if intrinsic == sym::powif16 {
codegen_f16_f128::f32_to_f16(fx, ret_val)
} else {
ret_val
}
}

_ => {
let input_tys: Vec<_> = args.iter().map(|_| AbiParam::new(clif_ty)).collect();
fx.lib_call(name, input_tys, vec![AbiParam::new(clif_ty)], args)[0]
Expand All @@ -455,6 +376,13 @@ fn codegen_float_intrinsic_call<'tcx>(
true
}

/// Used to distinguish fallbacks of float intrinsics. For some we have a codegen fallback,
/// while for others we fallback to an external libcall.
enum IntrinsicFallback {
Fallback(&'static str),
Codegen(Value),
}

fn codegen_regular_intrinsic_call<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
instance: Instance<'tcx>,
Expand Down Expand Up @@ -1163,7 +1091,94 @@ fn codegen_regular_intrinsic_call<'tcx>(
ret.write_cvalue(fx, old);
}

sym::fma | sym::fmuladd => {
intrinsic_args!(fx, args => (arg1, arg2, arg3); intrinsic);
let layout = arg1.layout();
let ty::Float(float_ty) = layout.ty.kind() else {
span_bug!(
source_info.span,
"expected float type for {:?} intrinsic: {:?}",
intrinsic,
layout.ty
);
};
use FloatTy::*;
use IntrinsicFallback::*;
let x = arg1.load_scalar(fx);
let y = arg2.load_scalar(fx);
let z = arg3.load_scalar(fx);
let res = match (intrinsic, float_ty) {
// FIXME: calling `fma` from libc without FMA target feature uses expensive
// sofware emulation, use cranelift intrinsic analogous to llvm.fmuladd.*.
(sym::fma | sym::fmuladd, F32 | F64) => Codegen(fx.bcx.ins().fma(x, y, z)),

(sym::fma, F128) => Fallback("fmaf128"),

(sym::fmuladd, F128) | (_, F16) => {
// We use the intrinsic fallback bodies for the rest
return Err(Instance::new_raw(instance.def_id(), instance.args));
}

_ => unreachable!(),
};
let val = match res {
Codegen(val) => val,
Fallback(name) => {
let ty = fx.clif_type(layout.ty).unwrap();
let arg = AbiParam::new(ty);
fx.lib_call(name, vec![arg, arg, arg], vec![arg], &[x, y, z])[0]
}
};
let val = CValue::by_val(val, layout);
ret.write_cvalue(fx, val);
}

sym::powf => {
intrinsic_args!(fx, args => (arg1, arg2); intrinsic);
let layout = arg1.layout();
let ty::Float(float_ty) = layout.ty.kind() else {
span_bug!(
source_info.span,
"expected float type for {:?} intrinsic: {:?}",
intrinsic,
layout.ty
);
};
use FloatTy::*;
use IntrinsicFallback::*;
let x = arg1.load_scalar(fx);
let y = arg2.load_scalar(fx);
let res = match (intrinsic, float_ty) {
(sym::powf, F32) => Fallback("powf"),
(sym::powf, F64) => Fallback("pow"),
(sym::powf, F128) => Fallback("powf128"),

(_, F16) => {
// We use the intrinsic fallback bodies for the rest
return Err(Instance::new_raw(instance.def_id(), instance.args));
}

_ => unreachable!(),
};
let val = match res {
Codegen(val) => val,
Fallback(name) => {
let ty = fx.clif_type(layout.ty).unwrap();
let arg = AbiParam::new(ty);
fx.lib_call(name, vec![arg, arg], vec![arg], &[x, y])[0]
}
};

@bjorn3 bjorn3 Sep 9, 2026 •
edited by rustbot
Loading

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The codegen variant is never used here.

View changes since the review

let val = CValue::by_val(val, layout);
ret.write_cvalue(fx, val);
}

sym::fabs
| sym::sqrt
| sym::floor
| sym::ceil
| sym::trunc
| sym::round
| sym::round_ties_even
| sym::exp
| sym::exp2
| sym::log
Expand All @@ -1176,23 +1191,35 @@ fn codegen_regular_intrinsic_call<'tcx>(
let ty::Float(float_ty) = layout.ty.kind() else {
span_bug!(
source_info.span,
"expected float type for fabs intrinsic: {:?}",
"expected float type for {:?} intrinsic: {:?}",
intrinsic,
layout.ty
);
};
enum IntrinsicFallback {
Fallback(&'static str),
Codegen(Value),
}
use FloatTy::*;
use IntrinsicFallback::*;
let x = arg.load_scalar(fx);
let res = match (intrinsic, float_ty) {
(sym::fabs, F32 | F64) => Codegen(fx.bcx.ins().fabs(x)),
// FIXME(bytecodealliance/wasmtime#8312): Use `fabsf16` once Cranelift
// backend lowerings are implemented.
(sym::fabs, F16) => Codegen(codegen_f16_f128::abs_f16(fx, x)),
(sym::fabs, F128) => Codegen(codegen_f16_f128::abs_f128(fx, x)),

(sym::sqrt, F32 | F64) => Codegen(fx.bcx.ins().sqrt(x)),
(sym::sqrt, F128) => Fallback("sqrtf128"),

(sym::floor, F32 | F64) => Codegen(fx.bcx.ins().floor(x)),
(sym::floor, F128) => Fallback("floorf128"),

(sym::ceil, F32 | F64) => Codegen(fx.bcx.ins().ceil(x)),
(sym::ceil, F128) => Fallback("ceilf128"),

(sym::trunc, F32 | F64) => Codegen(fx.bcx.ins().trunc(x)),
(sym::trunc, F128) => Fallback("truncf128"),

(sym::round_ties_even, F32 | F64) => Codegen(fx.bcx.ins().nearest(x)),
(sym::round_ties_even, F128) => Fallback("rintf128"),

(sym::round, F32) => Fallback("roundf"),
(sym::round, F64) => Fallback("round"),
(sym::round, F128) => Fallback("roundf128"),

(sym::exp, F32) => Fallback("expf"),
(sym::exp, F64) => Fallback("exp"),
Expand Down Expand Up @@ -1222,8 +1249,10 @@ fn codegen_regular_intrinsic_call<'tcx>(
(sym::cos, F64) => Fallback("cos"),
(sym::cos, F128) => Fallback("cosf128"),

(_, F16) => {
// We implement fallbacks for other f16 intrinsics via f32
(sym::fabs, F128) | (_, F16) => {
// FIXME(bytecodealliance/wasmtime#8312): Use the native operations once
// Cranelift backend lowerings for `f16` are implemented.
// We use the intrinsic fallback bodies for the rest
return Err(Instance::new_raw(instance.def_id(), instance.args));
}

Expand Down
Loading
Loading