Skip to content

Commit ea75827

Browse files
folkertdevfolkertdev
committed
Redesign VaList 
1 parent 1c4dd04 commit ea75827

File tree

15 files changed

+353
-209
lines changed

15 files changed

+353
-209
lines changed

‎compiler/rustc_codegen_llvm/src/va_arg.rs

Lines changed: 40 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
1-
use rustc_abi::{Align, BackendRepr, Endian, ExternAbi, HasDataLayout, Size};
1+
use rustc_abi::{Align, BackendRepr, Endian, HasDataLayout, Size};
22
use rustc_codegen_ssa::common::IntPredicate;
33
use rustc_codegen_ssa::mir::operand::OperandRef;
44
use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods, ConstCodegenMethods};
55
use rustc_middle::ty::Ty;
66
use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf};
7+
use rustc_target::spec::VaListFlavor;
78

89
use crate::builder::Builder;
910
use crate::type_::Type;
@@ -557,37 +558,43 @@ pub(super) fn emit_va_arg<'ll, 'tcx>(
557558
// Determine the va_arg implementation to use. The LLVM va_arg instruction
558559
// is lacking in some instances, so we should only use it as a fallback.
559560
let target = &bx.cx.tcx.sess.target;
560-
let arch = &bx.cx.tcx.sess.target.arch;
561-
match &**arch {
562-
// Windows x86
563-
"x86" if target.is_like_windows => {
564-
emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(4).unwrap(), false)
565-
}
566-
// Generic x86
567-
"x86" => emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(4).unwrap(), true),
568-
// Windows AArch64
569-
"aarch64" | "arm64ec" if target.is_like_windows => {
570-
emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(8).unwrap(), false)
571-
}
572-
// macOS / iOS AArch64
573-
"aarch64" if target.is_like_darwin => {
574-
emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(8).unwrap(), true)
575-
}
576-
"aarch64" => emit_aapcs_va_arg(bx, addr, target_ty),
577-
"s390x" => emit_s390x_va_arg(bx, addr, target_ty),
578-
// Windows x86_64
579-
"x86_64" if target.is_like_windows => {
580-
let target_ty_size = bx.cx.size_of(target_ty).bytes();
581-
let indirect: bool = target_ty_size > 8 || !target_ty_size.is_power_of_two();
582-
emit_ptr_va_arg(bx, addr, target_ty, indirect, Align::from_bytes(8).unwrap(), false)
583-
}
584-
"x86_64" if target.is_abi_supported(ExternAbi::SysV64 { unwind: false }) => {
585-
emit_x86_64_sysv64_va_arg(bx, addr, target_ty)
586-
}
587-
"xtensa" => emit_xtensa_va_arg(bx, addr, target_ty),
588-
// For all other architecture/OS combinations fall back to using
589-
// the LLVM va_arg instruction.
590-
// https://llvm.org/docs/LangRef.html#va-arg-instruction
591-
_ => bx.va_arg(addr.immediate(), bx.cx.layout_of(target_ty).llvm_type(bx.cx)),
561+
let arch = &*target.arch;
562+
563+
let align_8 = Align::from_bytes(8).unwrap();
564+
let align_4 = Align::from_bytes(4).unwrap();
565+
566+
// For all other architecture/OS combinations fall back to using the LLVM va_arg instruction.
567+
// https://llvm.org/docs/LangRef.html#va-arg-instruction
568+
let mut fallback = || bx.va_arg(addr.immediate(), bx.cx.layout_of(target_ty).llvm_type(bx.cx));
569+
570+
match VaListFlavor::from_target(target) {
571+
VaListFlavor::Darwin => match arch {
572+
// macOS / iOS AArch64
573+
"aarch64" => emit_ptr_va_arg(bx, addr, target_ty, false, align_8, true),
574+
_ => fallback(),
575+
},
576+
577+
VaListFlavor::Windows => match arch {
578+
// Windows x86
579+
"x86" => emit_ptr_va_arg(bx, addr, target_ty, false, align_4, false),
580+
// Windows AArch64
581+
"aarch64" | "arm64ec" => emit_ptr_va_arg(bx, addr, target_ty, false, align_8, false),
582+
// Windows x86_64
583+
"x86_64" => {
584+
let target_ty_size = bx.cx.size_of(target_ty).bytes();
585+
let indirect: bool = target_ty_size > 8 || !target_ty_size.is_power_of_two();
586+
emit_ptr_va_arg(bx, addr, target_ty, indirect, align_8, false)
587+
}
588+
_ => fallback(),
589+
},
590+
VaListFlavor::GenericAarch64 => match arch {
591+
"aarch64" => emit_aapcs_va_arg(bx, addr, target_ty),
592+
_ => fallback(),
593+
},
594+
VaListFlavor::SystemV => emit_x86_64_sysv64_va_arg(bx, addr, target_ty),
595+
VaListFlavor::GenericX86 => emit_ptr_va_arg(bx, addr, target_ty, false, align_4, true),
596+
VaListFlavor::S390x => emit_s390x_va_arg(bx, addr, target_ty),
597+
VaListFlavor::Extensa => emit_xtensa_va_arg(bx, addr, target_ty),
598+
VaListFlavor::PowerPc | VaListFlavor::Uefi | VaListFlavor::Unknown => fallback(),
592599
}
593600
}

‎compiler/rustc_codegen_ssa/src/mir/block.rs

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -503,7 +503,22 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
503503
let va_list_arg_idx = self.fn_abi.args.len();
504504
match self.locals[mir::Local::from_usize(1 + va_list_arg_idx)] {
505505
LocalRef::Place(va_list) => {
506-
bx.va_end(va_list.val.llval);
506+
if bx.cx().tcx().sess.target.va_list_flavor().va_list_on_stack() {
507+
// call `va_end` on the `&mut VaListTag` that is stored in `va_list`.
508+
let inner_field = va_list.project_field(bx, 0);
509+
510+
let tag_ptr = bx.load(
511+
bx.backend_type(inner_field.layout),
512+
inner_field.val.llval,
513+
inner_field.layout.align.abi,
514+
);
515+
516+
bx.va_end(tag_ptr);
517+
} else {
518+
// call `va_end` on the `VaListTag` that is stored in `va_list`.
519+
let tag_ptr = va_list.project_field(bx, 0);
520+
bx.va_end(tag_ptr.val.llval);
521+
}
507522
}
508523
_ => bug!("C-variadic function must have a `VaList` place"),
509524
}

‎compiler/rustc_codegen_ssa/src/mir/mod.rs

Lines changed: 35 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -436,10 +436,42 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
436436
}
437437

438438
if fx.fn_abi.c_variadic && arg_index == fx.fn_abi.args.len() {
439-
let va_list = PlaceRef::alloca(bx, bx.layout_of(arg_ty));
440-
bx.va_start(va_list.val.llval);
439+
use rustc_hir::LangItem;
441440

442-
return LocalRef::Place(va_list);
441+
let va_list_tag_ty = {
442+
let did = bx.tcx().require_lang_item(LangItem::VaListTag, None);
443+
let ty = bx.tcx().type_of(did).instantiate_identity();
444+
bx.tcx().normalize_erasing_regions(fx.cx.typing_env(), ty)
445+
};
446+
let va_list_tag_layout = bx.layout_of(va_list_tag_ty);
447+
448+
// Construct the `VaListTag` on the stack.
449+
let va_list_tag = PlaceRef::alloca(bx, va_list_tag_layout);
450+
451+
// Initialize the alloca.
452+
bx.va_start(va_list_tag.val.llval);
453+
454+
if fx.cx.tcx().sess.target.va_list_flavor().va_list_on_stack() {
455+
// Roughly `struct VaList<'a>(&'a mut VaListTag<'a>);`
456+
let va_list_layout = bx.layout_of(arg_ty);
457+
458+
let tmp = PlaceRef::alloca(bx, va_list_layout);
459+
bx.store_to_place(va_list_tag.val.llval, tmp.val);
460+
return LocalRef::Place(tmp);
461+
} else {
462+
// Roughly `struct VaList<'a>(VaListTag<'a>);`
463+
let va_list_layout = bx.layout_of(arg_ty);
464+
465+
let va_list_tag = bx.load(
466+
bx.backend_type(va_list_tag_layout),
467+
va_list_tag.val.llval,
468+
va_list_tag.layout.align.abi,
469+
);
470+
471+
let tmp = PlaceRef::alloca(bx, va_list_layout);
472+
bx.store_to_place(va_list_tag, tmp.val);
473+
return LocalRef::Place(tmp);
474+
}
443475
}
444476

445477
let arg = &fx.fn_abi.args[idx];

‎compiler/rustc_hir/src/lang_items.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -228,6 +228,7 @@ language_item_table! {
228228
UnsafePinned, sym::unsafe_pinned, unsafe_pinned_type, Target::Struct, GenericRequirement::None;
229229

230230
VaList, sym::va_list, va_list, Target::Struct, GenericRequirement::None;
231+
VaListTag, sym::va_list_tag, va_list_tag, Target::Struct, GenericRequirement::None;
231232

232233
Deref, sym::deref, deref_trait, Target::Trait, GenericRequirement::Exact(0);
233234
DerefMut, sym::deref_mut, deref_mut_trait, Target::Trait, GenericRequirement::Exact(0);

‎compiler/rustc_hir_analysis/src/check/intrinsic.rs

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -179,8 +179,8 @@ pub(crate) fn check_intrinsic_type(
179179
ty::BoundVariableKind::Region(ty::BoundRegionKind::Anon),
180180
ty::BoundVariableKind::Region(ty::BoundRegionKind::ClosureEnv),
181181
]);
182-
let mk_va_list_ty = |mutbl| {
183-
let did = tcx.require_lang_item(LangItem::VaList, Some(span));
182+
let mk_va_list_tag_ty = |mutbl| {
183+
let did = tcx.require_lang_item(LangItem::VaListTag, Some(span));
184184
let region = ty::Region::new_bound(
185185
tcx,
186186
ty::INNERMOST,
@@ -565,16 +565,16 @@ pub(crate) fn check_intrinsic_type(
565565
}
566566

567567
sym::va_start | sym::va_end => {
568-
(0, 0, vec![mk_va_list_ty(hir::Mutability::Mut).0], tcx.types.unit)
568+
(0, 0, vec![mk_va_list_tag_ty(hir::Mutability::Mut).0], tcx.types.unit)
569569
}
570570

571571
sym::va_copy => {
572-
let (va_list_ref_ty, va_list_ty) = mk_va_list_ty(hir::Mutability::Not);
572+
let (va_list_ref_ty, va_list_ty) = mk_va_list_tag_ty(hir::Mutability::Not);
573573
let va_list_ptr_ty = Ty::new_mut_ptr(tcx, va_list_ty);
574574
(0, 0, vec![va_list_ptr_ty, va_list_ref_ty], tcx.types.unit)
575575
}
576576

577-
sym::va_arg => (1, 0, vec![mk_va_list_ty(hir::Mutability::Mut).0], param(0)),
577+
sym::va_arg => (1, 0, vec![mk_va_list_tag_ty(hir::Mutability::Mut).0], param(0)),
578578

579579
sym::nontemporal_store => {
580580
(1, 0, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], tcx.types.unit)

‎compiler/rustc_span/src/symbol.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2268,6 +2268,7 @@ symbols! {
22682268
va_copy,
22692269
va_end,
22702270
va_list,
2271+
va_list_tag,
22712272
va_start,
22722273
val,
22732274
validity,

‎compiler/rustc_target/src/spec/mod.rs

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2666,6 +2666,54 @@ pub struct TargetOptions {
26662666
small_data_threshold_support: SmallDataThresholdSupport,
26672667
}
26682668

2669+
/// How C variadics work for a target.
2670+
pub enum VaListFlavor {
2671+
Darwin,
2672+
Windows,
2673+
GenericAarch64,
2674+
SystemV,
2675+
GenericX86,
2676+
S390x,
2677+
PowerPc,
2678+
Extensa,
2679+
Uefi,
2680+
Unknown,
2681+
}
2682+
2683+
impl VaListFlavor {
2684+
pub fn from_target(target: &Target) -> Self {
2685+
let arch = &target.arch;
2686+
2687+
match arch.as_ref() {
2688+
_ if target.is_like_windows => Self::Windows,
2689+
_ if target.is_like_darwin => Self::Darwin,
2690+
_ if target.os == "uefi" => Self::Uefi,
2691+
_ if target.is_abi_supported(ExternAbi::SysV64 { unwind: false }) => Self::SystemV,
2692+
"aarch64" => Self::GenericAarch64,
2693+
"x86" => Self::GenericX86,
2694+
"powerpc" => Self::PowerPc,
2695+
"s390x" => Self::S390x,
2696+
"xtensa" => Self::Extensa,
2697+
_ => Self::Unknown,
2698+
}
2699+
}
2700+
2701+
pub fn va_list_on_stack(&self) -> bool {
2702+
match self {
2703+
// These use a pointer to the function's stack to represent the `va_list`.
2704+
Self::GenericAarch64 | Self::SystemV | Self::S390x | Self::PowerPc | Self::Extensa => {
2705+
true
2706+
}
2707+
2708+
// These use just an opaque pointer to represent the `va_list`.
2709+
Self::Darwin | Self::Windows | Self::GenericX86 | Self::Uefi => false,
2710+
2711+
// By default we assume the opaque pointer approach.
2712+
Self::Unknown => true,
2713+
}
2714+
}
2715+
}
2716+
26692717
/// Add arguments for the given flavor and also for its "twin" flavors
26702718
/// that have a compatible command line interface.
26712719
fn add_link_args_iter(
@@ -3024,6 +3072,10 @@ impl Target {
30243072
}
30253073
}
30263074

3075+
pub fn va_list_flavor(&self) -> VaListFlavor {
3076+
VaListFlavor::from_target(self)
3077+
}
3078+
30273079
/// Minimum integer size in bits that this target can perform atomic
30283080
/// operations on.
30293081
pub fn min_atomic_width(&self) -> u64 {

‎library/core/src/ffi/mod.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ pub mod c_str;
2828
issue = "44930",
2929
reason = "the `c_variadic` feature has not been properly tested on all supported platforms"
3030
)]
31-
pub use self::va_list::{VaArgSafe, VaList, VaListImpl};
31+
pub use self::va_list::{VaArgSafe, VaList, va_copy};
3232

3333
#[unstable(
3434
feature = "c_variadic",

0 commit comments

Comments
 (0)