Skip to content

Commit 197fa64

Browse files
borsbors
committed
Auto merge of #136035 - SpecificProtagonist:miri-zeroed-alloc, r=oli-obk
miri: optimize zeroed alloc When allocating zero-initialized memory in MIR interpretation, rustc allocates zeroed memory, marks it as initialized and then re-zeroes it. Remove the last step. I don't expect this to have much of an effect on performance normally, but in my case in which I'm creating a large allocation via mmap it gets in the way.
2 parents a1ef041 + d6b5471 commit 197fa64

File tree

6 files changed

+29
-49
lines changed

6 files changed

+29
-49
lines changed

‎src/shims/alloc.rs‎

Lines changed: 6 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
use std::iter;
2-
31
use rustc_abi::{Align, Size};
42
use rustc_ast::expand::allocator::AllocatorKind;
53

@@ -80,18 +78,10 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
8078
}
8179
}
8280

83-
fn malloc(&mut self, size: u64, zero_init: bool) -> InterpResult<'tcx, Pointer> {
81+
fn malloc(&mut self, size: u64, init: AllocInit) -> InterpResult<'tcx, Pointer> {
8482
let this = self.eval_context_mut();
8583
let align = this.malloc_align(size);
86-
let ptr = this.allocate_ptr(Size::from_bytes(size), align, MiriMemoryKind::C.into())?;
87-
if zero_init {
88-
// We just allocated this, the access is definitely in-bounds and fits into our address space.
89-
this.write_bytes_ptr(
90-
ptr.into(),
91-
iter::repeat(0u8).take(usize::try_from(size).unwrap()),
92-
)
93-
.unwrap();
94-
}
84+
let ptr = this.allocate_ptr(Size::from_bytes(size), align, MiriMemoryKind::C.into(), init)?;
9585
interp_ok(ptr.into())
9686
}
9787

@@ -115,6 +105,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
115105
Size::from_bytes(size),
116106
Align::from_bytes(align).unwrap(),
117107
MiriMemoryKind::C.into(),
108+
AllocInit::Uninit
118109
)?;
119110
this.write_pointer(ptr, &memptr)?;
120111
interp_ok(Scalar::from_i32(0))
@@ -134,7 +125,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
134125
let new_align = this.malloc_align(new_size);
135126
if this.ptr_is_null(old_ptr)? {
136127
// Here we must behave like `malloc`.
137-
self.malloc(new_size, /*zero_init*/ false)
128+
self.malloc(new_size, AllocInit::Uninit)
138129
} else {
139130
if new_size == 0 {
140131
// C, in their infinite wisdom, made this UB.
@@ -147,6 +138,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
147138
Size::from_bytes(new_size),
148139
new_align,
149140
MiriMemoryKind::C.into(),
141+
AllocInit::Uninit
150142
)?;
151143
interp_ok(new_ptr.into())
152144
}
@@ -187,6 +179,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
187179
Size::from_bytes(size),
188180
Align::from_bytes(align).unwrap(),
189181
MiriMemoryKind::C.into(),
182+
AllocInit::Uninit
190183
)?;
191184
interp_ok(ptr.into())
192185
}

‎src/shims/foreign_items.rs‎

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
use std::collections::hash_map::Entry;
22
use std::io::Write;
3-
use std::iter;
43
use std::path::Path;
54

65
use rustc_abi::{Align, AlignFromBytesError, Size};
@@ -9,6 +8,7 @@ use rustc_ast::expand::allocator::alloc_error_handler_name;
98
use rustc_hir::def::DefKind;
109
use rustc_hir::def_id::CrateNum;
1110
use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
11+
use rustc_middle::mir::interpret::AllocInit;
1212
use rustc_middle::ty::Ty;
1313
use rustc_middle::{mir, ty};
1414
use rustc_span::Symbol;
@@ -442,7 +442,7 @@ trait EvalContextExtPriv<'tcx>: crate::MiriInterpCxExt<'tcx> {
442442
let [size] = this.check_shim(abi, Conv::C, link_name, args)?;
443443
let size = this.read_target_usize(size)?;
444444
if size <= this.max_size_of_val().bytes() {
445-
let res = this.malloc(size, /*zero_init:*/ false)?;
445+
let res = this.malloc(size, AllocInit::Uninit)?;
446446
this.write_pointer(res, dest)?;
447447
} else {
448448
// If this does not fit in an isize, return null and, on Unix, set errno.
@@ -457,7 +457,7 @@ trait EvalContextExtPriv<'tcx>: crate::MiriInterpCxExt<'tcx> {
457457
let items = this.read_target_usize(items)?;
458458
let elem_size = this.read_target_usize(elem_size)?;
459459
if let Some(size) = this.compute_size_in_bytes(Size::from_bytes(elem_size), items) {
460-
let res = this.malloc(size.bytes(), /*zero_init:*/ true)?;
460+
let res = this.malloc(size.bytes(), AllocInit::Zero)?;
461461
this.write_pointer(res, dest)?;
462462
} else {
463463
// On size overflow, return null and, on Unix, set errno.
@@ -509,6 +509,7 @@ trait EvalContextExtPriv<'tcx>: crate::MiriInterpCxExt<'tcx> {
509509
Size::from_bytes(size),
510510
Align::from_bytes(align).unwrap(),
511511
memory_kind.into(),
512+
AllocInit::Uninit
512513
)?;
513514

514515
ecx.write_pointer(ptr, dest)
@@ -537,14 +538,8 @@ trait EvalContextExtPriv<'tcx>: crate::MiriInterpCxExt<'tcx> {
537538
Size::from_bytes(size),
538539
Align::from_bytes(align).unwrap(),
539540
MiriMemoryKind::Rust.into(),
541+
AllocInit::Zero
540542
)?;
541-
542-
// We just allocated this, the access is definitely in-bounds.
543-
this.write_bytes_ptr(
544-
ptr.into(),
545-
iter::repeat(0u8).take(usize::try_from(size).unwrap()),
546-
)
547-
.unwrap();
548543
this.write_pointer(ptr, dest)
549544
});
550545
}
@@ -604,6 +599,7 @@ trait EvalContextExtPriv<'tcx>: crate::MiriInterpCxExt<'tcx> {
604599
Size::from_bytes(new_size),
605600
align,
606601
MiriMemoryKind::Rust.into(),
602+
AllocInit::Uninit
607603
)?;
608604
this.write_pointer(new_ptr, dest)
609605
});

‎src/shims/unix/fs.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1109,6 +1109,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
11091109
Size::from_bytes(size),
11101110
dirent_layout.align.abi,
11111111
MiriMemoryKind::Runtime.into(),
1112+
AllocInit::Uninit
11121113
)?;
11131114
let entry: Pointer = entry.into();
11141115

‎src/shims/unix/linux/mem.rs‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -49,16 +49,8 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
4949
Size::from_bytes(new_size),
5050
align,
5151
MiriMemoryKind::Mmap.into(),
52+
AllocInit::Zero
5253
)?;
53-
if let Some(increase) = new_size.checked_sub(old_size) {
54-
// We just allocated this, the access is definitely in-bounds and fits into our address space.
55-
// mmap guarantees new mappings are zero-init.
56-
this.write_bytes_ptr(
57-
ptr.wrapping_offset(Size::from_bytes(old_size), this).into(),
58-
std::iter::repeat(0u8).take(usize::try_from(increase).unwrap()),
59-
)
60-
.unwrap();
61-
}
6254

6355
interp_ok(Scalar::from_pointer(ptr, this))
6456
}

‎src/shims/unix/mem.rs‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -111,15 +111,13 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
111111
return interp_ok(this.eval_libc("MAP_FAILED"));
112112
}
113113

114-
let ptr =
115-
this.allocate_ptr(Size::from_bytes(map_length), align, MiriMemoryKind::Mmap.into())?;
116-
// We just allocated this, the access is definitely in-bounds and fits into our address space.
117-
// mmap guarantees new mappings are zero-init.
118-
this.write_bytes_ptr(
119-
ptr.into(),
120-
std::iter::repeat(0u8).take(usize::try_from(map_length).unwrap()),
121-
)
122-
.unwrap();
114+
let ptr = this.allocate_ptr(
115+
Size::from_bytes(map_length),
116+
align,
117+
MiriMemoryKind::Mmap.into(),
118+
// mmap guarantees new mappings are zero-init.
119+
AllocInit::Zero
120+
)?;
123121

124122
interp_ok(Scalar::from_pointer(ptr, this))
125123
}

‎src/shims/windows/foreign_items.rs‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -253,22 +253,21 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
253253
this.read_target_isize(handle)?;
254254
let flags = this.read_scalar(flags)?.to_u32()?;
255255
let size = this.read_target_usize(size)?;
256-
let heap_zero_memory = 0x00000008; // HEAP_ZERO_MEMORY
257-
let zero_init = (flags & heap_zero_memory) == heap_zero_memory;
256+
const HEAP_ZERO_MEMORY: u32 = 0x00000008;
257+
let init = if (flags & HEAP_ZERO_MEMORY) == HEAP_ZERO_MEMORY {
258+
AllocInit::Zero
259+
} else {
260+
AllocInit::Uninit
261+
};
258262
// Alignment is twice the pointer size.
259263
// Source: <https://learn.microsoft.com/en-us/windows/win32/api/heapapi/nf-heapapi-heapalloc>
260264
let align = this.tcx.pointer_size().bytes().strict_mul(2);
261265
let ptr = this.allocate_ptr(
262266
Size::from_bytes(size),
263267
Align::from_bytes(align).unwrap(),
264268
MiriMemoryKind::WinHeap.into(),
269+
init
265270
)?;
266-
if zero_init {
267-
this.write_bytes_ptr(
268-
ptr.into(),
269-
iter::repeat(0u8).take(usize::try_from(size).unwrap()),
270-
)?;
271-
}
272271
this.write_pointer(ptr, dest)?;
273272
}
274273
"HeapFree" => {
@@ -300,6 +299,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
300299
Size::from_bytes(size),
301300
Align::from_bytes(align).unwrap(),
302301
MiriMemoryKind::WinHeap.into(),
302+
AllocInit::Uninit
303303
)?;
304304
this.write_pointer(new_ptr, dest)?;
305305
}

0 commit comments

Comments
 (0)