Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

compiler-rt: re-enable memcpy runtime safety in tests #22875

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
+156 −147
Open

compiler-rt: re-enable memcpy runtime safety in tests #22875

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
303 changes: 156 additions & 147 deletions lib/compiler_rt/memcpy.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,9 @@ comptime {
};

if (builtin.mode == .ReleaseSmall)
@export(&memcpySmall, export_options)
@export(&impl(false).memcpySmall, export_options)
else
@export(&memcpyFast, export_options);
@export(&impl(false).memcpyFast, export_options);
}
}

Expand All @@ -33,174 +33,183 @@ comptime {
assert(std.math.isPowerOfTwo(@sizeOf(Element)));
}

fn memcpySmall(noalias dest: ?[*]u8, noalias src: ?[*]const u8, len: usize) callconv(.C) ?[*]u8 {
@setRuntimeSafety(false);
fn impl(comptime enable_safety: bool) type {
return struct {
fn memcpySmall(noalias dest: ?[*]u8, noalias src: ?[*]const u8, len: usize) callconv(.C) ?[*]u8 {
@setRuntimeSafety(enable_safety);

for (0..len) |i| {
dest.?[i] = src.?[i];
}
for (0..len) |i| {
dest.?[i] = src.?[i];
}

return dest;
}
return dest;
}

fn memcpyFast(noalias dest: ?[*]u8, noalias src: ?[*]const u8, len: usize) callconv(.C) ?[*]u8 {
@setRuntimeSafety(false);
fn memcpyFast(noalias dest: ?[*]u8, noalias src: ?[*]const u8, len: usize) callconv(.C) ?[*]u8 {
@setRuntimeSafety(enable_safety);

const small_limit = 2 * @sizeOf(Element);
const small_limit = 2 * @sizeOf(Element);

if (copySmallLength(small_limit, dest.?, src.?, len)) return dest;
if (copySmallLength(small_limit, dest.?, src.?, len)) return dest;

copyForwards(dest.?, src.?, len);
copyForwards(dest.?, src.?, len);

return dest;
}
return dest;
}

inline fn copySmallLength(
comptime small_limit: comptime_int,
dest: [*]u8,
src: [*]const u8,
len: usize,
) bool {
if (len < 16) {
copyLessThan16(dest, src, len);
return true;
}
inline fn copySmallLength(
comptime small_limit: comptime_int,
dest: [*]u8,
src: [*]const u8,
len: usize,
) bool {
if (len < 16) {
copyLessThan16(dest, src, len);
return true;
}

if (comptime 2 < (std.math.log2(small_limit) + 1) / 2) {
if (copy16ToSmallLimit(small_limit, dest, src, len)) return true;
}
if (comptime 2 < (std.math.log2(small_limit) + 1) / 2) {
if (copy16ToSmallLimit(small_limit, dest, src, len)) return true;
}

return false;
}
return false;
}

inline fn copyLessThan16(
dest: [*]u8,
src: [*]const u8,
len: usize,
) void {
@setRuntimeSafety(false);
if (len < 4) {
if (len == 0) return;
dest[0] = src[0];
dest[len / 2] = src[len / 2];
dest[len - 1] = src[len - 1];
return;
}
copyRange4(4, dest, src, len);
}
inline fn copyLessThan16(
dest: [*]u8,
src: [*]const u8,
len: usize,
) void {
@setRuntimeSafety(enable_safety);
if (len < 4) {
if (len == 0) return;
dest[0] = src[0];
dest[len / 2] = src[len / 2];
dest[len - 1] = src[len - 1];
return;
}
copyRange4(4, dest, src, len);
}

inline fn copy16ToSmallLimit(
comptime small_limit: comptime_int,
dest: [*]u8,
src: [*]const u8,
len: usize,
) bool {
@setRuntimeSafety(false);
inline for (2..(std.math.log2(small_limit) + 1) / 2 + 1) |p| {
const limit = 1 << (2 * p);
if (len < limit) {
copyRange4(limit / 4, dest, src, len);
return true;
inline fn copy16ToSmallLimit(
comptime small_limit: comptime_int,
dest: [*]u8,
src: [*]const u8,
len: usize,
) bool {
@setRuntimeSafety(enable_safety);
inline for (2..(std.math.log2(small_limit) + 1) / 2 + 1) |p| {
const limit = 1 << (2 * p);
if (len < limit) {
copyRange4(limit / 4, dest, src, len);
return true;
}
}
return false;
}
}
return false;
}

inline fn copyForwards(
noalias dest: [*]u8,
noalias src: [*]const u8,
len: usize,
) void {
@setRuntimeSafety(false);

copyFixedLength(dest, src, @sizeOf(Element));
const alignment_offset = @alignOf(Element) - @intFromPtr(src) % @alignOf(Element);
const n = len - alignment_offset;
const d = dest + alignment_offset;
const s = src + alignment_offset;

copyBlocksAlignedSource(@ptrCast(d), @alignCast(@ptrCast(s)), n);

// copy last `@sizeOf(Element)` bytes unconditionally, since block copy
// methods only copy a multiple of `@sizeOf(Element)` bytes.
const offset = len - @sizeOf(Element);
copyFixedLength(dest + offset, src + offset, @sizeOf(Element));
}
inline fn copyForwards(
noalias dest: [*]u8,
noalias src: [*]const u8,
len: usize,
) void {
@setRuntimeSafety(enable_safety);
if (enable_safety) assert(len >= 2 * @sizeOf(Element));

copyFixedLength(dest, src, @sizeOf(Element));
const alignment_offset = @alignOf(Element) - @intFromPtr(src) % @alignOf(Element);
const n = len - alignment_offset;
const d = dest + alignment_offset;
const s = src + alignment_offset;

copyBlocksAlignedSource(@ptrCast(d), @alignCast(@ptrCast(s)), n);

// copy last `@sizeOf(Element)` bytes unconditionally, since block copy
// methods only copy a multiple of `@sizeOf(Element)` bytes.
const offset = len - @sizeOf(Element);
copyFixedLength(dest + offset, src + offset, @sizeOf(Element));
}

inline fn copyBlocksAlignedSource(
noalias dest: [*]align(1) Element,
noalias src: [*]const Element,
max_bytes: usize,
) void {
copyBlocks(dest, src, max_bytes);
}
inline fn copyBlocksAlignedSource(
noalias dest: [*]align(1) Element,
noalias src: [*]const Element,
max_bytes: usize,
) void {
copyBlocks(dest, src, max_bytes);
}

/// Copies the largest multiple of `@sizeOf(T)` bytes from `src` to `dest`,
/// that is less than `max_bytes` where `T` is the child type of `src` and
/// `dest`.
inline fn copyBlocks(
noalias dest: anytype,
noalias src: anytype,
max_bytes: usize,
) void {
@setRuntimeSafety(false);
/// Copies the largest multiple of `@sizeOf(T)` bytes from `src` to `dest`,
/// that is less than `max_bytes` where `T` is the child type of `src` and
/// `dest`.
inline fn copyBlocks(
noalias dest: anytype,
noalias src: anytype,
max_bytes: usize,
) void {
@setRuntimeSafety(enable_safety);

const T = @typeInfo(@TypeOf(dest)).pointer.child;
comptime assert(T == @typeInfo(@TypeOf(src)).pointer.child);
const T = @typeInfo(@TypeOf(dest)).pointer.child;
comptime assert(T == @typeInfo(@TypeOf(src)).pointer.child);

const loop_count = max_bytes / @sizeOf(T);
const loop_count = max_bytes / @sizeOf(T);

for (dest[0..loop_count], src[0..loop_count]) |*d, s| {
d.* = s;
}
}
for (dest[0..loop_count], src[0..loop_count]) |*d, s| {
d.* = s;
}
}

inline fn copyFixedLength(
noalias dest: [*]u8,
noalias src: [*]const u8,
comptime len: comptime_int,
) void {
@setRuntimeSafety(false);
comptime assert(std.math.isPowerOfTwo(len));
inline fn copyFixedLength(
noalias dest: [*]u8,
noalias src: [*]const u8,
comptime len: comptime_int,
) void {
@setRuntimeSafety(enable_safety);
comptime assert(std.math.isPowerOfTwo(len));

const T = if (len >= @sizeOf(Element))
Element
else if (len > @sizeOf(usize))
@Vector(len, u8)
else
@Type(.{ .int = .{ .signedness = .unsigned, .bits = len * 8 } });
const T = if (len >= @sizeOf(Element))
Element
else if (len > @sizeOf(usize))
@Vector(len, u8)
else
@Type(.{ .int = .{ .signedness = .unsigned, .bits = len * 8 } });

const loop_count = @divExact(len, @sizeOf(T));
const loop_count = @divExact(len, @sizeOf(T));

const d: [*]align(1) T = @ptrCast(dest);
const s: [*]align(1) const T = @ptrCast(src);
const d: [*]align(1) T = @ptrCast(dest);
const s: [*]align(1) const T = @ptrCast(src);

inline for (0..loop_count) |i| {
d[i] = s[i];
}
}
inline for (0..loop_count) |i| {
d[i] = s[i];
}
}

/// copy `len` bytes from `src` to `dest`; `len` must be in the range
/// `[copy_len, 4 * copy_len)`.
inline fn copyRange4(
comptime copy_len: comptime_int,
noalias dest: [*]u8,
noalias src: [*]const u8,
len: usize,
) void {
@setRuntimeSafety(enable_safety);
comptime assert(std.math.isPowerOfTwo(copy_len));
if (enable_safety) {
assert(len >= copy_len);
assert(len < 4 * copy_len);
}

/// copy `len` bytes from `src` to `dest`; `len` must be in the range
/// `[copy_len, 4 * copy_len)`.
inline fn copyRange4(
comptime copy_len: comptime_int,
noalias dest: [*]u8,
noalias src: [*]const u8,
len: usize,
) void {
@setRuntimeSafety(false);
comptime assert(std.math.isPowerOfTwo(copy_len));

const a = len & (copy_len * 2);
const b = a / 2;

const last = len - copy_len;
const pen = last - b;

copyFixedLength(dest, src, copy_len);
copyFixedLength(dest + b, src + b, copy_len);
copyFixedLength(dest + pen, src + pen, copy_len);
copyFixedLength(dest + last, src + last, copy_len);
const a = len & (copy_len * 2);
const b = a / 2;

const last = len - copy_len;
const pen = last - b;

copyFixedLength(dest, src, copy_len);
copyFixedLength(dest + b, src + b, copy_len);
copyFixedLength(dest + pen, src + pen, copy_len);
copyFixedLength(dest + last, src + last, copy_len);
}
};
}

test "memcpy" {
Expand Down Expand Up @@ -232,6 +241,6 @@ test "memcpy" {
}
};

try S.testFunc(memcpySmall);
try S.testFunc(memcpyFast);
try S.testFunc(impl(true).memcpySmall);
try S.testFunc(impl(true).memcpyFast);
}
Loading