Merge pull request #25965 from alexrp/s390x

`s390x-linux` and general big-endian stuff

Author: alexrp

.forgejo/workflows/ci.yaml

@@ -73,6 +73,26 @@ jobs:
       - name: Build and Test
         run: sh ci/riscv64-linux-release.sh
         timeout-minutes: 420
+  s390x-linux-debug:
+    runs-on: [self-hosted, s390x-linux]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Build and Test
+        run: sh ci/s390x-linux-debug.sh
+        timeout-minutes: 240
+  s390x-linux-release:
+    runs-on: [self-hosted, s390x-linux]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Build and Test
+        run: sh ci/s390x-linux-release.sh
+        timeout-minutes: 180
   x86_64-freebsd-debug:
     runs-on: [self-hosted, x86_64-freebsd]
     steps:

ci/s390x-linux-debug.sh

@@ -0,0 +1,65 @@
+#!/bin/sh
+
+# Requires cmake ninja-build
+
+set -x
+set -e
+
+TARGET="s390x-linux-musl"
+MCPU="z15"
+CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.16.0-dev.1354+94e98bfe8"
+PREFIX="$HOME/deps/$CACHE_BASENAME"
+ZIG="$PREFIX/bin/zig"
+
+# Override the cache directories because they won't actually help other CI runs
+# which will be testing alternate versions of zig, and ultimately would just
+# fill up space on the hard drive for no reason.
+export ZIG_GLOBAL_CACHE_DIR="$PWD/zig-global-cache"
+export ZIG_LOCAL_CACHE_DIR="$PWD/zig-local-cache"
+
+mkdir build-debug
+cd build-debug
+
+export CC="$ZIG cc -target $TARGET -mcpu=$MCPU"
+export CXX="$ZIG c++ -target $TARGET -mcpu=$MCPU"
+
+cmake .. \
+  -DCMAKE_INSTALL_PREFIX="stage3-debug" \
+  -DCMAKE_PREFIX_PATH="$PREFIX" \
+  -DCMAKE_BUILD_TYPE=Debug \
+  -DZIG_TARGET_TRIPLE="$TARGET" \
+  -DZIG_TARGET_MCPU="$MCPU" \
+  -DZIG_STATIC=ON \
+  -DZIG_NO_LIB=ON \
+  -GNinja \
+  -DCMAKE_C_LINKER_DEPFILE_SUPPORTED=FALSE \
+  -DCMAKE_CXX_LINKER_DEPFILE_SUPPORTED=FALSE
+# https://github.com/ziglang/zig/issues/22213
+
+# Now cmake will use zig as the C/C++ compiler. We reset the environment variables
+# so that installation and testing do not get affected by them.
+unset CC
+unset CXX
+
+ninja install
+
+# No -fqemu and -fwasmtime here as they're covered by the x86_64-linux scripts.
+stage3-debug/bin/zig build test docs \
+  --maxrss 30064771072 \
+  -Dstatic-llvm \
+  -Dskip-non-native \
+  -Dtarget=native-native-musl \
+  --search-prefix "$PREFIX" \
+  --zig-lib-dir "$PWD/../lib" \
+  --test-timeout 4m
+
+stage3-debug/bin/zig build \
+  --prefix stage4-debug \
+  -Denable-llvm \
+  -Dno-lib \
+  -Dtarget=$TARGET \
+  -Dcpu=$MCPU \
+  -Duse-zig-libcxx \
+  -Dversion-string="$(stage3-debug/bin/zig version)"
+
+stage4-debug/bin/zig test ../test/behavior.zig

ci/s390x-linux-release.sh

@@ -0,0 +1,71 @@
+#!/bin/sh
+
+# Requires cmake ninja-build
+
+set -x
+set -e
+
+TARGET="s390x-linux-musl"
+MCPU="z15"
+CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.16.0-dev.1354+94e98bfe8"
+PREFIX="$HOME/deps/$CACHE_BASENAME"
+ZIG="$PREFIX/bin/zig"
+
+# Override the cache directories because they won't actually help other CI runs
+# which will be testing alternate versions of zig, and ultimately would just
+# fill up space on the hard drive for no reason.
+export ZIG_GLOBAL_CACHE_DIR="$PWD/zig-global-cache"
+export ZIG_LOCAL_CACHE_DIR="$PWD/zig-local-cache"
+
+mkdir build-release
+cd build-release
+
+export CC="$ZIG cc -target $TARGET -mcpu=$MCPU"
+export CXX="$ZIG c++ -target $TARGET -mcpu=$MCPU"
+
+cmake .. \
+  -DCMAKE_INSTALL_PREFIX="stage3-release" \
+  -DCMAKE_PREFIX_PATH="$PREFIX" \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DZIG_TARGET_TRIPLE="$TARGET" \
+  -DZIG_TARGET_MCPU="$MCPU" \
+  -DZIG_STATIC=ON \
+  -DZIG_NO_LIB=ON \
+  -GNinja \
+  -DCMAKE_C_LINKER_DEPFILE_SUPPORTED=FALSE \
+  -DCMAKE_CXX_LINKER_DEPFILE_SUPPORTED=FALSE
+# https://github.com/ziglang/zig/issues/22213
+
+# Now cmake will use zig as the C/C++ compiler. We reset the environment variables
+# so that installation and testing do not get affected by them.
+unset CC
+unset CXX
+
+ninja install
+
+# No -fqemu and -fwasmtime here as they're covered by the x86_64-linux scripts.
+stage3-release/bin/zig build test docs \
+  --maxrss 30064771072 \
+  -Dstatic-llvm \
+  -Dskip-non-native \
+  -Dtarget=native-native-musl \
+  --search-prefix "$PREFIX" \
+  --zig-lib-dir "$PWD/../lib" \
+  --test-timeout 4m
+
+# Ensure that stage3 and stage4 are byte-for-byte identical.
+stage3-release/bin/zig build \
+  --prefix stage4-release \
+  -Denable-llvm \
+  -Dno-lib \
+  -Doptimize=ReleaseFast \
+  -Dstrip \
+  -Dtarget=$TARGET \
+  -Dcpu=$MCPU \
+  -Duse-zig-libcxx \
+  -Dversion-string="$(stage3-release/bin/zig version)"
+
+# diff returns an error code if the files differ.
+echo "If the following command fails, it means nondeterminism has been"
+echo "introduced, making stage3 and stage4 no longer byte-for-byte identical."
+diff stage3-release/bin/zig stage4-release/bin/zig

doc/langref/runtime_shrExact_overflow.zig

@@ -7,7 +7,7 @@ pub fn main() void {
     const y = @shrExact(x, 2);
     std.debug.print("value: {}\n", .{y});
 
-    if ((builtin.cpu.arch.isRISCV() or builtin.cpu.arch.isLoongArch()) and builtin.zig_backend == .stage2_llvm) @panic("https://github.com/ziglang/zig/issues/24304");
+    if ((builtin.cpu.arch.isRISCV() or builtin.cpu.arch.isLoongArch() or builtin.cpu.arch == .s390x) and builtin.zig_backend == .stage2_llvm) @panic("https://github.com/ziglang/zig/issues/24304");
 }
 
 // exe=fail

doc/langref/test_defining_variadic_function.zig

@@ -22,6 +22,10 @@ test "defining a variadic function" {
         // https://github.com/ziglang/zig/issues/16961
         return error.SkipZigTest;
     }
+    if (builtin.cpu.arch == .s390x) {
+        // https://github.com/ziglang/zig/issues/21350#issuecomment-3543006475
+        return error.SkipZigTest;
+    }
 
     try std.testing.expectEqual(@as(c_int, 0), add(0));
     try std.testing.expectEqual(@as(c_int, 1), add(1, @as(c_int, 1)));

lib/compiler/aro/aro/Compilation.zig

@@ -774,6 +774,11 @@ fn generateSystemDefines(comp: *Compilation, w: *Io.Writer) !void {
                 try w.print("#define __wasm_{s}__ 1\n", .{feature.name});
             }
         },
+        .s390x => {
+            try define(w, "__s390__");
+            try define(w, "__s390x__");
+            try define(w, "__zarch__");
+        },
         else => {},
     }
 

lib/compiler/std-docs.zig

@@ -348,12 +348,12 @@ fn buildWasmBinary(
                 result_error_bundle = try std.zig.Server.allocErrorBundle(arena, body);
             },
             .emit_digest => {
-                const EmitDigest = std.zig.Server.Message.EmitDigest;
-                const emit_digest = @as(*align(1) const EmitDigest, @ptrCast(body));
+                var r: std.Io.Reader = .fixed(body);
+                const emit_digest = r.takeStruct(std.zig.Server.Message.EmitDigest, .little) catch unreachable;
                 if (!emit_digest.flags.cache_hit) {
                     std.log.info("source changes detected; rebuilt wasm component", .{});
                 }
-                const digest = body[@sizeOf(EmitDigest)..][0..Cache.bin_digest_len];
+                const digest = r.takeArray(Cache.bin_digest_len) catch unreachable;
                 result = .{
                     .root_dir = Cache.Directory.cwd(),
                     .sub_path = try std.fs.path.join(arena, &.{
@@ -415,7 +415,10 @@ fn sendMessage(file: std.fs.File, tag: std.zig.Client.Message.Tag) !void {
         .tag = tag,
         .bytes_len = 0,
     };
-    try file.writeAll(std.mem.asBytes(&header));
+    var w = file.writer(&.{});
+    w.interface.writeStruct(header, .little) catch |err| switch (err) {
+        error.WriteFailed => return w.err.?,
+    };
 }
 
 fn openBrowserTab(gpa: Allocator, url: []const u8) !void {

lib/std/Build/Step.zig

@@ -680,7 +680,10 @@ fn sendMessage(file: std.fs.File, tag: std.zig.Client.Message.Tag) !void {
         .tag = tag,
         .bytes_len = 0,
     };
-    try file.writeAll(std.mem.asBytes(&header));
+    var w = file.writer(&.{});
+    w.interface.writeStruct(header, .little) catch |err| switch (err) {
+        error.WriteFailed => return w.err.?,
+    };
 }
 
 pub fn handleVerbose(

lib/std/Build/Step/CheckObject.zig

@@ -1857,15 +1857,26 @@ const ElfDumper = struct {
 
     fn parseAndDumpObject(step: *Step, check: Check, bytes: []const u8) ![]const u8 {
         const gpa = step.owner.allocator;
-        var reader: std.Io.Reader = .fixed(bytes);
 
-        const hdr = try reader.takeStruct(elf.Elf64_Ehdr, .little);
-        if (!mem.eql(u8, hdr.e_ident[0..4], "\x7fELF")) {
-            return error.InvalidMagicNumber;
+        // `std.elf.Header` takes care of endianness issues for us.
+        var reader: std.Io.Reader = .fixed(bytes);
+        const hdr = try elf.Header.read(&reader);
+
+        var shdrs = try gpa.alloc(elf.Elf64_Shdr, hdr.shnum);
+        defer gpa.free(shdrs);
+        {
+            var shdr_it = hdr.iterateSectionHeadersBuffer(bytes);
+            var shdr_i: usize = 0;
+            while (try shdr_it.next()) |shdr| : (shdr_i += 1) shdrs[shdr_i] = shdr;
         }
 
-        const shdrs = @as([*]align(1) const elf.Elf64_Shdr, @ptrCast(bytes.ptr + hdr.e_shoff))[0..hdr.e_shnum];
-        const phdrs = @as([*]align(1) const elf.Elf64_Phdr, @ptrCast(bytes.ptr + hdr.e_phoff))[0..hdr.e_phnum];
+        var phdrs = try gpa.alloc(elf.Elf64_Phdr, hdr.shnum);
+        defer gpa.free(phdrs);
+        {
+            var phdr_it = hdr.iterateProgramHeadersBuffer(bytes);
+            var phdr_i: usize = 0;
+            while (try phdr_it.next()) |phdr| : (phdr_i += 1) phdrs[phdr_i] = phdr;
+        }
 
         var ctx = ObjectContext{
             .gpa = gpa,
@@ -1875,13 +1886,21 @@ const ElfDumper = struct {
             .phdrs = phdrs,
             .shstrtab = undefined,
         };
-        ctx.shstrtab = ctx.getSectionContents(ctx.hdr.e_shstrndx);
+        ctx.shstrtab = ctx.getSectionContents(ctx.hdr.shstrndx);
+
+        defer gpa.free(ctx.symtab.symbols);
+        defer gpa.free(ctx.dysymtab.symbols);
+        defer gpa.free(ctx.dyns);
 
         for (ctx.shdrs, 0..) |shdr, i| switch (shdr.sh_type) {
             elf.SHT_SYMTAB, elf.SHT_DYNSYM => {
                 const raw = ctx.getSectionContents(i);
                 const nsyms = @divExact(raw.len, @sizeOf(elf.Elf64_Sym));
-                const symbols = @as([*]align(1) const elf.Elf64_Sym, @ptrCast(raw.ptr))[0..nsyms];
+                const symbols = try gpa.alloc(elf.Elf64_Sym, nsyms);
+
+                var r: std.Io.Reader = .fixed(raw);
+                for (0..nsyms) |si| symbols[si] = r.takeStruct(elf.Elf64_Sym, ctx.hdr.endian) catch unreachable;
+
                 const strings = ctx.getSectionContents(shdr.sh_link);
 
                 switch (shdr.sh_type) {
@@ -1900,6 +1919,17 @@ const ElfDumper = struct {
                     else => unreachable,
                 }
             },
+            elf.SHT_DYNAMIC => {
+                const raw = ctx.getSectionContents(i);
+                const ndyns = @divExact(raw.len, @sizeOf(elf.Elf64_Dyn));
+                const dyns = try gpa.alloc(elf.Elf64_Dyn, ndyns);
+
+                var r: std.Io.Reader = .fixed(raw);
+                for (0..ndyns) |si| dyns[si] = r.takeStruct(elf.Elf64_Dyn, ctx.hdr.endian) catch unreachable;
+
+                ctx.dyns = dyns;
+                ctx.dyns_strings = ctx.getSectionContents(shdr.sh_link);
+            },
 
             else => {},
         };
@@ -1923,9 +1953,9 @@ const ElfDumper = struct {
                 try ctx.dumpSymtab(.dysymtab, writer);
             } else return step.fail("no dynamic symbol table found", .{}),
 
-            .dynamic_section => if (ctx.getSectionByName(".dynamic")) |shndx| {
-                try ctx.dumpDynamicSection(shndx, writer);
-            } else return step.fail("no .dynamic section found", .{}),
+            .dynamic_section => if (ctx.dyns.len > 0) {
+                try ctx.dumpDynamicSection(writer);
+            } else return step.fail("no dynamic section found", .{}),
 
             .dump_section => {
                 const name = mem.sliceTo(@as([*:0]const u8, @ptrCast(check.data.items.ptr + check.payload.dump_section)), 0);
@@ -1942,17 +1972,19 @@ const ElfDumper = struct {
     const ObjectContext = struct {
         gpa: Allocator,
         data: []const u8,
-        hdr: elf.Elf64_Ehdr,
-        shdrs: []align(1) const elf.Elf64_Shdr,
-        phdrs: []align(1) const elf.Elf64_Phdr,
+        hdr: elf.Header,
+        shdrs: []const elf.Elf64_Shdr,
+        phdrs: []const elf.Elf64_Phdr,
         shstrtab: []const u8,
         symtab: Symtab = .{},
         dysymtab: Symtab = .{},
+        dyns: []const elf.Elf64_Dyn = &.{},
+        dyns_strings: []const u8 = &.{},
 
         fn dumpHeader(ctx: ObjectContext, writer: anytype) !void {
             try writer.writeAll("header\n");
-            try writer.print("type {s}\n", .{@tagName(ctx.hdr.e_type)});
-            try writer.print("entry {x}\n", .{ctx.hdr.e_entry});
+            try writer.print("type {s}\n", .{@tagName(ctx.hdr.type)});
+            try writer.print("entry {x}\n", .{ctx.hdr.entry});
         }
 
         fn dumpPhdrs(ctx: ObjectContext, writer: anytype) !void {
@@ -2011,16 +2043,10 @@ const ElfDumper = struct {
             }
         }
 
-        fn dumpDynamicSection(ctx: ObjectContext, shndx: usize, writer: anytype) !void {
-            const shdr = ctx.shdrs[shndx];
-            const strtab = ctx.getSectionContents(shdr.sh_link);
-            const data = ctx.getSectionContents(shndx);
-            const nentries = @divExact(data.len, @sizeOf(elf.Elf64_Dyn));
-            const entries = @as([*]align(1) const elf.Elf64_Dyn, @ptrCast(data.ptr))[0..nentries];
-
+        fn dumpDynamicSection(ctx: ObjectContext, writer: anytype) !void {
             try writer.writeAll(ElfDumper.dynamic_section_label ++ "\n");
 
-            for (entries) |entry| {
+            for (ctx.dyns) |entry| {
                 const key = @as(u64, @bitCast(entry.d_tag));
                 const value = entry.d_val;
 
@@ -2067,7 +2093,7 @@ const ElfDumper = struct {
                     elf.DT_RPATH,
                     elf.DT_RUNPATH,
                     => {
-                        const name = getString(strtab, @intCast(value));
+                        const name = getString(ctx.dyns_strings, @intCast(value));
                         try writer.print(" {s}", .{name});
                     },
 
@@ -2256,8 +2282,8 @@ const ElfDumper = struct {
     };
 
     const Symtab = struct {
-        symbols: []align(1) const elf.Elf64_Sym = &[0]elf.Elf64_Sym{},
-        strings: []const u8 = &[0]u8{},
+        symbols: []const elf.Elf64_Sym = &.{},
+        strings: []const u8 = &.{},
 
         fn get(st: Symtab, index: usize) ?elf.Elf64_Sym {
             if (index >= st.symbols.len) return null;