diff --git a/Makefile b/Makefile index d3ec41fa93..0413695f24 100644 --- a/Makefile +++ b/Makefile @@ -68,7 +68,7 @@ build.examples.tinygo: $(tinygo_sources) done # We use zig to build C as it is easy to install and embeds a copy of zig-cc. -c_sources := imports/wasi_snapshot_preview1/example/testdata/zig-cc/cat.c +c_sources := imports/wasi_snapshot_preview1/example/testdata/zig-cc/cat.c imports/wasi_snapshot_preview1/testdata/zig-cc/ls.c .PHONY: build.examples.zig-cc build.examples.zig-cc: $(c_sources) @for f in $^; do \ @@ -103,9 +103,10 @@ build.examples.emscripten: $(emscripten_sources) %/greet.wasm : cargo_target := wasm32-unknown-unknown %/cat.wasm : cargo_target := wasm32-wasi +%/ls.wasm : cargo_target := wasm32-wasi .PHONY: build.examples.rust -build.examples.rust: examples/allocation/rust/testdata/greet.wasm imports/wasi_snapshot_preview1/example/testdata/cargo-wasi/cat.wasm +build.examples.rust: examples/allocation/rust/testdata/greet.wasm imports/wasi_snapshot_preview1/example/testdata/cargo-wasi/cat.wasm imports/wasi_snapshot_preview1/testdata/cargo-wasi/ls.wasm # Builds rust using cargo normally, or cargo-wasi. %.wasm: %.rs diff --git a/imports/wasi_snapshot_preview1/example/testdata/zig-cc/cat.wasm b/imports/wasi_snapshot_preview1/example/testdata/zig-cc/cat.wasm index 82088713e9..bcfc20b479 100755 Binary files a/imports/wasi_snapshot_preview1/example/testdata/zig-cc/cat.wasm and b/imports/wasi_snapshot_preview1/example/testdata/zig-cc/cat.wasm differ diff --git a/imports/wasi_snapshot_preview1/fs.go b/imports/wasi_snapshot_preview1/fs.go index fe0238ab69..6ab67110a2 100644 --- a/imports/wasi_snapshot_preview1/fs.go +++ b/imports/wasi_snapshot_preview1/fs.go @@ -6,6 +6,7 @@ import ( "errors" "io" "io/fs" + "math" "path" "syscall" @@ -643,11 +644,268 @@ func fdRead_shouldContinueRead(n, l uint32, err error) (bool, Errno) { // entries from a directory. // // See https://github.com/WebAssembly/WASI/blob/snapshot-01/phases/snapshot/docs.md#-fd_readdirfd-fd-buf-pointeru8-buf_len-size-cookie-dircookie---errno-size -var fdReaddir = stubFunction( - functionFdReaddir, - []wasm.ValueType{i32, i32, i32, i64, i32}, - []string{"fd", "buf", "buf_len", "cookie", "result.bufused"}, -) +var fdReaddir = &wasm.HostFunc{ + ExportNames: []string{functionFdReaddir}, + Name: functionFdReaddir, + ParamTypes: []wasm.ValueType{i32, i32, i32, i64, i32}, + ParamNames: []string{"fd", "buf", "buf_len", "cookie", "result.bufused"}, + ResultTypes: []api.ValueType{i32}, + Code: &wasm.Code{ + IsHostFunction: true, + GoFunc: wasiFunc(fdReaddirFn), + }, +} + +func fdReaddirFn(ctx context.Context, mod api.Module, params []uint64) Errno { + fd := uint32(params[0]) + buf := uint32(params[1]) + bufLen := uint32(params[2]) + // We control the value of the cookie, and it should never be negative. + // However, we coerce it to signed to ensure the caller doesn't manipulate + // it in such a way that becomes negative. + cookie := int64(params[3]) + resultBufused := uint32(params[4]) + + // Validate the FD is a directory + rd, dir, errno := openedDir(ctx, mod, fd) + if errno != ErrnoSuccess { + return errno + } + + // expect a cookie only if we are continuing a read. + if cookie == 0 && dir.CountRead > 0 { + return ErrnoInval // invalid as a cookie is minimally one. + } + + // First, determine the maximum directory entries that can be encoded as + // dirents. The total size is direntSize(24) + nameSize, for each file. + // Since a zero-length file name is invalid, the minimum size entry is + // 25 (direntSize + 1 character). + maxDirEntries := int(bufLen/direntSize + 1) + + // While unlikely maxDirEntries will fit into bufLen, add one more just in + // case, as we need to know if we hit the end of the directory or not to + // write the correct bufused (e.g. == bufLen unless EOF). + // >> If less than the size of the read buffer, the end of the + // >> directory has been reached. + maxDirEntries += 1 + + // The host keeps state for any unread entries from the prior call because + // we cannot seek to a previous directory position. Collect these entries. + entries, errno := lastDirEntries(dir, cookie) + if errno != ErrnoSuccess { + return errno + } + + // Check if we have maxDirEntries, and read more from the FS as needed. + if entryCount := len(entries); entryCount < maxDirEntries { + if l, err := rd.ReadDir(maxDirEntries - entryCount); err != io.EOF { + if err != nil { + return ErrnoIo + } + dir.CountRead += uint64(len(l)) + entries = append(entries, l...) + // Replace the cache with up to maxDirEntries, starting at cookie. + dir.Entries = entries + } + } + + mem := mod.Memory() + + // Determine how many dirents we can write, excluding a potentially + // truncated entry. + bufused, direntCount, writeTruncatedEntry := maxDirents(entries, bufLen) + + // Now, write entries to the underlying buffer. + if bufused > 0 { + + // d_next is the index of the next file in the list, so it should + // always be one higher than the requested cookie. + d_next := uint64(cookie + 1) + // ^^ yes this can overflow to negative, which means our implementation + // doesn't support writing greater than max int64 entries. + + dirents, ok := mem.Read(ctx, buf, bufused) + if !ok { + return ErrnoFault + } + + writeDirents(entries, direntCount, writeTruncatedEntry, dirents, d_next) + } + + if !mem.WriteUint32Le(ctx, resultBufused, bufused) { + return ErrnoFault + } + return ErrnoSuccess +} + +const largestDirent = int64(math.MaxUint32 - direntSize) + +// lastDirEntries is broken out from fdReaddirFn for testability. +func lastDirEntries(dir *internalsys.ReadDir, cookie int64) (entries []fs.DirEntry, errno Errno) { + if cookie < 0 { + errno = ErrnoInval // invalid as we will never send a negative cookie. + return + } + + entryCount := int64(len(dir.Entries)) + if entryCount == 0 { // there was no prior call + if cookie != 0 { + errno = ErrnoInval // invalid as we haven't sent that cookie + } + return + } + + // Get the first absolute position in our window of results + firstPos := int64(dir.CountRead) - entryCount + cookiePos := cookie - firstPos + + switch { + case cookiePos < 0: // cookie is asking for results outside our window. + errno = ErrnoNosys // we can't implement directory seeking backwards. + case cookiePos == 0: // cookie is asking for the next page. + case cookiePos > entryCount: + errno = ErrnoInval // invalid as we read that far, yet. + case cookiePos > 0: // truncate so to avoid large lists. + entries = dir.Entries[cookiePos:] + default: + entries = dir.Entries + } + if len(entries) == 0 { + entries = nil + } + return +} + +// direntSize is the size of the dirent struct, which should be followed by the +// length of a file name. +const direntSize = uint32(24) + +// maxDirents returns the maximum count and total entries that can fit in +// maxLen bytes. +// +// truncatedEntryLen is the amount of bytes past bufLen needed to write the +// next entry. We have to return bufused == bufLen unless the directory is +// exhausted. +// +// See https://github.com/WebAssembly/WASI/blob/snapshot-01/phases/snapshot/docs.md#fd_readdir +// See https://github.com/WebAssembly/wasi-libc/blob/659ff414560721b1660a19685110e484a081c3d4/libc-bottom-half/cloudlibc/src/libc/dirent/readdir.c#L44 +func maxDirents(entries []fs.DirEntry, bufLen uint32) (bufused, direntCount uint32, writeTruncatedEntry bool) { + lenRemaining := bufLen + for _, e := range entries { + if lenRemaining < direntSize { + // We don't have enough space in bufLen for another struct, + // entry. A caller who wants more will retry. + + // bufused == bufLen means more entries exist, which is the case + // when the dirent is larger than bytes remaining. + bufused = bufLen + break + } + + // use int64 to guard against huge filenames + nameLen := int64(len(e.Name())) + var entryLen uint32 + + // Check to see if direntSize + nameLen overflows, or if it would be + // larger than possible to encode. + if el := int64(direntSize) + nameLen; el < 0 || el > largestDirent { + // panic, as testing is difficult. ex we would have to extract a + // function to get size of a string or allocate a 2^32 size one! + panic("invalid filename: too large") + } else { // we know this can fit into a uint32 + entryLen = uint32(el) + } + + if entryLen > lenRemaining { + // We haven't room to write the entry, and docs say to write the + // header. This helps especially when there is an entry with a very + // long filename. Ex if bufLen is 4096 and the filename is 4096, + // we need to write direntSize(24) + 4096 bytes to write the entry. + // In this case, we only write up to direntSize(24) to allow the + // caller to resize. + + // bufused == bufLen means more entries exist, which is the case + // when the next entry is larger than bytes remaining. + bufused = bufLen + + // We do have enough space to write the header, this value will be + // passed on to writeDirents to only write the header for this entry. + writeTruncatedEntry = true + break + } + + // This won't go negative because we checked entryLen <= lenRemaining. + lenRemaining -= entryLen + bufused += entryLen + direntCount++ + } + return +} + +// writeDirents writes the directory entries to the buffer, which is pre-sized +// based on maxDirents. truncatedEntryLen means write one past entryCount, +// without its name. See maxDirents for why +func writeDirents( + entries []fs.DirEntry, + entryCount uint32, + writeTruncatedEntry bool, + dirents []byte, + d_next uint64, +) { + pos, i := uint32(0), uint32(0) + for ; i < entryCount; i++ { + e := entries[i] + nameLen := uint32(len(e.Name())) + + writeDirent(dirents[pos:], d_next, nameLen, e.IsDir()) + pos += direntSize + + copy(dirents[pos:], e.Name()) + pos += nameLen + d_next++ + } + + if !writeTruncatedEntry { + return + } + + // Write a dirent without its name + dirent := make([]byte, direntSize) + e := entries[i] + writeDirent(dirent, d_next, uint32(len(e.Name())), e.IsDir()) + + // Potentially truncate it + copy(dirents[pos:], dirent) +} + +// writeDirent writes direntSize bytes +func writeDirent(buf []byte, dNext uint64, dNamlen uint32, dType bool) { + binary.LittleEndian.PutUint64(buf, dNext) // d_next + binary.LittleEndian.PutUint64(buf[8:], 0) // no d_ino + binary.LittleEndian.PutUint32(buf[16:], dNamlen) // d_namlen + + filetype := wasiFiletypeRegularFile + if dType { + filetype = wasiFiletypeDirectory + } + binary.LittleEndian.PutUint32(buf[20:], uint32(filetype)) // d_type +} + +// openedDir returns the directory and ErrnoSuccess if the fd points to a readable directory. +func openedDir(ctx context.Context, mod api.Module, fd uint32) (fs.ReadDirFile, *internalsys.ReadDir, Errno) { + fsc := mod.(*wasm.CallContext).Sys.FS(ctx) + if f, ok := fsc.OpenedFile(ctx, fd); !ok { + return nil, nil, ErrnoBadf + } else if d, ok := f.File.(fs.ReadDirFile); !ok { + return nil, nil, ErrnoNotdir + } else { + if f.ReadDir == nil { + f.ReadDir = &internalsys.ReadDir{} + } + return d, f.ReadDir, ErrnoSuccess + } +} // fdRenumber is the WASI function named functionFdRenumber which atomically // replaces a file descriptor by renumbering another file descriptor. diff --git a/imports/wasi_snapshot_preview1/fs_test.go b/imports/wasi_snapshot_preview1/fs_test.go index cb6a686a26..6931971d37 100644 --- a/imports/wasi_snapshot_preview1/fs_test.go +++ b/imports/wasi_snapshot_preview1/fs_test.go @@ -2,6 +2,7 @@ package wasi_snapshot_preview1 import ( "bytes" + _ "embed" "io" "io/fs" "math" @@ -13,6 +14,7 @@ import ( "github.com/tetratelabs/wazero" "github.com/tetratelabs/wazero/api" + internalsys "github.com/tetratelabs/wazero/internal/sys" "github.com/tetratelabs/wazero/internal/testing/require" "github.com/tetratelabs/wazero/internal/wasm" ) @@ -1004,15 +1006,712 @@ func Test_fdRead_shouldContinueRead(t *testing.T) { } } -// Test_fdReaddir only tests it is stubbed for GrainLang per #271 +var ( + fdReadDirFs = fstest.MapFS{ + "notdir": {}, + "emptydir": {Mode: fs.ModeDir}, + "dir": {Mode: fs.ModeDir}, + "dir/-": {}, // len = 24+1 = 25 + "dir/a-": {Mode: fs.ModeDir}, // len = 24+2 = 26 + "dir/ab-": {}, // len = 24+3 = 27 + } + + testDirEntries = func() []fs.DirEntry { + entries, err := fdReadDirFs.ReadDir("dir") + if err != nil { + panic(err) + } + return entries + }() + + dirent1 = []byte{ + 1, 0, 0, 0, 0, 0, 0, 0, // d_next = 1 + 0, 0, 0, 0, 0, 0, 0, 0, // d_ino = 0 + 1, 0, 0, 0, // d_namlen = 1 character + 4, 0, 0, 0, // d_type = regular_file + '-', // name + } + dirent2 = []byte{ + 2, 0, 0, 0, 0, 0, 0, 0, // d_next = 2 + 0, 0, 0, 0, 0, 0, 0, 0, // d_ino = 0 + 2, 0, 0, 0, // d_namlen = 1 character + 3, 0, 0, 0, // d_type = directory + 'a', '-', // name + } + dirent3 = []byte{ + 3, 0, 0, 0, 0, 0, 0, 0, // d_next = 3 + 0, 0, 0, 0, 0, 0, 0, 0, // d_ino = 0 + 3, 0, 0, 0, // d_namlen = 3 characters + 4, 0, 0, 0, // d_type = regular_file + 'a', 'b', '-', // name + } +) + func Test_fdReaddir(t *testing.T) { - log := requireErrnoNosys(t, functionFdReaddir, 0, 0, 0, 0, 0) - require.Equal(t, ` ---> proxy.fd_readdir(fd=0,buf=0,buf_len=0,cookie=0,result.bufused=0) - --> wasi_snapshot_preview1.fd_readdir(fd=0,buf=0,buf_len=0,cookie=0,result.bufused=0) - <-- ENOSYS -<-- (52) -`, log) + mod, r, log := requireProxyModule(t, wazero.NewModuleConfig().WithFS(fdReadDirFs)) + defer r.Close(testCtx) + + fsc := mod.(*wasm.CallContext).Sys.FS(testCtx) + + fd, err := fsc.OpenFile(testCtx, "dir") + require.NoError(t, err) + + tests := []struct { + name string + dir func() *internalsys.FileEntry + buf, bufLen uint32 + cookie int64 + expectedMem []byte + expectedMemSize int + expectedBufused uint32 + expectedReadDir *internalsys.ReadDir + }{ + { + name: "empty dir", + dir: func() *internalsys.FileEntry { + dir, err := fdReadDirFs.Open("emptydir") + require.NoError(t, err) + + return &internalsys.FileEntry{File: dir} + }, + buf: 0, bufLen: 1, + cookie: 0, + expectedBufused: 0, + expectedMem: []byte{}, + expectedReadDir: &internalsys.ReadDir{}, + }, + { + name: "full read", + dir: func() *internalsys.FileEntry { + dir, err := fdReadDirFs.Open("dir") + require.NoError(t, err) + + return &internalsys.FileEntry{File: dir} + }, + buf: 0, bufLen: 4096, + cookie: 0, + expectedBufused: 78, // length of all entries + expectedMem: append(append(dirent1, dirent2...), dirent3...), + expectedReadDir: &internalsys.ReadDir{ + CountRead: 3, + Entries: testDirEntries, + }, + }, + { + name: "can't read", + dir: func() *internalsys.FileEntry { + dir, err := fdReadDirFs.Open("dir") + require.NoError(t, err) + + return &internalsys.FileEntry{File: dir} + }, + buf: 0, bufLen: 23, // length is too short for header + cookie: 0, + expectedBufused: 23, // == bufLen which is the size of the dirent + expectedMem: nil, + expectedReadDir: &internalsys.ReadDir{ + CountRead: 2, + Entries: testDirEntries[:2], + }, + }, + { + name: "can't read name", + dir: func() *internalsys.FileEntry { + dir, err := fdReadDirFs.Open("dir") + require.NoError(t, err) + + return &internalsys.FileEntry{File: dir} + }, + buf: 0, bufLen: 24, // length is long enough for first, but not the name. + cookie: 0, + expectedBufused: 24, // == bufLen which is the size of the dirent + expectedMem: dirent1[:24], // header without name + expectedReadDir: &internalsys.ReadDir{ + CountRead: 3, + Entries: testDirEntries, + }, + }, + { + name: "read exactly first", + dir: func() *internalsys.FileEntry { + dir, err := fdReadDirFs.Open("dir") + require.NoError(t, err) + + return &internalsys.FileEntry{File: dir} + }, + buf: 0, bufLen: 25, // length is long enough for first + the name, but not more. + cookie: 0, + expectedBufused: 25, // length to read exactly first. + expectedMem: dirent1, + expectedReadDir: &internalsys.ReadDir{ + CountRead: 3, + Entries: testDirEntries, + }, + }, + { + name: "read exactly second", + dir: func() *internalsys.FileEntry { + dir, err := fdReadDirFs.Open("dir") + require.NoError(t, err) + entry, err := dir.(fs.ReadDirFile).ReadDir(1) + require.NoError(t, err) + + return &internalsys.FileEntry{ + File: dir, + ReadDir: &internalsys.ReadDir{ + CountRead: 1, + Entries: entry, + }, + } + }, + buf: 0, bufLen: 26, // length is long enough for exactly second. + cookie: 1, // d_next of first + expectedBufused: 26, // length to read exactly second. + expectedMem: dirent2, + expectedReadDir: &internalsys.ReadDir{ + CountRead: 3, + Entries: testDirEntries[1:], + }, + }, + { + name: "read second and a little more", + dir: func() *internalsys.FileEntry { + dir, err := fdReadDirFs.Open("dir") + require.NoError(t, err) + entry, err := dir.(fs.ReadDirFile).ReadDir(1) + require.NoError(t, err) + + return &internalsys.FileEntry{ + File: dir, + ReadDir: &internalsys.ReadDir{ + CountRead: 1, + Entries: entry, + }, + } + }, + buf: 0, bufLen: 30, // length is longer than the second entry, but not long enough for a header. + cookie: 1, // d_next of first + expectedBufused: 30, // length to read some more, but not enough for a header, so buf was exhausted. + expectedMem: dirent2, + expectedMemSize: len(dirent2), // we do not want to compare the full buffer since we don't know what the leftover 4 bytes will contain. + expectedReadDir: &internalsys.ReadDir{ + CountRead: 3, + Entries: testDirEntries[1:], + }, + }, + { + name: "read second and header of third", + dir: func() *internalsys.FileEntry { + dir, err := fdReadDirFs.Open("dir") + require.NoError(t, err) + entry, err := dir.(fs.ReadDirFile).ReadDir(1) + require.NoError(t, err) + + return &internalsys.FileEntry{ + File: dir, + ReadDir: &internalsys.ReadDir{ + CountRead: 1, + Entries: entry, + }, + } + }, + buf: 0, bufLen: 50, // length is longer than the second entry + enough for the header of third. + cookie: 1, // d_next of first + expectedBufused: 50, // length to read exactly second and the header of third. + expectedMem: append(dirent2, dirent3[0:24]...), + expectedReadDir: &internalsys.ReadDir{ + CountRead: 3, + Entries: testDirEntries[1:], + }, + }, + { + name: "read second and third", + dir: func() *internalsys.FileEntry { + dir, err := fdReadDirFs.Open("dir") + require.NoError(t, err) + entry, err := dir.(fs.ReadDirFile).ReadDir(1) + require.NoError(t, err) + + return &internalsys.FileEntry{ + File: dir, + ReadDir: &internalsys.ReadDir{ + CountRead: 1, + Entries: entry, + }, + } + }, + buf: 0, bufLen: 53, // length is long enough for second and third. + cookie: 1, // d_next of first + expectedBufused: 53, // length to read exactly one second and third. + expectedMem: append(dirent2, dirent3...), + expectedReadDir: &internalsys.ReadDir{ + CountRead: 3, + Entries: testDirEntries[1:], + }, + }, + { + name: "read exactly third", + dir: func() *internalsys.FileEntry { + dir, err := fdReadDirFs.Open("dir") + require.NoError(t, err) + two, err := dir.(fs.ReadDirFile).ReadDir(2) + require.NoError(t, err) + + return &internalsys.FileEntry{ + File: dir, + ReadDir: &internalsys.ReadDir{ + CountRead: 2, + Entries: two[1:], + }, + } + }, + buf: 0, bufLen: 27, // length is long enough for exactly third. + cookie: 2, // d_next of second. + expectedBufused: 27, // length to read exactly third. + expectedMem: dirent3, + expectedReadDir: &internalsys.ReadDir{ + CountRead: 3, + Entries: testDirEntries[2:], + }, + }, + { + name: "read third and beyond", + dir: func() *internalsys.FileEntry { + dir, err := fdReadDirFs.Open("dir") + require.NoError(t, err) + two, err := dir.(fs.ReadDirFile).ReadDir(2) + require.NoError(t, err) + + return &internalsys.FileEntry{ + File: dir, + ReadDir: &internalsys.ReadDir{ + CountRead: 2, + Entries: two[1:], + }, + } + }, + buf: 0, bufLen: 100, // length is long enough for third and more, but there is nothing more. + cookie: 2, // d_next of second. + expectedBufused: 27, // length to read exactly third. + expectedMem: dirent3, + expectedReadDir: &internalsys.ReadDir{ + CountRead: 3, + Entries: testDirEntries[2:], + }, + }, + } + + for _, tt := range tests { + tc := tt + t.Run(tc.name, func(t *testing.T) { + defer log.Reset() + + // Assign the state we are testing + file, ok := fsc.OpenedFile(testCtx, fd) + require.True(t, ok) + dir := tc.dir() + defer dir.File.Close() + + file.File = dir.File + file.ReadDir = dir.ReadDir + + maskMemory(t, testCtx, mod, int(tc.bufLen)) + + // use an arbitrarily high value for the buf used position. + resultBufused := uint32(16192) + requireErrno(t, ErrnoSuccess, mod, functionFdReaddir, + uint64(fd), uint64(tc.buf), uint64(tc.bufLen), uint64(tc.cookie), uint64(resultBufused)) + + // read back the bufused and compare memory against it + bufUsed, ok := mod.Memory().ReadUint32Le(testCtx, resultBufused) + require.True(t, ok) + require.Equal(t, tc.expectedBufused, bufUsed) + + mem, ok := mod.Memory().Read(testCtx, tc.buf, bufUsed) + require.True(t, ok) + + if tc.expectedMem != nil { + if tc.expectedMemSize == 0 { + tc.expectedMemSize = len(tc.expectedMem) + } + require.Equal(t, tc.expectedMem, mem[:tc.expectedMemSize]) + } + + require.Equal(t, tc.expectedReadDir, file.ReadDir) + }) + } +} + +func Test_fdReaddir_Errors(t *testing.T) { + mod, r, log := requireProxyModule(t, wazero.NewModuleConfig().WithFS(fdReadDirFs)) + defer r.Close(testCtx) + memLen := mod.Memory().Size(testCtx) + + fsc := mod.(*wasm.CallContext).Sys.FS(testCtx) + + dirFD, err := fsc.OpenFile(testCtx, "dir") + require.NoError(t, err) + + fileFD, err := fsc.OpenFile(testCtx, "notdir") + require.NoError(t, err) + + tests := []struct { + name string + dir func() *internalsys.FileEntry + fd, buf, bufLen, resultBufused uint32 + cookie int64 + readDir *internalsys.ReadDir + expectedErrno Errno + expectedLog string + }{ + { + name: "out-of-memory reading buf", + fd: dirFD, + buf: memLen, + bufLen: 1000, + expectedErrno: ErrnoFault, + expectedLog: ` +--> proxy.fd_readdir(fd=4,buf=65536,buf_len=1000,cookie=0,result.bufused=0) + ==> wasi_snapshot_preview1.fd_readdir(fd=4,buf=65536,buf_len=1000,cookie=0,result.bufused=0) + <== EFAULT +<-- (21) +`, + }, + { + name: "invalid fd", + fd: 42, // arbitrary invalid fd + expectedErrno: ErrnoBadf, + expectedLog: ` +--> proxy.fd_readdir(fd=42,buf=0,buf_len=0,cookie=0,result.bufused=0) + ==> wasi_snapshot_preview1.fd_readdir(fd=42,buf=0,buf_len=0,cookie=0,result.bufused=0) + <== EBADF +<-- (8) +`, + }, + { + name: "not a dir", + fd: fileFD, + expectedErrno: ErrnoNotdir, + expectedLog: ` +--> proxy.fd_readdir(fd=5,buf=0,buf_len=0,cookie=0,result.bufused=0) + ==> wasi_snapshot_preview1.fd_readdir(fd=5,buf=0,buf_len=0,cookie=0,result.bufused=0) + <== ENOTDIR +<-- (54) +`, + }, + { + name: "out-of-memory reading buf", + fd: dirFD, + buf: memLen, + bufLen: 1000, + expectedErrno: ErrnoFault, + expectedLog: ` +--> proxy.fd_readdir(fd=4,buf=65536,buf_len=1000,cookie=0,result.bufused=0) + ==> wasi_snapshot_preview1.fd_readdir(fd=4,buf=65536,buf_len=1000,cookie=0,result.bufused=0) + <== EFAULT +<-- (21) +`, + }, + { + name: "out-of-memory reading bufLen", + fd: dirFD, + buf: memLen - 1, + bufLen: 1000, + expectedErrno: ErrnoFault, + expectedLog: ` +--> proxy.fd_readdir(fd=4,buf=65535,buf_len=1000,cookie=0,result.bufused=0) + ==> wasi_snapshot_preview1.fd_readdir(fd=4,buf=65535,buf_len=1000,cookie=0,result.bufused=0) + <== EFAULT +<-- (21) +`, + }, + { + name: "resultBufused is outside memory", + fd: dirFD, + buf: 0, bufLen: 1, + resultBufused: memLen, + expectedErrno: ErrnoFault, + expectedLog: ` +--> proxy.fd_readdir(fd=4,buf=0,buf_len=1,cookie=0,result.bufused=65536) + ==> wasi_snapshot_preview1.fd_readdir(fd=4,buf=0,buf_len=1,cookie=0,result.bufused=65536) + <== EFAULT +<-- (21) +`, + }, + { + name: "cookie invalid when no prior state", + fd: dirFD, + buf: 0, bufLen: 1000, + cookie: 1, + resultBufused: 2000, + expectedErrno: ErrnoInval, + expectedLog: ` +--> proxy.fd_readdir(fd=4,buf=0,buf_len=1000,cookie=1,result.bufused=2000) + ==> wasi_snapshot_preview1.fd_readdir(fd=4,buf=0,buf_len=1000,cookie=1,result.bufused=2000) + <== EINVAL +<-- (28) +`, + }, + { + name: "negative cookie invalid", + fd: dirFD, + buf: 0, bufLen: 1000, + cookie: -1, + readDir: &internalsys.ReadDir{CountRead: 1}, + resultBufused: 2000, + expectedErrno: ErrnoInval, + expectedLog: ` +--> proxy.fd_readdir(fd=4,buf=0,buf_len=1000,cookie=18446744073709551615,result.bufused=2000) + ==> wasi_snapshot_preview1.fd_readdir(fd=4,buf=0,buf_len=1000,cookie=18446744073709551615,result.bufused=2000) + <== EINVAL +<-- (28) +`, + }, + } + + for _, tt := range tests { + tc := tt + t.Run(tc.name, func(t *testing.T) { + defer log.Reset() + + // Reset the directory so that tests don't taint each other. + if file, ok := fsc.OpenedFile(testCtx, tc.fd); ok && tc.fd == dirFD { + dir, err := fdReadDirFs.Open("dir") + require.NoError(t, err) + defer dir.Close() + + file.File = dir + file.ReadDir = nil + } + + requireErrno(t, tc.expectedErrno, mod, functionFdReaddir, + uint64(tc.fd), uint64(tc.buf), uint64(tc.bufLen), uint64(tc.cookie), uint64(tc.resultBufused)) + require.Equal(t, tc.expectedLog, "\n"+log.String()) + }) + } +} + +func Test_lastDirEntries(t *testing.T) { + tests := []struct { + name string + f *internalsys.ReadDir + cookie int64 + expectedEntries []fs.DirEntry + expectedErrno Errno + }{ + { + name: "no prior call", + }, + { + name: "no prior call, but passed a cookie", + cookie: 1, + expectedErrno: ErrnoInval, + }, + { + name: "cookie is negative", + f: &internalsys.ReadDir{ + CountRead: 3, + Entries: testDirEntries, + }, + cookie: -1, + expectedErrno: ErrnoInval, + }, + { + name: "cookie is greater than last d_next", + f: &internalsys.ReadDir{ + CountRead: 3, + Entries: testDirEntries, + }, + cookie: 5, + expectedErrno: ErrnoInval, + }, + { + name: "cookie is last pos", + f: &internalsys.ReadDir{ + CountRead: 3, + Entries: testDirEntries, + }, + cookie: 3, + expectedEntries: nil, + }, + { + name: "cookie is one before last pos", + f: &internalsys.ReadDir{ + CountRead: 3, + Entries: testDirEntries, + }, + cookie: 2, + expectedEntries: testDirEntries[2:], + }, + { + name: "cookie is before current entries", + f: &internalsys.ReadDir{ + CountRead: 5, + Entries: testDirEntries, + }, + cookie: 1, + expectedErrno: ErrnoNosys, // not implemented + }, + } + + for _, tt := range tests { + tc := tt + + t.Run(tc.name, func(t *testing.T) { + f := tc.f + if f == nil { + f = &internalsys.ReadDir{} + } + entries, errno := lastDirEntries(f, tc.cookie) + require.Equal(t, tc.expectedErrno, errno) + require.Equal(t, tc.expectedEntries, entries) + }) + } +} + +func Test_maxDirents(t *testing.T) { + tests := []struct { + name string + entries []fs.DirEntry + maxLen uint32 + expectedCount uint32 + expectedwriteTruncatedEntry bool + expectedBufused uint32 + }{ + { + name: "no entries", + }, + { + name: "can't fit one", + entries: testDirEntries, + maxLen: 23, + expectedBufused: 23, + expectedwriteTruncatedEntry: false, + }, + { + name: "only fits header", + entries: testDirEntries, + maxLen: 24, + expectedBufused: 24, + expectedwriteTruncatedEntry: true, + }, + { + name: "one", + entries: testDirEntries, + maxLen: 25, + expectedCount: 1, + expectedBufused: 25, + }, + { + name: "one but not room for two's name", + entries: testDirEntries, + maxLen: 25 + 25, + expectedCount: 1, + expectedwriteTruncatedEntry: true, // can write direntSize + expectedBufused: 25 + 25, + }, + { + name: "two", + entries: testDirEntries, + maxLen: 25 + 26, + expectedCount: 2, + expectedBufused: 25 + 26, + }, + { + name: "two but not three's dirent", + entries: testDirEntries, + maxLen: 25 + 26 + 20, + expectedCount: 2, + expectedwriteTruncatedEntry: false, // 20 + 4 == direntSize + expectedBufused: 25 + 26 + 20, + }, + { + name: "two but not three's name", + entries: testDirEntries, + maxLen: 25 + 26 + 26, + expectedCount: 2, + expectedwriteTruncatedEntry: true, // can write direntSize + expectedBufused: 25 + 26 + 26, + }, + { + name: "three", + entries: testDirEntries, + maxLen: 25 + 26 + 27, + expectedCount: 3, + expectedwriteTruncatedEntry: false, // end of dir + expectedBufused: 25 + 26 + 27, + }, + { + name: "max", + entries: testDirEntries, + maxLen: 100, + expectedCount: 3, + expectedwriteTruncatedEntry: false, // end of dir + expectedBufused: 25 + 26 + 27, + }, + } + + for _, tt := range tests { + tc := tt + + t.Run(tc.name, func(t *testing.T) { + bufused, direntCount, writeTruncatedEntry := maxDirents(tc.entries, tc.maxLen) + require.Equal(t, tc.expectedCount, direntCount) + require.Equal(t, tc.expectedwriteTruncatedEntry, writeTruncatedEntry) + require.Equal(t, tc.expectedBufused, bufused) + }) + } +} + +func Test_writeDirents(t *testing.T) { + tests := []struct { + name string + entries []fs.DirEntry + entryCount uint32 + writeTruncatedEntry bool + expectedEntriesBuf []byte + }{ + { + name: "none", + entries: testDirEntries, + }, + { + name: "one", + entries: testDirEntries, + entryCount: 1, + expectedEntriesBuf: dirent1, + }, + { + name: "two", + entries: testDirEntries, + entryCount: 2, + expectedEntriesBuf: append(dirent1, dirent2...), + }, + { + name: "two with truncated", + entries: testDirEntries, + entryCount: 2, + writeTruncatedEntry: true, + expectedEntriesBuf: append(append(dirent1, dirent2...), dirent3[0:10]...), + }, + { + name: "three", + entries: testDirEntries, + entryCount: 3, + expectedEntriesBuf: append(append(dirent1, dirent2...), dirent3...), + }, + } + + for _, tt := range tests { + tc := tt + + t.Run(tc.name, func(t *testing.T) { + cookie := uint64(1) + entriesBuf := make([]byte, len(tc.expectedEntriesBuf)) + writeDirents(tc.entries, tc.entryCount, tc.writeTruncatedEntry, entriesBuf, cookie) + require.Equal(t, tc.expectedEntriesBuf, entriesBuf) + }) + } } // Test_fdRenumber only tests it is stubbed for GrainLang per #271 diff --git a/imports/wasi_snapshot_preview1/testdata/cargo-wasi/.gitignore b/imports/wasi_snapshot_preview1/testdata/cargo-wasi/.gitignore new file mode 100644 index 0000000000..96ef6c0b94 --- /dev/null +++ b/imports/wasi_snapshot_preview1/testdata/cargo-wasi/.gitignore @@ -0,0 +1,2 @@ +/target +Cargo.lock diff --git a/imports/wasi_snapshot_preview1/testdata/cargo-wasi/Cargo.toml b/imports/wasi_snapshot_preview1/testdata/cargo-wasi/Cargo.toml new file mode 100644 index 0000000000..0ee71c53bd --- /dev/null +++ b/imports/wasi_snapshot_preview1/testdata/cargo-wasi/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "ls" +version = "0.1.0" +edition = "2021" + +[[bin]] +name = "ls" +path = "ls.rs" diff --git a/imports/wasi_snapshot_preview1/testdata/cargo-wasi/ls.rs b/imports/wasi_snapshot_preview1/testdata/cargo-wasi/ls.rs new file mode 100644 index 0000000000..b5b6435c01 --- /dev/null +++ b/imports/wasi_snapshot_preview1/testdata/cargo-wasi/ls.rs @@ -0,0 +1,7 @@ +use std::fs; + +fn main() { + for path in fs::read_dir(".").unwrap() { + println!("{}", path.unwrap().path().display()) + } +} diff --git a/imports/wasi_snapshot_preview1/testdata/cargo-wasi/ls.wasm b/imports/wasi_snapshot_preview1/testdata/cargo-wasi/ls.wasm new file mode 100644 index 0000000000..babbb7af0a Binary files /dev/null and b/imports/wasi_snapshot_preview1/testdata/cargo-wasi/ls.wasm differ diff --git a/imports/wasi_snapshot_preview1/testdata/zig-cc/ls.c b/imports/wasi_snapshot_preview1/testdata/zig-cc/ls.c new file mode 100644 index 0000000000..6077d3061f --- /dev/null +++ b/imports/wasi_snapshot_preview1/testdata/zig-cc/ls.c @@ -0,0 +1,15 @@ +#include +#include + +int main(void) { + DIR *d; + struct dirent *dir; + d = opendir("."); + if (d) { + while ((dir = readdir(d)) != NULL) { + printf("./%s\n", dir->d_name); + } + closedir(d); + } + return(0); +} diff --git a/imports/wasi_snapshot_preview1/testdata/zig-cc/ls.wasm b/imports/wasi_snapshot_preview1/testdata/zig-cc/ls.wasm new file mode 100755 index 0000000000..79059e7cab Binary files /dev/null and b/imports/wasi_snapshot_preview1/testdata/zig-cc/ls.wasm differ diff --git a/imports/wasi_snapshot_preview1/wasi_stdlib_test.go b/imports/wasi_snapshot_preview1/wasi_stdlib_test.go new file mode 100644 index 0000000000..60bc9f5fc5 --- /dev/null +++ b/imports/wasi_snapshot_preview1/wasi_stdlib_test.go @@ -0,0 +1,103 @@ +package wasi_snapshot_preview1 + +import ( + "bytes" + _ "embed" + "io/fs" + "strconv" + "strings" + "testing" + "testing/fstest" + + "github.com/tetratelabs/wazero" + "github.com/tetratelabs/wazero/internal/testing/require" + "github.com/tetratelabs/wazero/sys" +) + +// lsWasmCargoWasi was compiled from testdata/cargo-wasi/ls.rs +// +//go:embed testdata/cargo-wasi/ls.wasm +var lsWasmCargoWasi []byte + +// lsZigCc was compiled from testdata/zig-cc/ls.c +// +//go:embed testdata/zig-cc/ls.wasm +var lsZigCc []byte + +// Test_fdReaddir_ls ensures that the behavior we've implemented not only +// matches the wasi spec, but also at least two compilers use of sdks. +func Test_fdReaddir_ls(t *testing.T) { + for toolchain, bin := range map[string][]byte{ + "cargo-wasi": lsWasmCargoWasi, + "zig-cc": lsZigCc, + } { + toolchain := toolchain + bin := bin + t.Run(toolchain, func(t *testing.T) { + testFdReaddirLs(t, bin) + }) + } +} + +func testFdReaddirLs(t *testing.T, bin []byte) { + t.Run("empty directory", func(t *testing.T) { + stdout, stderr := compileAndRun(t, wazero.NewModuleConfig(). + WithFS(fstest.MapFS{}), bin) + + require.Zero(t, stderr) + require.Zero(t, stdout) + }) + + t.Run("directory with entries", func(t *testing.T) { + stdout, stderr := compileAndRun(t, wazero.NewModuleConfig(). + WithFS(fstest.MapFS{ + "-": {}, + "a-": {Mode: fs.ModeDir}, + "ab-": {}, + }), bin) + + require.Zero(t, stderr) + require.Equal(t, `./- +./a- +./ab- +`, stdout) + }) + + t.Run("directory with tons of entries", func(t *testing.T) { + testFS := fstest.MapFS{} + count := 8096 + for i := 0; i < count; i++ { + testFS[strconv.Itoa(i)] = &fstest.MapFile{} + } + stdout, stderr := compileAndRun(t, wazero.NewModuleConfig(). + WithFS(testFS), bin) + + require.Zero(t, stderr) + lines := strings.Split(stdout, "\n") + require.Equal(t, count+1 /* trailing newline */, len(lines)) + }) +} + +func compileAndRun(t *testing.T, config wazero.ModuleConfig, bin []byte) (stdout, stderr string) { + var stdoutBuf, stderrBuf bytes.Buffer + + r := wazero.NewRuntime(testCtx) + defer r.Close(testCtx) + + _, err := Instantiate(testCtx, r) + require.NoError(t, err) + + compiled, err := r.CompileModule(testCtx, bin) + require.NoError(t, err) + + _, err = r.InstantiateModule(testCtx, compiled, config.WithStdout(&stdoutBuf).WithStderr(&stderrBuf)) + if exitErr, ok := err.(*sys.ExitError); ok { + require.Zero(t, exitErr.ExitCode()) + } else { + require.NoError(t, err) + } + + stdout = stdoutBuf.String() + stderr = stderrBuf.String() + return +} diff --git a/internal/sys/fs.go b/internal/sys/fs.go index e24f06bc70..4ace070d34 100644 --- a/internal/sys/fs.go +++ b/internal/sys/fs.go @@ -39,9 +39,26 @@ func (f *emptyFS) Open(name string) (fs.File, error) { // FileEntry maps a path to an open file in a file system. type FileEntry struct { + // Path was the argument to FSContext.OpenFile Path string + // File when nil this is the root "/" (fd=3) File fs.File + + // ReadDir is present when this File is a fs.ReadDirFile and `ReadDir` + // was called. + ReadDir *ReadDir +} + +// ReadDir is the status of a prior fs.ReadDirFile call. +type ReadDir struct { + // CountRead is the total count of files read including Entries. + CountRead uint64 + + // Entries is the contents of the last fs.ReadDirFile call. Notably, + // directory listing are not rewindable, so we keep entries around in case + // the caller mis-estimated their buffer and needs a few still cached. + Entries []fs.DirEntry } type FSContext struct {