diff --git a/internal/fastwalk/fastwalk.go b/internal/fastwalk/fastwalk.go deleted file mode 100644 index c40c7e93106..00000000000 --- a/internal/fastwalk/fastwalk.go +++ /dev/null @@ -1,196 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package fastwalk provides a faster version of [filepath.Walk] for file system -// scanning tools. -package fastwalk - -import ( - "errors" - "os" - "path/filepath" - "runtime" - "sync" -) - -// ErrTraverseLink is used as a return value from WalkFuncs to indicate that the -// symlink named in the call may be traversed. -var ErrTraverseLink = errors.New("fastwalk: traverse symlink, assuming target is a directory") - -// ErrSkipFiles is a used as a return value from WalkFuncs to indicate that the -// callback should not be called for any other files in the current directory. -// Child directories will still be traversed. -var ErrSkipFiles = errors.New("fastwalk: skip remaining files in directory") - -// Walk is a faster implementation of [filepath.Walk]. -// -// [filepath.Walk]'s design necessarily calls [os.Lstat] on each file, -// even if the caller needs less info. -// Many tools need only the type of each file. -// On some platforms, this information is provided directly by the readdir -// system call, avoiding the need to stat each file individually. -// fastwalk_unix.go contains a fork of the syscall routines. -// -// See golang.org/issue/16399. -// -// Walk walks the file tree rooted at root, calling walkFn for -// each file or directory in the tree, including root. -// -// If Walk returns [filepath.SkipDir], the directory is skipped. -// -// Unlike [filepath.Walk]: -// - file stat calls must be done by the user. -// The only provided metadata is the file type, which does not include -// any permission bits. -// - multiple goroutines stat the filesystem concurrently. The provided -// walkFn must be safe for concurrent use. -// - Walk can follow symlinks if walkFn returns the TraverseLink -// sentinel error. It is the walkFn's responsibility to prevent -// Walk from going into symlink cycles. -func Walk(root string, walkFn func(path string, typ os.FileMode) error) error { - // TODO(bradfitz): make numWorkers configurable? We used a - // minimum of 4 to give the kernel more info about multiple - // things we want, in hopes its I/O scheduling can take - // advantage of that. Hopefully most are in cache. Maybe 4 is - // even too low of a minimum. Profile more. - numWorkers := 4 - if n := runtime.NumCPU(); n > numWorkers { - numWorkers = n - } - - // Make sure to wait for all workers to finish, otherwise - // walkFn could still be called after returning. This Wait call - // runs after close(e.donec) below. - var wg sync.WaitGroup - defer wg.Wait() - - w := &walker{ - fn: walkFn, - enqueuec: make(chan walkItem, numWorkers), // buffered for performance - workc: make(chan walkItem, numWorkers), // buffered for performance - donec: make(chan struct{}), - - // buffered for correctness & not leaking goroutines: - resc: make(chan error, numWorkers), - } - defer close(w.donec) - - for i := 0; i < numWorkers; i++ { - wg.Add(1) - go w.doWork(&wg) - } - todo := []walkItem{{dir: root}} - out := 0 - for { - workc := w.workc - var workItem walkItem - if len(todo) == 0 { - workc = nil - } else { - workItem = todo[len(todo)-1] - } - select { - case workc <- workItem: - todo = todo[:len(todo)-1] - out++ - case it := <-w.enqueuec: - todo = append(todo, it) - case err := <-w.resc: - out-- - if err != nil { - return err - } - if out == 0 && len(todo) == 0 { - // It's safe to quit here, as long as the buffered - // enqueue channel isn't also readable, which might - // happen if the worker sends both another unit of - // work and its result before the other select was - // scheduled and both w.resc and w.enqueuec were - // readable. - select { - case it := <-w.enqueuec: - todo = append(todo, it) - default: - return nil - } - } - } - } -} - -// doWork reads directories as instructed (via workc) and runs the -// user's callback function. -func (w *walker) doWork(wg *sync.WaitGroup) { - defer wg.Done() - for { - select { - case <-w.donec: - return - case it := <-w.workc: - select { - case <-w.donec: - return - case w.resc <- w.walk(it.dir, !it.callbackDone): - } - } - } -} - -type walker struct { - fn func(path string, typ os.FileMode) error - - donec chan struct{} // closed on fastWalk's return - workc chan walkItem // to workers - enqueuec chan walkItem // from workers - resc chan error // from workers -} - -type walkItem struct { - dir string - callbackDone bool // callback already called; don't do it again -} - -func (w *walker) enqueue(it walkItem) { - select { - case w.enqueuec <- it: - case <-w.donec: - } -} - -func (w *walker) onDirEnt(dirName, baseName string, typ os.FileMode) error { - joined := dirName + string(os.PathSeparator) + baseName - if typ == os.ModeDir { - w.enqueue(walkItem{dir: joined}) - return nil - } - - err := w.fn(joined, typ) - if typ == os.ModeSymlink { - if err == ErrTraverseLink { - // Set callbackDone so we don't call it twice for both the - // symlink-as-symlink and the symlink-as-directory later: - w.enqueue(walkItem{dir: joined, callbackDone: true}) - return nil - } - if err == filepath.SkipDir { - // Permit SkipDir on symlinks too. - return nil - } - } - return err -} - -func (w *walker) walk(root string, runUserCallback bool) error { - if runUserCallback { - err := w.fn(root, os.ModeDir) - if err == filepath.SkipDir { - return nil - } - if err != nil { - return err - } - } - - return readDir(root, w.onDirEnt) -} diff --git a/internal/fastwalk/fastwalk_darwin.go b/internal/fastwalk/fastwalk_darwin.go deleted file mode 100644 index 0ca55e0d56f..00000000000 --- a/internal/fastwalk/fastwalk_darwin.go +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build darwin && cgo -// +build darwin,cgo - -package fastwalk - -/* -#include - -// fastwalk_readdir_r wraps readdir_r so that we don't have to pass a dirent** -// result pointer which triggers CGO's "Go pointer to Go pointer" check unless -// we allocat the result dirent* with malloc. -// -// fastwalk_readdir_r returns 0 on success, -1 upon reaching the end of the -// directory, or a positive error number to indicate failure. -static int fastwalk_readdir_r(DIR *fd, struct dirent *entry) { - struct dirent *result; - int ret = readdir_r(fd, entry, &result); - if (ret == 0 && result == NULL) { - ret = -1; // EOF - } - return ret; -} -*/ -import "C" - -import ( - "os" - "syscall" - "unsafe" -) - -func readDir(dirName string, fn func(dirName, entName string, typ os.FileMode) error) error { - fd, err := openDir(dirName) - if err != nil { - return &os.PathError{Op: "opendir", Path: dirName, Err: err} - } - defer C.closedir(fd) - - skipFiles := false - var dirent syscall.Dirent - for { - ret := int(C.fastwalk_readdir_r(fd, (*C.struct_dirent)(unsafe.Pointer(&dirent)))) - if ret != 0 { - if ret == -1 { - break // EOF - } - if ret == int(syscall.EINTR) { - continue - } - return &os.PathError{Op: "readdir", Path: dirName, Err: syscall.Errno(ret)} - } - if dirent.Ino == 0 { - continue - } - typ := dtToType(dirent.Type) - if skipFiles && typ.IsRegular() { - continue - } - name := (*[len(syscall.Dirent{}.Name)]byte)(unsafe.Pointer(&dirent.Name))[:] - name = name[:dirent.Namlen] - for i, c := range name { - if c == 0 { - name = name[:i] - break - } - } - // Check for useless names before allocating a string. - if string(name) == "." || string(name) == ".." { - continue - } - if err := fn(dirName, string(name), typ); err != nil { - if err != ErrSkipFiles { - return err - } - skipFiles = true - } - } - - return nil -} - -func dtToType(typ uint8) os.FileMode { - switch typ { - case syscall.DT_BLK: - return os.ModeDevice - case syscall.DT_CHR: - return os.ModeDevice | os.ModeCharDevice - case syscall.DT_DIR: - return os.ModeDir - case syscall.DT_FIFO: - return os.ModeNamedPipe - case syscall.DT_LNK: - return os.ModeSymlink - case syscall.DT_REG: - return 0 - case syscall.DT_SOCK: - return os.ModeSocket - } - return ^os.FileMode(0) -} - -// openDir wraps opendir(3) and handles any EINTR errors. The returned *DIR -// needs to be closed with closedir(3). -func openDir(path string) (*C.DIR, error) { - name, err := syscall.BytePtrFromString(path) - if err != nil { - return nil, err - } - for { - fd, err := C.opendir((*C.char)(unsafe.Pointer(name))) - if err != syscall.EINTR { - return fd, err - } - } -} diff --git a/internal/fastwalk/fastwalk_dirent_fileno.go b/internal/fastwalk/fastwalk_dirent_fileno.go deleted file mode 100644 index d58595dbd3f..00000000000 --- a/internal/fastwalk/fastwalk_dirent_fileno.go +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build freebsd || openbsd || netbsd -// +build freebsd openbsd netbsd - -package fastwalk - -import "syscall" - -func direntInode(dirent *syscall.Dirent) uint64 { - return uint64(dirent.Fileno) -} diff --git a/internal/fastwalk/fastwalk_dirent_ino.go b/internal/fastwalk/fastwalk_dirent_ino.go deleted file mode 100644 index d3922890b0b..00000000000 --- a/internal/fastwalk/fastwalk_dirent_ino.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build (linux || (darwin && !cgo)) && !appengine -// +build linux darwin,!cgo -// +build !appengine - -package fastwalk - -import "syscall" - -func direntInode(dirent *syscall.Dirent) uint64 { - return dirent.Ino -} diff --git a/internal/fastwalk/fastwalk_dirent_namlen_bsd.go b/internal/fastwalk/fastwalk_dirent_namlen_bsd.go deleted file mode 100644 index 38a4db6af3a..00000000000 --- a/internal/fastwalk/fastwalk_dirent_namlen_bsd.go +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build (darwin && !cgo) || freebsd || openbsd || netbsd -// +build darwin,!cgo freebsd openbsd netbsd - -package fastwalk - -import "syscall" - -func direntNamlen(dirent *syscall.Dirent) uint64 { - return uint64(dirent.Namlen) -} diff --git a/internal/fastwalk/fastwalk_dirent_namlen_linux.go b/internal/fastwalk/fastwalk_dirent_namlen_linux.go deleted file mode 100644 index c82e57df85e..00000000000 --- a/internal/fastwalk/fastwalk_dirent_namlen_linux.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build linux && !appengine -// +build linux,!appengine - -package fastwalk - -import ( - "bytes" - "syscall" - "unsafe" -) - -func direntNamlen(dirent *syscall.Dirent) uint64 { - const fixedHdr = uint16(unsafe.Offsetof(syscall.Dirent{}.Name)) - nameBuf := (*[unsafe.Sizeof(dirent.Name)]byte)(unsafe.Pointer(&dirent.Name[0])) - const nameBufLen = uint16(len(nameBuf)) - limit := dirent.Reclen - fixedHdr - if limit > nameBufLen { - limit = nameBufLen - } - nameLen := bytes.IndexByte(nameBuf[:limit], 0) - if nameLen < 0 { - panic("failed to find terminating 0 byte in dirent") - } - return uint64(nameLen) -} diff --git a/internal/fastwalk/fastwalk_portable.go b/internal/fastwalk/fastwalk_portable.go deleted file mode 100644 index 27e860243e1..00000000000 --- a/internal/fastwalk/fastwalk_portable.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build appengine || (!linux && !darwin && !freebsd && !openbsd && !netbsd) -// +build appengine !linux,!darwin,!freebsd,!openbsd,!netbsd - -package fastwalk - -import ( - "os" -) - -// readDir calls fn for each directory entry in dirName. -// It does not descend into directories or follow symlinks. -// If fn returns a non-nil error, readDir returns with that error -// immediately. -func readDir(dirName string, fn func(dirName, entName string, typ os.FileMode) error) error { - fis, err := os.ReadDir(dirName) - if err != nil { - return err - } - skipFiles := false - for _, fi := range fis { - info, err := fi.Info() - if err != nil { - return err - } - if info.Mode().IsRegular() && skipFiles { - continue - } - if err := fn(dirName, fi.Name(), info.Mode()&os.ModeType); err != nil { - if err == ErrSkipFiles { - skipFiles = true - continue - } - return err - } - } - return nil -} diff --git a/internal/fastwalk/fastwalk_test.go b/internal/fastwalk/fastwalk_test.go deleted file mode 100644 index b5c82bc5293..00000000000 --- a/internal/fastwalk/fastwalk_test.go +++ /dev/null @@ -1,251 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package fastwalk_test - -import ( - "bytes" - "flag" - "fmt" - "os" - "path/filepath" - "reflect" - "runtime" - "sort" - "strings" - "sync" - "testing" - - "golang.org/x/tools/internal/fastwalk" -) - -func formatFileModes(m map[string]os.FileMode) string { - var keys []string - for k := range m { - keys = append(keys, k) - } - sort.Strings(keys) - var buf bytes.Buffer - for _, k := range keys { - fmt.Fprintf(&buf, "%-20s: %v\n", k, m[k]) - } - return buf.String() -} - -func testFastWalk(t *testing.T, files map[string]string, callback func(path string, typ os.FileMode) error, want map[string]os.FileMode) { - tempdir, err := os.MkdirTemp("", "test-fast-walk") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tempdir) - - symlinks := map[string]string{} - for path, contents := range files { - file := filepath.Join(tempdir, "/src", path) - if err := os.MkdirAll(filepath.Dir(file), 0755); err != nil { - t.Fatal(err) - } - var err error - if strings.HasPrefix(contents, "LINK:") { - symlinks[file] = filepath.FromSlash(strings.TrimPrefix(contents, "LINK:")) - } else { - err = os.WriteFile(file, []byte(contents), 0644) - } - if err != nil { - t.Fatal(err) - } - } - - // Create symlinks after all other files. Otherwise, directory symlinks on - // Windows are unusable (see https://golang.org/issue/39183). - for file, dst := range symlinks { - err = os.Symlink(dst, file) - if err != nil { - if writeErr := os.WriteFile(file, []byte(dst), 0644); writeErr == nil { - // Couldn't create symlink, but could write the file. - // Probably this filesystem doesn't support symlinks. - // (Perhaps we are on an older Windows and not running as administrator.) - t.Skipf("skipping because symlinks appear to be unsupported: %v", err) - } - } - } - - got := map[string]os.FileMode{} - var mu sync.Mutex - err = fastwalk.Walk(tempdir, func(path string, typ os.FileMode) error { - mu.Lock() - defer mu.Unlock() - if !strings.HasPrefix(path, tempdir) { - t.Errorf("bogus prefix on %q, expect %q", path, tempdir) - } - key := filepath.ToSlash(strings.TrimPrefix(path, tempdir)) - if old, dup := got[key]; dup { - t.Errorf("callback called twice for key %q: %v -> %v", key, old, typ) - } - got[key] = typ - return callback(path, typ) - }) - - if err != nil { - t.Fatalf("callback returned: %v", err) - } - if !reflect.DeepEqual(got, want) { - t.Errorf("walk mismatch.\n got:\n%v\nwant:\n%v", formatFileModes(got), formatFileModes(want)) - } -} - -func TestFastWalk_Basic(t *testing.T) { - testFastWalk(t, map[string]string{ - "foo/foo.go": "one", - "bar/bar.go": "two", - "skip/skip.go": "skip", - }, - func(path string, typ os.FileMode) error { - return nil - }, - map[string]os.FileMode{ - "": os.ModeDir, - "/src": os.ModeDir, - "/src/bar": os.ModeDir, - "/src/bar/bar.go": 0, - "/src/foo": os.ModeDir, - "/src/foo/foo.go": 0, - "/src/skip": os.ModeDir, - "/src/skip/skip.go": 0, - }) -} - -func TestFastWalk_LongFileName(t *testing.T) { - longFileName := strings.Repeat("x", 255) - - testFastWalk(t, map[string]string{ - longFileName: "one", - }, - func(path string, typ os.FileMode) error { - return nil - }, - map[string]os.FileMode{ - "": os.ModeDir, - "/src": os.ModeDir, - "/src/" + longFileName: 0, - }, - ) -} - -func TestFastWalk_Symlink(t *testing.T) { - testFastWalk(t, map[string]string{ - "foo/foo.go": "one", - "bar/bar.go": "LINK:../foo/foo.go", - "symdir": "LINK:foo", - "broken/broken.go": "LINK:../nonexistent", - }, - func(path string, typ os.FileMode) error { - return nil - }, - map[string]os.FileMode{ - "": os.ModeDir, - "/src": os.ModeDir, - "/src/bar": os.ModeDir, - "/src/bar/bar.go": os.ModeSymlink, - "/src/foo": os.ModeDir, - "/src/foo/foo.go": 0, - "/src/symdir": os.ModeSymlink, - "/src/broken": os.ModeDir, - "/src/broken/broken.go": os.ModeSymlink, - }) -} - -func TestFastWalk_SkipDir(t *testing.T) { - testFastWalk(t, map[string]string{ - "foo/foo.go": "one", - "bar/bar.go": "two", - "skip/skip.go": "skip", - }, - func(path string, typ os.FileMode) error { - if typ == os.ModeDir && strings.HasSuffix(path, "skip") { - return filepath.SkipDir - } - return nil - }, - map[string]os.FileMode{ - "": os.ModeDir, - "/src": os.ModeDir, - "/src/bar": os.ModeDir, - "/src/bar/bar.go": 0, - "/src/foo": os.ModeDir, - "/src/foo/foo.go": 0, - "/src/skip": os.ModeDir, - }) -} - -func TestFastWalk_SkipFiles(t *testing.T) { - // Directory iteration order is undefined, so there's no way to know - // which file to expect until the walk happens. Rather than mess - // with the test infrastructure, just mutate want. - var mu sync.Mutex - want := map[string]os.FileMode{ - "": os.ModeDir, - "/src": os.ModeDir, - "/src/zzz": os.ModeDir, - "/src/zzz/c.go": 0, - } - - testFastWalk(t, map[string]string{ - "a_skipfiles.go": "a", - "b_skipfiles.go": "b", - "zzz/c.go": "c", - }, - func(path string, typ os.FileMode) error { - if strings.HasSuffix(path, "_skipfiles.go") { - mu.Lock() - defer mu.Unlock() - want["/src/"+filepath.Base(path)] = 0 - return fastwalk.ErrSkipFiles - } - return nil - }, - want) - if len(want) != 5 { - t.Errorf("saw too many files: wanted 5, got %v (%v)", len(want), want) - } -} - -func TestFastWalk_TraverseSymlink(t *testing.T) { - testFastWalk(t, map[string]string{ - "foo/foo.go": "one", - "bar/bar.go": "two", - "skip/skip.go": "skip", - "symdir": "LINK:foo", - }, - func(path string, typ os.FileMode) error { - if typ == os.ModeSymlink { - return fastwalk.ErrTraverseLink - } - return nil - }, - map[string]os.FileMode{ - "": os.ModeDir, - "/src": os.ModeDir, - "/src/bar": os.ModeDir, - "/src/bar/bar.go": 0, - "/src/foo": os.ModeDir, - "/src/foo/foo.go": 0, - "/src/skip": os.ModeDir, - "/src/skip/skip.go": 0, - "/src/symdir": os.ModeSymlink, - "/src/symdir/foo.go": 0, - }) -} - -var benchDir = flag.String("benchdir", runtime.GOROOT(), "The directory to scan for BenchmarkFastWalk") - -func BenchmarkFastWalk(b *testing.B) { - b.ReportAllocs() - for i := 0; i < b.N; i++ { - err := fastwalk.Walk(*benchDir, func(path string, typ os.FileMode) error { return nil }) - if err != nil { - b.Fatal(err) - } - } -} diff --git a/internal/fastwalk/fastwalk_unix.go b/internal/fastwalk/fastwalk_unix.go deleted file mode 100644 index f12f1a734cc..00000000000 --- a/internal/fastwalk/fastwalk_unix.go +++ /dev/null @@ -1,153 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build (linux || freebsd || openbsd || netbsd || (darwin && !cgo)) && !appengine -// +build linux freebsd openbsd netbsd darwin,!cgo -// +build !appengine - -package fastwalk - -import ( - "fmt" - "os" - "syscall" - "unsafe" -) - -const blockSize = 8 << 10 - -// unknownFileMode is a sentinel (and bogus) os.FileMode -// value used to represent a syscall.DT_UNKNOWN Dirent.Type. -const unknownFileMode os.FileMode = os.ModeNamedPipe | os.ModeSocket | os.ModeDevice - -func readDir(dirName string, fn func(dirName, entName string, typ os.FileMode) error) error { - fd, err := open(dirName, 0, 0) - if err != nil { - return &os.PathError{Op: "open", Path: dirName, Err: err} - } - defer syscall.Close(fd) - - // The buffer must be at least a block long. - buf := make([]byte, blockSize) // stack-allocated; doesn't escape - bufp := 0 // starting read position in buf - nbuf := 0 // end valid data in buf - skipFiles := false - for { - if bufp >= nbuf { - bufp = 0 - nbuf, err = readDirent(fd, buf) - if err != nil { - return os.NewSyscallError("readdirent", err) - } - if nbuf <= 0 { - return nil - } - } - consumed, name, typ := parseDirEnt(buf[bufp:nbuf]) - bufp += consumed - if name == "" || name == "." || name == ".." { - continue - } - // Fallback for filesystems (like old XFS) that don't - // support Dirent.Type and have DT_UNKNOWN (0) there - // instead. - if typ == unknownFileMode { - fi, err := os.Lstat(dirName + "/" + name) - if err != nil { - // It got deleted in the meantime. - if os.IsNotExist(err) { - continue - } - return err - } - typ = fi.Mode() & os.ModeType - } - if skipFiles && typ.IsRegular() { - continue - } - if err := fn(dirName, name, typ); err != nil { - if err == ErrSkipFiles { - skipFiles = true - continue - } - return err - } - } -} - -func parseDirEnt(buf []byte) (consumed int, name string, typ os.FileMode) { - // golang.org/issue/37269 - dirent := &syscall.Dirent{} - copy((*[unsafe.Sizeof(syscall.Dirent{})]byte)(unsafe.Pointer(dirent))[:], buf) - if v := unsafe.Offsetof(dirent.Reclen) + unsafe.Sizeof(dirent.Reclen); uintptr(len(buf)) < v { - panic(fmt.Sprintf("buf size of %d smaller than dirent header size %d", len(buf), v)) - } - if len(buf) < int(dirent.Reclen) { - panic(fmt.Sprintf("buf size %d < record length %d", len(buf), dirent.Reclen)) - } - consumed = int(dirent.Reclen) - if direntInode(dirent) == 0 { // File absent in directory. - return - } - switch dirent.Type { - case syscall.DT_REG: - typ = 0 - case syscall.DT_DIR: - typ = os.ModeDir - case syscall.DT_LNK: - typ = os.ModeSymlink - case syscall.DT_BLK: - typ = os.ModeDevice - case syscall.DT_FIFO: - typ = os.ModeNamedPipe - case syscall.DT_SOCK: - typ = os.ModeSocket - case syscall.DT_UNKNOWN: - typ = unknownFileMode - default: - // Skip weird things. - // It's probably a DT_WHT (http://lwn.net/Articles/325369/) - // or something. Revisit if/when this package is moved outside - // of goimports. goimports only cares about regular files, - // symlinks, and directories. - return - } - - nameBuf := (*[unsafe.Sizeof(dirent.Name)]byte)(unsafe.Pointer(&dirent.Name[0])) - nameLen := direntNamlen(dirent) - - // Special cases for common things: - if nameLen == 1 && nameBuf[0] == '.' { - name = "." - } else if nameLen == 2 && nameBuf[0] == '.' && nameBuf[1] == '.' { - name = ".." - } else { - name = string(nameBuf[:nameLen]) - } - return -} - -// According to https://golang.org/doc/go1.14#runtime -// A consequence of the implementation of preemption is that on Unix systems, including Linux and macOS -// systems, programs built with Go 1.14 will receive more signals than programs built with earlier releases. -// -// This causes syscall.Open and syscall.ReadDirent sometimes fail with EINTR errors. -// We need to retry in this case. -func open(path string, mode int, perm uint32) (fd int, err error) { - for { - fd, err := syscall.Open(path, mode, perm) - if err != syscall.EINTR { - return fd, err - } - } -} - -func readDirent(fd int, buf []byte) (n int, err error) { - for { - nbuf, err := syscall.ReadDirent(fd, buf) - if err != syscall.EINTR { - return nbuf, err - } - } -} diff --git a/internal/gopathwalk/walk.go b/internal/gopathwalk/walk.go index 452e342c559..bca83d1a11a 100644 --- a/internal/gopathwalk/walk.go +++ b/internal/gopathwalk/walk.go @@ -9,13 +9,12 @@ package gopathwalk import ( "bufio" "bytes" + "io/fs" "log" "os" "path/filepath" "strings" "time" - - "golang.org/x/tools/internal/fastwalk" ) // Options controls the behavior of a Walk call. @@ -78,14 +77,25 @@ func walkDir(root Root, add func(Root, string), skip func(root Root, dir string) if opts.Logf != nil { opts.Logf("scanning %s", root.Path) } + w := &walker{ - root: root, - add: add, - skip: skip, - opts: opts, + root: root, + add: add, + skip: skip, + opts: opts, + added: make(map[string]bool), } w.init() - if err := fastwalk.Walk(root.Path, w.walk); err != nil { + + // Add a trailing path separator to cause filepath.WalkDir to traverse symlinks. + path := root.Path + if len(path) == 0 { + path = "." + string(filepath.Separator) + } else if !os.IsPathSeparator(path[len(path)-1]) { + path = path + string(filepath.Separator) + } + + if err := filepath.WalkDir(path, w.walk); err != nil { logf := opts.Logf if logf == nil { logf = log.Printf @@ -106,6 +116,8 @@ type walker struct { opts Options // Options passed to Walk by the user. ignoredDirs []os.FileInfo // The ignored directories, loaded from .goimportsignore files. + + added map[string]bool } // init initializes the walker based on its Options @@ -164,6 +176,13 @@ func (w *walker) getIgnoredDirs(path string) []string { // shouldSkipDir reports whether the file should be skipped or not. func (w *walker) shouldSkipDir(fi os.FileInfo, dir string) bool { for _, ignoredDir := range w.ignoredDirs { + // TODO(bcmills): Given that we already ought to be preserving the + // user-provided paths for directories encountered via symlinks, and that we + // don't expect GOROOT/src or GOPATH/src to itself contain symlinks, + // os.SameFile seems needlessly expensive here — it forces the caller to + // obtain an os.FileInfo instead of a potentially much cheaper fs.DirEntry. + // + // Can we drop this and use a lexical comparison instead? if os.SameFile(fi, ignoredDir) { return true } @@ -176,20 +195,25 @@ func (w *walker) shouldSkipDir(fi os.FileInfo, dir string) bool { } // walk walks through the given path. -func (w *walker) walk(path string, typ os.FileMode) error { +func (w *walker) walk(path string, d fs.DirEntry, err error) error { + typ := d.Type() if typ.IsRegular() { + if !strings.HasSuffix(path, ".go") { + return nil + } + dir := filepath.Dir(path) if dir == w.root.Path && (w.root.Type == RootGOROOT || w.root.Type == RootGOPATH) { // Doesn't make sense to have regular files // directly in your $GOPATH/src or $GOROOT/src. - return fastwalk.ErrSkipFiles - } - if !strings.HasSuffix(path, ".go") { return nil } - w.add(w.root, dir) - return fastwalk.ErrSkipFiles + if !w.added[dir] { + w.add(w.root, dir) + w.added[dir] = true + } + return nil } if typ == os.ModeDir { base := filepath.Base(path) @@ -199,20 +223,67 @@ func (w *walker) walk(path string, typ os.FileMode) error { (!w.opts.ModulesEnabled && base == "node_modules") { return filepath.SkipDir } - fi, err := os.Lstat(path) + fi, err := d.Info() if err == nil && w.shouldSkipDir(fi, path) { return filepath.SkipDir } return nil } - if typ == os.ModeSymlink { + if typ == os.ModeSymlink && err == nil { + // TODO(bcmills): 'go list all' itself ignores symlinks within GOROOT/src + // and GOPATH/src. Do we really need to traverse them here? If so, why? + + if os.IsPathSeparator(path[len(path)-1]) { + // The OS was supposed to resolve a directory symlink but didn't. + // + // On macOS this may be caused by a known libc/kernel bug; + // see https://go.dev/issue/59586. + // + // On Windows before Go 1.21, this may be caused by a bug in + // os.Lstat (fixed in https://go.dev/cl/463177). + // + // In either case, we can work around the bug by walking this level + // explicitly: first the symlink target itself, then its contents. + + fi, err := os.Stat(path) + if err != nil || !fi.IsDir() { + return nil + } + err = w.walk(path, fs.FileInfoToDirEntry(fi), nil) + if err == filepath.SkipDir { + return nil + } else if err != nil { + return err + } + + ents, _ := os.ReadDir(path) // ignore error if unreadable + for _, d := range ents { + nextPath := filepath.Join(path, d.Name()) + var err error + if d.IsDir() { + err = filepath.WalkDir(nextPath, w.walk) + } else { + err = w.walk(nextPath, d, nil) + if err == filepath.SkipDir { + break + } + } + if err != nil { + return err + } + } + return nil + } + base := filepath.Base(path) if strings.HasPrefix(base, ".#") { // Emacs noise. return nil } if w.shouldTraverse(path) { - return fastwalk.ErrTraverseLink + // Add a trailing separator to traverse the symlink. + nextPath := path + string(filepath.Separator) + return filepath.WalkDir(nextPath, w.walk) } } return nil