Skip to content

Commit 96516c2

Browse files
charlieviethcharlievieth
committed
WIP: add support for fs.SkipAll
NB: This seems to impact performace by 5-8% on Mac/Linux. Need to do more benchmarking to see if this is really the case.
1 parent 940d64f commit 96516c2

File tree

5 files changed

+222
-11
lines changed

5 files changed

+222
-11
lines changed

‎fastwalk.go

Lines changed: 17 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -60,9 +60,10 @@ var ErrSkipFiles = errors.New("fastwalk: skip remaining files in directory")
6060
// as an error by any function.
6161
var SkipDir = fs.SkipDir
6262

63-
// TODO(charlie): Look into implementing the fs.SkipAll behavior of
64-
// filepath.Walk and filepath.WalkDir. This may not be possible without taking
65-
// a performance hit.
63+
// SkipAll is used as a return value from [WalkDirFunc] to indicate that
64+
// all remaining files and directories are to be skipped. It is not returned
65+
// as an error by any function.
66+
var SkipAll = fs.SkipAll
6667

6768
// DefaultNumWorkers returns the default number of worker goroutines to use in
6869
// [Walk] and is the value of [runtime.GOMAXPROCS](-1) clamped to a range
@@ -577,13 +578,18 @@ func (w *walker) joinPaths(dir, base string) string {
577578

578579
func (w *walker) onDirEnt(dirName, baseName string, de DirEntry) error {
579580
joined := w.joinPaths(dirName, baseName)
581+
err := w.fn(joined, de, nil)
580582
typ := de.Type()
581583
if typ == os.ModeDir {
582-
w.enqueue(walkItem{dir: joined, info: de})
584+
if err != nil {
585+
if err == SkipDir {
586+
return nil
587+
}
588+
return err // May be SkipAll
589+
}
590+
w.enqueue(walkItem{dir: joined, info: de, callbackDone: true})
583591
return nil
584592
}
585-
586-
err := w.fn(joined, de, nil)
587593
if typ == os.ModeSymlink {
588594
if err == ErrTraverseLink {
589595
if !w.follow {
@@ -594,8 +600,8 @@ func (w *walker) onDirEnt(dirName, baseName string, de DirEntry) error {
594600
}
595601
err = nil // Ignore ErrTraverseLink when Follow is true.
596602
}
597-
if err == filepath.SkipDir {
598-
// Permit SkipDir on symlinks too.
603+
if err == SkipDir {
604+
// Permit SkipDir and SkipAll on symlinks too.
599605
return nil
600606
}
601607
if err == nil && w.follow && w.shouldTraverse(joined, de) {
@@ -609,10 +615,10 @@ func (w *walker) onDirEnt(dirName, baseName string, de DirEntry) error {
609615
func (w *walker) walk(root string, info DirEntry, runUserCallback bool) error {
610616
if runUserCallback {
611617
err := w.fn(root, info, nil)
612-
if err == filepath.SkipDir {
613-
return nil
614-
}
615618
if err != nil {
619+
if err == SkipDir || err == SkipAll {
620+
return nil
621+
}
616622
return err
617623
}
618624
}

‎fastwalk_darwin.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,9 @@ func (w *walker) readDir(dirName string) (err error) {
7171
de := newUnixDirent(dirName, nm, typ)
7272
if w.sortMode == SortNone {
7373
if err := w.onDirEnt(dirName, nm, de); err != nil {
74+
if err == SkipAll {
75+
return nil
76+
}
7477
if err != ErrSkipFiles {
7578
return err
7679
}
@@ -92,6 +95,9 @@ func (w *walker) readDir(dirName string) (err error) {
9295
continue
9396
}
9497
if err := w.onDirEnt(dirName, d.Name(), d); err != nil {
98+
if err == SkipAll {
99+
return nil
100+
}
95101
if err != ErrSkipFiles {
96102
return err
97103
}

‎fastwalk_portable.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,9 @@ func (w *walker) readDir(dirName string) error {
3636
e := newDirEntry(dirName, d)
3737
if w.sortMode == SortNone {
3838
if err := w.onDirEnt(dirName, d.Name(), e); err != nil {
39+
if err == SkipAll {
40+
return nil
41+
}
3942
if err != ErrSkipFiles {
4043
return err
4144
}
@@ -57,6 +60,9 @@ func (w *walker) readDir(dirName string) error {
5760
continue
5861
}
5962
if err := w.onDirEnt(dirName, d.Name(), d); err != nil {
63+
if err == SkipAll {
64+
return nil
65+
}
6066
if err != ErrSkipFiles {
6167
return err
6268
}

‎fastwalk_test.go

Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -444,6 +444,60 @@ func TestFastWalk_DirEntryType(t *testing.T) {
444444
})
445445
}
446446

447+
func TestFastWalk_DirEntryStat(t *testing.T) {
448+
testFastWalk(t, map[string]string{
449+
"foo/foo.go": "one",
450+
"bar/bar.go": "LINK:../foo/foo.go",
451+
"symdir": "LINK:foo",
452+
},
453+
func(path string, d fs.DirEntry, err error) error {
454+
requireNoError(t, err)
455+
de := d.(fastwalk.DirEntry)
456+
if _, ok := de.(fastwalk.DirEntry); !ok {
457+
t.Errorf("%q: not a fastwalk.DirEntry: %T", path, de)
458+
}
459+
ls1, err := os.Lstat(path)
460+
if err != nil {
461+
t.Error(err)
462+
}
463+
ls2, err := de.Info()
464+
if err != nil {
465+
t.Error(err)
466+
}
467+
if !os.SameFile(ls1, ls2) {
468+
t.Errorf("Info(%q) = %v; want: %v", path, ls2, ls1)
469+
}
470+
st1, err := os.Stat(path)
471+
if err != nil {
472+
t.Error(err)
473+
}
474+
st2, err := de.Stat()
475+
if err != nil {
476+
t.Error(err)
477+
}
478+
if !os.SameFile(st1, st2) {
479+
t.Errorf("Stat(%q) = %v; want: %v", path, st2, st1)
480+
}
481+
if de.Name() != filepath.Base(path) {
482+
t.Errorf("Name() = %q; want: %q", de.Name(), filepath.Base(path))
483+
}
484+
if de.Type() != de.Type().Type() {
485+
t.Errorf("%s: type mismatch got: %q want: %q",
486+
path, de.Type(), de.Type().Type())
487+
}
488+
return nil
489+
},
490+
map[string]os.FileMode{
491+
"": os.ModeDir,
492+
"/src": os.ModeDir,
493+
"/src/bar": os.ModeDir,
494+
"/src/bar/bar.go": os.ModeSymlink,
495+
"/src/foo": os.ModeDir,
496+
"/src/foo/foo.go": 0,
497+
"/src/symdir": os.ModeSymlink,
498+
})
499+
}
500+
447501
func TestFastWalk_SkipDir(t *testing.T) {
448502
test := func(t *testing.T, mode fastwalk.SortMode) {
449503
conf := fastwalk.DefaultConfig.Copy()
@@ -485,6 +539,28 @@ func TestFastWalk_SkipDir(t *testing.T) {
485539
}
486540
}
487541

542+
// Test that returning SkipDir for the root directory aborts the walk
543+
func TestFastWalk_SkipDir_Root(t *testing.T) {
544+
want := map[string]os.FileMode{
545+
"": os.ModeDir,
546+
}
547+
conf := fastwalk.DefaultConfig.Copy()
548+
conf.Sort = fastwalk.SortLexical // Needed for ordering
549+
testFastWalkConf(t, conf, map[string]string{
550+
"a.go": "a",
551+
"b.go": "b",
552+
},
553+
func(path string, de fs.DirEntry, err error) error {
554+
requireNoError(t, err)
555+
return fastwalk.SkipDir
556+
},
557+
want)
558+
if len(want) != 1 {
559+
t.Errorf("invalid number of files visited: wanted 1, got %v (%q)",
560+
len(want), want)
561+
}
562+
}
563+
488564
func TestFastWalk_SkipFiles(t *testing.T) {
489565
mapKeys := func(m map[string]os.FileMode) []string {
490566
a := make([]string, 0, len(m))
@@ -542,6 +618,117 @@ func TestFastWalk_SkipFiles(t *testing.T) {
542618
}
543619
}
544620

621+
func TestFastWalk_SkipAll(t *testing.T) {
622+
mapKeys := func(m map[string]os.FileMode) []string {
623+
a := make([]string, 0, len(m))
624+
for k := range m {
625+
a = append(a, k)
626+
}
627+
return a
628+
}
629+
630+
t.Run("Root", func(t *testing.T) {
631+
want := map[string]os.FileMode{
632+
"": os.ModeDir,
633+
}
634+
conf := fastwalk.DefaultConfig.Copy()
635+
conf.Sort = fastwalk.SortLexical // Needed for ordering
636+
testFastWalkConf(t, conf, map[string]string{
637+
"a.go": "a",
638+
"b.go": "b",
639+
},
640+
func(path string, de fs.DirEntry, err error) error {
641+
requireNoError(t, err)
642+
return fastwalk.SkipAll
643+
},
644+
want)
645+
if len(want) != 1 {
646+
t.Errorf("invalid number of files visited: wanted 1, got %v (%q)",
647+
len(want), mapKeys(want))
648+
}
649+
})
650+
651+
t.Run("File", func(t *testing.T) {
652+
want := map[string]os.FileMode{
653+
"": os.ModeDir,
654+
"/src": os.ModeDir,
655+
"/src/a.go": 0,
656+
}
657+
conf := fastwalk.DefaultConfig.Copy()
658+
conf.Sort = fastwalk.SortLexical // Needed for ordering
659+
testFastWalkConf(t, conf, map[string]string{
660+
"a.go": "a",
661+
"b.go": "b",
662+
},
663+
func(path string, de fs.DirEntry, err error) error {
664+
requireNoError(t, err)
665+
if de.Name() == "a.go" {
666+
return fastwalk.SkipAll
667+
}
668+
return nil
669+
},
670+
want)
671+
if len(want) != 3 {
672+
t.Errorf("invalid number of files visited: wanted 3, got %v (%q)",
673+
len(want), mapKeys(want))
674+
}
675+
})
676+
677+
t.Run("Directory", func(t *testing.T) {
678+
want := map[string]os.FileMode{
679+
"": os.ModeDir,
680+
"/src": os.ModeDir,
681+
"/src/dir1": os.ModeDir,
682+
}
683+
conf := fastwalk.DefaultConfig.Copy()
684+
conf.Sort = fastwalk.SortDirsFirst // Needed for ordering
685+
testFastWalkConf(t, conf, map[string]string{
686+
"dir1/a.go": "a",
687+
"dir2/a.go": "a",
688+
},
689+
func(path string, de fs.DirEntry, err error) error {
690+
requireNoError(t, err)
691+
if de.Name() == "dir1" {
692+
return fastwalk.SkipAll
693+
}
694+
return nil
695+
},
696+
want)
697+
if len(want) != 3 {
698+
t.Errorf("invalid number of files visited: wanted 3, got %v (%q)",
699+
len(want), mapKeys(want))
700+
}
701+
})
702+
703+
t.Run("Symlink", func(t *testing.T) {
704+
want := map[string]os.FileMode{
705+
"": os.ModeDir,
706+
"/src": os.ModeDir,
707+
"/src/a.go": 0,
708+
"/src/symdir": os.ModeSymlink,
709+
}
710+
conf := fastwalk.DefaultConfig.Copy()
711+
conf.Sort = fastwalk.SortFilesFirst // Needed for ordering
712+
testFastWalkConf(t, conf, map[string]string{
713+
"a.go": "a",
714+
"foo/foo.go": "one",
715+
"symdir": "LINK:foo",
716+
},
717+
func(path string, de fs.DirEntry, err error) error {
718+
requireNoError(t, err)
719+
if de.Type()&fs.ModeSymlink != 0 {
720+
return fastwalk.SkipAll
721+
}
722+
return nil
723+
},
724+
want)
725+
if len(want) != 4 {
726+
t.Errorf("invalid number of files visited: wanted 4, got %v (%q)",
727+
len(want), mapKeys(want))
728+
}
729+
})
730+
}
731+
545732
func TestFastWalk_TraverseSymlink(t *testing.T) {
546733
testFastWalk(t, map[string]string{
547734
"foo/foo.go": "one",

‎fastwalk_unix.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,9 @@ func (w *walker) readDir(dirName string) error {
7575
de := newUnixDirent(dirName, name, typ)
7676
if w.sortMode == SortNone {
7777
if err := w.onDirEnt(dirName, name, de); err != nil {
78+
if err == SkipAll {
79+
return nil
80+
}
7881
if err == ErrSkipFiles {
7982
skipFiles = true
8083
continue
@@ -97,6 +100,9 @@ func (w *walker) readDir(dirName string) error {
97100
continue
98101
}
99102
if err := w.onDirEnt(dirName, d.Name(), d); err != nil {
103+
if err == SkipAll {
104+
return nil
105+
}
100106
if err != ErrSkipFiles {
101107
return err
102108
}

0 commit comments

Comments
 (0)