From aedb63b2796c18566525300f477cab49f27181ad Mon Sep 17 00:00:00 2001 From: Caleb Brown Date: Tue, 28 Feb 2023 09:45:45 +1100 Subject: [PATCH] Improve input iteration by adding more tests. (#336) * Improve input iteration by adding more tests. Signed-off-by: Caleb Brown * Check the errors from inputiter.New during tests. Signed-off-by: Caleb Brown * Figure out what type the Windows file error is. Signed-off-by: Caleb Brown * Add windows-specific code to handle invalid filename errors. Signed-off-by: Caleb Brown * Add missing errors import Signed-off-by: Caleb Brown * Remove ErrInvalid as it is not normally returned. Signed-off-by: Caleb Brown * Add copyright notices. Signed-off-by: Caleb Brown * Close files opened in tests so they can be cleaned up properly. Signed-off-by: Caleb Brown * After creating the filename, move to a different dir. Signed-off-by: Caleb Brown * Skip tests that fail on Windows. Signed-off-by: Caleb Brown * Minor clean-up and test improvements. Signed-off-by: Caleb Brown * Add some comments to clarify why we Getwd + Chdir Signed-off-by: Caleb Brown * Split unit-test and scorecard-version check into separate workflow jobs. Signed-off-by: Caleb Brown --------- Signed-off-by: Caleb Brown --- .github/workflows/ci.yml | 15 +- cmd/criticality_score/input.go | 139 -------------- .../inputiter/err_windows.go | 30 +++ cmd/criticality_score/inputiter/iterator.go | 95 ++++++++++ .../inputiter/iterator_test.go | 172 ++++++++++++++++++ cmd/criticality_score/inputiter/new.go | 84 +++++++++ cmd/criticality_score/inputiter/new_test.go | 163 +++++++++++++++++ cmd/criticality_score/main.go | 11 +- cmd/enumerate_github/marker/type_test.go | 2 + cmd/enumerate_github/marker/write_test.go | 2 + go.mod | 2 +- go.sum | 2 + go.work.sum | 24 +-- internal/cloudstorage/cloudstorage.go | 2 +- internal/cloudstorage/cloudstorage_test.go | 10 +- internal/outfile/outfile_test.go | 14 +- 16 files changed, 598 insertions(+), 169 deletions(-) delete mode 100644 cmd/criticality_score/input.go create mode 100644 cmd/criticality_score/inputiter/err_windows.go create mode 100644 cmd/criticality_score/inputiter/iterator.go create mode 100644 cmd/criticality_score/inputiter/iterator_test.go create mode 100644 cmd/criticality_score/inputiter/new.go create mode 100644 cmd/criticality_score/inputiter/new_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 29a5348d9..9baa6b8f6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,14 +10,17 @@ permissions: read-all jobs: run-tests: - runs-on: ubuntu-latest + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 with: go-version: 1.19 - name: Run tests - run: make test + run: make test/unit env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run-linter: @@ -29,3 +32,11 @@ jobs: go-version: 1.19 - name: golangci-lint uses: golangci/golangci-lint-action@08e2f20817b15149a52b5b3ebe7de50aff2ba8c5 + check-scorecard-version: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c + - name: Run tests + run: make test/scorecard-version + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/cmd/criticality_score/input.go b/cmd/criticality_score/input.go deleted file mode 100644 index abd4a21ac..000000000 --- a/cmd/criticality_score/input.go +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright 2022 Criticality Score Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "bufio" - "context" - "errors" - "io" - "net/url" - "os" - - "github.com/ossf/criticality_score/internal/infile" -) - -// iter is a simple interface for iterating across a list of items. -// -// This interface is modeled on the bufio.Scanner behavior. -type iter[T any] interface { - // Item returns the current item in the iterator - Item() T - - // Next advances the iterator to the next item and returns true if there is - // an item to consume, and false if the end of the input has been reached, - // or there has been an error. - // - // Next must be called before each call to Item. - Next() bool - - // Err returns any error produced while iterating. - Err() error -} - -// iterCloser is an iter, but also embeds the io.Closer interface, so it can be -// used to wrap a file for iterating through. -type iterCloser[T any] interface { - iter[T] - io.Closer -} - -// scannerIter implements iter using a bufio.Scanner to iterate through lines in -// a file. -type scannerIter struct { - r io.ReadCloser - scanner *bufio.Scanner -} - -func (i *scannerIter) Item() string { - return i.scanner.Text() -} - -func (i *scannerIter) Next() bool { - return i.scanner.Scan() -} - -func (i *scannerIter) Err() error { - return i.scanner.Err() -} - -func (i *scannerIter) Close() error { - return i.r.Close() -} - -// sliceIter implements iter using a slice for iterating. -type sliceIter[T any] struct { - values []T - next int - size int -} - -func (i *sliceIter[T]) Item() T { - return i.values[i.next-1] -} - -func (i *sliceIter[T]) Next() bool { - if i.next <= i.size { - i.next++ - } - return i.next <= i.size -} - -func (i *sliceIter[T]) Err() error { - return nil -} - -func (i *sliceIter[T]) Close() error { - return nil -} - -// initInput returns an iterCloser for iterating across repositories for -// collecting signals. -// -// If only one arg is specified, the code will treat it as a file and attempt to -// open it. If the file doesn't exist, and is parseable as a URL the arg will be -// treated as a repo. -// -// If more than one arg is specified they are all considered to be repos. -// -// TODO: support the ability to force args to be interpreted as either a file, -// or a list of repos. -func initInput(args []string) (iterCloser[string], error) { - if len(args) == 1 { - // If there is 1 arg, attempt to open it as a file. - fileOrRepo := args[0] - _, err := url.Parse(fileOrRepo) - urlParseFailed := err != nil - - // Open the in-file for reading - r, err := infile.Open(context.Background(), fileOrRepo) - if err == nil { - return &scannerIter{ - r: r, - scanner: bufio.NewScanner(r), - }, nil - } else if err != nil && (urlParseFailed || !errors.Is(err, os.ErrNotExist)) { - // Only report errors if the file doesn't appear to be a URL, or if - // it doesn't exist. - return nil, err - } - } - // If file loading failed, or there are 2 or more args, treat args as a list - // of repos. - return &sliceIter[string]{ - size: len(args), - values: args, - }, nil -} diff --git a/cmd/criticality_score/inputiter/err_windows.go b/cmd/criticality_score/inputiter/err_windows.go new file mode 100644 index 000000000..bb047bd7d --- /dev/null +++ b/cmd/criticality_score/inputiter/err_windows.go @@ -0,0 +1,30 @@ +// Copyright 2022 Criticality Score Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build windows + +package inputiter + +import ( + "errors" + + "golang.org/x/sys/windows" +) + +func init() { + // Windows can return an additional error number when there is a failure. + osErrorWithFilename = func(err error) bool { + return errors.Is(err, windows.ERROR_INVALID_NAME) + } +} diff --git a/cmd/criticality_score/inputiter/iterator.go b/cmd/criticality_score/inputiter/iterator.go new file mode 100644 index 000000000..577d21a67 --- /dev/null +++ b/cmd/criticality_score/inputiter/iterator.go @@ -0,0 +1,95 @@ +// Copyright 2022 Criticality Score Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package inputiter + +import ( + "bufio" + "io" +) + +// Iter is a simple interface for iterating across a list of items. +// +// This interface is modeled on the bufio.Scanner behavior. +type Iter[T any] interface { + // Item returns the current item in the iterator. + // + // Next() must be called before calling Item(). + Item() T + + // Next advances the iterator to the next item and returns true if there is + // an item to consume, and false if the end of the input has been reached, + // or there has been an error. + // + // Next must be called before each call to Item. + Next() bool + + // Err returns any error produced while iterating. + Err() error +} + +// IterCloser is an iter, but also embeds the io.Closer interface, so it can be +// used to wrap a file for iterating through. +type IterCloser[T any] interface { + Iter[T] + io.Closer +} + +// scannerIter implements Iter[string] using a bufio.Scanner to iterate through +// lines in a file. +type scannerIter struct { + c io.Closer + scanner *bufio.Scanner +} + +func (i *scannerIter) Item() string { + return i.scanner.Text() +} + +func (i *scannerIter) Next() bool { + return i.scanner.Scan() +} + +func (i *scannerIter) Err() error { + return i.scanner.Err() +} + +func (i *scannerIter) Close() error { + return i.c.Close() +} + +// sliceIter implements iter using a slice for iterating. +type sliceIter[T any] struct { + values []T + next int +} + +func (i *sliceIter[T]) Item() T { + return i.values[i.next-1] +} + +func (i *sliceIter[T]) Next() bool { + if i.next <= len(i.values) { + i.next++ + } + return i.next <= len(i.values) +} + +func (i *sliceIter[T]) Err() error { + return nil +} + +func (i *sliceIter[T]) Close() error { + return nil +} diff --git a/cmd/criticality_score/inputiter/iterator_test.go b/cmd/criticality_score/inputiter/iterator_test.go new file mode 100644 index 000000000..1e92ded18 --- /dev/null +++ b/cmd/criticality_score/inputiter/iterator_test.go @@ -0,0 +1,172 @@ +// Copyright 2022 Criticality Score Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package inputiter + +import ( + "bufio" + "bytes" + "errors" + "io" + "strings" + "testing" + "testing/iotest" + + "golang.org/x/exp/slices" +) + +func TestSliceIter_Empty(t *testing.T) { + i := &sliceIter[int]{ + values: []int{}, + } + + if got := i.Next(); got { + t.Errorf("Next() = %v; want false", got) + } +} + +func TestSliceIter_SingleEntry(t *testing.T) { + want := 42 + i := &sliceIter[int]{ + values: []int{want}, + } + + if got := i.Next(); !got { + t.Errorf("Next() = %v; want true", got) + } + if got := i.Item(); got != want { + t.Errorf("Item() = %v; want %v", got, want) + } + if got := i.Next(); got { + t.Errorf("Next()#2 = %v; want false", got) + } +} + +func TestSliceIter_MultiEntry(t *testing.T) { + want := []int{1, 2, 3, 42, 1337} + i := &sliceIter[int]{ + values: want, + } + + var got []int + for i.Next() { + got = append(got, i.Item()) + } + + if !slices.Equal(got, want) { + t.Errorf("Iterator returned %v, want %v", got, want) + } +} + +func TestScannerIter_Empty(t *testing.T) { + var b bytes.Buffer + i := scannerIter{ + c: io.NopCloser(&b), + scanner: bufio.NewScanner(&b), + } + + if got := i.Next(); got { + t.Errorf("Next() = %v; want false", got) + } + if err := i.Err(); err != nil { + t.Errorf("Err() = %v; want no error", err) + } +} + +func TestScannerIter_SingleLine(t *testing.T) { + want := "test line" + b := bytes.NewBuffer([]byte(want)) + i := scannerIter{ + c: io.NopCloser(b), + scanner: bufio.NewScanner(b), + } + + if got := i.Next(); !got { + t.Errorf("Next() = %v; want true", got) + } + if err := i.Err(); err != nil { + t.Errorf("Err() = %v; want no error", err) + } + if got := i.Item(); got != want { + t.Errorf("Item() = %v; want %v", got, want) + } + if got := i.Next(); got { + t.Errorf("Next()#2 = %v; want false", got) + } + if err := i.Err(); err != nil { + t.Errorf("Err()#2 = %v; want no error", err) + } +} + +func TestScannerIter_MultiLine(t *testing.T) { + want := []string{"line one", "line two", "line three"} + b := bytes.NewBuffer([]byte(strings.Join(want, "\n"))) + i := scannerIter{ + c: io.NopCloser(b), + scanner: bufio.NewScanner(b), + } + + var got []string + for i.Next() { + item := i.Item() + got = append(got, item) + } + if err := i.Err(); err != nil { + t.Errorf("Err() = %v; want no error", err) + } + if !slices.Equal(got, want) { + t.Errorf("Iterator returned %v, want %v", got, want) + } +} + +func TestScannerIter_Error(t *testing.T) { + want := errors.New("error") + r := iotest.ErrReader(want) + i := scannerIter{ + c: io.NopCloser(r), + scanner: bufio.NewScanner(r), + } + + if got := i.Next(); got { + t.Errorf("Next() = %v; want false", got) + } + if err := i.Err(); err == nil || !errors.Is(err, want) { + t.Errorf("Err() = %v; want %v", err, want) + } +} + +type closerFn func() error + +func (c closerFn) Close() error { + return c() +} + +func TestScannerIter_Close(t *testing.T) { + got := 0 + i := scannerIter{ + c: closerFn(func() error { + got++ + return nil + }), + scanner: bufio.NewScanner(&bytes.Buffer{}), + } + err := i.Close() + + if got != 1 { + t.Errorf("Close() called %d times; want 1", got) + } + if err != nil { + t.Errorf("Err() = %v; want no error", err) + } +} diff --git a/cmd/criticality_score/inputiter/new.go b/cmd/criticality_score/inputiter/new.go new file mode 100644 index 000000000..d407d4aaa --- /dev/null +++ b/cmd/criticality_score/inputiter/new.go @@ -0,0 +1,84 @@ +// Copyright 2022 Criticality Score Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package inputiter + +import ( + "bufio" + "context" + "errors" + "net/url" + "os" + + "github.com/ossf/criticality_score/internal/infile" +) + +// osErrorWithFilename is an os-specific helper for determining if a particular +// error is related to the filename of the file. +var osErrorWithFilename func(err error) bool + +// errWithFilename determines if the given error is the result of an error +// caused by the filename being invalid, or pointing to a filename that doesn't +// exist. +func errWithFilename(err error) bool { + switch { + case errors.Is(err, os.ErrNotExist): + return true + case osErrorWithFilename != nil && osErrorWithFilename(err): + return true + default: + return false + } +} + +// Iterator returns an IterCloser for iterating across repositories for +// collecting signals. +// +// If only one arg is specified, the code will treat it as a file and attempt to +// open it. If the file doesn't exist, and is parseable as a URL the arg will be +// treated as a repo. +// +// If more than one arg is specified they are all considered to be repos. +// +// TODO: support the ability to force args to be interpreted as either a file, +// or a list of repos. +func New(args []string) (IterCloser[string], error) { + if len(args) == 1 { + // If there is 1 arg, attempt to open it as a file. + fileOrRepo := args[0] + urlParseFailed := false + if _, err := url.Parse(fileOrRepo); err != nil { + urlParseFailed = true + } + + // Open the in-file for reading + r, err := infile.Open(context.Background(), fileOrRepo) + if err == nil { + return &scannerIter{ + c: r, + scanner: bufio.NewScanner(r), + }, nil + } + if urlParseFailed || !errWithFilename(err) { + // Only report errors if the file doesn't appear to be a URL, if the + // filename doesn't exist, or the filename is invalid. + return nil, err + } + } + // If file loading failed, or there are 2 or more args, treat args as a list + // of repos. + return &sliceIter[string]{ + values: args, + }, nil +} diff --git a/cmd/criticality_score/inputiter/new_test.go b/cmd/criticality_score/inputiter/new_test.go new file mode 100644 index 000000000..96a623fdf --- /dev/null +++ b/cmd/criticality_score/inputiter/new_test.go @@ -0,0 +1,163 @@ +// Copyright 2022 Criticality Score Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package inputiter_test + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "golang.org/x/exp/slices" + + "github.com/ossf/criticality_score/cmd/criticality_score/inputiter" +) + +func TestNew_SingleURL(t *testing.T) { + want := "https://github.com/ossf/criticality_score" + i, err := inputiter.New([]string{want}) + if err != nil { + t.Fatalf("New() = %#v; want no error", err) + } + defer i.Close() + + // Move to the first item + if got := i.Next(); !got { + t.Errorf("Next() = %v; want true", got) + } + if err := i.Err(); err != nil { + t.Errorf("Err() = %v; want no error", err) + } + + // Get the single item + got := i.Item() + if got != want { + t.Errorf("Item() = %v; want %v", got, want) + } + + // Ensure the iterator is now empty + if got := i.Next(); got { + t.Errorf("Next()#2 = %v; want false", got) + } + if err := i.Err(); err != nil { + t.Errorf("Err()#2 = %v; want no error", err) + } +} + +func TestNew_MultipleURL(t *testing.T) { + want := []string{ + "https://github.com/ossf/criticality_score", + "https://github.com/ossf/scorecard", + } + i, err := inputiter.New(want) + if err != nil { + t.Fatalf("New() = %v; want no error", err) + } + defer i.Close() + + var got []string + for i.Next() { + item := i.Item() + got = append(got, item) + } + + if err := i.Err(); err != nil { + t.Errorf("Err() = %v; want no err", err) + } + + if !slices.Equal(got, want) { + t.Errorf("Iterator return %v; want %v", got, want) + } +} + +func TestNew_MissingFileIsURL(t *testing.T) { + want := "this/is/a/file/that/doesnt/exists" + i, err := inputiter.New([]string{want}) + if err != nil { + t.Fatalf("New() = %v; want no error", err) + } + defer i.Close() + + // Move to the first item + if got := i.Next(); !got { + t.Errorf("Next() = %v; want true", got) + } + if err := i.Err(); err != nil { + t.Errorf("Err() = %v; want no error", err) + } + + // Get the single item + got := i.Item() + if got != want { + t.Errorf("Item() = %v; want %v", got, want) + } + + // Ensure the iterator is now empty + if got := i.Next(); got { + t.Errorf("Next()#2 = %v; want false", got) + } + if err := i.Err(); err != nil { + t.Errorf("Err()#2 = %v; want no error", err) + } +} + +func TestNew_URLFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "urls.txt") + want := []string{ + "https://github.com/ossf/criticality_score", + "https://github.com/ossf/scorecard", + } + f, err := os.Create(path) + if err != nil { + t.Fatalf("Failed to open test file: %v", err) + } + defer f.Close() + for _, url := range want { + if _, err := fmt.Fprintln(f, url); err != nil { + t.Fatalf("Failed to write to test file: %v", err) + } + } + + i, err := inputiter.New([]string{path}) + if err != nil { + t.Fatalf("New() = %v; want no error", err) + } + defer i.Close() + + var got []string + for i.Next() { + item := i.Item() + got = append(got, item) + } + + if err := i.Err(); err != nil { + t.Errorf("Err() = %v; want no err", err) + } + + if !slices.Equal(got, want) { + t.Errorf("Iterator return %v; want %v", got, want) + } +} + +func TestNew_InvalidURL(t *testing.T) { + want := ":this.is/not/a/url" + i, err := inputiter.New([]string{want}) + if err == nil { + t.Errorf("New() = %#v; want error", err) + } + if err == nil { + defer i.Close() + } +} diff --git a/cmd/criticality_score/main.go b/cmd/criticality_score/main.go index b6a982549..1587547e1 100644 --- a/cmd/criticality_score/main.go +++ b/cmd/criticality_score/main.go @@ -29,6 +29,7 @@ import ( "go.uber.org/zap" "go.uber.org/zap/zapcore" + "github.com/ossf/criticality_score/cmd/criticality_score/inputiter" "github.com/ossf/criticality_score/internal/collector" log "github.com/ossf/criticality_score/internal/log" "github.com/ossf/criticality_score/internal/outfile" @@ -162,14 +163,14 @@ func main() { } // Prepare the input for reading - inputIter, err := initInput(flag.Args()) + iter, err := inputiter.New(flag.Args()) if err != nil { logger.With( zap.Error(err), ).Error("Failed to prepare input") os.Exit(2) } - defer inputIter.Close() + defer iter.Close() // Open the out-file for writing w, err := outfile.Open(context.Background()) @@ -229,8 +230,8 @@ func main() { }) // Read in each repo from the input - for inputIter.Next() { - line := inputIter.Item() + for iter.Next() { + line := iter.Item() u, err := url.Parse(strings.TrimSpace(line)) if err != nil { @@ -247,7 +248,7 @@ func main() { // Send the url to the workers repos <- u } - if err := inputIter.Err(); err != nil { + if err := iter.Err(); err != nil { logger.With( zap.Error(err), ).Error("Failed while reading input") diff --git a/cmd/enumerate_github/marker/type_test.go b/cmd/enumerate_github/marker/type_test.go index 26e1fd3e7..0a9f19ede 100644 --- a/cmd/enumerate_github/marker/type_test.go +++ b/cmd/enumerate_github/marker/type_test.go @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !windows + package marker_test import ( diff --git a/cmd/enumerate_github/marker/write_test.go b/cmd/enumerate_github/marker/write_test.go index 9d0c50066..40b0b12df 100644 --- a/cmd/enumerate_github/marker/write_test.go +++ b/cmd/enumerate_github/marker/write_test.go @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !windows + package marker_test import ( diff --git a/go.mod b/go.mod index 72a025436..455942d33 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( go.uber.org/zap v1.23.0 gocloud.dev v0.26.0 golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561 + golang.org/x/sys v0.5.0 google.golang.org/api v0.102.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -76,7 +77,6 @@ require ( golang.org/x/net v0.4.0 // indirect golang.org/x/oauth2 v0.1.0 // indirect golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.3.0 // indirect golang.org/x/text v0.5.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum b/go.sum index 0ada404d8..a065ced7d 100644 --- a/go.sum +++ b/go.sum @@ -769,6 +769,8 @@ golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/go.work.sum b/go.work.sum index 77d2fa54b..03041cb4d 100644 --- a/go.work.sum +++ b/go.work.sum @@ -199,6 +199,7 @@ github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8c github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= +github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0= github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8= @@ -222,6 +223,7 @@ github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqE github.com/googleapis/go-type-adapters v1.0.0 h1:9XdMn+d/G57qq1s8dNc5IesGCXHf6V2HZ2JwRxfA2tA= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8 h1:tlyzajkF3030q6M8SvmJSemC9DTHL/xaMa18b65+JM4= github.com/gookit/color v1.5.1 h1:Vjg2VEcdHpwq+oY63s/ksHrgJYCTo0bwWvmmYWdE9fQ= +github.com/gookit/color v1.5.2 h1:uLnfXcaFjlrDnQDT+NCBcfhrXqYTx/rcCa6xn01Y8yI= github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= @@ -239,8 +241,6 @@ github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+l github.com/hashicorp/serf v0.9.7 h1:hkdgbqizGQHuU5IPqYM1JdSMV8nKfpuOnZYXssk9muY= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639 h1:mV02weKRL81bEnm8A0HT1/CAelMQDBuQIfLw8n+d6xI= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= -github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= -github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= github.com/jackc/pgconn v1.11.0 h1:HiHArx4yFbwl91X3qqIHtUFoiIfLNJXCQRsnzkiwwaQ= @@ -255,6 +255,7 @@ github.com/jackc/pgx/v4 v4.15.0 h1:B7dTkXsdILD3MF987WGGCcg+tvLW6bZJdEcqVFeU//w= github.com/jackc/puddle v1.2.1 h1:gI8os0wpRXFd4FiAY2dWiqRK037tjj3t7rKFeO4X5iw= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= +github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -270,6 +271,7 @@ github.com/kr/pty v1.1.8 h1:AkaSdXYQOWeaO3neb8EM634ahkXXe3jYbVh/F9lq+GI= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/lib/pq v1.10.6 h1:jbk+ZieJ0D7EVGJYpL9QTz7/YW6UHbmdnZWYyK5cdBs= +github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/magefile/mage v1.14.0 h1:6QDX3g6z1YvJ4olPhT1wksUcSa/V0a1B+pJb73fBjyo= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= @@ -291,13 +293,14 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034= github.com/otiai10/curr v1.0.0 h1:TJIWdbX0B+kpNagQrjgq8bCMrbhiuX73M2XwgtDMoOI= github.com/otiai10/mint v1.3.1 h1:BCmzIS3n71sGfHB5NMNDB3lHYPz8fWSkCAErHed//qc= +github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d h1:CdDQnGF8Nq9ocOS/xlSptM1N3BbrA6/kmaep5ggwaIA= github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 h1:49lOXmGaUpV9Fz3gd7TFZY106KVlPVa5jcYD1gaQf98= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A= github.com/pkg/sftp v1.13.1 h1:I2qBYMChEhIjOgazfJmV3/mZM256btk6wkCDRmW7JYs= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34= github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= github.com/quasilyte/go-ruleguard/dsl v0.3.21 h1:vNkC6fC6qMLzCOGbnIHOd5ixUGgTbp3Z4fGnUgULlDA= +github.com/quasilyte/go-ruleguard/dsl v0.3.22 h1:wd8zkOhSNr+I+8Qeciml08ivDt1pSXe60+5DqOpCjPE= github.com/quasilyte/go-ruleguard/rules v0.0.0-20211022131956-028d6511ab71 h1:CNooiryw5aisadVfzneSZPswRWvnVW8hF1bS/vo8ReI= github.com/remyoudompheng/go-dbus v0.0.0-20121104212943-b7232d34b1d5 h1:CvqZS4QYHBRvx7AeFdimd16HCbLlYsvQMcKDACpJW/c= github.com/remyoudompheng/go-liblzma v0.0.0-20190506200333-81bf2d431b96 h1:J8J/cgLDRuqXJnwIrRDBvtl+LLsdg7De74znW/BRRq4= @@ -316,15 +319,15 @@ github.com/sagikazarmark/crypt v0.6.0 h1:REOEXCs/NFY/1jOCEouMuT4zEniE5YoXbvpC5X/ github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/shirou/gopsutil/v3 v3.22.9 h1:yibtJhIVEMcdw+tCTbOPiF1VcsuDeTE4utJ8Dm4c5eA= +github.com/shirou/gopsutil/v3 v3.23.1 h1:a9KKO+kGLKEvcPIs4W62v0nu3sciVDOOOPUD0Hz7z/4= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e h1:MZM7FHLqUHYI0Y/mQAt3d2aYa0SiNms/hFqC9qJYolM= github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041 h1:llrF3Fs4018ePo4+G/HV/uQUqEI1HMDjCeOf2V6puPc= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= -github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= -github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw= +github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= github.com/tklauser/numcpus v0.4.0 h1:E53Dm1HjH1/R2/aoCtXtPgzmElmn51aOkhCFSuZq//o= +github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= @@ -352,9 +355,6 @@ golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDA golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 h1:4+4C/Iv2U4fMZBiMCc98MG1In4gJY5YRhtpDNeDeHWs= golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I= -golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= -golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA= -golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= @@ -363,15 +363,11 @@ golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0 h1:z85xZCsEl7bi/KwbNADeBYoOP0++7W1ipu+aGnpwzRM= -golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= -golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= +golang.org/x/term v0.4.0 h1:O7UWfv5+A2qiuulQk30kVinPoMtoIPeVaKLEgLpVkvg= golang.org/x/time v0.0.0-20220922220347-f3bd1da661af h1:Yx9k8YCG3dvF87UAn2tu2HQLf2dt/eR1bXxpLMWeH+Y= golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE= -golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.3.0 h1:SrNbZl6ECOS1qFzgTdQfWXZM9XBkiA6tkFrH9YSTPHM= -golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= google.golang.org/api v0.81.0 h1:o8WF5AvfidafWbFjsRyupxyEQJNUWxLZJCK5NXrxZZ8= google.golang.org/api v0.93.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= diff --git a/internal/cloudstorage/cloudstorage.go b/internal/cloudstorage/cloudstorage.go index b58f302be..a99fe2257 100644 --- a/internal/cloudstorage/cloudstorage.go +++ b/internal/cloudstorage/cloudstorage.go @@ -78,7 +78,7 @@ func NewWriter(ctx context.Context, rawURL string) (io.WriteCloser, error) { b, err := blob.OpenBucket(ctx, bucket) if err != nil { - return nil, fmt.Errorf("failed to opening %s: %w", bucket, err) + return nil, fmt.Errorf("failed opening %s: %w", bucket, err) } w, err := b.NewWriter(ctx, prefix, nil) if err != nil { diff --git a/internal/cloudstorage/cloudstorage_test.go b/internal/cloudstorage/cloudstorage_test.go index 3472c6e88..05a1821bb 100644 --- a/internal/cloudstorage/cloudstorage_test.go +++ b/internal/cloudstorage/cloudstorage_test.go @@ -91,20 +91,20 @@ func assertBucket(t *testing.T, bucket, wantScheme, wantHost, wantPath string, w t.Fatalf("Bucket is not a valid url: %v", err) } if u.Scheme != wantScheme { - t.Fatalf("Bucket scheme = %q, want %q", u.Scheme, wantScheme) + t.Errorf("Bucket scheme = %q, want %q", u.Scheme, wantScheme) } if u.Host != wantHost { - t.Fatalf("Bucket host = %q, want %q", u.Host, wantHost) + t.Errorf("Bucket host = %q, want %q", u.Host, wantHost) } if u.Path != wantPath { - t.Fatalf("Bucket path = %q, want %q", u.Path, wantPath) + t.Errorf("Bucket path = %q, want %q", u.Path, wantPath) } for k, want := range wantQuery { if !u.Query().Has(k) { - t.Fatalf("Bucket query has no key %q", k) + t.Errorf("Bucket query has no key %q", k) } if got := u.Query().Get(k); got != want { - t.Fatalf("Bucket query %q = %q, want %q", k, got, want) + t.Errorf("Bucket query %q = %q, want %q", k, got, want) } } } diff --git a/internal/outfile/outfile_test.go b/internal/outfile/outfile_test.go index 7fd84ab28..318b0274e 100644 --- a/internal/outfile/outfile_test.go +++ b/internal/outfile/outfile_test.go @@ -52,9 +52,14 @@ func newTestOpener(t *testing.T) *testOpener { return nil, o.openErr } else { dir := t.TempDir() + cwd, err := os.Getwd() // Save the CWD so we can restore it later. + if err != nil { + return nil, err + } if err := os.Chdir(dir); err != nil { return nil, err } + defer os.Chdir(cwd) // Restore the CWD so the temp dir can be cleaned up on Windows. return os.Create(filename) } } @@ -95,6 +100,7 @@ func TestOpenBucketUrl(t *testing.T) { if err != nil { t.Fatalf("Open() == %v, want nil", err) } + defer f.Close() if o.lastOpen != nil { t.Fatal("Open(...) called instead of bucket code") } @@ -106,8 +112,9 @@ func TestOpenBucketUrl(t *testing.T) { func TestOpenBucketUrlNoForceFlag(t *testing.T) { o := newTestOpener(t) o.flag.Parse([]string{"-out=mem://bucket/prefix"}) - _, err := o.opener.Open(context.Background()) + f, err := o.opener.Open(context.Background()) if err == nil { + defer f.Close() t.Fatalf("Open() == nil, want an error") } } @@ -151,6 +158,7 @@ func TestOpenFlagTest(t *testing.T) { if err != nil { t.Fatalf("Open() == %v, want nil", err) } + defer f.Close() if f == nil { t.Fatal("Open() == nil, want a file") } @@ -167,8 +175,9 @@ func TestOpenFlagTest(t *testing.T) { o := newTestOpener(t) o.flag.Parse(append(test.args, "-out=testfile")) o.openErr = errors.New("test error") - _, err := o.opener.Open(context.Background()) + f, err := o.opener.Open(context.Background()) if err == nil { + defer f.Close() t.Fatalf("Open() is nil, want %v", o.openErr) } }) @@ -200,6 +209,7 @@ func TestFilenameTransform(t *testing.T) { if err != nil { t.Fatalf("Open() == %v, want nil", err) } + defer f.Close() if f == nil { t.Fatal("Open() == nil, want a file") }