[SumDB] Non-verifiable SumDB info support

This allows users to run this without a verifiable index. If you fully
trust SumDB and don't want to verify it, then this is probably good
enough.

Author: mhutchinson

vindex/cmd/sumdbverify/README.md

@@ -1,8 +1,8 @@
 ## SumDB Verify
 
 > [!IMPORTANT]
-> This tool requires a [SumDB VIndex](../sumdb/) to be running.
-> Functionality may be added to support reading SumDB contents from non-verifiable endpoints.
+> Proper use of this tool requires a [SumDB VIndex](../sumdb/) to be running.
+> See [Running non-verifiably](#running-non-verifiably) for the quick and dirty way.
 
 This tool checks that the contents for a module in SumDB match the state as represented in a local git repository.
 The command below shows the output for this command querying a local checkout of `github.com/transparency-dev/tessera`:
@@ -27,11 +27,31 @@ v1.0.0      43930254  ✅      ✅       ✅
 
 The output shows all versions present in SumDB, and for each:
  - INDEX is the leaf index of this `module@version` in SumDB
- - FOUND shows that a tag with the same version string was found in the git version
+ - FOUND shows that a tag with the same version string was found in the git repository
  - go.mod shows that the hashes for the `go.mod` file match. In addition to the green tick, there are two other states:
    - ⚠️: no `go.mod` file was found in the git repo at the tagged version; this _could_ be a release from before modules were adopted
    - ❌: a `go.mod` file was found in the git repo, but the hash doesn't match that in SumDB. Either the tag was changed, or SumDB is hosting bad content.
  - zip shows that the hashes for the zip containing the source code match. In addition to the green tick, there are two other states:
    - ⚠️: no `go.mod` file was found in the git repo at the tagged version; this _could_ be a release from before modules were adopted
    - ❌: the zip file hash did not match that in SumDB. Either the tag was changed, or SumDB is hosting bad content.
 
+### Running non-verifiably
+
+By omitting the `--base_url` and `--out_log_pub_key` flags, the SumDB information will be fetched from non-verifiable endpoints.
+This is useful for casual testing before a public-good instance of the SumDB verifiable index is available.
+
+```shell
+go run ./vindex/cmd/sumdbverify --mod_root ~/git/tessera
+
+W1002 13:31:48.254094 2883468 client.go:84] --base_url is not provided. Using NON-VERIFIABLE lookup to source SumDB data.
+github.com/transparency-dev/tessera (./go.mod)
+VERSION     INDEX     FOUND  go.mod  zip
+v0.1.0      37258761  ✅      ✅       ✅
+v0.1.1      37258762  ✅      ✅       ✅
+v0.1.2      37258746  ✅      ✅       ✅
+v0.2.0      38108519  ✅      ✅       ✅
+v1.0.0-rc1  41510961  ✅      ✅       ✅
+v1.0.0-rc2  42710781  ✅      ✅       ✅
+v1.0.0-rc3  43267373  ✅      ✅       ✅
+v1.0.0      43930254  ✅      ✅       ✅
+```

vindex/cmd/sumdbverify/client.go

@@ -18,6 +18,7 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"flag"
@@ -71,24 +72,63 @@ func main() {
 }
 
 func run(ctx context.Context) error {
-	if *baseURL == "" {
-		return errors.New("base_url flag must be provided")
-	}
-	if *outLogPubKey == "" {
-		return errors.New("out_log_pub_key flag must be provided")
-	}
 	if *modRoot == "" {
 		return errors.New("mod_root flag must be provided")
 	}
 
-	// TODO(mhutchinson): Support a non-VIndex version of this that reads the non-verifiable proxy endpoints:
-	// 1) https://proxy.golang.org/github.com/transparency-dev/tessera/@v/list
-	// 2) https://sum.golang.org/lookup/github.com/transparency-dev/tessera@v1.0.0
-	// This will provide a way to use this tool before the VIndex is widely available
-	sumFetcher := func(ctx context.Context, modName string) (map[string]modData, error) {
-		vic := newVIndexClientFromFlags()
+	var sumFetcher func(ctx context.Context, modName string) (map[string]modData, error)
+	if *baseURL == "" {
+		klog.Warningf("--base_url is not provided. Using NON-VERIFIABLE lookup to source SumDB data.")
+
+		// This constructs the map non-verifiably by calling similar URLs to these:
+		// 1) https://proxy.golang.org/github.com/transparency-dev/tessera/@v/list
+		// 2) https://sum.golang.org/lookup/github.com/transparency-dev/tessera@v1.0.0
+		sumFetcher = func(ctx context.Context, modName string) (map[string]modData, error) {
+			result := make(map[string]modData)
+			resp, err := http.Get(fmt.Sprintf("https://proxy.golang.org/%s/@v/list", modName))
+			if err != nil {
+				return nil, fmt.Errorf("failed to get module listing: %v", err)
+			}
+			body, err := io.ReadAll(resp.Body)
+			if err != nil {
+				return nil, fmt.Errorf("failed to get module listing: %v", err)
+			}
+			for v := range strings.Lines(string(body)) {
+				v = strings.TrimSpace(v)
+				resp, err = http.Get(fmt.Sprintf("https://sum.golang.org/lookup/%s@%s", modName, v))
+				if err != nil {
+					return nil, fmt.Errorf("failed to get version info: %v", err)
+				}
+				body, err = io.ReadAll(resp.Body)
+				if err != nil {
+					return nil, fmt.Errorf("failed to get version info: %v", err)
+				}
+				lines := bytes.Split(body, []byte{'\n'})
+				idx, err := strconv.ParseInt(string(lines[0]), 10, 64)
+				if err != nil {
+					return nil, fmt.Errorf("failed to parse index: %v", err)
+				}
+				leaf := append(append(append(lines[1], byte('\n')), lines[2]...), byte('\n'))
+				v2, md, err := parseLeaf(uint64(idx), leaf)
+				if err != nil {
+					return nil, fmt.Errorf("failed to parse leaf: %v", err)
+				}
+				if v != v2 {
+					return nil, fmt.Errorf("performed lookup for %s@%s but got version %s", modName, v, v2)
+				}
+				result[v] = md
+			}
+			return result, nil
+		}
 
-		return queryIndex(ctx, vic, modName)
+	} else {
+		if *outLogPubKey == "" {
+			return errors.New("out_log_pub_key flag must be provided if --base_url is provided")
+		}
+		sumFetcher = func(ctx context.Context, modName string) (map[string]modData, error) {
+			vic := newVIndexClientFromFlags()
+			return queryIndex(ctx, vic, modName)
+		}
 	}
 
 	report, reportErr := getReport(ctx, *modRoot, sumFetcher)