update: add incremental bundle on top of previous bundles

Be sure to parse the existing bundle headers in order to get the
previous refs, then use those as prerequisites for the new incremental
bundle. It's not enough to only use the refs from the previous layer,
since perhaps not all refs were updated or maybe a ref was force-pushed.

Author: derrickstolee

cmd/git-bundle-server/init.go

@@ -35,10 +35,13 @@ func (Init) run(args []string) error {
 	bundle := bundles.CreateInitialBundle(repo)
 	fmt.Printf("Constructing base bundle file at %s\n", bundle.Filename)
 
-	gitErr = git.GitCommand("-C", repo.RepoDir, "bundle", "create", bundle.Filename, "--all")
+	written, gitErr := git.CreateBundle(repo, bundle)
 	if gitErr != nil {
 		return fmt.Errorf("failed to create bundle: %w", gitErr)
 	}
+	if !written {
+		return fmt.Errorf("refused to write empty bundle. Is the repo empty?")
+	}
 
 	list := bundles.SingletonList(bundle)
 	listErr := bundles.WriteBundleList(list, repo)

cmd/git-bundle-server/subcommand.go

@@ -8,5 +8,6 @@ type Subcommand interface {
 func all() []Subcommand {
 	return []Subcommand{
 		Init{},
+		Update{},
 	}
 }

cmd/git-bundle-server/update.go

@@ -1,7 +1,11 @@
 package main
 
 import (
+	"errors"
 	"fmt"
+	"git-bundle-server/internal/bundles"
+	"git-bundle-server/internal/core"
+	"git-bundle-server/internal/git"
 )
 
 type Update struct{}
@@ -11,10 +15,39 @@ func (Update) subcommand() string {
 }
 
 func (Update) run(args []string) error {
-	fmt.Printf("Found Update method!\n")
+	if len(args) != 1 {
+		// TODO: allow parsing <route> out of <url>
+		return errors.New("usage: git-bundle-server update <route>")
+	}
+
+	route := args[0]
+	repo := core.GetRepository(route)
+
+	list, err := bundles.GetBundleList(repo)
+	if err != nil {
+		return fmt.Errorf("failed to load bundle list: %w", err)
+	}
+
+	bundle := bundles.CreateDistinctBundle(repo, *list)
+
+	fmt.Printf("Constructing incremental bundle file at %s\n", bundle.Filename)
+
+	written, err := git.CreateIncrementalBundle(repo, bundle, *list)
+	if err != nil {
+		return fmt.Errorf("failed to create incremental bundle: %w", err)
+	}
+
+	// Nothing to update
+	if !written {
+		return nil
+	}
+
+	list.Bundles[bundle.CreationToken] = bundle
 
-	for _, arg := range args {
-		fmt.Printf("%s\n", arg)
+	fmt.Printf("Writing updated bundle list\n")
+	listErr := bundles.WriteBundleList(*list, repo)
+	if listErr != nil {
+		return fmt.Errorf("failed to write bundle list: %w", listErr)
 	}
 
 	return nil

internal/bundles/bundles.go

@@ -5,10 +5,24 @@ import (
 	"encoding/json"
 	"fmt"
 	"git-bundle-server/internal/core"
+	"io"
 	"os"
+	"strconv"
+	"strings"
 	"time"
 )
 
+type BundleHeader struct {
+	Version int64
+
+	// The Refs map is given as Refs[<refname>] = <oid>.
+	Refs map[string]string
+
+	// The PrereqCommits map is given as
+	// PrereqCommits[<oid>] = <commit-msg>
+	PrereqCommits map[string]string
+}
+
 type Bundle struct {
 	URI           string
 	Filename      string
@@ -38,6 +52,27 @@ func CreateInitialBundle(repo core.Repository) Bundle {
 	return bundle
 }
 
+func CreateDistinctBundle(repo core.Repository, list BundleList) Bundle {
+	timestamp := time.Now().UTC().Unix()
+
+	_, c := list.Bundles[timestamp]
+
+	for c {
+		timestamp++
+		_, c = list.Bundles[timestamp]
+	}
+
+	bundleName := "bundle-" + fmt.Sprint(timestamp) + ".bundle"
+	bundleFile := repo.WebDir + "/" + bundleName
+	bundle := Bundle{
+		URI:           "./" + bundleName,
+		Filename:      bundleFile,
+		CreationToken: timestamp,
+	}
+
+	return bundle
+}
+
 func SingletonList(bundle Bundle) BundleList {
 	list := BundleList{1, "all", make(map[int64]Bundle)}
 
@@ -104,3 +139,104 @@ func WriteBundleList(list BundleList, repo core.Repository) error {
 
 	return os.Rename(listFile+".lock", listFile)
 }
+
+func GetBundleList(repo core.Repository) (*BundleList, error) {
+	jsonFile := repo.RepoDir + "/bundle-list.json"
+
+	reader, err := os.Open(jsonFile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open file: %w", err)
+	}
+
+	var list BundleList
+	err = json.NewDecoder(reader).Decode(&list)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse JSON from file: %w", err)
+	}
+
+	return &list, nil
+}
+
+func GetBundleHeader(bundle Bundle) (*BundleHeader, error) {
+	file, err := os.Open(bundle.Filename)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open bundle file: %w", err)
+	}
+
+	header := BundleHeader{
+		Version:       0,
+		Refs:          make(map[string]string),
+		PrereqCommits: make(map[string]string),
+	}
+
+	reader := bufio.NewReader(file)
+	line, err := reader.ReadString('\n')
+	if line == "" ||
+		(err != nil && err != io.EOF) ||
+		!strings.HasPrefix(line, "# v") ||
+		!strings.HasSuffix(line, " git bundle\n") {
+		return nil, fmt.Errorf("bundle file has invalid header: '%s'", line)
+	}
+
+	header.Version, err = strconv.ParseInt(line[3:len(line)-len(" git bundle\n")], 10, 64)
+	if err != nil {
+		return nil, fmt.Errorf(("bundle file has invalid version: %w"), err)
+	}
+
+	// Parse until we reach the "PACK" information
+	for {
+		line, err = reader.ReadString('\n')
+		if line == "" || line[0] == '\n' ||
+			(err != nil && err != io.EOF) ||
+			strings.HasPrefix(line, "PACK") {
+			break
+		}
+
+		if line[0] == '@' {
+			// This is a capability. Ignore for now.
+			continue
+		}
+
+		if line[0] == '-' {
+			// This is a prerequisite
+			space := strings.Index(line, " ")
+			if space < 0 {
+				return nil, fmt.Errorf("failed to parse rerequisite '%s'", line)
+			}
+
+			oid := line[0:space]
+			message := line[space+1 : len(line)-1]
+			header.PrereqCommits[oid] = message
+		} else {
+			// This is a tip
+			space := strings.Index(line, " ")
+
+			if space < 0 {
+				return nil, fmt.Errorf("failed to parse tip '%s'", line)
+			}
+
+			oid := line[0:space]
+			ref := line[space+1 : len(line)-1]
+			header.Refs[ref] = oid
+		}
+	}
+
+	return &header, nil
+}
+
+func GetAllPrereqsForIncrementalBundle(list BundleList) ([]string, error) {
+	prereqs := []string{}
+
+	for _, bundle := range list.Bundles {
+		header, err := GetBundleHeader(bundle)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse bundle file %s: %w", bundle.Filename, err)
+		}
+
+		for _, oid := range header.Refs {
+			prereqs = append(prereqs, "^"+oid)
+		}
+	}
+
+	return prereqs, nil
+}

internal/git/git.go

@@ -1,8 +1,13 @@
 package git
 
 import (
-	"log"
+	"bytes"
+	"fmt"
+	"git-bundle-server/internal/bundles"
+	"git-bundle-server/internal/core"
+	"os"
 	"os/exec"
+	"strings"
 )
 
 func GitCommand(args ...string) error {
@@ -13,15 +18,88 @@ func GitCommand(args ...string) error {
 	}
 
 	cmd := exec.Command(git, args...)
+	cmd.Stderr = os.Stderr
+	cmd.Stdout = os.Stdout
+
 	err := cmd.Start()
 	if err != nil {
-		log.Fatal("Git command failed to start: ", err)
+		return fmt.Errorf("git command failed to start: %w", err)
 	}
 
 	err = cmd.Wait()
 	if err != nil {
-		log.Fatal("Git command returned a failure: ", err)
+		return fmt.Errorf("git command returned a failure: %w", err)
 	}
 
 	return err
 }
+
+func GitCommandWithStdin(stdinLines []string, args ...string) error {
+	git, lookErr := exec.LookPath("git")
+
+	if lookErr != nil {
+		return lookErr
+	}
+
+	buffer := bytes.Buffer{}
+	for line := range stdinLines {
+		buffer.Write([]byte(stdinLines[line] + "\n"))
+	}
+
+	cmd := exec.Command(git, args...)
+
+	cmd.Stdin = &buffer
+
+	errorBuffer := bytes.Buffer{}
+	cmd.Stderr = &errorBuffer
+	cmd.Stdout = os.Stdout
+
+	err := cmd.Start()
+	if err != nil {
+		return fmt.Errorf("git command failed to start: %w", err)
+	}
+
+	err = cmd.Wait()
+	if err != nil {
+		return fmt.Errorf("git command returned a failure: %w\nstderr: %s", err, errorBuffer.String())
+	}
+
+	return err
+}
+
+func CreateBundle(repo core.Repository, bundle bundles.Bundle) (bool, error) {
+	err := GitCommand(
+		"-C", repo.RepoDir, "bundle", "create",
+		bundle.Filename, "--all")
+	if err != nil {
+		if strings.Contains(err.Error(), "Refusing to create empty bundle") {
+			return false, nil
+		}
+		return false, err
+	}
+
+	return true, nil
+}
+
+func CreateIncrementalBundle(repo core.Repository, bundle bundles.Bundle, list bundles.BundleList) (bool, error) {
+	lines, err := bundles.GetAllPrereqsForIncrementalBundle(list)
+	if err != nil {
+		return false, err
+	}
+
+	for _, line := range lines {
+		fmt.Printf("Sending prereq: %s\n", line)
+	}
+
+	err = GitCommandWithStdin(
+		lines, "-C", repo.RepoDir, "bundle", "create",
+		bundle.Filename, "--stdin", "--all")
+	if err != nil {
+		if strings.Contains(err.Error(), "Refusing to create empty bundle") {
+			return false, nil
+		}
+		return false, err
+	}
+
+	return true, nil
+}