diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 4017ed82ca4341..c19530b086311a 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -1,8 +1,10 @@
- - [ ] I was not able to find an [open](https://github.com/git-for-windows/git/issues?q=is%3Aopen) or [closed](https://github.com/git-for-windows/git/issues?q=is%3Aclosed) issue matching what I'm seeing
+ - [ ] I was not able to find an [open](https://github.com/microsoft/git/issues?q=is%3Aopen)
+ or [closed](https://github.com/microsoft/git/issues?q=is%3Aclosed) issue matching
+ what I'm seeing, including in [the `git-for-windows/git` tracker](https://github.com/git-for-windows/git/issues).
### Setup
- - Which version of Git for Windows are you using? Is it 32-bit or 64-bit?
+ - Which version of `microsoft/git` are you using? Is it 32-bit or 64-bit?
```
$ git --version --build-options
@@ -10,24 +12,22 @@ $ git --version --build-options
** insert your machine's response here **
```
- - Which version of Windows are you running? Vista, 7, 8, 10? Is it 32-bit or 64-bit?
+Are you using Scalar or VFS for Git?
+
+** insert your answer here **
+
+If VFS for Git, then what version?
```
-$ cmd.exe /c ver
+$ gvfs version
** insert your machine's response here **
```
- - What options did you set as part of the installation? Or did you choose the
- defaults?
+ - Which version of Windows are you running? Vista, 7, 8, 10? Is it 32-bit or 64-bit?
```
-# One of the following:
-> type "C:\Program Files\Git\etc\install-options.txt"
-> type "C:\Program Files (x86)\Git\etc\install-options.txt"
-> type "%USERPROFILE%\AppData\Local\Programs\Git\etc\install-options.txt"
-> type "$env:USERPROFILE\AppData\Local\Programs\Git\etc\install-options.txt"
-$ cat /etc/install-options.txt
+$ cmd.exe /c ver
** insert your machine's response here **
```
@@ -58,7 +58,11 @@ $ cat /etc/install-options.txt
** insert here **
- - If the problem was occurring with a specific repository, can you provide the
- URL to that repository to help us with testing?
+ - If the problem was occurring with a specific repository, can you specify
+ the repository?
-** insert URL here **
+ * [ ] Public repo: **insert URL here**
+ * [ ] Windows monorepo
+ * [ ] Office monorepo
+ * [ ] Other Microsoft-internal repo: **insert name here**
+ * [ ] Other internal repo.
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 7baf31f2c471ec..3cb48d8582f31c 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -1,22 +1,10 @@
Thanks for taking the time to contribute to Git!
-Those seeking to contribute to the Git for Windows fork should see
-http://gitforwindows.org/#contribute on how to contribute Windows specific
-enhancements.
-
-If your contribution is for the core Git functions and documentation
-please be aware that the Git community does not use the github.com issues
-or pull request mechanism for their contributions.
-
-Instead, we use the Git mailing list (git@vger.kernel.org) for code and
-documentation submissions, code reviews, and bug reports. The
-mailing list is plain text only (anything with HTML is sent directly
-to the spam folder).
-
-Nevertheless, you can use GitGitGadget (https://gitgitgadget.github.io/)
-to conveniently send your Pull Requests commits to our mailing list.
-
-For a single-commit pull request, please *leave the pull request description
-empty*: your commit message itself should describe your changes.
-
-Please read the "guidelines for contributing" linked above!
+This fork contains changes specific to monorepo scenarios. If you are an
+external contributor, then please detail your reason for submitting to
+this fork:
+
+* [ ] This is an early version of work already under review upstream.
+* [ ] This change only applies to interactions with Azure DevOps and the
+ GVFS Protocol.
+* [ ] This change only applies to the virtualization hook and VFS for Git.
diff --git a/.github/macos-installer/Makefile b/.github/macos-installer/Makefile
new file mode 100644
index 00000000000000..1a06f6200e62dc
--- /dev/null
+++ b/.github/macos-installer/Makefile
@@ -0,0 +1,157 @@
+SHELL := /bin/bash
+SUDO := sudo
+C_INCLUDE_PATH := /usr/include
+CPLUS_INCLUDE_PATH := /usr/include
+LD_LIBRARY_PATH := /usr/lib
+
+OSX_VERSION := $(shell sw_vers -productVersion)
+TARGET_FLAGS := -mmacosx-version-min=$(OSX_VERSION) -DMACOSX_DEPLOYMENT_TARGET=$(OSX_VERSION)
+
+uname_M := $(shell sh -c 'uname -m 2>/dev/null || echo not')
+
+ARCH_UNIV := universal
+ARCH_FLAGS := -arch x86_64 -arch arm64
+
+CFLAGS := $(TARGET_FLAGS) $(ARCH_FLAGS)
+LDFLAGS := $(TARGET_FLAGS) $(ARCH_FLAGS)
+
+PREFIX := /usr/local
+GIT_PREFIX := $(PREFIX)/git
+
+BUILD_DIR := $(GITHUB_WORKSPACE)/payload
+DESTDIR := $(PWD)/stage/git-$(ARCH_UNIV)-$(VERSION)
+ARTIFACTDIR := build-artifacts
+SUBMAKE := $(MAKE) C_INCLUDE_PATH="$(C_INCLUDE_PATH)" CPLUS_INCLUDE_PATH="$(CPLUS_INCLUDE_PATH)" LD_LIBRARY_PATH="$(LD_LIBRARY_PATH)" TARGET_FLAGS="$(TARGET_FLAGS)" CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS)" NO_GETTEXT=1 NO_DARWIN_PORTS=1 prefix=$(GIT_PREFIX) GIT_BUILT_FROM_COMMIT="$(GIT_BUILT_FROM_COMMIT)" DESTDIR=$(DESTDIR)
+CORES := $(shell bash -c "sysctl hw.ncpu | awk '{print \$$2}'")
+
+# Guard against environment variables
+APPLE_APP_IDENTITY =
+APPLE_INSTALLER_IDENTITY =
+APPLE_KEYCHAIN_PROFILE =
+
+.PHONY: image pkg payload codesign notarize
+
+.SECONDARY:
+
+$(DESTDIR)$(GIT_PREFIX)/VERSION-$(VERSION)-$(ARCH_UNIV):
+ rm -f $(BUILD_DIR)/git-$(VERSION)/osx-installed*
+ mkdir -p $(DESTDIR)$(GIT_PREFIX)
+ touch $@
+
+$(BUILD_DIR)/git-$(VERSION)/osx-built-keychain:
+ cd $(BUILD_DIR)/git-$(VERSION)/contrib/credential/osxkeychain; $(SUBMAKE) CFLAGS="$(CFLAGS) -g -O2 -Wall"
+ touch $@
+
+$(BUILD_DIR)/git-$(VERSION)/osx-built:
+ [ -d $(DESTDIR)$(GIT_PREFIX) ] && $(SUDO) rm -rf $(DESTDIR) || echo ok
+ cd $(BUILD_DIR)/git-$(VERSION); $(SUBMAKE) -j $(CORES) all strip
+ echo "================"
+ echo "Dumping Linkage"
+ cd $(BUILD_DIR)/git-$(VERSION); ./git version
+ echo "===="
+ cd $(BUILD_DIR)/git-$(VERSION); /usr/bin/otool -L ./git
+ echo "===="
+ cd $(BUILD_DIR)/git-$(VERSION); /usr/bin/otool -L ./git-http-fetch
+ echo "===="
+ cd $(BUILD_DIR)/git-$(VERSION); /usr/bin/otool -L ./git-http-push
+ echo "===="
+ cd $(BUILD_DIR)/git-$(VERSION); /usr/bin/otool -L ./git-remote-http
+ echo "===="
+ cd $(BUILD_DIR)/git-$(VERSION); /usr/bin/otool -L ./git-gvfs-helper
+ echo "================"
+ touch $@
+
+$(BUILD_DIR)/git-$(VERSION)/osx-installed-bin: $(BUILD_DIR)/git-$(VERSION)/osx-built $(BUILD_DIR)/git-$(VERSION)/osx-built-keychain
+ cd $(BUILD_DIR)/git-$(VERSION); $(SUBMAKE) install
+ cp $(BUILD_DIR)/git-$(VERSION)/contrib/credential/osxkeychain/git-credential-osxkeychain $(DESTDIR)$(GIT_PREFIX)/bin/git-credential-osxkeychain
+ mkdir -p $(DESTDIR)$(GIT_PREFIX)/contrib/completion
+ cp $(BUILD_DIR)/git-$(VERSION)/contrib/completion/git-completion.bash $(DESTDIR)$(GIT_PREFIX)/contrib/completion/
+ cp $(BUILD_DIR)/git-$(VERSION)/contrib/completion/git-completion.zsh $(DESTDIR)$(GIT_PREFIX)/contrib/completion/
+ cp $(BUILD_DIR)/git-$(VERSION)/contrib/completion/git-prompt.sh $(DESTDIR)$(GIT_PREFIX)/contrib/completion/
+ # This is needed for Git-Gui, GitK
+ mkdir -p $(DESTDIR)$(GIT_PREFIX)/lib/perl5/site_perl
+ [ ! -f $(DESTDIR)$(GIT_PREFIX)/lib/perl5/site_perl/Error.pm ] && cp $(BUILD_DIR)/git-$(VERSION)/perl/private-Error.pm $(DESTDIR)$(GIT_PREFIX)/lib/perl5/site_perl/Error.pm || echo done
+ touch $@
+
+$(BUILD_DIR)/git-$(VERSION)/osx-installed-man: $(BUILD_DIR)/git-$(VERSION)/osx-installed-bin
+ mkdir -p $(DESTDIR)$(GIT_PREFIX)/share/man
+ cp -R $(GITHUB_WORKSPACE)/manpages/ $(DESTDIR)$(GIT_PREFIX)/share/man
+ touch $@
+
+$(BUILD_DIR)/git-$(VERSION)/osx-built-subtree:
+ cd $(BUILD_DIR)/git-$(VERSION)/contrib/subtree; $(SUBMAKE) XML_CATALOG_FILES="$(XML_CATALOG_FILES)" all git-subtree.1
+ touch $@
+
+$(BUILD_DIR)/git-$(VERSION)/osx-installed-subtree: $(BUILD_DIR)/git-$(VERSION)/osx-built-subtree
+ mkdir -p $(DESTDIR)
+ cd $(BUILD_DIR)/git-$(VERSION)/contrib/subtree; $(SUBMAKE) XML_CATALOG_FILES="$(XML_CATALOG_FILES)" install install-man
+ touch $@
+
+$(BUILD_DIR)/git-$(VERSION)/osx-installed-assets: $(BUILD_DIR)/git-$(VERSION)/osx-installed-bin
+ mkdir -p $(DESTDIR)$(GIT_PREFIX)/etc
+ cat assets/etc/gitconfig.osxkeychain >> $(DESTDIR)$(GIT_PREFIX)/etc/gitconfig
+ cp assets/uninstall.sh $(DESTDIR)$(GIT_PREFIX)/uninstall.sh
+ sh -c "echo .DS_Store >> $(DESTDIR)$(GIT_PREFIX)/share/git-core/templates/info/exclude"
+
+symlinks:
+ mkdir -p $(ARTIFACTDIR)$(PREFIX)/bin
+ cd $(ARTIFACTDIR)$(PREFIX)/bin; find ../git/bin -type f -exec ln -sf {} \;
+ for man in man1 man3 man5 man7; do mkdir -p $(ARTIFACTDIR)$(PREFIX)/share/man/$$man; (cd $(ARTIFACTDIR)$(PREFIX)/share/man/$$man; ln -sf ../../../git/share/man/$$man/* ./); done
+ ruby ../scripts/symlink-git-hardlinks.rb $(ARTIFACTDIR)
+ touch $@
+
+$(BUILD_DIR)/git-$(VERSION)/osx-installed: $(DESTDIR)$(GIT_PREFIX)/VERSION-$(VERSION)-$(ARCH_UNIV) $(BUILD_DIR)/git-$(VERSION)/osx-installed-man $(BUILD_DIR)/git-$(VERSION)/osx-installed-assets $(BUILD_DIR)/git-$(VERSION)/osx-installed-subtree
+ find $(DESTDIR)$(GIT_PREFIX) -type d -exec chmod ugo+rx {} \;
+ find $(DESTDIR)$(GIT_PREFIX) -type f -exec chmod ugo+r {} \;
+ touch $@
+
+$(BUILD_DIR)/git-$(VERSION)/osx-built-assert-$(ARCH_UNIV): $(BUILD_DIR)/git-$(VERSION)/osx-built
+ File $(BUILD_DIR)/git-$(VERSION)/git
+ File $(BUILD_DIR)/git-$(VERSION)/contrib/credential/osxkeychain/git-credential-osxkeychain
+ touch $@
+
+disk-image/VERSION-$(VERSION)-$(ARCH_UNIV):
+ rm -f disk-image/*.pkg disk-image/VERSION-* disk-image/.DS_Store
+ mkdir disk-image
+ touch "$@"
+
+pkg_cmd := pkgbuild --identifier com.git.pkg --version $(VERSION) \
+ --root $(ARTIFACTDIR)$(PREFIX) --scripts assets/scripts \
+ --install-location $(PREFIX) --component-plist ./assets/git-components.plist
+
+ifdef APPLE_INSTALLER_IDENTITY
+ pkg_cmd += --sign "$(APPLE_INSTALLER_IDENTITY)"
+endif
+
+pkg_cmd += disk-image/git-$(VERSION)-$(ARCH_UNIV).pkg
+disk-image/git-$(VERSION)-$(ARCH_UNIV).pkg: disk-image/VERSION-$(VERSION)-$(ARCH_UNIV) symlinks
+ $(pkg_cmd)
+
+git-%-$(ARCH_UNIV).dmg:
+ hdiutil create git-$(VERSION)-$(ARCH_UNIV).uncompressed.dmg -fs HFS+ -srcfolder disk-image -volname "Git $(VERSION) $(ARCH_UNIV)" -ov 2>&1 | tee err || { \
+ grep "Resource busy" err && \
+ sleep 5 && \
+ hdiutil create git-$(VERSION)-$(ARCH_UNIV).uncompressed.dmg -fs HFS+ -srcfolder disk-image -volname "Git $(VERSION) $(ARCH_UNIV)" -ov; }
+ hdiutil convert -format UDZO -o $@ git-$(VERSION)-$(ARCH_UNIV).uncompressed.dmg
+ rm -f git-$(VERSION)-$(ARCH_UNIV).uncompressed.dmg
+
+payload: $(BUILD_DIR)/git-$(VERSION)/osx-installed $(BUILD_DIR)/git-$(VERSION)/osx-built-assert-$(ARCH_UNIV)
+
+pkg: disk-image/git-$(VERSION)-$(ARCH_UNIV).pkg
+
+image: git-$(VERSION)-$(ARCH_UNIV).dmg
+
+ifdef APPLE_APP_IDENTITY
+codesign:
+ @$(CURDIR)/../scripts/codesign.sh --payload="build-artifacts/usr/local/git" \
+ --identity="$(APPLE_APP_IDENTITY)" \
+ --entitlements="$(CURDIR)/entitlements.xml"
+endif
+
+# Notarization can only happen if the package is fully signed
+ifdef APPLE_KEYCHAIN_PROFILE
+notarize:
+ @$(CURDIR)/../scripts/notarize.sh \
+ --package="disk-image/git-$(VERSION)-$(ARCH_UNIV).pkg" \
+ --keychain-profile="$(APPLE_KEYCHAIN_PROFILE)"
+endif
diff --git a/.github/macos-installer/assets/etc/gitconfig.osxkeychain b/.github/macos-installer/assets/etc/gitconfig.osxkeychain
new file mode 100644
index 00000000000000..788266b3a40a9d
--- /dev/null
+++ b/.github/macos-installer/assets/etc/gitconfig.osxkeychain
@@ -0,0 +1,2 @@
+[credential]
+ helper = osxkeychain
diff --git a/.github/macos-installer/assets/git-components.plist b/.github/macos-installer/assets/git-components.plist
new file mode 100644
index 00000000000000..78db36777df3ed
--- /dev/null
+++ b/.github/macos-installer/assets/git-components.plist
@@ -0,0 +1,18 @@
+
+
+
+
+
+ BundleHasStrictIdentifier
+
+ BundleIsRelocatable
+
+ BundleIsVersionChecked
+
+ BundleOverwriteAction
+ upgrade
+ RootRelativeBundlePath
+ git/share/git-gui/lib/Git Gui.app
+
+
+
diff --git a/.github/macos-installer/assets/scripts/postinstall b/.github/macos-installer/assets/scripts/postinstall
new file mode 100755
index 00000000000000..94056db9b7b864
--- /dev/null
+++ b/.github/macos-installer/assets/scripts/postinstall
@@ -0,0 +1,62 @@
+#!/bin/bash
+INSTALL_DST="$2"
+SCALAR_C_CMD="$INSTALL_DST/git/bin/scalar"
+SCALAR_DOTNET_CMD="/usr/local/scalar/scalar"
+SCALAR_UNINSTALL_SCRIPT="/usr/local/scalar/uninstall_scalar.sh"
+
+function cleanupScalar()
+{
+ echo "checking whether Scalar was installed"
+ if [ ! -f "$SCALAR_C_CMD" ]; then
+ echo "Scalar not installed; exiting..."
+ return 0
+ fi
+ echo "Scalar is installed!"
+
+ echo "looking for Scalar.NET"
+ if [ ! -f "$SCALAR_DOTNET_CMD" ]; then
+ echo "Scalar.NET not found; exiting..."
+ return 0
+ fi
+ echo "Scalar.NET found!"
+
+ currentUser=$(echo "show State:/Users/ConsoleUser" | scutil | awk '/Name :/ { print $3 }')
+
+ # Re-register Scalar.NET repositories with the newly-installed Scalar
+ for repo in $($SCALAR_DOTNET_CMD list); do
+ (
+ PATH="$INSTALL_DST/git/bin:$PATH"
+ sudo -u "$currentUser" scalar register $repo || \
+ echo "warning: skipping re-registration of $repo"
+ )
+ done
+
+ # Uninstall Scalar.NET
+ echo "removing Scalar.NET"
+
+ # Add /usr/local/bin to path - default install location of Homebrew
+ PATH="/usr/local/bin:$PATH"
+ if (sudo -u "$currentUser" brew list --cask scalar); then
+ # Remove from Homebrew
+ sudo -u "$currentUser" brew remove --cask scalar || echo "warning: Scalar.NET uninstall via Homebrew completed with code $?"
+ echo "Scalar.NET uninstalled via Homebrew!"
+ elif (sudo -u "$currentUser" brew list --cask scalar-azrepos); then
+ sudo -u "$currentUser" brew remove --cask scalar-azrepos || echo "warning: Scalar.NET with GVFS uninstall via Homebrew completed with code $?"
+ echo "Scalar.NET with GVFS uninstalled via Homebrew!"
+ elif [ -f $SCALAR_UNINSTALL_SCRIPT ]; then
+ # If not installed with Homebrew, manually remove package
+ sudo -S sh $SCALAR_UNINSTALL_SCRIPT || echo "warning: Scalar.NET uninstall completed with code $?"
+ echo "Scalar.NET uninstalled!"
+ else
+ echo "warning: Scalar.NET uninstall script not found"
+ fi
+
+ # Re-create the Scalar symlink, in case it was removed by the Scalar.NET uninstall operation
+ mkdir -p $INSTALL_DST/bin
+ /bin/ln -Fs "$SCALAR_C_CMD" "$INSTALL_DST/bin/scalar"
+}
+
+# Run Scalar cleanup (will exit if not applicable)
+cleanupScalar
+
+exit 0
\ No newline at end of file
diff --git a/.github/macos-installer/assets/uninstall.sh b/.github/macos-installer/assets/uninstall.sh
new file mode 100755
index 00000000000000..4fc79fbaa2e652
--- /dev/null
+++ b/.github/macos-installer/assets/uninstall.sh
@@ -0,0 +1,34 @@
+#!/bin/bash -e
+if [ ! -r "/usr/local/git" ]; then
+ echo "Git doesn't appear to be installed via this installer. Aborting"
+ exit 1
+fi
+
+if [ "$1" != "--yes" ]; then
+ echo "This will uninstall git by removing /usr/local/git/, and symlinks"
+ printf "Type 'yes' if you are sure you wish to continue: "
+ read response
+else
+ response="yes"
+fi
+
+if [ "$response" == "yes" ]; then
+ # remove all of the symlinks we've created
+ pkgutil --files com.git.pkg | grep bin | while read f; do
+ if [ -L /usr/local/$f ]; then
+ sudo rm /usr/local/$f
+ fi
+ done
+
+ # forget receipts.
+ pkgutil --packages | grep com.git.pkg | xargs -I {} sudo pkgutil --forget {}
+ echo "Uninstalled"
+
+ # The guts all go here.
+ sudo rm -rf /usr/local/git/
+else
+ echo "Aborted"
+ exit 1
+fi
+
+exit 0
diff --git a/.github/macos-installer/entitlements.xml b/.github/macos-installer/entitlements.xml
new file mode 100644
index 00000000000000..46f675661149b6
--- /dev/null
+++ b/.github/macos-installer/entitlements.xml
@@ -0,0 +1,12 @@
+
+
+
+
+ com.apple.security.cs.allow-jit
+
+ com.apple.security.cs.allow-unsigned-executable-memory
+
+ com.apple.security.cs.disable-library-validation
+
+
+
diff --git a/.github/scripts/codesign.sh b/.github/scripts/codesign.sh
new file mode 100755
index 00000000000000..076b29f93be45e
--- /dev/null
+++ b/.github/scripts/codesign.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+
+sign_directory () {
+ (
+ cd "$1"
+ for f in *
+ do
+ macho=$(file --mime $f | grep mach)
+ # Runtime sign dylibs and Mach-O binaries
+ if [[ $f == *.dylib ]] || [ ! -z "$macho" ];
+ then
+ echo "Runtime Signing $f"
+ codesign -s "$IDENTITY" $f --timestamp --force --options=runtime --entitlements $ENTITLEMENTS_FILE
+ elif [ -d "$f" ];
+ then
+ echo "Signing files in subdirectory $f"
+ sign_directory "$f"
+
+ else
+ echo "Signing $f"
+ codesign -s "$IDENTITY" $f --timestamp --force
+ fi
+ done
+ )
+}
+
+for i in "$@"
+do
+case "$i" in
+ --payload=*)
+ SIGN_DIR="${i#*=}"
+ shift # past argument=value
+ ;;
+ --identity=*)
+ IDENTITY="${i#*=}"
+ shift # past argument=value
+ ;;
+ --entitlements=*)
+ ENTITLEMENTS_FILE="${i#*=}"
+ shift # past argument=value
+ ;;
+ *)
+ die "unknown option '$i'"
+ ;;
+esac
+done
+
+if [ -z "$SIGN_DIR" ]; then
+ echo "error: missing directory argument"
+ exit 1
+elif [ -z "$IDENTITY" ]; then
+ echo "error: missing signing identity argument"
+ exit 1
+elif [ -z "$ENTITLEMENTS_FILE" ]; then
+ echo "error: missing entitlements file argument"
+ exit 1
+fi
+
+echo "======== INPUTS ========"
+echo "Directory: $SIGN_DIR"
+echo "Signing identity: $IDENTITY"
+echo "Entitlements: $ENTITLEMENTS_FILE"
+echo "======== END INPUTS ========"
+
+sign_directory "$SIGN_DIR"
diff --git a/.github/scripts/notarize.sh b/.github/scripts/notarize.sh
new file mode 100755
index 00000000000000..9315d688afbd49
--- /dev/null
+++ b/.github/scripts/notarize.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+for i in "$@"
+do
+case "$i" in
+ --package=*)
+ PACKAGE="${i#*=}"
+ shift # past argument=value
+ ;;
+ --keychain-profile=*)
+ KEYCHAIN_PROFILE="${i#*=}"
+ shift # past argument=value
+ ;;
+ *)
+ die "unknown option '$i'"
+ ;;
+esac
+done
+
+if [ -z "$PACKAGE" ]; then
+ echo "error: missing package argument"
+ exit 1
+elif [ -z "$KEYCHAIN_PROFILE" ]; then
+ echo "error: missing keychain profile argument"
+ exit 1
+fi
+
+# Exit as soon as any line fails
+set -e
+
+# Send the notarization request
+xcrun notarytool submit -v "$PACKAGE" -p "$KEYCHAIN_PROFILE" --wait
+
+# Staple the notarization ticket (to allow offline installation)
+xcrun stapler staple -v "$PACKAGE"
diff --git a/.github/scripts/symlink-git-hardlinks.rb b/.github/scripts/symlink-git-hardlinks.rb
new file mode 100644
index 00000000000000..174802ccc85d93
--- /dev/null
+++ b/.github/scripts/symlink-git-hardlinks.rb
@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+
+install_prefix = ARGV[0]
+puts install_prefix
+git_binary = File.join(install_prefix, '/usr/local/git/bin/git')
+
+[
+ ['git' , File.join(install_prefix, '/usr/local/git/bin')],
+ ['../../bin/git', File.join(install_prefix, '/usr/local/git/libexec/git-core')]
+].each do |link, path|
+ Dir.glob(File.join(path, '*')).each do |file|
+ next if file == git_binary
+ puts "#{file} #{File.size(file)} == #{File.size(git_binary)}"
+ next unless File.size(file) == File.size(git_binary)
+ puts "Symlinking #{file}"
+ puts `ln -sf #{link} #{file}`
+ exit $?.exitstatus if $?.exitstatus != 0
+ end
+end
\ No newline at end of file
diff --git a/.github/workflows/build-git-installers.yml b/.github/workflows/build-git-installers.yml
new file mode 100644
index 00000000000000..de13e7df7239eb
--- /dev/null
+++ b/.github/workflows/build-git-installers.yml
@@ -0,0 +1,812 @@
+name: build-git-installers
+
+on:
+ push:
+ tags:
+ - 'v[0-9]*vfs*' # matches "vvfs"
+
+permissions:
+ id-token: write # required for Azure login via OIDC
+
+jobs:
+ # Check prerequisites for the workflow
+ prereqs:
+ runs-on: ubuntu-latest
+ environment: release
+ outputs:
+ tag_name: ${{ steps.tag.outputs.name }} # The full name of the tag, e.g. v2.32.0.vfs.0.0
+ tag_version: ${{ steps.tag.outputs.version }} # The version number (without preceding "v"), e.g. 2.32.0.vfs.0.0
+ steps:
+ - name: Validate tag
+ run: |
+ echo "$GITHUB_REF" |
+ grep -E '^refs/tags/v2\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.vfs\.0\.(0|[1-9][0-9]*)(\.rc[0-9])?$' || {
+ echo "::error::${GITHUB_REF#refs/tags/} is not of the form v2...vfs.0.[.rc]" >&2
+ exit 1
+ }
+ - name: Determine tag to build
+ run: |
+ echo "name=${GITHUB_REF#refs/tags/}" >>$GITHUB_OUTPUT
+ echo "version=${GITHUB_REF#refs/tags/v}" >>$GITHUB_OUTPUT
+ id: tag
+ - name: Clone git
+ uses: actions/checkout@v4
+ - name: Validate the tag identified with trigger
+ run: |
+ die () {
+ echo "::error::$*" >&2
+ exit 1
+ }
+
+ # `actions/checkout` only downloads the peeled tag (i.e. the commit)
+ git fetch origin +$GITHUB_REF:$GITHUB_REF
+
+ # Verify that the tag is annotated
+ test $(git cat-file -t "$GITHUB_REF") == "tag" || die "Tag ${{ steps.tag.outputs.name }} is not annotated"
+
+ # Verify tag follows rules in GIT-VERSION-GEN (i.e., matches the specified "DEF_VER" in
+ # GIT-VERSION-FILE) and matches tag determined from trigger
+ make GIT-VERSION-FILE
+ test "${{ steps.tag.outputs.version }}" == "$(sed -n 's/^GIT_VERSION = //p'< GIT-VERSION-FILE)" || die "GIT-VERSION-FILE tag does not match ${{ steps.tag.outputs.name }}"
+ # End check prerequisites for the workflow
+
+ # Build Windows installers (x86_64 & aarch64; installer & portable)
+ windows_pkg:
+ environment: release
+ needs: prereqs
+ strategy:
+ fail-fast: false
+ matrix:
+ arch:
+ - name: x86_64
+ artifact: pkg-x86_64
+ toolchain: x86_64
+ mingwprefix: mingw64
+ runner: windows-2019
+ - name: aarch64
+ artifact: pkg-aarch64
+ toolchain: clang-aarch64
+ mingwprefix: clangarm64
+ runner: ['self-hosted', '1ES.Pool=github-arm64-pool']
+ runs-on: ${{ matrix.arch.runner }}
+ env:
+ GPG_OPTIONS: "--batch --yes --no-tty --list-options no-show-photos --verify-options no-show-photos --pinentry-mode loopback"
+ HOME: "${{github.workspace}}\\home"
+ USERPROFILE: "${{github.workspace}}\\home"
+ steps:
+ - name: Configure user
+ shell: bash
+ run:
+ USER_NAME="${{github.actor}}" &&
+ USER_EMAIL="${{github.actor}}@users.noreply.github.com" &&
+ mkdir -p "$HOME" &&
+ git config --global user.name "$USER_NAME" &&
+ git config --global user.email "$USER_EMAIL" &&
+ echo "PACKAGER=$USER_NAME <$USER_EMAIL>" >>$GITHUB_ENV
+ - uses: git-for-windows/setup-git-for-windows-sdk@v1
+ with:
+ flavor: build-installers
+ architecture: ${{ matrix.arch.name }}
+ - name: Clone build-extra
+ shell: bash
+ run: |
+ git clone --filter=blob:none --single-branch -b main https://github.com/git-for-windows/build-extra /usr/src/build-extra
+ - name: Clone git
+ shell: bash
+ run: |
+ # Since we cannot directly clone a specified tag (as we would a branch with `git clone -b `),
+ # this clone has to be done manually (via init->fetch->reset).
+
+ tag_name="${{ needs.prereqs.outputs.tag_name }}" &&
+ git -c init.defaultBranch=main init &&
+ git remote add -f origin https://github.com/git-for-windows/git &&
+ git fetch "https://github.com/${{github.repository}}" refs/tags/${tag_name}:refs/tags/${tag_name} &&
+ git reset --hard ${tag_name}
+ - name: Prepare home directory for code-signing
+ env:
+ CODESIGN_P12: ${{secrets.CODESIGN_P12}}
+ CODESIGN_PASS: ${{secrets.CODESIGN_PASS}}
+ if: env.CODESIGN_P12 != '' && env.CODESIGN_PASS != ''
+ shell: bash
+ run: |
+ cd home &&
+ mkdir -p .sig &&
+ echo -n "$CODESIGN_P12" | tr % '\n' | base64 -d >.sig/codesign.p12 &&
+ echo -n "$CODESIGN_PASS" >.sig/codesign.pass
+ git config --global alias.signtool '!sh "/usr/src/build-extra/signtool.sh"'
+ - name: Prepare home directory for GPG signing
+ if: env.GPGKEY != ''
+ shell: bash
+ run: |
+ # This section ensures that the identity for the GPG key matches the git user identity, otherwise
+ # signing will fail
+
+ echo '${{secrets.PRIVGPGKEY}}' | tr % '\n' | gpg $GPG_OPTIONS --import &&
+ info="$(gpg --list-keys --with-colons "${GPGKEY%% *}" | cut -d : -f 1,10 | sed -n '/^uid/{s|uid:||p;q}')" &&
+ git config --global user.name "${info% <*}" &&
+ git config --global user.email "<${info#*<}"
+ env:
+ GPGKEY: ${{secrets.GPGKEY}}
+ - name: Build mingw-w64-${{matrix.arch.toolchain}}-git
+ env:
+ GPGKEY: "${{secrets.GPGKEY}}"
+ shell: bash
+ run: |
+ set -x
+
+ # Make sure that there is a `/usr/bin/git` that can be used by `makepkg-mingw`
+ printf '#!/bin/sh\n\nexec /${{matrix.arch.mingwprefix}}/bin/git.exe "$@"\n' >/usr/bin/git &&
+
+ sh -x /usr/src/build-extra/please.sh build-mingw-w64-git --only-${{matrix.arch.name}} --build-src-pkg -o artifacts HEAD &&
+ if test -n "$GPGKEY"
+ then
+ for tar in artifacts/*.tar*
+ do
+ /usr/src/build-extra/gnupg-with-gpgkey.sh --detach-sign --no-armor $tar
+ done
+ fi &&
+
+ b=$PWD/artifacts &&
+ version=${{ needs.prereqs.outputs.tag_name }} &&
+ (cd /usr/src/MINGW-packages/mingw-w64-git &&
+ cp PKGBUILD.$version PKGBUILD &&
+ git commit -s -m "mingw-w64-git: new version ($version)" PKGBUILD &&
+ git bundle create "$b"/MINGW-packages.bundle origin/main..main)
+ - name: Publish mingw-w64-${{matrix.arch.toolchain}}-git
+ uses: actions/upload-artifact@v4
+ with:
+ name: "${{ matrix.arch.artifact }}"
+ path: artifacts
+ windows_artifacts:
+ environment: release
+ needs: [prereqs, windows_pkg]
+ env:
+ HOME: "${{github.workspace}}\\home"
+ strategy:
+ fail-fast: false
+ matrix:
+ arch:
+ - name: x86_64
+ artifact: pkg-x86_64
+ toolchain: x86_64
+ mingwprefix: mingw64
+ runner: windows-2019
+ - name: aarch64
+ artifact: pkg-aarch64
+ toolchain: clang-aarch64
+ mingwprefix: clangarm64
+ runner: ['self-hosted', '1ES.Pool=github-arm64-pool']
+ type:
+ - name: installer
+ fileprefix: Git
+ - name: portable
+ fileprefix: PortableGit
+ runs-on: ${{ matrix.arch.runner }}
+ steps:
+ - name: Download ${{ matrix.arch.artifact }}
+ uses: actions/download-artifact@v4
+ with:
+ name: ${{ matrix.arch.artifact }}
+ path: ${{ matrix.arch.artifact }}
+ - uses: git-for-windows/setup-git-for-windows-sdk@v1
+ with:
+ flavor: build-installers
+ architecture: ${{ matrix.arch.name }}
+ - name: Clone build-extra
+ shell: bash
+ run: |
+ git clone --filter=blob:none --single-branch -b main https://github.com/git-for-windows/build-extra /usr/src/build-extra
+ - name: Prepare home directory for code-signing
+ env:
+ CODESIGN_P12: ${{secrets.CODESIGN_P12}}
+ CODESIGN_PASS: ${{secrets.CODESIGN_PASS}}
+ if: env.CODESIGN_P12 != '' && env.CODESIGN_PASS != ''
+ shell: bash
+ run: |
+ mkdir -p home/.sig &&
+ echo -n "$CODESIGN_P12" | tr % '\n' | base64 -d >home/.sig/codesign.p12 &&
+ echo -n "$CODESIGN_PASS" >home/.sig/codesign.pass &&
+ git config --global alias.signtool '!sh "/usr/src/build-extra/signtool.sh"'
+ - name: Retarget auto-update to microsoft/git
+ shell: bash
+ run: |
+ set -x
+
+ b=/usr/src/build-extra &&
+
+ filename=$b/git-update-git-for-windows.config
+ tr % '\t' >$filename <<-\EOF &&
+ [update]
+ %fromFork = microsoft/git
+ EOF
+
+ sed -i -e '/^#include "file-list.iss"/a\
+ Source: {#SourcePath}\\..\\git-update-git-for-windows.config; DestDir: {app}\\${{matrix.arch.mingwprefix}}\\bin; Flags: replacesameversion; AfterInstall: DeleteFromVirtualStore' \
+ -e '/^Type: dirifempty; Name: {app}\\{#MINGW_BITNESS}$/i\
+ Type: files; Name: {app}\\{#MINGW_BITNESS}\\bin\\git-update-git-for-windows.config\
+ Type: dirifempty; Name: {app}\\{#MINGW_BITNESS}\\bin' \
+ $b/installer/install.iss
+ - name: Set alerts to continue until upgrade is taken
+ shell: bash
+ run: |
+ set -x
+
+ b=/${{matrix.arch.mingwprefix}}/bin &&
+
+ sed -i -e '6 a use_recently_seen=no' \
+ $b/git-update-git-for-windows
+ - name: Set the installer Publisher to the Git Client team
+ shell: bash
+ run: |
+ b=/usr/src/build-extra &&
+ sed -i -e 's/^\(AppPublisher=\).*/\1The Git Client Team at Microsoft/' $b/installer/install.iss
+ - name: Let the installer configure Visual Studio to use the installed Git
+ shell: bash
+ run: |
+ set -x
+
+ b=/usr/src/build-extra &&
+
+ sed -i -e '/^ *InstallAutoUpdater();$/a\
+ CustomPostInstall();' \
+ -e '/^ *UninstallAutoUpdater();$/a\
+ CustomPostUninstall();' \
+ $b/installer/install.iss &&
+
+ cat >>$b/installer/helpers.inc.iss <<\EOF
+
+ procedure CustomPostInstall();
+ begin
+ if not RegWriteStringValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\15.0\TeamFoundation\GitSourceControl','GitPath',ExpandConstant('{app}')) or
+ not RegWriteStringValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\16.0\TeamFoundation\GitSourceControl','GitPath',ExpandConstant('{app}')) or
+ not RegWriteStringValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\17.0\TeamFoundation\GitSourceControl','GitPath',ExpandConstant('{app}')) or
+ not RegWriteStringValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\18.0\TeamFoundation\GitSourceControl','GitPath',ExpandConstant('{app}')) or
+ not RegWriteStringValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\19.0\TeamFoundation\GitSourceControl','GitPath',ExpandConstant('{app}')) or
+ not RegWriteStringValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\20.0\TeamFoundation\GitSourceControl','GitPath',ExpandConstant('{app}')) then
+ LogError('Could not register TeamFoundation\GitSourceControl');
+ end;
+
+ procedure CustomPostUninstall();
+ begin
+ if not RegDeleteValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\15.0\TeamFoundation\GitSourceControl','GitPath') or
+ not RegDeleteValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\16.0\TeamFoundation\GitSourceControl','GitPath') or
+ not RegDeleteValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\17.0\TeamFoundation\GitSourceControl','GitPath') or
+ not RegDeleteValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\18.0\TeamFoundation\GitSourceControl','GitPath') or
+ not RegDeleteValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\19.0\TeamFoundation\GitSourceControl','GitPath') or
+ not RegDeleteValue(HKEY_CURRENT_USER,'Software\Microsoft\VSCommon\20.0\TeamFoundation\GitSourceControl','GitPath') then
+ LogError('Could not register TeamFoundation\GitSourceControl');
+ end;
+ EOF
+ - name: Enable Scalar/C and the auto-updater in the installer by default
+ shell: bash
+ run: |
+ set -x
+
+ b=/usr/src/build-extra &&
+
+ sed -i -e "/ChosenOptions:=''/a\\
+ if (ExpandConstant('{param:components|/}')='/') then begin\n\
+ WizardSelectComponents('autoupdate');\n\
+ #ifdef WITH_SCALAR\n\
+ WizardSelectComponents('scalar');\n\
+ #endif\n\
+ end;" $b/installer/install.iss
+ - name: Build ${{matrix.type.name}} (${{matrix.arch.name}})
+ shell: bash
+ run: |
+ set -x
+
+ # Copy the PDB archive to the directory where `--include-pdbs` expects it
+ b=/usr/src/build-extra &&
+ mkdir -p $b/cached-source-packages &&
+ cp ${{matrix.arch.artifact}}/*-pdb* $b/cached-source-packages/ &&
+
+ # Build the installer, embedding PDBs
+ eval $b/please.sh make_installers_from_mingw_w64_git --include-pdbs \
+ --version=${{ needs.prereqs.outputs.tag_version }} \
+ -o artifacts --${{matrix.type.name}} \
+ --pkg=${{matrix.arch.artifact}}/mingw-w64-${{matrix.arch.toolchain}}-git-[0-9]*.tar.xz \
+ --pkg=${{matrix.arch.artifact}}/mingw-w64-${{matrix.arch.toolchain}}-git-doc-html-[0-9]*.tar.xz &&
+
+ if test portable = '${{matrix.type.name}}' && test -n "$(git config alias.signtool)"
+ then
+ git signtool artifacts/PortableGit-*.exe
+ fi &&
+ openssl dgst -sha256 artifacts/${{matrix.type.fileprefix}}-*.exe | sed "s/.* //" >artifacts/sha-256.txt
+ - name: Verify that .exe files are code-signed
+ if: env.CODESIGN_P12 != '' && env.CODESIGN_PASS != ''
+ shell: bash
+ run: |
+ PATH=$PATH:"/c/Program Files (x86)/Windows Kits/10/App Certification Kit/" \
+ signtool verify //pa artifacts/${{matrix.type.fileprefix}}-*.exe
+ - name: Publish ${{matrix.type.name}}-${{matrix.arch.name}}
+ uses: actions/upload-artifact@v4
+ with:
+ name: win-${{matrix.type.name}}-${{matrix.arch.name}}
+ path: artifacts
+ # End build Windows installers
+
+ # Build and sign Mac OSX installers & upload artifacts
+ create-macos-artifacts:
+ runs-on: macos-latest-xl-arm64
+ needs: prereqs
+ env:
+ VERSION: "${{ needs.prereqs.outputs.tag_version }}"
+ environment: release
+ steps:
+ - name: Check out repository
+ uses: actions/checkout@v4
+ with:
+ path: 'git'
+
+ - name: Install Git dependencies
+ run: |
+ set -ex
+
+ # Install x86_64 packages
+ arch -x86_64 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
+ arch -x86_64 /usr/local/bin/brew install gettext
+
+ # Install arm64 packages
+ brew install automake asciidoc xmlto docbook
+ brew link --force gettext
+
+ # Make universal gettext library
+ lipo -create -output libintl.a /usr/local/opt/gettext/lib/libintl.a /opt/homebrew/opt/gettext/lib/libintl.a
+
+ - name: Set up signing/notarization infrastructure
+ env:
+ A1: ${{ secrets.APPLICATION_CERTIFICATE_BASE64 }}
+ A2: ${{ secrets.APPLICATION_CERTIFICATE_PASSWORD }}
+ I1: ${{ secrets.INSTALLER_CERTIFICATE_BASE64 }}
+ I2: ${{ secrets.INSTALLER_CERTIFICATE_PASSWORD }}
+ N1: ${{ secrets.APPLE_TEAM_ID }}
+ N2: ${{ secrets.APPLE_DEVELOPER_ID }}
+ N3: ${{ secrets.APPLE_DEVELOPER_PASSWORD }}
+ N4: ${{ secrets.APPLE_KEYCHAIN_PROFILE }}
+ run: |
+ echo "Setting up signing certificates"
+ security create-keychain -p pwd $RUNNER_TEMP/buildagent.keychain
+ security default-keychain -s $RUNNER_TEMP/buildagent.keychain
+ security unlock-keychain -p pwd $RUNNER_TEMP/buildagent.keychain
+ # Prevent re-locking
+ security set-keychain-settings $RUNNER_TEMP/buildagent.keychain
+
+ echo "$A1" | base64 -D > $RUNNER_TEMP/cert.p12
+ security import $RUNNER_TEMP/cert.p12 \
+ -k $RUNNER_TEMP/buildagent.keychain \
+ -P "$A2" \
+ -T /usr/bin/codesign
+ security set-key-partition-list \
+ -S apple-tool:,apple:,codesign: \
+ -s -k pwd \
+ $RUNNER_TEMP/buildagent.keychain
+
+ echo "$I1" | base64 -D > $RUNNER_TEMP/cert.p12
+ security import $RUNNER_TEMP/cert.p12 \
+ -k $RUNNER_TEMP/buildagent.keychain \
+ -P "$I2" \
+ -T /usr/bin/pkgbuild
+ security set-key-partition-list \
+ -S apple-tool:,apple:,pkgbuild: \
+ -s -k pwd \
+ $RUNNER_TEMP/buildagent.keychain
+
+ echo "Setting up notarytool"
+ xcrun notarytool store-credentials \
+ --team-id "$N1" \
+ --apple-id "$N2" \
+ --password "$N3" \
+ "$N4"
+
+ - name: Build, sign, and notarize artifacts
+ env:
+ A3: ${{ secrets.APPLE_APPLICATION_SIGNING_IDENTITY }}
+ I3: ${{ secrets.APPLE_INSTALLER_SIGNING_IDENTITY }}
+ N4: ${{ secrets.APPLE_KEYCHAIN_PROFILE }}
+ run: |
+ die () {
+ echo "$*" >&2
+ exit 1
+ }
+
+ # Trace execution, stop on error
+ set -ex
+
+ # Write to "version" file to force match with trigger payload version
+ echo "${{ needs.prereqs.outputs.tag_version }}" >>git/version
+
+ # Configure universal build
+ cat >git/config.mak <>git/config.mak <>git/config.mak <>git/config.mak
+
+ # To make use of the catalogs...
+ export XML_CATALOG_FILES=$homebrew_prefix/etc/xml/catalog
+
+ make -C git -j$(sysctl -n hw.physicalcpu) GIT-VERSION-FILE dist dist-doc
+
+ export GIT_BUILT_FROM_COMMIT=$(gunzip -c git/git-$VERSION.tar.gz | git get-tar-commit-id) ||
+ die "Could not determine commit for build"
+
+ # Extract tarballs
+ mkdir payload manpages
+ tar -xvf git/git-$VERSION.tar.gz -C payload
+ tar -xvf git/git-manpages-$VERSION.tar.gz -C manpages
+
+ # Lay out payload
+ cp git/config.mak payload/git-$VERSION/config.mak
+ make -C git/.github/macos-installer V=1 payload
+
+ # Codesign payload
+ cp -R stage/git-universal-$VERSION/ \
+ git/.github/macos-installer/build-artifacts
+ make -C git/.github/macos-installer V=1 codesign \
+ APPLE_APP_IDENTITY="$A3" || die "Creating signed payload failed"
+
+ # Build and sign pkg
+ make -C git/.github/macos-installer V=1 pkg \
+ APPLE_INSTALLER_IDENTITY="$I3" \
+ || die "Creating signed pkg failed"
+
+ # Notarize pkg
+ make -C git/.github/macos-installer V=1 notarize \
+ APPLE_INSTALLER_IDENTITY="$I3" APPLE_KEYCHAIN_PROFILE="$N4" \
+ || die "Creating signed and notarized pkg failed"
+
+ # Create DMG
+ make -C git/.github/macos-installer V=1 image || die "Creating DMG failed"
+
+ # Move all artifacts into top-level directory
+ mv git/.github/macos-installer/disk-image/*.pkg git/.github/macos-installer/
+
+ - name: Upload artifacts
+ uses: actions/upload-artifact@v4
+ with:
+ name: macos-artifacts
+ path: |
+ git/.github/macos-installer/*.dmg
+ git/.github/macos-installer/*.pkg
+ # End build and sign Mac OSX installers
+
+ # Build and sign Debian package
+ create-linux-artifacts:
+ runs-on: ubuntu-latest
+ needs: prereqs
+ environment: release
+ steps:
+ - name: Install git dependencies
+ run: |
+ set -ex
+ sudo apt-get update -q
+ sudo apt-get install -y -q --no-install-recommends gettext libcurl4-gnutls-dev libpcre3-dev asciidoc xmlto
+
+ - name: Clone git
+ uses: actions/checkout@v4
+ with:
+ path: git
+
+ - name: Build and create Debian package
+ run: |
+ set -ex
+
+ die () {
+ echo "$*" >&2
+ exit 1
+ }
+
+ echo "${{ needs.prereqs.outputs.tag_version }}" >>git/version
+ make -C git GIT-VERSION-FILE
+
+ VERSION="${{ needs.prereqs.outputs.tag_version }}"
+
+ ARCH="$(dpkg-architecture -q DEB_HOST_ARCH)"
+ if test -z "$ARCH"; then
+ die "Could not determine host architecture!"
+ fi
+
+ PKGNAME="microsoft-git_$VERSION"
+ PKGDIR="$(dirname $(pwd))/$PKGNAME"
+
+ rm -rf "$PKGDIR"
+ mkdir -p "$PKGDIR"
+
+ DESTDIR="$PKGDIR" make -C git -j5 V=1 DEVELOPER=1 \
+ USE_LIBPCRE=1 \
+ NO_CROSS_DIRECTORY_HARDLINKS=1 \
+ ASCIIDOC8=1 ASCIIDOC_NO_ROFF=1 \
+ ASCIIDOC='TZ=UTC asciidoc' \
+ prefix=/usr/local \
+ gitexecdir=/usr/local/lib/git-core \
+ libexecdir=/usr/local/lib/git-core \
+ htmldir=/usr/local/share/doc/git/html \
+ install install-doc install-html
+
+ cd ..
+ mkdir "$PKGNAME/DEBIAN"
+
+ # Based on https://packages.ubuntu.com/xenial/vcs/git
+ cat >"$PKGNAME/DEBIAN/control" <
+ Description: Git client built from the https://github.com/microsoft/git repository,
+ specialized in supporting monorepo scenarios. Includes the Scalar CLI.
+ EOF
+
+ dpkg-deb -Zxz --build "$PKGNAME"
+ # Move Debian package for later artifact upload
+ mv "$PKGNAME.deb" "$GITHUB_WORKSPACE"
+
+ - name: Log into Azure
+ uses: azure/login@v2
+ with:
+ client-id: ${{ secrets.AZURE_CLIENT_ID }}
+ tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+ subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+
+ - name: Prepare for GPG signing
+ env:
+ AZURE_VAULT: ${{ secrets.AZURE_VAULT }}
+ GPG_KEY_SECRET_NAME: ${{ secrets.GPG_KEY_SECRET_NAME }}
+ GPG_PASSPHRASE_SECRET_NAME: ${{ secrets.GPG_PASSPHRASE_SECRET_NAME }}
+ GPG_KEYGRIP_SECRET_NAME: ${{ secrets.GPG_KEYGRIP_SECRET_NAME }}
+ run: |
+ # Install debsigs
+ sudo apt install debsigs
+
+ # Download GPG key, passphrase, and keygrip from Azure Key Vault
+ key=$(az keyvault secret show --name $GPG_KEY_SECRET_NAME --vault-name $AZURE_VAULT --query "value")
+ passphrase=$(az keyvault secret show --name $GPG_PASSPHRASE_SECRET_NAME --vault-name $AZURE_VAULT --query "value")
+ keygrip=$(az keyvault secret show --name $GPG_KEYGRIP_SECRET_NAME --vault-name $AZURE_VAULT --query "value")
+
+ # Remove quotes from downloaded values
+ key=$(sed -e 's/^"//' -e 's/"$//' <<<"$key")
+ passphrase=$(sed -e 's/^"//' -e 's/"$//' <<<"$passphrase")
+ keygrip=$(sed -e 's/^"//' -e 's/"$//' <<<"$keygrip")
+
+ # Import GPG key
+ echo "$key" | base64 -d | gpg --import --no-tty --batch --yes
+
+ # Configure GPG
+ echo "allow-preset-passphrase" > ~/.gnupg/gpg-agent.conf
+ gpg-connect-agent RELOADAGENT /bye
+ /usr/lib/gnupg2/gpg-preset-passphrase --preset "$keygrip" <<<"$passphrase"
+
+ - name: Sign Debian package
+ run: |
+ # Sign Debian package
+ version="${{ needs.prereqs.outputs.tag_version }}"
+ debsigs --sign=origin --verify --check microsoft-git_"$version".deb
+
+ - name: Upload artifacts
+ uses: actions/upload-artifact@v4
+ with:
+ name: linux-artifacts
+ path: |
+ *.deb
+ # End build and sign Debian package
+
+ # Validate installers
+ validate-installers:
+ name: Validate installers
+ strategy:
+ matrix:
+ component:
+ - os: ubuntu-latest
+ artifact: linux-artifacts
+ command: git
+ - os: macos-latest-xl-arm64
+ artifact: macos-artifacts
+ command: git
+ - os: macos-latest
+ artifact: macos-artifacts
+ command: git
+ - os: windows-latest
+ artifact: win-installer-x86_64
+ command: $PROGRAMFILES\Git\cmd\git.exe
+ - os: ['self-hosted', '1ES.Pool=github-arm64-pool']
+ artifact: win-installer-aarch64
+ command: $PROGRAMFILES\Git\cmd\git.exe
+ runs-on: ${{ matrix.component.os }}
+ needs: [prereqs, windows_artifacts, create-macos-artifacts, create-linux-artifacts]
+ steps:
+ - name: Download artifacts
+ uses: actions/download-artifact@v4
+ with:
+ name: ${{ matrix.component.artifact }}
+
+ - name: Install Windows
+ if: contains(matrix.component.artifact, 'win-installer')
+ shell: pwsh
+ run: |
+ $exePath = Get-ChildItem -Path ./*.exe | %{$_.FullName}
+ Start-Process -Wait -FilePath "$exePath" -ArgumentList "/SILENT /VERYSILENT /NORESTART /SUPPRESSMSGBOXES /ALLOWDOWNGRADE=1"
+
+ - name: Install Linux
+ if: contains(matrix.component.artifact, 'linux')
+ run: |
+ debpath=$(find ./*.deb)
+ sudo apt install $debpath
+
+ - name: Install macOS
+ if: contains(matrix.component.artifact, 'macos')
+ run: |
+ # avoid letting Homebrew's `git` in `/opt/homebrew/bin` override `/usr/local/bin/git`
+ arch="$(uname -m)"
+ test arm64 != "$arch" ||
+ brew uninstall git
+
+ pkgpath=$(find ./*universal*.pkg)
+ sudo installer -pkg $pkgpath -target /
+
+ - name: Validate
+ shell: bash
+ run: |
+ "${{ matrix.component.command }}" --version | sed 's/git version //' >actual
+ echo ${{ needs.prereqs.outputs.tag_version }} >expect
+ cmp expect actual || exit 1
+
+ - name: Validate universal binary CPU architecture
+ if: contains(matrix.component.os, 'macos')
+ shell: bash
+ run: |
+ set -ex
+ git version --build-options >actual
+ cat actual
+ grep "cpu: $(uname -m)" actual
+ # End validate installers
+
+ create-github-release:
+ runs-on: ubuntu-latest
+ permissions:
+ contents: write
+ id-token: write # required for Azure login via OIDC
+ needs:
+ - validate-installers
+ - create-linux-artifacts
+ - create-macos-artifacts
+ - windows_artifacts
+ - prereqs
+ env:
+ AZURE_VAULT: ${{ secrets.AZURE_VAULT }}
+ GPG_PUBLIC_KEY_SECRET_NAME: ${{ secrets.GPG_PUBLIC_KEY_SECRET_NAME }}
+ environment: release
+ if: |
+ success() ||
+ (needs.create-linux-artifacts.result == 'skipped' &&
+ needs.create-macos-artifacts.result == 'success' &&
+ needs.windows_artifacts.result == 'success')
+ steps:
+ - name: Download Windows portable (x86_64)
+ uses: actions/download-artifact@v4
+ with:
+ name: win-portable-x86_64
+ path: win-portable-x86_64
+
+ - name: Download Windows portable (aarch64)
+ uses: actions/download-artifact@v4
+ with:
+ name: win-portable-aarch64
+ path: win-portable-aarch64
+
+ - name: Download Windows installer (x86_64)
+ uses: actions/download-artifact@v4
+ with:
+ name: win-installer-x86_64
+ path: win-installer-x86_64
+
+ - name: Download Windows installer (aarch64)
+ uses: actions/download-artifact@v4
+ with:
+ name: win-installer-aarch64
+ path: win-installer-aarch64
+
+ - name: Download macOS artifacts
+ uses: actions/download-artifact@v4
+ with:
+ name: macos-artifacts
+ path: macos-artifacts
+
+ - name: Download Debian package
+ uses: actions/download-artifact@v4
+ with:
+ name: linux-artifacts
+ path: deb-package
+
+ - name: Log into Azure
+ uses: azure/login@v2
+ with:
+ client-id: ${{ secrets.AZURE_CLIENT_ID }}
+ tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+ subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+
+ - name: Download GPG public key signature file
+ run: |
+ az keyvault secret show --name "$GPG_PUBLIC_KEY_SECRET_NAME" \
+ --vault-name "$AZURE_VAULT" --query "value" \
+ | sed -e 's/^"//' -e 's/"$//' | base64 -d >msft-git-public.asc
+ mv msft-git-public.asc deb-package
+
+ - uses: actions/github-script@v6
+ with:
+ script: |
+ const fs = require('fs');
+ const path = require('path');
+
+ var releaseMetadata = {
+ owner: context.repo.owner,
+ repo: context.repo.repo
+ };
+
+ // Create the release
+ var tagName = "${{ needs.prereqs.outputs.tag_name }}";
+ var createdRelease = await github.rest.repos.createRelease({
+ ...releaseMetadata,
+ draft: true,
+ tag_name: tagName,
+ name: tagName
+ });
+ releaseMetadata.release_id = createdRelease.data.id;
+
+ // Uploads contents of directory to the release created above
+ async function uploadDirectoryToRelease(directory, includeExtensions=[]) {
+ return fs.promises.readdir(directory)
+ .then(async(files) => Promise.all(
+ files.filter(file => {
+ return includeExtensions.length==0 || includeExtensions.includes(path.extname(file).toLowerCase());
+ })
+ .map(async (file) => {
+ var filePath = path.join(directory, file);
+ github.rest.repos.uploadReleaseAsset({
+ ...releaseMetadata,
+ name: file,
+ headers: {
+ "content-length": (await fs.promises.stat(filePath)).size
+ },
+ data: fs.createReadStream(filePath)
+ });
+ }))
+ );
+ }
+
+ await Promise.all([
+ // Upload Windows x86_64 artifacts
+ uploadDirectoryToRelease('win-installer-x86_64', ['.exe']),
+ uploadDirectoryToRelease('win-portable-x86_64', ['.exe']),
+
+ // Upload Windows aarch64 artifacts
+ uploadDirectoryToRelease('win-installer-aarch64', ['.exe']),
+ uploadDirectoryToRelease('win-portable-aarch64', ['.exe']),
+
+ // Upload Mac artifacts
+ uploadDirectoryToRelease('macos-artifacts'),
+
+ // Upload Ubuntu artifacts
+ uploadDirectoryToRelease('deb-package')
+ ]);
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index b1ed3794e2b2c8..787343b3b88db9 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -164,7 +164,7 @@ jobs:
vs-build:
name: win+VS build
needs: ci-config
- if: github.event.repository.owner.login == 'git-for-windows' && needs.ci-config.outputs.enabled == 'yes'
+ if: github.event.repository.owner.login == 'microsoft' && needs.ci-config.outputs.enabled == 'yes'
env:
NO_PERL: 1
GIT_CONFIG_PARAMETERS: "'user.name=CI' 'user.email=ci@git'"
diff --git a/.github/workflows/monitor-components.yml b/.github/workflows/monitor-components.yml
deleted file mode 100644
index fedc6add69a9e0..00000000000000
--- a/.github/workflows/monitor-components.yml
+++ /dev/null
@@ -1,98 +0,0 @@
-name: Monitor component updates
-
-# Git for Windows is a slightly modified subset of MSYS2. Some of its
-# components are maintained by Git for Windows, others by MSYS2. To help
-# keeping the former up to date, this workflow monitors the Atom/RSS feeds
-# and opens new tickets for each new component version.
-
-on:
- schedule:
- - cron: "23 8,11,14,17 * * *"
- workflow_dispatch:
-
-env:
- CHARACTER_LIMIT: 5000
- MAX_AGE: 7d
-
-jobs:
- job:
- # Only run this in Git for Windows' fork
- if: github.event.repository.owner.login == 'git-for-windows'
- runs-on: ubuntu-latest
- permissions:
- issues: write
- strategy:
- matrix:
- component:
- - label: git
- feed: https://github.com/git/git/tags.atom
- - label: git-lfs
- feed: https://github.com/git-lfs/git-lfs/tags.atom
- - label: git-credential-manager
- feed: https://github.com/git-ecosystem/git-credential-manager/tags.atom
- - label: tig
- feed: https://github.com/jonas/tig/tags.atom
- - label: cygwin
- feed: https://github.com/cygwin/cygwin/releases.atom
- title-pattern: ^(?!.*newlib)
- - label: msys2-runtime-package
- feed: https://github.com/msys2/MSYS2-packages/commits/master/msys2-runtime.atom
- - label: msys2-runtime
- feed: https://github.com/msys2/msys2-runtime/commits/HEAD.atom
- aggregate: true
- - label: openssh
- feed: https://github.com/openssh/openssh-portable/tags.atom
- - label: libfido2
- feed: https://github.com/Yubico/libfido2/tags.atom
- - label: libcbor
- feed: https://github.com/PJK/libcbor/tags.atom
- - label: openssl
- feed: https://github.com/openssl/openssl/tags.atom
- title-pattern: ^(?!.*alpha)
- - label: gnutls
- feed: https://gnutls.org/news.atom
- - label: heimdal
- feed: https://github.com/heimdal/heimdal/tags.atom
- - label: git-sizer
- feed: https://github.com/github/git-sizer/tags.atom
- - label: gitflow
- feed: https://github.com/petervanderdoes/gitflow-avh/tags.atom
- - label: curl
- feed: https://github.com/curl/curl/tags.atom
- - label: libgpg-error
- feed: https://github.com/gpg/libgpg-error/releases.atom
- title-pattern: ^libgpg-error-[0-9\.]*$
- - label: libgcrypt
- feed: https://github.com/gpg/libgcrypt/releases.atom
- title-pattern: ^libgcrypt-[0-9\.]*$
- - label: gpg
- feed: https://github.com/gpg/gnupg/releases.atom
- - label: mintty
- feed: https://github.com/mintty/mintty/releases.atom
- - label: 7-zip
- feed: https://sourceforge.net/projects/sevenzip/rss?path=/7-Zip
- aggregate: true
- - label: bash
- feed: https://git.savannah.gnu.org/cgit/bash.git/atom/?h=master
- aggregate: true
- - label: perl
- feed: https://github.com/Perl/perl5/tags.atom
- title-pattern: ^(?!.*(5\.[0-9]+[13579]|RC))
- - label: pcre2
- feed: https://github.com/PCRE2Project/pcre2/tags.atom
- - label: mingw-w64-llvm
- feed: https://github.com/msys2/MINGW-packages/commits/master/mingw-w64-llvm.atom
- - label: innosetup
- feed: https://github.com/jrsoftware/issrc/tags.atom
- fail-fast: false
- steps:
- - uses: git-for-windows/rss-to-issues@v0
- with:
- feed: ${{matrix.component.feed}}
- prefix: "[New ${{matrix.component.label}} version]"
- labels: component-update
- github-token: ${{ secrets.GITHUB_TOKEN }}
- character-limit: ${{ env.CHARACTER_LIMIT }}
- max-age: ${{ env.MAX_AGE }}
- aggregate: ${{matrix.component.aggregate}}
- title-pattern: ${{matrix.component.title-pattern}}
diff --git a/.github/workflows/release-homebrew.yml b/.github/workflows/release-homebrew.yml
new file mode 100644
index 00000000000000..e00b90d8c07579
--- /dev/null
+++ b/.github/workflows/release-homebrew.yml
@@ -0,0 +1,51 @@
+name: Update Homebrew Tap
+on:
+ release:
+ types: [released]
+
+permissions:
+ id-token: write # required for Azure login via OIDC
+
+jobs:
+ release:
+ runs-on: ubuntu-latest
+ environment: release
+ steps:
+ - id: version
+ name: Compute version number
+ run: |
+ echo "result=$(echo $GITHUB_REF | sed -e "s/^refs\/tags\/v//")" >>$GITHUB_OUTPUT
+ - id: hash
+ name: Compute release asset hash
+ uses: mjcheetham/asset-hash@v1.1
+ with:
+ asset: /git-(.*)\.pkg/
+ hash: sha256
+ token: ${{ secrets.GITHUB_TOKEN }}
+ - name: Log into Azure
+ uses: azure/login@v2
+ with:
+ client-id: ${{ secrets.AZURE_CLIENT_ID }}
+ tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+ subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+ - name: Retrieve token
+ id: token
+ run: |
+ az keyvault secret show \
+ --name ${{ secrets.HOMEBREW_TOKEN_SECRET_NAME }} \
+ --vault-name ${{ secrets.AZURE_VAULT }} \
+ --query "value" -o tsv >token &&
+ # avoid outputting the token under `set -x` by using `sed` instead of `echo`
+ sed s/^/::add-mask::/ >$GITHUB_OUTPUT &&
+ rm token
+ - name: Update scalar Cask
+ uses: mjcheetham/update-homebrew@v1.4
+ with:
+ token: ${{ steps.token.outputs.result }}
+ tap: microsoft/git
+ name: microsoft-git
+ type: cask
+ version: ${{ steps.version.outputs.result }}
+ sha256: ${{ steps.hash.outputs.result }}
+ alwaysUsePullRequest: false
diff --git a/.github/workflows/release-winget.yml b/.github/workflows/release-winget.yml
new file mode 100644
index 00000000000000..d6edab844d05b5
--- /dev/null
+++ b/.github/workflows/release-winget.yml
@@ -0,0 +1,51 @@
+name: "release-winget"
+on:
+ release:
+ types: [released]
+
+ workflow_dispatch:
+ inputs:
+ release:
+ description: 'Release Id'
+ required: true
+ default: 'latest'
+
+permissions:
+ id-token: write # required for Azure login via OIDC
+
+jobs:
+ release:
+ runs-on: windows-latest
+ environment: release
+ steps:
+ - name: Log into Azure
+ uses: azure/login@v2
+ with:
+ client-id: ${{ secrets.AZURE_CLIENT_ID }}
+ tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+ subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+
+ - name: Publish manifest with winget-create
+ run: |
+ # Get correct release asset
+ $github = Get-Content '${{ github.event_path }}' | ConvertFrom-Json
+ $asset = $github.release.assets | Where-Object -Property name -match '64-bit.exe$'
+
+ # Remove 'v' and 'vfs' from the version
+ $github.release.tag_name -match '\d.*'
+ $version = $Matches[0] -replace ".vfs",""
+
+ # Download wingetcreate and create manifests
+ Invoke-WebRequest https://aka.ms/wingetcreate/latest -OutFile wingetcreate.exe
+ .\wingetcreate.exe update Microsoft.Git -u $asset.browser_download_url -v $version -o manifests
+
+ # Manually substitute the name of the default branch in the License
+ # and Copyright URLs since the tooling cannot do that for us.
+ $shortenedVersion = $version -replace ".{4}$"
+ $manifestPath = dir -Path ./manifests -Filter Microsoft.Git.locale.en-US.yaml -Recurse | %{$_.FullName}
+ sed -i "s/vfs-[.0-9]*/vfs-$shortenedVersion/g" "$manifestPath"
+
+ # Submit manifests
+ $manifestDirectory = Split-Path "$manifestPath"
+ .\wingetcreate.exe submit -t "(az keyvault secret show --name ${{ secrets.WINGET_TOKEN_SECRET_NAME }} --vault-name ${{ secrets.AZURE_VAULT }} --query "value")" $manifestDirectory
+ shell: powershell
diff --git a/.github/workflows/scalar-functional-tests.yml b/.github/workflows/scalar-functional-tests.yml
new file mode 100644
index 00000000000000..a5946bc33939d6
--- /dev/null
+++ b/.github/workflows/scalar-functional-tests.yml
@@ -0,0 +1,220 @@
+name: Scalar Functional Tests
+
+env:
+ SCALAR_REPOSITORY: microsoft/scalar
+ SCALAR_REF: main
+ DEBUG_WITH_TMATE: false
+ SCALAR_TEST_SKIP_VSTS_INFO: true
+
+on:
+ push:
+ branches: [ vfs-*, tentative/vfs-* ]
+ pull_request:
+ branches: [ vfs-*, features/* ]
+
+jobs:
+ scalar:
+ name: "Scalar Functional Tests"
+
+ strategy:
+ fail-fast: false
+ matrix:
+ # Order by runtime (in descending order)
+ os: [windows-2019, macos-13, ubuntu-20.04, ubuntu-22.04]
+ # Scalar.NET used to be tested using `features: [false, experimental]`
+ # But currently, Scalar/C ignores `feature.scalar` altogether, so let's
+ # save some electrons and run only one of them...
+ features: [ignored]
+ exclude:
+ # The built-in FSMonitor is not (yet) supported on Linux
+ - os: ubuntu-20.04
+ features: experimental
+ - os: ubuntu-22.04
+ features: experimental
+ runs-on: ${{ matrix.os }}
+
+ env:
+ BUILD_FRAGMENT: bin/Release/netcoreapp3.1
+ GIT_FORCE_UNTRACKED_CACHE: 1
+
+ steps:
+ - name: Check out Git's source code
+ uses: actions/checkout@v4
+
+ - name: Setup build tools on Windows
+ if: runner.os == 'Windows'
+ uses: git-for-windows/setup-git-for-windows-sdk@v1
+
+ - name: Provide a minimal `install` on Windows
+ if: runner.os == 'Windows'
+ shell: bash
+ run: |
+ test -x /usr/bin/install ||
+ tr % '\t' >/usr/bin/install <<-\EOF
+ #!/bin/sh
+
+ cmd=cp
+ while test $# != 0
+ do
+ %case "$1" in
+ %-d) cmd="mkdir -p";;
+ %-m) shift;; # ignore mode
+ %*) break;;
+ %esac
+ %shift
+ done
+
+ exec $cmd "$@"
+ EOF
+
+ - name: Install build dependencies for Git (Linux)
+ if: runner.os == 'Linux'
+ run: |
+ sudo apt-get update
+ sudo apt-get -q -y install libssl-dev libcurl4-openssl-dev gettext
+
+ - name: Build and install Git
+ shell: bash
+ env:
+ NO_TCLTK: Yup
+ run: |
+ # We do require a VFS version
+ def_ver="$(sed -n 's/DEF_VER=\(.*vfs.*\)/\1/p' GIT-VERSION-GEN)"
+ test -n "$def_ver"
+
+ # Ensure that `git version` reflects DEF_VER
+ case "$(git describe --match "v[0-9]*vfs*" HEAD)" in
+ ${def_ver%%.vfs.*}.vfs.*) ;; # okay, we can use this
+ *) git -c user.name=ci -c user.email=ci@github tag -m for-testing ${def_ver}.NNN.g$(git rev-parse --short HEAD);;
+ esac
+
+ SUDO=
+ extra=
+ case "${{ runner.os }}" in
+ Windows)
+ extra=DESTDIR=/c/Progra~1/Git
+ cygpath -aw "/c/Program Files/Git/cmd" >>$GITHUB_PATH
+ ;;
+ Linux)
+ SUDO=sudo
+ extra=prefix=/usr
+ ;;
+ macOS)
+ SUDO=sudo
+ extra=prefix=/usr/local
+ ;;
+ esac
+
+ $SUDO make -j5 $extra install
+
+ - name: Ensure that we use the built Git and Scalar
+ shell: bash
+ run: |
+ type -p git
+ git version
+ case "$(git version)" in *.vfs.*) echo Good;; *) exit 1;; esac
+ type -p scalar
+ scalar version
+ case "$(scalar version 2>&1)" in *.vfs.*) echo Good;; *) exit 1;; esac
+
+ - name: Check out Scalar's source code
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 0 # Indicate full history so Nerdbank.GitVersioning works.
+ path: scalar
+ repository: ${{ env.SCALAR_REPOSITORY }}
+ ref: ${{ env.SCALAR_REF }}
+
+ - name: Setup .NET Core
+ uses: actions/setup-dotnet@v4
+ with:
+ dotnet-version: '3.1.x'
+
+ - name: Install dependencies
+ run: dotnet restore
+ working-directory: scalar
+ env:
+ DOTNET_NOLOGO: 1
+
+ - name: Build
+ working-directory: scalar
+ run: dotnet build --configuration Release --no-restore -p:UseAppHost=true # Force generation of executable on macOS.
+
+ - name: Setup platform (Linux)
+ if: runner.os == 'Linux'
+ run: |
+ echo "BUILD_PLATFORM=${{ runner.os }}" >>$GITHUB_ENV
+ echo "TRACE2_BASENAME=Trace2.${{ github.run_id }}__${{ github.run_number }}__${{ matrix.os }}__${{ matrix.features }}" >>$GITHUB_ENV
+
+ - name: Setup platform (Mac)
+ if: runner.os == 'macOS'
+ run: |
+ echo 'BUILD_PLATFORM=Mac' >>$GITHUB_ENV
+ echo "TRACE2_BASENAME=Trace2.${{ github.run_id }}__${{ github.run_number }}__${{ matrix.os }}__${{ matrix.features }}" >>$GITHUB_ENV
+
+ - name: Setup platform (Windows)
+ if: runner.os == 'Windows'
+ run: |
+ echo "BUILD_PLATFORM=${{ runner.os }}" >>$env:GITHUB_ENV
+ echo 'BUILD_FILE_EXT=.exe' >>$env:GITHUB_ENV
+ echo "TRACE2_BASENAME=Trace2.${{ github.run_id }}__${{ github.run_number }}__${{ matrix.os }}__${{ matrix.features }}" >>$env:GITHUB_ENV
+
+ - name: Configure feature.scalar
+ run: git config --global feature.scalar ${{ matrix.features }}
+
+ - id: functional_test
+ name: Functional test
+ timeout-minutes: 60
+ working-directory: scalar
+ shell: bash
+ run: |
+ export GIT_TRACE2_EVENT="$PWD/$TRACE2_BASENAME/Event"
+ export GIT_TRACE2_PERF="$PWD/$TRACE2_BASENAME/Perf"
+ export GIT_TRACE2_EVENT_BRIEF=true
+ export GIT_TRACE2_PERF_BRIEF=true
+ mkdir -p "$TRACE2_BASENAME"
+ mkdir -p "$TRACE2_BASENAME/Event"
+ mkdir -p "$TRACE2_BASENAME/Perf"
+ git version --build-options
+ cd ../out
+ Scalar.FunctionalTests/$BUILD_FRAGMENT/Scalar.FunctionalTests$BUILD_FILE_EXT --test-scalar-on-path --test-git-on-path --timeout=300000 --full-suite
+
+ - name: Force-stop FSMonitor daemons and Git processes (Windows)
+ if: runner.os == 'Windows' && (success() || failure())
+ shell: bash
+ run: |
+ set -x
+ wmic process get CommandLine,ExecutablePath,HandleCount,Name,ParentProcessID,ProcessID
+ wmic process where "CommandLine Like '%fsmonitor--daemon %run'" delete
+ wmic process where "ExecutablePath Like '%git.exe'" delete
+
+ - id: trace2_zip_unix
+ if: runner.os != 'Windows' && ( success() || failure() ) && ( steps.functional_test.conclusion == 'success' || steps.functional_test.conclusion == 'failure' )
+ name: Zip Trace2 Logs (Unix)
+ shell: bash
+ working-directory: scalar
+ run: zip -q -r $TRACE2_BASENAME.zip $TRACE2_BASENAME/
+
+ - id: trace2_zip_windows
+ if: runner.os == 'Windows' && ( success() || failure() ) && ( steps.functional_test.conclusion == 'success' || steps.functional_test.conclusion == 'failure' )
+ name: Zip Trace2 Logs (Windows)
+ working-directory: scalar
+ run: Compress-Archive -DestinationPath ${{ env.TRACE2_BASENAME }}.zip -Path ${{ env.TRACE2_BASENAME }}
+
+ - name: Archive Trace2 Logs
+ if: ( success() || failure() ) && ( steps.trace2_zip_unix.conclusion == 'success' || steps.trace2_zip_windows.conclusion == 'success' )
+ uses: actions/upload-artifact@v3
+ with:
+ name: ${{ env.TRACE2_BASENAME }}.zip
+ path: scalar/${{ env.TRACE2_BASENAME }}.zip
+ retention-days: 3
+
+ # The GitHub Action `action-tmate` allows developers to connect to the running agent
+ # using SSH (it will be a `tmux` session; on Windows agents it will be inside the MSYS2
+ # environment in `C:\msys64`, therefore it can be slightly tricky to interact with
+ # Git for Windows, which runs a slightly incompatible MSYS2 runtime).
+ - name: action-tmate
+ if: env.DEBUG_WITH_TMATE == 'true' && failure()
+ uses: mxschmitt/action-tmate@v3
+ with:
+ limit-access-to-actor: true
diff --git a/.gitignore b/.gitignore
index 9dd1fbc61e6c82..3e0b4d016cb1ce 100644
--- a/.gitignore
+++ b/.gitignore
@@ -74,6 +74,7 @@
/git-gc
/git-get-tar-commit-id
/git-grep
+/git-gvfs-helper
/git-hash-object
/git-help
/git-hook
@@ -173,6 +174,7 @@
/git-unpack-file
/git-unpack-objects
/git-update-index
+/git-update-microsoft-git
/git-update-ref
/git-update-server-info
/git-upload-archive
diff --git a/BRANCHES.md b/BRANCHES.md
new file mode 100644
index 00000000000000..364158375e7d55
--- /dev/null
+++ b/BRANCHES.md
@@ -0,0 +1,59 @@
+Branches used in this repo
+==========================
+
+The document explains the branching structure that we are using in the VFSForGit repository as well as the forking strategy that we have adopted for contributing.
+
+Repo Branches
+-------------
+
+1. `vfs-#`
+
+ These branches are used to track the specific version that match Git for Windows with the VFSForGit specific patches on top. When a new version of Git for Windows is released, the VFSForGit patches will be rebased on that windows version and a new gvfs-# branch created to create pull requests against.
+
+ #### Examples
+
+ ```
+ vfs-2.27.0
+ vfs-2.30.0
+ ```
+
+ The versions of git for VFSForGit are based on the Git for Windows versions. v2.20.0.vfs.1 will correspond with the v2.20.0.windows.1 with the VFSForGit specific patches applied to the windows version.
+
+2. `vfs-#-exp`
+
+ These branches are for releasing experimental features to early adopters. They
+ should contain everything within the corresponding `vfs-#` branch; if the base
+ branch updates, then merge into the `vfs-#-exp` branch as well.
+
+Tags
+----
+
+We are using annotated tags to build the version number for git. The build will look back through the commit history to find the first tag matching `v[0-9]*vfs*` and build the git version number using that tag.
+
+Full releases are of the form `v2.XX.Y.vfs.Z.W` where `v2.XX.Y` comes from the
+upstream version and `Z.W` are custom updates within our fork. Specifically,
+the `.Z` value represents the "compatibility level" with VFS for Git. Only
+increase this version when making a breaking change with a released version
+of VFS for Git. The `.W` version is used for minor updates between major
+versions.
+
+Experimental releases are of the form `v2.XX.Y.vfs.Z.W.exp`. The `.exp`
+suffix indicates that experimental features are available. The rest of the
+version string comes from the full release tag. These versions will only
+be made available as pre-releases on the releases page, never a full release.
+
+Forking
+-------
+
+A personal fork of this repository and a branch in that repository should be used for development.
+
+These branches should be based on the latest vfs-# branch. If there are work in progress pull requests that you have based on a previous version branch when a new version branch is created, you will need to move your patches to the new branch to get them in that latest version.
+
+#### Example
+
+```
+git clone
+git remote add ms https://github.com/Microsoft/git.git
+git checkout -b my-changes ms/vfs-2.20.0 --no-track
+git push -fu origin HEAD
+```
diff --git a/Documentation/config.txt b/Documentation/config.txt
index 939cc1387992f8..b87cb7a593b368 100644
--- a/Documentation/config.txt
+++ b/Documentation/config.txt
@@ -448,6 +448,8 @@ include::config/gui.txt[]
include::config/guitool.txt[]
+include::config/gvfs.txt[]
+
include::config/help.txt[]
include::config/http.txt[]
diff --git a/Documentation/config/core.txt b/Documentation/config/core.txt
index b633053f799a9c..ba5b246dfe55d0 100644
--- a/Documentation/config/core.txt
+++ b/Documentation/config/core.txt
@@ -111,6 +111,14 @@ Version 2 uses an opaque string so that the monitor can return
something that can be used to determine what files have changed
without race conditions.
+core.virtualFilesystem::
+ If set, the value of this variable is used as a command which
+ will identify all files and directories that are present in
+ the working directory. Git will only track and update files
+ listed in the virtual file system. Using the virtual file system
+ will supersede the sparse-checkout settings which will be ignored.
+ See the "virtual file system" section of linkgit:githooks[5].
+
core.trustctime::
If false, the ctime differences between the index and the
working tree are ignored; useful when the inode change time
@@ -743,6 +751,55 @@ core.multiPackIndex::
single index. See linkgit:git-multi-pack-index[1] for more
information. Defaults to true.
+core.gvfs::
+ Enable the features needed for GVFS. This value can be set to true
+ to indicate all features should be turned on or the bit values listed
+ below can be used to turn on specific features.
++
+--
+ GVFS_SKIP_SHA_ON_INDEX::
+ Bit value 1
+ Disables the calculation of the sha when writing the index
+ GVFS_MISSING_OK::
+ Bit value 4
+ Normally git write-tree ensures that the objects referenced by the
+ directory exist in the object database. This option disables this check.
+ GVFS_NO_DELETE_OUTSIDE_SPARSECHECKOUT::
+ Bit value 8
+ When marking entries to remove from the index and the working
+ directory this option will take into account what the
+ skip-worktree bit was set to so that if the entry has the
+ skip-worktree bit set it will not be removed from the working
+ directory. This will allow virtualized working directories to
+ detect the change to HEAD and use the new commit tree to show
+ the files that are in the working directory.
+ GVFS_FETCH_SKIP_REACHABILITY_AND_UPLOADPACK::
+ Bit value 16
+ While performing a fetch with a virtual file system we know
+ that there will be missing objects and we don't want to download
+ them just because of the reachability of the commits. We also
+ don't want to download a pack file with commits, trees, and blobs
+ since these will be downloaded on demand. This flag will skip the
+ checks on the reachability of objects during a fetch as well as
+ the upload pack so that extraneous objects don't get downloaded.
+ GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS::
+ Bit value 64
+ With a virtual file system we only know the file size before any
+ CRLF or smudge/clean filters processing is done on the client.
+ To prevent file corruption due to truncation or expansion with
+ garbage at the end, these filters must not run when the file
+ is first accessed and brought down to the client. Git.exe can't
+ currently tell the first access vs subsequent accesses so this
+ flag just blocks them from occurring at all.
+ GVFS_PREFETCH_DURING_FETCH::
+ Bit value 128
+ While performing a `git fetch` command, use the gvfs-helper to
+ perform a "prefetch" of commits and trees.
+--
+
+core.useGvfsHelper::
+ TODO
+
core.sparseCheckout::
Enable "sparse checkout" feature. See linkgit:git-sparse-checkout[1]
for more information.
@@ -777,3 +834,12 @@ core.WSLCompat::
The default value is false. When set to true, Git will set the mode
bits of the file in the way of wsl, so that the executable flag of
files can be set or read correctly.
+
+core.configWriteLockTimeoutMS::
+ When processes try to write to the config concurrently, it is likely
+ that one process "wins" and the other process(es) fail to lock the
+ config file. By configuring a timeout larger than zero, Git can be
+ told to try to lock the config again a couple times within the
+ specified timeout. If the timeout is configure to zero (which is the
+ default), Git will fail immediately when the config is already
+ locked.
diff --git a/Documentation/config/gvfs.txt b/Documentation/config/gvfs.txt
new file mode 100644
index 00000000000000..7224939ac0b270
--- /dev/null
+++ b/Documentation/config/gvfs.txt
@@ -0,0 +1,10 @@
+gvfs.cache-server::
+ TODO
+
+gvfs.sharedcache::
+ TODO
+
+gvfs.fallback::
+ If set to `false`, then never fallback to the origin server when the cache
+ server fails to connect. This will alert users to failures with the cache
+ server, but avoid causing throttling on the origin server.
diff --git a/Documentation/config/index.txt b/Documentation/config/index.txt
index 3eff42036033ea..0d6d05b70ce03d 100644
--- a/Documentation/config/index.txt
+++ b/Documentation/config/index.txt
@@ -1,3 +1,9 @@
+index.deleteSparseDirectories::
+ When enabled, the cone mode sparse-checkout feature will delete
+ directories that are outside of the sparse-checkout cone, unless
+ such a directory contains an untracked, non-ignored file. Defaults
+ to true.
+
index.recordEndOfIndexEntries::
Specifies whether the index file should include an "End Of Index
Entry" section. This reduces index load time on multiprocessor
diff --git a/Documentation/config/status.txt b/Documentation/config/status.txt
index 8caf90f51c19a3..4d863fdaaec2eb 100644
--- a/Documentation/config/status.txt
+++ b/Documentation/config/status.txt
@@ -77,3 +77,25 @@ status.submoduleSummary::
the --ignore-submodules=dirty command-line option or the 'git
submodule summary' command, which shows a similar output but does
not honor these settings.
+
+status.deserializePath::
+ EXPERIMENTAL, Pathname to a file containing cached status results
+ generated by `--serialize`. This will be overridden by
+ `--deserialize=` on the command line. If the cache file is
+ invalid or stale, git will fall-back and compute status normally.
+
+status.deserializeWait::
+ EXPERIMENTAL, Specifies what `git status --deserialize` should do
+ if the serialization cache file is stale and whether it should
+ fall-back and compute status normally. This will be overridden by
+ `--deserialize-wait=` on the command line.
++
+--
+* `fail` - cause git to exit with an error when the status cache file
+is stale; this is intended for testing and debugging.
+* `block` - cause git to spin and periodically retry the cache file
+every 100 ms; this is intended to help coordinate with another git
+instance concurrently computing the cache file.
+* `no` - to immediately fall-back if cache file is stale. This is the default.
+* `` - time (in tenths of a second) to spin and retry.
+--
diff --git a/Documentation/config/survey.txt b/Documentation/config/survey.txt
index 9e594a2092f225..f3ae768933fe1b 100644
--- a/Documentation/config/survey.txt
+++ b/Documentation/config/survey.txt
@@ -4,6 +4,10 @@ survey.*::
background with these options.
+
--
+ survey.namerev::
+ Boolean to show/hide `git name-rev` information for each
+ reported commit and the containing commit of each
+ reported tree and blob.
verbose::
This boolean value implies the `--[no-]verbose` option.
progress::
@@ -11,4 +15,33 @@ survey.*::
top::
This integer value implies `--top=`, specifying the
number of entries in the detail tables.
+ showBlobSizes::
+ A non-negative integer value. Requests details on the
+ largest file blobs by size in bytes. Provides a
+ default value for `--blob-sizes=` in
+ linkgit:git-survey[1].
+ showCommitParents::
+ A non-negative integer value. Requests details on the
+ commits with the most number of parents. Provides a
+ default value for `--commit-parents=` in
+ linkgit:git-survey[1].
+ showCommitSizes::
+ A non-negative integer value. Requests details on the
+ largest commits by size in bytes. Generally, these
+ are the commits with the largest commit messages.
+ Provides a default value for `--commit-sizes=` in
+ linkgit:git-survey[1].
+ showTreeEntries::
+ A non-negative integer value. Requests details on the
+ trees (directories) with the most number of entries
+ (files and subdirectories). Provides a default value
+ for `--tree-entries=` in linkgit:git-survey[1].
+ showTreeSizes::
+ A non-negative integer value. Requests details on the
+ largest trees (directories) by size in bytes. This
+ will set will usually be equal to the
+ `survey.showTreeEntries` set, but may be skewed by very
+ long file or subdirectory entry names. Provides a
+ default value for `--tree-sizes=` in
+ linkgit:git-survey[1].
--
diff --git a/Documentation/git-status.txt b/Documentation/git-status.txt
index 9a376886a5867a..62254617dd7fc1 100644
--- a/Documentation/git-status.txt
+++ b/Documentation/git-status.txt
@@ -151,6 +151,21 @@ ignored, then the directory is not shown, but all contents are shown.
threshold.
See also linkgit:git-diff[1] `--find-renames`.
+--serialize[=]::
+ (EXPERIMENTAL) Serialize raw status results to a file or stdout
+ in a format suitable for use by `--deserialize`. If a path is
+ given, serialize data will be written to that path *and* normal
+ status output will be written to stdout. If path is omitted,
+ only binary serialization data will be written to stdout.
+
+--deserialize[=]::
+ (EXPERIMENTAL) Deserialize raw status results from a file or
+ stdin rather than scanning the worktree. If `` is omitted
+ and `status.deserializePath` is unset, input is read from stdin.
+--no-deserialize::
+ (EXPERIMENTAL) Disable implicit deserialization of status results
+ from the value of `status.deserializePath`.
+
...::
See the 'pathspec' entry in linkgit:gitglossary[7].
@@ -424,6 +439,26 @@ quoted as explained for the configuration variable `core.quotePath`
(see linkgit:git-config[1]).
+SERIALIZATION and DESERIALIZATION (EXPERIMENTAL)
+------------------------------------------------
+
+The `--serialize` option allows git to cache the result of a
+possibly time-consuming status scan to a binary file. A local
+service/daemon watching file system events could use this to
+periodically pre-compute a fresh status result.
+
+Interactive users could then use `--deserialize` to simply
+(and immediately) print the last-known-good result without
+waiting for the status scan.
+
+The binary serialization file format includes some worktree state
+information allowing `--deserialize` to reject the cached data
+and force a normal status scan if, for example, the commit, branch,
+or status modes/options change. The format cannot, however, indicate
+when the cached data is otherwise stale -- that coordination belongs
+to the task driving the serializations.
+
+
CONFIGURATION
-------------
diff --git a/Documentation/git-survey.txt b/Documentation/git-survey.txt
index 44f3a0568b7697..dc670508e09e2b 100644
--- a/Documentation/git-survey.txt
+++ b/Documentation/git-survey.txt
@@ -32,6 +32,10 @@ OPTIONS
--progress::
Show progress. This is automatically enabled when interactive.
+--[no-]name-rev::
+ Print `git name-rev` output for each commit, tree, and blob.
+ Defaults to true.
+
Ref Selection
~~~~~~~~~~~~~
@@ -59,6 +63,32 @@ only refs for the given options are added.
--other::
Add notes (`refs/notes/`) and stashes (`refs/stash/`) to the set.
+Large Item Selection
+~~~~~~~~~~~~~~~~~~~~
+
+The following options control the optional display of large items under
+various dimensions of scale. The OID of the largest `n` objects will be
+displayed in reverse sorted order. For each, `n` defaults to 10.
+
+--commit-parents::
+ Shows the OIDs of the commits with the most parent commits.
+
+--commit-sizes::
+ Shows the OIDs of the largest commits by size in bytes. This is
+ usually the ones with the largest commit messages.
+
+--tree-entries::
+ Shows the OIDs of the trees with the most number of entries. These
+ are the directories with the most number of files or subdirectories.
+
+--tree-sizes::
+ Shows the OIDs of the largest trees by size in bytes. This set
+ will usually be the same as the vector of number of entries unless
+ skewed by very long entry names.
+
+--blob-sizes::
+ Shows the OIDs of the largest blobs by size in bytes.
+
OUTPUT
------
@@ -78,6 +108,11 @@ Reachable Object Summary
The reachable object summary shows the total number of each kind of Git
object, including tags, commits, trees, and blobs.
+CONFIGURATION
+-------------
+
+include::config/survey.txt[]
+
GIT
---
Part of the linkgit:git[1] suite
diff --git a/Documentation/git-update-microsoft-git.txt b/Documentation/git-update-microsoft-git.txt
new file mode 100644
index 00000000000000..724bfc172f8ab7
--- /dev/null
+++ b/Documentation/git-update-microsoft-git.txt
@@ -0,0 +1,24 @@
+git-update-microsoft-git(1)
+===========================
+
+NAME
+----
+git-update-microsoft-git - Update the installed version of Git
+
+
+SYNOPSIS
+--------
+[verse]
+'git update-microsoft-git'
+
+DESCRIPTION
+-----------
+This version of Git is based on the Microsoft fork of Git, which
+has custom capabilities focused on supporting monorepos. This
+command checks for the latest release of that fork and installs
+it on your machine.
+
+
+GIT
+---
+Part of the linkgit:git[1] suite
diff --git a/Documentation/githooks.txt b/Documentation/githooks.txt
index 0397dec64d7315..86c78b3b4825e2 100644
--- a/Documentation/githooks.txt
+++ b/Documentation/githooks.txt
@@ -758,6 +758,26 @@ and "0" meaning they were not.
Only one parameter should be set to "1" when the hook runs. The hook
running passing "1", "1" should not be possible.
+virtualFilesystem
+~~~~~~~~~~~~~~~~~~
+
+"Virtual File System" allows populating the working directory sparsely.
+The projection data is typically automatically generated by an external
+process. Git will limit what files it checks for changes as well as which
+directories are checked for untracked files based on the path names given.
+Git will also only update those files listed in the projection.
+
+The hook is invoked when the configuration option core.virtualFilesystem
+is set. It takes one argument, a version (currently 1).
+
+The hook should output to stdout the list of all files in the working
+directory that git should track. The paths are relative to the root
+of the working directory and are separated by a single NUL. Full paths
+('dir1/a.txt') as well as directories are supported (ie 'dir1/').
+
+The exit status determines whether git will use the data from the
+hook. On error, git will abort the command with an error message.
+
SEE ALSO
--------
linkgit:git-hook[1]
diff --git a/Documentation/lint-manpages.sh b/Documentation/lint-manpages.sh
index 92cfc0a15abd56..2622a493566950 100755
--- a/Documentation/lint-manpages.sh
+++ b/Documentation/lint-manpages.sh
@@ -27,6 +27,8 @@ check_missing_docs () (
git-init-db) continue;;
git-remote-*) continue;;
git-stage) continue;;
+ git-gvfs-helper) continue;;
+ git-update-microsoft-git) continue;;
git-legacy-*) continue;;
git-?*--?* ) continue ;;
esac
diff --git a/Documentation/scalar.txt b/Documentation/scalar.txt
index 7e4259c6743f9b..17cc2d500ca40b 100644
--- a/Documentation/scalar.txt
+++ b/Documentation/scalar.txt
@@ -9,7 +9,8 @@ SYNOPSIS
--------
[verse]
scalar clone [--single-branch] [--branch ] [--full-clone]
- [--[no-]src] []
+ [--[no-]src] [--local-cache-path ] [--cache-server-url ]
+ []
scalar list
scalar register []
scalar unregister []
@@ -17,6 +18,7 @@ scalar run ( all | config | commit-graph | fetch | loose-objects | pack-files )
scalar reconfigure [ --all | ]
scalar diagnose []
scalar delete
+scalar cache-server ( --get | --set | --list [] ) []
DESCRIPTION
-----------
@@ -97,6 +99,37 @@ cloning. If the HEAD at the remote did not point at any branch when
A sparse-checkout is initialized by default. This behavior can be
turned off via `--full-clone`.
+--local-cache-path ::
+ Override the path to the local cache root directory; Pre-fetched objects
+ are stored into a repository-dependent subdirectory of that path.
++
+The default is `:\.scalarCache` on Windows (on the same drive as the
+clone), and `~/.scalarCache` on macOS.
+
+--cache-server-url ::
+ Retrieve missing objects from the specified remote, which is expected to
+ understand the GVFS protocol.
+
+--[no-]gvfs-protocol::
+ When cloning from a `` with either `dev.azure.com` or
+ `visualstudio.com` in the name, `scalar clone` will attempt to use the GVFS
+ Protocol to access Git objects, specifically from a cache server when
+ available, and will fail to clone if there is an error over that protocol.
+
+ To enable the GVFS Protocol regardless of the origin ``, use
+ `--gvfs-protocol`. This will cause `scalar clone` to fail when the origin
+ server fails to provide a valid response to the `gvfs/config` endpoint.
+
+ To disable the GVFS Protocol, use `--no-gvfs-protocol` and `scalar clone`
+ will only use the Git protocol, starting with a partial clone. This can be
+ helpful if your `` points to Azure Repos but the repository does not
+ have GVFS cache servers enabled. It is likely more efficient to use its
+ partial clone functionality through the Git protocol.
+
+ Previous versions of `scalar clone` could fall back to a partial clone over
+ the Git protocol if there is any issue gathering GVFS configuration
+ information from the origin server.
+
List
~~~~
@@ -170,6 +203,27 @@ delete ::
This subcommand lets you delete an existing Scalar enlistment from your
local file system, unregistering the repository.
+Cache-server
+~~~~~~~~~~~~
+
+cache-server ( --get | --set | --list [] ) []::
+ This command lets you query or set the GVFS-enabled cache server used
+ to fetch missing objects.
+
+--get::
+ This is the default command mode: query the currently-configured cache
+ server URL, if any.
+
+--list::
+ Access the `gvfs/info` endpoint of the specified remote (default:
+ `origin`) to figure out which cache servers are available, if any.
++
+In contrast to the `--get` command mode (which only accesses the local
+repository), this command mode triggers a request via the network that
+potentially requires authentication. If authentication is required, the
+configured credential helper is employed (see linkgit:git-credential[1]
+for details).
+
SEE ALSO
--------
linkgit:git-clone[1], linkgit:git-maintenance[1].
diff --git a/Documentation/technical/api-path-walk.txt b/Documentation/technical/api-path-walk.txt
index 83bfe3d665e9fb..85d8e262401486 100644
--- a/Documentation/technical/api-path-walk.txt
+++ b/Documentation/technical/api-path-walk.txt
@@ -65,6 +65,14 @@ better off using the revision walk API instead.
the revision walk so that the walk emits commits marked with the
`UNINTERESTING` flag.
+`edge_aggressive`::
+ For performance reasons, usually only the boundary commits are
+ explored to find UNINTERESTING objects. However, in the case of
+ shallow clones it can be helpful to mark all trees and blobs
+ reachable from UNINTERESTING tip commits as UNINTERESTING. This
+ matches the behavior of `--objects-edge-aggressive` in the
+ revision API.
+
`pl`::
This pattern list pointer allows focusing the path-walk search to
a set of patterns, only emitting paths that match the given
diff --git a/Documentation/technical/read-object-protocol.txt b/Documentation/technical/read-object-protocol.txt
new file mode 100644
index 00000000000000..a893b46e7c28a9
--- /dev/null
+++ b/Documentation/technical/read-object-protocol.txt
@@ -0,0 +1,102 @@
+Read Object Process
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The read-object process enables Git to read all missing blobs with a
+single process invocation for the entire life of a single Git command.
+This is achieved by using a packet format (pkt-line, see technical/
+protocol-common.txt) based protocol over standard input and standard
+output as follows. All packets, except for the "*CONTENT" packets and
+the "0000" flush packet, are considered text and therefore are
+terminated by a LF.
+
+Git starts the process when it encounters the first missing object that
+needs to be retrieved. After the process is started, Git sends a welcome
+message ("git-read-object-client"), a list of supported protocol version
+numbers, and a flush packet. Git expects to read a welcome response
+message ("git-read-object-server"), exactly one protocol version number
+from the previously sent list, and a flush packet. All further
+communication will be based on the selected version.
+
+The remaining protocol description below documents "version=1". Please
+note that "version=42" in the example below does not exist and is only
+there to illustrate how the protocol would look with more than one
+version.
+
+After the version negotiation Git sends a list of all capabilities that
+it supports and a flush packet. Git expects to read a list of desired
+capabilities, which must be a subset of the supported capabilities list,
+and a flush packet as response:
+------------------------
+packet: git> git-read-object-client
+packet: git> version=1
+packet: git> version=42
+packet: git> 0000
+packet: git< git-read-object-server
+packet: git< version=1
+packet: git< 0000
+packet: git> capability=get
+packet: git> capability=have
+packet: git> capability=put
+packet: git> capability=not-yet-invented
+packet: git> 0000
+packet: git< capability=get
+packet: git< 0000
+------------------------
+The only supported capability in version 1 is "get".
+
+Afterwards Git sends a list of "key=value" pairs terminated with a flush
+packet. The list will contain at least the command (based on the
+supported capabilities) and the sha1 of the object to retrieve. Please
+note, that the process must not send any response before it received the
+final flush packet.
+
+When the process receives the "get" command, it should make the requested
+object available in the git object store and then return success. Git will
+then check the object store again and this time find it and proceed.
+------------------------
+packet: git> command=get
+packet: git> sha1=0a214a649e1b3d5011e14a3dc227753f2bd2be05
+packet: git> 0000
+------------------------
+
+The process is expected to respond with a list of "key=value" pairs
+terminated with a flush packet. If the process does not experience
+problems then the list must contain a "success" status.
+------------------------
+packet: git< status=success
+packet: git< 0000
+------------------------
+
+In case the process cannot or does not want to process the content, it
+is expected to respond with an "error" status.
+------------------------
+packet: git< status=error
+packet: git< 0000
+------------------------
+
+In case the process cannot or does not want to process the content as
+well as any future content for the lifetime of the Git process, then it
+is expected to respond with an "abort" status at any point in the
+protocol.
+------------------------
+packet: git< status=abort
+packet: git< 0000
+------------------------
+
+Git neither stops nor restarts the process in case the "error"/"abort"
+status is set.
+
+If the process dies during the communication or does not adhere to the
+protocol then Git will stop the process and restart it with the next
+object that needs to be processed.
+
+After the read-object process has processed an object it is expected to
+wait for the next "key=value" list containing a command. Git will close
+the command pipe on exit. The process is expected to detect EOF and exit
+gracefully on its own. Git will wait until the process has stopped.
+
+A long running read-object process demo implementation can be found in
+`contrib/long-running-read-object/example.pl` located in the Git core
+repository. If you develop your own long running process then the
+`GIT_TRACE_PACKET` environment variables can be very helpful for
+debugging (see linkgit:git[1]).
diff --git a/Documentation/technical/sparse-index.txt b/Documentation/technical/sparse-index.txt
index 3b24c1a219f811..c466dbddc930a9 100644
--- a/Documentation/technical/sparse-index.txt
+++ b/Documentation/technical/sparse-index.txt
@@ -206,3 +206,10 @@ Here are some commands that might be useful to update:
* `git am`
* `git clean`
* `git stash`
+
+In order to help identify the cases where remaining index expansion is
+occurring in user machines, calls to `ensure_full_index()` have been
+replaced with `ensure_full_index_with_reason()` or with
+`ensure_full_index_unaudited()`. These versions add tracing that should
+help identify the reason for the index expansion without needing full
+access to someone's repository.
diff --git a/Documentation/technical/status-serialization-format.txt b/Documentation/technical/status-serialization-format.txt
new file mode 100644
index 00000000000000..475ae814495581
--- /dev/null
+++ b/Documentation/technical/status-serialization-format.txt
@@ -0,0 +1,107 @@
+Git status serialization format
+===============================
+
+Git status serialization enables git to dump the results of a status scan
+to a binary file. This file can then be loaded by later status invocations
+to print the cached status results.
+
+The file contains the essential fields from:
+() the index
+() the "struct wt_status" for the overall results
+() the contents of "struct wt_status_change_data" for tracked changed files
+() the list of untracked and ignored files
+
+Version 1 Format:
+=================
+
+The V1 file begins with a required header section followed by optional
+sections for each type of item (changed, untracked, ignored). Individual
+item sections are only present if necessary. Each item section begins
+with an item-type header with the number of items in the section.
+
+Each "line" in the format is encoded using pkt-line with a final LF.
+Flush packets are used to terminate sections.
+
+-----------------
+PKT-LINE("version" SP "1")
+
+[]
+[]
+[]
+-----------------
+
+
+V1 Header
+---------
+
+The v1-header-section fields are taken directly from "struct wt_status".
+Each field is printed on a separate pkt-line. Lines for NULL string
+values are omitted. All integers are printed with "%d". OIDs are
+printed in hex.
+
+v1-header-section =
+
+ PKT-LINE()
+
+v1-index-headers = PKT-LINE("index_mtime" SP SP LF)
+
+v1-wt-status-headers = PKT-LINE("is_initial" SP LF)
+ [ PKT-LINE("branch" SP LF) ]
+ [ PKT-LINE("reference" SP LF) ]
+ PKT-LINE("show_ignored_files" SP LF)
+ PKT-LINE("show_untracked_files" SP LF)
+ PKT-LINE("show_ignored_directory" SP LF)
+ [ PKT-LINE("ignore_submodule_arg" SP LF) ]
+ PKT-LINE("detect_rename" SP LF)
+ PKT-LINE("rename_score" SP LF)
+ PKT-LINE("rename_limit" SP LF)
+ PKT-LINE("detect_break" SP LF)
+ PKT-LINE("sha1_commit" SP LF)
+ PKT-LINE("committable" SP LF)
+ PKT-LINE("workdir_dirty" SP LF)
+
+
+V1 Changed Items
+----------------
+
+The v1-changed-item-section lists all of the changed items with one
+item per pkt-line. Each pkt-line contains: a binary block of data
+from "struct wt_status_serialize_data_fixed" in a fixed header where
+integers are in network byte order and OIDs are in raw (non-hex) form.
+This is followed by one or two raw pathnames (not c-quoted) with NUL
+terminators (both NULs are always present even if there is no rename).
+
+v1-changed-item-section = PKT-LINE("changed" SP LF)
+ [ PKT-LINE( LF) ]+
+ PKT-LINE()
+
+changed_item =
+
+
+
+
+
+
+
+
+
+
+
+ NUL
+ [ ]
+ NUL
+
+
+V1 Untracked and Ignored Items
+------------------------------
+
+These sections are simple lists of pathnames. They ARE NOT
+c-quoted.
+
+v1-untracked-item-section = PKT-LINE("untracked" SP LF)
+ [ PKT-LINE( LF) ]+
+ PKT-LINE()
+
+v1-ignored-item-section = PKT-LINE("ignored" SP LF)
+ [ PKT-LINE( LF) ]+
+ PKT-LINE()
diff --git a/GIT-VERSION-GEN b/GIT-VERSION-GEN
index 194ec0f9ad32fe..e961f845618414 100755
--- a/GIT-VERSION-GEN
+++ b/GIT-VERSION-GEN
@@ -2,6 +2,9 @@
DEF_VER=v2.48.0-rc1
+# Identify microsoft/git via a distinct version suffix
+DEF_VER=$DEF_VER.vfs.0.0
+
LF='
'
@@ -39,10 +42,15 @@ then
test -d "${GIT_DIR:-.git}" ||
test -f "$SOURCE_DIR"/.git;
} &&
- VN=$(git -C "$SOURCE_DIR" describe --match "v[0-9]*" HEAD 2>/dev/null) &&
+ VN=$(git -C "$SOURCE_DIR" describe --match "v[0-9]*vfs*" HEAD 2>/dev/null) &&
case "$VN" in
*$LF*) (exit 1) ;;
v[0-9]*)
+ if test "${VN%%.vfs.*}" != "${DEF_VER%%.vfs.*}"
+ then
+ echo "Found version $VN, which is not based on $DEF_VER" >&2
+ exit 1
+ fi
git -C "$SOURCE_DIR" update-index -q --refresh
test -z "$(git -C "$SOURCE_DIR" diff-index --name-only HEAD --)" ||
VN="$VN-dirty" ;;
diff --git a/Makefile b/Makefile
index ab42aaf126936e..a86e6b262c84a3 100644
--- a/Makefile
+++ b/Makefile
@@ -1038,6 +1038,8 @@ LIB_OBJS += git-zlib.o
LIB_OBJS += gpg-interface.o
LIB_OBJS += graph.o
LIB_OBJS += grep.o
+LIB_OBJS += gvfs.o
+LIB_OBJS += gvfs-helper-client.o
LIB_OBJS += hash-lookup.o
LIB_OBJS += hashmap.o
LIB_OBJS += help.o
@@ -1196,6 +1198,7 @@ LIB_OBJS += utf8.o
LIB_OBJS += varint.o
LIB_OBJS += version.o
LIB_OBJS += versioncmp.o
+LIB_OBJS += virtualfilesystem.o
LIB_OBJS += walker.o
LIB_OBJS += wildmatch.o
LIB_OBJS += worktree.o
@@ -1203,6 +1206,8 @@ LIB_OBJS += wrapper.o
LIB_OBJS += write-or-die.o
LIB_OBJS += ws.o
LIB_OBJS += wt-status.o
+LIB_OBJS += wt-status-deserialize.o
+LIB_OBJS += wt-status-serialize.o
LIB_OBJS += xdiff-interface.o
BUILTIN_OBJS += builtin/add.o
@@ -1321,6 +1326,7 @@ BUILTIN_OBJS += builtin/tag.o
BUILTIN_OBJS += builtin/unpack-file.o
BUILTIN_OBJS += builtin/unpack-objects.o
BUILTIN_OBJS += builtin/update-index.o
+BUILTIN_OBJS += builtin/update-microsoft-git.o
BUILTIN_OBJS += builtin/update-ref.o
BUILTIN_OBJS += builtin/update-server-info.o
BUILTIN_OBJS += builtin/upload-archive.o
@@ -1682,6 +1688,9 @@ endif
endif
BASIC_CFLAGS += $(CURL_CFLAGS)
+ PROGRAM_OBJS += gvfs-helper.o
+ TEST_PROGRAMS_NEED_X += test-gvfs-protocol
+
REMOTE_CURL_PRIMARY = git-remote-http$X
REMOTE_CURL_ALIASES = git-remote-https$X git-remote-ftp$X git-remote-ftps$X
REMOTE_CURL_NAMES = $(REMOTE_CURL_PRIMARY) $(REMOTE_CURL_ALIASES)
@@ -2777,6 +2786,7 @@ GIT_OBJS += git.o
.PHONY: git-objs
git-objs: $(GIT_OBJS)
+SCALAR_OBJS := json-parser.o
SCALAR_OBJS += scalar.o
.PHONY: scalar-objs
scalar-objs: $(SCALAR_OBJS)
@@ -2877,7 +2887,7 @@ gettext.sp gettext.s gettext.o: GIT-PREFIX
gettext.sp gettext.s gettext.o: EXTRA_CPPFLAGS = \
-DGIT_LOCALE_PATH='"$(localedir_relative_SQ)"'
-http-push.sp http.sp http-walker.sp remote-curl.sp imap-send.sp: SP_EXTRA_FLAGS += \
+http-push.sp http.sp http-walker.sp remote-curl.sp imap-send.sp gvfs-helper.sp: SP_EXTRA_FLAGS += \
-DCURL_DISABLE_TYPECHECK
pack-revindex.sp: SP_EXTRA_FLAGS += -Wno-memcpy-max-count
@@ -2928,10 +2938,14 @@ $(REMOTE_CURL_PRIMARY): remote-curl.o http.o http-walker.o $(LAZYLOAD_LIBCURL_OB
$(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \
$(CURL_LIBCURL) $(EXPAT_LIBEXPAT) $(LIBS)
-scalar$X: scalar.o GIT-LDFLAGS $(GITLIBS)
+scalar$X: $(SCALAR_OBJS) GIT-LDFLAGS $(GITLIBS)
$(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) \
$(filter %.o,$^) $(LIBS)
+git-gvfs-helper$X: gvfs-helper.o http.o GIT-LDFLAGS $(GITLIBS) $(LAZYLOAD_LIBCURL_OBJ)
+ $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \
+ $(CURL_LIBCURL) $(EXPAT_LIBEXPAT) $(LIBS)
+
$(LIB_FILE): $(LIB_OBJS)
$(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^
@@ -3694,7 +3708,7 @@ dist: git-archive$(X) configure
@$(MAKE) -C git-gui TARDIR=../.dist-tmp-dir/git-gui dist-version
./git-archive --format=tar \
$(GIT_ARCHIVE_EXTRA_FILES) \
- --prefix=$(GIT_TARNAME)/ HEAD^{tree} > $(GIT_TARNAME).tar
+ --prefix=$(GIT_TARNAME)/ HEAD > $(GIT_TARNAME).tar
@$(RM) -r .dist-tmp-dir
gzip -f -9 $(GIT_TARNAME).tar
diff --git a/README.md b/README.md
index 4eabce53d89e1f..b39764e9ad1dcd 100644
--- a/README.md
+++ b/README.md
@@ -1,148 +1,225 @@
-Git for Windows
-===============
-
-[](CODE_OF_CONDUCT.md)
-[](https://open.vscode.dev/git-for-windows/git)
-[](https://github.com/git-for-windows/git/actions?query=branch%3Amain+event%3Apush)
-[](https://gitter.im/git-for-windows/git?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
-
-This is [Git for Windows](http://git-for-windows.github.io/), the Windows port
-of [Git](http://git-scm.com/).
-
-The Git for Windows project is run using a [governance
-model](http://git-for-windows.github.io/governance-model.html). If you
-encounter problems, you can report them as [GitHub
-issues](https://github.com/git-for-windows/git/issues), discuss them on Git
-for Windows' [Google Group](http://groups.google.com/group/git-for-windows),
-and [contribute bug
-fixes](https://github.com/git-for-windows/git/wiki/How-to-participate).
-
-To build Git for Windows, please either install [Git for Windows'
-SDK](https://gitforwindows.org/#download-sdk), start its `git-bash.exe`, `cd`
-to your Git worktree and run `make`, or open the Git worktree as a folder in
-Visual Studio.
-
-To verify that your build works, use one of the following methods:
-
-- If you want to test the built executables within Git for Windows' SDK,
- prepend `/bin-wrappers` to the `PATH`.
-- Alternatively, run `make install` in the Git worktree.
-- If you need to test this in a full installer, run `sdk build
- git-and-installer`.
-- You can also "install" Git into an existing portable Git via `make install
- DESTDIR=` where `` refers to the top-level directory of the
- portable Git. In this instance, you will want to prepend that portable Git's
- `/cmd` directory to the `PATH`, or test by running that portable Git's
- `git-bash.exe` or `git-cmd.exe`.
-- If you built using a recent Visual Studio, you can use the menu item
- `Build>Install git` (you will want to click on `Project>CMake Settings for
- Git` first, then click on `Edit JSON` and then point `installRoot` to the
- `mingw64` directory of an already-unpacked portable Git).
-
- As in the previous bullet point, you will then prepend `/cmd` to the `PATH`
- or run using the portable Git's `git-bash.exe` or `git-cmd.exe`.
-- If you want to run the built executables in-place, but in a CMD instead of
- inside a Bash, you can run a snippet like this in the `git-bash.exe` window
- where Git was built (ensure that the `EOF` line has no leading spaces), and
- then paste into the CMD window what was put in the clipboard:
-
- ```sh
- clip.exe <
-including full documentation and Git related tools.
-
-See [Documentation/gittutorial.txt][] to get started, then see
-[Documentation/giteveryday.txt][] for a useful minimum set of commands, and
-`Documentation/git-.txt` for documentation of each command.
-If git has been correctly installed, then the tutorial can also be
-read with `man gittutorial` or `git help tutorial`, and the
-documentation of each command with `man git-` or `git help
-`.
-
-CVS users may also want to read [Documentation/gitcvs-migration.txt][]
-(`man gitcvs-migration` or `git help cvs-migration` if git is
-installed).
-
-The user discussion and development of core Git take place on the Git
-mailing list -- everyone is welcome to post bug reports, feature
-requests, comments and patches to git@vger.kernel.org (read
-[Documentation/SubmittingPatches][] for instructions on patch submission
-and [Documentation/CodingGuidelines][]).
-
-Those wishing to help with error message, usage and informational message
-string translations (localization l10) should see [po/README.md][]
-(a `po` file is a Portable Object file that holds the translations).
-
-To subscribe to the list, send an email to
-(see https://subspace.kernel.org/subscribing.html for details). The mailing
-list archives are available at ,
- and other archival sites.
-The core git mailing list is plain text (no HTML!).
-
-Issues which are security relevant should be disclosed privately to
-the Git Security mailing list .
-
-The maintainer frequently sends the "What's cooking" reports that
-list the current status of various development topics to the mailing
-list. The discussion following them give a good reference for
-project status, development direction and remaining tasks.
-
-The name "git" was given by Linus Torvalds when he wrote the very
-first version. He described the tool as "the stupid content tracker"
-and the name as (depending on your mood):
-
- - random three-letter combination that is pronounceable, and not
- actually used by any common UNIX command. The fact that it is a
- mispronunciation of "get" may or may not be relevant.
- - stupid. contemptible and despicable. simple. Take your pick from the
- dictionary of slang.
- - "global information tracker": you're in a good mood, and it actually
- works for you. Angels sing, and a light suddenly fills the room.
- - "goddamn idiotic truckload of sh*t": when it breaks
-
-[INSTALL]: INSTALL
-[Documentation/gittutorial.txt]: Documentation/gittutorial.txt
-[Documentation/giteveryday.txt]: Documentation/giteveryday.txt
-[Documentation/gitcvs-migration.txt]: Documentation/gitcvs-migration.txt
-[Documentation/SubmittingPatches]: Documentation/SubmittingPatches
-[Documentation/CodingGuidelines]: Documentation/CodingGuidelines
-[po/README.md]: po/README.md
+If you're working in a monorepo and want to take advantage of the performance boosts in
+`microsoft/git`, then you can download the latest version installer for your OS from the
+[Releases page](https://github.com/microsoft/git/releases). Alternatively, you can opt to install
+via the command line, using the below instructions for supported OSes:
+
+## Windows
+
+__Note:__ Winget is still in public preview, meaning you currently
+[need to take special installation steps](https://docs.microsoft.com/en-us/windows/package-manager/winget/#install-winget):
+Either manually install the `.appxbundle` available at the
+[preview version of App Installer](https://www.microsoft.com/p/app-installer/9nblggh4nns1?ocid=9nblggh4nns1_ORSEARCH_Bing&rtc=1&activetab=pivot:overviewtab),
+or participate in the
+[Windows Insider flight ring](https://insider.windows.com/https://insider.windows.com/)
+since `winget` is available by default on preview versions of Windows.
+
+To install with Winget, run
+
+```shell
+winget install --id microsoft.git
+```
+
+Double-check that you have the right version by running these commands,
+which should have the same output:
+
+```shell
+git version
+scalar version
+```
+
+To upgrade `microsoft/git`, use the following Git command, which will download and install the latest
+release.
+
+```shell
+git update-microsoft-git
+```
+
+You may also be alerted with a notification to upgrade, which presents a single-click process for
+running `git update-microsoft-git`.
+
+## macOS
+
+To install `microsoft/git` on macOS, first [be sure that Homebrew is installed](https://brew.sh/) then
+install the `microsoft-git` cask with these steps:
+
+```shell
+brew tap microsoft/git
+brew install --cask microsoft-git
+```
+
+Double-check that you have the right version by running these commands,
+which should have the same output:
+
+```shell
+git version
+scalar version
+```
+
+To upgrade microsoft/git, you can run the necessary `brew` commands:
+
+```shell
+brew update
+brew upgrade --cask microsoft-git
+```
+
+Or you can run the `git update-microsoft-git` command, which will run those brew commands for you.
+
+## Linux
+### Ubuntu/Debian distributions
+
+On newer distributions*, you can install using the most recent Debian package.
+To download and validate the signature of this package, run the following:
+
+```shell
+# Install needed packages
+sudo apt-get install -y curl debsig-verify
+
+# Download public key signature file
+curl -s https://api.github.com/repos/microsoft/git/releases/latest \
+| grep -E 'browser_download_url.*msft-git-public.asc' \
+| cut -d : -f 2,3 \
+| tr -d \" \
+| xargs -I 'url' curl -L -o msft-git-public.asc 'url'
+
+# De-armor public key signature file
+gpg --output msft-git-public.gpg --dearmor msft-git-public.asc
+
+# Note that the fingerprint of this key is "B8F12E25441124E1", which you can
+# determine by running:
+gpg --show-keys msft-git-public.asc | head -n 2 | tail -n 1 | tail -c 17
+
+# Copy de-armored public key to debsig keyring folder
+sudo mkdir /usr/share/debsig/keyrings/B8F12E25441124E1
+sudo mv msft-git-public.gpg /usr/share/debsig/keyrings/B8F12E25441124E1/
+
+# Create an appropriate policy file
+sudo mkdir /etc/debsig/policies/B8F12E25441124E1
+cat > generic.pol << EOL
+
+
+
+
+
+
+
+
+
+
+
+EOL
+
+sudo mv generic.pol /etc/debsig/policies/B8F12E25441124E1/generic.pol
+
+# Download Debian package
+curl -s https://api.github.com/repos/microsoft/git/releases/latest \
+| grep "browser_download_url.*deb" \
+| cut -d : -f 2,3 \
+| tr -d \" \
+| xargs -I 'url' curl -L -o msft-git.deb 'url'
+
+# Verify
+debsig-verify msft-git.deb
+
+# Install
+sudo dpkg -i msft-git.deb
+```
+
+Double-check that you have the right version by running these commands,
+which should have the same output:
+
+```shell
+git version
+scalar version
+```
+
+To upgrade, you will need to repeat these steps to reinstall.
+
+*Older distributions are missing some required dependencies. Even
+though the package may appear to install successfully, `microsoft/
+git` will not function as expected. If you are running `Ubuntu 20.04` or
+older, please follow the install from source instructions below
+instead of installing the debian package.
+
+### Installing From Source
+
+On older or other distros you will need to compile and install `microsoft/git` from source:
+
+```shell
+git clone https://github.com/microsoft/git microsoft-git
+cd microsoft-git
+make -j12 prefix=/usr/local
+sudo make -j12 prefix=/usr/local install
+```
+
+For more assistance building Git from source, see
+[the INSTALL file in the core Git project](https://github.com/git/git/blob/master/INSTALL).
+
+#### Common Debian based dependencies
+While the INSTALL file covers dependencies in detail, here is a shortlist of common required dependencies on older Debian/Ubuntu distros:
+
+```shell
+sudo apt-get update
+sudo apt-get install libz-dev libssl-dev libcurl4-gnutls-dev libexpat1-dev gettext cmake gcc
+```
+
+Contributing
+=========================================================
+
+This project welcomes contributions and suggestions. Most contributions require you to agree to a
+Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
+the rights to use your contribution. For details, visit .
+
+When you submit a pull request, a CLA-bot will automatically determine whether you need to provide
+a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions
+provided by the bot. You will only need to do this once across all repos using our CLA.
+
+This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
+For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or
+contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
diff --git a/abspath.c b/abspath.c
index 0c17e98654e4b0..e899f46d02097a 100644
--- a/abspath.c
+++ b/abspath.c
@@ -14,7 +14,7 @@ int is_directory(const char *path)
}
/* removes the last path component from 'path' except if 'path' is root */
-static void strip_last_component(struct strbuf *path)
+void strip_last_path_component(struct strbuf *path)
{
size_t offset = offset_1st_component(path->buf);
size_t len = path->len;
@@ -119,7 +119,7 @@ static char *strbuf_realpath_1(struct strbuf *resolved, const char *path,
continue; /* '.' component */
} else if (next.len == 2 && !strcmp(next.buf, "..")) {
/* '..' component; strip the last path component */
- strip_last_component(resolved);
+ strip_last_path_component(resolved);
continue;
}
@@ -171,7 +171,7 @@ static char *strbuf_realpath_1(struct strbuf *resolved, const char *path,
* strip off the last component since it will
* be replaced with the contents of the symlink
*/
- strip_last_component(resolved);
+ strip_last_path_component(resolved);
}
/*
diff --git a/abspath.h b/abspath.h
index 4653080d5e4b7a..06241ba13cf646 100644
--- a/abspath.h
+++ b/abspath.h
@@ -10,6 +10,11 @@ char *real_pathdup(const char *path, int die_on_error);
const char *absolute_path(const char *path);
char *absolute_pathdup(const char *path);
+/**
+ * Remove the last path component from 'path' except if 'path' is root.
+ */
+void strip_last_path_component(struct strbuf *path);
+
/*
* Concatenate "prefix" (if len is non-zero) and "path", with no
* connecting characters (so "prefix" should end with a "/").
diff --git a/apply.c b/apply.c
index 3b4cb3e042112a..df1ae11f44be54 100644
--- a/apply.c
+++ b/apply.c
@@ -3370,6 +3370,24 @@ static int checkout_target(struct index_state *istate,
{
struct checkout costate = CHECKOUT_INIT;
+ /*
+ * Do not checkout the entry if the skipworktree bit is set
+ *
+ * Both callers of this method (check_preimage and load_current)
+ * check for the existance of the file before calling this
+ * method so we know that the file doesn't exist at this point
+ * and we don't need to perform that check again here.
+ * We just need to check the skip-worktree and return.
+ *
+ * This is to prevent git from creating a file in the
+ * working directory that has the skip-worktree bit on,
+ * then updating the index from the patch and not keeping
+ * the working directory version up to date with what it
+ * changed the index version to be.
+ */
+ if (ce_skip_worktree(ce))
+ return 0;
+
costate.refresh_cache = 1;
costate.istate = istate;
if (checkout_entry(ce, &costate, NULL, NULL) ||
diff --git a/bin-wrappers/.gitignore b/bin-wrappers/.gitignore
index 1c6c90458b7586..e481f5a45a7a0d 100644
--- a/bin-wrappers/.gitignore
+++ b/bin-wrappers/.gitignore
@@ -6,4 +6,5 @@
/git-upload-pack
/scalar
/test-fake-ssh
+/test-gvfs-protocol
/test-tool
diff --git a/builtin.h b/builtin.h
index 5f64730cf0273d..f99519a65bcba6 100644
--- a/builtin.h
+++ b/builtin.h
@@ -239,6 +239,7 @@ int cmd_tag(int argc, const char **argv, const char *prefix, struct repository *
int cmd_unpack_file(int argc, const char **argv, const char *prefix, struct repository *repo);
int cmd_unpack_objects(int argc, const char **argv, const char *prefix, struct repository *repo);
int cmd_update_index(int argc, const char **argv, const char *prefix, struct repository *repo);
+int cmd_update_microsoft_git(int argc, const char **argv, const char *prefix, struct repository *repo);
int cmd_update_ref(int argc, const char **argv, const char *prefix, struct repository *repo);
int cmd_update_server_info(int argc, const char **argv, const char *prefix, struct repository *repo);
int cmd_upload_archive(int argc, const char **argv, const char *prefix, struct repository *repo);
diff --git a/builtin/add.c b/builtin/add.c
index fc2866fad8afbe..d85b2bd81974d7 100644
--- a/builtin/add.c
+++ b/builtin/add.c
@@ -1,3 +1,5 @@
+#define USE_THE_REPOSITORY_VARIABLE
+
/*
* "git add" builtin command
*
@@ -5,6 +7,7 @@
*/
#include "builtin.h"
+#include "environment.h"
#include "advice.h"
#include "config.h"
#include "lockfile.h"
@@ -47,6 +50,7 @@ static int chmod_pathspec(struct repository *repo,
int err;
if (!include_sparse &&
+ !core_virtualfilesystem &&
(ce_skip_worktree(ce) ||
!path_in_sparse_checkout(ce->name, repo->index)))
continue;
@@ -132,8 +136,9 @@ static int refresh(struct repository *repo, int verbose, const struct pathspec *
if (!seen[i]) {
const char *path = pathspec->items[i].original;
- if (matches_skip_worktree(pathspec, i, &skip_worktree_seen) ||
- !path_in_sparse_checkout(path, repo->index)) {
+ if (!core_virtualfilesystem &&
+ (matches_skip_worktree(pathspec, i, &skip_worktree_seen) ||
+ !path_in_sparse_checkout(path, repo->index))) {
string_list_append(&only_match_skip_worktree,
pathspec->items[i].original);
} else {
@@ -143,7 +148,11 @@ static int refresh(struct repository *repo, int verbose, const struct pathspec *
}
}
- if (only_match_skip_worktree.nr) {
+ /*
+ * When using a virtual filesystem, we might re-add a path
+ * that is currently virtual and we want that to succeed.
+ */
+ if (!core_virtualfilesystem && only_match_skip_worktree.nr) {
advise_on_updating_sparse_paths(&only_match_skip_worktree);
ret = 1;
}
@@ -529,7 +538,11 @@ int cmd_add(int argc,
if (seen[i])
continue;
- if (!include_sparse &&
+ /*
+ * When using a virtual filesystem, we might re-add a path
+ * that is currently virtual and we want that to succeed.
+ */
+ if (!include_sparse && !core_virtualfilesystem &&
matches_skip_worktree(&pathspec, i, &skip_worktree_seen)) {
string_list_append(&only_match_skip_worktree,
pathspec.items[i].original);
@@ -553,7 +566,6 @@ int cmd_add(int argc,
}
}
-
if (only_match_skip_worktree.nr) {
advise_on_updating_sparse_paths(&only_match_skip_worktree);
exit_status = 1;
diff --git a/builtin/checkout-index.c b/builtin/checkout-index.c
index a81501098d9fdb..b7c176f1790f0a 100644
--- a/builtin/checkout-index.c
+++ b/builtin/checkout-index.c
@@ -156,7 +156,8 @@ static int checkout_all(const char *prefix, int prefix_length)
* first entry inside the expanded sparse directory).
*/
if (ignore_skip_worktree) {
- ensure_full_index(the_repository->index);
+ ensure_full_index_with_reason(the_repository->index,
+ "checkout-index");
ce = the_repository->index->cache[i];
}
}
diff --git a/builtin/checkout.c b/builtin/checkout.c
index 0ff27deffeee97..f19bc559505fdc 100644
--- a/builtin/checkout.c
+++ b/builtin/checkout.c
@@ -20,6 +20,7 @@
#include "merge-recursive.h"
#include "object-name.h"
#include "object-store-ll.h"
+#include "packfile.h"
#include "parse-options.h"
#include "path.h"
#include "preload-index.h"
@@ -1049,8 +1050,16 @@ static void update_refs_for_switch(const struct checkout_opts *opts,
strbuf_release(&msg);
if (!opts->quiet &&
!opts->force_detach &&
- (new_branch_info->path || !strcmp(new_branch_info->name, "HEAD")))
+ (new_branch_info->path || !strcmp(new_branch_info->name, "HEAD"))) {
+ unsigned long nr_unpack_entry_at_start;
+
+ trace2_region_enter("tracking", "report_tracking", the_repository);
+ nr_unpack_entry_at_start = get_nr_unpack_entry();
report_tracking(new_branch_info);
+ trace2_data_intmax("tracking", NULL, "report_tracking/nr_unpack_entries",
+ (intmax_t)(get_nr_unpack_entry() - nr_unpack_entry_at_start));
+ trace2_region_leave("tracking", "report_tracking", the_repository);
+ }
}
static int add_pending_uninteresting_ref(const char *refname, const char *referent UNUSED,
diff --git a/builtin/commit.c b/builtin/commit.c
index f4f87d01d5a90b..5ca91dcecb7ac4 100644
--- a/builtin/commit.c
+++ b/builtin/commit.c
@@ -42,6 +42,7 @@
#include "commit-graph.h"
#include "pretty.h"
#include "trailer.h"
+#include "trace2.h"
static const char * const builtin_commit_usage[] = {
N_("git commit [-a | --interactive | --patch] [-s] [-v] [-u] [--amend]\n"
@@ -164,6 +165,122 @@ static int opt_parse_porcelain(const struct option *opt, const char *arg, int un
return 0;
}
+static int do_serialize = 0;
+static char *serialize_path = NULL;
+
+static int reject_implicit = 0;
+static int do_implicit_deserialize = 0;
+static int do_explicit_deserialize = 0;
+static char *deserialize_path = NULL;
+
+static enum wt_status_deserialize_wait implicit_deserialize_wait = DESERIALIZE_WAIT__UNSET;
+static enum wt_status_deserialize_wait explicit_deserialize_wait = DESERIALIZE_WAIT__UNSET;
+
+/*
+ * --serialize | --serialize=
+ *
+ * Request that we serialize status output rather than or in addition to
+ * printing in any of the established formats.
+ *
+ * Without a path, we write binary serialization data to stdout (and omit
+ * the normal status output).
+ *
+ * With a path, we write binary serialization data to the and then
+ * write normal status output.
+ */
+static int opt_parse_serialize(const struct option *opt, const char *arg, int unset)
+{
+ enum wt_status_format *value = (enum wt_status_format *)opt->value;
+ if (unset || !arg)
+ *value = STATUS_FORMAT_SERIALIZE_V1;
+
+ if (arg) {
+ free(serialize_path);
+ serialize_path = xstrdup(arg);
+ }
+
+ if (do_explicit_deserialize)
+ die("cannot mix --serialize and --deserialize");
+ do_implicit_deserialize = 0;
+
+ do_serialize = 1;
+ return 0;
+}
+
+/*
+ * --deserialize | --deserialize= |
+ * --no-deserialize
+ *
+ * Request that we deserialize status data from some existing resource
+ * rather than performing a status scan.
+ *
+ * The input source can come from stdin or a path given here -- or be
+ * inherited from the config settings.
+ */
+static int opt_parse_deserialize(const struct option *opt UNUSED, const char *arg, int unset)
+{
+ if (unset) {
+ do_implicit_deserialize = 0;
+ do_explicit_deserialize = 0;
+ } else {
+ if (do_serialize)
+ die("cannot mix --serialize and --deserialize");
+ if (arg) {
+ /* override config or stdin */
+ free(deserialize_path);
+ deserialize_path = xstrdup(arg);
+ }
+ if (!deserialize_path || !*deserialize_path)
+ do_explicit_deserialize = 1; /* read stdin */
+ else if (wt_status_deserialize_access(deserialize_path, R_OK) == 0)
+ do_explicit_deserialize = 1; /* can read from this file */
+ else {
+ /*
+ * otherwise, silently fallback to the normal
+ * collection scan
+ */
+ do_implicit_deserialize = 0;
+ do_explicit_deserialize = 0;
+ }
+ }
+
+ return 0;
+}
+
+static enum wt_status_deserialize_wait parse_dw(const char *arg)
+{
+ int tenths;
+
+ if (!strcmp(arg, "fail"))
+ return DESERIALIZE_WAIT__FAIL;
+ else if (!strcmp(arg, "block"))
+ return DESERIALIZE_WAIT__BLOCK;
+ else if (!strcmp(arg, "no"))
+ return DESERIALIZE_WAIT__NO;
+
+ /*
+ * Otherwise, assume it is a timeout in tenths of a second.
+ * If it contains a bogus value, atol() will return zero
+ * which is OK.
+ */
+ tenths = atol(arg);
+ if (tenths < 0)
+ tenths = DESERIALIZE_WAIT__NO;
+ return tenths;
+}
+
+static int opt_parse_deserialize_wait(const struct option *opt UNUSED,
+ const char *arg,
+ int unset)
+{
+ if (unset)
+ explicit_deserialize_wait = DESERIALIZE_WAIT__UNSET;
+ else
+ explicit_deserialize_wait = parse_dw(arg);
+
+ return 0;
+}
+
static int opt_parse_m(const struct option *opt, const char *arg, int unset)
{
struct strbuf *buf = opt->value;
@@ -268,7 +385,7 @@ static int list_paths(struct string_list *list, const char *with_tree,
}
/* TODO: audit for interaction with sparse-index. */
- ensure_full_index(the_repository->index);
+ ensure_full_index_unaudited(the_repository->index);
for (i = 0; i < the_repository->index->cache_nr; i++) {
const struct cache_entry *ce = the_repository->index->cache[i];
struct string_list_item *item;
@@ -1016,7 +1133,7 @@ static int prepare_to_commit(const char *index_file, const char *prefix,
int i, ita_nr = 0;
/* TODO: audit for interaction with sparse-index. */
- ensure_full_index(the_repository->index);
+ ensure_full_index_unaudited(the_repository->index);
for (i = 0; i < the_repository->index->cache_nr; i++)
if (ce_intent_to_add(the_repository->index->cache[i]))
ita_nr++;
@@ -1186,6 +1303,8 @@ static enum untracked_status_type parse_untracked_setting_name(const char *u)
return SHOW_NORMAL_UNTRACKED_FILES;
else if (!strcmp(u, "all"))
return SHOW_ALL_UNTRACKED_FILES;
+ else if (!strcmp(u,"complete"))
+ return SHOW_COMPLETE_UNTRACKED_FILES;
else
return SHOW_UNTRACKED_FILES_ERROR;
}
@@ -1481,6 +1600,28 @@ static int git_status_config(const char *k, const char *v,
s->relative_paths = git_config_bool(k, v);
return 0;
}
+ if (!strcmp(k, "status.deserializepath")) {
+ /*
+ * Automatically assume deserialization if this is
+ * set in the config and the file exists. Do not
+ * complain if the file does not exist, because we
+ * silently fall back to normal mode.
+ */
+ if (v && *v && access(v, R_OK) == 0) {
+ do_implicit_deserialize = 1;
+ deserialize_path = xstrdup(v);
+ } else {
+ reject_implicit = 1;
+ }
+ return 0;
+ }
+ if (!strcmp(k, "status.deserializewait")) {
+ if (!v || !*v)
+ implicit_deserialize_wait = DESERIALIZE_WAIT__UNSET;
+ else
+ implicit_deserialize_wait = parse_dw(v);
+ return 0;
+ }
if (!strcmp(k, "status.showuntrackedfiles")) {
enum untracked_status_type u;
@@ -1520,7 +1661,8 @@ struct repository *repo UNUSED)
static const char *rename_score_arg = (const char *)-1;
static struct wt_status s;
unsigned int progress_flag = 0;
- int fd;
+ int try_deserialize;
+ int fd = -1;
struct object_id oid;
static struct option builtin_status_options[] = {
OPT__VERBOSE(&verbose, N_("be verbose")),
@@ -1535,6 +1677,15 @@ struct repository *repo UNUSED)
OPT_CALLBACK_F(0, "porcelain", &status_format,
N_("version"), N_("machine-readable output"),
PARSE_OPT_OPTARG, opt_parse_porcelain),
+ { OPTION_CALLBACK, 0, "serialize", &status_format,
+ N_("path"), N_("serialize raw status data to path or stdout"),
+ PARSE_OPT_OPTARG | PARSE_OPT_NONEG, opt_parse_serialize },
+ { OPTION_CALLBACK, 0, "deserialize", NULL,
+ N_("path"), N_("deserialize raw status data from file"),
+ PARSE_OPT_OPTARG, opt_parse_deserialize },
+ { OPTION_CALLBACK, 0, "deserialize-wait", NULL,
+ N_("fail|block|no"), N_("how to wait if status cache file is invalid"),
+ PARSE_OPT_OPTARG, opt_parse_deserialize_wait },
OPT_SET_INT(0, "long", &status_format,
N_("show status in long format (default)"),
STATUS_FORMAT_LONG),
@@ -1579,10 +1730,53 @@ struct repository *repo UNUSED)
s.show_untracked_files == SHOW_NO_UNTRACKED_FILES)
die(_("Unsupported combination of ignored and untracked-files arguments"));
+ if (s.show_untracked_files == SHOW_COMPLETE_UNTRACKED_FILES &&
+ s.show_ignored_mode == SHOW_NO_IGNORED)
+ die(_("Complete Untracked only supported with ignored files"));
+
parse_pathspec(&s.pathspec, 0,
PATHSPEC_PREFER_FULL,
prefix, argv);
+ /*
+ * If we want to try to deserialize status data from a cache file,
+ * we need to re-order the initialization code. The problem is that
+ * this makes for a very nasty diff and causes merge conflicts as we
+ * carry it forward. And it easy to mess up the merge, so we
+ * duplicate some code here to hopefully reduce conflicts.
+ */
+ try_deserialize = (!do_serialize &&
+ (do_implicit_deserialize || do_explicit_deserialize));
+
+ /*
+ * Disable deserialize when verbose is set because it causes us to
+ * print diffs for each modified file, but that requires us to have
+ * the index loaded and we don't want to do that (at least not now for
+ * this seldom used feature). My fear is that would further tangle
+ * the merge conflict with upstream.
+ *
+ * TODO Reconsider this in the future.
+ */
+ if (try_deserialize && verbose) {
+ trace2_data_string("status", the_repository, "deserialize/reject",
+ "args/verbose");
+ try_deserialize = 0;
+ }
+
+ if (try_deserialize)
+ goto skip_init;
+ /*
+ * If we implicitly received a status cache pathname from the config
+ * and the file does not exist, we silently reject it and do the normal
+ * status "collect". Fake up some trace2 messages to reflect this and
+ * assist post-processors know this case is different.
+ */
+ if (!do_serialize && reject_implicit) {
+ trace2_cmd_mode("implicit-deserialize");
+ trace2_data_string("status", the_repository, "deserialize/reject",
+ "status-cache/access");
+ }
+
enable_fscache(0);
if (status_format != STATUS_FORMAT_PORCELAIN &&
status_format != STATUS_FORMAT_PORCELAIN_V2)
@@ -1597,6 +1791,7 @@ struct repository *repo UNUSED)
else
fd = -1;
+skip_init:
s.is_initial = repo_get_oid(the_repository, s.reference, &oid) ? 1 : 0;
if (!s.is_initial)
oidcpy(&s.oid_commit, &oid);
@@ -1613,6 +1808,36 @@ struct repository *repo UNUSED)
s.rename_score = parse_rename_score(&rename_score_arg);
}
+ if (try_deserialize) {
+ int result;
+ enum wt_status_deserialize_wait dw = implicit_deserialize_wait;
+ if (explicit_deserialize_wait != DESERIALIZE_WAIT__UNSET)
+ dw = explicit_deserialize_wait;
+ if (dw == DESERIALIZE_WAIT__UNSET)
+ dw = DESERIALIZE_WAIT__NO;
+
+ if (s.relative_paths)
+ s.prefix = prefix;
+
+ trace2_cmd_mode("deserialize");
+ result = wt_status_deserialize(&s, deserialize_path, dw);
+ if (result == DESERIALIZE_OK)
+ return 0;
+ if (dw == DESERIALIZE_WAIT__FAIL)
+ die(_("Rejected status serialization cache"));
+
+ /* deserialize failed, so force the initialization we skipped above. */
+ enable_fscache(1);
+ repo_read_index_preload(the_repository, &s.pathspec, 0);
+ refresh_index(the_repository->index, REFRESH_QUIET|REFRESH_UNMERGED, &s.pathspec, NULL, NULL);
+
+ if (use_optional_locks())
+ fd = repo_hold_locked_index(the_repository, &index_lock, 0);
+ else
+ fd = -1;
+ }
+
+ trace2_cmd_mode("collect");
wt_status_collect(&s);
if (0 <= fd)
@@ -1621,6 +1846,17 @@ struct repository *repo UNUSED)
if (s.relative_paths)
s.prefix = prefix;
+ if (serialize_path) {
+ int fd_serialize = xopen(serialize_path,
+ O_WRONLY | O_CREAT | O_TRUNC, 0666);
+ if (fd_serialize < 0)
+ die_errno(_("could not serialize to '%s'"),
+ serialize_path);
+ trace2_cmd_mode("serialize");
+ wt_status_serialize_v1(fd_serialize, &s);
+ close(fd_serialize);
+ }
+
wt_status_print(&s);
wt_status_collect_free_buffers(&s);
diff --git a/builtin/difftool.c b/builtin/difftool.c
index fbd7537b1be769..fc5811c43eb57e 100644
--- a/builtin/difftool.c
+++ b/builtin/difftool.c
@@ -592,7 +592,7 @@ static int run_dir_diff(const char *extcmd, int symlinks, const char *prefix,
ret = run_command(&cmd);
/* TODO: audit for interaction with sparse-index. */
- ensure_full_index(&wtindex);
+ ensure_full_index_unaudited(&wtindex);
/*
* If the diff includes working copy files and those
diff --git a/builtin/fetch.c b/builtin/fetch.c
index 2d37a378ba7650..52235b510c10de 100644
--- a/builtin/fetch.c
+++ b/builtin/fetch.c
@@ -21,6 +21,9 @@
#include "string-list.h"
#include "remote.h"
#include "transport.h"
+#include "gvfs.h"
+#include "gvfs-helper-client.h"
+#include "packfile.h"
#include "run-command.h"
#include "parse-options.h"
#include "sigchain.h"
@@ -1160,6 +1163,13 @@ static int store_updated_refs(struct display_state *display_state,
opt.exclude_hidden_refs_section = "fetch";
rm = ref_map;
+
+ /*
+ * Before checking connectivity, be really sure we have the
+ * latest pack-files loaded into memory.
+ */
+ reprepare_packed_git(the_repository);
+
if (check_connected(iterate_ref_map, &rm, &opt)) {
rc = error(_("%s did not send all necessary objects"),
display_state->url);
@@ -2549,6 +2559,9 @@ int cmd_fetch(int argc,
}
string_list_remove_duplicates(&list, 0);
+ if (core_gvfs & GVFS_PREFETCH_DURING_FETCH)
+ gh_client__prefetch(0, NULL);
+
if (negotiate_only) {
struct oidset acked_commits = OIDSET_INIT;
struct oidset_iter iter;
diff --git a/builtin/fsck.c b/builtin/fsck.c
index 0196c54eb68ee5..e86176f6a75c1b 100644
--- a/builtin/fsck.c
+++ b/builtin/fsck.c
@@ -821,7 +821,7 @@ static void fsck_index(struct index_state *istate, const char *index_path,
unsigned int i;
/* TODO: audit for interaction with sparse-index. */
- ensure_full_index(istate);
+ ensure_full_index_unaudited(istate);
for (i = 0; i < istate->cache_nr; i++) {
unsigned int mode;
struct blob *blob;
diff --git a/builtin/gc.c b/builtin/gc.c
index a9b1c36de27da2..5d8f1561c12f88 100644
--- a/builtin/gc.c
+++ b/builtin/gc.c
@@ -18,6 +18,7 @@
#include "date.h"
#include "environment.h"
#include "hex.h"
+#include "gvfs.h"
#include "config.h"
#include "tempfile.h"
#include "lockfile.h"
@@ -749,6 +750,9 @@ struct repository *repo UNUSED)
if (quiet)
strvec_push(&repack, "-q");
+ if ((!opts.auto_flag || (opts.auto_flag && cfg.gc_auto_threshold > 0)) && gvfs_config_is_set(GVFS_BLOCK_COMMANDS))
+ die(_("'git gc' is not supported on a GVFS repo"));
+
if (opts.auto_flag) {
if (cfg.detach_auto && opts.detach < 0)
opts.detach = 1;
@@ -1150,6 +1154,8 @@ static int write_loose_object_to_stdin(const struct object_id *oid,
return ++(d->count) > d->batch_size;
}
+static const char *object_dir = NULL;
+
static int pack_loose(struct maintenance_run_opts *opts)
{
struct repository *r = the_repository;
@@ -1157,11 +1163,14 @@ static int pack_loose(struct maintenance_run_opts *opts)
struct write_loose_object_data data;
struct child_process pack_proc = CHILD_PROCESS_INIT;
+ if (!object_dir)
+ object_dir = r->objects->odb->path;
+
/*
* Do not start pack-objects process
* if there are no loose objects.
*/
- if (!for_each_loose_file_in_objdir(r->objects->odb->path,
+ if (!for_each_loose_file_in_objdir(object_dir,
bail_on_loose,
NULL, NULL, NULL))
return 0;
@@ -1171,7 +1180,7 @@ static int pack_loose(struct maintenance_run_opts *opts)
strvec_push(&pack_proc.args, "pack-objects");
if (opts->quiet)
strvec_push(&pack_proc.args, "--quiet");
- strvec_pushf(&pack_proc.args, "%s/pack/loose", r->objects->odb->path);
+ strvec_pushf(&pack_proc.args, "%s/pack/loose", object_dir);
pack_proc.in = -1;
@@ -1190,7 +1199,7 @@ static int pack_loose(struct maintenance_run_opts *opts)
data.count = 0;
data.batch_size = 50000;
- for_each_loose_file_in_objdir(r->objects->odb->path,
+ for_each_loose_file_in_objdir(object_dir,
write_loose_object_to_stdin,
NULL,
NULL,
@@ -1580,6 +1589,7 @@ static int maintenance_run(int argc, const char **argv, const char *prefix,
int i;
struct maintenance_run_opts opts = MAINTENANCE_RUN_OPTS_INIT;
struct gc_config cfg = GC_CONFIG_INIT;
+ const char *tmp_obj_dir = NULL;
struct option builtin_maintenance_run_options[] = {
OPT_BOOL(0, "auto", &opts.auto_flag,
N_("run tasks based on the state of the repository")),
@@ -1617,6 +1627,17 @@ static int maintenance_run(int argc, const char **argv, const char *prefix,
usage_with_options(builtin_maintenance_run_usage,
builtin_maintenance_run_options);
+ /*
+ * To enable the VFS for Git/Scalar shared object cache, use
+ * the gvfs.sharedcache config option to redirect the
+ * maintenance to that location.
+ */
+ if (!git_config_get_value("gvfs.sharedcache", &tmp_obj_dir) &&
+ tmp_obj_dir) {
+ object_dir = xstrdup(tmp_obj_dir);
+ setenv(DB_ENVIRONMENT, object_dir, 1);
+ }
+
ret = maintenance_run_tasks(&opts, &cfg);
gc_config_release(&cfg);
return ret;
diff --git a/builtin/index-pack.c b/builtin/index-pack.c
index d773809c4c9660..0c11a1bcb49352 100644
--- a/builtin/index-pack.c
+++ b/builtin/index-pack.c
@@ -889,7 +889,7 @@ static void sha1_object(const void *data, struct object_entry *obj_entry,
read_lock();
collision_test_needed =
repo_has_object_file_with_flags(the_repository, oid,
- OBJECT_INFO_QUICK);
+ OBJECT_INFO_FOR_PREFETCH);
read_unlock();
}
@@ -1890,6 +1890,7 @@ int cmd_index_pack(int argc,
unsigned foreign_nr = 1; /* zero is a "good" value, assume bad */
int report_end_of_input = 0;
int hash_algo = 0;
+ int dash_o = 0;
/*
* index-pack never needs to fetch missing objects except when
@@ -1983,6 +1984,7 @@ int cmd_index_pack(int argc,
if (index_name || (i+1) >= argc)
usage(index_pack_usage);
index_name = argv[++i];
+ dash_o = 1;
} else if (starts_with(arg, "--index-version=")) {
char *c;
opts.version = strtoul(arg + 16, &c, 10);
@@ -2036,6 +2038,8 @@ int cmd_index_pack(int argc,
repo_set_hash_algo(the_repository, GIT_HASH_SHA1);
opts.flags &= ~(WRITE_REV | WRITE_REV_VERIFY);
+ if (rev_index && dash_o && !ends_with(index_name, ".idx"))
+ rev_index = 0;
if (rev_index) {
opts.flags |= verify ? WRITE_REV_VERIFY : WRITE_REV;
if (index_name)
diff --git a/builtin/ls-files.c b/builtin/ls-files.c
index 15499cd12b6bd5..2411fe523eb89e 100644
--- a/builtin/ls-files.c
+++ b/builtin/ls-files.c
@@ -413,7 +413,7 @@ static void show_files(struct repository *repo, struct dir_struct *dir)
return;
if (!show_sparse_dirs)
- ensure_full_index(repo->index);
+ ensure_full_index_with_reason(repo->index, "ls-files");
for (i = 0; i < repo->index->cache_nr; i++) {
const struct cache_entry *ce = repo->index->cache[i];
diff --git a/builtin/merge-index.c b/builtin/merge-index.c
index 342699edb77c97..6a1d7966626692 100644
--- a/builtin/merge-index.c
+++ b/builtin/merge-index.c
@@ -66,7 +66,7 @@ static void merge_all(void)
{
int i;
/* TODO: audit for interaction with sparse-index. */
- ensure_full_index(the_repository->index);
+ ensure_full_index_unaudited(the_repository->index);
for (i = 0; i < the_repository->index->cache_nr; i++) {
const struct cache_entry *ce = the_repository->index->cache[i];
if (!ce_stage(ce))
@@ -93,7 +93,7 @@ int cmd_merge_index(int argc,
repo_read_index(the_repository);
/* TODO: audit for interaction with sparse-index. */
- ensure_full_index(the_repository->index);
+ ensure_full_index_unaudited(the_repository->index);
i = 1;
if (!strcmp(argv[i], "-o")) {
diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c
index 4eeaf788b117f8..8c5e39d73f886d 100644
--- a/builtin/pack-objects.c
+++ b/builtin/pack-objects.c
@@ -203,6 +203,7 @@ static int keep_unreachable, unpack_unreachable, include_tag;
static timestamp_t unpack_unreachable_expiration;
static int pack_loose_unreachable;
static int cruft;
+static int shallow = 0;
static timestamp_t cruft_expiration;
static int local;
static int have_non_local_packs;
@@ -4429,6 +4430,7 @@ static void get_object_list_path_walk(struct rev_info *revs)
* base objects.
*/
info.prune_all_uninteresting = sparse;
+ info.edge_aggressive = shallow;
if (walk_objects_by_path(&info))
die(_("failed to pack objects via path-walk"));
@@ -4630,7 +4632,6 @@ int cmd_pack_objects(int argc,
struct repository *repo UNUSED)
{
int use_internal_rev_list = 0;
- int shallow = 0;
int all_progress_implied = 0;
struct strvec rp = STRVEC_INIT;
int rev_list_unpacked = 0, rev_list_all = 0, rev_list_reflog = 0;
@@ -4818,10 +4819,6 @@ int cmd_pack_objects(int argc,
warning(_("cannot use delta islands with --path-walk"));
path_walk = 0;
}
- if (path_walk && shallow) {
- warning(_("cannot use --shallow with --path-walk"));
- path_walk = 0;
- }
if (path_walk) {
strvec_push(&rp, "--boundary");
/*
diff --git a/builtin/push.c b/builtin/push.c
index 90de3746b5229f..f49f436dd389b1 100644
--- a/builtin/push.c
+++ b/builtin/push.c
@@ -619,6 +619,10 @@ int cmd_push(int argc,
else if (recurse_submodules == RECURSE_SUBMODULES_ONLY)
flags |= TRANSPORT_RECURSE_SUBMODULES_ONLY;
+ prepare_repo_settings(the_repository);
+ if (the_repository->settings.pack_use_path_walk)
+ flags |= TRANSPORT_PUSH_NO_REUSE_DELTA;
+
if (tags)
refspec_append(&rs, "refs/tags/*");
diff --git a/builtin/read-tree.c b/builtin/read-tree.c
index d2a807a828b6ab..e84cd4ee4d2d9a 100644
--- a/builtin/read-tree.c
+++ b/builtin/read-tree.c
@@ -226,7 +226,8 @@ int cmd_read_tree(int argc,
setup_work_tree();
if (opts.skip_sparse_checkout)
- ensure_full_index(the_repository->index);
+ ensure_full_index_with_reason(the_repository->index,
+ "read-tree");
if (opts.merge) {
switch (stage - 1) {
diff --git a/builtin/reset.c b/builtin/reset.c
index 6cfab674e40541..4e9a058766f382 100644
--- a/builtin/reset.c
+++ b/builtin/reset.c
@@ -40,6 +40,8 @@
#include "add-interactive.h"
#include "strbuf.h"
#include "quote.h"
+#include "dir.h"
+#include "entry.h"
#define REFRESH_INDEX_DELAY_WARNING_IN_MS (2 * 1000)
@@ -160,9 +162,48 @@ static void update_index_from_diff(struct diff_queue_struct *q,
for (i = 0; i < q->nr; i++) {
int pos;
+ int respect_skip_worktree = 1;
struct diff_filespec *one = q->queue[i]->one;
+ struct diff_filespec *two = q->queue[i]->two;
int is_in_reset_tree = one->mode && !is_null_oid(&one->oid);
+ int is_missing = !(one->mode && !is_null_oid(&one->oid));
+ int was_missing = !two->mode && is_null_oid(&two->oid);
struct cache_entry *ce;
+ struct cache_entry *ceBefore;
+ struct checkout state = CHECKOUT_INIT;
+
+ /*
+ * When using the virtual filesystem feature, the cache entries that are
+ * added here will not have the skip-worktree bit set.
+ *
+ * Without this code there is data that is lost because the files that
+ * would normally be in the working directory are not there and show as
+ * deleted for the next status or in the case of added files just disappear.
+ * We need to create the previous version of the files in the working
+ * directory so that they will have the right content and the next
+ * status call will show modified or untracked files correctly.
+ */
+ if (core_virtualfilesystem && !file_exists(two->path))
+ {
+ respect_skip_worktree = 0;
+ pos = index_name_pos(the_repository->index, two->path, strlen(two->path));
+
+ if ((pos >= 0 && ce_skip_worktree(the_repository->index->cache[pos])) &&
+ (is_missing || !was_missing))
+ {
+ state.force = 1;
+ state.refresh_cache = 1;
+ state.istate = the_repository->index;
+ ceBefore = make_cache_entry(the_repository->index, two->mode,
+ &two->oid, two->path,
+ 0, 0);
+ if (!ceBefore)
+ die(_("make_cache_entry failed for path '%s'"),
+ two->path);
+
+ checkout_entry(ceBefore, &state, NULL, NULL);
+ }
+ }
if (!is_in_reset_tree && !intent_to_add) {
remove_file_from_index(the_repository->index, one->path);
@@ -181,8 +222,14 @@ static void update_index_from_diff(struct diff_queue_struct *q,
* to properly construct the reset sparse directory.
*/
pos = index_name_pos(the_repository->index, one->path, strlen(one->path));
- if ((pos >= 0 && ce_skip_worktree(the_repository->index->cache[pos])) ||
- (pos < 0 && !path_in_sparse_checkout(one->path, the_repository->index)))
+
+ /*
+ * Do not add the SKIP_WORKTREE bit back if we populated the
+ * file on purpose in a virtual filesystem scenario.
+ */
+ if (respect_skip_worktree &&
+ ((pos >= 0 && ce_skip_worktree(the_repository->index->cache[pos])) ||
+ (pos < 0 && !path_in_sparse_checkout(one->path, the_repository->index))))
ce->ce_flags |= CE_SKIP_WORKTREE;
if (!ce)
@@ -215,7 +262,8 @@ static int read_from_tree(const struct pathspec *pathspec,
opt.add_remove = diff_addremove;
if (pathspec->nr && pathspec_needs_expanded_index(the_repository->index, pathspec))
- ensure_full_index(the_repository->index);
+ ensure_full_index_with_reason(the_repository->index,
+ "reset pathspec");
if (do_diff_cache(tree_oid, &opt))
return 1;
diff --git a/builtin/rm.c b/builtin/rm.c
index 12ae086a556ce3..043f95240388c1 100644
--- a/builtin/rm.c
+++ b/builtin/rm.c
@@ -8,6 +8,7 @@
#define DISABLE_SIGN_COMPARE_WARNINGS
#include "builtin.h"
+#include "environment.h"
#include "advice.h"
#include "config.h"
#include "lockfile.h"
@@ -312,12 +313,13 @@ int cmd_rm(int argc,
seen = xcalloc(pathspec.nr, 1);
if (pathspec_needs_expanded_index(the_repository->index, &pathspec))
- ensure_full_index(the_repository->index);
+ ensure_full_index_with_reason(the_repository->index,
+ "rm pathspec");
for (i = 0; i < the_repository->index->cache_nr; i++) {
const struct cache_entry *ce = the_repository->index->cache[i];
- if (!include_sparse &&
+ if (!include_sparse && !core_virtualfilesystem &&
(ce_skip_worktree(ce) ||
!path_in_sparse_checkout(ce->name, the_repository->index)))
continue;
@@ -354,7 +356,11 @@ int cmd_rm(int argc,
*original ? original : ".");
}
- if (only_match_skip_worktree.nr) {
+ /*
+ * When using a virtual filesystem, we might re-add a path
+ * that is currently virtual and we want that to succeed.
+ */
+ if (!core_virtualfilesystem && only_match_skip_worktree.nr) {
advise_on_updating_sparse_paths(&only_match_skip_worktree);
ret = 1;
}
diff --git a/builtin/sparse-checkout.c b/builtin/sparse-checkout.c
index 14dcace5f8ff7c..ec1a7de9995fce 100644
--- a/builtin/sparse-checkout.c
+++ b/builtin/sparse-checkout.c
@@ -111,7 +111,7 @@ static int sparse_checkout_list(int argc, const char **argv, const char *prefix,
static void clean_tracked_sparse_directories(struct repository *r)
{
- int i, was_full = 0;
+ int i, value, was_full = 0;
struct strbuf path = STRBUF_INIT;
size_t pathlen;
struct string_list_item *item;
@@ -127,6 +127,13 @@ static void clean_tracked_sparse_directories(struct repository *r)
!r->index->sparse_checkout_patterns->use_cone_patterns)
return;
+ /*
+ * Users can disable this behavior.
+ */
+ if (!repo_config_get_bool(r, "index.deletesparsedirectories", &value) &&
+ !value)
+ return;
+
/*
* Use the sparse index as a data structure to assist finding
* directories that are safe to delete. This conversion to a
@@ -200,7 +207,8 @@ static void clean_tracked_sparse_directories(struct repository *r)
strbuf_release(&path);
if (was_full)
- ensure_full_index(r->index);
+ ensure_full_index_with_reason(r->index,
+ "sparse-checkout:was full");
}
static int update_working_directory(struct pattern_list *pl)
@@ -430,7 +438,8 @@ static int update_modes(int *cone_mode, int *sparse_index)
the_repository->index->updated_workdir = 1;
if (!*sparse_index)
- ensure_full_index(the_repository->index);
+ ensure_full_index_with_reason(the_repository->index,
+ "sparse-checkout:disabling sparse index");
}
return 0;
diff --git a/builtin/stash.c b/builtin/stash.c
index dbaa999cf171a7..79fec5612edb6c 100644
--- a/builtin/stash.c
+++ b/builtin/stash.c
@@ -403,7 +403,7 @@ static int restore_untracked(struct object_id *u_tree)
child_process_init(&cp);
cp.git_cmd = 1;
- strvec_pushl(&cp.args, "checkout-index", "--all", NULL);
+ strvec_pushl(&cp.args, "checkout-index", "--all", "-f", NULL);
strvec_pushf(&cp.env, "GIT_INDEX_FILE=%s",
stash_index_path.buf);
@@ -1560,7 +1560,7 @@ static int do_push_stash(const struct pathspec *ps, const char *stash_msg, int q
char *ps_matched = xcalloc(ps->nr, 1);
/* TODO: audit for interaction with sparse-index. */
- ensure_full_index(the_repository->index);
+ ensure_full_index_unaudited(the_repository->index);
for (size_t i = 0; i < the_repository->index->cache_nr; i++)
ce_path_match(the_repository->index, the_repository->index->cache[i], ps,
ps_matched);
diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c
index f9b970f8a64a54..5a708639452993 100644
--- a/builtin/submodule--helper.c
+++ b/builtin/submodule--helper.c
@@ -3400,7 +3400,7 @@ static void die_on_index_match(const char *path, int force)
char *ps_matched = xcalloc(ps.nr, 1);
/* TODO: audit for interaction with sparse-index. */
- ensure_full_index(the_repository->index);
+ ensure_full_index_unaudited(the_repository->index);
/*
* Since there is only one pathspec, we just need to
diff --git a/builtin/survey.c b/builtin/survey.c
index a86b728d6a2671..c063b7bc44f0c3 100644
--- a/builtin/survey.c
+++ b/builtin/survey.c
@@ -13,11 +13,12 @@
#include "ref-filter.h"
#include "refs.h"
#include "revision.h"
+#include "run-command.h"
#include "strbuf.h"
#include "strvec.h"
-#include "tag.h"
#include "trace2.h"
-#include "color.h"
+#include "tree.h"
+#include "tree-walk.h"
static const char * const survey_usage[] = {
N_("(EXPERIMENTAL!) git survey "),
@@ -41,6 +42,16 @@ static struct survey_refs_wanted default_ref_options = {
struct survey_opts {
int verbose;
int show_progress;
+ int show_name_rev;
+
+ int show_largest_commits_by_nr_parents;
+ int show_largest_commits_by_size_bytes;
+
+ int show_largest_trees_by_nr_entries;
+ int show_largest_trees_by_size_bytes;
+
+ int show_largest_blobs_by_size_bytes;
+
int top_nr;
struct survey_refs_wanted refs;
};
@@ -53,6 +64,312 @@ struct survey_report_ref_summary {
size_t tags_annotated_nr;
size_t others_nr;
size_t unknown_nr;
+
+ size_t cnt_symref;
+
+ size_t cnt_packed;
+ size_t cnt_loose;
+
+ /*
+ * Measure the length of the refnames. We can look for
+ * potential platform limits. The partial sums may help us
+ * estimate the size of a haves/wants conversation, since each
+ * refname and a SHA must be transmitted.
+ */
+ size_t len_max_local_refname;
+ size_t len_sum_local_refnames;
+ size_t len_max_remote_refname;
+ size_t len_sum_remote_refnames;
+};
+
+/*
+ * HBIN -- hex binning (histogram bucketing).
+ *
+ * We create histograms for various counts and sums. Since we have a
+ * wide range of values (objects range in size from 1 to 4G bytes), a
+ * linear bucketing is not interesting. Instead, lets use a
+ * log16()-based bucketing. This gives us a better spread on the low
+ * and middle range and a coarse bucketing on the high end.
+ *
+ * The idea here is that it doesn't matter if you have n 1GB blobs or
+ * n/2 1GB blobs and n/2 1.5GB blobs -- either way you have a scaling
+ * problem that we want to report on.
+ */
+#define HBIN_LEN (sizeof(unsigned long) * 2)
+#define HBIN_MASK (0xF)
+#define HBIN_SHIFT (4)
+
+static int hbin(unsigned long value)
+{
+ for (size_t k = 0; k < HBIN_LEN; k++) {
+ if ((value & ~(HBIN_MASK)) == 0)
+ return k;
+ value >>= HBIN_SHIFT;
+ }
+
+ return 0; /* should not happen */
+}
+
+/*
+ * QBIN -- base4 binning (histogram bucketing).
+ *
+ * This is the same idea as the above, but we want better granularity
+ * in the low end and don't expect as many large values.
+ */
+#define QBIN_LEN (sizeof(unsigned long) * 4)
+#define QBIN_MASK (0x3)
+#define QBIN_SHIFT (2)
+
+static int qbin(unsigned long value)
+{
+ for (size_t k = 0; k < QBIN_LEN; k++) {
+ if ((value & ~(QBIN_MASK)) == 0)
+ return k;
+ value >>= (QBIN_SHIFT);
+ }
+
+ return 0; /* should not happen */
+}
+
+/*
+ * histogram bin for objects.
+ */
+struct obj_hist_bin {
+ uint64_t sum_size; /* sum(object_size) for all objects in this bin */
+ uint64_t sum_disk_size; /* sum(on_disk_size) for all objects in this bin */
+ uint32_t cnt_seen; /* number seen in this bin */
+};
+
+static void incr_obj_hist_bin(struct obj_hist_bin *pbin,
+ unsigned long object_length,
+ off_t disk_sizep)
+{
+ pbin->sum_size += object_length;
+ pbin->sum_disk_size += disk_sizep;
+ pbin->cnt_seen++;
+}
+
+/*
+ * Remember the largest n objects for some scaling dimension. This
+ * could be the observed object size or number of entries in a tree.
+ * We'll use this to generate a sorted vector in the output for that
+ * dimension.
+ */
+struct large_item {
+ uint64_t size;
+ struct object_id oid;
+
+ /*
+ * For blobs and trees the name field is the pathname of the
+ * file or directory. Root trees will have a zero-length
+ * name. The name field is not currenly used for commits.
+ */
+ struct strbuf name;
+
+ /*
+ * For blobs and trees remember the transient commit from
+ * the treewalk so that we can say that this large item
+ * first appeared in this commit (relative to the treewalk
+ * order).
+ */
+ struct object_id containing_commit_oid;
+
+ /*
+ * Lookup `containing_commit_oid` using `git name-rev`.
+ * Lazy allocate this post-treewalk.
+ */
+ struct strbuf name_rev;
+};
+
+struct large_item_vec {
+ char *dimension_label;
+ char *item_label;
+ uint64_t nr_items;
+ struct large_item items[FLEX_ARRAY]; /* nr_items */
+};
+
+static struct large_item_vec *alloc_large_item_vec(const char *dimension_label,
+ const char *item_label,
+ uint64_t nr_items)
+{
+ struct large_item_vec *vec;
+ size_t flex_len = nr_items * sizeof(struct large_item);
+ size_t k;
+
+ if (!nr_items)
+ return NULL;
+
+ vec = xcalloc(1, (sizeof(struct large_item_vec) + flex_len));
+ vec->dimension_label = strdup(dimension_label);
+ vec->item_label = strdup(item_label);
+ vec->nr_items = nr_items;
+
+ for (k = 0; k < nr_items; k++)
+ strbuf_init(&vec->items[k].name, 0);
+
+ return vec;
+}
+
+static void free_large_item_vec(struct large_item_vec *vec)
+{
+ if (!vec)
+ return;
+
+ for (size_t k = 0; k < vec->nr_items; k++) {
+ strbuf_release(&vec->items[k].name);
+ strbuf_release(&vec->items[k].name_rev);
+ }
+
+ free(vec->dimension_label);
+ free(vec->item_label);
+ free(vec);
+}
+
+static void maybe_insert_large_item(struct large_item_vec *vec,
+ uint64_t size,
+ struct object_id *oid,
+ const char *name,
+ const struct object_id *containing_commit_oid)
+{
+ size_t rest_len;
+ size_t k;
+
+ if (!vec || !vec->nr_items)
+ return;
+
+ /*
+ * Since the odds an object being among the largest n
+ * is small, shortcut and see if it is smaller than
+ * the smallest one in our set and quickly reject it.
+ */
+ if (size < vec->items[vec->nr_items - 1].size)
+ return;
+
+ for (k = 0; k < vec->nr_items; k++) {
+ if (size < vec->items[k].size)
+ continue;
+
+ /*
+ * The last large_item in the vector is about to be
+ * overwritten by the previous one during the shift.
+ * Steal its allocated strbuf and reuse it.
+ *
+ * We can ignore .name_rev because it will not be
+ * allocated until after the treewalk.
+ */
+ strbuf_release(&vec->items[vec->nr_items - 1].name);
+
+ /* push items[k..] down one and insert data for this item here */
+
+ rest_len = (vec->nr_items - k - 1) * sizeof(struct large_item);
+ if (rest_len)
+ memmove(&vec->items[k + 1], &vec->items[k], rest_len);
+
+ memset(&vec->items[k], 0, sizeof(struct large_item));
+ vec->items[k].size = size;
+ oidcpy(&vec->items[k].oid, oid);
+ oidcpy(&vec->items[k].containing_commit_oid, containing_commit_oid ? containing_commit_oid : null_oid());
+ strbuf_init(&vec->items[k].name, 0);
+ if (name && *name)
+ strbuf_addstr(&vec->items[k].name, name);
+
+ return;
+ }
+}
+
+/*
+ * Common fields for any type of object.
+ */
+struct survey_stats_base_object {
+ uint32_t cnt_seen;
+
+ uint32_t cnt_missing; /* we may have a partial clone. */
+
+ /*
+ * Number of objects grouped by where they are stored on disk.
+ * This is a function of how the ODB is packed.
+ */
+ uint32_t cnt_cached; /* see oi.whence */
+ uint32_t cnt_loose; /* see oi.whence */
+ uint32_t cnt_packed; /* see oi.whence */
+ uint32_t cnt_dbcached; /* see oi.whence */
+
+ uint64_t sum_size; /* sum(object_size) */
+ uint64_t sum_disk_size; /* sum(disk_size) */
+
+ /*
+ * A histogram of the count of objects, the observed size, and
+ * the on-disk size grouped by the observed size.
+ */
+ struct obj_hist_bin size_hbin[HBIN_LEN];
+};
+
+/*
+ * PBIN -- parent vector binning (histogram bucketing).
+ *
+ * We create a histogram based upon the number of parents
+ * in a commit. This is a simple linear vector. It starts
+ * at zero for "initial" commits.
+ *
+ * If a commit has more parents, just put it in the last bin.
+ */
+#define PBIN_VEC_LEN (32)
+
+struct survey_stats_commits {
+ struct survey_stats_base_object base;
+
+ /*
+ * Count of commits with k parents.
+ */
+ uint32_t parent_cnt_pbin[PBIN_VEC_LEN];
+
+ struct large_item_vec *vec_largest_by_nr_parents;
+ struct large_item_vec *vec_largest_by_size_bytes;
+};
+
+/*
+ * Stats for reachable trees.
+ */
+struct survey_stats_trees {
+ struct survey_stats_base_object base;
+
+ /*
+ * Keep a vector of the trees with the most number of entries.
+ * This gives us a feel for the width of a tree when there are
+ * gigantic directories.
+ */
+ struct large_item_vec *vec_largest_by_nr_entries;
+
+ /*
+ * Keep a vector of the trees with the largest size in bytes.
+ * The contents of this may or may not match items in the other
+ * vector, since entryname length can alter the results.
+ */
+ struct large_item_vec *vec_largest_by_size_bytes;
+
+ /*
+ * Computing the sum of the number of entries across all trees
+ * is probably not that interesting.
+ */
+ uint64_t sum_entries; /* sum(nr_entries) -- sum across all trees */
+
+ /*
+ * A histogram of the count of trees, the observed size, and
+ * the on-disk size grouped by the number of entries in the tree.
+ */
+ struct obj_hist_bin entry_qbin[QBIN_LEN];
+};
+
+/*
+ * Stats for reachable blobs.
+ */
+struct survey_stats_blobs {
+ struct survey_stats_base_object base;
+
+ /*
+ * Remember the OIDs of the largest n blobs.
+ */
+ struct large_item_vec *vec_largest_by_size_bytes;
};
struct survey_report_object_summary {
@@ -60,6 +377,10 @@ struct survey_report_object_summary {
size_t tags_nr;
size_t trees_nr;
size_t blobs_nr;
+
+ struct survey_stats_commits commits;
+ struct survey_stats_trees trees;
+ struct survey_stats_blobs blobs;
};
/**
@@ -229,6 +550,12 @@ struct survey_context {
static void clear_survey_context(struct survey_context *ctx)
{
+ free_large_item_vec(ctx->report.reachable_objects.commits.vec_largest_by_nr_parents);
+ free_large_item_vec(ctx->report.reachable_objects.commits.vec_largest_by_size_bytes);
+ free_large_item_vec(ctx->report.reachable_objects.trees.vec_largest_by_nr_entries);
+ free_large_item_vec(ctx->report.reachable_objects.trees.vec_largest_by_size_bytes);
+ free_large_item_vec(ctx->report.reachable_objects.blobs.vec_largest_by_size_bytes);
+
ref_array_clear(&ctx->ref_array);
strvec_clear(&ctx->refs);
}
@@ -349,6 +676,128 @@ static void print_table_plaintext(struct survey_table *table)
free(column_widths);
}
+static void pretty_print_bin_table(const char *title_caption,
+ const char *bucket_header,
+ struct obj_hist_bin *bin,
+ uint64_t bin_len, int bin_shift, uint64_t bin_mask)
+{
+ struct survey_table table = SURVEY_TABLE_INIT;
+ struct strbuf bucket = STRBUF_INIT, cnt_seen = STRBUF_INIT;
+ struct strbuf sum_size = STRBUF_INIT, sum_disk_size = STRBUF_INIT;
+ uint64_t lower = 0;
+ uint64_t upper = bin_mask;
+
+ table.table_name = title_caption;
+ strvec_pushl(&table.header, bucket_header, "Count", "Size", "Disk Size", NULL);
+
+ for (size_t k = 0; k < bin_len; k++) {
+ struct obj_hist_bin *p = bin + k;
+ uintmax_t lower_k = lower;
+ uintmax_t upper_k = upper;
+
+ lower = upper+1;
+ upper = (upper << bin_shift) + bin_mask;
+
+ if (!p->cnt_seen)
+ continue;
+
+ strbuf_reset(&bucket);
+ strbuf_addf(&bucket, "%"PRIuMAX"..%"PRIuMAX, lower_k, upper_k);
+
+ strbuf_reset(&cnt_seen);
+ strbuf_addf(&cnt_seen, "%"PRIuMAX, (uintmax_t)p->cnt_seen);
+
+ strbuf_reset(&sum_size);
+ strbuf_addf(&sum_size, "%"PRIuMAX, (uintmax_t)p->sum_size);
+
+ strbuf_reset(&sum_disk_size);
+ strbuf_addf(&sum_disk_size, "%"PRIuMAX, (uintmax_t)p->sum_disk_size);
+
+ insert_table_rowv(&table, bucket.buf,
+ cnt_seen.buf, sum_size.buf, sum_disk_size.buf, NULL);
+ }
+ strbuf_release(&bucket);
+ strbuf_release(&cnt_seen);
+ strbuf_release(&sum_size);
+ strbuf_release(&sum_disk_size);
+
+ print_table_plaintext(&table);
+ clear_table(&table);
+}
+
+static void survey_report_hbin(const char *title_caption,
+ struct obj_hist_bin *bin)
+{
+ pretty_print_bin_table(title_caption,
+ "Byte Range",
+ bin,
+ HBIN_LEN, HBIN_SHIFT, HBIN_MASK);
+}
+
+static void survey_report_tree_lengths(struct survey_context *ctx)
+{
+ pretty_print_bin_table(_("TREE HISTOGRAM BY NUMBER OF ENTRIES"),
+ "Entry Range",
+ ctx->report.reachable_objects.trees.entry_qbin,
+ QBIN_LEN, QBIN_SHIFT, QBIN_MASK);
+}
+
+static void survey_report_commit_parents(struct survey_context *ctx)
+{
+ struct survey_stats_commits *psc = &ctx->report.reachable_objects.commits;
+ struct survey_table table = SURVEY_TABLE_INIT;
+ struct strbuf parents = STRBUF_INIT, counts = STRBUF_INIT;
+
+ table.table_name = _("HISTOGRAM BY NUMBER OF COMMIT PARENTS");
+ strvec_pushl(&table.header, "Parents", "Counts", NULL);
+
+ for (int k = 0; k < PBIN_VEC_LEN; k++)
+ if (psc->parent_cnt_pbin[k]) {
+ strbuf_reset(&parents);
+ strbuf_addf(&parents, "%02d", k);
+
+ strbuf_reset(&counts);
+ strbuf_addf(&counts, "%14"PRIuMAX, (uintmax_t)psc->parent_cnt_pbin[k]);
+
+ insert_table_rowv(&table, parents.buf, counts.buf, NULL);
+ }
+ strbuf_release(&parents);
+ strbuf_release(&counts);
+
+ print_table_plaintext(&table);
+ clear_table(&table);
+}
+
+static void survey_report_largest_vec(struct survey_context *ctx, struct large_item_vec *vec)
+{
+ struct survey_table table = SURVEY_TABLE_INIT;
+ struct strbuf size = STRBUF_INIT;
+
+ if (!vec || !vec->nr_items)
+ return;
+
+ table.table_name = vec->dimension_label;
+ strvec_pushl(&table.header, "Size", "OID", "Name", "Commit", ctx->opts.show_name_rev ? "Name-Rev" : NULL, NULL);
+
+ for (size_t k = 0; k < vec->nr_items; k++) {
+ struct large_item *pk = &vec->items[k];
+ if (!is_null_oid(&pk->oid)) {
+ strbuf_reset(&size);
+ strbuf_addf(&size, "%"PRIuMAX, (uintmax_t)pk->size);
+
+ insert_table_rowv(&table, size.buf, oid_to_hex(&pk->oid), pk->name.buf,
+ is_null_oid(&pk->containing_commit_oid) ?
+ "" : oid_to_hex(&pk->containing_commit_oid),
+ !ctx->opts.show_name_rev ? NULL : pk->name_rev.len ? pk->name_rev.buf : "",
+ NULL);
+ }
+ }
+ strbuf_release(&size);
+
+ print_table_plaintext(&table);
+ clear_table(&table);
+}
+
static void survey_report_plaintext_refs(struct survey_context *ctx)
{
struct survey_report_ref_summary *refs = &ctx->report.refs;
@@ -380,6 +829,42 @@ static void survey_report_plaintext_refs(struct survey_context *ctx)
free(fmt);
}
+ /*
+ * SymRefs are somewhat orthogonal to the above classification (e.g.
+ * "HEAD" --> detached and "refs/remotes/origin/HEAD" --> remote) so the
+ * above classified counts will already include them, but it is less
+ * confusing to display them here than to create a whole new section.
+ */
+ if (ctx->report.refs.cnt_symref) {
+ char *fmt = xstrfmt("%"PRIuMAX"", (uintmax_t)refs->cnt_symref);
+ insert_table_rowv(&table, _("Symbolic refs"), fmt, NULL);
+ free(fmt);
+ }
+
+ if (ctx->report.refs.cnt_loose || ctx->report.refs.cnt_packed) {
+ char *fmt = xstrfmt("%"PRIuMAX"", (uintmax_t)refs->cnt_loose);
+ insert_table_rowv(&table, _("Loose refs"), fmt, NULL);
+ free(fmt);
+ fmt = xstrfmt("%"PRIuMAX"", (uintmax_t)refs->cnt_packed);
+ insert_table_rowv(&table, _("Packed refs"), fmt, NULL);
+ free(fmt);
+ }
+
+ if (ctx->report.refs.len_max_local_refname || ctx->report.refs.len_max_remote_refname) {
+ char *fmt = xstrfmt("%"PRIuMAX"", (uintmax_t)refs->len_max_local_refname);
+ insert_table_rowv(&table, _("Max local refname length"), fmt, NULL);
+ free(fmt);
+ fmt = xstrfmt("%"PRIuMAX"", (uintmax_t)refs->len_sum_local_refnames);
+ insert_table_rowv(&table, _("Sum local refnames length"), fmt, NULL);
+ free(fmt);
+ fmt = xstrfmt("%"PRIuMAX"", (uintmax_t)refs->len_max_remote_refname);
+ insert_table_rowv(&table, _("Max remote refname length"), fmt, NULL);
+ free(fmt);
+ fmt = xstrfmt("%"PRIuMAX"", (uintmax_t)refs->len_sum_remote_refnames);
+ insert_table_rowv(&table, _("Sum remote refnames length"), fmt, NULL);
+ free(fmt);
+ }
+
print_table_plaintext(&table);
clear_table(&table);
}
@@ -465,6 +950,19 @@ static void survey_report_plaintext(struct survey_context *ctx)
ctx->report.by_type,
REPORT_TYPE_COUNT);
+ survey_report_commit_parents(ctx);
+
+ survey_report_hbin(_("COMMITS HISTOGRAM BY SIZE IN BYTES"),
+ ctx->report.reachable_objects.commits.base.size_hbin);
+
+ survey_report_tree_lengths(ctx);
+
+ survey_report_hbin(_("TREES HISTOGRAM BY SIZE IN BYTES"),
+ ctx->report.reachable_objects.trees.base.size_hbin);
+
+ survey_report_hbin(_("BLOBS HISTOGRAM BY SIZE IN BYTES"),
+ ctx->report.reachable_objects.blobs.base.size_hbin);
+
survey_report_plaintext_sorted_size(
&ctx->report.top_paths_by_count[REPORT_TYPE_TREE]);
survey_report_plaintext_sorted_size(
@@ -479,6 +977,12 @@ static void survey_report_plaintext(struct survey_context *ctx)
&ctx->report.top_paths_by_inflate[REPORT_TYPE_TREE]);
survey_report_plaintext_sorted_size(
&ctx->report.top_paths_by_inflate[REPORT_TYPE_BLOB]);
+
+ survey_report_largest_vec(ctx, ctx->report.reachable_objects.commits.vec_largest_by_nr_parents);
+ survey_report_largest_vec(ctx, ctx->report.reachable_objects.commits.vec_largest_by_size_bytes);
+ survey_report_largest_vec(ctx, ctx->report.reachable_objects.trees.vec_largest_by_nr_entries);
+ survey_report_largest_vec(ctx, ctx->report.reachable_objects.trees.vec_largest_by_size_bytes);
+ survey_report_largest_vec(ctx, ctx->report.reachable_objects.blobs.vec_largest_by_size_bytes);
}
/*
@@ -550,6 +1054,31 @@ static int survey_load_config_cb(const char *var, const char *value,
ctx->opts.show_progress = git_config_bool(var, value);
return 0;
}
+ if (!strcmp(var, "survey.namerev")) {
+ ctx->opts.show_name_rev = git_config_bool(var, value);
+ return 0;
+ }
+ if (!strcmp(var, "survey.showcommitparents")) {
+ ctx->opts.show_largest_commits_by_nr_parents = git_config_ulong(var, value, cctx->kvi);
+ return 0;
+ }
+ if (!strcmp(var, "survey.showcommitsizes")) {
+ ctx->opts.show_largest_commits_by_size_bytes = git_config_ulong(var, value, cctx->kvi);
+ return 0;
+ }
+
+ if (!strcmp(var, "survey.showtreeentries")) {
+ ctx->opts.show_largest_trees_by_nr_entries = git_config_ulong(var, value, cctx->kvi);
+ return 0;
+ }
+ if (!strcmp(var, "survey.showtreesizes")) {
+ ctx->opts.show_largest_trees_by_size_bytes = git_config_ulong(var, value, cctx->kvi);
+ return 0;
+ }
+ if (!strcmp(var, "survey.showblobsizes")) {
+ ctx->opts.show_largest_blobs_by_size_bytes = git_config_ulong(var, value, cctx->kvi);
+ return 0;
+ }
if (!strcmp(var, "survey.top")) {
ctx->opts.top_nr = git_config_bool(var, value);
return 0;
@@ -614,6 +1143,80 @@ static void do_load_refs(struct survey_context *ctx,
ref_sorting_release(sorting);
}
+/*
+ * Try to run `git name-rev` on each of the containing-commit-oid's
+ * in this large-item-vec to get a pretty name for each OID. Silently
+ * ignore errors if it fails because this info is nice to have but not
+ * essential.
+ */
+static void large_item_vec_lookup_name_rev(struct survey_context *ctx,
+ struct large_item_vec *vec)
+{
+ struct child_process cp = CHILD_PROCESS_INIT;
+ struct strbuf in = STRBUF_INIT;
+ struct strbuf out = STRBUF_INIT;
+ const char *line;
+ size_t k;
+
+ if (!vec || !vec->nr_items)
+ return;
+
+ ctx->progress_total += vec->nr_items;
+ display_progress(ctx->progress, ctx->progress_total);
+
+ for (k = 0; k < vec->nr_items; k++)
+ strbuf_addf(&in, "%s\n", oid_to_hex(&vec->items[k].containing_commit_oid));
+
+ cp.git_cmd = 1;
+ strvec_pushl(&cp.args, "name-rev", "--name-only", "--annotate-stdin", NULL);
+ if (pipe_command(&cp, in.buf, in.len, &out, 0, NULL, 0)) {
+ strbuf_release(&in);
+ strbuf_release(&out);
+ return;
+ }
+
+ line = out.buf;
+ k = 0;
+ while (*line) {
+ const char *eol = strchrnul(line, '\n');
+
+ strbuf_init(&vec->items[k].name_rev, 0);
+ strbuf_add(&vec->items[k].name_rev, line, (eol - line));
+
+ line = eol + 1;
+ k++;
+ }
+
+ strbuf_release(&in);
+ strbuf_release(&out);
+}
+
+static void do_lookup_name_rev(struct survey_context *ctx)
+{
+ /*
+ * `git name-rev` can be very expensive when there are lots of
+ * refs, so make it optional.
+ */
+ if (!ctx->opts.show_name_rev)
+ return;
+
+ if (ctx->opts.show_progress) {
+ ctx->progress_total = 0;
+ ctx->progress = start_progress(_("Resolving name-revs..."), 0);
+ }
+
+ large_item_vec_lookup_name_rev(ctx, ctx->report.reachable_objects.commits.vec_largest_by_nr_parents);
+ large_item_vec_lookup_name_rev(ctx, ctx->report.reachable_objects.commits.vec_largest_by_size_bytes);
+
+ large_item_vec_lookup_name_rev(ctx, ctx->report.reachable_objects.trees.vec_largest_by_nr_entries);
+ large_item_vec_lookup_name_rev(ctx, ctx->report.reachable_objects.trees.vec_largest_by_size_bytes);
+
+ large_item_vec_lookup_name_rev(ctx, ctx->report.reachable_objects.blobs.vec_largest_by_size_bytes);
+
+ if (ctx->opts.show_progress)
+ stop_progress(&ctx->progress);
+}
+
/*
* The REFS phase:
*
@@ -637,6 +1240,7 @@ static void survey_phase_refs(struct survey_context *ctx)
for (int i = 0; i < ctx->ref_array.nr; i++) {
unsigned long size;
struct ref_array_item *item = ctx->ref_array.items[i];
+ size_t len = strlen(item->refname);
switch (item->kind) {
case FILTER_REFS_TAGS:
@@ -663,6 +1267,33 @@ static void survey_phase_refs(struct survey_context *ctx)
ctx->report.refs.unknown_nr++;
break;
}
+
+ /*
+ * SymRefs are somewhat orthogonal to the above
+ * classification (e.g. "HEAD" --> detached
+ * and "refs/remotes/origin/HEAD" --> remote) so
+ * our totals will already include them.
+ */
+ if (item->flag & REF_ISSYMREF)
+ ctx->report.refs.cnt_symref++;
+
+ /*
+ * Where/how is the ref stored in GITDIR.
+ */
+ if (item->flag & REF_ISPACKED)
+ ctx->report.refs.cnt_packed++;
+ else
+ ctx->report.refs.cnt_loose++;
+
+ if (item->kind == FILTER_REFS_REMOTES) {
+ ctx->report.refs.len_sum_remote_refnames += len;
+ if (len > ctx->report.refs.len_max_remote_refname)
+ ctx->report.refs.len_max_remote_refname = len;
+ } else {
+ ctx->report.refs.len_sum_local_refnames += len;
+ if (len > ctx->report.refs.len_max_local_refname)
+ ctx->report.refs.len_max_local_refname = len;
+ }
}
trace2_region_leave("survey", "phase/refs", ctx->repo);
@@ -697,7 +1328,8 @@ static void increment_object_counts(
static void increment_totals(struct survey_context *ctx,
struct oid_array *oids,
- struct survey_report_object_size_summary *summary)
+ struct survey_report_object_size_summary *summary,
+ const char *path)
{
for (size_t i = 0; i < oids->nr; i++) {
struct object_info oi = OBJECT_INFO_INIT;
@@ -705,6 +1337,8 @@ static void increment_totals(struct survey_context *ctx,
unsigned long object_length = 0;
off_t disk_sizep = 0;
enum object_type type;
+ struct survey_stats_base_object *base;
+ int hb;
oi.typep = &type;
oi.sizep = &object_length;
@@ -713,11 +1347,86 @@ static void increment_totals(struct survey_context *ctx,
if (oid_object_info_extended(ctx->repo, &oids->oid[i],
&oi, oi_flags) < 0) {
summary->num_missing++;
- } else {
- summary->nr++;
- summary->disk_size += disk_sizep;
- summary->inflated_size += object_length;
+ continue;
+ }
+
+ summary->nr++;
+ summary->disk_size += disk_sizep;
+ summary->inflated_size += object_length;
+
+ switch (type) {
+ case OBJ_COMMIT: {
+ struct commit *commit = lookup_commit(ctx->repo, &oids->oid[i]);
+ unsigned k = commit_list_count(commit->parents);
+
+ if (k >= PBIN_VEC_LEN)
+ k = PBIN_VEC_LEN - 1;
+
+ ctx->report.reachable_objects.commits.parent_cnt_pbin[k]++;
+ base = &ctx->report.reachable_objects.commits.base;
+
+ maybe_insert_large_item(ctx->report.reachable_objects.commits.vec_largest_by_nr_parents, k, &commit->object.oid, NULL, &commit->object.oid);
+ maybe_insert_large_item(ctx->report.reachable_objects.commits.vec_largest_by_size_bytes, object_length, &commit->object.oid, NULL, &commit->object.oid);
+ break;
+ }
+ case OBJ_TREE: {
+ struct tree *tree = lookup_tree(ctx->repo, &oids->oid[i]);
+ if (tree) {
+ struct survey_stats_trees *pst = &ctx->report.reachable_objects.trees;
+ struct tree_desc desc;
+ struct name_entry entry;
+ int nr_entries;
+ int qb;
+
+ parse_tree(tree);
+ init_tree_desc(&desc, &oids->oid[i], tree->buffer, tree->size);
+ nr_entries = 0;
+ while (tree_entry(&desc, &entry))
+ nr_entries++;
+
+ pst->sum_entries += nr_entries;
+
+ maybe_insert_large_item(pst->vec_largest_by_nr_entries, nr_entries, &tree->object.oid, path, NULL);
+ maybe_insert_large_item(pst->vec_largest_by_size_bytes, object_length, &tree->object.oid, path, NULL);
+
+ qb = qbin(nr_entries);
+ incr_obj_hist_bin(&pst->entry_qbin[qb], object_length, disk_sizep);
+ }
+ base = &ctx->report.reachable_objects.trees.base;
+ break;
+ }
+ case OBJ_BLOB:
+ base = &ctx->report.reachable_objects.blobs.base;
+
+ maybe_insert_large_item(ctx->report.reachable_objects.blobs.vec_largest_by_size_bytes, object_length, &oids->oid[i], path, NULL);
+ break;
+ default:
+ continue;
+ }
+
+ switch (oi.whence) {
+ case OI_CACHED:
+ base->cnt_cached++;
+ break;
+ case OI_LOOSE:
+ base->cnt_loose++;
+ break;
+ case OI_PACKED:
+ base->cnt_packed++;
+ break;
+ case OI_DBCACHED:
+ base->cnt_dbcached++;
+ break;
+ default:
+ break;
}
+
+ base->sum_size += object_length;
+ base->sum_disk_size += disk_sizep;
+
+ hb = hbin(object_length);
+ incr_obj_hist_bin(&base->size_hbin[hb], object_length, disk_sizep);
+
}
}
@@ -729,7 +1438,7 @@ static void increment_object_totals(struct survey_context *ctx,
struct survey_report_object_size_summary *total;
struct survey_report_object_size_summary summary = { 0 };
- increment_totals(ctx, oids, &summary);
+ increment_totals(ctx, oids, &summary, path);
switch (type) {
case OBJ_COMMIT:
@@ -861,6 +1570,12 @@ static void survey_phase_objects(struct survey_context *ctx)
release_revisions(&revs);
trace2_region_leave("survey", "phase/objects", ctx->repo);
+
+ if (ctx->opts.show_name_rev) {
+ trace2_region_enter("survey", "phase/namerev", the_repository);
+ do_lookup_name_rev(ctx);
+ trace2_region_enter("survey", "phase/namerev", the_repository);
+ }
}
int cmd_survey(int argc, const char **argv, const char *prefix, struct repository *repo)
@@ -885,6 +1600,7 @@ int cmd_survey(int argc, const char **argv, const char *prefix, struct repositor
static struct option survey_options[] = {
OPT__VERBOSE(&ctx.opts.verbose, N_("verbose output")),
OPT_BOOL(0, "progress", &ctx.opts.show_progress, N_("show progress")),
+ OPT_BOOL(0, "name-rev", &ctx.opts.show_name_rev, N_("run name-rev on each reported commit")),
OPT_INTEGER('n', "top", &ctx.opts.top_nr,
N_("number of entries to include in detail tables")),
@@ -896,6 +1612,14 @@ int cmd_survey(int argc, const char **argv, const char *prefix, struct repositor
OPT_BOOL_F(0, "detached", &ctx.opts.refs.want_detached, N_("include detached HEAD"), PARSE_OPT_NONEG),
OPT_BOOL_F(0, "other", &ctx.opts.refs.want_other, N_("include notes and stashes"), PARSE_OPT_NONEG),
+ OPT_INTEGER_F(0, "commit-parents", &ctx.opts.show_largest_commits_by_nr_parents, N_("show N largest commits by parent count"), PARSE_OPT_NONEG),
+ OPT_INTEGER_F(0, "commit-sizes", &ctx.opts.show_largest_commits_by_size_bytes, N_("show N largest commits by size in bytes"), PARSE_OPT_NONEG),
+
+ OPT_INTEGER_F(0, "tree-entries", &ctx.opts.show_largest_trees_by_nr_entries, N_("show N largest trees by entry count"), PARSE_OPT_NONEG),
+ OPT_INTEGER_F(0, "tree-sizes", &ctx.opts.show_largest_trees_by_size_bytes, N_("show N largest trees by size in bytes"), PARSE_OPT_NONEG),
+
+ OPT_INTEGER_F(0, "blob-sizes", &ctx.opts.show_largest_blobs_by_size_bytes, N_("show N largest blobs by size in bytes"), PARSE_OPT_NONEG),
+
OPT_END(),
};
@@ -919,6 +1643,39 @@ int cmd_survey(int argc, const char **argv, const char *prefix, struct repositor
fixup_refs_wanted(&ctx);
+ if (ctx.opts.show_largest_commits_by_nr_parents)
+ ctx.report.reachable_objects.commits.vec_largest_by_nr_parents =
+ alloc_large_item_vec(
+ "largest_commits_by_nr_parents",
+ "nr_parents",
+ ctx.opts.show_largest_commits_by_nr_parents);
+ if (ctx.opts.show_largest_commits_by_size_bytes)
+ ctx.report.reachable_objects.commits.vec_largest_by_size_bytes =
+ alloc_large_item_vec(
+ "largest_commits_by_size_bytes",
+ "size",
+ ctx.opts.show_largest_commits_by_size_bytes);
+
+ if (ctx.opts.show_largest_trees_by_nr_entries)
+ ctx.report.reachable_objects.trees.vec_largest_by_nr_entries =
+ alloc_large_item_vec(
+ "largest_trees_by_nr_entries",
+ "nr_entries",
+ ctx.opts.show_largest_trees_by_nr_entries);
+ if (ctx.opts.show_largest_trees_by_size_bytes)
+ ctx.report.reachable_objects.trees.vec_largest_by_size_bytes =
+ alloc_large_item_vec(
+ "largest_trees_by_size_bytes",
+ "size",
+ ctx.opts.show_largest_trees_by_size_bytes);
+
+ if (ctx.opts.show_largest_blobs_by_size_bytes)
+ ctx.report.reachable_objects.blobs.vec_largest_by_size_bytes =
+ alloc_large_item_vec(
+ "largest_blobs_by_size_bytes",
+ "size",
+ ctx.opts.show_largest_blobs_by_size_bytes);
+
survey_phase_refs(&ctx);
survey_phase_objects(&ctx);
@@ -928,3 +1685,143 @@ int cmd_survey(int argc, const char **argv, const char *prefix, struct repositor
clear_survey_context(&ctx);
return 0;
}
+
+/*
+ * NEEDSWORK: So far, I only have iteration on the requested set of
+ * refs and treewalk/reachable objects on that set of refs. The
+ * following is a bit of a laundry list of things that I'd like to
+ * add.
+ *
+ * [] Dump stats on all of the packfiles. The number and size of each.
+ * Whether each is in the .git directory or in an alternate. The
+ * state of the IDX or MIDX files and etc. Delta chain stats. All
+ * of this data is relative to the "lived-in" state of the
+ * repository. Stuff that may change after a GC or repack.
+ *
+ * [] Clone and Index stats. partial, shallow, sparse-checkout,
+ * sparse-index, etc. Hydration stats.
+ *
+ * [] Dump stats on each remote. When we fetch from a remote the size
+ * of the response is related to the set of haves on the server.
+ * You can see this in `GIT_TRACE_CURL=1 git fetch`. We get a
+ * `ls-refs` payload that lists all of the branches and tags on the
+ * server, so at a minimum the RefName and SHA for each. But for
+ * annotated tags we also get the peeled SHA. The size of this
+ * overhead on every fetch is proporational to the size of the `git
+ * ls-remote` response (roughly, although the latter repeats the
+ * RefName of the peeled tag). If, for example, you have 500K refs
+ * on a remote, you're going to have a long "haves" message, so
+ * every fetch will be slow just because of that overhead (not
+ * counting new objects to be downloaded).
+ *
+ * Note that the local set of tags in "refs/tags/" is a union over
+ * all remotes. However, since most people only have one remote,
+ * we can probaly estimate the overhead value directly from the
+ * size of the set of "refs/tags/" that we visited while building
+ * the `ref_info` and `ref_array` and not need to ask the remote.
+ *
+ * [] Should the "string length of refnames / remote refs", for
+ * example, be sub-divided by remote so we can project the
+ * cost of the haves/wants overhead a fetch.
+ *
+ * [] Can we examine the merge commits and classify them as clean or
+ * dirty? (ie. ones with merge conflicts that needed to be
+ * addressed during the merge itself.)
+ *
+ * [] Do dirty merges affect performance of later operations?
+ *
+ * [] Dump info on the complexity of the DAG. Criss-cross merges.
+ * The number of edges that must be touched to compute merge bases.
+ * Edge length. The number of parallel lanes in the history that
+ * must be navigated to get to the merge base. What affects the
+ * cost of the Ahead/Behind computation? How often do
+ * criss-crosses occur and do they cause various operations to slow
+ * down?
+ *
+ * [] If there are primary branches (like "main" or "master") are they
+ * always on the left side of merges? Does the graph have a clean
+ * left edge? Or are there normal and "backwards" merges? Do
+ * these cause problems at scale?
+ *
+ * [] If we have a hierarchy of FI/RI branches like "L1", "L2, ...,
+ * can we learn anything about the shape of the repo around these
+ * FI and RI integrations?
+ *
+ * [] Do we need a no-PII flag to omit pathnames or branch/tag names
+ * in the various histograms? (This would turn off --name-rev
+ * too.)
+ *
+ * [] I have so far avoided adding opinions about individual fields
+ * (such as the way `git-sizer` prints a row of stars or bangs in
+ * the last column).
+ *
+ * I'm wondering if that is a job of this executable or if it
+ * should be done in a post-processing step using the JSON output.
+ *
+ * My problem with the `git-sizer` approach is that it doesn't give
+ * the (casual) user any information on why it has stars or bangs.
+ * And there isn't a good way to print detailed information in the
+ * ASCII-art tables that would be easy to understand.
+ *
+ * [] For example, a large number of refs does not define a cliff.
+ * Performance will drop off (linearly, quadratically, ... ??).
+ * The tool should refer them to article(s) talking about the
+ * different problems that it could cause. So should `git
+ * survey` just print the number and (implicitly) refer them to
+ * the man page (chapter/verse) or to a tool that will interpret
+ * the number and explain it?
+ *
+ * [] Alternatively, should `git survey` do that analysis too and
+ * just print footnotes for each large number?
+ *
+ * [] The computation of the raw survey JSON data can take HOURS on
+ * a very large repo (like Windows), so I'm wondering if we
+ * want to keep the opinion portion separate.
+ *
+ * [] In addition to opinions based on the static data, I would like
+ * to dump the JSON results (or the Trace2 telemetry) into a DB and
+ * aggregate it with other users.
+ *
+ * Granted, they should all see the same DAG and the same set of
+ * reachable objects, but we could average across all datasets
+ * generated on a particular date and detect outlier users.
+ *
+ * [] Maybe someone cloned from the `_full` endpoint rather than
+ * the limited refs endpoint.
+ *
+ * [] Maybe that user is having problems with repacking / GC /
+ * maintenance without knowing it.
+ *
+ * [] I'd also like to dump use the DB to compare survey datasets over
+ * a time. How fast is their repository growing and in what ways?
+ *
+ * [] I'd rather have the delta analysis NOT be inside `git
+ * survey`, so it makes sense to consider having all of it in a
+ * post-process step.
+ *
+ * [] Another reason to put the opinion analysis in a post-process
+ * is that it would be easier to generate plots on the data tables.
+ * Granted, we can get plots from telemetry, but a stand-alone user
+ * could run the JSON thru python or jq or something and generate
+ * something nicer than ASCII-art and it could handle cross-referencing
+ * and hyperlinking to helpful information on each issue.
+ *
+ * [] I think there are several classes of data that we can report on:
+ *
+ * [] The "inherit repo properties", such as the shape and size of
+ * the DAG -- these should be universal in each enlistment.
+ *
+ * [] The "ODB lived in properties", such as the efficiency
+ * of the repack and things like partial and shallow clone.
+ * These will vary, but indicate health of the ODB.
+ *
+ * [] The "index related properties", such as sparse-checkout,
+ * sparse-index, cache-tree, untracked-cache, fsmonitor, and
+ * etc. These will also vary, but are more like knobs for
+ * the user to adjust.
+ *
+ * [] I want to compare these with Matt's "dimensions of scale"
+ * notes and see if there are other pieces of data that we
+ * could compute/consider.
+ *
+ */
diff --git a/builtin/update-index.c b/builtin/update-index.c
index 74bbad9f87d86d..073c55e280296d 100644
--- a/builtin/update-index.c
+++ b/builtin/update-index.c
@@ -8,6 +8,7 @@
#define DISABLE_SIGN_COMPARE_WARNINGS
#include "builtin.h"
+#include "gvfs.h"
#include "bulk-checkin.h"
#include "config.h"
#include "environment.h"
@@ -713,7 +714,9 @@ static int do_reupdate(const char **paths,
* to process each path individually
*/
if (S_ISSPARSEDIR(ce->ce_mode)) {
- ensure_full_index(the_repository->index);
+ const char *fmt = "update-index:modified sparse dir '%s'";
+ ensure_full_index_with_reason(the_repository->index,
+ fmt, ce->name);
goto redo;
}
@@ -1115,7 +1118,13 @@ int cmd_update_index(int argc,
argc = parse_options_end(&ctx);
getline_fn = nul_term_line ? strbuf_getline_nul : strbuf_getline_lf;
+ if (mark_skip_worktree_only && gvfs_config_is_set(GVFS_BLOCK_COMMANDS))
+ die(_("modifying the skip worktree bit is not supported on a GVFS repo"));
+
if (preferred_index_format) {
+ if (preferred_index_format != 4 && gvfs_config_is_set(GVFS_BLOCK_COMMANDS))
+ die(_("changing the index version is not supported on a GVFS repo"));
+
if (preferred_index_format < 0) {
printf(_("%d\n"), the_repository->index->version);
} else if (preferred_index_format < INDEX_FORMAT_LB ||
@@ -1161,6 +1170,9 @@ int cmd_update_index(int argc,
end_odb_transaction();
if (split_index > 0) {
+ if (gvfs_config_is_set(GVFS_BLOCK_COMMANDS))
+ die(_("split index is not supported on a GVFS repo"));
+
if (repo_config_get_split_index(the_repository) == 0)
warning(_("core.splitIndex is set to false; "
"remove or change it, if you really want to "
diff --git a/builtin/update-microsoft-git.c b/builtin/update-microsoft-git.c
new file mode 100644
index 00000000000000..54e196b70116f2
--- /dev/null
+++ b/builtin/update-microsoft-git.c
@@ -0,0 +1,69 @@
+#include "builtin.h"
+#include "repository.h"
+#include "parse-options.h"
+#include "run-command.h"
+#include "strvec.h"
+
+#if defined(GIT_WINDOWS_NATIVE)
+/*
+ * On Windows, run 'git update-git-for-windows' which
+ * is installed by the installer, based on the script
+ * in git-for-windows/build-extra.
+ */
+static int platform_specific_upgrade(void)
+{
+ struct child_process cp = CHILD_PROCESS_INIT;
+
+ strvec_push(&cp.args, "git-update-git-for-windows");
+ return run_command(&cp);
+}
+#elif defined(__APPLE__)
+/*
+ * On macOS, we expect the user to have the microsoft-git
+ * cask installed via Homebrew. We check using these
+ * commands:
+ *
+ * 1. 'brew update' to get latest versions.
+ * 2. 'brew upgrade --cask microsoft-git' to get the
+ * latest version.
+ */
+static int platform_specific_upgrade(void)
+{
+ int res;
+ struct child_process update = CHILD_PROCESS_INIT;
+ struct child_process upgrade = CHILD_PROCESS_INIT;
+
+ printf("Updating Homebrew with 'brew update'\n");
+
+ strvec_pushl(&update.args, "brew", "update", NULL);
+ res = run_command(&update);
+
+ if (res) {
+ error(_("'brew update' failed; is brew installed?"));
+ return 1;
+ }
+
+ printf("Upgrading microsoft-git with 'brew upgrade --cask microsoft-git'\n");
+ strvec_pushl(&upgrade.args, "brew", "upgrade", "--cask", "microsoft-git", NULL);
+ res = run_command(&upgrade);
+
+ return res;
+}
+#else
+static int platform_specific_upgrade(void)
+{
+ error(_("update-microsoft-git is not supported on this platform"));
+ return 1;
+}
+#endif
+
+static const char builtin_update_microsoft_git_usage[] =
+ N_("git update-microsoft-git");
+
+int cmd_update_microsoft_git(int argc, const char **argv, const char *prefix UNUSED, struct repository *repo UNUSED)
+{
+ if (argc == 2 && !strcmp(argv[1], "-h"))
+ usage(builtin_update_microsoft_git_usage);
+
+ return platform_specific_upgrade();
+}
diff --git a/builtin/worktree.c b/builtin/worktree.c
index c043d4d523f578..900e7101dbe43f 100644
--- a/builtin/worktree.c
+++ b/builtin/worktree.c
@@ -4,6 +4,7 @@
#include "builtin.h"
#include "abspath.h"
#include "advice.h"
+#include "gvfs.h"
#include "checkout.h"
#include "config.h"
#include "copy.h"
@@ -1429,6 +1430,13 @@ int cmd_worktree(int ac,
git_config(git_worktree_config, NULL);
+ /*
+ * git-worktree is special-cased to work in Scalar repositories
+ * even when they use the GVFS Protocol.
+ */
+ if (core_gvfs & GVFS_USE_VIRTUAL_FILESYSTEM)
+ die("'git %s' is not supported on a GVFS repo", "worktree");
+
if (!prefix)
prefix = "";
diff --git a/cache-tree.c b/cache-tree.c
index bcbcad3d61a09c..03d23737523c52 100644
--- a/cache-tree.c
+++ b/cache-tree.c
@@ -4,6 +4,7 @@
#include "git-compat-util.h"
#include "gettext.h"
#include "hex.h"
+#include "gvfs.h"
#include "lockfile.h"
#include "tree.h"
#include "tree-walk.h"
@@ -233,7 +234,7 @@ static void discard_unused_subtrees(struct cache_tree *it)
}
}
-int cache_tree_fully_valid(struct cache_tree *it)
+static int cache_tree_fully_valid_1(struct cache_tree *it)
{
int i;
if (!it)
@@ -241,7 +242,7 @@ int cache_tree_fully_valid(struct cache_tree *it)
if (it->entry_count < 0 || !repo_has_object_file(the_repository, &it->oid))
return 0;
for (i = 0; i < it->subtree_nr; i++) {
- if (!cache_tree_fully_valid(it->down[i]->cache_tree))
+ if (!cache_tree_fully_valid_1(it->down[i]->cache_tree))
return 0;
}
return 1;
@@ -252,6 +253,17 @@ static int must_check_existence(const struct cache_entry *ce)
return !(repo_has_promisor_remote(the_repository) && ce_skip_worktree(ce));
}
+int cache_tree_fully_valid(struct cache_tree *it)
+{
+ int result;
+
+ trace2_region_enter("cache_tree", "fully_valid", NULL);
+ result = cache_tree_fully_valid_1(it);
+ trace2_region_leave("cache_tree", "fully_valid", NULL);
+
+ return result;
+}
+
static int update_one(struct cache_tree *it,
struct cache_entry **cache,
int entries,
@@ -261,7 +273,8 @@ static int update_one(struct cache_tree *it,
int flags)
{
struct strbuf buffer;
- int missing_ok = flags & WRITE_TREE_MISSING_OK;
+ int missing_ok = gvfs_config_is_set(GVFS_MISSING_OK) ?
+ WRITE_TREE_MISSING_OK : (flags & WRITE_TREE_MISSING_OK);
int dryrun = flags & WRITE_TREE_DRY_RUN;
int repair = flags & WRITE_TREE_REPAIR;
int to_invalidate = 0;
@@ -430,7 +443,29 @@ static int update_one(struct cache_tree *it,
continue;
strbuf_grow(&buffer, entlen + 100);
- strbuf_addf(&buffer, "%o %.*s%c", mode, entlen, path + baselen, '\0');
+
+ switch (mode) {
+ case 0100644:
+ strbuf_add(&buffer, "100644 ", 7);
+ break;
+ case 0100664:
+ strbuf_add(&buffer, "100664 ", 7);
+ break;
+ case 0100755:
+ strbuf_add(&buffer, "100755 ", 7);
+ break;
+ case 0120000:
+ strbuf_add(&buffer, "120000 ", 7);
+ break;
+ case 0160000:
+ strbuf_add(&buffer, "160000 ", 7);
+ break;
+ default:
+ strbuf_addf(&buffer, "%o ", mode);
+ break;
+ }
+ strbuf_add(&buffer, path + baselen, entlen);
+ strbuf_addch(&buffer, '\0');
strbuf_add(&buffer, oid->hash, the_hash_algo->rawsz);
#if DEBUG_CACHE_TREE
diff --git a/commit.c b/commit.c
index a127fe60c5e83c..e5dbffae13a794 100644
--- a/commit.c
+++ b/commit.c
@@ -1,6 +1,7 @@
#define USE_THE_REPOSITORY_VARIABLE
#include "git-compat-util.h"
+#include "gvfs.h"
#include "tag.h"
#include "commit.h"
#include "commit-graph.h"
@@ -556,13 +557,17 @@ int repo_parse_commit_internal(struct repository *r,
.sizep = &size,
.contentp = &buffer,
};
+ int ret;
/*
* Git does not support partial clones that exclude commits, so set
* OBJECT_INFO_SKIP_FETCH_OBJECT to fail fast when an object is missing.
*/
int flags = OBJECT_INFO_LOOKUP_REPLACE | OBJECT_INFO_SKIP_FETCH_OBJECT |
- OBJECT_INFO_DIE_IF_CORRUPT;
- int ret;
+ OBJECT_INFO_DIE_IF_CORRUPT;
+
+ /* But the GVFS Protocol _does_ support missing commits! */
+ if (gvfs_config_is_set(GVFS_MISSING_OK))
+ flags ^= OBJECT_INFO_SKIP_FETCH_OBJECT;
if (!item)
return -1;
diff --git a/compat/mingw.c b/compat/mingw.c
index b1bd925c701ed0..1e46fa2ae8f57b 100644
--- a/compat/mingw.c
+++ b/compat/mingw.c
@@ -3718,31 +3718,44 @@ static void setup_windows_environment(void)
has_symlinks = 0;
}
-static PSID get_current_user_sid(void)
+static void get_current_user_sid(PSID *sid, HANDLE *linked_token)
{
HANDLE token;
DWORD len = 0;
- PSID result = NULL;
+ TOKEN_ELEVATION_TYPE elevationType;
+ DWORD size;
+
+ *sid = NULL;
+ *linked_token = NULL;
if (!OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, &token))
- return NULL;
+ return;
if (!GetTokenInformation(token, TokenUser, NULL, 0, &len)) {
TOKEN_USER *info = xmalloc((size_t)len);
if (GetTokenInformation(token, TokenUser, info, len, &len)) {
len = GetLengthSid(info->User.Sid);
- result = xmalloc(len);
- if (!CopySid(len, result, info->User.Sid)) {
+ *sid = xmalloc(len);
+ if (!CopySid(len, *sid, info->User.Sid)) {
error(_("failed to copy SID (%ld)"),
GetLastError());
- FREE_AND_NULL(result);
+ FREE_AND_NULL(*sid);
}
}
FREE_AND_NULL(info);
}
- CloseHandle(token);
- return result;
+ if (GetTokenInformation(token, TokenElevationType, &elevationType, sizeof(elevationType), &size) &&
+ elevationType == TokenElevationTypeLimited) {
+ /*
+ * The current process is run by a member of the Administrators
+ * group, but is not running elevated.
+ */
+ if (!GetTokenInformation(token, TokenLinkedToken, linked_token, sizeof(*linked_token), &size))
+ linked_token = NULL; /* there is no linked token */
+ }
+
+ CloseHandle(token);
}
static BOOL user_sid_to_user_name(PSID sid, LPSTR *str)
@@ -3821,18 +3834,22 @@ int is_path_owned_by_current_sid(const char *path, struct strbuf *report)
if (err == ERROR_SUCCESS && sid && IsValidSid(sid)) {
/* Now, verify that the SID matches the current user's */
static PSID current_user_sid;
+ static HANDLE linked_token;
BOOL is_member;
if (!current_user_sid)
- current_user_sid = get_current_user_sid();
+ get_current_user_sid(¤t_user_sid, &linked_token);
if (current_user_sid &&
IsValidSid(current_user_sid) &&
EqualSid(sid, current_user_sid))
result = 1;
else if (IsWellKnownSid(sid, WinBuiltinAdministratorsSid) &&
- CheckTokenMembership(NULL, sid, &is_member) &&
- is_member)
+ ((CheckTokenMembership(NULL, sid, &is_member) &&
+ is_member) ||
+ (linked_token &&
+ CheckTokenMembership(linked_token, sid, &is_member) &&
+ is_member)))
/*
* If owned by the Administrators group, and the
* current user is an administrator, we consider that
@@ -4238,6 +4255,8 @@ int wmain(int argc, const wchar_t **wargv)
SetConsoleCtrlHandler(handle_ctrl_c, TRUE);
+ trace2_initialize_clock();
+
maybe_redirect_std_handles();
adjust_symlink_flags();
fsync_object_files = 1;
diff --git a/config.c b/config.c
index 95b7de0522d180..8b5df6a05652ed 100644
--- a/config.c
+++ b/config.c
@@ -13,6 +13,7 @@
#include "abspath.h"
#include "advice.h"
#include "date.h"
+#include "gvfs.h"
#include "branch.h"
#include "config.h"
#include "parse.h"
@@ -42,6 +43,7 @@
#include "wildmatch.h"
#include "ws.h"
#include "write-or-die.h"
+#include "transport.h"
struct config_source {
struct config_source *prev;
@@ -1621,8 +1623,22 @@ int git_default_core_config(const char *var, const char *value,
return 0;
}
+ if (!strcmp(var, "core.gvfs")) {
+ gvfs_load_config_value(value);
+ return 0;
+ }
+
+ if (!strcmp(var, "core.usegvfshelper")) {
+ core_use_gvfs_helper = git_config_bool(var, value);
+ return 0;
+ }
+
if (!strcmp(var, "core.sparsecheckout")) {
- core_apply_sparse_checkout = git_config_bool(var, value);
+ /* virtual file system relies on the sparse checkout logic so force it on */
+ if (core_virtualfilesystem)
+ core_apply_sparse_checkout = 1;
+ else
+ core_apply_sparse_checkout = git_config_bool(var, value);
return 0;
}
@@ -1651,6 +1667,11 @@ int git_default_core_config(const char *var, const char *value,
return 0;
}
+ if (!strcmp(var, "core.virtualizeobjects")) {
+ core_virtualize_objects = git_config_bool(var, value);
+ return 0;
+ }
+
/* Add other config variables here and to Documentation/config.txt. */
return platform_core_config(var, value, ctx, cb);
}
@@ -1763,6 +1784,37 @@ static int git_default_mailmap_config(const char *var, const char *value)
return 0;
}
+static int git_default_gvfs_config(const char *var, const char *value)
+{
+ if (!strcmp(var, "gvfs.cache-server")) {
+ char *v2 = NULL;
+
+ if (!git_config_string(&v2, var, value) && v2 && *v2) {
+ free(gvfs_cache_server_url);
+ gvfs_cache_server_url = transport_anonymize_url(v2);
+ }
+ free(v2);
+ return 0;
+ }
+
+ if (!strcmp(var, "gvfs.sharedcache") && value && *value) {
+ strbuf_setlen(&gvfs_shared_cache_pathname, 0);
+ strbuf_addstr(&gvfs_shared_cache_pathname, value);
+ if (strbuf_normalize_path(&gvfs_shared_cache_pathname) < 0) {
+ /*
+ * Pretend it wasn't set. This will cause us to
+ * fallback to ".git/objects" effectively.
+ */
+ strbuf_release(&gvfs_shared_cache_pathname);
+ return 0;
+ }
+ strbuf_trim_trailing_dir_sep(&gvfs_shared_cache_pathname);
+ return 0;
+ }
+
+ return 0;
+}
+
static int git_default_attr_config(const char *var, const char *value)
{
if (!strcmp(var, "attr.tree")) {
@@ -1830,6 +1882,9 @@ int git_default_config(const char *var, const char *value,
if (starts_with(var, "sparse."))
return git_default_sparse_config(var, value);
+ if (starts_with(var, "gvfs."))
+ return git_default_gvfs_config(var, value);
+
/* Add other config variables here and to Documentation/config.txt. */
return 0;
}
@@ -2708,6 +2763,44 @@ int repo_config_get_max_percent_split_change(struct repository *r)
return -1; /* default value */
}
+int repo_config_get_virtualfilesystem(struct repository *r)
+{
+ /* Run only once. */
+ static int virtual_filesystem_result = -1;
+ if (virtual_filesystem_result >= 0)
+ return virtual_filesystem_result;
+
+ if (repo_config_get_pathname(r, "core.virtualfilesystem", &core_virtualfilesystem))
+ core_virtualfilesystem = xstrdup_or_null(getenv("GIT_VIRTUALFILESYSTEM_TEST"));
+
+ if (core_virtualfilesystem && !*core_virtualfilesystem)
+ FREE_AND_NULL(core_virtualfilesystem);
+
+ if (core_virtualfilesystem) {
+ /*
+ * Some git commands spawn helpers and redirect the index to a different
+ * location. These include "difftool -d" and the sequencer
+ * (i.e. `git rebase -i`, `git cherry-pick` and `git revert`) and others.
+ * In those instances we don't want to update their temporary index with
+ * our virtualization data.
+ */
+ char *default_index_file = xstrfmt("%s/%s", the_repository->gitdir, "index");
+ int should_run_hook = !strcmp(default_index_file, the_repository->index_file);
+
+ free(default_index_file);
+ if (should_run_hook) {
+ /* virtual file system relies on the sparse checkout logic so force it on */
+ core_apply_sparse_checkout = 1;
+ virtual_filesystem_result = 1;
+ return 1;
+ }
+ FREE_AND_NULL(core_virtualfilesystem);
+ }
+
+ virtual_filesystem_result = 0;
+ return 0;
+}
+
int repo_config_get_index_threads(struct repository *r, int *dest)
{
int is_bool, val;
@@ -3182,6 +3275,7 @@ int repo_config_set_multivar_in_file_gently(struct repository *r,
const char *comment,
unsigned flags)
{
+ static unsigned long timeout_ms = ULONG_MAX;
int fd = -1, in_fd = -1;
int ret;
struct lock_file lock = LOCK_INIT;
@@ -3202,11 +3296,16 @@ int repo_config_set_multivar_in_file_gently(struct repository *r,
if (!config_filename)
config_filename = filename_buf = repo_git_path(r, "config");
+ if ((long)timeout_ms < 0 &&
+ git_config_get_ulong("core.configWriteLockTimeoutMS", &timeout_ms))
+ timeout_ms = 0;
+
/*
* The lock serves a purpose in addition to locking: the new
* contents of .git/config will be written into it.
*/
- fd = hold_lock_file_for_update(&lock, config_filename, 0);
+ fd = hold_lock_file_for_update_timeout(&lock, config_filename, 0,
+ timeout_ms);
if (fd < 0) {
error_errno(_("could not lock config file %s"), config_filename);
ret = CONFIG_NO_LOCK;
diff --git a/config.h b/config.h
index e4199bbdc07685..b640ded10f822f 100644
--- a/config.h
+++ b/config.h
@@ -684,6 +684,8 @@ int repo_config_get_index_threads(struct repository *r, int *dest);
int repo_config_get_split_index(struct repository *r);
int repo_config_get_max_percent_split_change(struct repository *r);
+int repo_config_get_virtualfilesystem(struct repository *r);
+
/* This dies if the configured or default date is in the future */
int repo_config_get_expiry(struct repository *r, const char *key, char **output);
diff --git a/connected.c b/connected.c
index 3099da84f3397f..1ce9cf9521ed90 100644
--- a/connected.c
+++ b/connected.c
@@ -1,8 +1,10 @@
#define USE_THE_REPOSITORY_VARIABLE
#include "git-compat-util.h"
+#include "environment.h"
#include "gettext.h"
#include "hex.h"
+#include "gvfs.h"
#include "object-store-ll.h"
#include "run-command.h"
#include "sigchain.h"
@@ -34,6 +36,26 @@ int check_connected(oid_iterate_fn fn, void *cb_data,
struct transport *transport;
size_t base_len;
+ /*
+ * Running a virtual file system there will be objects that are
+ * missing locally and we don't want to download a bunch of
+ * commits, trees, and blobs just to make sure everything is
+ * reachable locally so this option will skip reachablility
+ * checks below that use rev-list. This will stop the check
+ * before uploadpack runs to determine if there is anything to
+ * fetch. Returning zero for the first check will also prevent the
+ * uploadpack from happening. It will also skip the check after
+ * the fetch is finished to make sure all the objects where
+ * downloaded in the pack file. This will allow the fetch to
+ * run and get all the latest tip commit ids for all the branches
+ * in the fetch but not pull down commits, trees, or blobs via
+ * upload pack.
+ */
+ if (gvfs_config_is_set(GVFS_FETCH_SKIP_REACHABILITY_AND_UPLOADPACK))
+ return 0;
+ if (core_virtualize_objects)
+ return 0;
+
if (!opt)
opt = &defaults;
transport = opt->transport;
diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt
index 5b2c600c2db1b7..b7d0ca62973267 100644
--- a/contrib/buildsystems/CMakeLists.txt
+++ b/contrib/buildsystems/CMakeLists.txt
@@ -635,7 +635,7 @@ if(NOT CURL_FOUND)
add_compile_definitions(NO_CURL)
message(WARNING "git-http-push and git-http-fetch will not be built")
else()
- list(APPEND PROGRAMS_BUILT git-http-fetch git-http-push git-imap-send git-remote-http)
+ list(APPEND PROGRAMS_BUILT git-http-fetch git-http-push git-imap-send git-remote-http git-gvfs-helper)
if(CURL_VERSION_STRING VERSION_GREATER_EQUAL 7.34.0)
add_compile_definitions(USE_CURL_FOR_IMAP_SEND)
endif()
@@ -804,7 +804,7 @@ target_link_libraries(git-sh-i18n--envsubst common-main)
add_executable(git-shell ${CMAKE_SOURCE_DIR}/shell.c)
target_link_libraries(git-shell common-main)
-add_executable(scalar ${CMAKE_SOURCE_DIR}/scalar.c)
+add_executable(scalar ${CMAKE_SOURCE_DIR}/scalar.c ${CMAKE_SOURCE_DIR}/json-parser.c)
target_link_libraries(scalar common-main)
if(CURL_FOUND)
@@ -823,6 +823,9 @@ if(CURL_FOUND)
add_executable(git-http-push ${CMAKE_SOURCE_DIR}/http-push.c)
target_link_libraries(git-http-push http_obj common-main ${CURL_LIBRARIES} ${EXPAT_LIBRARIES})
endif()
+
+ add_executable(git-gvfs-helper ${CMAKE_SOURCE_DIR}/gvfs-helper.c)
+ target_link_libraries(git-gvfs-helper http_obj common-main ${CURL_LIBRARIES} )
endif()
parse_makefile_for_executables(git_builtin_extra "BUILT_INS")
@@ -1116,6 +1119,20 @@ set(wrapper_scripts
set(wrapper_test_scripts
test-fake-ssh test-tool)
+if(CURL_FOUND)
+ list(APPEND wrapper_test_scripts test-gvfs-protocol)
+
+ add_executable(test-gvfs-protocol ${CMAKE_SOURCE_DIR}/t/helper/test-gvfs-protocol.c)
+ target_link_libraries(test-gvfs-protocol common-main)
+
+ if(MSVC)
+ set_target_properties(test-gvfs-protocol
+ PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG ${CMAKE_BINARY_DIR}/t/helper)
+ set_target_properties(test-gvfs-protocol
+ PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_BINARY_DIR}/t/helper)
+ endif()
+endif()
+
foreach(script ${wrapper_scripts})
file(STRINGS ${CMAKE_SOURCE_DIR}/bin-wrappers/wrap-for-bin.sh content NEWLINE_CONSUME)
diff --git a/contrib/completion/git-completion.bash b/contrib/completion/git-completion.bash
index b3b6aa3bae2919..8cb868301057fe 100644
--- a/contrib/completion/git-completion.bash
+++ b/contrib/completion/git-completion.bash
@@ -1802,7 +1802,7 @@ _git_clone ()
esac
}
-__git_untracked_file_modes="all no normal"
+__git_untracked_file_modes="all no normal complete"
__git_trailer_tokens ()
{
diff --git a/contrib/long-running-read-object/example.pl b/contrib/long-running-read-object/example.pl
new file mode 100644
index 00000000000000..b8f37f836a813c
--- /dev/null
+++ b/contrib/long-running-read-object/example.pl
@@ -0,0 +1,114 @@
+#!/usr/bin/perl
+#
+# Example implementation for the Git read-object protocol version 1
+# See Documentation/technical/read-object-protocol.txt
+#
+# Allows you to test the ability for blobs to be pulled from a host git repo
+# "on demand." Called when git needs a blob it couldn't find locally due to
+# a lazy clone that only cloned the commits and trees.
+#
+# A lazy clone can be simulated via the following commands from the host repo
+# you wish to create a lazy clone of:
+#
+# cd /host_repo
+# git rev-parse HEAD
+# git init /guest_repo
+# git cat-file --batch-check --batch-all-objects | grep -v 'blob' |
+# cut -d' ' -f1 | git pack-objects /guest_repo/.git/objects/pack/noblobs
+# cd /guest_repo
+# git config core.virtualizeobjects true
+# git reset --hard
+#
+# Please note, this sample is a minimal skeleton. No proper error handling
+# was implemented.
+#
+
+use strict;
+use warnings;
+
+#
+# Point $DIR to the folder where your host git repo is located so we can pull
+# missing objects from it
+#
+my $DIR = "/host_repo/.git/";
+
+sub packet_bin_read {
+ my $buffer;
+ my $bytes_read = read STDIN, $buffer, 4;
+ if ( $bytes_read == 0 ) {
+
+ # EOF - Git stopped talking to us!
+ exit();
+ }
+ elsif ( $bytes_read != 4 ) {
+ die "invalid packet: '$buffer'";
+ }
+ my $pkt_size = hex($buffer);
+ if ( $pkt_size == 0 ) {
+ return ( 1, "" );
+ }
+ elsif ( $pkt_size > 4 ) {
+ my $content_size = $pkt_size - 4;
+ $bytes_read = read STDIN, $buffer, $content_size;
+ if ( $bytes_read != $content_size ) {
+ die "invalid packet ($content_size bytes expected; $bytes_read bytes read)";
+ }
+ return ( 0, $buffer );
+ }
+ else {
+ die "invalid packet size: $pkt_size";
+ }
+}
+
+sub packet_txt_read {
+ my ( $res, $buf ) = packet_bin_read();
+ unless ( $buf =~ s/\n$// ) {
+ die "A non-binary line MUST be terminated by an LF.";
+ }
+ return ( $res, $buf );
+}
+
+sub packet_bin_write {
+ my $buf = shift;
+ print STDOUT sprintf( "%04x", length($buf) + 4 );
+ print STDOUT $buf;
+ STDOUT->flush();
+}
+
+sub packet_txt_write {
+ packet_bin_write( $_[0] . "\n" );
+}
+
+sub packet_flush {
+ print STDOUT sprintf( "%04x", 0 );
+ STDOUT->flush();
+}
+
+( packet_txt_read() eq ( 0, "git-read-object-client" ) ) || die "bad initialize";
+( packet_txt_read() eq ( 0, "version=1" ) ) || die "bad version";
+( packet_bin_read() eq ( 1, "" ) ) || die "bad version end";
+
+packet_txt_write("git-read-object-server");
+packet_txt_write("version=1");
+packet_flush();
+
+( packet_txt_read() eq ( 0, "capability=get" ) ) || die "bad capability";
+( packet_bin_read() eq ( 1, "" ) ) || die "bad capability end";
+
+packet_txt_write("capability=get");
+packet_flush();
+
+while (1) {
+ my ($command) = packet_txt_read() =~ /^command=([^=]+)$/;
+
+ if ( $command eq "get" ) {
+ my ($sha1) = packet_txt_read() =~ /^sha1=([0-9a-f]{40})$/;
+ packet_bin_read();
+
+ system ('git --git-dir="' . $DIR . '" cat-file blob ' . $sha1 . ' | git -c core.virtualizeobjects=false hash-object -w --stdin >/dev/null 2>&1');
+ packet_txt_write(($?) ? "status=error" : "status=success");
+ packet_flush();
+ } else {
+ die "bad command '$command'";
+ }
+}
diff --git a/contrib/scalar/docs/faq.md b/contrib/scalar/docs/faq.md
new file mode 100644
index 00000000000000..a14f78a996d5d5
--- /dev/null
+++ b/contrib/scalar/docs/faq.md
@@ -0,0 +1,51 @@
+Frequently Asked Questions
+==========================
+
+Using Scalar
+------------
+
+### I don't want a sparse clone, I want every file after I clone!
+
+Run `scalar clone --full-clone ` to initialize your repo to include
+every file. You can switch to a sparse-checkout later by running
+`git sparse-checkout init --cone`.
+
+### I already cloned without `--full-clone`. How do I get everything?
+
+Run `git sparse-checkout disable`.
+
+Scalar Design Decisions
+-----------------------
+
+There may be many design decisions within Scalar that are confusing at first
+glance. Some of them may cause friction when you use Scalar with your existing
+repos and existing habits.
+
+> Scalar has the most benefit when users design repositories
+> with efficient patterns.
+
+For example: Scalar uses the sparse-checkout feature to limit the size of the
+working directory within a large monorepo. It is designed to work efficiently
+with monorepos that are highly componentized, allowing most developers to
+need many fewer files in their daily work.
+
+### Why does `scalar clone` create a `/src` folder?
+
+Scalar uses a file system watcher to keep track of changes under this `src` folder.
+Any activity in this folder is assumed to be important to Git operations. By
+creating the `src` folder, we are making it easy for your build system to
+create output folders outside the `src` directory. We commonly see systems
+create folders for build outputs and package downloads. Scalar itself creates
+these folders during its builds.
+
+Your build system may create build artifacts such as `.obj` or `.lib` files
+next to your source code. These are commonly "hidden" from Git using
+`.gitignore` files. Having such artifacts in your source tree creates
+additional work for Git because it needs to look at these files and match them
+against the `.gitignore` patterns.
+
+By following the `src` pattern Scalar tries to establish and placing your build
+intermediates and outputs parallel with the `src` folder and not inside it,
+you can help optimize Git command performance for developers in the repository
+by limiting the number of files Git needs to consider for many common
+operations.
diff --git a/contrib/scalar/docs/getting-started.md b/contrib/scalar/docs/getting-started.md
new file mode 100644
index 00000000000000..d5125330320d2c
--- /dev/null
+++ b/contrib/scalar/docs/getting-started.md
@@ -0,0 +1,109 @@
+Getting Started
+===============
+
+Registering existing Git repos
+------------------------------
+
+To add a repository to the list of registered repos, run `scalar register []`.
+If `` is not provided, then the "current repository" is discovered from
+the working directory by scanning the parent paths for a path containing a `.git`
+folder, possibly inside a `src` folder.
+
+To see which repositories are currently tracked by the service, run
+`scalar list`.
+
+Run `scalar unregister []` to remove the repo from this list.
+
+Creating a new Scalar clone
+---------------------------------------------------
+
+The `clone` verb creates a local enlistment of a remote repository using the
+partial clone feature available e.g. on GitHub, or using the
+[GVFS protocol](https://github.com/microsoft/VFSForGit/blob/HEAD/Protocol.md),
+such as Azure Repos.
+
+```
+scalar clone [options] []
+```
+
+Create a local copy of the repository at ``. If specified, create the ``
+directory and place the repository there. Otherwise, the last section of the ``
+will be used for ``.
+
+At the end, the repo is located at `/src`. By default, the sparse-checkout
+feature is enabled and the only files present are those in the root of your
+Git repository. Use `git sparse-checkout set` to expand the set of directories
+you want to see, or `git sparse-checkout disable` to expand to all files. You
+can explore the subdirectories outside your sparse-checkout specification using
+`git ls-tree HEAD`.
+
+### Sparse Repo Mode
+
+By default, Scalar reduces your working directory to only the files at the
+root of the repository. You need to add the folders you care about to build up
+to your working set.
+
+* `scalar clone `
+ * Please choose the **Clone with HTTPS** option in the `Clone Repository` dialog in Azure Repos, not **Clone with SSH**.
+* `cd \src`
+* At this point, your `src` directory only contains files that appear in your root
+ tree. No folders are populated.
+* Set the directory list for your sparse-checkout using:
+ 1. `git sparse-checkout set ...`
+ 2. `git sparse-checkout set --stdin < dir-list.txt`
+* Run git commands as you normally would.
+* To fully populate your working directory, run `git sparse-checkout disable`.
+
+If instead you want to start with all files on-disk, you can clone with the
+`--full-clone` option. To enable sparse-checkout after the fact, run
+`git sparse-checkout init --cone`. This will initialize your sparse-checkout
+patterns to only match the files at root.
+
+If you are unfamiliar with what directories are available in the repository,
+then you can run `git ls-tree -d --name-only HEAD` to discover the directories
+at root, or `git ls-tree -d --name-only HEAD ` to discover the directories
+in ``.
+
+### Options
+
+These options allow a user to customize their initial enlistment.
+
+* `--full-clone`: If specified, do not initialize the sparse-checkout feature.
+ All files will be present in your `src` directory. This behaves very similar
+ to a Git partial clone in that blobs are downloaded on demand. However, it
+ will use the GVFS protocol to download all Git objects.
+
+* `--cache-server-url=`: If specified, set the intended cache server to
+ the specified ``. All object queries will use the GVFS protocol to this
+ `` instead of the origin remote. If the remote supplies a list of
+ cache servers via the `/gvfs/config` endpoint, then the `clone` command
+ will select a nearby cache server from that list.
+
+* `--branch=[`: Specify the branch to checkout after clone.
+
+* `--local-cache-path=`: Use this option to override the path for the
+ local Scalar cache. If not specified, then Scalar will select a default
+ path to share objects with your other enlistments. On Windows, this path
+ is a subdirectory of `:\.scalarCache\`. On Mac, this path is a
+ subdirectory of `~/.scalarCache/`. The default cache path is recommended so
+ multiple enlistments of the same remote repository share objects on the
+ same device.
+
+### Advanced Options
+
+The options below are not intended for use by a typical user. These are
+usually used by build machines to create a temporary enlistment that
+operates on a single commit.
+
+* `--single-branch`: Use this option to only download metadata for the branch
+ that will be checked out. This is helpful for build machines that target
+ a remote with many branches. Any `git fetch` commands after the clone will
+ still ask for all branches.
+
+Removing a Scalar Clone
+-----------------------
+
+Since the `scalar clone` command sets up a file-system watcher (when available),
+that watcher could prevent deleting the enlistment. Run `scalar delete `
+from outside of your enlistment to unregister the enlistment from the filesystem
+watcher and delete the enlistment at ``.
diff --git a/contrib/scalar/docs/index.md b/contrib/scalar/docs/index.md
new file mode 100644
index 00000000000000..4f56e2b0ebbac6
--- /dev/null
+++ b/contrib/scalar/docs/index.md
@@ -0,0 +1,54 @@
+Scalar: Enabling Git at Scale
+=============================
+
+Scalar is a tool that helps Git scale to some of the largest Git repositories.
+It achieves this by enabling some advanced Git features, such as:
+
+* *Partial clone:* reduces time to get a working repository by not
+ downloading all Git objects right away.
+
+* *Background prefetch:* downloads Git object data from all remotes every
+ hour, reducing the amount of time for foreground `git fetch` calls.
+
+* *Sparse-checkout:* limits the size of your working directory.
+
+* *File system monitor:* tracks the recently modified files and eliminates
+ the need for Git to scan the entire worktree.
+
+* *Commit-graph:* accelerates commit walks and reachability calculations,
+ speeding up commands like `git log`.
+
+* *Multi-pack-index:* enables fast object lookups across many pack-files.
+
+* *Incremental repack:* Repacks the packed Git data into fewer pack-file
+ without disrupting concurrent commands by using the multi-pack-index.
+
+By running `scalar register` in any Git repo, Scalar will automatically enable
+these features for that repo (except partial clone) and start running suggested
+maintenance in the background using
+[the `git maintenance` feature](https://git-scm.com/docs/git-maintenance).
+
+Repos cloned with the `scalar clone` command use partial clone or the
+[GVFS protocol](https://github.com/microsoft/VFSForGit/blob/HEAD/Protocol.md)
+to significantly reduce the amount of data required to get started
+using a repository. By delaying all blob downloads until they are required,
+Scalar allows you to work with very large repositories quickly. The GVFS
+protocol allows a network of _cache servers_ to serve objects with lower
+latency and higher throughput. The cache servers also reduce load on the
+central server.
+
+Documentation
+-------------
+
+* [Getting Started](getting-started.md): Get started with Scalar.
+ Includes `scalar register`, `scalar unregister`, `scalar clone`, and
+ `scalar delete`.
+
+* [Troubleshooting](troubleshooting.md):
+ Collect diagnostic information or update custom settings. Includes
+ `scalar diagnose` and `scalar cache-server`.
+
+* [The Philosophy of Scalar](philosophy.md): Why does Scalar work the way
+ it does, and how do we make decisions about its future?
+
+* [Frequently Asked Questions](faq.md)
diff --git a/contrib/scalar/docs/philosophy.md b/contrib/scalar/docs/philosophy.md
new file mode 100644
index 00000000000000..e3dfa025a2504c
--- /dev/null
+++ b/contrib/scalar/docs/philosophy.md
@@ -0,0 +1,71 @@
+The Philosophy of Scalar
+========================
+
+The team building Scalar has **opinions** about Git performance. Scalar
+takes out the guesswork by automatically configuring your Git repositories
+to take advantage of the latest and greatest features. It is difficult to
+say that these are the absolute best settings for every repository, but
+these settings do work for some of the largest repositories in the world.
+
+Scalar intends to do very little more than the standard Git client. We
+actively implement new features into Git instead of Scalar, then update
+Scalar only to configure those new settings. In particular, we ported
+features like background maintenance to Git to make Scalar simpler and
+make Git more powerful.
+
+Scalar ships inside [a custom version of Git][microsoft-git], but we are
+working to make it available in other forks of Git. The only feature
+that is not intended to ever reach the standard Git client is Scalar's use
+of [the GVFS Protocol][gvfs-protocol], which is essentially an older
+version of [Git's partial clone feature](https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/)
+that was available first in Azure Repos. Services such as GitHub support
+only partial clone instead of the GVFS protocol because that is the
+standard adopted by the Git project. If your hosting service supports
+partial clone, then we absolutely recommend it as a way to greatly speed
+up your clone and fetch times and to reduce how much disk space your Git
+repository requires. Scalar will help with this!
+
+If you don't use the GVFS Protocol, then most of the value of Scalar can
+be found in the core Git client. However, most of the advanced features
+that really optimize Git's performance are off by default for compatibility
+reasons. To really take advantage of Git's latest and greatest features,
+you either need to study the [`git config` documentation](https://git-scm.com/docs/git-config)
+and regularly read [the Git release notes](https://github.com/git/git/tree/master/Documentation/RelNotes).
+Even if you do all that work and customize your Git settings on your machines,
+you likely will want to share those settings with other team members.
+Or, you can just use Scalar!
+
+Using `scalar register` on an existing Git repository will give you these
+benefits:
+
+* Additional compression of your `.git/index` file.
+* Hourly background `git fetch` operations, keeping you in-sync with your
+ remotes.
+* Advanced data structures, such as the `commit-graph` and `multi-pack-index`
+ are updated automatically in the background.
+* If using macOS or Windows, then Scalar configures Git's builtin File System
+ Monitor, providing faster commands such as `git status` or `git add`.
+
+Additionally, if you use `scalar clone` to create a new repository, then
+you will automatically get these benefits:
+
+* Use Git's partial clone feature to only download the files you need for
+ your current checkout.
+* Use Git's [sparse-checkout feature][sparse-checkout] to minimize the
+ number of files required in your working directory.
+ [Read more about sparse-checkout here.][sparse-checkout-blog]
+* Create the Git repository inside `/src` to make it easy to
+ place build artifacts outside of the Git repository, such as in
+ `/bin` or `/packages`.
+
+We also admit that these **opinions** can always be improved! If you have
+an idea of how to improve our setup, consider
+[creating an issue](https://github.com/microsoft/scalar/issues/new) or
+contributing a pull request! Some [existing](https://github.com/microsoft/scalar/issues/382)
+[issues](https://github.com/microsoft/scalar/issues/388) have already
+improved our configuration settings and roadmap!
+
+[gvfs-protocol]: https://github.com/microsoft/VFSForGit/blob/HEAD/Protocol.md
+[microsoft-git]: https://github.com/microsoft/git
+[sparse-checkout]: https://git-scm.com/docs/git-sparse-checkout
+[sparse-checkout-blog]: https://github.blog/2020-01-17-bring-your-monorepo-down-to-size-with-sparse-checkout/
diff --git a/contrib/scalar/docs/troubleshooting.md b/contrib/scalar/docs/troubleshooting.md
new file mode 100644
index 00000000000000..c54d2438f22523
--- /dev/null
+++ b/contrib/scalar/docs/troubleshooting.md
@@ -0,0 +1,40 @@
+Troubleshooting
+===============
+
+Diagnosing Issues
+-----------------
+
+The `scalar diagnose` command collects logs and config details for the current
+repository. The resulting zip file helps root-cause issues.
+
+When run inside your repository, creates a zip file containing several important
+files for that repository. This includes:
+
+* Configuration files from your `.git` folder, such as the `config` file,
+ `index`, `hooks`, and `refs`.
+
+* A summary of your Git object database, including the number of loose objects
+ and the names and sizes of pack-files.
+
+As the `diagnose` command completes, it provides the path of the resulting
+zip file. This zip can be attached to bug reports to make the analysis easier.
+
+Modifying Configuration Values
+------------------------------
+
+The Scalar-specific configuration is only available for repos using the
+GVFS protocol.
+
+### Cache Server URL
+
+When using an enlistment cloned with `scalar clone` and the GVFS protocol,
+you will have a value called the cache server URL. Cache servers are a feature
+of the GVFS protocol to provide low-latency access to the on-demand object
+requests. This modifies the `gvfs.cache-server` setting in your local Git config
+file.
+
+Run `scalar cache-server --get` to see the current cache server.
+
+Run `scalar cache-server --list` to see the available cache server URLs.
+
+Run `scalar cache-server --set=` to set your cache server to ``.
diff --git a/convert.c b/convert.c
index 9cc0ca20ca0776..22486ab42b8cb7 100644
--- a/convert.c
+++ b/convert.c
@@ -3,6 +3,7 @@
#include "git-compat-util.h"
#include "advice.h"
+#include "gvfs.h"
#include "config.h"
#include "convert.h"
#include "copy.h"
@@ -563,6 +564,9 @@ static int crlf_to_git(struct index_state *istate,
if (!buf)
return 1;
+ if (gvfs_config_is_set(GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS))
+ die("CRLF conversions not supported when running under GVFS");
+
/* only grow if not in place */
if (strbuf_avail(buf) + buf->len < len)
strbuf_grow(buf, len - buf->len);
@@ -602,6 +606,9 @@ static int crlf_to_worktree(const char *src, size_t len, struct strbuf *buf,
if (!will_convert_lf_to_crlf(&stats, crlf_action))
return 0;
+ if (gvfs_config_is_set(GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS))
+ die("CRLF conversions not supported when running under GVFS");
+
/* are we "faking" in place editing ? */
if (src == buf->buf)
to_free = strbuf_detach(buf, NULL);
@@ -711,6 +718,9 @@ static int apply_single_file_filter(const char *path, const char *src, size_t le
struct async async;
struct filter_params params;
+ if (gvfs_config_is_set(GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS))
+ die("Filter \"%s\" not supported when running under GVFS", cmd);
+
memset(&async, 0, sizeof(async));
async.proc = filter_buffer_or_fd;
async.data = ¶ms;
@@ -1130,6 +1140,9 @@ static int ident_to_git(const char *src, size_t len,
if (!buf)
return 1;
+ if (gvfs_config_is_set(GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS))
+ die("ident conversions not supported when running under GVFS");
+
/* only grow if not in place */
if (strbuf_avail(buf) + buf->len < len)
strbuf_grow(buf, len - buf->len);
@@ -1177,6 +1190,9 @@ static int ident_to_worktree(const char *src, size_t len,
if (!cnt)
return 0;
+ if (gvfs_config_is_set(GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS))
+ die("ident conversions not supported when running under GVFS");
+
/* are we "faking" in place editing ? */
if (src == buf->buf)
to_free = strbuf_detach(buf, NULL);
@@ -1629,6 +1645,9 @@ static int lf_to_crlf_filter_fn(struct stream_filter *filter,
size_t count, o = 0;
struct lf_to_crlf_filter *lf_to_crlf = (struct lf_to_crlf_filter *)filter;
+ if (gvfs_config_is_set(GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS))
+ die("CRLF conversions not supported when running under GVFS");
+
/*
* We may be holding onto the CR to see if it is followed by a
* LF, in which case we would need to go to the main loop.
@@ -1873,6 +1892,9 @@ static int ident_filter_fn(struct stream_filter *filter,
struct ident_filter *ident = (struct ident_filter *)filter;
static const char head[] = "$Id";
+ if (gvfs_config_is_set(GVFS_BLOCK_FILTERS_AND_EOL_CONVERSIONS))
+ die("ident conversions not supported when running under GVFS");
+
if (!input) {
/* drain upon eof */
switch (ident->state) {
diff --git a/credential.c b/credential.c
index a995031c5f5d84..a00eedba31a97e 100644
--- a/credential.c
+++ b/credential.c
@@ -444,6 +444,8 @@ static int run_credential_helper(struct credential *c,
else
helper.no_stdout = 1;
+ helper.trace2_child_class = "cred";
+
if (start_command(&helper) < 0)
return -1;
diff --git a/diagnose.c b/diagnose.c
index b11931df86c4ba..e367255be178b0 100644
--- a/diagnose.c
+++ b/diagnose.c
@@ -13,6 +13,7 @@
#include "packfile.h"
#include "parse-options.h"
#include "write-or-die.h"
+#include "config.h"
struct archive_dir {
const char *path;
@@ -72,6 +73,39 @@ static int dir_file_stats(struct object_directory *object_dir, void *data)
return 0;
}
+static void dir_stats(struct strbuf *buf, const char *path)
+{
+ DIR *dir = opendir(path);
+ struct dirent *e;
+ struct stat e_stat;
+ struct strbuf file_path = STRBUF_INIT;
+ size_t base_path_len;
+
+ if (!dir)
+ return;
+
+ strbuf_addstr(buf, "Contents of ");
+ strbuf_add_absolute_path(buf, path);
+ strbuf_addstr(buf, ":\n");
+
+ strbuf_add_absolute_path(&file_path, path);
+ strbuf_addch(&file_path, '/');
+ base_path_len = file_path.len;
+
+ while ((e = readdir(dir)) != NULL)
+ if (!is_dot_or_dotdot(e->d_name) && e->d_type == DT_REG) {
+ strbuf_setlen(&file_path, base_path_len);
+ strbuf_addstr(&file_path, e->d_name);
+ if (!stat(file_path.buf, &e_stat))
+ strbuf_addf(buf, "%-70s %16"PRIuMAX"\n",
+ e->d_name,
+ (uintmax_t)e_stat.st_size);
+ }
+
+ strbuf_release(&file_path);
+ closedir(dir);
+}
+
static int count_files(struct strbuf *path)
{
DIR *dir = opendir(path->buf);
@@ -184,7 +218,8 @@ int create_diagnostics_archive(struct strbuf *zip_path, enum diagnose_mode mode)
struct strvec archiver_args = STRVEC_INIT;
char **argv_copy = NULL;
int stdout_fd = -1, archiver_fd = -1;
- struct strbuf buf = STRBUF_INIT;
+ char *cache_server_url = NULL, *shared_cache = NULL;
+ struct strbuf buf = STRBUF_INIT, path = STRBUF_INIT;
int res;
struct archive_dir archive_dirs[] = {
{ ".git", 0 },
@@ -219,6 +254,13 @@ int create_diagnostics_archive(struct strbuf *zip_path, enum diagnose_mode mode)
get_version_info(&buf, 1);
strbuf_addf(&buf, "Repository root: %s\n", the_repository->worktree);
+
+ git_config_get_string("gvfs.cache-server", &cache_server_url);
+ git_config_get_string("gvfs.sharedCache", &shared_cache);
+ strbuf_addf(&buf, "Cache Server: %s\nLocal Cache: %s\n\n",
+ cache_server_url ? cache_server_url : "None",
+ shared_cache ? shared_cache : "None");
+
get_disk_info(&buf);
write_or_die(stdout_fd, buf.buf, buf.len);
strvec_pushf(&archiver_args,
@@ -249,6 +291,52 @@ int create_diagnostics_archive(struct strbuf *zip_path, enum diagnose_mode mode)
}
}
+ if (shared_cache) {
+ size_t path_len;
+
+ strbuf_reset(&buf);
+ strbuf_addf(&path, "%s/pack", shared_cache);
+ strbuf_reset(&buf);
+ strbuf_addstr(&buf, "--add-virtual-file=packs-cached.txt:");
+ dir_stats(&buf, path.buf);
+ strvec_push(&archiver_args, buf.buf);
+
+ strbuf_reset(&buf);
+ strbuf_addstr(&buf, "--add-virtual-file=objects-cached.txt:");
+ loose_objs_stats(&buf, shared_cache);
+ strvec_push(&archiver_args, buf.buf);
+
+ strbuf_reset(&path);
+ strbuf_addf(&path, "%s/info", shared_cache);
+ path_len = path.len;
+
+ if (is_directory(path.buf)) {
+ DIR *dir = opendir(path.buf);
+ struct dirent *e;
+
+ while ((e = readdir(dir))) {
+ if (!strcmp(".", e->d_name) || !strcmp("..", e->d_name))
+ continue;
+ if (e->d_type == DT_DIR)
+ continue;
+
+ strbuf_reset(&buf);
+ strbuf_addf(&buf, "--add-virtual-file=info/%s:", e->d_name);
+
+ strbuf_setlen(&path, path_len);
+ strbuf_addch(&path, '/');
+ strbuf_addstr(&path, e->d_name);
+
+ if (strbuf_read_file(&buf, path.buf, 0) < 0) {
+ res = error_errno(_("could not read '%s'"), path.buf);
+ goto diagnose_cleanup;
+ }
+ strvec_push(&archiver_args, buf.buf);
+ }
+ closedir(dir);
+ }
+ }
+
strvec_pushl(&archiver_args, "--prefix=",
oid_to_hex(the_hash_algo->empty_tree), "--", NULL);
@@ -262,10 +350,13 @@ int create_diagnostics_archive(struct strbuf *zip_path, enum diagnose_mode mode)
goto diagnose_cleanup;
}
- fprintf(stderr, "\n"
- "Diagnostics complete.\n"
- "All of the gathered info is captured in '%s'\n",
- zip_path->buf);
+ strbuf_reset(&buf);
+ strbuf_addf(&buf, "\n"
+ "Diagnostics complete.\n"
+ "All of the gathered info is captured in '%s'\n",
+ zip_path->buf);
+ write_or_die(stdout_fd, buf.buf, buf.len);
+ write_or_die(2, buf.buf, buf.len);
diagnose_cleanup:
if (archiver_fd >= 0) {
@@ -276,6 +367,8 @@ int create_diagnostics_archive(struct strbuf *zip_path, enum diagnose_mode mode)
free(argv_copy);
strvec_clear(&archiver_args);
strbuf_release(&buf);
+ free(cache_server_url);
+ free(shared_cache);
return res;
}
diff --git a/diff.c b/diff.c
index d28b4114c8dffb..b555b2c35a618d 100644
--- a/diff.c
+++ b/diff.c
@@ -4046,6 +4046,13 @@ static int reuse_worktree_file(struct index_state *istate,
has_object_pack(istate->repo, oid))
return 0;
+ /*
+ * If this path does not match our sparse-checkout definition,
+ * then the file will not be in the working directory.
+ */
+ if (!path_in_sparse_checkout(name, istate))
+ return 0;
+
/*
* Similarly, if we'd have to convert the file contents anyway, that
* makes the optimization not worthwhile.
diff --git a/dir.c b/dir.c
index 1ed84b7a2f67ac..66f4e4f2b1077c 100644
--- a/dir.c
+++ b/dir.c
@@ -11,6 +11,7 @@
#include "git-compat-util.h"
#include "abspath.h"
+#include "virtualfilesystem.h"
#include "config.h"
#include "convert.h"
#include "dir.h"
@@ -1480,6 +1481,19 @@ enum pattern_match_result path_matches_pattern_list(
int result = NOT_MATCHED;
size_t slash_pos;
+ if (core_virtualfilesystem) {
+ /*
+ * The virtual file system data is used to prevent git from traversing
+ * any part of the tree that is not in the virtual file system. Return
+ * 1 to exclude the entry if it is not found in the virtual file system,
+ * else fall through to the regular excludes logic as it may further exclude.
+ */
+ if (*dtype == DT_UNKNOWN)
+ *dtype = resolve_dtype(DT_UNKNOWN, istate, pathname, pathlen);
+ if (is_excluded_from_virtualfilesystem(pathname, pathlen, *dtype) > 0)
+ return 1;
+ }
+
if (!pl->use_cone_patterns) {
pattern = last_matching_pattern_from_list(pathname, pathlen, basename,
dtype, pl, istate);
@@ -1569,6 +1583,13 @@ static int path_in_sparse_checkout_1(const char *path,
enum pattern_match_result match = UNDECIDED;
const char *end, *slash;
+ /*
+ * When using a virtual filesystem, there aren't really patterns
+ * to follow, but be extra careful to skip this check.
+ */
+ if (core_virtualfilesystem)
+ return 1;
+
/*
* We default to accepting a path if the path is empty, there are no
* patterns, or the patterns are of the wrong type.
@@ -1824,8 +1845,22 @@ struct path_pattern *last_matching_pattern(struct dir_struct *dir,
int is_excluded(struct dir_struct *dir, struct index_state *istate,
const char *pathname, int *dtype_p)
{
- struct path_pattern *pattern =
- last_matching_pattern(dir, istate, pathname, dtype_p);
+ struct path_pattern *pattern;
+
+ if (core_virtualfilesystem) {
+ /*
+ * The virtual file system data is used to prevent git from traversing
+ * any part of the tree that is not in the virtual file system. Return
+ * 1 to exclude the entry if it is not found in the virtual file system,
+ * else fall through to the regular excludes logic as it may further exclude.
+ */
+ if (*dtype_p == DT_UNKNOWN)
+ *dtype_p = resolve_dtype(DT_UNKNOWN, istate, pathname, strlen(pathname));
+ if (is_excluded_from_virtualfilesystem(pathname, strlen(pathname), *dtype_p) > 0)
+ return 1;
+ }
+
+ pattern = last_matching_pattern(dir, istate, pathname, dtype_p);
if (pattern)
return pattern->flags & PATTERN_FLAG_NEGATIVE ? 0 : 1;
return 0;
@@ -2443,6 +2478,8 @@ static enum path_treatment treat_path(struct dir_struct *dir,
ignore_case);
if (dtype != DT_DIR && has_path_in_index)
return path_none;
+ if (is_excluded_from_virtualfilesystem(path->buf, path->len, dtype) > 0)
+ return path_excluded;
/*
* When we are looking at a directory P in the working tree,
@@ -2647,6 +2684,8 @@ static void add_path_to_appropriate_result_list(struct dir_struct *dir,
/* add the path to the appropriate result list */
switch (state) {
case path_excluded:
+ if (is_excluded_from_virtualfilesystem(path->buf, path->len, DT_DIR) > 0)
+ break;
if (dir->flags & DIR_SHOW_IGNORED)
dir_add_name(dir, istate, path->buf, path->len);
else if ((dir->flags & DIR_SHOW_IGNORED_TOO) ||
@@ -3194,6 +3233,8 @@ static int cmp_icase(char a, char b)
{
if (a == b)
return 0;
+ if (is_dir_sep(a))
+ return is_dir_sep(b) ? 0 : -1;
if (ignore_case)
return toupper(a) - toupper(b);
return a - b;
diff --git a/entry.c b/entry.c
index 358379a94cf6ec..ac5eff43e8493f 100644
--- a/entry.c
+++ b/entry.c
@@ -453,7 +453,7 @@ static void mark_colliding_entries(const struct checkout *state,
ce->ce_flags |= CE_MATCHED;
/* TODO: audit for interaction with sparse-index. */
- ensure_full_index(state->istate);
+ ensure_full_index_unaudited(state->istate);
for (size_t i = 0; i < state->istate->cache_nr; i++) {
struct cache_entry *dup = state->istate->cache[i];
diff --git a/environment.c b/environment.c
index 8389a272700eac..d7832244da25c9 100644
--- a/environment.c
+++ b/environment.c
@@ -68,9 +68,12 @@ int grafts_keep_true_parents;
int core_apply_sparse_checkout;
int core_sparse_checkout_cone;
int sparse_expect_files_outside_of_patterns;
+int core_gvfs;
+char *core_virtualfilesystem;
int merge_log_config = -1;
int precomposed_unicode = -1; /* see probe_utf8_pathname_composition() */
unsigned long pack_size_limit_cfg;
+int core_virtualize_objects;
int max_allowed_tree_depth =
#ifdef _MSC_VER
/*
@@ -95,6 +98,9 @@ int protect_hfs = PROTECT_HFS_DEFAULT;
#define PROTECT_NTFS_DEFAULT 1
#endif
int protect_ntfs = PROTECT_NTFS_DEFAULT;
+int core_use_gvfs_helper;
+char *gvfs_cache_server_url;
+struct strbuf gvfs_shared_cache_pathname = STRBUF_INIT;
/*
* The character that begins a commented line in user-editable file
diff --git a/environment.h b/environment.h
index 2f43340f0b553a..abb3366957fb1d 100644
--- a/environment.h
+++ b/environment.h
@@ -170,9 +170,14 @@ extern unsigned long pack_size_limit_cfg;
extern int max_allowed_tree_depth;
extern int core_preload_index;
+extern char *core_virtualfilesystem;
+extern int core_gvfs;
extern int precomposed_unicode;
extern int protect_hfs;
extern int protect_ntfs;
+extern int core_use_gvfs_helper;
+extern char *gvfs_cache_server_url;
+extern struct strbuf gvfs_shared_cache_pathname;
extern int core_apply_sparse_checkout;
extern int core_sparse_checkout_cone;
@@ -224,5 +229,6 @@ extern const char *comment_line_str;
extern char *comment_line_str_to_free;
extern int auto_comment_line_char;
+extern int core_virtualize_objects;
# endif /* USE_THE_REPOSITORY_VARIABLE */
#endif /* ENVIRONMENT_H */
diff --git a/git.c b/git.c
index 71f4a9c37236ab..c24ba4924ee5c4 100644
--- a/git.c
+++ b/git.c
@@ -1,6 +1,7 @@
#define USE_THE_REPOSITORY_VARIABLE
#include "builtin.h"
+#include "gvfs.h"
#include "config.h"
#include "environment.h"
#include "exec-cmd.h"
@@ -17,6 +18,8 @@
#include "shallow.h"
#include "trace.h"
#include "trace2.h"
+#include "dir.h"
+#include "hook.h"
#define RUN_SETUP (1<<0)
#define RUN_SETUP_GENTLY (1<<1)
@@ -28,6 +31,7 @@
#define NEED_WORK_TREE (1<<3)
#define DELAY_PAGER_CONFIG (1<<4)
#define NO_PARSEOPT (1<<5) /* parse-options is not used */
+#define BLOCK_ON_GVFS_REPO (1<<6) /* command not allowed in GVFS repos */
struct cmd_struct {
const char *cmd;
@@ -437,6 +441,68 @@ static int handle_alias(struct strvec *args)
return ret;
}
+/* Runs pre/post-command hook */
+static struct strvec sargv = STRVEC_INIT;
+static int run_post_hook = 0;
+static int exit_code = -1;
+
+static int run_pre_command_hook(struct repository *r, const char **argv)
+{
+ char *lock;
+ int ret = 0;
+ struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT;
+
+ /*
+ * Ensure the global pre/post command hook is only called for
+ * the outer command and not when git is called recursively
+ * or spawns multiple commands (like with the alias command)
+ */
+ lock = getenv("COMMAND_HOOK_LOCK");
+ if (lock && !strcmp(lock, "true"))
+ return 0;
+ setenv("COMMAND_HOOK_LOCK", "true", 1);
+
+ /* call the hook proc */
+ strvec_pushv(&sargv, argv);
+ strvec_pushf(&sargv, "--git-pid=%"PRIuMAX, (uintmax_t)getpid());
+ strvec_pushv(&opt.args, sargv.v);
+ ret = run_hooks_opt(r, "pre-command", &opt);
+
+ if (!ret)
+ run_post_hook = 1;
+ return ret;
+}
+
+static int run_post_command_hook(struct repository *r)
+{
+ char *lock;
+ int ret = 0;
+ struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT;
+
+ /*
+ * Only run post_command if pre_command succeeded in this process
+ */
+ if (!run_post_hook)
+ return 0;
+ lock = getenv("COMMAND_HOOK_LOCK");
+ if (!lock || strcmp(lock, "true"))
+ return 0;
+
+ strvec_pushv(&opt.args, sargv.v);
+ strvec_pushf(&opt.args, "--exit_code=%u", exit_code);
+ ret = run_hooks_opt(r, "post-command", &opt);
+
+ run_post_hook = 0;
+ strvec_clear(&sargv);
+ setenv("COMMAND_HOOK_LOCK", "false", 1);
+ return ret;
+}
+
+static void post_command_hook_atexit(void)
+{
+ run_post_command_hook(the_repository);
+}
+
static int run_builtin(struct cmd_struct *p, int argc, const char **argv, struct repository *repo)
{
int status, help;
@@ -473,16 +539,24 @@ static int run_builtin(struct cmd_struct *p, int argc, const char **argv, struct
if (!help && p->option & NEED_WORK_TREE)
setup_work_tree();
+ if (!help && p->option & BLOCK_ON_GVFS_REPO && gvfs_config_is_set(GVFS_BLOCK_COMMANDS))
+ die("'git %s' is not supported on a GVFS repo", p->cmd);
+
+ if (run_pre_command_hook(the_repository, argv))
+ die("pre-command hook aborted command");
+
trace_argv_printf(argv, "trace: built-in: git");
trace2_cmd_name(p->cmd);
validate_cache_entries(repo->index);
- status = p->fn(argc, argv, prefix, no_repo ? NULL : repo);
+ exit_code = status = p->fn(argc, argv, prefix, no_repo ? NULL : repo);
validate_cache_entries(repo->index);
if (status)
return status;
+ run_post_command_hook(the_repository);
+
/* Somebody closed stdout? */
if (fstat(fileno(stdout), &st))
return 0;
@@ -551,7 +625,7 @@ static struct cmd_struct commands[] = {
{ "for-each-ref", cmd_for_each_ref, RUN_SETUP },
{ "for-each-repo", cmd_for_each_repo, RUN_SETUP_GENTLY },
{ "format-patch", cmd_format_patch, RUN_SETUP },
- { "fsck", cmd_fsck, RUN_SETUP },
+ { "fsck", cmd_fsck, RUN_SETUP | BLOCK_ON_GVFS_REPO},
{ "fsck-objects", cmd_fsck, RUN_SETUP },
{ "fsmonitor--daemon", cmd_fsmonitor__daemon, RUN_SETUP },
{ "gc", cmd_gc, RUN_SETUP },
@@ -592,7 +666,7 @@ static struct cmd_struct commands[] = {
{ "pack-refs", cmd_pack_refs, RUN_SETUP },
{ "patch-id", cmd_patch_id, RUN_SETUP_GENTLY | NO_PARSEOPT },
{ "pickaxe", cmd_blame, RUN_SETUP },
- { "prune", cmd_prune, RUN_SETUP },
+ { "prune", cmd_prune, RUN_SETUP | BLOCK_ON_GVFS_REPO},
{ "prune-packed", cmd_prune_packed, RUN_SETUP },
{ "pull", cmd_pull, RUN_SETUP | NEED_WORK_TREE },
{ "push", cmd_push, RUN_SETUP },
@@ -605,7 +679,7 @@ static struct cmd_struct commands[] = {
{ "remote", cmd_remote, RUN_SETUP },
{ "remote-ext", cmd_remote_ext, NO_PARSEOPT },
{ "remote-fd", cmd_remote_fd, NO_PARSEOPT },
- { "repack", cmd_repack, RUN_SETUP },
+ { "repack", cmd_repack, RUN_SETUP | BLOCK_ON_GVFS_REPO },
{ "replace", cmd_replace, RUN_SETUP },
{ "replay", cmd_replay, RUN_SETUP },
{ "rerere", cmd_rerere, RUN_SETUP },
@@ -626,7 +700,7 @@ static struct cmd_struct commands[] = {
{ "stash", cmd_stash, RUN_SETUP | NEED_WORK_TREE },
{ "status", cmd_status, RUN_SETUP | NEED_WORK_TREE },
{ "stripspace", cmd_stripspace },
- { "submodule--helper", cmd_submodule__helper, RUN_SETUP },
+ { "submodule--helper", cmd_submodule__helper, RUN_SETUP | BLOCK_ON_GVFS_REPO },
{ "survey", cmd_survey, RUN_SETUP },
{ "switch", cmd_switch, RUN_SETUP | NEED_WORK_TREE },
{ "symbolic-ref", cmd_symbolic_ref, RUN_SETUP },
@@ -634,6 +708,7 @@ static struct cmd_struct commands[] = {
{ "unpack-file", cmd_unpack_file, RUN_SETUP | NO_PARSEOPT },
{ "unpack-objects", cmd_unpack_objects, RUN_SETUP | NO_PARSEOPT },
{ "update-index", cmd_update_index, RUN_SETUP },
+ { "update-microsoft-git", cmd_update_microsoft_git },
{ "update-ref", cmd_update_ref, RUN_SETUP },
{ "update-server-info", cmd_update_server_info, RUN_SETUP },
{ "upload-archive", cmd_upload_archive, NO_PARSEOPT },
@@ -770,13 +845,16 @@ static void execv_dashed_external(const char **argv)
*/
trace_argv_printf(cmd.args.v, "trace: exec:");
+ if (run_pre_command_hook(the_repository, cmd.args.v))
+ die("pre-command hook aborted command");
+
/*
* If we fail because the command is not found, it is
* OK to return. Otherwise, we just pass along the status code,
* or our usual generic code if we were not even able to exec
* the program.
*/
- status = run_command(&cmd);
+ exit_code = status = run_command(&cmd);
/*
* If the child process ran and we are now going to exit, emit a
@@ -787,6 +865,8 @@ static void execv_dashed_external(const char **argv)
exit(status);
else if (errno != ENOENT)
exit(128);
+
+ run_post_command_hook(the_repository);
}
static int run_argv(struct strvec *args)
@@ -894,6 +974,7 @@ int cmd_main(int argc, const char **argv)
}
trace_command_performance(argv);
+ atexit(post_command_hook_atexit);
/*
* "git-xxxx" is the same as "git xxxx", but we obviously:
@@ -921,10 +1002,14 @@ int cmd_main(int argc, const char **argv)
if (!argc) {
/* The user didn't specify a command; give them help */
commit_pager_choice();
+ if (run_pre_command_hook(the_repository, argv))
+ die("pre-command hook aborted command");
printf(_("usage: %s\n\n"), git_usage_string);
list_common_cmds_help();
printf("\n%s\n", _(git_more_info_string));
- exit(1);
+ exit_code = 1;
+ run_post_command_hook(the_repository);
+ exit(exit_code);
}
if (!strcmp("--version", argv[0]) || !strcmp("-v", argv[0]))
diff --git a/gvfs-helper-client.c b/gvfs-helper-client.c
new file mode 100644
index 00000000000000..c8ab947f5f9cc4
--- /dev/null
+++ b/gvfs-helper-client.c
@@ -0,0 +1,582 @@
+#define USE_THE_REPOSITORY_VARIABLE
+#include "git-compat-util.h"
+#include "environment.h"
+#include "hex.h"
+#include "strvec.h"
+#include "trace2.h"
+#include "oidset.h"
+#include "object.h"
+#include "object-store.h"
+#include "gvfs-helper-client.h"
+#include "sub-process.h"
+#include "sigchain.h"
+#include "pkt-line.h"
+#include "quote.h"
+#include "packfile.h"
+#include "hex.h"
+#include "config.h"
+
+static struct oidset gh_client__oidset_queued = OIDSET_INIT;
+static unsigned long gh_client__oidset_count;
+
+struct gh_server__process {
+ struct subprocess_entry subprocess; /* must be first */
+ unsigned int supported_capabilities;
+};
+
+static int gh_server__subprocess_map_initialized;
+static struct hashmap gh_server__subprocess_map;
+static struct object_directory *gh_client__chosen_odb;
+
+/*
+ * The "objects" capability has verbs: "get" and "post" and "prefetch".
+ */
+#define CAP_OBJECTS (1u<<1)
+#define CAP_OBJECTS_NAME "objects"
+
+#define CAP_OBJECTS__VERB_GET1_NAME "get"
+#define CAP_OBJECTS__VERB_POST_NAME "post"
+#define CAP_OBJECTS__VERB_PREFETCH_NAME "prefetch"
+
+static int gh_client__start_fn(struct subprocess_entry *subprocess)
+{
+ static int versions[] = {1, 0};
+ static struct subprocess_capability capabilities[] = {
+ { CAP_OBJECTS_NAME, CAP_OBJECTS },
+ { NULL, 0 }
+ };
+
+ struct gh_server__process *entry = (struct gh_server__process *)subprocess;
+
+ return subprocess_handshake(subprocess, "gvfs-helper", versions,
+ NULL, capabilities,
+ &entry->supported_capabilities);
+}
+
+/*
+ * Send the queued OIDs in the OIDSET to gvfs-helper for it to
+ * fetch from the cache-server or main Git server using "/gvfs/objects"
+ * POST semantics.
+ *
+ * objects.post LF
+ * ( LF)*
+ *
+ *
+ */
+static int gh_client__send__objects_post(struct child_process *process)
+{
+ struct oidset_iter iter;
+ struct object_id *oid;
+ int err;
+
+ /*
+ * We assume that all of the packet_ routines call error()
+ * so that we don't have to.
+ */
+
+ err = packet_write_fmt_gently(
+ process->in,
+ (CAP_OBJECTS_NAME "." CAP_OBJECTS__VERB_POST_NAME "\n"));
+ if (err)
+ return err;
+
+ oidset_iter_init(&gh_client__oidset_queued, &iter);
+ while ((oid = oidset_iter_next(&iter))) {
+ err = packet_write_fmt_gently(process->in, "%s\n",
+ oid_to_hex(oid));
+ if (err)
+ return err;
+ }
+
+ err = packet_flush_gently(process->in);
+ if (err)
+ return err;
+
+ return 0;
+}
+
+/*
+ * Send the given OID to gvfs-helper for it to fetch from the
+ * cache-server or main Git server using "/gvfs/objects" GET
+ * semantics.
+ *
+ * This ignores any queued OIDs.
+ *
+ * objects.get LF
+ * LF
+ *
+ *
+ */
+static int gh_client__send__objects_get(struct child_process *process,
+ const struct object_id *oid)
+{
+ int err;
+
+ /*
+ * We assume that all of the packet_ routines call error()
+ * so that we don't have to.
+ */
+
+ err = packet_write_fmt_gently(
+ process->in,
+ (CAP_OBJECTS_NAME "." CAP_OBJECTS__VERB_GET1_NAME "\n"));
+ if (err)
+ return err;
+
+ err = packet_write_fmt_gently(process->in, "%s\n",
+ oid_to_hex(oid));
+ if (err)
+ return err;
+
+ err = packet_flush_gently(process->in);
+ if (err)
+ return err;
+
+ return 0;
+}
+
+/*
+ * Send a request to gvfs-helper to prefetch packfiles from either the
+ * cache-server or the main Git server using "/gvfs/prefetch".
+ *
+ * objects.prefetch LF
+ * [ LF]
+ *
+ */
+static int gh_client__send__objects_prefetch(struct child_process *process,
+ timestamp_t seconds_since_epoch)
+{
+ int err;
+
+ /*
+ * We assume that all of the packet_ routines call error()
+ * so that we don't have to.
+ */
+
+ err = packet_write_fmt_gently(
+ process->in,
+ (CAP_OBJECTS_NAME "." CAP_OBJECTS__VERB_PREFETCH_NAME "\n"));
+ if (err)
+ return err;
+
+ if (seconds_since_epoch) {
+ err = packet_write_fmt_gently(process->in, "%" PRItime "\n",
+ seconds_since_epoch);
+ if (err)
+ return err;
+ }
+
+ err = packet_flush_gently(process->in);
+ if (err)
+ return err;
+
+ return 0;
+}
+
+/*
+ * Update the loose object cache to include the newly created
+ * object.
+ */
+static void gh_client__update_loose_cache(const char *line)
+{
+ const char *v1_oid;
+ struct object_id oid;
+
+ if (!skip_prefix(line, "loose ", &v1_oid))
+ BUG("update_loose_cache: invalid line '%s'", line);
+
+ if (get_oid_hex(v1_oid, &oid))
+ BUG("update_loose_cache: invalid line '%s'", line);
+
+ odb_loose_cache_add_new_oid(gh_client__chosen_odb, &oid);
+}
+
+/*
+ * Update the packed-git list to include the newly created packfile.
+ */
+static void gh_client__update_packed_git(const char *line)
+{
+ struct strbuf path = STRBUF_INIT;
+ const char *v1_filename;
+ struct packed_git *p;
+ int is_local;
+
+ if (!skip_prefix(line, "packfile ", &v1_filename))
+ BUG("update_packed_git: invalid line '%s'", line);
+
+ /*
+ * ODB[0] is the local .git/objects. All others are alternates.
+ */
+ is_local = (gh_client__chosen_odb == the_repository->objects->odb);
+
+ strbuf_addf(&path, "%s/pack/%s",
+ gh_client__chosen_odb->path, v1_filename);
+ strbuf_strip_suffix(&path, ".pack");
+ strbuf_addstr(&path, ".idx");
+
+ p = add_packed_git(the_repository, path.buf, path.len, is_local);
+ if (p)
+ install_packed_git_and_mru(the_repository, p);
+ strbuf_release(&path);
+}
+
+/*
+ * CAP_OBJECTS verbs return the same format response:
+ *
+ *
+ * *
+ *
+ *
+ *
+ * Where:
+ *
+ * ::= odb SP LF
+ *
+ * ::= /
+ *
+ * ::= packfile SP LF
+ *
+ * ::= loose SP LF
+ *
+ * ::= ok LF
+ * / partial LF
+ * / error SP LF
+ *
+ * Note that `gvfs-helper` controls how/if it chunks the request when
+ * it talks to the cache-server and/or main Git server. So it is
+ * possible for us to receive many packfiles and/or loose objects *AND
+ * THEN* get a hard network error or a 404 on an individual object.
+ *
+ * If we get a partial result, we can let the caller try to continue
+ * -- for example, maybe an immediate request for a tree object was
+ * grouped with a queued request for a blob. The tree-walk *might* be
+ * able to continue and let the 404 blob be handled later.
+ */
+static int gh_client__objects__receive_response(
+ struct child_process *process,
+ enum gh_client__created *p_ghc,
+ int *p_nr_loose, int *p_nr_packfile)
+{
+ enum gh_client__created ghc = GHC__CREATED__NOTHING;
+ const char *v1;
+ char *line;
+ int len;
+ int nr_loose = 0;
+ int nr_packfile = 0;
+ int err = 0;
+
+ while (1) {
+ /*
+ * Warning: packet_read_line_gently() calls die()
+ * despite the _gently moniker.
+ */
+ len = packet_read_line_gently(process->out, NULL, &line);
+ if ((len < 0) || !line)
+ break;
+
+ if (starts_with(line, "odb")) {
+ /* trust that this matches what we expect */
+ }
+
+ else if (starts_with(line, "packfile")) {
+ gh_client__update_packed_git(line);
+ ghc |= GHC__CREATED__PACKFILE;
+ nr_packfile++;
+ }
+
+ else if (starts_with(line, "loose")) {
+ gh_client__update_loose_cache(line);
+ ghc |= GHC__CREATED__LOOSE;
+ nr_loose++;
+ }
+
+ else if (starts_with(line, "ok"))
+ ;
+ else if (starts_with(line, "partial"))
+ ;
+ else if (skip_prefix(line, "error ", &v1)) {
+ error("gvfs-helper error: '%s'", v1);
+ err = -1;
+ }
+ }
+
+ *p_ghc = ghc;
+ *p_nr_loose = nr_loose;
+ *p_nr_packfile = nr_packfile;
+
+ return err;
+}
+
+/*
+ * Select the preferred ODB for fetching missing objects.
+ * This should be the alternate with the same directory
+ * name as set in `gvfs.sharedCache`.
+ *
+ * Fallback to .git/objects if necessary.
+ */
+static void gh_client__choose_odb(void)
+{
+ struct object_directory *odb;
+
+ if (gh_client__chosen_odb)
+ return;
+
+ prepare_alt_odb(the_repository);
+ gh_client__chosen_odb = the_repository->objects->odb;
+
+ if (!gvfs_shared_cache_pathname.len)
+ return;
+
+ for (odb = the_repository->objects->odb->next; odb; odb = odb->next) {
+ if (!fspathcmp(odb->path, gvfs_shared_cache_pathname.buf)) {
+ gh_client__chosen_odb = odb;
+ return;
+ }
+ }
+}
+
+static struct gh_server__process *gh_client__find_long_running_process(
+ unsigned int cap_needed)
+{
+ struct gh_server__process *entry;
+ struct strvec argv = STRVEC_INIT;
+ struct strbuf quoted = STRBUF_INIT;
+ int fallback;
+
+ gh_client__choose_odb();
+
+ /*
+ * TODO decide what defaults we want.
+ */
+ strvec_push(&argv, "gvfs-helper");
+ strvec_push(&argv, "--cache-server=trust");
+ strvec_pushf(&argv, "--shared-cache=%s",
+ gh_client__chosen_odb->path);
+
+ /* If gvfs.fallback=false, then don't add --fallback. */
+ if (!git_config_get_bool("gvfs.fallback", &fallback) &&
+ !fallback)
+ strvec_push(&argv, "--no-fallback");
+ else
+ strvec_push(&argv, "--fallback");
+
+ strvec_push(&argv, "server");
+
+ sq_quote_argv_pretty("ed, argv.v);
+
+ /*
+ * Find an existing long-running process with the above command
+ * line -or- create a new long-running process for this and
+ * subsequent requests.
+ */
+ if (!gh_server__subprocess_map_initialized) {
+ gh_server__subprocess_map_initialized = 1;
+ hashmap_init(&gh_server__subprocess_map,
+ (hashmap_cmp_fn)cmd2process_cmp, NULL, 0);
+ entry = NULL;
+ } else
+ entry = (struct gh_server__process *)subprocess_find_entry(
+ &gh_server__subprocess_map, quoted.buf);
+
+ if (!entry) {
+ entry = xmalloc(sizeof(*entry));
+ entry->supported_capabilities = 0;
+
+ if (subprocess_start_strvec(&gh_server__subprocess_map,
+ &entry->subprocess, 1,
+ &argv, gh_client__start_fn))
+ FREE_AND_NULL(entry);
+ }
+
+ if (entry &&
+ (entry->supported_capabilities & cap_needed) != cap_needed) {
+ error("gvfs-helper: does not support needed capabilities");
+ subprocess_stop(&gh_server__subprocess_map,
+ (struct subprocess_entry *)entry);
+ FREE_AND_NULL(entry);
+ }
+
+ strvec_clear(&argv);
+ strbuf_release("ed);
+
+ return entry;
+}
+
+void gh_client__queue_oid(const struct object_id *oid)
+{
+ /*
+ * Keep this trace as a printf only, so that it goes to the
+ * perf log, but not the event log. It is useful for interactive
+ * debugging, but generates way too much (unuseful) noise for the
+ * database.
+ */
+ if (trace2_is_enabled())
+ trace2_printf("gh_client__queue_oid: %s", oid_to_hex(oid));
+
+ if (!oidset_insert(&gh_client__oidset_queued, oid))
+ gh_client__oidset_count++;
+}
+
+/*
+ * This routine should actually take a "const struct oid_array *"
+ * rather than the component parts, but fetch_objects() uses
+ * this model (because of the call in sha1-file.c).
+ */
+void gh_client__queue_oid_array(const struct object_id *oids, int oid_nr)
+{
+ int k;
+
+ for (k = 0; k < oid_nr; k++)
+ gh_client__queue_oid(&oids[k]);
+}
+
+/*
+ * Bulk fetch all of the queued OIDs in the OIDSET.
+ */
+int gh_client__drain_queue(enum gh_client__created *p_ghc)
+{
+ struct gh_server__process *entry;
+ struct child_process *process;
+ int nr_loose = 0;
+ int nr_packfile = 0;
+ int err = 0;
+
+ *p_ghc = GHC__CREATED__NOTHING;
+
+ if (!gh_client__oidset_count)
+ return 0;
+
+ entry = gh_client__find_long_running_process(CAP_OBJECTS);
+ if (!entry)
+ return -1;
+
+ trace2_region_enter("gh-client", "objects/post", the_repository);
+
+ process = &entry->subprocess.process;
+
+ sigchain_push(SIGPIPE, SIG_IGN);
+
+ err = gh_client__send__objects_post(process);
+ if (!err)
+ err = gh_client__objects__receive_response(
+ process, p_ghc, &nr_loose, &nr_packfile);
+
+ sigchain_pop(SIGPIPE);
+
+ if (err) {
+ subprocess_stop(&gh_server__subprocess_map,
+ (struct subprocess_entry *)entry);
+ FREE_AND_NULL(entry);
+ }
+
+ trace2_data_intmax("gh-client", the_repository,
+ "objects/post/nr_objects", gh_client__oidset_count);
+ trace2_region_leave("gh-client", "objects/post", the_repository);
+
+ oidset_clear(&gh_client__oidset_queued);
+ gh_client__oidset_count = 0;
+
+ return err;
+}
+
+/*
+ * Get exactly 1 object immediately.
+ * Ignore any queued objects.
+ */
+int gh_client__get_immediate(const struct object_id *oid,
+ enum gh_client__created *p_ghc)
+{
+ struct gh_server__process *entry;
+ struct child_process *process;
+ int nr_loose = 0;
+ int nr_packfile = 0;
+ int err = 0;
+
+ /*
+ * Keep this trace as a printf only, so that it goes to the
+ * perf log, but not the event log. It is useful for interactive
+ * debugging, but generates way too much (unuseful) noise for the
+ * database.
+ */
+ if (trace2_is_enabled())
+ trace2_printf("gh_client__get_immediate: %s", oid_to_hex(oid));
+
+ entry = gh_client__find_long_running_process(CAP_OBJECTS);
+ if (!entry)
+ return -1;
+
+ trace2_region_enter("gh-client", "objects/get", the_repository);
+
+ process = &entry->subprocess.process;
+
+ sigchain_push(SIGPIPE, SIG_IGN);
+
+ err = gh_client__send__objects_get(process, oid);
+ if (!err)
+ err = gh_client__objects__receive_response(
+ process, p_ghc, &nr_loose, &nr_packfile);
+
+ sigchain_pop(SIGPIPE);
+
+ if (err) {
+ subprocess_stop(&gh_server__subprocess_map,
+ (struct subprocess_entry *)entry);
+ FREE_AND_NULL(entry);
+ }
+
+ trace2_region_leave("gh-client", "objects/get", the_repository);
+
+ return err;
+}
+
+/*
+ * Ask gvfs-helper to prefetch commits-and-trees packfiles since a
+ * given timestamp.
+ *
+ * If seconds_since_epoch is zero, gvfs-helper will scan the ODB for
+ * the last received prefetch and ask for ones newer than that.
+ */
+int gh_client__prefetch(timestamp_t seconds_since_epoch,
+ int *nr_packfiles_received)
+{
+ struct gh_server__process *entry;
+ struct child_process *process;
+ enum gh_client__created ghc;
+ int nr_loose = 0;
+ int nr_packfile = 0;
+ int err = 0;
+
+ entry = gh_client__find_long_running_process(CAP_OBJECTS);
+ if (!entry)
+ return -1;
+
+ trace2_region_enter("gh-client", "objects/prefetch", the_repository);
+ trace2_data_intmax("gh-client", the_repository, "prefetch/since",
+ seconds_since_epoch);
+
+ process = &entry->subprocess.process;
+
+ sigchain_push(SIGPIPE, SIG_IGN);
+
+ err = gh_client__send__objects_prefetch(process, seconds_since_epoch);
+ if (!err)
+ err = gh_client__objects__receive_response(
+ process, &ghc, &nr_loose, &nr_packfile);
+
+ sigchain_pop(SIGPIPE);
+
+ if (err) {
+ subprocess_stop(&gh_server__subprocess_map,
+ (struct subprocess_entry *)entry);
+ FREE_AND_NULL(entry);
+ }
+
+ trace2_data_intmax("gh-client", the_repository,
+ "prefetch/packfile_count", nr_packfile);
+ trace2_region_leave("gh-client", "objects/prefetch", the_repository);
+
+ if (nr_packfiles_received)
+ *nr_packfiles_received = nr_packfile;
+
+ return err;
+}
diff --git a/gvfs-helper-client.h b/gvfs-helper-client.h
new file mode 100644
index 00000000000000..7692534ecda54c
--- /dev/null
+++ b/gvfs-helper-client.h
@@ -0,0 +1,87 @@
+#ifndef GVFS_HELPER_CLIENT_H
+#define GVFS_HELPER_CLIENT_H
+
+struct repository;
+struct commit;
+struct object_id;
+
+enum gh_client__created {
+ /*
+ * The _get_ operation did not create anything. If doesn't
+ * matter if `gvfs-helper` had errors or not -- just that
+ * nothing was created.
+ */
+ GHC__CREATED__NOTHING = 0,
+
+ /*
+ * The _get_ operation created one or more packfiles.
+ */
+ GHC__CREATED__PACKFILE = 1<<1,
+
+ /*
+ * The _get_ operation created one or more loose objects.
+ * (Not necessarily the for the individual OID you requested.)
+ */
+ GHC__CREATED__LOOSE = 1<<2,
+
+ /*
+ * The _get_ operation created one or more packfilea *and*
+ * one or more loose objects.
+ */
+ GHC__CREATED__PACKFILE_AND_LOOSE = (GHC__CREATED__PACKFILE |
+ GHC__CREATED__LOOSE),
+};
+
+/*
+ * Ask `gvfs-helper server` to immediately fetch a single object
+ * using "/gvfs/objects" GET semantics.
+ *
+ * A long-running background process is used to make subsequent
+ * requests more efficient.
+ *
+ * A loose object will be created in the shared-cache ODB and
+ * in-memory cache updated.
+ */
+int gh_client__get_immediate(const struct object_id *oid,
+ enum gh_client__created *p_ghc);
+
+/*
+ * Queue this OID for a future fetch using `gvfs-helper service`.
+ * It does not wait.
+ *
+ * Callers should not rely on the queued object being on disk until
+ * the queue has been drained.
+ */
+void gh_client__queue_oid(const struct object_id *oid);
+void gh_client__queue_oid_array(const struct object_id *oids, int oid_nr);
+
+/*
+ * Ask `gvfs-helper server` to fetch the set of queued OIDs using
+ * "/gvfs/objects" POST semantics.
+ *
+ * A long-running background process is used to subsequent requests
+ * more efficient.
+ *
+ * One or more packfiles will be created in the shared-cache ODB.
+ */
+int gh_client__drain_queue(enum gh_client__created *p_ghc);
+
+/*
+ * Ask `gvfs-helper server` to fetch any "prefetch packs"
+ * available on the server more recent than the requested time.
+ *
+ * If seconds_since_epoch is zero, gvfs-helper will scan the ODB for
+ * the last received prefetch and ask for ones newer than that.
+ *
+ * A long-running background process is used to subsequent requests
+ * (either prefetch or regular immediate/queued requests) more efficient.
+ *
+ * One or more packfiles will be created in the shared-cache ODB.
+ *
+ * Returns 0 on success, -1 on error. Optionally also returns the
+ * number of prefetch packs received.
+ */
+int gh_client__prefetch(timestamp_t seconds_since_epoch,
+ int *nr_packfiles_received);
+
+#endif /* GVFS_HELPER_CLIENT_H */
diff --git a/gvfs-helper.c b/gvfs-helper.c
new file mode 100644
index 00000000000000..9615845411288b
--- /dev/null
+++ b/gvfs-helper.c
@@ -0,0 +1,4234 @@
+// TODO Write a man page. Here are some notes for dogfooding.
+// TODO
+//
+// Usage: git gvfs-helper [] []
+//
+// :
+//
+// --remote= // defaults to "origin"
+//
+// --fallback // boolean. defaults to off
+//
+// When a fetch from the cache-server fails, automatically
+// fallback to the main Git server. This option has no effect
+// if no cache-server is defined.
+//
+// --cache-server=]