Merge pull request juju#18794 from jack-w-shaw/3.6-into-main

juju#18794

Merges:
- juju#18745
- juju#18762
- juju#18298
- juju#18772
- juju#18773
- juju#18706
- juju#18790
- juju#18793

Conflicts:
- Makefile
- cmd/juju/main.go
- cmd/jujuc/main.go
- cmd/jujud/main.go
- docs/conf.py
- docs/contributor/unsorted/high_availability.md
- scripts/md-gen/hook-commands/main.go
 
Resolutions:
- cmd/juju and cmd/jujuc were resolved with simple import fixes
- cmd/jujud needed to import the debug changes (from juju#18298)
- Makefile had some simple but non-trivial merge conflicts (fromhttps://github.com/juju/pull/18298)
- docs/ and scripts/ were trivial fixes

Makefile

@@ -78,13 +78,18 @@ OCI_IMAGE_PLATFORMS ?= linux/$(GOARCH)
 # Build tags passed to go install/build.
 # Passing no-dqlite will disable building with dqlite.
 # Example: BUILD_TAGS="minimal provider_kubernetes"
-BUILD_TAGS ?= 
+BUILD_TAGS ?=
 
 # EXTRA_BUILD_TAGS is not passed in, but built up from context.
 EXTRA_BUILD_TAGS =
 ifeq (,$(findstring no-dqlite,$(BUILD_TAGS)))
-EXTRA_BUILD_TAGS += libsqlite3
-EXTRA_BUILD_TAGS += dqlite
+    EXTRA_BUILD_TAGS += libsqlite3
+    EXTRA_BUILD_TAGS += dqlite
+endif
+
+# Enable coverage collection.
+ifneq ($(COVERAGE_COLLECT_URL),)
+    EXTRA_BUILD_TAGS += cover
 endif
 
 ifdef DEBUG_JUJU
@@ -159,33 +164,28 @@ endef
 
 # Windows doesn't support the agent binaries
 ifeq ($(GOOS), windows)
-	INSTALL_TARGETS = juju \
+    INSTALL_TARGETS = juju \
                       juju-metadata
 endif
 
 # We only add pebble to the list of install targets if we are building for linux
 ifeq ($(GOOS), linux)
-	INSTALL_TARGETS += jujud-controller
-	INSTALL_TARGETS += pebble
+    INSTALL_TARGETS += jujud-controller
+    INSTALL_TARGETS += pebble
 endif
 
 # Allow the tests to take longer on restricted platforms.
 ifeq ($(shell echo "${GOARCH}" | sed -E 's/.*(arm|arm64|ppc64le|ppc64|s390x).*/golang/'), golang)
-	TEST_TIMEOUT ?= 5400s
+    TEST_TIMEOUT ?= 5400s
 else
-	TEST_TIMEOUT ?= 2700s
+    TEST_TIMEOUT ?= 2700s
 endif
 TEST_TIMEOUT := $(TEST_TIMEOUT)
 
 TEST_ARGS ?=
 # Limit concurrency on s390x.
 ifeq ($(shell echo "${GOARCH}" | sed -E 's/.*(s390x).*/golang/'), golang)
-	TEST_ARGS += -p 4
-endif
-
-# Enable coverage testing.
-ifeq ($(COVERAGE_CHECK), 1)
-	TEST_ARGS += -coverprofile=coverage.txt -covermode=atomic
+    TEST_ARGS += -p 4
 endif
 
 ifeq ($(FUZZ_CHECK), 1)
@@ -196,24 +196,33 @@ endif
 
 # Enable verbose testing for reporting.
 ifeq ($(VERBOSE_CHECK), 1)
-	CHECK_ARGS = -v
+    CHECK_ARGS = -v
 endif
 
 define link_flags_version
 -X $(PROJECT)/core/version.GitCommit=$(GIT_COMMIT) \
 -X $(PROJECT)/core/version.GitTreeState=$(GIT_TREE_STATE) \
 -X $(PROJECT)/core/version.build=$(JUJU_BUILD_NUMBER) \
--X $(PROJECT)/core/version.GoBuildTags=$(FINAL_BUILD_TAGS)
+-X $(PROJECT)/core/version.GoBuildTags=$(FINAL_BUILD_TAGS) \
+-X $(PROJECT)/internal/debug/coveruploader.putURL=$(COVERAGE_COLLECT_URL)
 endef
 
+# Enable coverage collection.
+ifneq ($(COVERAGE_COLLECT_URL),)
+    COVER_COMPILE_FLAGS = -cover -covermode=atomic
+    COVER_LINK_FLAGS = -checklinkname=0
+    COVER_CGO_LINK_FLAGS = -checklinkname=0
+endif
+
 # Compile with debug flags if requested.
 ifdef DEBUG_JUJU
-    COMPILE_FLAGS = -gcflags "all=-N -l"
-    LINK_FLAGS =  "$(link_flags_version)"
-	CGO_LINK_FLAGS = "-linkmode 'external' -extldflags '-static' $(link_flags_version)"
+    COMPILE_FLAGS = $(COVER_COMPILE_FLAGS) -gcflags "all=-N -l"
+    LINK_FLAGS = $(COVER_LINK_FLAGS) "$(link_flags_version)"
+    CGO_LINK_FLAGS = $(COVER_CGO_LINK_FLAGS) "-linkmode 'external' -extldflags '-static' $(link_flags_version)"
 else
-    LINK_FLAGS = "-s -w -extldflags '-static' $(link_flags_version)"
-	CGO_LINK_FLAGS = "-s -w -linkmode 'external' -extldflags '-static' $(link_flags_version)"
+    COMPILE_FLAGS = $(COVER_COMPILE_FLAGS)
+    LINK_FLAGS = "$(COVER_LINK_FLAGS) -s -w -extldflags '-static' $(link_flags_version)"
+    CGO_LINK_FLAGS = "$(COVER_CGO_LINK_FLAGS) -s -w -linkmode 'external' -extldflags '-static' $(link_flags_version)"
 endif
 
 define DEPENDENCIES
@@ -468,6 +477,10 @@ race-test:
 ## race-test: Verify Juju code using unit tests with the race detector enabled
 	+make run-tests TEST_ARGS="$(TEST_ARGS) -race"
 
+.PHONY: cover-test
+cover-test:
+	+make run-tests TEST_ARGS="$(TEST_ARGS) -cover -covermode=atomic" TEST_EXTRA_ARGS="$(TEST_EXTRA_ARGS) -test.gocoverdir=${GOCOVERDIR}"
+
 .PHONY: run-tests run-go-tests go-test-alias
 # Can't make the length of the TMP dir too long or it hits socket name length issues.
 run-tests: musl-install-if-missing dqlite-install-if-missing
@@ -477,7 +490,7 @@ run-tests: musl-install-if-missing dqlite-install-if-missing
 	$(eval BUILD_ARCH = $(subst ppc64el,ppc64le,${ARCH}))
 	$(eval TMP := $(shell mktemp -d $${TMPDIR:-/tmp}/jj-XXX))
 	$(eval TEST_PACKAGES := $(shell go list $(PROJECT)/... | sort | ([ -f "$(TEST_PACKAGE_LIST)" ] && comm -12 "$(TEST_PACKAGE_LIST)" - || cat) | grep -v $(PROJECT)$$ | grep -v $(PROJECT)/vendor/ | grep -v $(PROJECT)/generate/ | grep -v mocks))
-	@echo 'go test -mod=$(JUJU_GOMOD_MODE) -tags=$(TEST_BUILD_TAGS) $(TEST_ARGS) $(CHECK_ARGS) -test.timeout=$(TEST_TIMEOUT) $$TEST_PACKAGES -check.v'
+	@echo 'go test -mod=$(JUJU_GOMOD_MODE) -tags=$(TEST_BUILD_TAGS) $(TEST_ARGS) $(CHECK_ARGS) -test.timeout=$(TEST_TIMEOUT) $$TEST_PACKAGES -check.v $(TEST_EXTRA_ARGS)'
 	@TMPDIR=$(TMP) \
 		PATH="${MUSL_BIN_PATH}:${PATH}" \
 		CC="musl-gcc" \
@@ -486,7 +499,7 @@ run-tests: musl-install-if-missing dqlite-install-if-missing
 		CGO_LDFLAGS_ALLOW="(-Wl,-wrap,pthread_create)|(-Wl,-z,now)" \
 		LD_LIBRARY_PATH="${DQLITE_EXTRACTED_DEPS_ARCHIVE_PATH}" \
 		CGO_ENABLED=1 \
-		go test -v -mod=$(JUJU_GOMOD_MODE) -tags=$(TEST_BUILD_TAGS) $(TEST_ARGS) $(CHECK_ARGS) -ldflags ${CGO_LINK_FLAGS} -test.timeout=$(TEST_TIMEOUT) $(TEST_PACKAGES) -check.v
+		go test -v -mod=$(JUJU_GOMOD_MODE) -tags=$(TEST_BUILD_TAGS) $(TEST_ARGS) $(CHECK_ARGS) -ldflags ${CGO_LINK_FLAGS} -test.timeout=$(TEST_TIMEOUT) $(TEST_PACKAGES) -check.v $(TEST_EXTRA_ARGS)
 	@rm -r $(TMP)
 
 run-go-tests: musl-install-if-missing dqlite-install-if-missing
@@ -496,15 +509,15 @@ run-go-tests: musl-install-if-missing dqlite-install-if-missing
 	$(eval BUILD_ARCH = $(subst ppc64el,ppc64le,${ARCH}))
 	$(eval TEST_PACKAGES ?= "./...")
 	$(eval TEST_FILTER ?= "")
-	@echo 'go test -mod=$(JUJU_GOMOD_MODE) -tags=$(TEST_BUILD_TAGS) $(TEST_ARGS) $(CHECK_ARGS) -test.timeout=$(TEST_TIMEOUT) $$TEST_PACKAGES -check.v -check.f $(TEST_FILTER)'
+	@echo 'go test -mod=$(JUJU_GOMOD_MODE) -tags=$(TEST_BUILD_TAGS) $(TEST_ARGS) $(CHECK_ARGS) -test.timeout=$(TEST_TIMEOUT) $$TEST_PACKAGES -check.v -check.f $(TEST_FILTER) $(TEST_EXTRA_ARGS)'
 	@PATH="${MUSL_BIN_PATH}:${PATH}" \
 		CC="musl-gcc" \
 		CGO_CFLAGS="-I${DQLITE_EXTRACTED_DEPS_ARCHIVE_PATH}/include" \
 		CGO_LDFLAGS="-L${DQLITE_EXTRACTED_DEPS_ARCHIVE_PATH} -luv -ldqlite -llz4 -lsqlite3 -Wl,-z,stack-size=1048576" \
 		CGO_LDFLAGS_ALLOW="(-Wl,-wrap,pthread_create)|(-Wl,-z,now)" \
 		LD_LIBRARY_PATH="${DQLITE_EXTRACTED_DEPS_ARCHIVE_PATH}" \
 		CGO_ENABLED=1 \
-		go test -v -mod=$(JUJU_GOMOD_MODE) -tags=$(TEST_BUILD_TAGS) $(TEST_ARGS) $(CHECK_ARGS) -ldflags ${CGO_LINK_FLAGS} -test.timeout=$(TEST_TIMEOUT) ${TEST_PACKAGES} -check.v -check.f $(TEST_FILTER)
+		go test -v -mod=$(JUJU_GOMOD_MODE) -tags=$(TEST_BUILD_TAGS) $(TEST_ARGS) $(CHECK_ARGS) -ldflags ${CGO_LINK_FLAGS} -test.timeout=$(TEST_TIMEOUT) ${TEST_PACKAGES} -check.v -check.f $(TEST_FILTER) $(TEST_EXTRA_ARGS)
 
 go-test-alias: musl-install-if-missing dqlite-install-if-missing
 ## go-test-alias: Prints out an alias command for easy running of tests.
@@ -555,12 +568,12 @@ rebuild-schema:
 	@echo "Generating facade schema..."
 # GOOS and GOARCH environment variables are cleared in case the user is trying to cross architecture compilation.
 ifdef SCHEMA_PATH
-	@env GOOS= GOARCH= CGO_ENABLED=1 go run -tags="libsqlite3" $(COMPILE_FLAGS) $(PROJECT)/generate/schemagen -admin-facades -facade-group=client "$(SCHEMA_PATH)/schema.json"
-	@env GOOS= GOARCH= CGO_ENABLED=1 go run -tags="libsqlite3" $(COMPILE_FLAGS) $(PROJECT)/generate/schemagen -admin-facades -facade-group=agent "$(SCHEMA_PATH)/agent-schema.json"
+	@env GOOS= GOARCH= CGO_ENABLED=1 go run -tags="libsqlite3" $(PROJECT)/generate/schemagen -admin-facades -facade-group=client "$(SCHEMA_PATH)/schema.json"
+	@env GOOS= GOARCH= CGO_ENABLED=1 go run -tags="libsqlite3" $(PROJECT)/generate/schemagen -admin-facades -facade-group=agent "$(SCHEMA_PATH)/agent-schema.json"
 else
-	@env GOOS= GOARCH= CGO_ENABLED=1 go run -tags="libsqlite3" $(COMPILE_FLAGS) $(PROJECT)/generate/schemagen -admin-facades -facade-group=client \
+	@env GOOS= GOARCH= CGO_ENABLED=1 go run -tags="libsqlite3" $(PROJECT)/generate/schemagen -admin-facades -facade-group=client \
 		./apiserver/facades/schema.json
-	@env GOOS= GOARCH= CGO_ENABLED=1 go run -tags="libsqlite3" $(COMPILE_FLAGS) $(PROJECT)/generate/schemagen -admin-facades -facade-group=agent \
+	@env GOOS= GOARCH= CGO_ENABLED=1 go run -tags="libsqlite3" $(PROJECT)/generate/schemagen -admin-facades -facade-group=agent \
 		./apiserver/facades/agent-schema.json
 endif
 
@@ -785,5 +798,3 @@ docs-%:
 ## docs-run: Build and serve the documentation
 ## docs-clean: Clean the docs build artifacts
 	cd docs && $(MAKE) -f Makefile.sp sp-$* ALLFILES='*.md **/*.md'
-
-

cmd/juju/main.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/juju/juju/cmd/juju/commands"
 	"github.com/juju/juju/internal/cmd"
+	"github.com/juju/juju/internal/debug/coveruploader"
 	_ "github.com/juju/juju/internal/provider/all" // Import the providers.
 )
 
@@ -20,6 +21,7 @@ func init() {
 }
 
 func main() {
+	coveruploader.Enable()
 	_, err := loggo.ReplaceDefaultWriter(cmd.NewWarningWriter(os.Stderr))
 	if err != nil {
 		panic(err)

cmd/jujuc/main.go

@@ -19,6 +19,7 @@ import (
 	"github.com/juju/utils/v4/exec"
 
 	"github.com/juju/juju/internal/cmd"
+	"github.com/juju/juju/internal/debug/coveruploader"
 	"github.com/juju/juju/internal/featureflag"
 	internallogger "github.com/juju/juju/internal/logger"
 	"github.com/juju/juju/juju/osenv"
@@ -170,6 +171,7 @@ func hookToolMain(commandName string, ctx *cmd.Context, args []string) (code int
 }
 
 func main() {
+	coveruploader.Enable()
 	os.Exit(Main(os.Args))
 }
 

cmd/jujud/main.go

@@ -5,8 +5,13 @@
 
 package main
 
-import "os"
+import (
+	"os"
+
+	"github.com/juju/juju/internal/debug/coveruploader"
+)
 
 func main() {
+	coveruploader.Enable()
 	os.Exit(Main(os.Args))
 }

docs/conf.py

@@ -107,7 +107,8 @@
     # Product page URL; can be different from product docs URL
     "product_page": "juju.is",
     # Product tag image; the orange part of your logo, shown in the page header
-    'product_tag': 'logos/juju-logo.png',
+    # Assumes the current directory is .sphinx.
+    'product_tag': '_static/logos/juju-logo-no-text.png',
     # Your Discourse instance URL
     "discourse": "https://discourse.charmhub.com",
     # Your Mattermost channel URL
@@ -379,6 +380,46 @@ def generate_model_config_docs():
 
     print("generated model config key list")
 
+def generate_hook_command_docs():
+    hook_commands_reference_dir = 'user/reference/hook-commands/'
+    generated_hook_commands_dir = hook_commands_reference_dir + 'list-of-hook-commands/'
+    hook_index_header = hook_commands_reference_dir + 'hook_index'
+
+    # Remove existing hook command folder to regenerate it
+    if os.path.exists(generated_hook_commands_dir):
+        shutil.rmtree(generated_hook_commands_dir)
+
+    # Generate the hook commands doc using script.
+    result = subprocess.run(['go', 'run', '../scripts/md-gen/hook-commands/main.go', generated_hook_commands_dir],
+                            check=True)
+    if result.returncode != 0:
+        raise Exception("error auto-generating hook commands: " + result.stderr)
+
+    # Remove 'help' and 'documentaion' files as they are not needed.
+    if os.path.exists(generated_hook_commands_dir + 'help.md'):
+        os.remove(generated_hook_commands_dir + 'help.md')
+    if os.path.exists(generated_hook_commands_dir + 'documentation.md'):
+        os.remove(generated_hook_commands_dir + 'documentation.md')
+
+    for page in os.listdir(generated_hook_commands_dir):
+        title = "`" + page[:-3] + "`"
+        anchor = "hook-command-" + page[:-3]
+        # Add sphinx names to each file.
+        with open(os.path.join(generated_hook_commands_dir, page), 'r+') as mdfile:
+            content = mdfile.read()
+            # Remove trailing seperated (e.g. ----)
+            content = content.rstrip(" -\n")
+            mdfile.seek(0, 0)
+            mdfile.write('(' + anchor + ')=\n' +
+                         '# ' + title + '\n' +
+                         content)
+
+    # Add in the index file containing the command list.
+    subprocess.run(['cp', hook_index_header, generated_hook_commands_dir + 'index.md'])
+
+    print("generated hook command list")
+
 generate_cli_docs()
 generate_controller_config_docs()
 generate_model_config_docs()
+generate_hook_command_docs()

docs/contributor/unsorted/conventional-commits.md

@@ -1,8 +1,10 @@
 (conventional-commits)=
+# Conventional commits
+
 The [Conventional Commits standard](https://www.conventionalcommits.org/en/v1.0.0/) is adopted for the Juju project with the
 following structure:
 
-```  
+```
 <type>(optional <scope>): <description>
 
 [optional body]
@@ -55,14 +57,14 @@ feat(api): add user authentication feature
 
 # scope: Indicates the part of the codebase affected
 # Here, 'api' is specified as the scope
-  
+
 # description: A brief summary of the change in lower-case
 # "add user authentication feature" summarizes the change concisely
 
 This commit adds user authentication to the API. Users can now sign up,
 log in, and log out. Passwords are hashed using bcrypt. Token-based 
 authentication is implemented using JWT.
-  
+
 # body: Detailed explanation of the change
 # A more detailed explanation of what has been changed and why
 

docs/contributor/unsorted/cross-platform-development.md

@@ -1,5 +1,6 @@
 (cross-platform-development)=
-## Introduction
+# Cross-platform development
+
 It's often common practice that a juju developer needs to test out new juju code
 on machines that are not the same operating system or architecture to that of
 their current host. Alternatively in this workflow it is common place to want to
@@ -89,7 +90,7 @@ juju upgrade-controller --agent-version <version>
 number from above.**
 
 ## Current limitations & future changes
-- Does not support oci artifacts including upgrading of OCI deployed controllers 
+- Does not support oci artifacts including upgrading of OCI deployed controllers
   and agents.
 - Would like to integrate work by @hpidcock for a simplestreams server that
   continuously builds and updates.

docs/contributor/unsorted/death-and-destruction.md

@@ -1,6 +1,5 @@
 (death-and-destruction)=
-Death and Destruction
-=====================
+# Death and destruction
 
 This document describes in detail the operations associated with the destruction
 and removal of the fundamental state entities, and what agents are responsible

docs/contributor/unsorted/entity-creation.md

@@ -1,6 +1,6 @@
 (entity-creation)=
-Entity Creation
-===============
+# Entity creation
+
 
 This document describes the circumstances in which fundamental state entities
 are created, from the perspective of the CLI.

docs/contributor/unsorted/fixing-bugs.md

@@ -1,6 +1,5 @@
 (fixing-bugs)=
-juju and Bug Fixes
-==================
+# Juju and bug fixes
 
 When fixing a bug that affects more than a single branch the prefered
 process for fixing the bug is to perform the work against the oldest
@@ -13,8 +12,8 @@ you would use the pull request and review process.
 Then you would apply your fix to R2, and finally once your fix
 was merged in to R2, you would apply the fix to current master.
 
-How To Backport
-================
+## How to backport
+
 Once your initial fix has been merged by the bot (the back port) the process
 of applying your patch can be done with a few git commands. In this example I will show
 you applying the fix in to master.