Fix devhost multi machine

.gitignore

@@ -1,6 +1,9 @@
+*.orig
 *.pyc
+*.status
 *.sublime-*
 *src/batou.egg-info/
+.appenv
 .batou
 .batou-lock
 .coverage
@@ -10,11 +13,15 @@
 .vagrant
 /.tox
 /bin
+/build/
 /develop-eggs
 /doc/build
 /eggs
-/examples/*/work
+/examples/**/hostmap.json
+/examples/**/insecure-private.key
 /examples/*/.appenv
+/examples/*/ssh_config_*
+/examples/*/work
 /htmlcov
 /include
 /lib
@@ -25,12 +32,6 @@
 /src/batou/secrets/tests/fixture/gnupg/random_seed
 __pycache__
 pip-selfcheck.json
+pip-wheel-metadata
 pyvenv.cfg
 report.xml
-pip-wheel-metadata
-/examples/*/ssh_config_*
-/examples/**/hostmap.json
-/examples/**/insecure-private.key
-*.status
-*.orig
-/build/

CHANGES.d/20250121_145243_ct_fix_devhost_multi_machine.md

@@ -0,0 +1,2 @@
+- Correctly record all hosts in the SSH config file when provisioning
+  with the devhost provisioner.

src/batou/conftest.py

@@ -1,4 +1,6 @@
 import os.path
+import subprocess
+import tempfile
 
 import pytest
 
@@ -31,3 +33,18 @@ def reset_address_defaults():
     v4, v6 = batou.utils.Address.require_v4, batou.utils.Address.require_v6
     yield
     batou.utils.Address.require_v4, batou.utils.Address.require_v6 = v4, v6
+
+
+@pytest.fixture(scope="session")
+def git_main_branch() -> str:
+    if initial_branch := subprocess.check_output(
+        ["git", "config", "get", "init.defaultbranch"]
+    ).decode("ascii"):
+        return initial_branch
+    with tempfile.TemporaryDirectory() as tmpdir:
+        subprocess.check_call("git init .", shell=True)
+        return (
+            subprocess.check_output(["git", "branch", "--show-current"])
+            .decode("ascii")
+            .strip()
+        )

src/batou/lib/git.py

@@ -26,14 +26,14 @@ def exactly_one(*args):
 
 
 class Clone(Component):
-
     namevar = "url"
     target = "."
     revision = None
     branch = None
     tag = None
     vcs_update = True
     clobber = False
+    url: str
 
     def configure(self):
         if not exactly_one(self.revision, self.branch, self.tag):

src/batou/lib/tests/test_git.py

@@ -29,9 +29,9 @@ def repos_path2(root, name="upstream2"):
 
 
 @pytest.mark.slow
-def test_runs_git_to_clone_repository(root, repos_path):
+def test_runs_git_to_clone_repository(root, repos_path, git_main_branch):
     root.component += batou.lib.git.Clone(
-        repos_path, target="clone", branch="master"
+        repos_path, target="clone", branch=git_main_branch
     )
     root.component.deploy()
     assert os.path.isfile(
@@ -65,9 +65,11 @@ def test_directly_after_clone_nothing_is_merged(root, repos_path):
 
 
 @pytest.mark.slow
-def test_setting_branch_updates_on_incoming_changes(root, repos_path):
+def test_setting_branch_updates_on_incoming_changes(
+    root, repos_path, git_main_branch
+):
     root.component += batou.lib.git.Clone(
-        repos_path, target="clone", branch="master"
+        repos_path, target="clone", branch=git_main_branch
     )
     root.component.deploy()
     cmd(
@@ -149,8 +151,12 @@ def test_tag_does_switch_tag(root, repos_path):
 
 
 @pytest.mark.slow
-def test_has_changes_counts_changes_to_tracked_files(root, repos_path):
-    clone = batou.lib.git.Clone(repos_path, target="clone", branch="master")
+def test_has_changes_counts_changes_to_tracked_files(
+    root, repos_path, git_main_branch
+):
+    clone = batou.lib.git.Clone(
+        repos_path, target="clone", branch=git_main_branch
+    )
     root.component += clone
     root.component.deploy()
     assert not clone.has_changes()
@@ -160,8 +166,12 @@ def test_has_changes_counts_changes_to_tracked_files(root, repos_path):
 
 
 @pytest.mark.slow
-def test_has_changes_counts_untracked_files_as_changes(root, repos_path):
-    clone = batou.lib.git.Clone(repos_path, target="clone", branch="master")
+def test_has_changes_counts_untracked_files_as_changes(
+    root, repos_path, git_main_branch
+):
+    clone = batou.lib.git.Clone(
+        repos_path, target="clone", branch=git_main_branch
+    )
     root.component += clone
     root.component.deploy()
     assert not clone.has_changes()
@@ -170,9 +180,13 @@ def test_has_changes_counts_untracked_files_as_changes(root, repos_path):
 
 
 @pytest.mark.slow
-def test_clean_clone_updates_on_incoming_changes(root, repos_path):
+def test_clean_clone_updates_on_incoming_changes(
+    root, repos_path, git_main_branch
+):
     root.component += batou.lib.git.Clone(
-        repos_path, target="clone", branch="master"
+        repos_path,
+        target="clone",
+        branch=git_main_branch,
     )
     root.component.deploy()
     cmd(
@@ -185,9 +199,11 @@ def test_clean_clone_updates_on_incoming_changes(root, repos_path):
 
 
 @pytest.mark.slow
-def test_no_clobber_changes_protected_on_update_with_incoming(root, repos_path):
+def test_no_clobber_changes_protected_on_update_with_incoming(
+    root, repos_path, git_main_branch
+):
     root.component += batou.lib.git.Clone(
-        repos_path, target="clone", branch="master"
+        repos_path, target="clone", branch=git_main_branch
     )
     root.component.deploy()
     cmd(
@@ -205,10 +221,10 @@ def test_no_clobber_changes_protected_on_update_with_incoming(root, repos_path):
 
 @pytest.mark.slow
 def test_no_clobber_changes_protected_on_update_without_incoming(
-    root, repos_path
+    root, repos_path, git_main_branch
 ):
     root.component += batou.lib.git.Clone(
-        repos_path, target="clone", branch="master"
+        repos_path, target="clone", branch=git_main_branch
     )
     root.component.deploy()
     cmd("cd {dir}/clone; echo foobar >foo".format(dir=root.workdir))
@@ -220,9 +236,11 @@ def test_no_clobber_changes_protected_on_update_without_incoming(
 
 
 @pytest.mark.slow
-def test_no_clobber_untracked_files_are_kept_on_update(root, repos_path):
+def test_no_clobber_untracked_files_are_kept_on_update(
+    root, repos_path, git_main_branch
+):
     root.component += batou.lib.git.Clone(
-        repos_path, target="clone", branch="master"
+        repos_path, target="clone", branch=git_main_branch
     )
     root.component.deploy()
     cmd(
@@ -238,9 +256,11 @@ def test_no_clobber_untracked_files_are_kept_on_update(root, repos_path):
 
 
 @pytest.mark.slow
-def test_clobber_changes_lost_on_update_with_incoming(root, repos_path):
+def test_clobber_changes_lost_on_update_with_incoming(
+    root, repos_path, git_main_branch
+):
     root.component += batou.lib.git.Clone(
-        repos_path, target="clone", branch="master", clobber=True
+        repos_path, target="clone", branch=git_main_branch, clobber=True
     )
     root.component.deploy()
     cmd(
@@ -256,9 +276,11 @@ def test_clobber_changes_lost_on_update_with_incoming(root, repos_path):
 
 
 @pytest.mark.slow
-def test_clobber_changes_lost_on_update_without_incoming(root, repos_path):
+def test_clobber_changes_lost_on_update_without_incoming(
+    root, repos_path, git_main_branch
+):
     root.component += batou.lib.git.Clone(
-        repos_path, target="clone", branch="master", clobber=True
+        repos_path, target="clone", branch=git_main_branch, clobber=True
     )
     root.component.deploy()
     cmd("cd {dir}/clone; echo foobar >foo".format(dir=root.workdir))
@@ -268,9 +290,11 @@ def test_clobber_changes_lost_on_update_without_incoming(root, repos_path):
 
 
 @pytest.mark.slow
-def test_clobber_untracked_files_are_removed_on_update(root, repos_path):
+def test_clobber_untracked_files_are_removed_on_update(
+    root, repos_path, git_main_branch
+):
     root.component += batou.lib.git.Clone(
-        repos_path, target="clone", branch="master", clobber=True
+        repos_path, target="clone", branch=git_main_branch, clobber=True
     )
     root.component.deploy()
     cmd(
@@ -283,9 +307,11 @@ def test_clobber_untracked_files_are_removed_on_update(root, repos_path):
 
 
 @pytest.mark.slow
-def test_clean_clone_vcs_update_false_leaves_changes_intact(root, repos_path):
+def test_clean_clone_vcs_update_false_leaves_changes_intact(
+    root, repos_path, git_main_branch
+):
     root.component += batou.lib.git.Clone(
-        repos_path, target="clone", branch="master", vcs_update=False
+        repos_path, target="clone", branch=git_main_branch, vcs_update=False
     )
     root.component.deploy()
     cmd(
@@ -300,8 +326,12 @@ def test_clean_clone_vcs_update_false_leaves_changes_intact(root, repos_path):
 
 
 @pytest.mark.slow
-def test_changed_remote_is_updated(root, repos_path, repos_path2):
-    git = batou.lib.git.Clone(repos_path, target="clone", branch="master")
+def test_changed_remote_is_updated(
+    root, repos_path, repos_path2, git_main_branch
+):
+    git = batou.lib.git.Clone(
+        repos_path, target="clone", branch=git_main_branch
+    )
     root.component += git
 
     # Fresh, unrelated repo
@@ -318,8 +348,10 @@ def test_changed_remote_is_updated(root, repos_path, repos_path2):
 
 
 @pytest.mark.slow
-def test_clone_into_workdir_works(root, repos_path, repos_path2):
-    git = batou.lib.git.Clone(repos_path, branch="master")
+def test_clone_into_workdir_works(
+    root, repos_path, repos_path2, git_main_branch
+):
+    git = batou.lib.git.Clone(repos_path, branch=git_main_branch)
     with open(root.component.map("asdf"), "w") as f:
         f.write("foo")
     root.component += git
@@ -328,7 +360,7 @@ def test_clone_into_workdir_works(root, repos_path, repos_path2):
 
 
 @pytest.mark.slow
-def test_can_read_untracked_files_correctly(root, repos_path):
+def test_can_read_untracked_files_correctly(root, repos_path, git_main_branch):
     fun_strings = [
         "this string has spaces in it",
         "this string has a ' in it",
@@ -338,7 +370,7 @@ def test_can_read_untracked_files_correctly(root, repos_path):
         "this string has a \\ and a ' in it",
         "this string has a \\ and a & in it",
     ]
-    git = batou.lib.git.Clone(repos_path, branch="master")
+    git = batou.lib.git.Clone(repos_path, branch=git_main_branch)
     root.component += git
     root.component.deploy()
     for s in fun_strings:

src/batou/provision.py

@@ -34,6 +34,10 @@ class FCDevProvisioner(Provisioner):
     target_host = None
     aliases = ()
 
+    def __init__(self, name):
+        super().__init__(name)
+        self._known_ssh_hosts = {}
+
     def _prepare_ssh(self, host):
         # XXX application / user-specific files
         # https://unix.stackexchange.com/questions/312988/understanding-home-configuration-file-locations-config-and-local-sha
@@ -61,7 +65,7 @@ def _prepare_ssh(self, host):
                 f_target.write(f_packaged.read())
         os.chmod(local_insecure_key, 0o600)
 
-        ssh_config.append(
+        self._known_ssh_hosts[host.name] = (
             """
 Host {hostname} {aliases}
     HostName {hostname}
@@ -79,6 +83,11 @@ def _prepare_ssh(self, host):
             )
         )
 
+        # Gather all known hosts together - otherwise we can only access
+        # one provisioned host per environment.
+        for section in self._known_ssh_hosts.values():
+            ssh_config.append(section)
+
         # More generic includes need to go last because parameters are
         # set by a match-first and things like User settings for * may
         # otherwise collide with our very specific settings. See
@@ -151,10 +160,10 @@ def provision(self, host):
 
         rsync_path = ""
         if host.environment.service_user:
-            rsync_path = (
-                f'--rsync-path="sudo -u {host.environment.service_user} '
-                f'rsync"'
-            )
+            user_prefix = f"sudo -u {host.environment.service_user}"
+            ssh_cmd_prefix = user_prefix
+            rsync_path = f'--rsync-path="{user_prefix} rsync"'
+
         env = self._initial_provision_env(host)
         env["SSH_CONFIG"] = self.ssh_config_file
         env["RSYNC_RSH"] = "ssh -F {}".format(self.ssh_config_file)
@@ -245,6 +254,7 @@ def provision(self, host):
                     self.SEED_TEMPLATE.format(
                         seed_script=seed_script,
                         rsync_path=rsync_path,
+                        ssh_cmd_prefix=ssh_cmd_prefix,
                         ENV=batou.utils.export_environment_variables(env),
                     )
                 )
@@ -304,13 +314,13 @@ class FCDevContainer(FCDevProvisioner):
 
 RUN() {{
     cmd=$@
-    ssh -F $SSH_CONFIG $PROVISION_CONTAINER "$cmd"
+    ssh -F $SSH_CONFIG $PROVISION_CONTAINER "{ssh_cmd_prefix} $cmd"
 }}
 
 COPY() {{
     what=${{1?what to copy}}
     where=${{2?where to copy}}
-    rsync -avz --no-l --safe-links {rsync_path} $what $PROVISION_CONTAINER:$where
+    rsync -avz --no-owner --no-group --no-l --safe-links {rsync_path} $what $PROVISION_CONTAINER:$where
 }}
 
 if [ ${{PROVISION_REBUILD+x}} ]; then
@@ -385,13 +395,13 @@ class FCDevVM(FCDevProvisioner):
 
 RUN() {{
     cmd=$@
-    ssh -F $SSH_CONFIG $PROVISION_VM "$cmd"
+    ssh -F $SSH_CONFIG $PROVISION_VM "{ssh_cmd_prefix} $cmd"
 }}
 
 COPY() {{
     what=${{1?what to copy}}
     where=${{2?where to copy}}
-    rsync -avz --no-l --safe-links {rsync_path} $what $PROVISION_VM:$where
+    rsync -avz --no-owner --no-group --no-l --safe-links {rsync_path} $what $PROVISION_VM:$where
 }}
 
 if [ ${{PROVISION_REBUILD+x}} ]; then