From dbc6f44567ea706d7363a79411c9089c185414e4 Mon Sep 17 00:00:00 2001 From: Fyodor Soikin Date: Wed, 28 Aug 2024 12:58:09 -0400 Subject: [PATCH] Clone Git monorepos with several dependencies only once (#1275) --- CHANGELOG.md | 2 + core/src/Log.purs | 12 +- core/src/Prelude.purs | 2 +- src/Spago/Command/Fetch.purs | 64 ++++++---- src/Spago/Git.purs | 17 ++- src/Spago/Paths.purs | 6 + src/Spago/Prelude.purs | 2 +- test-fixtures/circular-dependencies.txt | 1 - .../expected-stderr/four-deps.txt | 7 ++ .../expected-stderr/three-deps-offline.txt | 6 + .../expected-stderr/three-deps.txt | 8 ++ .../expected-stderr/two-deps.txt | 8 ++ .../library/lib1/spago.yaml | 4 + .../library/lib2/spago.yaml | 5 + .../library/lib3/spago.yaml | 6 + .../library/lib4/spago.yaml | 5 + .../1208-no-double-cloning/library/spago.yaml | 3 + .../spago-four-deps.yaml | 27 +++++ .../spago-three-deps.yaml | 22 ++++ .../spago-two-deps.yaml | 17 +++ test/Prelude.purs | 18 ++- test/Spago/Build/Monorepo.purs | 113 ++++++++++++++++++ 22 files changed, 324 insertions(+), 31 deletions(-) create mode 100644 test-fixtures/monorepo/1208-no-double-cloning/expected-stderr/four-deps.txt create mode 100644 test-fixtures/monorepo/1208-no-double-cloning/expected-stderr/three-deps-offline.txt create mode 100644 test-fixtures/monorepo/1208-no-double-cloning/expected-stderr/three-deps.txt create mode 100644 test-fixtures/monorepo/1208-no-double-cloning/expected-stderr/two-deps.txt create mode 100644 test-fixtures/monorepo/1208-no-double-cloning/library/lib1/spago.yaml create mode 100644 test-fixtures/monorepo/1208-no-double-cloning/library/lib2/spago.yaml create mode 100644 test-fixtures/monorepo/1208-no-double-cloning/library/lib3/spago.yaml create mode 100644 test-fixtures/monorepo/1208-no-double-cloning/library/lib4/spago.yaml create mode 100644 test-fixtures/monorepo/1208-no-double-cloning/library/spago.yaml create mode 100644 test-fixtures/monorepo/1208-no-double-cloning/spago-four-deps.yaml create mode 100644 test-fixtures/monorepo/1208-no-double-cloning/spago-three-deps.yaml create mode 100644 test-fixtures/monorepo/1208-no-double-cloning/spago-two-deps.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index d194a6ed0..d6d6646de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,8 @@ Other improvements: overwrite the file if the marker isn't present, assuming that the file was manually created or edited, not generated by Spago itself. - migrated tests to the `spec-node` runner. +- when multiple dependencies share a monorepo, that repo is cloned only once and + cached locally. - `spago publish` now allows to publish a package with some test (but only test!) dependencies not present in the registry. diff --git a/core/src/Log.purs b/core/src/Log.purs index 8951f6f40..ea7eb4694 100644 --- a/core/src/Log.purs +++ b/core/src/Log.purs @@ -19,6 +19,8 @@ module Spago.Log , module DodoExport , output , prepareToDie + , rightOrDie + , rightOrDie_ , rightOrDieWith , rightOrDieWith' , toDoc @@ -180,14 +182,20 @@ justOrDieWith' value msg = case value of Nothing -> die' msg -rightOrDieWith :: forall a b m err x. MonadEffect m => MonadAsk (LogEnv b) m => Loggable a => Either err x -> (err -> a) -> m x +rightOrDie :: ∀ b m err x. MonadEffect m => MonadAsk (LogEnv b) m => Loggable err => Either err x -> m x +rightOrDie value = rightOrDieWith value identity + +rightOrDie_ :: ∀ b m err x. MonadEffect m => MonadAsk (LogEnv b) m => Loggable err => Either err x -> m Unit +rightOrDie_ = void <<< rightOrDie + +rightOrDieWith :: ∀ a b m err x. MonadEffect m => MonadAsk (LogEnv b) m => Loggable a => Either err x -> (err -> a) -> m x rightOrDieWith value toMsg = case value of Right a -> pure a Left err -> die $ toMsg err -rightOrDieWith' :: forall a b m err x. MonadEffect m => MonadAsk (LogEnv b) m => Loggable a => Either err x -> (err -> Array a) -> m x +rightOrDieWith' :: ∀ a b m err x. MonadEffect m => MonadAsk (LogEnv b) m => Loggable a => Either err x -> (err -> Array a) -> m x rightOrDieWith' value toMsg = case value of Right a -> pure a diff --git a/core/src/Prelude.purs b/core/src/Prelude.purs index d490731fc..5047a48d2 100644 --- a/core/src/Prelude.purs +++ b/core/src/Prelude.purs @@ -48,7 +48,7 @@ import Partial.Unsafe (unsafeCrashWith) import Registry.ManifestIndex (ManifestIndex) as Extra import Registry.Types (PackageName, Version, Range, Location, License, Manifest(..), Metadata(..), Sha256) as Extra import Spago.Json (printJson, parseJson) as Extra -import Spago.Log (logDebug, logError, logInfo, Docc, logSuccess, logWarn, die, die', justOrDieWith, justOrDieWith', rightOrDieWith, rightOrDieWith', toDoc, indent, indent2, output, LogEnv, LogOptions, OutputFormat(..)) as Extra +import Spago.Log (logDebug, logError, logInfo, Docc, logSuccess, logWarn, die, die', justOrDieWith, justOrDieWith', rightOrDie, rightOrDie_, rightOrDieWith, rightOrDieWith', toDoc, indent, indent2, output, LogEnv, LogOptions, OutputFormat(..)) as Extra import Spago.Yaml (YamlDoc, printYaml, parseYaml) as Extra newtype Spago env a = Spago (ReaderT env Extra.Aff a) diff --git a/src/Spago/Command/Fetch.purs b/src/Spago/Command/Fetch.purs index 8c249404d..410e85bd2 100644 --- a/src/Spago/Command/Fetch.purs +++ b/src/Spago/Command/Fetch.purs @@ -429,29 +429,49 @@ toAllDependencies = getGitPackageInLocalCache :: forall a. PackageName -> GitPackage -> Spago (Git.GitEnv a) Unit getGitPackageInLocalCache name package = do + ensureRepoCloned + ensureRefPresent + let localPackageLocation = Config.getPackageLocation name (GitPackage package) - tempDir <- mkTemp' (Just $ printJson Config.gitPackageCodec package) - logDebug $ "Cloning repo in " <> tempDir - Git.fetchRepo package tempDir >>= case _ of - Left err -> die err - Right _ -> do - logDebug $ "Repo cloned. Moving to " <> localPackageLocation - FS.mkdirp $ Path.concat [ Paths.localCachePackagesPath, PackageName.print name ] - FS.moveSync { src: tempDir, dst: localPackageLocation } - - -- Note: the package might have been cloned with a tag, but we stick the commit hash in the lockfiles - -- so we need to make a copy to a location that has the commit hash too. - -- So we run getRef here and then do a copy if the ref is different than the original one - -- (since it might be a commit to start with) - logDebug $ "Checking if we need to copy the package to a commit hash location..." - Git.getRef (Just localPackageLocation) >>= case _ of - Left err -> die err - Right ref -> do - when (ref /= package.ref) do - let commitHashLocation = Config.getPackageLocation name (GitPackage $ package { ref = ref }) - logDebug $ "Copying the repo also to " <> commitHashLocation - FS.mkdirp $ Path.concat [ Paths.localCachePackagesPath, PackageName.print name ] - FS.copyTree { src: localPackageLocation, dst: commitHashLocation } + logDebug $ "Copying repo to " <> localPackageLocation + FS.mkdirp $ Path.concat [ Paths.localCachePackagesPath, PackageName.print name ] + FS.copyTree { src: repoCacheLocation, dst: localPackageLocation } + logDebug $ "Checking out ref '" <> package.ref <> "'" + Git.checkout { repo: localPackageLocation, ref: package.ref } >>= rightOrDie_ + + -- Note: the package might have been cloned with a tag, but we stick the commit hash in the lockfiles + -- so we need to make a copy to a location that has the commit hash too. + -- So we run getRef here and then do a copy if the ref is different than the original one + -- (since it might be a commit to start with) + logDebug $ "Checking if we need to copy the package to a commit hash location..." + commitHash <- Git.getRef (Just localPackageLocation) >>= rightOrDie + when (commitHash /= package.ref) do + let commitHashLocation = Config.getPackageLocation name (GitPackage $ package { ref = commitHash }) + logDebug $ "Copying the repo also to " <> commitHashLocation + FS.copyTree { src: localPackageLocation, dst: commitHashLocation } + where + repoCacheLocation = Path.concat [ Paths.localCacheGitPath, Config.fileSystemCharEscape package.git ] + + ensureRepoCloned = unlessM (FS.exists repoCacheLocation) do + tempDir <- mkTemp' (Just $ printJson Config.gitPackageCodec package) + logDebug $ "Cloning repo in " <> tempDir + Git.fetchRepo package tempDir >>= rightOrDie_ + + logDebug $ "Repo cloned. Moving to " <> repoCacheLocation + FS.mkdirp $ Path.concat [ Paths.localCachePackagesPath, PackageName.print name ] + FS.moveSync { src: tempDir, dst: repoCacheLocation } + + ensureRefPresent = do + logDebug $ "Verifying ref " <> package.ref + { offline } <- ask + Git.getRefType { repo: repoCacheLocation, ref: package.ref } >>= case _, offline of + Right _, _ -> + pure unit + Left _, Offline -> + die $ "Repo " <> package.git <> " does not have ref " <> package.ref <> " in local cache. Cannot pull from origin in offline mode." + Left _, Online -> do + logDebug $ "Ref " <> package.ref <> " is not present, trying to pull from origin" + Git.fetch { repo: repoCacheLocation, remote: "origin" } >>= rightOrDie_ getPackageDependencies :: forall a. PackageName -> Package -> Spago (FetchEnv a) (Maybe (ByEnv (Map PackageName Range))) getPackageDependencies packageName package = case package of diff --git a/src/Spago/Git.purs b/src/Spago/Git.purs index eadcac8ed..713929328 100644 --- a/src/Spago/Git.purs +++ b/src/Spago/Git.purs @@ -6,6 +6,9 @@ module Spago.Git , getRef , getRemotes , getStatus + , checkout + , fetch + , getRefType , isIgnored , listTags , parseRemote @@ -48,7 +51,7 @@ runGit args cwd = ExceptT do Right r -> Right r.stdout Left r -> Left r.stderr -fetchRepo :: forall a b. { git :: String, ref :: String | a } -> FilePath -> Spago (GitEnv b) (Either (Array String) Unit) +fetchRepo :: ∀ a b. { git :: String, ref :: String | a } -> FilePath -> Spago (GitEnv b) (Either (Array String) Unit) fetchRepo { git, ref } path = do repoExists <- FS.exists path { offline } <- ask @@ -91,6 +94,18 @@ fetchRepo { git, ref } path = do logDebug $ "Successfully fetched the repo '" <> git <> "' at ref '" <> ref <> "'" pure $ Right unit +checkout :: ∀ a. { repo :: String, ref :: String } -> Spago (GitEnv a) (Either String Unit) +checkout { repo, ref } = Except.runExceptT $ void $ runGit [ "checkout", ref ] (Just repo) + +fetch :: ∀ a. { repo :: String, remote :: String } -> Spago (GitEnv a) (Either String Unit) +fetch { repo, remote } = do + remoteUrl <- runGit [ "remote", "get-url", remote ] (Just repo) # Except.runExceptT >>= rightOrDie + logInfo $ "Fetching from " <> remoteUrl + Except.runExceptT $ runGit_ [ "fetch", remote, "--tags" ] (Just repo) + +getRefType :: ∀ a. { repo :: String, ref :: String } -> Spago (GitEnv a) (Either String String) +getRefType { repo, ref } = Except.runExceptT $ runGit [ "cat-file", "-t", ref ] (Just repo) + listTags :: forall a. Maybe FilePath -> Spago (GitEnv a) (Either Docc (Array String)) listTags cwd = do let opts = Cmd.defaultExecOptions { pipeStdout = false, pipeStderr = false, cwd = cwd } diff --git a/src/Spago/Paths.purs b/src/Spago/Paths.purs index 224b004f3..e2b3ccc8a 100644 --- a/src/Spago/Paths.purs +++ b/src/Spago/Paths.purs @@ -32,12 +32,18 @@ localCachePath = toLocalCachePath cwd localCachePackagesPath :: FilePath localCachePackagesPath = toLocalCachePackagesPath cwd +localCacheGitPath :: FilePath +localCacheGitPath = toLocalCacheGitPath cwd + toLocalCachePath :: FilePath -> FilePath toLocalCachePath rootDir = Path.concat [ rootDir, ".spago" ] toLocalCachePackagesPath :: FilePath -> FilePath toLocalCachePackagesPath rootDir = Path.concat [ toLocalCachePath rootDir, "p" ] +toLocalCacheGitPath :: FilePath -> FilePath +toLocalCacheGitPath rootDir = Path.concat [ toLocalCachePath rootDir, "g" ] + registryPath ∷ FilePath registryPath = Path.concat [ globalCachePath, "registry" ] diff --git a/src/Spago/Prelude.purs b/src/Spago/Prelude.purs index 419661d5d..cda29c1c3 100644 --- a/src/Spago/Prelude.purs +++ b/src/Spago/Prelude.purs @@ -159,7 +159,7 @@ mkTemp' maybeSuffix = liftAff do sha <- Sha256.hashString $ show now <> fromMaybe "" maybeSuffix shaToHex sha -- Return the dir, but don't make it - that's the responsibility of the client - let tempDirPath = Path.concat [ Paths.paths.temp, random ] + let tempDirPath = Path.concat [ Paths.paths.temp, String.drop 50 random ] pure tempDirPath mkTemp :: forall m. MonadAff m => m FilePath diff --git a/test-fixtures/circular-dependencies.txt b/test-fixtures/circular-dependencies.txt index 2c24d9d17..ab2aa236e 100644 --- a/test-fixtures/circular-dependencies.txt +++ b/test-fixtures/circular-dependencies.txt @@ -2,7 +2,6 @@ Reading Spago workspace configuration... ✓ Selecting package to build: bbb -Cloning https://github.com/purescript/spago.git Cloning https://github.com/purescript/spago.git ✘ The following packages have circular dependencies: diff --git a/test-fixtures/monorepo/1208-no-double-cloning/expected-stderr/four-deps.txt b/test-fixtures/monorepo/1208-no-double-cloning/expected-stderr/four-deps.txt new file mode 100644 index 000000000..a5a63a2bb --- /dev/null +++ b/test-fixtures/monorepo/1208-no-double-cloning/expected-stderr/four-deps.txt @@ -0,0 +1,7 @@ +Reading Spago workspace configuration... + +✓ Selecting package to build: consumer + +Downloading dependencies... +No lockfile found, generating it... +Lockfile written to spago.lock. Please commit this file. diff --git a/test-fixtures/monorepo/1208-no-double-cloning/expected-stderr/three-deps-offline.txt b/test-fixtures/monorepo/1208-no-double-cloning/expected-stderr/three-deps-offline.txt new file mode 100644 index 000000000..5a8cfe2d4 --- /dev/null +++ b/test-fixtures/monorepo/1208-no-double-cloning/expected-stderr/three-deps-offline.txt @@ -0,0 +1,6 @@ +Reading Spago workspace configuration... + +✓ Selecting package to build: consumer + + +✘ Repo does not have ref v3 in local cache. Cannot pull from origin in offline mode. diff --git a/test-fixtures/monorepo/1208-no-double-cloning/expected-stderr/three-deps.txt b/test-fixtures/monorepo/1208-no-double-cloning/expected-stderr/three-deps.txt new file mode 100644 index 000000000..249fe7d91 --- /dev/null +++ b/test-fixtures/monorepo/1208-no-double-cloning/expected-stderr/three-deps.txt @@ -0,0 +1,8 @@ +Reading Spago workspace configuration... + +✓ Selecting package to build: consumer + +Fetching from +Downloading dependencies... +No lockfile found, generating it... +Lockfile written to spago.lock. Please commit this file. diff --git a/test-fixtures/monorepo/1208-no-double-cloning/expected-stderr/two-deps.txt b/test-fixtures/monorepo/1208-no-double-cloning/expected-stderr/two-deps.txt new file mode 100644 index 000000000..3cad05133 --- /dev/null +++ b/test-fixtures/monorepo/1208-no-double-cloning/expected-stderr/two-deps.txt @@ -0,0 +1,8 @@ +Reading Spago workspace configuration... + +✓ Selecting package to build: consumer + +Cloning +Downloading dependencies... +No lockfile found, generating it... +Lockfile written to spago.lock. Please commit this file. diff --git a/test-fixtures/monorepo/1208-no-double-cloning/library/lib1/spago.yaml b/test-fixtures/monorepo/1208-no-double-cloning/library/lib1/spago.yaml new file mode 100644 index 000000000..c95bb4a4a --- /dev/null +++ b/test-fixtures/monorepo/1208-no-double-cloning/library/lib1/spago.yaml @@ -0,0 +1,4 @@ +package: + name: lib1 + dependencies: + - prelude diff --git a/test-fixtures/monorepo/1208-no-double-cloning/library/lib2/spago.yaml b/test-fixtures/monorepo/1208-no-double-cloning/library/lib2/spago.yaml new file mode 100644 index 000000000..266558c12 --- /dev/null +++ b/test-fixtures/monorepo/1208-no-double-cloning/library/lib2/spago.yaml @@ -0,0 +1,5 @@ +package: + name: lib2 + dependencies: + - lib1 + - prelude diff --git a/test-fixtures/monorepo/1208-no-double-cloning/library/lib3/spago.yaml b/test-fixtures/monorepo/1208-no-double-cloning/library/lib3/spago.yaml new file mode 100644 index 000000000..4a4bc43cf --- /dev/null +++ b/test-fixtures/monorepo/1208-no-double-cloning/library/lib3/spago.yaml @@ -0,0 +1,6 @@ +package: + name: lib3 + dependencies: + - lib1 + - lib2 + - prelude diff --git a/test-fixtures/monorepo/1208-no-double-cloning/library/lib4/spago.yaml b/test-fixtures/monorepo/1208-no-double-cloning/library/lib4/spago.yaml new file mode 100644 index 000000000..1da44029c --- /dev/null +++ b/test-fixtures/monorepo/1208-no-double-cloning/library/lib4/spago.yaml @@ -0,0 +1,5 @@ +package: + name: lib4 + dependencies: + - lib1 + - prelude diff --git a/test-fixtures/monorepo/1208-no-double-cloning/library/spago.yaml b/test-fixtures/monorepo/1208-no-double-cloning/library/spago.yaml new file mode 100644 index 000000000..c5ebb16b5 --- /dev/null +++ b/test-fixtures/monorepo/1208-no-double-cloning/library/spago.yaml @@ -0,0 +1,3 @@ +workspace: + packageSet: + registry: 56.4.0 diff --git a/test-fixtures/monorepo/1208-no-double-cloning/spago-four-deps.yaml b/test-fixtures/monorepo/1208-no-double-cloning/spago-four-deps.yaml new file mode 100644 index 000000000..e729bb5dd --- /dev/null +++ b/test-fixtures/monorepo/1208-no-double-cloning/spago-four-deps.yaml @@ -0,0 +1,27 @@ +package: + name: consumer + dependencies: + - lib1 + - lib2 + - lib3 + - lib4 +workspace: + packageSet: + registry: 56.4.0 + extraPackages: + lib1: + git: + subdir: lib1 + ref: v1 + lib2: + git: + subdir: lib2 + ref: v2 + lib3: + git: + subdir: lib3 + ref: v3 + lib4: + git: + subdir: lib4 + ref: v4 diff --git a/test-fixtures/monorepo/1208-no-double-cloning/spago-three-deps.yaml b/test-fixtures/monorepo/1208-no-double-cloning/spago-three-deps.yaml new file mode 100644 index 000000000..0cfa1bd8d --- /dev/null +++ b/test-fixtures/monorepo/1208-no-double-cloning/spago-three-deps.yaml @@ -0,0 +1,22 @@ +package: + name: consumer + dependencies: + - lib1 + - lib2 + - lib3 +workspace: + packageSet: + registry: 56.4.0 + extraPackages: + lib1: + git: + subdir: lib1 + ref: v1 + lib2: + git: + subdir: lib2 + ref: v2 + lib3: + git: + subdir: lib3 + ref: v3 diff --git a/test-fixtures/monorepo/1208-no-double-cloning/spago-two-deps.yaml b/test-fixtures/monorepo/1208-no-double-cloning/spago-two-deps.yaml new file mode 100644 index 000000000..f6a62db4b --- /dev/null +++ b/test-fixtures/monorepo/1208-no-double-cloning/spago-two-deps.yaml @@ -0,0 +1,17 @@ +package: + name: consumer + dependencies: + - lib1 + - lib2 +workspace: + packageSet: + registry: 56.4.0 + extraPackages: + lib1: + git: + subdir: lib1 + ref: v1 + lib2: + git: + subdir: lib2 + ref: v2 diff --git a/test/Prelude.purs b/test/Prelude.purs index 2777fc004..eb73192d7 100644 --- a/test/Prelude.purs +++ b/test/Prelude.purs @@ -17,6 +17,7 @@ import Node.Path (dirname) import Node.Path as Path import Node.Platform as Platform import Node.Process as Process +import Record (merge) import Registry.PackageName as PackageName import Registry.Version as Version import Spago.Cmd (ExecResult, StdinConfig(..)) @@ -175,11 +176,22 @@ checkOutputs } -> Either ExecResult ExecResult -> Aff Unit -checkOutputs checkers execResult = do +checkOutputs args = checkOutputs' $ args `merge` { sanitize: String.trim } + +checkOutputs' + :: { stdoutFile :: Maybe FilePath + , stderrFile :: Maybe FilePath + , result :: (Either ExecResult ExecResult) -> Boolean + , sanitize :: String -> String + } + -> Either ExecResult ExecResult + -> Aff Unit +checkOutputs' checkers execResult = do let checkOrOverwrite = case _ of Nothing -> mempty - Just fixtureFileExpected -> \actual -> do + Just fixtureFileExpected -> \actual' -> do + let actual = checkers.sanitize actual' overwriteSpecFile <- liftEffect $ map isJust $ Process.lookupEnv "SPAGO_TEST_ACCEPT" if overwriteSpecFile then do Console.log $ "Overwriting fixture at path: " <> fixtureFileExpected @@ -187,7 +199,7 @@ checkOutputs checkers execResult = do unlessM (FS.exists parentDir) $ FS.mkdirp parentDir FS.writeTextFile fixtureFileExpected (actual <> "\n") else do - expected <- String.trim <$> FS.readTextFile fixtureFileExpected + expected <- checkers.sanitize <$> FS.readTextFile fixtureFileExpected actual `shouldEqualStr` expected check { stdout: checkOrOverwrite checkers.stdoutFile diff --git a/test/Spago/Build/Monorepo.purs b/test/Spago/Build/Monorepo.purs index 71f9146e3..7300f0ecf 100644 --- a/test/Spago/Build/Monorepo.purs +++ b/test/Spago/Build/Monorepo.purs @@ -5,7 +5,13 @@ import Test.Prelude import Data.Array as Array import Data.String (Pattern(..)) import Data.String as String +import Effect.Aff (bracket) +import Node.FS.Aff as FS.Aff +import Node.Path as Path +import Node.Process as Process +import Spago.Cmd as Cmd import Spago.FS as FS +import Spago.Paths as Paths import Test.Spec (SpecT) import Test.Spec as Spec import Test.Spec.Assertions as Assert @@ -246,3 +252,110 @@ spec = Spec.describe "monorepo" do spago [ "build" ] >>= shouldBeSuccess spago [ "build", "--pedantic-packages" ] >>= shouldBeFailureErr (fixture "monorepo/pedantic-cross-package-imports/expected-stderr.txt") + + Spec.it "#1208: clones a monorepo only once, even if multiple packages from it are needed" \{ spago, fixture, testCwd } -> do + -- A local file system Git repo to use as a remote for Spago to clone from + let createLibraryRepo = do + let libRepo = Path.concat [ Paths.paths.temp, "spago-1208" ] + whenM (FS.exists libRepo) $ rmRf libRepo + FS.copyTree { src: fixture "monorepo/1208-no-double-cloning/library", dst: libRepo } + git_ libRepo [ "init" ] + git_ libRepo [ "add", "." ] + git_ libRepo [ "config", "--global", "core.longpaths", "true" ] + git_ libRepo [ "config", "user.name", "test-user" ] + git_ libRepo [ "config", "user.email", "test-user@aol.com" ] + git_ libRepo [ "commit", "-m", "Initial commit" ] + git_ libRepo [ "tag", "v1" ] + git_ libRepo [ "tag", "v2" ] + pure libRepo + + bracket createLibraryRepo rmRf \libRepo -> do + let + recreateConsumerWorkspace = do + liftEffect $ Process.chdir testCwd + whenM (FS.exists "consumer") $ rmRf "consumer" + FS.mkdirp "consumer" + liftEffect $ Process.chdir "consumer" + copySpagoYaml "spago-two-deps.yaml" + + copySpagoYaml src = do + whenM (FS.exists "spago.yaml") $ FS.unlink "spago.yaml" + whenM (FS.exists "spago.lock") $ FS.unlink "spago.lock" + content <- FS.readTextFile $ fixture "monorepo/1208-no-double-cloning/" <> src + FS.writeTextFile "spago.yaml" $ String.replaceAll (String.Pattern "") (String.Replacement libRepo) content + + assertRefCheckedOut package ref = do + -- The `.spago/p//` should be a git repo checked out at `ref` + let path = Path.concat [ ".spago", "p", package, ref ] + commitHash <- git path [ "rev-parse", ref ] + git path [ "rev-parse", "HEAD" ] >>= flip shouldEqualStr commitHash + + -- And there should be a copy of that repo at + -- `.spago/p//`, checked out at the same commit. + let commitHashPath = Path.concat [ ".spago", "p", package, commitHash ] + git commitHashPath [ "rev-parse", "HEAD" ] >>= flip shouldEqualStr commitHash + + shouldBeSuccessErr' = checkOutputsWithPatchErr isRight + shouldBeFailureErr' = checkOutputsWithPatchErr isLeft + + checkOutputsWithPatchErr result expectedFixture = + checkOutputs' + { stdoutFile: Nothing + , stderrFile: Just $ fixture expectedFixture + , result + , sanitize: String.trim >>> String.replaceAll (String.Pattern libRepo) (String.Replacement "") + } + + -- First run `spago install` to make sure global cache is populated, + -- otherwise it may or may not appear in Spago's output and then we can't + -- reliably compare it to golden output. + recreateConsumerWorkspace + spago [ "ls", "packages" ] >>= shouldBeSuccess + + -- Nuke the cache after that so Spago can re-clone the repositories and we + -- can check that it's happening only once. + recreateConsumerWorkspace + spago [ "ls", "packages" ] >>= + shouldBeSuccessErr' "monorepo/1208-no-double-cloning/expected-stderr/two-deps.txt" + + -- Check that every package has the right ref checked out, as specified in + -- spago.yaml/extraPackages. + assertRefCheckedOut "lib1" "v1" + assertRefCheckedOut "lib2" "v2" + + -- Add lib3 to the config and check that Spago refuses to clone/pull + -- from the repo in offline more. + copySpagoYaml "spago-three-deps.yaml" + spago [ "ls", "packages", "--offline" ] >>= + shouldBeFailureErr' "monorepo/1208-no-double-cloning/expected-stderr/three-deps-offline.txt" + + -- Create new tags that lib3 and lib4 are pointing to + git_ libRepo [ "tag", "v3" ] + git_ libRepo [ "tag", "v4" ] + + -- Now that the remote repo has tags v3 and v4 defined, try again in + -- online mode and see that the repo is not cloned a second time, but + -- still pulled because the v3 tag is not in Spago's cache. + copySpagoYaml "spago-three-deps.yaml" + spago [ "ls", "packages" ] >>= + shouldBeSuccessErr' "monorepo/1208-no-double-cloning/expected-stderr/three-deps.txt" + assertRefCheckedOut "lib3" "v3" + + -- Add lib4 to the config and check that the repo is not cloned and not + -- pulled, but can be used in offline mode, because the v4 tag is already + -- in Spago's cache. + copySpagoYaml "spago-four-deps.yaml" + spago [ "ls", "packages", "--offline" ] >>= + shouldBeSuccessErr' "monorepo/1208-no-double-cloning/expected-stderr/four-deps.txt" + assertRefCheckedOut "lib4" "v4" + + where + git_ cwd = void <<< git cwd + + git cwd args = do + let opts = Cmd.defaultExecOptions { pipeStdout = false, pipeStderr = false, cwd = Just cwd } + res <- Cmd.exec "git" args opts + res # shouldBeSuccess + pure $ Cmd.getStdout res + + rmRf dir = liftAff $ FS.Aff.rm' dir { force: true, recursive: true, maxRetries: 5, retryDelay: 1000 }