Reimplement spago run for 0.15 (ES modules) (#866)

Co-authored-by: Fabrizio Ferrai <fabrizio.ferrai@gmail.com>
Co-authored-by: Jordan Martinez <jordanalex.martinez@protonmail.com>
Co-authored-by: Nicholas Yip <nicholasyip@obceit.cc>

Author: 4 people

.github/workflows/build.yml

@@ -95,7 +95,7 @@ jobs:
             $HOME\AppData\Local\Programs\stack\x86_64-windows
           key: ${{ runner.os }}-${{ hashFiles('stack.yaml') }}-1
 
-      - run: npm install -g purescript@0.14.0 psc-package@3.0.1 bower@1.8.8
+      - run: npm install -g purescript@0.14.0 psc-package@3.0.1 bower@1.8.8 esbuild@0.14.28
 
       - name: Install dependencies
         run: |
@@ -112,6 +112,13 @@ jobs:
         run: ./scripts/fix-home stack install
         shell: bash
 
-      - name: Run tests
+      - name: Run tests (PureScript < 0.15.0)
         run: ./scripts/fix-home stack test
         shell: bash
+
+      - name: Install PureScript 0.15.0
+        run: npm install -g purescript@next
+
+      - name: Run tests (PureScript >= 0.15.0)
+        shell: bash
+        run: ./scripts/fix-home stack test --ta "--match purs-0.15"

CHANGELOG.md

@@ -13,6 +13,10 @@ Features:
 - Support Glibc versions >= `2.24`
 - Retry git clone up to two times (#834, #873)
 
+Bugfixes
+- Fix `spago run` and `spago test` to accept command line arguments correctly, by writing a JS file to run (#865, #866)
+- Remove support for node versions older than 12.0.0 as they do not work with es modules (#866)
+
 ## [0.20.7] - 2022-02-12
 
 Bugfixes

spago.cabal

@@ -87,6 +87,7 @@ library
     Spago.Bower
     Spago.Build
     Spago.CLI
+    Spago.Cmd
     Spago.Command.Init
     Spago.Command.Ls
     Spago.Command.Path

src/Spago/Build.hs

@@ -17,6 +17,7 @@ import qualified Data.List.NonEmpty   as NonEmpty
 import qualified Data.Map             as Map
 import qualified Data.Set             as Set
 import qualified Data.Text            as Text
+import qualified Data.Versions        as Version
 import           System.Directory     (getCurrentDirectory)
 import           System.FilePath      (splitDirectories)
 import qualified System.FilePath.Glob as Glob
@@ -36,6 +37,7 @@ import qualified Spago.Packages       as Packages
 import qualified Spago.Purs           as Purs
 import qualified Spago.Templates      as Templates
 import qualified Spago.Watch          as Watch
+import qualified Spago.Cmd            as Cmd
 
 
 prepareBundleDefaults
@@ -50,7 +52,7 @@ prepareBundleDefaults maybeModuleName maybeTargetPath maybePlatform = (moduleNam
     platform = fromMaybe Browser maybePlatform
 
 --   eventually running some other action after the build
-build :: HasBuildEnv env => Maybe (RIO Env ()) -> RIO env ()
+build :: HasBuildEnv env => Maybe (RIO env ()) -> RIO env ()
 build maybePostBuild = do
   logDebug "Running `spago build`"
   BuildOptions{..} <- view (the @BuildOptions)
@@ -143,8 +145,7 @@ build maybePostBuild = do
         checkImports
 
       buildAction globs = do
-        env <- Run.getEnv
-        let action = buildBackend globs >> (runRIO env $ fromMaybe (pure ()) maybePostBuild)
+        let action = buildBackend globs >> (fromMaybe (pure ()) maybePostBuild)
         runCommands "Before" beforeCommands
         action `onException` (runCommands "Else" elseCommands)
         runCommands "Then" thenCommands
@@ -309,6 +310,22 @@ script modulePath tag packageDeps opts = do
 
 data RunDirectories = RunDirectories { sourceDir :: FilePath, executeDir :: FilePath }
 
+data NodeEsSupport = Unsupported Version.SemVer | Experimental | Supported
+
+hasNodeEsSupport :: (HasLogFunc env) => RIO env NodeEsSupport
+hasNodeEsSupport = do
+  nodeVersion <- Cmd.getCmdVersion "node"
+  case nodeVersion  of
+    Left err -> do
+      logDebug $ display $ "Unable to get Node.js version: " <> displayShow err
+      pure Supported
+    Right nv@Version.SemVer{} | Version._svMajor nv < 12 ->
+      pure $ Unsupported nv
+    Right nv@Version.SemVer{} | Version._svMajor nv >= 12 && Version._svMajor nv < 13 ->
+      pure Experimental
+    _ -> pure Supported
+
+
 -- | Run the project with node (or the chosen alternate backend):
 --   compile and run the provided ModuleName
 runBackend
@@ -323,15 +340,17 @@ runBackend
 runBackend maybeBackend RunDirectories{ sourceDir, executeDir } moduleName maybeSuccessMessage failureMessage extraArgs = do
   logDebug $ display $ "Running with backend: " <> fromMaybe "nodejs" maybeBackend
   BuildOptions{ pursArgs } <- view (the @BuildOptions)
-  isES <- Purs.hasMinPursVersion "0.15.0"
-  let postBuild = maybe (nodeAction isES $ Path.getOutputPath pursArgs) backendAction maybeBackend
+  let
+    postBuild = maybe (nodeAction $ Path.getOutputPath pursArgs) backendAction maybeBackend
   build (Just postBuild)
   where
     fromFilePath = Text.pack . Turtle.encodeString
+    runJsSource = fromFilePath (sourceDir Turtle.</> ".spago/run.js")
+    packageJson = fromFilePath (sourceDir Turtle.</> ".spago/package.json")
     nodeArgs = Text.intercalate " " $ map unBackendArg extraArgs
     esContents outputPath' =
       fold
-        [ "import { main } from '"
+        [ "import { main } from 'file://"
         , Text.replace "\\" "/" (fromFilePath sourceDir)
         , "/"
         , Text.pack outputPath'
@@ -352,19 +371,36 @@ runBackend maybeBackend RunDirectories{ sourceDir, executeDir } moduleName maybe
         , unModuleName moduleName
         , "').main()"
         ]
-    nodeCmd isES outputPath'=
-      if isES then
-        "node --input-type=module -e \"" <> esContents outputPath'  <> "\" -- " <> nodeArgs
-      else
-        "node -e \"" <> cjsContents outputPath' <> "\" -- " <> nodeArgs
-
-    nodeAction isES outputPath' = do
+    nodeContents isES outputPath' =
+      if isES then esContents outputPath'
+      else cjsContents outputPath'
+
+    packageJsonContents = "{\"type\":\"module\" }"
+
+    nodeCmd isES Experimental | isES = "node --experimental-modules \"" <> runJsSource <> "\" " <> nodeArgs
+    nodeCmd _ _ = "node \"" <> runJsSource <> "\" " <> nodeArgs
+
+    nodeAction outputPath' = do
+      isES <- Purs.hasMinPursVersion "0.15.0-alpha-01"
+      nodeVersion <- hasNodeEsSupport
+      case (isES, nodeVersion) of
+        (True, Unsupported nv) ->
+          die [ "Unsupported Node.js version: " <> display (Version.prettySemVer nv), "Required Node.js version >=12." ]
+        _ ->
+          pure ()
+      logDebug $ "Writing " <> displayShow @Text runJsSource
+      writeTextFile runJsSource (nodeContents isES outputPath')
+      void $ chmod executable $ pathFromText runJsSource
+      if isES then do
+        logDebug $ "Writing " <> displayShow @Text packageJson
+        writeTextFile packageJson packageJsonContents
+      else pure ()
       -- cd to executeDir in case it isn't the same as sourceDir
       logDebug $ "Executing from: " <> displayShow @FilePath executeDir
       Turtle.cd executeDir
       -- We build a process by hand here because we need to forward the stdin to the backend process
-      logDebug $ "Running node command: `" <> display (nodeCmd isES outputPath') <> "`"
-      let processWithStdin = (Process.shell (Text.unpack $ nodeCmd isES outputPath')) { Process.std_in = Process.Inherit }
+      logDebug $ "Running node command: `" <> display (nodeCmd isES nodeVersion) <> "`"
+      let processWithStdin = (Process.shell (Text.unpack $ nodeCmd isES nodeVersion)) { Process.std_in = Process.Inherit }
       Turtle.system processWithStdin empty >>= \case
         ExitSuccess   -> maybe (pure ()) (logInfo . display) maybeSuccessMessage
         ExitFailure n -> die [ display failureMessage <> "exit code: " <> repr n ]
@@ -376,26 +412,28 @@ runBackend maybeBackend RunDirectories{ sourceDir, executeDir } moduleName maybe
         ExitFailure n -> die [ display failureMessage <> "Backend " <> displayShow backend <> " exited with error:" <> repr n ]
 
 
-bundleWithEsbuild :: HasLogFunc env => WithMain -> ModuleName -> TargetPath -> Platform -> Minify -> RIO env ()
-bundleWithEsbuild withMain (ModuleName moduleName) (TargetPath targetPath) platform minify = do
+bundleWithEsbuild :: HasLogFunc env => WithMain -> WithSrcMap -> ModuleName -> TargetPath -> Platform -> Minify -> RIO env ()
+bundleWithEsbuild withMain srcMap (ModuleName moduleName) (TargetPath targetPath) platform minify = do
   esbuild <- getESBuild
   let
     platformOpt = case platform of
-      Browser -> "browser"
-      Node -> "node"
+      Browser -> ["--platform=browser"]
+      Node -> ["--platform=node"]
     minifyOpt = case minify of
-      NoMinify -> ""
-      Minify -> " --minify"
-    cmd = case withMain of
-      WithMain ->
-        "echo \"import { main } from './output/" <> moduleName <> "/index.js'\nmain()\" | "
-        <> esbuild <> " --platform=" <> platformOpt <> minifyOpt <> " --bundle "
-        <> " --outfile=" <> targetPath
+      NoMinify -> []
+      Minify -> ["--minify"]
+    srcMapOpt = case srcMap of
+      WithSrcMap -> ["--sourcemap"]
+      WithoutSrcMap -> []
+    esbuildBase = platformOpt <> minifyOpt <> srcMapOpt <> ["--format=esm", "--bundle", "--outfile=" <> targetPath]
+    (input, cmd) = case withMain of
+      WithMain -> do
+        let
+          echoLine = "import { main } from './output/" <> moduleName <> "/index.js'; main();"
+        (Turtle.textToLine echoLine, esbuild :| esbuildBase)
       WithoutMain ->
-        esbuild <> " --platform=" <> platformOpt <> minifyOpt <> " --bundle "
-        <> "output/" <> moduleName <> "/index.js"
-        <> " --outfile=" <> targetPath
-  runWithOutput cmd
+        (Nothing, esbuild :| esbuildBase <> ["output/" <> moduleName <> "/index.js"])
+  runProcessWithOutput cmd input
     ("Bundle succeeded and output file to " <> targetPath)
     "Bundle failed."
   where
@@ -414,12 +452,12 @@ bundleModule
   -> UsePsa
   -> RIO env ()
 bundleModule withMain BundleOptions { maybeModuleName, maybeTargetPath, maybePlatform, minify, noBuild }  buildOpts usePsa = do
-  isES <- Purs.hasMinPursVersion "0.15.0"
+  isES <- Purs.hasMinPursVersion "0.15.0-alpha-01"
   let
     (moduleName, targetPath, platform) = prepareBundleDefaults maybeModuleName maybeTargetPath maybePlatform
     bundleAction =
       if isES then
-        bundleWithEsbuild withMain moduleName targetPath platform minify
+        bundleWithEsbuild withMain (withSourceMap buildOpts) moduleName targetPath platform minify
       else case withMain of
         WithMain -> Purs.bundle WithMain (withSourceMap buildOpts) moduleName targetPath
         WithoutMain ->
@@ -437,7 +475,7 @@ bundleModule withMain BundleOptions { maybeModuleName, maybeTargetPath, maybePla
                   Left (n :: SomeException) -> die [ "Make module failed: " <> repr n ]
   case noBuild of
     DoBuild -> Run.withBuildEnv usePsa buildOpts $ build (Just bundleAction)
-    NoBuild -> Run.getEnv >>= (flip runRIO) bundleAction
+    NoBuild -> Run.withBuildEnv usePsa buildOpts bundleAction
 
 docsSearchTemplate :: (HasType LogFunc env, HasType PursCmd env) => RIO env Text
 docsSearchTemplate = ifM (Purs.hasMinPursVersion "0.14.0")

src/Spago/Cmd.hs

@@ -0,0 +1,26 @@
+module Spago.Cmd (getCmdVersion) where
+
+import qualified Spago.Messages as Messages
+import qualified Turtle.Bytes
+import           Spago.Prelude
+import qualified Data.Text      as Text
+import qualified Data.Text.Encoding as Text.Encoding
+import qualified Data.Text.Encoding.Error as Text.Encoding
+import qualified Data.Versions  as Version
+
+-- | Get the semantic version of a command, e.g. purs --version
+getCmdVersion :: forall io. MonadIO io => Text -> io (Either Text Version.SemVer)
+getCmdVersion cmd =
+  Turtle.Bytes.shellStrictWithErr (cmd <> " --version") empty >>= \case
+    (ExitSuccess, out, _err) -> do
+      let versionText = headMay $ Text.split (== ' ') (Text.strip $ Text.Encoding.decodeUtf8With lenientDecode out)
+          parsed = versionText >>= (\vt -> Text.stripPrefix "v" vt <|> Just vt) >>= (hush . Version.semver)
+
+      pure $ case parsed of
+        Nothing ->
+          Left $
+            Messages.failedToParseCommandOutput
+              (cmd <> " --version")
+              (Text.Encoding.decodeUtf8With Text.Encoding.lenientDecode out)
+        Just p -> Right p
+    (_, _out, _err) -> pure $ Left $ "Failed to run '" <> cmd <> " --version'"

src/Spago/Prelude.hs

@@ -67,6 +67,7 @@ module Spago.Prelude
   , findExecutableOrDie
   , findExecutable
   , runWithOutput
+  , runProcessWithOutput
   -- * Other
   , Dhall.Core.throws
   , repr
@@ -108,7 +109,7 @@ import           RIO                                   as X hiding (FilePath, fi
 import           RIO.Orphans                           as X
 import           Safe                                  (headMay, lastMay)
 import           System.FilePath                       (isAbsolute, pathSeparator, (</>))
-import           Turtle                                (FilePath, appendonly, chmod,
+import           Turtle                                (FilePath, Line, appendonly, chmod,
                                                         executable, mktree, repr, shell,
                                                         shellStrict, shellStrictWithErr,
                                                         systemStrictWithErr, testdir)
@@ -271,6 +272,14 @@ runWithOutput command success failure = do
     ExitSuccess -> logInfo $ display success
     ExitFailure _ -> die [ display failure ]
 
+-- | Run the given command.
+runProcessWithOutput :: HasLogFunc env => NonEmpty Text -> Maybe Line -> Text -> Text -> RIO env ()
+runProcessWithOutput (command :| arguments) input success failure = do
+  logDebug $ "Running command: `" <> display (Text.intercalate " " $ command : arguments) <> "`"
+  Turtle.proc command arguments (Turtle.select input) >>= \case
+    ExitSuccess -> logInfo $ display success
+    ExitFailure _ -> die [ display failure ]
+
 -- | Return the full path of the executable we're trying to call,
 --   or die trying
 findExecutableOrDie :: HasLogFunc env => String -> RIO env Text

src/Spago/Purs.hs

@@ -19,12 +19,11 @@ import           Spago.Env
 import qualified Data.ByteString.Lazy as BSL
 import qualified Data.Text      as Text
 import qualified Data.Text.Encoding as Text.Encoding
-import qualified Data.Text.Encoding.Error as Text.Encoding
 import qualified Data.Versions  as Version
 
 import qualified Spago.Messages as Messages
 import qualified Turtle.Bytes
-
+import qualified Spago.Cmd      as Cmd
 
 compile
   :: (HasPurs env, HasLogFunc env)
@@ -130,19 +129,7 @@ docs format sourcePaths = do
     "Docs generation failed."
 
 pursVersion :: RIO env (Either Text Version.SemVer)
-pursVersion = Turtle.Bytes.shellStrictWithErr (purs <> " --version") empty >>= \case
-  (ExitSuccess, out, _err) -> do
-    let versionText = headMay $ Text.split (== ' ') (Text.strip $ Text.Encoding.decodeUtf8With lenientDecode out)
-        parsed = versionText >>= (hush . Version.semver)
-
-    pure $ case parsed of
-      Nothing -> Left $ Messages.failedToParseCommandOutput 
-        (purs <> " --version") 
-        (Text.Encoding.decodeUtf8With Text.Encoding.lenientDecode out)
-      Just p -> Right p
-  (_, _out, _err) -> pure $ Left $ "Failed to run '" <> purs <> " --version'"
-  where
-    purs = "purs"
+pursVersion = Cmd.getCmdVersion "purs"
 
 hasMinPursVersion :: (HasLogFunc env, HasPurs env) => Text -> RIO env Bool
 hasMinPursVersion maybeMinVersion = do