Merge branch 'master' into server-ansible-cleanup

Author: ibodrov

.github/workflows/build.yml

@@ -42,3 +42,6 @@ jobs:
 
       - name: Build and test with Maven
         run: ./mvnw -B clean install -Pdocker -Pit -Pjdk${{ matrix.jdk_version }}
+        
+      - name: Build with debian docker images
+        run: ./mvnw -C -B -f docker-images install -DskipTests -Pdocker -Pdebian -Pjdk${{ matrix.jdk_version }}

CHANGELOG.md

@@ -1,6 +1,80 @@
 # Change log
 
-## [1.97.0] - Unreleased
+## [1.98.1] - 2022-12-22
+
+### Changed
+
+- concord-server: @Inject refactoring (part 1)
+([#658](https://github.com/walmartlabs/concord/pull/658));
+- concord-server, oidc: OIDC team/role mapping. Maps
+OpenID properties (e.g. `groups`) to Concord teams and
+roles
+([#682](https://github.com/walmartlabs/concord/pull/682));
+- concord-server: `process_queue` table split (part 1)
+([#668](https://github.com/walmartlabs/concord/pull/668));
+- runtime-v2: do not create log segments for expressions
+by default. Logs produced by expression blocks without
+`name` will no longer be displayed as a separate log
+"segment";
+([#689](https://github.com/walmartlabs/concord/pull/689));
+- concord-console: new compact view for the Log tab
+([#690](https://github.com/walmartlabs/concord/pull/690));
+- concord-server-db: a migration task to update secrets using
+the updated hashing algorithm
+([#691](https://github.com/walmartlabs/concord/pull/691));
+- concord-task: fix concurrency issue when collecting output
+of processes
+([#693](https://github.com/walmartlabs/concord/pull/693));
+- concord-server-db: pass secret salt as a base64 value
+([#694][https://github.com/walmartlabs/concord/pull/689]).
+
+
+
+## [1.98.0] - 2022-12-07
+
+### Added
+
+- runtime-v2: provide checkpoint name after restore
+([#677](https://github.com/walmartlabs/concord/pull/677));
+- policy: new policy to restrict raw payload
+([#679](https://github.com/walmartlabs/concord/pull/679));
+- concord-cli: provide default process configuration
+([#649](https://github.com/walmartlabs/concord/pull/649));
+- policy: policy to restrict runtime of process
+([#671](https://github.com/walmartlabs/concord/pull/671));
+- resource-task: add printJson() method
+([#676](https://github.com/walmartlabs/concord/pull/676));
+- server: cleanup agent commands
+([#674](https://github.com/walmartlabs/concord/pull/674));
+- policy-engine: `cron` trigger policy
+([#686](https://github.com/walmartlabs/concord/pull/686)).
+
+### Changed
+
+- runtime-v2: fix parallel loop execution when no out variable defined
+([#659](https://github.com/walmartlabs/concord/pull/659));
+- console2: repository list now with paging
+([#643](https://github.com/walmartlabs/concord/pull/643));
+- server: api for list project repositories with limit/offset
+([#643](https://github.com/walmartlabs/concord/pull/643));
+- runtime-v2: "throw" with a string value shouldn't produce a stacktrace
+([#673](https://github.com/walmartlabs/concord/pull/673));
+- concord-server: deprecate `process_queue.commit_msg`
+([#670](https://github.com/walmartlabs/concord/pull/670));
+- runtime-v2: move expression evaluator into sdk
+([#667](https://github.com/walmartlabs/concord/pull/667));
+- cli: log checkpoint instead of throwing Exception
+([#665](https://github.com/walmartlabs/concord/pull/665));
+- http-task: allow any value as json body
+([#675](https://github.com/walmartlabs/concord/pull/675));
+- docker-images: change the default shell to bash in
+Debian-based images
+([#644](https://github.com/walmartlabs/concord/pull/675));
+- runtime-v2: fix `entryPoint` calculation in effective YAML
+([#685](https://github.com/walmartlabs/concord/pull/685)).
+
+
+## [1.97.0] - 2022-10-11
 
 ### Added
 

agent/pom.xml

@@ -6,7 +6,7 @@
     <parent>
         <groupId>com.walmartlabs.concord</groupId>
         <artifactId>parent</artifactId>
-        <version>1.97.1-SNAPSHOT</version>
+        <version>1.98.2-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

agent/src/main/java/com/walmartlabs/concord/agent/cfg/ServerConfiguration.java

@@ -47,6 +47,8 @@ public class ServerConfiguration {
     private final long readTimeout;
     private final String userAgent;
     private final long maxNoHeartbeatInterval;
+    private final long processRequestDelay;
+    private final long reconnectDelay;
 
     @Inject
     public ServerConfiguration(Config cfg, AgentConfiguration agentCfg) {
@@ -71,6 +73,9 @@ public ServerConfiguration(Config cfg, AgentConfiguration agentCfg) {
         this.userAgent = getStringOrDefault(cfg, "server.userAgent", () -> "Concord-Agent: id=" + agentCfg.getAgentId());
 
         this.maxNoHeartbeatInterval = cfg.getDuration("server.maxNoHeartbeatInterval", TimeUnit.MILLISECONDS);
+
+        this.processRequestDelay = cfg.getDuration("server.processRequestDelay", TimeUnit.MILLISECONDS);
+        this.reconnectDelay = cfg.getDuration("server.reconnectDelay", TimeUnit.MILLISECONDS);
     }
 
     public String getApiBaseUrl() {
@@ -113,6 +118,14 @@ public long getMaxNoHeartbeatInterval() {
         return maxNoHeartbeatInterval;
     }
 
+    public long getProcessRequestDelay() {
+        return processRequestDelay;
+    }
+
+    public long getReconnectDelay() {
+        return reconnectDelay;
+    }
+
     private static String[] getWebsocketUrls(Config cfg) {
         // we had a silly typo ("websockeR") in our configs, so for backward compatibility we must check the old variant first
         String oldKey = "server.websockerUrl";

agent/src/main/java/com/walmartlabs/concord/agent/executors/runner/RunnerCommandBuilder.java

@@ -104,8 +104,10 @@ public String[] build() {
 
         // speeds up the start, we don't care much about all potential optimizations done by HotSpot
         l.add("-client");
-        // don't do bytecode verification
-        l.add("-noverify");
+        if (majorJavaVersion < 13) {
+            // don't do bytecode verification
+            l.add("-noverify");
+        }
         // enable support for calling vararg methods in JUEL
         l.add("-Djavax.el.varArgs=true");
         // avoid blocking on crypto

agent/src/main/java/com/walmartlabs/concord/agent/executors/runner/RunnerJobExecutor.java

@@ -415,9 +415,9 @@ private void validateDependencies(RunnerJob job, Collection<DependencyEntity> re
 
         CheckResult<DependencyRule, DependencyEntity> result = policyEngine.getDependencyPolicy().check(resolvedDepEntities);
         result.getWarn().forEach(d ->
-                processLog.warn("Potentially restricted artifact '{}' (dependency policy: {})", d.getEntity(), d.getRule().getMsg()));
+                processLog.warn("Potentially restricted artifact '{}' (dependency policy: {})", d.getEntity(), d.getRule().msg()));
         result.getDeny().forEach(d ->
-                processLog.warn("Artifact '{}' is forbidden by the dependency policy {}", d.getEntity(), d.getRule().getMsg()));
+                processLog.warn("Artifact '{}' is forbidden by the dependency policy {}", d.getEntity(), d.getRule().msg()));
 
         if (!result.getDeny().isEmpty()) {
             throw new ExecutionException("Found restricted dependencies");

agent/src/main/java/com/walmartlabs/concord/agent/remote/QueueClientProvider.java

@@ -54,6 +54,8 @@ public QueueClient get() {
                     .connectTimeout(serverCfg.getConnectTimeout())
                     .pingInterval(serverCfg.getPingInterval())
                     .maxNoActivityPeriod(serverCfg.getMaxNoActivityPeriod())
+                    .processRequestDelay(serverCfg.getProcessRequestDelay())
+                    .reconnectDelay(serverCfg.getReconnectDelay())
                     .build());
 
             queueClient.start();

agent/src/main/resources/concord-agent.conf

@@ -127,6 +127,12 @@ concord-agent {
 
         # maximum time interval without a heartbeat before the process fails
         maxNoHeartbeatInterval = "5 minutes"
+
+        # delay between successful polling attempts
+        processRequestDelay = "1 seconds"
+
+        # delay between re-connection attempts if the server is unreachable or unhealthy
+        reconnectDelay = "5 seconds"
     }
 
     docker {

cli/pom.xml

@@ -6,7 +6,7 @@
     <parent>
         <groupId>com.walmartlabs.concord</groupId>
         <artifactId>parent</artifactId>
-        <version>1.97.1-SNAPSHOT</version>
+        <version>1.98.2-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 
@@ -87,6 +87,10 @@
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.dataformat</groupId>
+            <artifactId>jackson-dataformat-yaml</artifactId>
+        </dependency>
 
         <dependency>
             <groupId>org.junit.jupiter</groupId>

cli/src/main/java/com/walmartlabs/concord/cli/Run.java

@@ -21,6 +21,8 @@
  */
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator;
 import com.google.inject.Injector;
 import com.walmartlabs.concord.cli.runner.*;
 import com.walmartlabs.concord.common.ConfigurationUtils;
@@ -196,7 +198,7 @@ public Integer call() throws Exception {
         Map<String, Object> overlayArgs = MapUtils.getMap(overlayCfg, Constants.Request.ARGUMENTS_KEY, Collections.emptyMap());
         Map<String, Object> args = ConfigurationUtils.deepMerge(cfg.arguments(), overlayArgs, extraVars);
         if (verbose) {
-            System.out.println("Process arguments: " + args);
+            dumpArguments(args);
         }
         args.put(Constants.Context.TX_ID_KEY, instanceId.toString());
         args.put(Constants.Context.WORK_DIR_KEY, targetDir.toAbsolutePath().toString());
@@ -231,7 +233,7 @@ public Integer call() throws Exception {
                 runnerCfg,
                 () -> cfg,
                 new ProcessDependenciesModule(targetDir, runnerCfg.dependencies(), cfg.debug()),
-                new CliServicesModule(secretStoreDir, targetDir, new VaultProvider(vaultDir, vaultId), dependencyManager))
+                new CliServicesModule(secretStoreDir, targetDir, new VaultProvider(vaultDir, vaultId), dependencyManager, verbose))
                 .create();
 
         Runner runner = injector.getInstance(Runner.class);
@@ -309,6 +311,16 @@ private DependencyManagerConfiguration getDependencyManagerConfiguration() {
         return DependencyManagerConfiguration.of(depsCacheDir);
     }
 
+    private static void dumpArguments(Map<String, Object> args) {
+        ObjectMapper om = new ObjectMapper(new YAMLFactory().disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER));
+        try {
+            System.out.println("Process arguments:");
+            System.out.println(om.writerWithDefaultPrettyPrinter().writeValueAsString(args));
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     private static class CopyNotifier implements FileVisitor {
 
         private final long notifyOnCount;

cli/src/main/java/com/walmartlabs/concord/cli/runner/CliCheckpointService.java

@@ -4,7 +4,7 @@
  * *****
  * Concord
  * -----
- * Copyright (C) 2017 - 2020 Walmart Inc.
+ * Copyright (C) 2017 - 2023 Walmart Inc.
  * -----
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,17 +20,25 @@
  * =====
  */
 
+import com.walmartlabs.concord.runtime.common.SerializationUtils;
 import com.walmartlabs.concord.runtime.v2.runner.ProcessSnapshot;
 import com.walmartlabs.concord.runtime.v2.runner.checkpoints.CheckpointService;
 import com.walmartlabs.concord.svm.Runtime;
 import com.walmartlabs.concord.svm.ThreadId;
 
+import java.io.ByteArrayOutputStream;
 import java.util.UUID;
 
 public class CliCheckpointService implements CheckpointService {
 
     @Override
     public void create(ThreadId threadId, UUID correlationId, String name, Runtime runtime, ProcessSnapshot snapshot) {
+        try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
+            SerializationUtils.serialize(baos, snapshot.vmState());
+        } catch (Exception e) {
+            throw new RuntimeException("Checkpoint create error", e);
+        }
+
         System.out.println("Checkpoint '" +  name +  "' ignored");
     }
 }

cli/src/main/java/com/walmartlabs/concord/cli/runner/CliServicesModule.java

@@ -30,6 +30,7 @@
 import com.walmartlabs.concord.runtime.v2.runner.guice.BaseRunnerModule;
 import com.walmartlabs.concord.runtime.v2.runner.logging.RunnerLogger;
 import com.walmartlabs.concord.runtime.v2.runner.logging.SimpleLogger;
+import com.walmartlabs.concord.runtime.v2.runner.remote.EventRecordingExecutionListener;
 import com.walmartlabs.concord.runtime.v2.sdk.DockerService;
 import com.walmartlabs.concord.runtime.v2.sdk.LockService;
 import com.walmartlabs.concord.runtime.v2.sdk.SecretService;
@@ -44,16 +45,19 @@ public class CliServicesModule extends AbstractModule {
     private final Path workDir;
     private final VaultProvider vaultProvider;
     private final DependencyManager dependencyManager;
+    private final boolean verbose;
 
     public CliServicesModule(Path secretStoreDir,
                              Path workDir,
                              VaultProvider vaultProvider,
-                             DependencyManager dependencyManager) {
+                             DependencyManager dependencyManager,
+                             boolean verbose) {
 
         this.secretStoreDir = secretStoreDir;
         this.workDir = workDir;
         this.vaultProvider = vaultProvider;
         this.dependencyManager = dependencyManager;
+        this.verbose = verbose;
     }
 
     @Override
@@ -76,7 +80,10 @@ protected void configure() {
         bind(DependencyManager.class).toInstance(dependencyManager);
         bind(com.walmartlabs.concord.runtime.v2.sdk.DependencyManager.class).to(DefaultDependencyManager.class).in(Singleton.class);
 
-        Multibinder.newSetBinder(binder(), ExecutionListener.class);
+        Multibinder<ExecutionListener> executionListeners = Multibinder.newSetBinder(binder(), ExecutionListener.class);
+        if (verbose) {
+            executionListeners.addBinding().to(EventLoggerExecutionListener.class);
+        }
     }
 
     @SuppressWarnings("unchecked")

cli/src/main/java/com/walmartlabs/concord/cli/runner/EventLoggerExecutionListener.java

@@ -0,0 +1,77 @@
+package com.walmartlabs.concord.cli.runner;
+
+/*-
+ * *****
+ * Concord
+ * -----
+ * Copyright (C) 2017 - 2023 Walmart Inc.
+ * -----
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =====
+ */
+
+import com.walmartlabs.concord.runtime.v2.model.*;
+import com.walmartlabs.concord.runtime.v2.runner.vm.StepCommand;
+import com.walmartlabs.concord.svm.Runtime;
+import com.walmartlabs.concord.svm.*;
+
+public class EventLoggerExecutionListener implements ExecutionListener {
+
+    @Override
+    public Result beforeCommand(Runtime runtime, VM vm, State state, ThreadId threadId, Command cmd) {
+        if (!(cmd instanceof StepCommand)) {
+            return Result.CONTINUE;
+        }
+
+        StepCommand<?> s = (StepCommand<?>) cmd;
+
+        Location loc = s.getStep().getLocation();
+
+        System.out.println(">>> '" + getDescription(s.getStep()) + "' at " + loc.fileName() + ":" + loc.lineNum());
+
+        return Result.CONTINUE;
+    }
+
+    private static String getDescription(Step step) {
+        // TODO: add 'description' into step? so we will not miss description for new steps...
+        if (step instanceof FlowCall) {
+            return "Flow call: " + ((FlowCall) step).getFlowName();
+        } else if (step instanceof Expression) {
+            return "Expression: " + ((Expression) step).getExpr();
+        } else if (step instanceof ScriptCall) {
+            return "Script: " + ((ScriptCall) step).getLanguageOrRef();
+        } else if (step instanceof IfStep) {
+            return "Check: " + ((IfStep) step).getExpression();
+        } else if (step instanceof SwitchStep) {
+            return "Switch: " + ((SwitchStep) step).getExpression();
+        } else if (step instanceof SetVariablesStep) {
+            return "Set variables";
+        } else if (step instanceof Checkpoint) {
+            return "Checkpoint: " + ((Checkpoint) step).getName();
+        } else if (step instanceof FormCall) {
+            return "Form call: " + ((FormCall) step).getName();
+        } else if (step instanceof GroupOfSteps) {
+            return "Group of steps";
+        } else if (step instanceof ParallelBlock) {
+            return "Parallel block";
+        } else if (step instanceof ExitStep) {
+            return "Exit";
+        } else if (step instanceof ReturnStep) {
+            return "Return";
+        } else if (step instanceof TaskCall) {
+            return "Task: " + ((TaskCall) step).getName();
+        }
+
+        return step.getClass().getName();
+    }
+}