concord-repository: support for mvn:// scheme

agent/src/main/java/com/walmartlabs/concord/agent/JobRequest.java

@@ -147,6 +147,7 @@ public String toString() {
                 ", repoUrl='" + repoUrl + '\'' +
                 ", repoPath='" + repoPath + '\'' +
                 ", commitId='" + commitId + '\'' +
+                ", repoBranch='" + repoBranch + '\'' +
                 ", secretName='" + secretName + '\'' +
                 ", imports='" + imports + '\'' +
                 '}';

agent/src/main/java/com/walmartlabs/concord/agent/RepositoryManager.java

@@ -27,11 +27,12 @@
 import com.walmartlabs.concord.imports.Import.SecretDefinition;
 import com.walmartlabs.concord.repository.*;
 import com.walmartlabs.concord.sdk.Secret;
+import com.walmartlabs.concord.dependencymanager.DependencyManager;
 
 import javax.inject.Inject;
 import java.io.IOException;
 import java.nio.file.Path;
-import java.util.Collections;
+import java.util.Arrays;
 import java.util.List;
 
 public class RepositoryManager {
@@ -45,7 +46,8 @@ public class RepositoryManager {
     public RepositoryManager(SecretClient secretClient,
                              GitConfiguration gitCfg,
                              RepositoryCacheConfiguration cacheCfg,
-                             ObjectMapper objectMapper) throws IOException {
+                             ObjectMapper objectMapper,
+                             DependencyManager dependencyManager) throws IOException {
 
         this.secretClient = secretClient;
         this.gitCfg = gitCfg;
@@ -60,7 +62,7 @@ public RepositoryManager(SecretClient secretClient,
                 .sshTimeoutRetryCount(gitCfg.getSshTimeoutRetryCount())
                 .build();
 
-        List<RepositoryProvider> providers = Collections.singletonList(new GitCliRepositoryProvider(clientCfg));
+        List<RepositoryProvider> providers = Arrays.asList(new MavenRepositoryProvider(dependencyManager), new GitCliRepositoryProvider(clientCfg));
         this.providers = new RepositoryProviders(providers);
 
         this.repositoryCache = new RepositoryCache(cacheCfg.getCacheDir(),

agent/src/main/java/com/walmartlabs/concord/agent/Worker.java

@@ -142,7 +142,9 @@ private void onStatusChange(UUID instanceId, StatusEnum status) {
     }
 
     private void fetchRepo(JobRequest r) throws Exception {
-        if (r.getRepoUrl() == null || r.getCommitId() == null) {
+        if (r.getRepoUrl() == null
+                || (r.getCommitId() == null && r.getRepoBranch() == null)
+                || r.getRepoUrl().startsWith("classpath://")) {
             return;
         }
 

console2/src/components/molecules/GitHubLink/index.tsx

@@ -23,6 +23,7 @@ import { REPOSITORY_SSH_URL_PATTERN } from '../../../validation';
 
 interface Props {
     url: string;
+    link?: string;
     path?: string;
     commitId?: string;
     text?: string;
@@ -59,9 +60,9 @@ const normalizePath = (s: string): string => {
 
 class GitHubLink extends React.PureComponent<Props> {
     render() {
-        const { url, commitId, path, text, branch } = this.props;
+        const { url, link, commitId, path, text, branch } = this.props;
 
-        let s = gitUrlParse(url);
+        let s = !link ? gitUrlParse(url) : link;
         if (!s) {
             return url;
         }

console2/src/components/molecules/RepositoryForm/index.tsx

@@ -71,7 +71,7 @@ interface State {
 
 const sourceOptions = [
     {
-        text: 'Branch/tag',
+        text: 'Branch/tag/version',
         value: RepositorySourceType.BRANCH_OR_TAG
     },
     {
@@ -224,7 +224,7 @@ class RepositoryForm extends React.Component<InjectedFormikProps<Props, FormValu
                         {values.sourceType === RepositorySourceType.BRANCH_OR_TAG && (
                             <FormikInput
                                 name="branch"
-                                label="Branch/Tag"
+                                label="Branch/Tag/Version"
                                 fluid={true}
                                 required={true}
                             />
@@ -348,10 +348,10 @@ const validator = async (values: FormValues, props: Props) => {
     }
 
     if (!values.withSecret) {
-        if (!values.url.startsWith('https://')) {
+        if (!values.url.startsWith('https://') && !values.url.startsWith('mvn://')) {
             return Promise.resolve({
                 url:
-                    "Invalid repository URL: must begin with 'https://'. SSH repository URLs require additional credentials to be specified."
+                    "Invalid repository URL: must begin with 'https://' or 'mvn://'. SSH repository URLs require additional credentials to be specified."
             });
         }
     } else {

console2/src/components/molecules/RepositoryList/index.tsx

@@ -26,6 +26,7 @@ import { ConcordKey } from '../../../api/common';
 import { RepositoryEntry } from '../../../api/org/project/repository';
 import { GitHubLink } from '../../molecules';
 import { RepositoryActionDropdown } from '../../organisms';
+import { gitUrlParse } from "../GitHubLink";
 
 interface ExternalProps {
     orgName: ConcordKey;
@@ -44,7 +45,7 @@ const RepositoryList = ({ orgName, projectName, data, loading, refresh }: Extern
                         <Table.HeaderCell collapsing={true} />
                         <Table.HeaderCell collapsing={true}>Name</Table.HeaderCell>
                         <Table.HeaderCell>Repository URL</Table.HeaderCell>
-                        <Table.HeaderCell collapsing={true}>Branch/Commit ID</Table.HeaderCell>
+                        <Table.HeaderCell collapsing={true}>Branch/Commit ID/Version</Table.HeaderCell>
                         <Table.HeaderCell singleLine={true}>Path</Table.HeaderCell>
                         <Table.HeaderCell collapsing={true} style={{ width: '8%' }}>
                             Secret
@@ -72,24 +73,35 @@ const RepositoryList = ({ orgName, projectName, data, loading, refresh }: Extern
 };
 
 const renderRepoPath = (r: RepositoryEntry) => {
+    const urlLink = gitUrlParse(r.url);
+    if (!urlLink) {
+        return r.path;
+    }
     if (r.commitId) {
         return (
             <GitHubLink
+                link={urlLink}
                 url={r.url!}
                 commitId={r.commitId}
                 path={r.path || '/'}
                 text={r.path || '/'}
             />
         );
     }
-    return <GitHubLink url={r.url!} branch={r.branch} path={r.path || '/'} text={r.path || '/'} />;
+    return <GitHubLink url={r.url!} link={urlLink} branch={r.branch} path={r.path || '/'} text={r.path || '/'} />;
 };
 
 const renderRepoCommitIdOrBranch = (r: RepositoryEntry) => {
+
+    const urlLink = gitUrlParse(r.url);
+    if (!urlLink) {
+        return r.branch;
+    }
+
     if (r.commitId) {
-        return <GitHubLink url={r.url!} commitId={r.commitId} text={r.commitId} />;
+        return <GitHubLink url={r.url!} link={urlLink} commitId={r.commitId} text={r.commitId}/>;
     }
-    return <GitHubLink url={r.url!} branch={r.branch} text={r.branch} />;
+    return <GitHubLink url={r.url!} link={urlLink} branch={r.branch} text={r.branch}/>;
 };
 
 const renderTableRow = (

console2/src/validation.ts

@@ -30,7 +30,7 @@ export const REPOSITORY_SSH_URL_PATTERN = /^(ssh:\/\/)?([a-zA-Z0-9\-_.]+)@([^:]+
 const requiredError = () => 'Required';
 const tooLongError = (n: number) => `Must be not more than ${n} characters.`;
 const invalidRepositoryUrlError = () =>
-    "Invalid repository URL: must begin with 'https://', 'ssh://' or use 'user@host:path' scheme.";
+    "Invalid repository URL: must begin with 'https://', 'mvn://', 'ssh://' or use 'user@host:path' scheme.";
 const invalidCommitIdError = () => 'Invalid commit ID: must be a valid revision.';
 const concordKeyPatternError = () =>
     "Must start with a digit or a letter, may contain only digits, letters, underscores, hyphens, tildes, '.' or '@' or. Must be between 3 and 128 characters in length.";
@@ -70,7 +70,7 @@ const repositoryUrlValidator = (v?: string) => {
         return requiredError();
     }
 
-    if (!v.startsWith('https://') && !v.match(REPOSITORY_SSH_URL_PATTERN)) {
+    if (!v.startsWith('https://') && !v.match(REPOSITORY_SSH_URL_PATTERN) && !v.startsWith('mvn://')) {
         return invalidRepositoryUrlError();
     }
 

it/server/pom.xml

@@ -34,7 +34,7 @@
         <oldap.image>osixia/openldap</oldap.image>
         <server.image>walmartlabs/concord-server</server.image>
         <skip.socat>true</skip.socat>
-        <socat.image>bobrik/socat</socat.image>
+        <socat.image>alpine/socat</socat.image>
         <tmp.dir.src.mount>${java.io.tmpdir}</tmp.dir.src.mount>
         <tmp.dir>${java.io.tmpdir}</tmp.dir>
     </properties>
@@ -250,6 +250,7 @@
                     <systemProperties>
                         <java.io.tmpdir>${tmp.dir}</java.io.tmpdir>
                         <isDocker>${is.docker.profile}</isDocker>
+                        <local.mvn.repo>${local.repository.src.mount}</local.mvn.repo>
                     </systemProperties>
                 </configuration>
             </plugin>
@@ -529,6 +530,9 @@
                                                 <volume>${base.dir}/src/test/resources/server.conf:/opt/concord/conf/server.conf:ro</volume>
                                                 <!-- provide the default process variables cfg -->
                                                 <volume>${base.dir}/src/test/resources/default_vars.yml:/opt/concord/conf/default_vars.yml:ro</volume>
+                                                <!-- share host artifacts -->
+                                                <volume>${local.repository.src.mount}:/host/.m2/repository:ro</volume>
+                                                <volume>${base.dir}/src/test/resources/mvn.json:/opt/concord/conf/mvn.json:ro</volume>
                                             </bind>
                                         </volumes>
                                         <env>
@@ -543,6 +547,7 @@
                                             <NODEROSTER_DB_PASSWORD>it</NODEROSTER_DB_PASSWORD>
                                             <NODEROSTER_DB_URL>jdbc:postgresql://${it.db.addr}/postgres</NODEROSTER_DB_URL>
                                             <NODEROSTER_DB_USERNAME>postgres</NODEROSTER_DB_USERNAME>
+                                            <CONCORD_MAVEN_CFG>/opt/concord/conf/mvn.json</CONCORD_MAVEN_CFG>
                                         </env>
                                         <wait>
                                             <!--suppress MavenModelInspection -->

it/server/src/test/java/com/walmartlabs/concord/it/server/MavenRepoIT.java

@@ -0,0 +1,81 @@
+package com.walmartlabs.concord.it.server;
+
+/*-
+ * *****
+ * Concord
+ * -----
+ * Copyright (C) 2017 - 2018 Walmart Inc.
+ * -----
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =====
+ */
+
+import com.walmartlabs.concord.client2.*;
+import com.walmartlabs.concord.common.IOUtils;
+import org.junit.jupiter.api.Test;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.util.Collections;
+
+import static com.walmartlabs.concord.it.common.ServerClient.assertLog;
+import static com.walmartlabs.concord.it.common.ServerClient.waitForCompletion;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class MavenRepoIT extends AbstractServerIT {
+
+    @Test
+    public void test() throws Exception {
+        // prepare local mvn repo
+        Path localMavenRepo;
+        try {
+            localMavenRepo = Path.of(System.getProperty("local.mvn.repo"));
+        } catch (NullPointerException e) {
+            localMavenRepo = Path.of(System.getProperty("user.home")).resolve(".m2/repository");
+        }
+        Path mvnDirectory = localMavenRepo.resolve("com/walmartlabs/concord/mvn-concord/0.0.1");
+
+        Path src = Paths.get(MavenRepoIT.class.getResource("mvnRepoFiles").toURI());
+        IOUtils.copy(src, mvnDirectory, StandardCopyOption.REPLACE_EXISTING);
+
+        String url = "mvn://com.walmartlabs.concord:mvn-concord:zip";
+        String projectName = "project_" + randomString();
+        String repoName = "repo_" + randomString();
+
+        ProjectsApi projectsApi = new ProjectsApi(getApiClient());
+        projectsApi.createOrUpdateProject("Default", new ProjectEntry()
+                .name(projectName)
+                .acceptsRawPayload(false)
+                .repositories(Collections.singletonMap(repoName, new RepositoryEntry()
+                        .name(repoName)
+                        .url(url)
+                        .branch("0.0.1"))));
+
+        // ---
+
+        StartProcessResponse spr = start("Default", projectName, repoName, null, null);
+        assertTrue(spr.getOk());
+
+        // ---
+
+        ProcessEntry pe = waitForCompletion(getApiClient(), spr.getInstanceId());
+        assertEquals(ProcessEntry.StatusEnum.FINISHED, pe.getStatus());
+
+        // ---
+
+        byte[] ab = getLog(pe.getInstanceId());
+        assertLog(".*OK.*", ab);
+    }
+}

repository/pom.xml

@@ -26,6 +26,10 @@
             <groupId>com.google.guava</groupId>
             <artifactId>guava</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.walmartlabs.concord</groupId>
+            <artifactId>concord-dependency-manager</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-api</artifactId>

repository/src/main/java/com/walmartlabs/concord/repository/MavenRepositoryProvider.java

@@ -0,0 +1,92 @@
+package com.walmartlabs.concord.repository;
+
+import com.walmartlabs.concord.common.IOUtils;
+import com.walmartlabs.concord.dependencymanager.DependencyManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * *****
+ * Concord
+ * -----
+ * Copyright (C) 2025 Walmart Inc.
+ * -----
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =====
+ */
+public class MavenRepositoryProvider implements RepositoryProvider {
+
+    private static final String URL_PREFIX = "mvn://";
+    private static final Logger log = LoggerFactory.getLogger(MavenRepositoryProvider.class);
+    private final DependencyManager dependencyManager;
+
+    public MavenRepositoryProvider(DependencyManager dependencyManager) {
+        this.dependencyManager = dependencyManager;
+    }
+
+    /**
+     * @param url maven repo url in format mvn://groupId:artifactId:extension
+     * @return boolean can handle or not
+     */
+    @Override
+    public boolean canHandle(String url) {
+        return url.startsWith(URL_PREFIX);
+    }
+
+    /**
+     * @param request fetchRequest
+     * @return fetchResult
+     */
+    @Override
+    public FetchResult fetch(FetchRequest request) {
+        Path dst = request.destination();
+        try {
+            URI uri = new URI(request.url().concat(":").concat(request.version().value()));
+            Path dependencyPath = dependencyManager.resolveSingle(uri).getPath();
+            IOUtils.unzip(dependencyPath, dst, false, StandardCopyOption.REPLACE_EXISTING);
+            return null;
+        } catch (URISyntaxException | IOException e) {
+            try {
+                IOUtils.deleteRecursively(request.destination());
+            } catch (IOException ee) {
+                log.warn("fetch ['{}', '{}', '{}'] -> cleanup error: {}",
+                        request.url(), request.version(), request.destination(), e.getMessage());
+            }
+            throw new RepositoryException("Error while fetching a repository", e);
+        }
+    }
+
+    /**
+     * @param src source of the fetched repo
+     * @param dst destination to be copied to
+     * @param ignorePatterns ignore some files while copying
+     * @return  snapshot of copied files
+     * @throws IOException exception during IO operation
+     */
+    @Override
+    public Snapshot export(Path src, Path dst, List<String> ignorePatterns) throws IOException {
+        LastModifiedSnapshot snapshot = new LastModifiedSnapshot();
+        List<String> allIgnorePatterns = new ArrayList<>();
+        allIgnorePatterns.addAll(ignorePatterns);
+        IOUtils.copy(src, dst, allIgnorePatterns, snapshot, StandardCopyOption.REPLACE_EXISTING);
+        return snapshot;
+    }
+}