diff --git a/agent/src/main/java/com/walmartlabs/concord/agent/JobRequest.java b/agent/src/main/java/com/walmartlabs/concord/agent/JobRequest.java index ef5deebd3e..ebc5d38dd4 100644 --- a/agent/src/main/java/com/walmartlabs/concord/agent/JobRequest.java +++ b/agent/src/main/java/com/walmartlabs/concord/agent/JobRequest.java @@ -147,6 +147,7 @@ public String toString() { ", repoUrl='" + repoUrl + '\'' + ", repoPath='" + repoPath + '\'' + ", commitId='" + commitId + '\'' + + ", repoBranch='" + repoBranch + '\'' + ", secretName='" + secretName + '\'' + ", imports='" + imports + '\'' + '}'; diff --git a/agent/src/main/java/com/walmartlabs/concord/agent/RepositoryManager.java b/agent/src/main/java/com/walmartlabs/concord/agent/RepositoryManager.java index 43415a71bb..e500c41d2c 100644 --- a/agent/src/main/java/com/walmartlabs/concord/agent/RepositoryManager.java +++ b/agent/src/main/java/com/walmartlabs/concord/agent/RepositoryManager.java @@ -27,11 +27,12 @@ import com.walmartlabs.concord.imports.Import.SecretDefinition; import com.walmartlabs.concord.repository.*; import com.walmartlabs.concord.sdk.Secret; +import com.walmartlabs.concord.dependencymanager.DependencyManager; import javax.inject.Inject; import java.io.IOException; import java.nio.file.Path; -import java.util.Collections; +import java.util.Arrays; import java.util.List; public class RepositoryManager { @@ -45,7 +46,8 @@ public class RepositoryManager { public RepositoryManager(SecretClient secretClient, GitConfiguration gitCfg, RepositoryCacheConfiguration cacheCfg, - ObjectMapper objectMapper) throws IOException { + ObjectMapper objectMapper, + DependencyManager dependencyManager) throws IOException { this.secretClient = secretClient; this.gitCfg = gitCfg; @@ -60,7 +62,7 @@ public RepositoryManager(SecretClient secretClient, .sshTimeoutRetryCount(gitCfg.getSshTimeoutRetryCount()) .build(); - List providers = Collections.singletonList(new GitCliRepositoryProvider(clientCfg)); + List providers = Arrays.asList(new MavenRepositoryProvider(dependencyManager), new GitCliRepositoryProvider(clientCfg)); this.providers = new RepositoryProviders(providers); this.repositoryCache = new RepositoryCache(cacheCfg.getCacheDir(), diff --git a/agent/src/main/java/com/walmartlabs/concord/agent/Worker.java b/agent/src/main/java/com/walmartlabs/concord/agent/Worker.java index c5218430fc..be3f6f5ac2 100644 --- a/agent/src/main/java/com/walmartlabs/concord/agent/Worker.java +++ b/agent/src/main/java/com/walmartlabs/concord/agent/Worker.java @@ -142,7 +142,9 @@ private void onStatusChange(UUID instanceId, StatusEnum status) { } private void fetchRepo(JobRequest r) throws Exception { - if (r.getRepoUrl() == null || r.getCommitId() == null) { + if (r.getRepoUrl() == null + || (r.getCommitId() == null && r.getRepoBranch() == null) + || r.getRepoUrl().startsWith("classpath://")) { return; } diff --git a/console2/src/components/molecules/GitHubLink/index.tsx b/console2/src/components/molecules/GitHubLink/index.tsx index 10a8eb530e..1a017ae097 100644 --- a/console2/src/components/molecules/GitHubLink/index.tsx +++ b/console2/src/components/molecules/GitHubLink/index.tsx @@ -23,6 +23,7 @@ import { REPOSITORY_SSH_URL_PATTERN } from '../../../validation'; interface Props { url: string; + link?: string; path?: string; commitId?: string; text?: string; @@ -59,9 +60,9 @@ const normalizePath = (s: string): string => { class GitHubLink extends React.PureComponent { render() { - const { url, commitId, path, text, branch } = this.props; + const { url, link, commitId, path, text, branch } = this.props; - let s = gitUrlParse(url); + let s = !link ? gitUrlParse(url) : link; if (!s) { return url; } diff --git a/console2/src/components/molecules/RepositoryForm/index.tsx b/console2/src/components/molecules/RepositoryForm/index.tsx index aa1e7d5550..d4b4fe5b19 100644 --- a/console2/src/components/molecules/RepositoryForm/index.tsx +++ b/console2/src/components/molecules/RepositoryForm/index.tsx @@ -71,7 +71,7 @@ interface State { const sourceOptions = [ { - text: 'Branch/tag', + text: 'Branch/tag/version', value: RepositorySourceType.BRANCH_OR_TAG }, { @@ -224,7 +224,7 @@ class RepositoryForm extends React.Component @@ -348,10 +348,10 @@ const validator = async (values: FormValues, props: Props) => { } if (!values.withSecret) { - if (!values.url.startsWith('https://')) { + if (!values.url.startsWith('https://') && !values.url.startsWith('mvn://')) { return Promise.resolve({ url: - "Invalid repository URL: must begin with 'https://'. SSH repository URLs require additional credentials to be specified." + "Invalid repository URL: must begin with 'https://' or 'mvn://'. SSH repository URLs require additional credentials to be specified." }); } } else { diff --git a/console2/src/components/molecules/RepositoryList/index.tsx b/console2/src/components/molecules/RepositoryList/index.tsx index c550a0637a..34bbfe166d 100644 --- a/console2/src/components/molecules/RepositoryList/index.tsx +++ b/console2/src/components/molecules/RepositoryList/index.tsx @@ -26,6 +26,7 @@ import { ConcordKey } from '../../../api/common'; import { RepositoryEntry } from '../../../api/org/project/repository'; import { GitHubLink } from '../../molecules'; import { RepositoryActionDropdown } from '../../organisms'; +import { gitUrlParse } from "../GitHubLink"; interface ExternalProps { orgName: ConcordKey; @@ -44,7 +45,7 @@ const RepositoryList = ({ orgName, projectName, data, loading, refresh }: Extern Name Repository URL - Branch/Commit ID + Branch/Commit ID/Version Path Secret @@ -72,9 +73,14 @@ const RepositoryList = ({ orgName, projectName, data, loading, refresh }: Extern }; const renderRepoPath = (r: RepositoryEntry) => { + const urlLink = gitUrlParse(r.url); + if (!urlLink) { + return r.path; + } if (r.commitId) { return ( { /> ); } - return ; + return ; }; const renderRepoCommitIdOrBranch = (r: RepositoryEntry) => { + + const urlLink = gitUrlParse(r.url); + if (!urlLink) { + return r.branch; + } + if (r.commitId) { - return ; + return ; } - return ; + return ; }; const renderTableRow = ( diff --git a/console2/src/validation.ts b/console2/src/validation.ts index c9b1b86099..274a92083d 100644 --- a/console2/src/validation.ts +++ b/console2/src/validation.ts @@ -30,7 +30,7 @@ export const REPOSITORY_SSH_URL_PATTERN = /^(ssh:\/\/)?([a-zA-Z0-9\-_.]+)@([^:]+ const requiredError = () => 'Required'; const tooLongError = (n: number) => `Must be not more than ${n} characters.`; const invalidRepositoryUrlError = () => - "Invalid repository URL: must begin with 'https://', 'ssh://' or use 'user@host:path' scheme."; + "Invalid repository URL: must begin with 'https://', 'mvn://', 'ssh://' or use 'user@host:path' scheme."; const invalidCommitIdError = () => 'Invalid commit ID: must be a valid revision.'; const concordKeyPatternError = () => "Must start with a digit or a letter, may contain only digits, letters, underscores, hyphens, tildes, '.' or '@' or. Must be between 3 and 128 characters in length."; @@ -70,7 +70,7 @@ const repositoryUrlValidator = (v?: string) => { return requiredError(); } - if (!v.startsWith('https://') && !v.match(REPOSITORY_SSH_URL_PATTERN)) { + if (!v.startsWith('https://') && !v.match(REPOSITORY_SSH_URL_PATTERN) && !v.startsWith('mvn://')) { return invalidRepositoryUrlError(); } diff --git a/it/server/pom.xml b/it/server/pom.xml index a45cefdf03..40cff907d7 100644 --- a/it/server/pom.xml +++ b/it/server/pom.xml @@ -34,7 +34,7 @@ osixia/openldap walmartlabs/concord-server true - bobrik/socat + alpine/socat ${java.io.tmpdir} ${java.io.tmpdir} @@ -250,6 +250,7 @@ ${tmp.dir} ${is.docker.profile} + ${local.repository.src.mount} @@ -529,6 +530,9 @@ ${base.dir}/src/test/resources/server.conf:/opt/concord/conf/server.conf:ro ${base.dir}/src/test/resources/default_vars.yml:/opt/concord/conf/default_vars.yml:ro + + ${local.repository.src.mount}:/host/.m2/repository:ro + ${base.dir}/src/test/resources/mvn.json:/opt/concord/conf/mvn.json:ro @@ -543,6 +547,7 @@ it jdbc:postgresql://${it.db.addr}/postgres postgres + /opt/concord/conf/mvn.json diff --git a/it/server/src/test/java/com/walmartlabs/concord/it/server/MavenRepoIT.java b/it/server/src/test/java/com/walmartlabs/concord/it/server/MavenRepoIT.java new file mode 100644 index 0000000000..229261fad6 --- /dev/null +++ b/it/server/src/test/java/com/walmartlabs/concord/it/server/MavenRepoIT.java @@ -0,0 +1,81 @@ +package com.walmartlabs.concord.it.server; + +/*- + * ***** + * Concord + * ----- + * Copyright (C) 2017 - 2018 Walmart Inc. + * ----- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ===== + */ + +import com.walmartlabs.concord.client2.*; +import com.walmartlabs.concord.common.IOUtils; +import org.junit.jupiter.api.Test; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.util.Collections; + +import static com.walmartlabs.concord.it.common.ServerClient.assertLog; +import static com.walmartlabs.concord.it.common.ServerClient.waitForCompletion; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class MavenRepoIT extends AbstractServerIT { + + @Test + public void test() throws Exception { + // prepare local mvn repo + Path localMavenRepo; + try { + localMavenRepo = Path.of(System.getProperty("local.mvn.repo")); + } catch (NullPointerException e) { + localMavenRepo = Path.of(System.getProperty("user.home")).resolve(".m2/repository"); + } + Path mvnDirectory = localMavenRepo.resolve("com/walmartlabs/concord/mvn-concord/0.0.1"); + + Path src = Paths.get(MavenRepoIT.class.getResource("mvnRepoFiles").toURI()); + IOUtils.copy(src, mvnDirectory, StandardCopyOption.REPLACE_EXISTING); + + String url = "mvn://com.walmartlabs.concord:mvn-concord:zip"; + String projectName = "project_" + randomString(); + String repoName = "repo_" + randomString(); + + ProjectsApi projectsApi = new ProjectsApi(getApiClient()); + projectsApi.createOrUpdateProject("Default", new ProjectEntry() + .name(projectName) + .acceptsRawPayload(false) + .repositories(Collections.singletonMap(repoName, new RepositoryEntry() + .name(repoName) + .url(url) + .branch("0.0.1")))); + + // --- + + StartProcessResponse spr = start("Default", projectName, repoName, null, null); + assertTrue(spr.getOk()); + + // --- + + ProcessEntry pe = waitForCompletion(getApiClient(), spr.getInstanceId()); + assertEquals(ProcessEntry.StatusEnum.FINISHED, pe.getStatus()); + + // --- + + byte[] ab = getLog(pe.getInstanceId()); + assertLog(".*OK.*", ab); + } +} diff --git a/it/server/src/test/resources/com/walmartlabs/concord/it/server/mvnRepoFiles/mvn-concord-0.0.1.zip b/it/server/src/test/resources/com/walmartlabs/concord/it/server/mvnRepoFiles/mvn-concord-0.0.1.zip new file mode 100644 index 0000000000..c5a74a77ac Binary files /dev/null and b/it/server/src/test/resources/com/walmartlabs/concord/it/server/mvnRepoFiles/mvn-concord-0.0.1.zip differ diff --git a/repository/pom.xml b/repository/pom.xml index 400679aaaa..fb88721251 100644 --- a/repository/pom.xml +++ b/repository/pom.xml @@ -26,6 +26,10 @@ com.google.guava guava + + com.walmartlabs.concord + concord-dependency-manager + org.slf4j slf4j-api diff --git a/repository/src/main/java/com/walmartlabs/concord/repository/MavenRepositoryProvider.java b/repository/src/main/java/com/walmartlabs/concord/repository/MavenRepositoryProvider.java new file mode 100644 index 0000000000..7e0ae5fa44 --- /dev/null +++ b/repository/src/main/java/com/walmartlabs/concord/repository/MavenRepositoryProvider.java @@ -0,0 +1,92 @@ +package com.walmartlabs.concord.repository; + +import com.walmartlabs.concord.common.IOUtils; +import com.walmartlabs.concord.dependencymanager.DependencyManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.List; + +/** + * ***** + * Concord + * ----- + * Copyright (C) 2025 Walmart Inc. + * ----- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ===== + */ +public class MavenRepositoryProvider implements RepositoryProvider { + + private static final String URL_PREFIX = "mvn://"; + private static final Logger log = LoggerFactory.getLogger(MavenRepositoryProvider.class); + private final DependencyManager dependencyManager; + + public MavenRepositoryProvider(DependencyManager dependencyManager) { + this.dependencyManager = dependencyManager; + } + + /** + * @param url maven repo url in format mvn://groupId:artifactId:extension + * @return boolean can handle or not + */ + @Override + public boolean canHandle(String url) { + return url.startsWith(URL_PREFIX); + } + + /** + * @param request fetchRequest + * @return fetchResult + */ + @Override + public FetchResult fetch(FetchRequest request) { + Path dst = request.destination(); + try { + URI uri = new URI(request.url().concat(":").concat(request.version().value())); + Path dependencyPath = dependencyManager.resolveSingle(uri).getPath(); + IOUtils.unzip(dependencyPath, dst, false, StandardCopyOption.REPLACE_EXISTING); + return null; + } catch (URISyntaxException | IOException e) { + try { + IOUtils.deleteRecursively(request.destination()); + } catch (IOException ee) { + log.warn("fetch ['{}', '{}', '{}'] -> cleanup error: {}", + request.url(), request.version(), request.destination(), e.getMessage()); + } + throw new RepositoryException("Error while fetching a repository", e); + } + } + + /** + * @param src source of the fetched repo + * @param dst destination to be copied to + * @param ignorePatterns ignore some files while copying + * @return snapshot of copied files + * @throws IOException exception during IO operation + */ + @Override + public Snapshot export(Path src, Path dst, List ignorePatterns) throws IOException { + LastModifiedSnapshot snapshot = new LastModifiedSnapshot(); + List allIgnorePatterns = new ArrayList<>(); + allIgnorePatterns.addAll(ignorePatterns); + IOUtils.copy(src, dst, allIgnorePatterns, snapshot, StandardCopyOption.REPLACE_EXISTING); + return snapshot; + } +} diff --git a/server/impl/src/main/java/com/walmartlabs/concord/server/process/pipelines/processors/RepositoryInfoUpdateProcessor.java b/server/impl/src/main/java/com/walmartlabs/concord/server/process/pipelines/processors/RepositoryInfoUpdateProcessor.java index 1e8705bbaf..5c2280e750 100644 --- a/server/impl/src/main/java/com/walmartlabs/concord/server/process/pipelines/processors/RepositoryInfoUpdateProcessor.java +++ b/server/impl/src/main/java/com/walmartlabs/concord/server/process/pipelines/processors/RepositoryInfoUpdateProcessor.java @@ -48,8 +48,8 @@ public Payload process(Chain chain, Payload payload) { return chain.process(payload); } - String commitId = null; - String commitBranch = null; + String commitId = i.getCommitId(); + String commitBranch = i.getBranch(); RepositoryProcessor.CommitInfo ci = i.getCommitInfo(); if (ci != null) { diff --git a/server/impl/src/main/java/com/walmartlabs/concord/server/repository/RepositoryManager.java b/server/impl/src/main/java/com/walmartlabs/concord/server/repository/RepositoryManager.java index cbff486af0..0efc396551 100644 --- a/server/impl/src/main/java/com/walmartlabs/concord/server/repository/RepositoryManager.java +++ b/server/impl/src/main/java/com/walmartlabs/concord/server/repository/RepositoryManager.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.walmartlabs.concord.common.IOUtils; +import com.walmartlabs.concord.dependencymanager.DependencyManager; import com.walmartlabs.concord.process.loader.ProjectLoader; import com.walmartlabs.concord.repository.*; import com.walmartlabs.concord.sdk.Secret; @@ -60,7 +61,8 @@ public RepositoryManager(ObjectMapper objectMapper, GitConfiguration gitCfg, RepositoryConfiguration repoCfg, ProjectDao projectDao, - SecretManager secretManager) throws IOException { + SecretManager secretManager, + DependencyManager dependencyManager) throws IOException { GitClientConfiguration gitCliCfg = GitClientConfiguration.builder() .oauthToken(gitCfg.getOauthToken()) @@ -72,7 +74,7 @@ public RepositoryManager(ObjectMapper objectMapper, .sshTimeoutRetryCount(gitCfg.getSshTimeoutRetryCount()) .build(); - List providers = Arrays.asList(new ClasspathRepositoryProvider(), new GitCliRepositoryProvider(gitCliCfg)); + List providers = Arrays.asList(new ClasspathRepositoryProvider(), new MavenRepositoryProvider(dependencyManager), new GitCliRepositoryProvider(gitCliCfg)); this.gitCfg = gitCfg; this.providers = new RepositoryProviders(providers);