Skip to content

Commit

Permalink
Added the possibility to download scan resources from a custom reposi…
Browse files Browse the repository at this point in the history 
…tory in Artifactory, instead of JFrog's repository.
  • Loading branch information
@asafgabai
asafgabai committed Nov 1, 2023
1 parent 72a836e commit 2001d7f
Show file tree
Hide file tree
Showing 19 changed files with 246 additions and 252 deletions.
6 changes: 5 additions & 1 deletion build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,11 @@ def idePluginsCommonVersion = '2.3.0'
dependencies {
implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml', version: '2.15.2'
implementation group: 'org.jfrog.buildinfo', name: 'build-info-extractor', version: buildInfoVersion
implementation group: 'com.jfrog.ide', name: 'ide-plugins-common', version: idePluginsCommonVersion
implementation('com.jfrog.ide:ide-plugins-common') {
version {
branch = 'air-gap-support'
}
}
implementation group: 'org.jfrog.buildinfo', name: 'build-info-client', version: buildInfoVersion
implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.15.2'
implementation group: 'org.jfrog.buildinfo', name: 'build-info-api', version: buildInfoVersion
Expand Down
6 changes: 6 additions & 0 deletions settings.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,10 @@ pluginManagement {
}
}

sourceControl {
gitRepository("https://github.com/asafgabai/ide-plugins-common.git") {
producesModule("com.jfrog.ide:ide-plugins-common")
}
}

rootProject.name = 'jfrog-idea-plugin'
2 changes: 2 additions & 0 deletions src/main/java/com/jfrog/ide/idea/configuration/GlobalSettings.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ public GlobalSettings getState() {
serverConfig.setWatches(this.serverConfig.getWatches());
serverConfig.setConnectionRetries(this.serverConfig.getConnectionRetries());
serverConfig.setConnectionTimeout(this.serverConfig.getConnectionTimeout());
serverConfig.setCustomResourcesRepo(this.serverConfig.getCustomResourcesRepo());

GlobalSettings settings = new GlobalSettings();
settings.serverConfig = serverConfig;
Expand Down Expand Up @@ -130,6 +131,7 @@ private void setAdvancedSettings(ServerConfigImpl serverConfig) {
this.serverConfig.setExcludedPaths(serverConfig.getExcludedPaths());
this.serverConfig.setConnectionRetries(serverConfig.getConnectionRetries());
this.serverConfig.setConnectionTimeout(serverConfig.getConnectionTimeout());
this.serverConfig.setCustomResourcesRepo(serverConfig.getCustomResourcesRepo());
this.serverConfig.setPolicyType(serverConfig.getPolicyType());
this.serverConfig.setProject(serverConfig.getProject());
this.serverConfig.setWatches(serverConfig.getWatches());
Expand Down
23 changes: 21 additions & 2 deletions src/main/java/com/jfrog/ide/idea/configuration/ServerConfigImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,8 @@ public enum ConnectionType {
private Integer connectionRetries;
@Tag
private Integer connectionTimeout;
@Tag
private String customResourcesRepo;
// The subsystem key of the plugin configuration in the PasswordSafe
@Transient
private String jfrogSettingsCredentialsKey = JFROG_SETTINGS_KEY;
Expand All @@ -121,6 +123,7 @@ public enum ConnectionType {
this.excludedPaths = builder.excludedPaths;
this.connectionRetries = builder.connectionRetries;
this.connectionTimeout = builder.connectionTimeout;
this.customResourcesRepo = builder.customResourcesRepo;
this.jfrogSettingsCredentialsKey = builder.jfrogSettingsCredentialsKey;
}

Expand Down Expand Up @@ -155,13 +158,14 @@ public boolean equals(Object o) {
Objects.equals(getWatches(), other.getWatches()) &&
Objects.equals(getExcludedPaths(), other.getExcludedPaths()) &&
getConnectionRetries() == other.getConnectionRetries() &&
getConnectionTimeout() == other.getConnectionTimeout();
getConnectionTimeout() == other.getConnectionTimeout() &&
getCustomResourcesRepo() == other.getCustomResourcesRepo();
}

@Override
public int hashCode() {
return Objects.hash(getConnectionType(), getUrl(), getXrayUrl(), getArtifactoryUrl(), getPassword(), getAccessToken(),
getUsername(), getProject(), getExcludedPaths(), getConnectionRetries(), getConnectionTimeout());
getUsername(), getProject(), getExcludedPaths(), getConnectionRetries(), getConnectionTimeout(), getCustomResourcesRepo());
}

@Override
Expand Down Expand Up @@ -257,6 +261,11 @@ public int getConnectionTimeout() {
return defaultIfNull(this.connectionTimeout, ConnectionTimeoutSpinner.RANGE.initial);
}

@Override
public String getCustomResourcesRepo() {
return this.customResourcesRepo;
}

public String getJFrogSettingsCredentialsKey() {
return this.jfrogSettingsCredentialsKey;
}
Expand Down Expand Up @@ -402,6 +411,10 @@ void setConnectionTimeout(int connectionTimeout) {
this.connectionTimeout = connectionTimeout;
}

void setCustomResourcesRepo(String customResourcesRepo) {
this.customResourcesRepo = customResourcesRepo;
}

public void setJFrogSettingsCredentialsKey(String jfrogSettingsCredentialsKey) {
this.jfrogSettingsCredentialsKey = jfrogSettingsCredentialsKey;
}
Expand Down Expand Up @@ -505,6 +518,7 @@ public static class Builder {
private String watches;
private int connectionRetries;
private int connectionTimeout;
private String customResourcesRepo;

public ServerConfigImpl build() {
return new ServerConfigImpl(this);
Expand Down Expand Up @@ -577,6 +591,11 @@ public Builder setConnectionTimeout(int connectionTimeout) {
return this;
}

public Builder setCustomResourcesRepo(String customResourcesRepo) {
this.customResourcesRepo = customResourcesRepo;
return this;
}

public Builder setJFrogSettingsCredentialsKey(String jfrogSettingsCredentialsKey) {
this.jfrogSettingsCredentialsKey = jfrogSettingsCredentialsKey;
return this;
Expand Down
9 changes: 2 additions & 7 deletions src/main/java/com/jfrog/ide/idea/scan/ApplicabilityScannerExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,8 @@ public class ApplicabilityScannerExecutor extends ScanBinaryExecutor {
private static final List<String> SCANNER_ARGS = List.of("ca");
private static final List<PackageManagerType> SUPPORTED_PACKAGE_TYPES = List.of(PackageManagerType.PYPI, PackageManagerType.NPM, PackageManagerType.YARN, PackageManagerType.GRADLE, PackageManagerType.MAVEN);


public ApplicabilityScannerExecutor(Log log, ServerConfig serverConfig) {
this(log, serverConfig, "", true);
}

public ApplicabilityScannerExecutor(Log log, ServerConfig serverConfig, String binaryDownloadUrl, boolean useJFrogReleases) {
super(SourceCodeScanType.CONTEXTUAL, binaryDownloadUrl, log, serverConfig, useJFrogReleases);
public ApplicabilityScannerExecutor(Log log) {
super(SourceCodeScanType.CONTEXTUAL, log);
supportedPackageTypes = SUPPORTED_PACKAGE_TYPES;
}

Expand Down
8 changes: 2 additions & 6 deletions src/main/java/com/jfrog/ide/idea/scan/IACScannerExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,8 @@ public class IACScannerExecutor extends ScanBinaryExecutor {
private static final List<String> SCANNER_ARGS = List.of("iac");
private static final String ISSUE_TITLE = "Infrastructure as Code Vulnerability";

public IACScannerExecutor(Log log, ServerConfig serverConfig) {
this(log, serverConfig, null, true);
}

public IACScannerExecutor(Log log, ServerConfig serverConfig, String binaryDownloadUrl, boolean useJFrogReleases) {
super(SourceCodeScanType.IAC, binaryDownloadUrl, log, serverConfig, useJFrogReleases);
public IACScannerExecutor(Log log) {
super(SourceCodeScanType.IAC, log);
}

public List<JFrogSecurityWarning> execute(ScanConfig.Builder inputFileBuilder, Runnable checkCanceled) throws IOException, InterruptedException {
Expand Down
8 changes: 2 additions & 6 deletions src/main/java/com/jfrog/ide/idea/scan/SastScannerExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,8 @@ public class SastScannerExecutor extends ScanBinaryExecutor {
private static final boolean RUN_WITH_NEW_CONFIG_FILE = true;
private static final List<PackageManagerType> SUPPORTED_PACKAGE_TYPES = List.of(PackageManagerType.PYPI, PackageManagerType.NPM, PackageManagerType.YARN, PackageManagerType.GRADLE, PackageManagerType.MAVEN);

public SastScannerExecutor(Log log, ServerConfig serverConfig) {
this(log, serverConfig, null, true);
}

public SastScannerExecutor(Log log, ServerConfig serverConfig, String binaryDownloadUrl, boolean useJFrogReleases) {
super(SourceCodeScanType.SAST, binaryDownloadUrl, log, serverConfig, useJFrogReleases);
public SastScannerExecutor(Log log) {
super(SourceCodeScanType.SAST, log);
}

public List<JFrogSecurityWarning> execute(ScanConfig.Builder inputFileBuilder, Runnable checkCanceled) throws IOException, InterruptedException {
Expand Down
108 changes: 56 additions & 52 deletions src/main/java/com/jfrog/ide/idea/scan/ScanBinaryExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ public abstract class ScanBinaryExecutor {
private static final Path BINARIES_DIR = HOME_PATH.resolve("dependencies").resolve("jfrog-security");
private static final String SCANNER_BINARY_NAME = "analyzerManager";
private static final String SCANNER_BINARY_VERSION = "1.3.2.2019257";
private static final String DEFAULT_BINARY_DOWNLOAD_URL = "xsc-gen-exe-analyzer-manager-local/v1/" + SCANNER_BINARY_VERSION;
private static final String BINARY_DOWNLOAD_URL = "xsc-gen-exe-analyzer-manager-local/v1/" + SCANNER_BINARY_VERSION;
private static final String DOWNLOAD_SCANNER_NAME = "analyzerManager.zip";
private static final String MINIMAL_XRAY_VERSION_SUPPORTED_FOR_ENTITLEMENT = "3.66.0";
private static final String ENV_PLATFORM = "JF_PLATFORM_URL";
Expand All @@ -70,27 +70,23 @@ public abstract class ScanBinaryExecutor {
private static final String ENV_HTTP_PROXY = "HTTP_PROXY";
private static final String ENV_HTTPS_PROXY = "HTTPS_PROXY";
private static final String JFROG_RELEASES = "https://releases.jfrog.io/artifactory/";
private final String BINARY_DOWNLOAD_URL;
private static Path binaryTargetPath;
private static Path archiveTargetPath;
@Getter
private static String osDistribution;
private static LocalDateTime nextUpdateCheck;
private final ArtifactoryManagerBuilder artifactoryManagerBuilder;
protected final SourceCodeScanType scanType;
protected Collection<PackageManagerType> supportedPackageTypes;
private final Log log;
private boolean notSupported;
private final static Object downloadLock = new Object();

ScanBinaryExecutor(SourceCodeScanType scanType, String binaryDownloadUrl, Log log, ServerConfig server, boolean useJFrogReleases) {
ScanBinaryExecutor(SourceCodeScanType scanType, Log log) {
this.scanType = scanType;
this.log = log;
BINARY_DOWNLOAD_URL = defaultIfEmpty(binaryDownloadUrl, DEFAULT_BINARY_DOWNLOAD_URL);
String executable = SystemUtils.IS_OS_WINDOWS ? SCANNER_BINARY_NAME + ".exe" : SCANNER_BINARY_NAME;
binaryTargetPath = BINARIES_DIR.resolve(SCANNER_BINARY_NAME).resolve(executable);
archiveTargetPath = BINARIES_DIR.resolve(DOWNLOAD_SCANNER_NAME);
artifactoryManagerBuilder = createManagerBuilder(useJFrogReleases, server);
setOsDistribution();
}

Expand All @@ -116,8 +112,12 @@ protected void setOsDistribution() {
}
}

public String getBinaryDownloadURL() {
return String.format("%s/%s/%s", BINARY_DOWNLOAD_URL, getOsDistribution(), DOWNLOAD_SCANNER_NAME);
String getBinaryDownloadURL(String customResourcesRepo) {
String downloadUrlPrefix = "";
if (!StringUtils.isEmpty(customResourcesRepo)) {
downloadUrlPrefix = String.format("%s/artifactory/", customResourcesRepo);
}
return String.format("%s%s/%s/%s", downloadUrlPrefix, BINARY_DOWNLOAD_URL, getOsDistribution(), DOWNLOAD_SCANNER_NAME);
}

abstract Feature getScannerFeatureName();
Expand Down Expand Up @@ -192,38 +192,44 @@ protected List<JFrogSecurityWarning> execute(ScanConfig.Builder inputFileBuilder
private void updateBinaryIfNeeded() throws IOException {
// Allow only one thread to check and update the binary at any time.
synchronized (downloadLock) {
if (!Files.exists(binaryTargetPath)) {
log.debug(String.format("Resource %s is not found. Downloading it.", binaryTargetPath));
downloadBinary();
LocalDateTime currentTime = LocalDateTime.now();
if (nextUpdateCheck != null && currentTime.isBefore(nextUpdateCheck)) {
return;
}
LocalDateTime currentTime = LocalDateTime.now();
if (nextUpdateCheck == null || currentTime.isAfter(nextUpdateCheck)) {
nextUpdateCheck = currentTime.plusDays(UPDATE_INTERVAL);
// Check for new version of the binary
try (FileInputStream archiveBinaryFile = new FileInputStream(archiveTargetPath.toFile())) {
String latestBinaryChecksum = getFileChecksumFromServer();
String currentBinaryCheckSum = DigestUtils.sha256Hex(archiveBinaryFile);
if (!latestBinaryChecksum.equals(currentBinaryCheckSum)) {
log.debug(String.format("Resource %s is not up to date. Downloading it.", archiveTargetPath));
downloadBinary();
ServerConfig server = GlobalSettings.getInstance().getServerConfig();
String customResourcesRepo = server.getCustomResourcesRepo();
ArtifactoryManagerBuilder artifactoryManagerBuilder = createManagerBuilder(StringUtils.isEmpty(customResourcesRepo), server);
try (ArtifactoryManager artifactoryManager = artifactoryManagerBuilder.build()) {
if (Files.exists(binaryTargetPath)) {
// Check for new version of the binary
try (FileInputStream archiveBinaryFile = new FileInputStream(archiveTargetPath.toFile())) {
String latestBinaryChecksum = getFileChecksumFromServer(artifactoryManager, customResourcesRepo);
String currentBinaryCheckSum = DigestUtils.sha256Hex(archiveBinaryFile);
if (latestBinaryChecksum.equals(currentBinaryCheckSum)) {
nextUpdateCheck = currentTime.plusDays(UPDATE_INTERVAL);
return;
} else {
log.debug(String.format("Resource %s is not up to date. Downloading it.", archiveTargetPath));
}
}
} else {
log.debug(String.format("Resource %s is not found. Downloading it.", binaryTargetPath));
}
downloadBinary(artifactoryManager, customResourcesRepo);
}
}
}

public String getFileChecksumFromServer() throws IOException {
try (ArtifactoryManager artifactoryManager = artifactoryManagerBuilder.build()) {
Header[] headers = artifactoryManager.downloadHeaders(getBinaryDownloadURL());
for (Header header : headers) {
if (StringUtils.equalsIgnoreCase(header.getName(), "x-checksum-sha256")) {
return header.getValue();
}
public String getFileChecksumFromServer(ArtifactoryManager artifactoryManager, String customResourcesRepo) throws IOException {
String url = getBinaryDownloadURL(customResourcesRepo);
Header[] headers = artifactoryManager.downloadHeaders(url);
for (Header header : headers) {
if (StringUtils.equalsIgnoreCase(header.getName(), "x-checksum-sha256")) {
return header.getValue();
}
log.warn(String.format("Failed to retrieve file checksum from: %s/%s ", artifactoryManager.getUrl(), getBinaryDownloadURL()));
return "";
}
log.warn(String.format("Failed to retrieve file checksum from: %s/%s ", artifactoryManager.getUrl(), url));
return "";
}

protected boolean shouldExecute() {
Expand Down Expand Up @@ -270,28 +276,26 @@ protected Output getOutputObj(Path outputFile) throws IOException {
return om.readValue(outputFile.toFile(), Output.class);
}

protected void downloadBinary() throws IOException {
try (ArtifactoryManager artifactoryManager = artifactoryManagerBuilder.build()) {
String downloadUrl = getBinaryDownloadURL();
File downloadArchive = artifactoryManager.downloadToFile(downloadUrl, archiveTargetPath.toString());
log.debug(String.format("Downloading: %s", downloadUrl));
if (downloadArchive == null) {
throw new IOException("An empty response received from Artifactory.");
}
// Delete current scanners
FileUtils.deleteDirectory(binaryTargetPath.toFile().getParentFile());
// Extract archive
UnzipParameters params = new UnzipParameters();
params.setExtractSymbolicLinks(false);
try (ZipFile zip = new ZipFile(archiveTargetPath.toFile())) {
zip.extractAll(binaryTargetPath.toFile().getParentFile().toString(), params);
} catch (ZipException exception) {
throw new IOException("An error occurred while trying to unarchived the JFrog executable:\n" + exception.getMessage());
}
// Set executable permissions to the downloaded scanner
if (!binaryTargetPath.toFile().setExecutable(true)) {
throw new IOException("An error occurred while trying to give access permissions to the JFrog executable.");
}
protected void downloadBinary(ArtifactoryManager artifactoryManager, String customResourcesRepo) throws IOException {
String downloadUrl = getBinaryDownloadURL(customResourcesRepo);
File downloadArchive = artifactoryManager.downloadToFile(downloadUrl, archiveTargetPath.toString());
log.debug(String.format("Downloading: %s", downloadUrl));
if (downloadArchive == null) {
throw new IOException("An empty response received from Artifactory.");
}
// Delete current scanners
FileUtils.deleteDirectory(binaryTargetPath.toFile().getParentFile());
// Extract archive
UnzipParameters params = new UnzipParameters();
params.setExtractSymbolicLinks(false);
try (ZipFile zip = new ZipFile(archiveTargetPath.toFile())) {
zip.extractAll(binaryTargetPath.toFile().getParentFile().toString(), params);
} catch (ZipException exception) {
throw new IOException("An error occurred while trying to unarchived the JFrog executable:\n" + exception.getMessage());
}
// Set executable permissions to the downloaded scanner
if (!binaryTargetPath.toFile().setExecutable(true)) {
throw new IOException("An error occurred while trying to give access permissions to the JFrog executable.");
}
}

Expand Down
8 changes: 2 additions & 6 deletions src/main/java/com/jfrog/ide/idea/scan/SecretsScannerExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,8 @@ public class SecretsScannerExecutor extends ScanBinaryExecutor {
private static final List<String> SCANNER_ARGS = List.of("sec");
private static final String ISSUE_TITLE = "Potential Secret";

public SecretsScannerExecutor(Log log, ServerConfig serverConfig) {
this(log, serverConfig, null, true);
}

public SecretsScannerExecutor(Log log, ServerConfig serverConfig, String binaryDownloadUrl, boolean useJFrogReleases) {
super(SourceCodeScanType.SECRETS, binaryDownloadUrl, log, serverConfig, useJFrogReleases);
public SecretsScannerExecutor(Log log) {
super(SourceCodeScanType.SECRETS, log);
}

public List<JFrogSecurityWarning> execute(ScanConfig.Builder inputFileBuilder, Runnable checkCanceled) throws IOException, InterruptedException {
Expand Down
Loading

0 comments on commit 2001d7f

Please sign in to comment.