From 638f49dfc534b824deacce8cc943b55af78ca4de Mon Sep 17 00:00:00 2001 From: masahiro331 Date: Sun, 16 May 2021 16:57:57 +0900 Subject: [PATCH 1/5] support maven dependency.txt --- pkg/maven/parse.go | 95 ++++++ pkg/maven/parse_test.go | 48 +++ pkg/maven/parse_testcase.go | 306 ++++++++++++++++++++ pkg/maven/pom.xml | 19 ++ pkg/maven/testdata/hadoop-cloud-storage.txt | 108 +++++++ pkg/maven/testdata/hadoop-hdfs-nfs.txt | 114 ++++++++ pkg/maven/testdata/none.txt | 4 + pkg/maven/testdata/spring.txt | 5 + 8 files changed, 699 insertions(+) create mode 100644 pkg/maven/parse.go create mode 100644 pkg/maven/parse_test.go create mode 100644 pkg/maven/parse_testcase.go create mode 100644 pkg/maven/pom.xml create mode 100644 pkg/maven/testdata/hadoop-cloud-storage.txt create mode 100644 pkg/maven/testdata/hadoop-hdfs-nfs.txt create mode 100644 pkg/maven/testdata/none.txt create mode 100644 pkg/maven/testdata/spring.txt diff --git a/pkg/maven/parse.go b/pkg/maven/parse.go new file mode 100644 index 00000000..c84d22cb --- /dev/null +++ b/pkg/maven/parse.go @@ -0,0 +1,95 @@ +package maven + +import ( + "bufio" + "fmt" + "io" + "strings" + + "github.com/aquasecurity/go-dep-parser/pkg/types" + "golang.org/x/xerrors" +) + +const ( + separator = ":" + resolvedSection = "The following files have been resolved:" +) + +type Artifact struct { + GroupID string + ArtifactID string + Type string + Classifier string + Version string + Scope string +} + +func (a Artifact) getName() string { + return a.GroupID + separator + a.ArtifactID +} + +func Parse(r io.Reader) ([]types.Library, error) { + scanner := bufio.NewScanner(r) + + var libs []types.Library + var resolvedSectionFlag bool + for scanner.Scan() { + + line := scanner.Text() + if line == resolvedSection { + resolvedSectionFlag = true + continue + } + if resolvedSectionFlag && line == "" { + break + } + if !resolvedSectionFlag { + continue + } + + line = strings.TrimSpace(line) + if line == "none" { + return []types.Library{}, nil + } + + artifact, err := parseArtifact(strings.Fields(line)[0]) + if err != nil { + return nil, xerrors.Errorf("failed to parse artifact: %w", err) + } + libs = append(libs, types.Library{ + Name: artifact.getName(), + Version: artifact.Version, + }) + } + if err := scanner.Err(); err != nil { + return nil, xerrors.Errorf("scan error: %w", err) + } + + return libs, nil +} + +func parseArtifact(s string) (Artifact, error) { + ss := strings.Split(s, separator) + if len(ss) < 4 { + fmt.Println(s) + return Artifact{}, xerrors.New("invalid format error") + } + artifact := Artifact{ + GroupID: ss[0], + ArtifactID: ss[1], + Type: ss[2], + } + switch len(ss) { + case 4: + artifact.Version = ss[3] + case 5: + artifact.Version = ss[3] + artifact.Scope = ss[4] + case 6: + artifact.Classifier = ss[3] + artifact.Version = ss[4] + artifact.Scope = ss[5] + } + + return artifact, nil +} diff --git a/pkg/maven/parse_test.go b/pkg/maven/parse_test.go new file mode 100644 index 00000000..6fc45845 --- /dev/null +++ b/pkg/maven/parse_test.go @@ -0,0 +1,48 @@ +package maven + +import ( + "os" + "path" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/aquasecurity/go-dep-parser/pkg/types" +) + +func TestParse(t *testing.T) { + vectors := []struct { + file string // Test input file + want []types.Library + }{ + { + file: "testdata/spring.txt", + want: MavenNormal, + }, + { + file: "testdata/hadoop-hdfs-nfs.txt", + want: MavenHadoopHDFS, + }, + { + file: "testdata/hadoop-cloud-storage.txt", + want: MavenHadoopCloudStorage, + }, + { + file: "testdata/none.txt", + want: MavenNone, + }, + } + + for _, v := range vectors { + t.Run(path.Base(v.file), func(t *testing.T) { + f, err := os.Open(v.file) + require.NoError(t, err) + + got, err := Parse(f) + require.NoError(t, err) + + assert.Equal(t, v.want, got) + }) + } +} diff --git a/pkg/maven/parse_testcase.go b/pkg/maven/parse_testcase.go new file mode 100644 index 00000000..defac599 --- /dev/null +++ b/pkg/maven/parse_testcase.go @@ -0,0 +1,306 @@ +package maven + +import "github.com/aquasecurity/go-dep-parser/pkg/types" + +var ( + + /* pom.xml + + + 4.0.0 + io.aquasecurity + trivy + 1.0 + jar + + + + org.springframework + spring-core + [5.0.1.RELEASE] + + + + */ + // mvn dependency:list -DoutputFile=dependency.txt + /* + cat dependency.txt | grep -v "^\s*$" | grep -v "The following files have been resolved" | \ + awk ' + { + cnt=split($1, path, ":") + if(cnt == 5){ + print "{\"" path[1] ":" path[2] "\", \"" path[4] "\"}," + } + if(cnt == 6){ + print "{\"" path[1] ":" path[2] "\", \"" path[5] "\"}," + } + } + ' + */ + MavenNormal = []types.Library{ + {"org.springframework:spring-jcl", "5.0.1.RELEASE"}, + {"org.springframework:spring-core", "5.0.1.RELEASE"}, + } + + // git clone https://github.com/apache/hadoop.git + // git checkout fdd20a3cf4cd9073f443b2bf07eef14f454d4c33 + // cd hadoop/hadoop-hdfs-project/hadoop-hdfs-nfs/ + // mvn dependency:list -DoutputFile=dependency.txt + /* + cat dependency.txt | grep -v "^\s*$" | grep -v "The following files have been resolved" | \ + awk ' + { + cnt=split($1, path, ":") + if(cnt == 5){ + print "{\"" path[1] ":" path[2] "\", \"" path[4] "\"}," + } + if(cnt == 6){ + print "{\"" path[1] ":" path[2] "\", \"" path[5] "\"}," + } + } + ' + */ + MavenHadoopHDFS = []types.Library{ + {"org.apache.hadoop:hadoop-annotations", "3.4.0-SNAPSHOT"}, + {"org.apache.hadoop:hadoop-auth", "3.4.0-SNAPSHOT"}, + {"org.slf4j:slf4j-api", "1.7.30"}, + {"org.apache.httpcomponents:httpclient", "4.5.13"}, + {"org.apache.httpcomponents:httpcore", "4.4.13"}, + {"com.nimbusds:nimbus-jose-jwt", "9.8.1"}, + {"com.github.stephenc.jcip:jcip-annotations", "1.0-1"}, + {"net.minidev:json-smart", "2.4.2"}, + {"net.minidev:accessors-smart", "2.4.2"}, + {"org.ow2.asm:asm", "5.0.4"}, + {"org.apache.zookeeper:zookeeper", "3.5.6"}, + {"org.apache.curator:curator-framework", "4.2.0"}, + {"org.apache.kerby:kerb-simplekdc", "1.0.1"}, + {"org.apache.kerby:kerb-client", "1.0.1"}, + {"org.apache.kerby:kerby-config", "1.0.1"}, + {"org.apache.kerby:kerb-common", "1.0.1"}, + {"org.apache.kerby:kerb-crypto", "1.0.1"}, + {"org.apache.kerby:kerb-util", "1.0.1"}, + {"org.apache.kerby:token-provider", "1.0.1"}, + {"org.apache.kerby:kerb-admin", "1.0.1"}, + {"org.apache.kerby:kerb-server", "1.0.1"}, + {"org.apache.kerby:kerb-identity", "1.0.1"}, + {"org.apache.kerby:kerby-xdr", "1.0.1"}, + {"org.apache.hadoop:hadoop-nfs", "3.4.0-SNAPSHOT"}, + {"io.netty:netty-all", "4.1.61.Final"}, + {"org.apache.hadoop:hadoop-hdfs", "3.4.0-SNAPSHOT"}, + {"org.eclipse.jetty:jetty-util-ajax", "9.4.40.v20210413"}, + {"io.netty:netty", "3.10.6.Final"}, + {"org.fusesource.leveldbjni:leveldbjni-all", "1.8"}, + {"com.fasterxml.jackson.core:jackson-databind", "2.10.5.1"}, + {"com.fasterxml.jackson.core:jackson-core", "2.10.5"}, + {"org.apache.hadoop:hadoop-hdfs-client", "3.4.0-SNAPSHOT"}, + {"com.squareup.okhttp:okhttp", "2.7.5"}, + {"com.squareup.okio:okio", "1.6.0"}, + {"com.fasterxml.jackson.core:jackson-annotations", "2.10.5"}, + {"org.apache.hadoop:hadoop-hdfs", "3.4.0-SNAPSHOT"}, + {"org.apache.hadoop:hadoop-common", "3.4.0-SNAPSHOT"}, + {"org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_7", "1.1.0-SNAPSHOT"}, + {"com.google.guava:guava", "27.0-jre"}, + {"com.google.guava:failureaccess", "1.0"}, + {"com.google.guava:listenablefuture", "9999.0-empty-to-avoid-conflict-with-guava"}, + {"org.checkerframework:checker-qual", "2.5.2"}, + {"com.google.j2objc:j2objc-annotations", "1.1"}, + {"org.codehaus.mojo:animal-sniffer-annotations", "1.17"}, + {"org.apache.commons:commons-math3", "3.1.1"}, + {"commons-net:commons-net", "3.6"}, + {"commons-collections:commons-collections", "3.2.2"}, + {"jakarta.activation:jakarta.activation-api", "1.2.1"}, + {"org.eclipse.jetty:jetty-servlet", "9.4.40.v20210413"}, + {"org.eclipse.jetty:jetty-security", "9.4.40.v20210413"}, + {"org.eclipse.jetty:jetty-webapp", "9.4.40.v20210413"}, + {"org.eclipse.jetty:jetty-xml", "9.4.40.v20210413"}, + {"javax.servlet.jsp:jsp-api", "2.1"}, + {"com.sun.jersey:jersey-servlet", "1.19"}, + {"com.sun.jersey:jersey-json", "1.19"}, + {"org.codehaus.jettison:jettison", "1.1"}, + {"com.sun.xml.bind:jaxb-impl", "2.2.3-1"}, + {"javax.xml.bind:jaxb-api", "2.2.11"}, + {"org.codehaus.jackson:jackson-core-asl", "1.9.13"}, + {"org.codehaus.jackson:jackson-mapper-asl", "1.9.13"}, + {"org.codehaus.jackson:jackson-jaxrs", "1.9.13"}, + {"org.codehaus.jackson:jackson-xc", "1.9.13"}, + {"commons-beanutils:commons-beanutils", "1.9.4"}, + {"org.apache.commons:commons-configuration2", "2.1.1"}, + {"org.apache.commons:commons-lang3", "3.7"}, + {"org.apache.commons:commons-text", "1.4"}, + {"org.apache.avro:avro", "1.7.7"}, + {"com.thoughtworks.paranamer:paranamer", "2.3"}, + {"com.google.re2j:re2j", "1.1"}, + {"com.google.code.gson:gson", "2.2.4"}, + {"com.jcraft:jsch", "0.1.55"}, + {"org.apache.curator:curator-client", "4.2.0"}, + {"org.apache.curator:curator-recipes", "4.2.0"}, + {"com.google.code.findbugs:jsr305", "3.0.2"}, + {"org.apache.commons:commons-compress", "1.19"}, + {"org.apache.kerby:kerb-core", "1.0.1"}, + {"org.apache.kerby:kerby-pkix", "1.0.1"}, + {"org.apache.kerby:kerby-asn1", "1.0.1"}, + {"org.apache.kerby:kerby-util", "1.0.1"}, + {"org.codehaus.woodstox:stax2-api", "4.2.1"}, + {"com.fasterxml.woodstox:woodstox-core", "5.3.0"}, + {"dnsjava:dnsjava", "2.1.7"}, + {"org.xerial.snappy:snappy-java", "1.1.8.2"}, + {"org.apache.hadoop:hadoop-common", "3.4.0-SNAPSHOT"}, + {"org.apache.zookeeper:zookeeper", "3.5.6"}, + {"org.apache.zookeeper:zookeeper-jute", "3.5.6"}, + {"org.apache.yetus:audience-annotations", "0.5.0"}, + {"org.apache.hadoop.thirdparty:hadoop-shaded-guava", "1.1.0-SNAPSHOT"}, + {"org.eclipse.jetty:jetty-server", "9.4.40.v20210413"}, + {"org.eclipse.jetty:jetty-http", "9.4.40.v20210413"}, + {"org.eclipse.jetty:jetty-io", "9.4.40.v20210413"}, + {"org.eclipse.jetty:jetty-util", "9.4.40.v20210413"}, + {"com.sun.jersey:jersey-core", "1.19"}, + {"javax.ws.rs:jsr311-api", "1.1.1"}, + {"com.sun.jersey:jersey-server", "1.19"}, + {"commons-cli:commons-cli", "1.2"}, + {"commons-codec:commons-codec", "1.15"}, + {"commons-io:commons-io", "2.8.0"}, + {"commons-logging:commons-logging", "1.1.3"}, + {"commons-daemon:commons-daemon", "1.0.13"}, + {"log4j:log4j", "1.2.17"}, + {"com.google.protobuf:protobuf-java", "2.5.0"}, + {"javax.servlet:javax.servlet-api", "3.1.0"}, + {"junit:junit", "4.13.1"}, + {"org.hamcrest:hamcrest-core", "1.3"}, + {"org.mockito:mockito-core", "2.28.2"}, + {"net.bytebuddy:byte-buddy", "1.9.10"}, + {"net.bytebuddy:byte-buddy-agent", "1.9.10"}, + {"org.objenesis:objenesis", "2.6"}, + {"org.slf4j:slf4j-log4j12", "1.7.30"}, + {"org.bouncycastle:bcprov-jdk15on", "1.68"}, + } + + // git clone https://github.com/apache/hadoop.git + // git checkout fdd20a3cf4cd9073f443b2bf07eef14f454d4c33 + // cd hadoop/hadoop-cloud-storage-project + // mvn dependency:list -DoutputFile=dependency.txt + /* + cat dependency.txt | grep -v "^\s*$" | grep -v "The following files have been resolved" | \ + awk ' + { + cnt=split($1, path, ":") + if(cnt == 5){ + print "{\"" path[1] ":" path[2] "\", \"" path[4] "\"}," + } + if(cnt == 6){ + print "{\"" path[1] ":" path[2] "\", \"" path[5] "\"}," + } + } + ' + */ + MavenHadoopCloudStorage = []types.Library{ + {"org.apache.hadoop:hadoop-annotations", "3.4.0-SNAPSHOT"}, + {"org.apache.hadoop:hadoop-common", "3.4.0-SNAPSHOT"}, + {"org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_7", "1.1.0-SNAPSHOT"}, + {"org.apache.hadoop.thirdparty:hadoop-shaded-guava", "1.1.0-SNAPSHOT"}, + {"com.google.guava:guava", "27.0-jre"}, + {"com.google.guava:failureaccess", "1.0"}, + {"com.google.guava:listenablefuture", "9999.0-empty-to-avoid-conflict-with-guava"}, + {"org.checkerframework:checker-qual", "2.5.2"}, + {"com.google.j2objc:j2objc-annotations", "1.1"}, + {"org.codehaus.mojo:animal-sniffer-annotations", "1.17"}, + {"commons-cli:commons-cli", "1.2"}, + {"org.apache.commons:commons-math3", "3.1.1"}, + {"org.apache.httpcomponents:httpclient", "4.5.13"}, + {"commons-codec:commons-codec", "1.15"}, + {"commons-io:commons-io", "2.8.0"}, + {"commons-net:commons-net", "3.6"}, + {"commons-collections:commons-collections", "3.2.2"}, + {"javax.servlet:javax.servlet-api", "3.1.0"}, + {"jakarta.activation:jakarta.activation-api", "1.2.1"}, + {"org.eclipse.jetty:jetty-server", "9.4.40.v20210413"}, + {"org.eclipse.jetty:jetty-http", "9.4.40.v20210413"}, + {"org.eclipse.jetty:jetty-io", "9.4.40.v20210413"}, + {"org.eclipse.jetty:jetty-util", "9.4.40.v20210413"}, + {"org.eclipse.jetty:jetty-servlet", "9.4.40.v20210413"}, + {"org.eclipse.jetty:jetty-security", "9.4.40.v20210413"}, + {"org.eclipse.jetty:jetty-webapp", "9.4.40.v20210413"}, + {"org.eclipse.jetty:jetty-xml", "9.4.40.v20210413"}, + {"javax.servlet.jsp:jsp-api", "2.1"}, + {"com.sun.jersey:jersey-servlet", "1.19"}, + {"commons-logging:commons-logging", "1.1.3"}, + {"log4j:log4j", "1.2.17"}, + {"commons-beanutils:commons-beanutils", "1.9.4"}, + {"org.apache.commons:commons-configuration2", "2.1.1"}, + {"org.apache.commons:commons-lang3", "3.7"}, + {"org.apache.commons:commons-text", "1.4"}, + {"org.slf4j:slf4j-api", "1.7.30"}, + {"org.slf4j:slf4j-log4j12", "1.7.30"}, + {"org.apache.avro:avro", "1.7.7"}, + {"com.thoughtworks.paranamer:paranamer", "2.3"}, + {"com.google.re2j:re2j", "1.1"}, + {"com.google.protobuf:protobuf-java", "2.5.0"}, + {"com.google.code.gson:gson", "2.2.4"}, + {"org.apache.hadoop:hadoop-auth", "3.4.0-SNAPSHOT"}, + {"com.nimbusds:nimbus-jose-jwt", "9.8.1"}, + {"com.github.stephenc.jcip:jcip-annotations", "1.0-1"}, + {"net.minidev:json-smart", "2.4.2"}, + {"net.minidev:accessors-smart", "2.4.2"}, + {"org.ow2.asm:asm", "5.0.4"}, + {"org.apache.curator:curator-framework", "4.2.0"}, + {"org.apache.kerby:kerb-simplekdc", "1.0.1"}, + {"org.apache.kerby:kerb-client", "1.0.1"}, + {"org.apache.kerby:kerby-config", "1.0.1"}, + {"org.apache.kerby:kerb-common", "1.0.1"}, + {"org.apache.kerby:kerb-crypto", "1.0.1"}, + {"org.apache.kerby:kerb-util", "1.0.1"}, + {"org.apache.kerby:token-provider", "1.0.1"}, + {"org.apache.kerby:kerb-admin", "1.0.1"}, + {"org.apache.kerby:kerb-server", "1.0.1"}, + {"org.apache.kerby:kerb-identity", "1.0.1"}, + {"org.apache.kerby:kerby-xdr", "1.0.1"}, + {"org.apache.curator:curator-client", "4.2.0"}, + {"org.apache.curator:curator-recipes", "4.2.0"}, + {"com.google.code.findbugs:jsr305", "3.0.2"}, + {"org.apache.commons:commons-compress", "1.19"}, + {"org.bouncycastle:bcprov-jdk15on", "1.68"}, + {"org.apache.kerby:kerb-core", "1.0.1"}, + {"org.apache.kerby:kerby-pkix", "1.0.1"}, + {"org.apache.kerby:kerby-asn1", "1.0.1"}, + {"org.apache.kerby:kerby-util", "1.0.1"}, + {"com.fasterxml.jackson.core:jackson-databind", "2.10.5.1"}, + {"com.fasterxml.jackson.core:jackson-core", "2.10.5"}, + {"org.codehaus.woodstox:stax2-api", "4.2.1"}, + {"com.fasterxml.woodstox:woodstox-core", "5.3.0"}, + {"dnsjava:dnsjava", "2.1.7"}, + {"org.xerial.snappy:snappy-java", "1.1.8.2"}, + {"org.apache.hadoop:hadoop-aliyun", "3.4.0-SNAPSHOT"}, + {"com.aliyun.oss:aliyun-sdk-oss", "3.4.1"}, + {"org.jdom:jdom", "1.1"}, + {"org.codehaus.jettison:jettison", "1.1"}, + {"com.aliyun:aliyun-java-sdk-core", "3.4.0"}, + {"com.aliyun:aliyun-java-sdk-ram", "3.0.0"}, + {"com.aliyun:aliyun-java-sdk-sts", "3.0.0"}, + {"com.aliyun:aliyun-java-sdk-ecs", "4.2.0"}, + {"org.apache.hadoop:hadoop-aws", "3.4.0-SNAPSHOT"}, + {"com.amazonaws:aws-java-sdk-bundle", "1.11.901"}, + {"org.wildfly.openssl:wildfly-openssl", "1.0.7.Final"}, + {"org.apache.hadoop:hadoop-azure", "3.4.0-SNAPSHOT"}, + {"com.microsoft.azure:azure-storage", "7.0.1"}, + {"com.microsoft.azure:azure-keyvault-core", "1.0.0"}, + {"org.eclipse.jetty:jetty-util-ajax", "9.4.40.v20210413"}, + {"org.codehaus.jackson:jackson-mapper-asl", "1.9.13"}, + {"org.codehaus.jackson:jackson-core-asl", "1.9.13"}, + {"org.apache.hadoop:hadoop-azure-datalake", "3.4.0-SNAPSHOT"}, + {"com.microsoft.azure:azure-data-lake-store-sdk", "2.3.9"}, + {"org.apache.hadoop:hadoop-openstack", "3.4.0-SNAPSHOT"}, + {"org.apache.httpcomponents:httpcore", "4.4.13"}, + {"com.fasterxml.jackson.core:jackson-annotations", "2.10.5"}, + {"org.apache.hadoop:hadoop-cos", "3.4.0-SNAPSHOT"}, + {"com.qcloud:cos_api-bundle", "5.6.19"}, + {"org.apache.hadoop:hadoop-huaweicloud", "3.4.0-SNAPSHOT"}, + {"com.huaweicloud:esdk-obs-java", "3.20.4.2"}, + {"com.jamesmurty.utils:java-xmlbuilder", "1.2"}, + {"com.squareup.okhttp3:okhttp", "3.14.2"}, + {"org.apache.logging.log4j:log4j-core", "2.12.0"}, + {"org.apache.logging.log4j:log4j-api", "2.12.0"}, + } + MavenNone = []types.Library{} +) diff --git a/pkg/maven/pom.xml b/pkg/maven/pom.xml new file mode 100644 index 00000000..2151f316 --- /dev/null +++ b/pkg/maven/pom.xml @@ -0,0 +1,19 @@ + + + 4.0.0 + org.codehaus.mojo + child-project + 1.0 + jar + + + + org.springframework + spring-core + 5.0.1.RELEASE + + + \ No newline at end of file diff --git a/pkg/maven/testdata/hadoop-cloud-storage.txt b/pkg/maven/testdata/hadoop-cloud-storage.txt new file mode 100644 index 00000000..58ca5c54 --- /dev/null +++ b/pkg/maven/testdata/hadoop-cloud-storage.txt @@ -0,0 +1,108 @@ + +The following files have been resolved: + org.apache.hadoop:hadoop-annotations:jar:3.4.0-SNAPSHOT:compile -- module hadoop.annotations (auto) + org.apache.hadoop:hadoop-common:jar:3.4.0-SNAPSHOT:provided -- module hadoop.common (auto) + org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_7:jar:1.1.0-SNAPSHOT:provided + org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:1.1.0-SNAPSHOT:compile -- module hadoop.shaded.guava (auto) + com.google.guava:guava:jar:27.0-jre:compile -- module com.google.common [auto] + com.google.guava:failureaccess:jar:1.0:compile -- module failureaccess (auto) + com.google.guava:listenablefuture:jar:9999.0-empty-to-avoid-conflict-with-guava:compile -- module listenablefuture (auto) + org.checkerframework:checker-qual:jar:2.5.2:compile -- module checker.qual (auto) + com.google.j2objc:j2objc-annotations:jar:1.1:compile -- module j2objc.annotations (auto) + org.codehaus.mojo:animal-sniffer-annotations:jar:1.17:compile -- module animal.sniffer.annotations (auto) + commons-cli:commons-cli:jar:1.2:provided -- module commons.cli (auto) + org.apache.commons:commons-math3:jar:3.1.1:provided -- module commons.math3 (auto) + org.apache.httpcomponents:httpclient:jar:4.5.13:compile -- module org.apache.httpcomponents.httpclient [auto] + commons-codec:commons-codec:jar:1.15:compile -- module org.apache.commons.codec [auto] + commons-io:commons-io:jar:2.8.0:provided -- module org.apache.commons.io [auto] + commons-net:commons-net:jar:3.6:provided -- module commons.net (auto) + commons-collections:commons-collections:jar:3.2.2:provided -- module commons.collections (auto) + javax.servlet:javax.servlet-api:jar:3.1.0:provided -- module javax.servlet.api (auto) + jakarta.activation:jakarta.activation-api:jar:1.2.1:provided -- module jakarta.activation [auto] + org.eclipse.jetty:jetty-server:jar:9.4.40.v20210413:provided -- module org.eclipse.jetty.server [auto] + org.eclipse.jetty:jetty-http:jar:9.4.40.v20210413:provided -- module org.eclipse.jetty.http [auto] + org.eclipse.jetty:jetty-io:jar:9.4.40.v20210413:provided -- module org.eclipse.jetty.io [auto] + org.eclipse.jetty:jetty-util:jar:9.4.40.v20210413:compile -- module org.eclipse.jetty.util [auto] + org.eclipse.jetty:jetty-servlet:jar:9.4.40.v20210413:provided -- module org.eclipse.jetty.servlet [auto] + org.eclipse.jetty:jetty-security:jar:9.4.40.v20210413:provided -- module org.eclipse.jetty.security [auto] + org.eclipse.jetty:jetty-webapp:jar:9.4.40.v20210413:provided -- module org.eclipse.jetty.webapp [auto] + org.eclipse.jetty:jetty-xml:jar:9.4.40.v20210413:provided -- module org.eclipse.jetty.xml [auto] + javax.servlet.jsp:jsp-api:jar:2.1:provided -- module jsp.api (auto) + com.sun.jersey:jersey-servlet:jar:1.19:provided -- module jersey.servlet (auto) + commons-logging:commons-logging:jar:1.1.3:compile -- module commons.logging (auto) + log4j:log4j:jar:1.2.17:provided -- module log4j (auto) + commons-beanutils:commons-beanutils:jar:1.9.4:provided -- module commons.beanutils (auto) + org.apache.commons:commons-configuration2:jar:2.1.1:provided -- module commons.configuration2 (auto) + org.apache.commons:commons-lang3:jar:3.7:provided -- module org.apache.commons.lang3 [auto] + org.apache.commons:commons-text:jar:1.4:provided -- module org.apache.commons.text [auto] + org.slf4j:slf4j-api:jar:1.7.30:compile -- module org.slf4j [auto] + org.slf4j:slf4j-log4j12:jar:1.7.30:provided -- module slf4j.log4j12 (auto) + org.apache.avro:avro:jar:1.7.7:provided -- module avro (auto) + com.thoughtworks.paranamer:paranamer:jar:2.3:provided -- module paranamer (auto) + com.google.re2j:re2j:jar:1.1:provided -- module re2j (auto) + com.google.protobuf:protobuf-java:jar:2.5.0:provided -- module protobuf.java (auto) + com.google.code.gson:gson:jar:2.2.4:provided -- module gson (auto) + org.apache.hadoop:hadoop-auth:jar:3.4.0-SNAPSHOT:provided -- module hadoop.auth (auto) + com.nimbusds:nimbus-jose-jwt:jar:9.8.1:compile -- module com.nimbusds.jose.jwt [auto] + com.github.stephenc.jcip:jcip-annotations:jar:1.0-1:compile -- module jcip.annotations (auto) + net.minidev:json-smart:jar:2.4.2:provided -- module json.smart (auto) + net.minidev:accessors-smart:jar:2.4.2:provided -- module accessors.smart (auto) + org.ow2.asm:asm:jar:5.0.4:provided -- module asm (auto) + org.apache.curator:curator-framework:jar:4.2.0:provided -- module curator.framework (auto) + org.apache.kerby:kerb-simplekdc:jar:1.0.1:provided -- module kerb.simplekdc (auto) + org.apache.kerby:kerb-client:jar:1.0.1:provided -- module kerb.client (auto) + org.apache.kerby:kerby-config:jar:1.0.1:provided -- module kerby.config (auto) + org.apache.kerby:kerb-common:jar:1.0.1:provided -- module kerb.common (auto) + org.apache.kerby:kerb-crypto:jar:1.0.1:provided -- module kerb.crypto (auto) + org.apache.kerby:kerb-util:jar:1.0.1:provided -- module kerb.util (auto) + org.apache.kerby:token-provider:jar:1.0.1:provided -- module token.provider (auto) + org.apache.kerby:kerb-admin:jar:1.0.1:provided -- module kerb.admin (auto) + org.apache.kerby:kerb-server:jar:1.0.1:provided -- module kerb.server (auto) + org.apache.kerby:kerb-identity:jar:1.0.1:provided -- module kerb.identity (auto) + org.apache.kerby:kerby-xdr:jar:1.0.1:provided -- module kerby.xdr (auto) + org.apache.curator:curator-client:jar:4.2.0:provided -- module curator.client (auto) + org.apache.curator:curator-recipes:jar:4.2.0:provided -- module curator.recipes (auto) + com.google.code.findbugs:jsr305:jar:3.0.2:compile -- module jsr305 (auto) + org.apache.commons:commons-compress:jar:1.19:provided -- module org.apache.commons.compress [auto] + org.bouncycastle:bcprov-jdk15on:jar:1.68:provided -- module org.bouncycastle.provider + org.apache.kerby:kerb-core:jar:1.0.1:provided -- module kerb.core (auto) + org.apache.kerby:kerby-pkix:jar:1.0.1:provided -- module kerby.pkix (auto) + org.apache.kerby:kerby-asn1:jar:1.0.1:provided -- module kerby.asn1 (auto) + org.apache.kerby:kerby-util:jar:1.0.1:provided -- module kerby.util (auto) + com.fasterxml.jackson.core:jackson-databind:jar:2.10.5.1:compile -- module com.fasterxml.jackson.databind + com.fasterxml.jackson.core:jackson-core:jar:2.10.5:compile -- module com.fasterxml.jackson.core + org.codehaus.woodstox:stax2-api:jar:4.2.1:provided -- module org.codehaus.stax2 + com.fasterxml.woodstox:woodstox-core:jar:5.3.0:provided -- module com.ctc.wstx [auto] + dnsjava:dnsjava:jar:2.1.7:provided + org.xerial.snappy:snappy-java:jar:1.1.8.2:provided -- module snappy.java (auto) + org.apache.hadoop:hadoop-aliyun:jar:3.4.0-SNAPSHOT:compile -- module hadoop.aliyun (auto) + com.aliyun.oss:aliyun-sdk-oss:jar:3.4.1:compile -- module aliyun.sdk.oss (auto) + org.jdom:jdom:jar:1.1:compile + org.codehaus.jettison:jettison:jar:1.1:compile -- module jettison (auto) + com.aliyun:aliyun-java-sdk-core:jar:3.4.0:compile -- module aliyun.java.sdk.core (auto) + com.aliyun:aliyun-java-sdk-ram:jar:3.0.0:compile -- module aliyun.java.sdk.ram (auto) + com.aliyun:aliyun-java-sdk-sts:jar:3.0.0:compile -- module aliyun.java.sdk.sts (auto) + com.aliyun:aliyun-java-sdk-ecs:jar:4.2.0:compile -- module aliyun.java.sdk.ecs (auto) + org.apache.hadoop:hadoop-aws:jar:3.4.0-SNAPSHOT:compile -- module hadoop.aws (auto) + com.amazonaws:aws-java-sdk-bundle:jar:1.11.901:compile + org.wildfly.openssl:wildfly-openssl:jar:1.0.7.Final:compile -- module wildfly.openssl (auto) + org.apache.hadoop:hadoop-azure:jar:3.4.0-SNAPSHOT:compile -- module hadoop.azure (auto) + com.microsoft.azure:azure-storage:jar:7.0.1:compile -- module azure.storage (auto) + com.microsoft.azure:azure-keyvault-core:jar:1.0.0:compile -- module azure.keyvault.core (auto) + org.eclipse.jetty:jetty-util-ajax:jar:9.4.40.v20210413:compile -- module org.eclipse.jetty.util.ajax [auto] + org.codehaus.jackson:jackson-mapper-asl:jar:1.9.13:compile -- module jackson.mapper.asl (auto) + org.codehaus.jackson:jackson-core-asl:jar:1.9.13:compile -- module jackson.core.asl (auto) + org.apache.hadoop:hadoop-azure-datalake:jar:3.4.0-SNAPSHOT:compile -- module hadoop.azure.datalake (auto) + com.microsoft.azure:azure-data-lake-store-sdk:jar:2.3.9:compile -- module azure.data.lake.store.sdk (auto) + org.apache.hadoop:hadoop-openstack:jar:3.4.0-SNAPSHOT:compile -- module hadoop.openstack (auto) + org.apache.httpcomponents:httpcore:jar:4.4.13:compile -- module org.apache.httpcomponents.httpcore [auto] + com.fasterxml.jackson.core:jackson-annotations:jar:2.10.5:compile -- module com.fasterxml.jackson.annotation + org.apache.hadoop:hadoop-cos:jar:3.4.0-SNAPSHOT:compile -- module hadoop.cos (auto) + com.qcloud:cos_api-bundle:jar:5.6.19:compile + org.apache.hadoop:hadoop-huaweicloud:jar:3.4.0-SNAPSHOT:compile -- module hadoop.huaweicloud (auto) + com.huaweicloud:esdk-obs-java:jar:3.20.4.2:compile -- module esdk.obs.java (auto) + com.jamesmurty.utils:java-xmlbuilder:jar:1.2:compile -- module java.xmlbuilder (auto) + com.squareup.okhttp3:okhttp:jar:3.14.2:compile -- module okhttp3 [auto] + org.apache.logging.log4j:log4j-core:jar:2.12.0:compile -- module org.apache.logging.log4j.core [auto] + org.apache.logging.log4j:log4j-api:jar:2.12.0:compile -- module org.apache.logging.log4j + diff --git a/pkg/maven/testdata/hadoop-hdfs-nfs.txt b/pkg/maven/testdata/hadoop-hdfs-nfs.txt new file mode 100644 index 00000000..12c1f98f --- /dev/null +++ b/pkg/maven/testdata/hadoop-hdfs-nfs.txt @@ -0,0 +1,114 @@ + +The following files have been resolved: + org.apache.hadoop:hadoop-annotations:jar:3.4.0-SNAPSHOT:provided -- module hadoop.annotations (auto) + org.apache.hadoop:hadoop-auth:jar:3.4.0-SNAPSHOT:provided -- module hadoop.auth (auto) + org.slf4j:slf4j-api:jar:1.7.30:provided -- module org.slf4j [auto] + org.apache.httpcomponents:httpclient:jar:4.5.13:provided -- module org.apache.httpcomponents.httpclient [auto] + org.apache.httpcomponents:httpcore:jar:4.4.13:provided -- module org.apache.httpcomponents.httpcore [auto] + com.nimbusds:nimbus-jose-jwt:jar:9.8.1:compile -- module com.nimbusds.jose.jwt [auto] + com.github.stephenc.jcip:jcip-annotations:jar:1.0-1:compile -- module jcip.annotations (auto) + net.minidev:json-smart:jar:2.4.2:provided -- module json.smart (auto) + net.minidev:accessors-smart:jar:2.4.2:provided -- module accessors.smart (auto) + org.ow2.asm:asm:jar:5.0.4:provided -- module asm (auto) + org.apache.zookeeper:zookeeper:jar:3.5.6:provided -- module zookeeper (auto) + org.apache.curator:curator-framework:jar:4.2.0:provided -- module curator.framework (auto) + org.apache.kerby:kerb-simplekdc:jar:1.0.1:provided -- module kerb.simplekdc (auto) + org.apache.kerby:kerb-client:jar:1.0.1:provided -- module kerb.client (auto) + org.apache.kerby:kerby-config:jar:1.0.1:provided -- module kerby.config (auto) + org.apache.kerby:kerb-common:jar:1.0.1:provided -- module kerb.common (auto) + org.apache.kerby:kerb-crypto:jar:1.0.1:provided -- module kerb.crypto (auto) + org.apache.kerby:kerb-util:jar:1.0.1:provided -- module kerb.util (auto) + org.apache.kerby:token-provider:jar:1.0.1:provided -- module token.provider (auto) + org.apache.kerby:kerb-admin:jar:1.0.1:provided -- module kerb.admin (auto) + org.apache.kerby:kerb-server:jar:1.0.1:provided -- module kerb.server (auto) + org.apache.kerby:kerb-identity:jar:1.0.1:provided -- module kerb.identity (auto) + org.apache.kerby:kerby-xdr:jar:1.0.1:provided -- module kerby.xdr (auto) + org.apache.hadoop:hadoop-nfs:jar:3.4.0-SNAPSHOT:provided -- module hadoop.nfs (auto) + io.netty:netty-all:jar:4.1.61.Final:compile -- module io.netty.all [auto] + org.apache.hadoop:hadoop-hdfs:jar:3.4.0-SNAPSHOT:compile -- module hadoop.hdfs (auto) + org.eclipse.jetty:jetty-util-ajax:jar:9.4.40.v20210413:compile -- module org.eclipse.jetty.util.ajax [auto] + io.netty:netty:jar:3.10.6.Final:compile -- module netty (auto) + org.fusesource.leveldbjni:leveldbjni-all:jar:1.8:compile -- module leveldbjni.all (auto) + com.fasterxml.jackson.core:jackson-databind:jar:2.10.5.1:compile -- module com.fasterxml.jackson.databind + com.fasterxml.jackson.core:jackson-core:jar:2.10.5:compile -- module com.fasterxml.jackson.core + org.apache.hadoop:hadoop-hdfs-client:jar:3.4.0-SNAPSHOT:provided -- module hadoop.hdfs.client (auto) + com.squareup.okhttp:okhttp:jar:2.7.5:provided -- module okhttp (auto) + com.squareup.okio:okio:jar:1.6.0:provided -- module okio (auto) + com.fasterxml.jackson.core:jackson-annotations:jar:2.10.5:compile -- module com.fasterxml.jackson.annotation + org.apache.hadoop:hadoop-hdfs:test-jar:tests:3.4.0-SNAPSHOT:test -- module hadoop.hdfs (auto) + org.apache.hadoop:hadoop-common:jar:3.4.0-SNAPSHOT:provided -- module hadoop.common (auto) + org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_7:jar:1.1.0-SNAPSHOT:provided + com.google.guava:guava:jar:27.0-jre:provided -- module com.google.common [auto] + com.google.guava:failureaccess:jar:1.0:provided -- module failureaccess (auto) + com.google.guava:listenablefuture:jar:9999.0-empty-to-avoid-conflict-with-guava:provided -- module listenablefuture (auto) + org.checkerframework:checker-qual:jar:2.5.2:provided -- module checker.qual (auto) + com.google.j2objc:j2objc-annotations:jar:1.1:provided -- module j2objc.annotations (auto) + org.codehaus.mojo:animal-sniffer-annotations:jar:1.17:provided -- module animal.sniffer.annotations (auto) + org.apache.commons:commons-math3:jar:3.1.1:provided -- module commons.math3 (auto) + commons-net:commons-net:jar:3.6:provided -- module commons.net (auto) + commons-collections:commons-collections:jar:3.2.2:provided -- module commons.collections (auto) + jakarta.activation:jakarta.activation-api:jar:1.2.1:provided -- module jakarta.activation [auto] + org.eclipse.jetty:jetty-servlet:jar:9.4.40.v20210413:provided -- module org.eclipse.jetty.servlet [auto] + org.eclipse.jetty:jetty-security:jar:9.4.40.v20210413:provided -- module org.eclipse.jetty.security [auto] + org.eclipse.jetty:jetty-webapp:jar:9.4.40.v20210413:provided -- module org.eclipse.jetty.webapp [auto] + org.eclipse.jetty:jetty-xml:jar:9.4.40.v20210413:provided -- module org.eclipse.jetty.xml [auto] + javax.servlet.jsp:jsp-api:jar:2.1:provided -- module jsp.api (auto) + com.sun.jersey:jersey-servlet:jar:1.19:provided -- module jersey.servlet (auto) + com.sun.jersey:jersey-json:jar:1.19:provided -- module jersey.json (auto) + org.codehaus.jettison:jettison:jar:1.1:provided -- module jettison (auto) + com.sun.xml.bind:jaxb-impl:jar:2.2.3-1:provided -- module jaxb.impl (auto) + javax.xml.bind:jaxb-api:jar:2.2.11:provided -- module jaxb.api (auto) + org.codehaus.jackson:jackson-core-asl:jar:1.9.13:provided -- module jackson.core.asl (auto) + org.codehaus.jackson:jackson-mapper-asl:jar:1.9.13:provided -- module jackson.mapper.asl (auto) + org.codehaus.jackson:jackson-jaxrs:jar:1.9.13:provided -- module jackson.jaxrs (auto) + org.codehaus.jackson:jackson-xc:jar:1.9.13:provided -- module jackson.xc (auto) + commons-beanutils:commons-beanutils:jar:1.9.4:provided -- module commons.beanutils (auto) + org.apache.commons:commons-configuration2:jar:2.1.1:provided -- module commons.configuration2 (auto) + org.apache.commons:commons-lang3:jar:3.7:provided -- module org.apache.commons.lang3 [auto] + org.apache.commons:commons-text:jar:1.4:provided -- module org.apache.commons.text [auto] + org.apache.avro:avro:jar:1.7.7:provided -- module avro (auto) + com.thoughtworks.paranamer:paranamer:jar:2.3:provided -- module paranamer (auto) + com.google.re2j:re2j:jar:1.1:provided -- module re2j (auto) + com.google.code.gson:gson:jar:2.2.4:provided -- module gson (auto) + com.jcraft:jsch:jar:0.1.55:provided -- module jsch (auto) + org.apache.curator:curator-client:jar:4.2.0:provided -- module curator.client (auto) + org.apache.curator:curator-recipes:jar:4.2.0:provided -- module curator.recipes (auto) + com.google.code.findbugs:jsr305:jar:3.0.2:provided -- module jsr305 (auto) + org.apache.commons:commons-compress:jar:1.19:provided -- module org.apache.commons.compress [auto] + org.apache.kerby:kerb-core:jar:1.0.1:provided -- module kerb.core (auto) + org.apache.kerby:kerby-pkix:jar:1.0.1:provided -- module kerby.pkix (auto) + org.apache.kerby:kerby-asn1:jar:1.0.1:provided -- module kerby.asn1 (auto) + org.apache.kerby:kerby-util:jar:1.0.1:provided -- module kerby.util (auto) + org.codehaus.woodstox:stax2-api:jar:4.2.1:provided -- module org.codehaus.stax2 + com.fasterxml.woodstox:woodstox-core:jar:5.3.0:provided -- module com.ctc.wstx [auto] + dnsjava:dnsjava:jar:2.1.7:provided + org.xerial.snappy:snappy-java:jar:1.1.8.2:provided -- module snappy.java (auto) + org.apache.hadoop:hadoop-common:test-jar:tests:3.4.0-SNAPSHOT:test -- module hadoop.common (auto) + org.apache.zookeeper:zookeeper:test-jar:tests:3.5.6:test -- module zookeeper (auto) + org.apache.zookeeper:zookeeper-jute:jar:3.5.6:provided -- module zookeeper.jute (auto) + org.apache.yetus:audience-annotations:jar:0.5.0:provided -- module audience.annotations (auto) + org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:1.1.0-SNAPSHOT:compile -- module hadoop.shaded.guava (auto) + org.eclipse.jetty:jetty-server:jar:9.4.40.v20210413:compile -- module org.eclipse.jetty.server [auto] + org.eclipse.jetty:jetty-http:jar:9.4.40.v20210413:compile -- module org.eclipse.jetty.http [auto] + org.eclipse.jetty:jetty-io:jar:9.4.40.v20210413:compile -- module org.eclipse.jetty.io [auto] + org.eclipse.jetty:jetty-util:jar:9.4.40.v20210413:compile -- module org.eclipse.jetty.util [auto] + com.sun.jersey:jersey-core:jar:1.19:compile -- module jersey.core (auto) + javax.ws.rs:jsr311-api:jar:1.1.1:compile -- module jsr311.api (auto) + com.sun.jersey:jersey-server:jar:1.19:compile -- module jersey.server (auto) + commons-cli:commons-cli:jar:1.2:compile -- module commons.cli (auto) + commons-codec:commons-codec:jar:1.15:compile -- module org.apache.commons.codec [auto] + commons-io:commons-io:jar:2.8.0:compile -- module org.apache.commons.io [auto] + commons-logging:commons-logging:jar:1.1.3:compile -- module commons.logging (auto) + commons-daemon:commons-daemon:jar:1.0.13:compile -- module commons.daemon (auto) + log4j:log4j:jar:1.2.17:compile -- module log4j (auto) + com.google.protobuf:protobuf-java:jar:2.5.0:compile -- module protobuf.java (auto) + javax.servlet:javax.servlet-api:jar:3.1.0:compile -- module javax.servlet.api (auto) + junit:junit:jar:4.13.1:test -- module junit [auto] + org.hamcrest:hamcrest-core:jar:1.3:test -- module hamcrest.core (auto) + org.mockito:mockito-core:jar:2.28.2:test -- module org.mockito [auto] + net.bytebuddy:byte-buddy:jar:1.9.10:test -- module net.bytebuddy + net.bytebuddy:byte-buddy-agent:jar:1.9.10:test -- module net.bytebuddy.agent + org.objenesis:objenesis:jar:2.6:test -- module objenesis (auto) + org.slf4j:slf4j-log4j12:jar:1.7.30:provided -- module slf4j.log4j12 (auto) + org.bouncycastle:bcprov-jdk15on:jar:1.68:test -- module org.bouncycastle.provider + diff --git a/pkg/maven/testdata/none.txt b/pkg/maven/testdata/none.txt new file mode 100644 index 00000000..dbc926fc --- /dev/null +++ b/pkg/maven/testdata/none.txt @@ -0,0 +1,4 @@ + +The following files have been resolved: + none + diff --git a/pkg/maven/testdata/spring.txt b/pkg/maven/testdata/spring.txt new file mode 100644 index 00000000..71ed7881 --- /dev/null +++ b/pkg/maven/testdata/spring.txt @@ -0,0 +1,5 @@ + +The following files have been resolved: + org.springframework:spring-jcl:jar:5.0.1.RELEASE:compile + org.springframework:spring-core:jar:5.0.1.RELEASE:compile + From e629ed809af7ed27460e4dd32867429f7c17932d Mon Sep 17 00:00:00 2001 From: masahiro331 Date: Sun, 16 May 2021 17:00:23 +0900 Subject: [PATCH 2/5] update maven testcase --- pkg/maven/parse_testcase.go | 72 ++++++++++++++++++------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/pkg/maven/parse_testcase.go b/pkg/maven/parse_testcase.go index defac599..9e47a2b4 100644 --- a/pkg/maven/parse_testcase.go +++ b/pkg/maven/parse_testcase.go @@ -27,18 +27,18 @@ var ( */ // mvn dependency:list -DoutputFile=dependency.txt /* - cat dependency.txt | grep -v "^\s*$" | grep -v "The following files have been resolved" | \ - awk ' - { - cnt=split($1, path, ":") - if(cnt == 5){ - print "{\"" path[1] ":" path[2] "\", \"" path[4] "\"}," - } - if(cnt == 6){ - print "{\"" path[1] ":" path[2] "\", \"" path[5] "\"}," - } - } - ' + cat dependency.txt | grep -v "^\s*$" | grep -v "The following files have been resolved" | \ + awk ' + { + cnt=split($1, path, ":") + if(cnt == 5){ + print "{\"" path[1] ":" path[2] "\", \"" path[4] "\"}," + } + if(cnt == 6){ + print "{\"" path[1] ":" path[2] "\", \"" path[5] "\"}," + } + } + ' */ MavenNormal = []types.Library{ {"org.springframework:spring-jcl", "5.0.1.RELEASE"}, @@ -50,18 +50,18 @@ var ( // cd hadoop/hadoop-hdfs-project/hadoop-hdfs-nfs/ // mvn dependency:list -DoutputFile=dependency.txt /* - cat dependency.txt | grep -v "^\s*$" | grep -v "The following files have been resolved" | \ - awk ' - { - cnt=split($1, path, ":") - if(cnt == 5){ - print "{\"" path[1] ":" path[2] "\", \"" path[4] "\"}," - } - if(cnt == 6){ - print "{\"" path[1] ":" path[2] "\", \"" path[5] "\"}," - } - } - ' + cat dependency.txt | grep -v "^\s*$" | grep -v "The following files have been resolved" | \ + awk ' + { + cnt=split($1, path, ":") + if(cnt == 5){ + print "{\"" path[1] ":" path[2] "\", \"" path[4] "\"}," + } + if(cnt == 6){ + print "{\"" path[1] ":" path[2] "\", \"" path[5] "\"}," + } + } + ' */ MavenHadoopHDFS = []types.Library{ {"org.apache.hadoop:hadoop-annotations", "3.4.0-SNAPSHOT"}, @@ -182,18 +182,18 @@ var ( // cd hadoop/hadoop-cloud-storage-project // mvn dependency:list -DoutputFile=dependency.txt /* - cat dependency.txt | grep -v "^\s*$" | grep -v "The following files have been resolved" | \ - awk ' - { - cnt=split($1, path, ":") - if(cnt == 5){ - print "{\"" path[1] ":" path[2] "\", \"" path[4] "\"}," - } - if(cnt == 6){ - print "{\"" path[1] ":" path[2] "\", \"" path[5] "\"}," - } - } - ' + cat dependency.txt | grep -v "^\s*$" | grep -v "The following files have been resolved" | \ + awk ' + { + cnt=split($1, path, ":") + if(cnt == 5){ + print "{\"" path[1] ":" path[2] "\", \"" path[4] "\"}," + } + if(cnt == 6){ + print "{\"" path[1] ":" path[2] "\", \"" path[5] "\"}," + } + } + ' */ MavenHadoopCloudStorage = []types.Library{ {"org.apache.hadoop:hadoop-annotations", "3.4.0-SNAPSHOT"}, From 0d4f74f39b4dc4689caaab2b3eb4552361fd3adb Mon Sep 17 00:00:00 2001 From: masahiro331 Date: Tue, 18 May 2021 23:37:46 +0900 Subject: [PATCH 3/5] add isValidScope --- pkg/maven/parse.go | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/pkg/maven/parse.go b/pkg/maven/parse.go index c84d22cb..4b41324b 100644 --- a/pkg/maven/parse.go +++ b/pkg/maven/parse.go @@ -15,6 +15,14 @@ const ( resolvedSection = "The following files have been resolved:" ) +var scopes = []string{ + "compile", + "provided", + "runtime", + "test", + "system", +} + type Artifact struct { GroupID string ArtifactID string @@ -83,8 +91,13 @@ func parseArtifact(s string) (Artifact, error) { case 4: artifact.Version = ss[3] case 5: - artifact.Version = ss[3] - artifact.Scope = ss[4] + if isValidScope(ss[4]) { + artifact.Version = ss[3] + artifact.Scope = ss[4] + } else { + artifact.Classifier = ss[3] + artifact.Version = ss[4] + } case 6: artifact.Classifier = ss[3] artifact.Version = ss[4] @@ -93,3 +106,12 @@ func parseArtifact(s string) (Artifact, error) { return artifact, nil } + +func isValidScope(s string) bool { + for _, scope := range scopes { + if s == scope { + return true + } + } + return false +} \ No newline at end of file From 0790142221f84dd81c5c70a7517de39b1cfb930c Mon Sep 17 00:00:00 2001 From: masahiro331 Date: Tue, 18 May 2021 23:55:46 +0900 Subject: [PATCH 4/5] replace escape code --- pkg/maven/parse.go | 6 ++++-- pkg/maven/parse_test.go | 4 ++++ pkg/maven/parse_testcase.go | 5 +++++ pkg/maven/testdata/classifier.txt | 5 +++++ 4 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 pkg/maven/testdata/classifier.txt diff --git a/pkg/maven/parse.go b/pkg/maven/parse.go index 4b41324b..3c9c7d10 100644 --- a/pkg/maven/parse.go +++ b/pkg/maven/parse.go @@ -60,7 +60,9 @@ func Parse(r io.Reader) ([]types.Library, error) { return []types.Library{}, nil } - artifact, err := parseArtifact(strings.Fields(line)[0]) + // Replace Escape code + as := strings.Fields(strings.ReplaceAll(line, "\x1b", " ")) + artifact, err := parseArtifact(as[0]) if err != nil { return nil, xerrors.Errorf("failed to parse artifact: %w", err) } @@ -114,4 +116,4 @@ func isValidScope(s string) bool { } } return false -} \ No newline at end of file +} diff --git a/pkg/maven/parse_test.go b/pkg/maven/parse_test.go index 6fc45845..396c04ed 100644 --- a/pkg/maven/parse_test.go +++ b/pkg/maven/parse_test.go @@ -32,6 +32,10 @@ func TestParse(t *testing.T) { file: "testdata/none.txt", want: MavenNone, }, + { + file: "testdata/classifier.txt", + want: MavenClassifier, + }, } for _, v := range vectors { diff --git a/pkg/maven/parse_testcase.go b/pkg/maven/parse_testcase.go index 9e47a2b4..81cb41aa 100644 --- a/pkg/maven/parse_testcase.go +++ b/pkg/maven/parse_testcase.go @@ -303,4 +303,9 @@ var ( {"org.apache.logging.log4j:log4j-api", "2.12.0"}, } MavenNone = []types.Library{} + + MavenClassifier = []types.Library{ + {"org.springframework:spring-jcl", "5.0.1.RELEASE"}, + {"org.springframework:spring-core", "5.0.1.RELEASE"}, + } ) diff --git a/pkg/maven/testdata/classifier.txt b/pkg/maven/testdata/classifier.txt new file mode 100644 index 00000000..ce21727e --- /dev/null +++ b/pkg/maven/testdata/classifier.txt @@ -0,0 +1,5 @@ + +The following files have been resolved: + org.springframework:spring-jcl:jar:classifier:5.0.1.RELEASE + org.springframework:spring-core:jar:classifier:5.0.1.RELEASE + From f69082234a62e46fb9bc24a7f5d799aab2dd65bf Mon Sep 17 00:00:00 2001 From: masahiro331 Date: Mon, 31 May 2021 19:07:51 +0900 Subject: [PATCH 5/5] fix review --- pkg/maven/parse.go | 6 ++---- pkg/maven/pom.xml | 19 ------------------- 2 files changed, 2 insertions(+), 23 deletions(-) delete mode 100644 pkg/maven/pom.xml diff --git a/pkg/maven/parse.go b/pkg/maven/parse.go index 3c9c7d10..9b155886 100644 --- a/pkg/maven/parse.go +++ b/pkg/maven/parse.go @@ -2,7 +2,6 @@ package maven import ( "bufio" - "fmt" "io" "strings" @@ -81,8 +80,7 @@ func Parse(r io.Reader) ([]types.Library, error) { func parseArtifact(s string) (Artifact, error) { ss := strings.Split(s, separator) if len(ss) < 4 { - fmt.Println(s) - return Artifact{}, xerrors.New("invalid format error") + return Artifact{}, xerrors.Errorf("invalid format: %s", s) } artifact := Artifact{ GroupID: ss[0], @@ -116,4 +114,4 @@ func isValidScope(s string) bool { } } return false -} +} \ No newline at end of file diff --git a/pkg/maven/pom.xml b/pkg/maven/pom.xml deleted file mode 100644 index 2151f316..00000000 --- a/pkg/maven/pom.xml +++ /dev/null @@ -1,19 +0,0 @@ - - - 4.0.0 - org.codehaus.mojo - child-project - 1.0 - jar - - - - org.springframework - spring-core - 5.0.1.RELEASE - - - \ No newline at end of file