diff --git a/core/src/main/java/org/jboss/pnc/build/finder/core/DistributionAnalyzer.java b/core/src/main/java/org/jboss/pnc/build/finder/core/DistributionAnalyzer.java index 9054a17d..c2f3caf6 100644 --- a/core/src/main/java/org/jboss/pnc/build/finder/core/DistributionAnalyzer.java +++ b/core/src/main/java/org/jboss/pnc/build/finder/core/DistributionAnalyzer.java @@ -29,7 +29,17 @@ import static org.jboss.pnc.build.finder.core.LicenseSource.POM_XML; import static org.jboss.pnc.build.finder.core.LicenseSource.TEXT; import static org.jboss.pnc.build.finder.core.LicenseUtils.NOASSERTION; +import static org.jboss.pnc.build.finder.core.LicenseUtils.getBundleLicenseFromManifest; +import static org.jboss.pnc.build.finder.core.LicenseUtils.getCurrentLicenseId; +import static org.jboss.pnc.build.finder.core.LicenseUtils.getFirstNonBlankString; +import static org.jboss.pnc.build.finder.core.LicenseUtils.getMatchingLicense; +import static org.jboss.pnc.build.finder.core.LicenseUtils.getNumberOfSPDXLicenses; +import static org.jboss.pnc.build.finder.core.LicenseUtils.getSPDXLicenseListVersion; +import static org.jboss.pnc.build.finder.core.LicenseUtils.isLicenseFile; +import static org.jboss.pnc.build.finder.core.LicenseUtils.isLicenseFileName; +import static org.jboss.pnc.build.finder.core.LicenseUtils.isManifestMfFileName; import static org.jboss.pnc.build.finder.core.LicenseUtils.isUrl; +import static org.jboss.pnc.build.finder.core.LicenseUtils.loadLicenseMapping; import static org.jboss.pnc.build.finder.core.MavenUtils.getLicenses; import static org.jboss.pnc.build.finder.core.MavenUtils.isPom; import static org.jboss.pnc.build.finder.core.MavenUtils.isPomXml; @@ -119,8 +129,6 @@ public class DistributionAnalyzer implements Callable inputs; private final MultiValuedMap inverseMap; @@ -155,7 +163,7 @@ public DistributionAnalyzer(List inputs, BuildConfig config) { public DistributionAnalyzer(List inputs, BuildConfig config, BasicCacheContainer cacheManager) { try { - Map> mapping = LicenseUtils.loadLicenseMapping(); + Map> mapping = loadLicenseMapping(); if (LOGGER.isInfoEnabled()) { LOGGER.info( @@ -164,7 +172,10 @@ public DistributionAnalyzer(List inputs, BuildConfig config, BasicCacheC green(String.join(", ", mapping.keySet()))); } } catch (IOException e) { - LOGGER.error("Error loading SPDX license URL mappings: {}", boldRed(getAllErrorMessages(e))); + if (LOGGER.isErrorEnabled()) { + LOGGER.error("Error loading SPDX license URL mappings: {}", boldRed(getAllErrorMessages(e))); + } + LOGGER.debug("Error", e); } @@ -173,8 +184,8 @@ public DistributionAnalyzer(List inputs, BuildConfig config, BasicCacheC checksumTypesToCheck = EnumSet.copyOf(config.getChecksumTypes()); map = new EnumMap<>(ChecksumType.class); licensesMap = new TreeMap<>(); - String licenseListVersion = LicenseUtils.getSPDXLicenseListVersion(); - int licenseListSize = LicenseUtils.getNumberOfSPDXLicenses(); + String licenseListVersion = getSPDXLicenseListVersion(); + int licenseListSize = getNumberOfSPDXLicenses(); LOGGER.info( "Using SPDX License List {} containing {} licenses", green(licenseListVersion), @@ -241,11 +252,14 @@ public Map> checksumFiles() thro new Checksum(checksumType, entry.getKey(), entry.getValue())); } } catch (ClassCastException e) { - LOGGER.error( - "Error loading cache {}: {}. The cache format has changed" - + " and you will have to manually delete the existing cache", - boldRed(ConfigDefaults.CACHE_LOCATION), - boldRed(getAllErrorMessages(e))); + if (LOGGER.isErrorEnabled()) { + LOGGER.error( + "Error loading cache {}: {}. The cache format has changed" + + " and you will have to manually delete the existing cache", + boldRed(ConfigDefaults.CACHE_LOCATION), + boldRed(getAllErrorMessages(e))); + } + throw e; } @@ -271,17 +285,20 @@ public Map> checksumFiles() thro if (listener != null) { listener.checksumsComputed(new ChecksumsComputedEvent(size)); } - - LOGGER.info( - "Loaded {} checksums for file: {} (checksum: {}) from cache", - green(size), - green(fo.getName()), - green(value)); + if (LOGGER.isInfoEnabled()) { + LOGGER.info( + "Loaded {} checksums for file: {} (checksum: {}) from cache", + green(size), + green(normalizePath(fo, root)), + green(value)); + } } else { - LOGGER.info( - "File: {} (checksum: {}) not found in cache", - green(fo.getName()), - green(value)); + if (LOGGER.isInfoEnabled()) { + LOGGER.info( + "File: {} (checksum: {}) not found in cache", + green(normalizePath(fo, root)), + green(value)); + } } } } @@ -297,7 +314,7 @@ public Map> checksumFiles() thro checksumTypesToCheck.stream() .map(String::valueOf) .collect(Collectors.toUnmodifiableSet()))), - green(fo.getName())); + green(normalizePath(fo, root))); } listChildren(fo); @@ -602,7 +619,12 @@ private void listChildren(FileObject fo) throws IOException { Map> map = getLicenses(root, file, source); putLicenses(map.keySet().iterator().next(), licenseInfos); } catch (XmlPullParserException | InterpolationException e) { - LOGGER.warn("Error parsing POM file {}: {}", red(file), red(getAllErrorMessages(e))); + if (LOGGER.isErrorEnabled()) { + LOGGER.error( + "Error parsing POM file {}: {}", + boldRed(file), + boldRed(getAllErrorMessages(e))); + } } } @@ -660,9 +682,9 @@ private List addLicensesFromJar(FileObject jar, FileObject localFil try { if (isPomXml(localFile)) { licenseInfos = addLicensesFromPom(localFile, POM_XML); - } else if (LicenseUtils.isManifestMfFileName(localFile)) { + } else if (isManifestMfFileName(localFile)) { licenseInfos = addLicensesFromBundleLicense(localFile); - } else if (LicenseUtils.isLicenseFileName(localFile.getName().getBaseName())) { + } else if (isLicenseFile(localFile)) { licenseInfos = addLicenseFromTextFile(jar, localFile); } else { licenseInfos = Collections.emptyList(); @@ -671,33 +693,46 @@ private List addLicensesFromJar(FileObject jar, FileObject localFil licenseInfos = Collections.emptyList(); } - return licenseInfos; + // Second license pass which looks for relative file URLs, e.g., names such as META-INF/LICENSE + licenseInfos.stream() + .filter(licenseInfo -> NOASSERTION.equals(licenseInfo.getSpdxLicenseId())) + .forEach(licenseInfo -> handleRelativeURL(jar, localFile, licenseInfo)); + + // If there are any licenses still unmatched, print them, but ignore unmatched files that were already checked + // in the last step + if (LOGGER.isWarnEnabled()) { + licenseInfos.stream() + .filter(licenseInfo -> NOASSERTION.equals(licenseInfo.getSpdxLicenseId())) + .forEach(licenseInfo -> checkMissingMapping(localFile, licenseInfo)); + } + + return Collections.unmodifiableList(licenseInfos); } private static List addLicenseFromTextFile(FileObject jar, FileObject licenseFile) throws IOException { - String licenseId = LicenseUtils.getMatchingLicense(licenseFile); + String licenseId = getMatchingLicense(licenseFile); LicenseInfo licenseInfo = new LicenseInfo( null, jar.getName().getRelativeName(licenseFile.getName()), - LicenseUtils.getCurrentLicenseId(licenseId), + getCurrentLicenseId(licenseId), TEXT); - return List.of(licenseInfo); + return Collections.singletonList(licenseInfo); } private static List addLicensesFromBundleLicense(FileObject fileObject) throws IOException { List licenses = new ArrayList<>(3); - List bundlesLicenses = LicenseUtils.getBundleLicenseFromManifest(fileObject); + List bundlesLicenses = getBundleLicenseFromManifest(fileObject); for (BundleLicense bundleLicense : bundlesLicenses) { String licenseIdentifier = bundleLicense.getLicenseIdentifier(); String description = bundleLicense.getDescription(); - String name = LicenseUtils.getFirstNonBlankString(licenseIdentifier, description); + String name = getFirstNonBlankString(licenseIdentifier, description); String url = bundleLicense.getLink(); LicenseInfo licenseInfo = new LicenseInfo(name, url, BUNDLE_LICENSE); licenses.add(licenseInfo); } - return licenses; + return Collections.unmodifiableList(licenses); } private List addLicensesFromPom(FileObject fileObject, LicenseSource source) throws IOException { @@ -723,33 +758,119 @@ private List addLicensesFromPom(FileObject fileObject, LicenseSourc .collect(Collectors.toUnmodifiableSet()))); } - return licenseInfos; + return Collections.unmodifiableList(licenseInfos); } catch (XmlPullParserException | InterpolationException e) { - LOGGER.warn("Unable to read licenses from file {}: {}", red(fileObject), red(getAllErrorMessages(e))); + if (LOGGER.isErrorEnabled()) { + LOGGER.error( + "Unable to read licenses from file {}: {}", + boldRed(fileObject), + boldRed(getAllErrorMessages(e))); + } + throw new IOException(e); } } - private void putLicenses(String pomOrJarFile, Collection licenseInfos) { - Collection existingLicenses = licensesMap.get(pomOrJarFile); + private void checkMissingMapping(FileObject localFile, LicenseInfo licenseInfo) { + String name = licenseInfo.getName(); + String url = licenseInfo.getUrl(); - if (existingLicenses != null) { - existingLicenses.addAll(licenseInfos); - } else { - licensesMap.put(pomOrJarFile, licenseInfos); + if (name == null && url == null) { + return; + } + + if (isLicenseFileName(name)) { + return; + } + + if (isLicenseFileName(url)) { + return; } if (LOGGER.isWarnEnabled()) { - for (LicenseInfo licenseInfo : licenseInfos) { - if (licenseInfo.getSpdxLicenseId().equals(NOASSERTION)) { - String name = licenseInfo.getName(); - String url = licenseInfo.getUrl(); + LOGGER.warn( + "Missing SPDX license mapping for name: {}, URL: {}, filename: {}", + red(name), + red(url), + red(normalizePath(localFile, root))); + } + } - if (name != null || isUrl(url)) { - LOGGER.warn("Missing SPDX license mapping for name: {}, URL: {}", red(name), red(url)); - } + private void handleRelativeURL(FileObject jar, FileObject localFile, LicenseInfo licenseInfo) { + String name = licenseInfo.getName(); + String url = licenseInfo.getUrl(); + + if (name == null && url == null) { + return; + } + + // URL is not relative, ignore + if (isUrl(url)) { + return; + } + + if (name == null) { + name = url; + } + + try { + // If the URL is absent, consider the possibility that the name refers to a file inside the JAR + FileObject licenseFile = jar.resolveFile(name); + + if (!isLicenseFile(licenseFile)) { + return; + } + + if (licenseFile.isFolder()) { + return; + } + + if (!licenseFile.exists()) { + if (LOGGER.isWarnEnabled()) { + LOGGER.warn( + "License file {} from {} does not exist", + red(name), + red(normalizePath(localFile, root))); } + + return; } + + List licenseInfos = addLicenseFromTextFile(jar, licenseFile); + + if (licenseInfos.isEmpty()) { + if (LOGGER.isWarnEnabled()) { + LOGGER.warn( + "Failed to add licenses from file {} located in JAR {}", + red(normalizePath(licenseFile, root)), + red(normalizePath(jar, root))); + } + + return; + } + + // XXX: Currently, the API returns either 0 or 1 licenses, which we rely on here + LicenseInfo licenseInfo2 = licenseInfos.get(0); + String spdxLicenseId = licenseInfo2.getSpdxLicenseId(); + licenseInfo.setSpdxLicense(spdxLicenseId); + } catch (IOException e) { + if (LOGGER.isErrorEnabled()) { + LOGGER.error( + "Error adding relative license URL name {} for {}: {}", + boldRed(name), + boldRed(normalizePath(jar, root)), + boldRed(getAllErrorMessages(e))); + } + } + } + + private void putLicenses(String pomOrJarFile, Collection licenseInfos) { + Collection existingLicenses = licensesMap.get(pomOrJarFile); + + if (existingLicenses != null) { + existingLicenses.addAll(licenseInfos); + } else { + licensesMap.put(pomOrJarFile, licenseInfos); } } diff --git a/core/src/main/java/org/jboss/pnc/build/finder/core/LicenseInfo.java b/core/src/main/java/org/jboss/pnc/build/finder/core/LicenseInfo.java index 69643b66..03fa819d 100644 --- a/core/src/main/java/org/jboss/pnc/build/finder/core/LicenseInfo.java +++ b/core/src/main/java/org/jboss/pnc/build/finder/core/LicenseInfo.java @@ -29,7 +29,7 @@ public class LicenseInfo implements Comparable { private final String url; - private final String spdxLicenseId; + private String spdxLicenseId; private final LicenseSource source; @@ -80,6 +80,10 @@ public String getSpdxLicenseId() { return spdxLicenseId; } + public void setSpdxLicense(String spdxLicenseId) { + this.spdxLicenseId = spdxLicenseId; + } + public LicenseSource getSource() { return source; } diff --git a/core/src/main/java/org/jboss/pnc/build/finder/core/LicenseUtils.java b/core/src/main/java/org/jboss/pnc/build/finder/core/LicenseUtils.java index fe2738af..3368cdca 100644 --- a/core/src/main/java/org/jboss/pnc/build/finder/core/LicenseUtils.java +++ b/core/src/main/java/org/jboss/pnc/build/finder/core/LicenseUtils.java @@ -53,6 +53,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.vfs2.FileContent; import org.apache.commons.vfs2.FileObject; +import org.apache.commons.vfs2.FileSystemException; import org.spdx.library.InvalidSPDXAnalysisException; import org.spdx.library.model.license.AnyLicenseInfo; import org.spdx.library.model.license.InvalidLicenseStringException; @@ -80,14 +81,14 @@ public final class LicenseUtils { private static final Pattern IDSTRING_PATTERN = Pattern.compile("[a-zA-Z0-9-.]+"); - private static final Pattern LICENSE_FILE_PATTERN = Pattern.compile("^([A-Z-]+)?LICENSE(.md|.txt)?$"); - private static final Pattern MANIFEST_MF_PATTERN = Pattern.compile("^.*META-INF/MANIFEST.MF$"); private static final int EXPECTED_NUM_SPDX_LICENSES = 1024; private static final String BUNDLE_LICENSE = "Bundle-License"; + private static final String LICENSE = "LICENSE"; + private static final Pattern PUNCT_PATTERN = Pattern.compile("\\p{Punct}"); private static final Pattern WHITESPACE_PATTERN = Pattern.compile("\\s+"); @@ -113,7 +114,9 @@ public final class LicenseUtils { private static final String URL_MARKER = ":/"; - private static final List EXTENSIONS_TO_REMOVE = List.of(".html", ".md", ".php", ".txt"); + private static final String DOT = "."; + + private static final List TEXT_EXTENSIONS = List.of(".html", ".md", ".php", ".txt"); private static final Pattern NAME_VERSION_PATTERN = Pattern .compile("(?[A-Z-a-z])[Vv]?(?[1-9]+)(\\.(?[0-9]+))?"); @@ -269,7 +272,7 @@ static String normalizeLicenseUrl(String licenseUrl) { path = LETTER_DIGIT_PATTERN.matcher(path).replaceAll("$1-$2"); path = path.replace("cc-0", "cc0"); - for (String extension : EXTENSIONS_TO_REMOVE) { + for (String extension : TEXT_EXTENSIONS) { path = StringUtils.removeEnd(path, extension); } @@ -361,9 +364,8 @@ public static String getSPDXLicenseId(String name, String url) { } static Optional findSPDXLicenseId(Map> mapping, String name, String url) { - Optional optSPDXLicenseId = LicenseUtils.findLicenseMapping(mapping, url) - .or(() -> LicenseUtils.findMatchingLicense(name, url)); - return optSPDXLicenseId.or(() -> LicenseUtils.findLicenseMapping(mapping, name)); + Optional optSPDXLicenseId = findLicenseMapping(mapping, url).or(() -> findMatchingLicense(name, url)); + return optSPDXLicenseId.or(() -> findLicenseMapping(mapping, name)); } /** @@ -552,6 +554,53 @@ public static boolean isManifestMfFileName(FileObject fileObject) { return MANIFEST_MF_PATTERN.matcher(fileObject.getName().getPath()).matches(); } + /** + * Returns whether the given file name is a license text file. Matches files such as + * + *
    + *
  • LICENSE.md
    • + *
  • LICENSE
    • + *
  • LICENSE.txt
    • + *
  • MIT-LICENSE
    • + *
  • <SPDX-LICENSE-ID>.txt
    • + *
+ * + * @param fileObject the file object + * @return whether the given file object is a license text file + */ + public static boolean isLicenseFile(FileObject fileObject) { + try { + if (!fileObject.isFile()) { + return false; + } + } catch (FileSystemException e) { + return false; + } + + String path = fileObject.getName().getPath(); + return isLicenseFileName(path); + } + + private static Optional findSPDXIdentifierFromFileName(FileObject fileObject) { + try { + if (!fileObject.isFile()) { + return Optional.empty(); + } + } catch (FileSystemException e) { + return Optional.empty(); + } + + String path = fileObject.getName().getPath(); + String name = FilenameUtils.getName(path); + String baseName = FilenameUtils.removeExtension(name); + + if (isKnownLicenseId(baseName)) { + return Optional.of(baseName); + } + + return Optional.empty(); + } + /** * Returns whether the given file name is a license text file. Matches files such as * @@ -567,12 +616,35 @@ public static boolean isManifestMfFileName(FileObject fileObject) { * @return whether the given file name is a license text file */ public static boolean isLicenseFileName(String fileName) { - if (LICENSE_FILE_PATTERN.matcher(fileName).matches()) { + if (StringUtils.isBlank(fileName)) { + return false; + } + + String extension = FilenameUtils.getExtension(fileName); + + if (!isTextExtension(extension)) { + return false; + } + + String name = FilenameUtils.getName(fileName); + String baseName = FilenameUtils.removeExtension(name); + + return StringUtils.containsIgnoreCase(baseName, LICENSE) || isKnownLicenseId(baseName); + } + + /** + * Returns whether this extension is a plain-text file extension. + * + * @param extension the extension + * @return whether this extension is a plain-text file extension + */ + public static boolean isTextExtension(String extension) { + if (EMPTY.equals(extension)) { return true; } - String baseName = FilenameUtils.removeExtension(fileName); - return LicenseUtils.isKnownLicenseId(baseName); + String ext = StringUtils.prependIfMissing(extension, DOT); + return TEXT_EXTENSIONS.contains(ext); } /** @@ -582,6 +654,12 @@ public static boolean isLicenseFileName(String fileName) { * @return the matching license identifier, if any */ public static Optional findMatchingLicense(FileObject licenseFileObject) { + Optional optionalId = findSPDXIdentifierFromFileName(licenseFileObject); + + if (optionalId.isPresent()) { + return optionalId; + } + try (FileContent fc = licenseFileObject.getContent(); InputStream in = fc.getInputStream()) { String licenseText = new String(in.readAllBytes(), UTF_8); return LICENSE_ID_TEXT_LIST.stream() diff --git a/core/src/test/java/org/jboss/pnc/build/finder/core/LicenseUtilsTest.java b/core/src/test/java/org/jboss/pnc/build/finder/core/LicenseUtilsTest.java index ce9ed905..a16ce9ff 100644 --- a/core/src/test/java/org/jboss/pnc/build/finder/core/LicenseUtilsTest.java +++ b/core/src/test/java/org/jboss/pnc/build/finder/core/LicenseUtilsTest.java @@ -52,12 +52,11 @@ import org.spdx.library.model.license.WithExceptionOperator; class LicenseUtilsTest { - private static Map> MAPPING; @BeforeAll static void setup() throws IOException { - MAPPING = LicenseUtils.loadLicenseMapping(); - assertThat(MAPPING).isNotEmpty(); + Map> mapping = LicenseUtils.loadLicenseMapping(); + assertThat(mapping).isNotEmpty(); } @Test @@ -322,11 +321,26 @@ void testGetMatchingLicense() throws IOException, InvalidSPDXAnalysisException { } @ParameterizedTest - @ValueSource(strings = { "LICENSE", "LICENSE.md", "LICENSE.txt", "MIT-LICENSE", "MIT", "MIT.md", "MIT.txt" }) + @ValueSource( + strings = { + "LICENSE", + "LICENSE.md", + "LICENSE.txt", + "LICENSE-2.0.txt", + "MIT-LICENSE", + "MIT", + "MIT.md", + "MIT.txt", + "META-INF/LGPL-3.0.txt" }) void testIsLicenseFileName(String fileName) { assertThat(LicenseUtils.isLicenseFileName(fileName)).isTrue(); } + @Test + void testIsLicenseFileName2() { + assertThat(LicenseUtils.isLicenseFileName("org/dom4j/xpp")).isTrue(); + } + @Test void testGetSPDXLicenseName() { assertThat(LicenseUtils.getSPDXLicenseName("Apache-2.0")).isEqualTo("Apache License 2.0");