diff --git a/build-tools-internal/build.gradle b/build-tools-internal/build.gradle index 10e39be490cef..26da39c0070a4 100644 --- a/build-tools-internal/build.gradle +++ b/build-tools-internal/build.gradle @@ -110,6 +110,10 @@ gradlePlugin { id = 'elasticsearch.jdk-download' implementationClass = 'org.elasticsearch.gradle.internal.JdkDownloadPlugin' } + releaseTools { + id = 'elasticsearch.release-tools' + implementationClass = 'org.elasticsearch.gradle.internal.release.ReleaseToolsPlugin' + } repositories { id = 'elasticsearch.repositories' implementationClass = 'org.elasticsearch.gradle.internal.RepositoriesSetupPlugin' diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ValidateJsonAgainstSchemaTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ValidateJsonAgainstSchemaTask.java index 60b609b3145fa..db6d2c1135b90 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ValidateJsonAgainstSchemaTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ValidateJsonAgainstSchemaTask.java @@ -20,9 +20,11 @@ import org.gradle.api.file.FileCollection; import org.gradle.api.tasks.InputFile; import org.gradle.api.tasks.InputFiles; +import org.gradle.api.tasks.Internal; import org.gradle.api.tasks.OutputFile; import org.gradle.api.tasks.TaskAction; import org.gradle.work.ChangeType; +import org.gradle.work.FileChange; import org.gradle.work.Incremental; import org.gradle.work.InputChanges; @@ -42,8 +44,6 @@ * Incremental task to validate a set of JSON files against against a schema. */ public class ValidateJsonAgainstSchemaTask extends DefaultTask { - - private final ObjectMapper mapper = new ObjectMapper(); private File jsonSchema; private File report; private FileCollection inputFiles; @@ -76,28 +76,36 @@ public File getReport() { return this.report; } + @Internal + protected ObjectMapper getMapper() { + return new ObjectMapper(); + } + + @Internal + protected String getFileType() { + return "JSON"; + } + @TaskAction public void validate(InputChanges inputChanges) throws IOException { - File jsonSchemaOnDisk = getJsonSchema(); - getLogger().debug("JSON schema : [{}]", jsonSchemaOnDisk.getAbsolutePath()); - SchemaValidatorsConfig config = new SchemaValidatorsConfig(); - JsonSchemaFactory factory = JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V7); - JsonSchema jsonSchema = factory.getSchema(mapper.readTree(jsonSchemaOnDisk), config); - Map> errors = new LinkedHashMap<>(); + final File jsonSchemaOnDisk = getJsonSchema(); + final JsonSchema jsonSchema = buildSchemaObject(jsonSchemaOnDisk); + + final Map> errors = new LinkedHashMap<>(); + final ObjectMapper mapper = this.getMapper(); + // incrementally evaluate input files + // validate all files and hold on to errors for a complete report if there are failures StreamSupport.stream(inputChanges.getFileChanges(getInputFiles()).spliterator(), false) .filter(f -> f.getChangeType() != ChangeType.REMOVED) - .forEach(fileChange -> { - File file = fileChange.getFile(); - if (file.isDirectory() == false) { - // validate all files and hold on to errors for a complete report if there are failures - getLogger().debug("Validating JSON [{}]", file.getName()); - try { - Set validationMessages = jsonSchema.validate(mapper.readTree(file)); - maybeLogAndCollectError(validationMessages, errors, file); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + .map(FileChange::getFile) + .filter(file -> file.isDirectory() == false) + .forEach(file -> { + try { + Set validationMessages = jsonSchema.validate(mapper.readTree(file)); + maybeLogAndCollectError(validationMessages, errors, file); + } catch (IOException e) { + throw new UncheckedIOException(e); } }); if (errors.isEmpty()) { @@ -119,9 +127,17 @@ public void validate(InputChanges inputChanges) throws IOException { } } + private JsonSchema buildSchemaObject(File jsonSchemaOnDisk) throws IOException { + final ObjectMapper jsonMapper = new ObjectMapper(); + final SchemaValidatorsConfig config = new SchemaValidatorsConfig(); + final JsonSchemaFactory factory = JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V7); + return factory.getSchema(jsonMapper.readTree(jsonSchemaOnDisk), config); + } + private void maybeLogAndCollectError(Set messages, Map> errors, File file) { + final String fileType = getFileType(); for (ValidationMessage message : messages) { - getLogger().error("[validate JSON][ERROR][{}][{}]", file.getName(), message.toString()); + getLogger().error("[validate {}][ERROR][{}][{}]", fileType, file.getName(), message.toString()); errors.computeIfAbsent(file, k -> new LinkedHashSet<>()) .add(String.format("%s: %s", file.getAbsolutePath(), message.toString())); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ValidateYamlAgainstSchemaTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ValidateYamlAgainstSchemaTask.java new file mode 100644 index 0000000000000..c4233fc26166c --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ValidateYamlAgainstSchemaTask.java @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.precommit; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; + +/** + * Incremental task to validate a set of YAML files against against a schema. + */ +public class ValidateYamlAgainstSchemaTask extends ValidateJsonAgainstSchemaTask { + @Override + protected String getFileType() { + return "YAML"; + } + + protected ObjectMapper getMapper() { + return new ObjectMapper(new YAMLFactory()); + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BreakingChangesGenerator.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BreakingChangesGenerator.java new file mode 100644 index 0000000000000..691aa47d9ebbc --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/BreakingChangesGenerator.java @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.release; + +import groovy.text.SimpleTemplateEngine; + +import com.google.common.annotations.VisibleForTesting; + +import org.elasticsearch.gradle.Version; +import org.elasticsearch.gradle.VersionProperties; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.TreeMap; +import java.util.stream.Collectors; + +/** + * Generates the page that lists the breaking changes and deprecations for a minor version release. + */ +public class BreakingChangesGenerator { + + static void update(File templateFile, File outputFile, List entries) throws IOException { + try (FileWriter output = new FileWriter(outputFile)) { + generateFile(Files.readString(templateFile.toPath()), output, entries); + } + } + + @VisibleForTesting + private static void generateFile(String template, FileWriter outputWriter, List entries) throws IOException { + final Version version = VersionProperties.getElasticsearchVersion(); + + final Map>> breakingChangesByNotabilityByArea = entries.stream() + .map(ChangelogEntry::getBreaking) + .filter(Objects::nonNull) + .collect( + Collectors.groupingBy( + ChangelogEntry.Breaking::isNotable, + Collectors.groupingBy(ChangelogEntry.Breaking::getArea, TreeMap::new, Collectors.toList()) + ) + ); + + final Map> deprecationsByArea = entries.stream() + .map(ChangelogEntry::getDeprecation) + .filter(Objects::nonNull) + .collect(Collectors.groupingBy(ChangelogEntry.Deprecation::getArea, TreeMap::new, Collectors.toList())); + + final Map bindings = new HashMap<>(); + bindings.put("breakingChangesByNotabilityByArea", breakingChangesByNotabilityByArea); + bindings.put("deprecationsByArea", deprecationsByArea); + bindings.put("isElasticsearchSnapshot", VersionProperties.isElasticsearchSnapshot()); + bindings.put("majorDotMinor", version.getMajor() + "." + version.getMinor()); + bindings.put("majorMinor", String.valueOf(version.getMajor()) + version.getMinor()); + bindings.put("nextMajor", (version.getMajor() + 1) + ".0"); + bindings.put("version", version); + + try { + final SimpleTemplateEngine engine = new SimpleTemplateEngine(); + engine.createTemplate(template).make(bindings).writeTo(outputWriter); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogEntry.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogEntry.java new file mode 100644 index 0000000000000..08b03b35ccd63 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogEntry.java @@ -0,0 +1,374 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.release; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; + +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * This class models the contents of a changelog YAML file. We validate it using a + * JSON Schema, as well as some programmatic checks in {@link ValidateChangelogEntryTask}. + * + */ +public class ChangelogEntry { + private Integer pr; + private List issues; + private String area; + private String type; + private String summary; + private Highlight highlight; + private Breaking breaking; + private Deprecation deprecation; + private List versions; + + private static final ObjectMapper yamlMapper = new ObjectMapper(new YAMLFactory()); + + public static ChangelogEntry parse(File file) { + try { + return yamlMapper.readValue(file, ChangelogEntry.class); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public Integer getPr() { + return pr; + } + + public void setPr(Integer pr) { + this.pr = pr; + } + + public List getIssues() { + return issues; + } + + public void setIssues(List issues) { + this.issues = issues; + } + + public String getArea() { + return area; + } + + public void setArea(String area) { + this.area = area; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getSummary() { + return summary; + } + + public void setSummary(String summary) { + this.summary = summary; + } + + public Highlight getHighlight() { + return highlight; + } + + public void setHighlight(Highlight highlight) { + this.highlight = highlight; + } + + public Breaking getBreaking() { + return breaking; + } + + public void setBreaking(Breaking breaking) { + this.breaking = breaking; + } + + public Deprecation getDeprecation() { + return deprecation; + } + + public void setDeprecation(Deprecation deprecation) { + this.deprecation = deprecation; + } + + public List getVersions() { + return versions; + } + + public void setVersions(List versions) { + this.versions = versions; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ChangelogEntry that = (ChangelogEntry) o; + return Objects.equals(pr, that.pr) + && Objects.equals(issues, that.issues) + && Objects.equals(area, that.area) + && Objects.equals(type, that.type) + && Objects.equals(summary, that.summary) + && Objects.equals(highlight, that.highlight) + && Objects.equals(breaking, that.breaking) + && Objects.equals(versions, that.versions); + } + + @Override + public int hashCode() { + return Objects.hash(pr, issues, area, type, summary, highlight, breaking, versions); + } + + @Override + public String toString() { + return String.format( + Locale.ROOT, + "ChangelogEntry{pr=%d, issues=%s, area='%s', type='%s', summary='%s', highlight=%s, breaking=%s, deprecation=%s versions=%s}", + pr, + issues, + area, + type, + summary, + highlight, + breaking, + deprecation, + versions + ); + } + + public static class Highlight { + private boolean notable; + private String title; + private String body; + + public boolean isNotable() { + return notable; + } + + public void setNotable(boolean notable) { + this.notable = notable; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public String getBody() { + return body; + } + + public void setBody(String body) { + this.body = body; + } + + public String getAnchor() { + return generatedAnchor(this.title); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Highlight highlight = (Highlight) o; + return Objects.equals(notable, highlight.notable) + && Objects.equals(title, highlight.title) + && Objects.equals(body, highlight.body); + } + + @Override + public int hashCode() { + return Objects.hash(notable, title, body); + } + + @Override + public String toString() { + return String.format(Locale.ROOT, "Highlight{notable=%s, title='%s', body='%s'}", notable, title, body); + } + } + + public static class Breaking { + private String area; + private String title; + private String details; + private String impact; + private boolean notable; + + public String getArea() { + return area; + } + + public void setArea(String area) { + this.area = area; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public String getDetails() { + return details; + } + + public void setDetails(String details) { + this.details = details; + } + + public String getImpact() { + return impact; + } + + public void setImpact(String impact) { + this.impact = impact; + } + + public boolean isNotable() { + return notable; + } + + public void setNotable(boolean notable) { + this.notable = notable; + } + + public String getAnchor() { + return generatedAnchor(this.title); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Breaking breaking = (Breaking) o; + return notable == breaking.notable + && Objects.equals(area, breaking.area) + && Objects.equals(title, breaking.title) + && Objects.equals(details, breaking.details) + && Objects.equals(impact, breaking.impact); + } + + @Override + public int hashCode() { + return Objects.hash(area, title, details, impact, notable); + } + + @Override + public String toString() { + return String.format( + "Breaking{area='%s', title='%s', details='%s', impact='%s', isNotable=%s}", + area, + title, + details, + impact, + notable + ); + } + } + + public static class Deprecation { + private String area; + private String title; + private String body; + + public String getArea() { + return area; + } + + public void setArea(String area) { + this.area = area; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public String getBody() { + return body; + } + + public void setBody(String body) { + this.body = body; + } + + public String getAnchor() { + return generatedAnchor(this.title); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Deprecation that = (Deprecation) o; + return Objects.equals(area, that.area) && Objects.equals(title, that.title) && Objects.equals(body, that.body); + } + + @Override + public int hashCode() { + return Objects.hash(area, title, body); + } + + @Override + public String toString() { + return String.format("Deprecation{area='%s', title='%s', body='%s'}", area, title, body); + } + } + + private static String generatedAnchor(String input) { + final List excludes = List.of("the", "is", "a"); + + final String[] words = input.toLowerCase(Locale.ROOT) + .replaceAll("[^\\w]+", "_") + .replaceFirst("^_+", "") + .replaceFirst("_+$", "") + .split("_+"); + return Arrays.stream(words).filter(word -> excludes.contains(word) == false).collect(Collectors.joining("_")); + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java new file mode 100644 index 0000000000000..5d5e1edf9b99e --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/GenerateReleaseNotesTask.java @@ -0,0 +1,195 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.release; + +import org.elasticsearch.gradle.Version; +import org.elasticsearch.gradle.VersionProperties; +import org.gradle.api.DefaultTask; +import org.gradle.api.file.ConfigurableFileCollection; +import org.gradle.api.file.FileCollection; +import org.gradle.api.file.RegularFile; +import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.logging.Logger; +import org.gradle.api.logging.Logging; +import org.gradle.api.model.ObjectFactory; +import org.gradle.api.tasks.InputFile; +import org.gradle.api.tasks.InputFiles; +import org.gradle.api.tasks.OutputFile; +import org.gradle.api.tasks.TaskAction; + +import javax.inject.Inject; +import java.io.IOException; +import java.util.List; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +/** + * Orchestrates the steps required to generate or update various release notes files. + */ +public class GenerateReleaseNotesTask extends DefaultTask { + private static final Logger LOGGER = Logging.getLogger(GenerateReleaseNotesTask.class); + + private final ConfigurableFileCollection changelogs; + + private final RegularFileProperty releaseNotesIndexTemplate; + private final RegularFileProperty releaseNotesTemplate; + private final RegularFileProperty releaseHighlightsTemplate; + private final RegularFileProperty breakingChangesTemplate; + + private final RegularFileProperty releaseNotesIndexFile; + private final RegularFileProperty releaseNotesFile; + private final RegularFileProperty releaseHighlightsFile; + private final RegularFileProperty breakingChangesFile; + + @Inject + public GenerateReleaseNotesTask(ObjectFactory objectFactory) { + changelogs = objectFactory.fileCollection(); + + releaseNotesIndexTemplate = objectFactory.fileProperty(); + releaseNotesTemplate = objectFactory.fileProperty(); + releaseHighlightsTemplate = objectFactory.fileProperty(); + breakingChangesTemplate = objectFactory.fileProperty(); + + releaseNotesIndexFile = objectFactory.fileProperty(); + releaseNotesFile = objectFactory.fileProperty(); + releaseHighlightsFile = objectFactory.fileProperty(); + breakingChangesFile = objectFactory.fileProperty(); + } + + @TaskAction + public void executeTask() throws IOException { + LOGGER.info("Finding changelog files..."); + + final Version checkoutVersion = VersionProperties.getElasticsearchVersion(); + + final List entries = this.changelogs.getFiles() + .stream() + .map(ChangelogEntry::parse) + .filter( + // Only process changelogs that are included in this minor version series of ES. + // If this change was released in an earlier major or minor version of Elasticsearch, do not + // include it in the notes. An earlier patch version is OK, the release notes include changes + // for every patch release in a minor series. + log -> { + final List versionsForChangelogFile = log.getVersions() + .stream() + .map(v -> Version.fromString(v, Version.Mode.RELAXED)) + .collect(Collectors.toList()); + + final Predicate includedInSameMinor = v -> v.getMajor() == checkoutVersion.getMajor() + && v.getMinor() == checkoutVersion.getMinor(); + + final Predicate includedInEarlierMajorOrMinor = v -> v.getMajor() < checkoutVersion.getMajor() + || (v.getMajor() == checkoutVersion.getMajor() && v.getMinor() < checkoutVersion.getMinor()); + + boolean includedInThisMinor = versionsForChangelogFile.stream().anyMatch(includedInSameMinor); + + if (includedInThisMinor) { + return versionsForChangelogFile.stream().noneMatch(includedInEarlierMajorOrMinor); + } else { + return false; + } + } + ) + .collect(Collectors.toList()); + + LOGGER.info("Updating release notes index..."); + ReleaseNotesIndexUpdater.update(this.releaseNotesIndexTemplate.get().getAsFile(), this.releaseNotesIndexFile.get().getAsFile()); + + LOGGER.info("Generating release notes..."); + ReleaseNotesGenerator.update(this.releaseNotesTemplate.get().getAsFile(), this.releaseNotesFile.get().getAsFile(), entries); + + LOGGER.info("Generating release highlights..."); + ReleaseHighlightsGenerator.update(this.releaseHighlightsTemplate.get().getAsFile(), this.releaseHighlightsFile.get().getAsFile(), entries); + + LOGGER.info("Generating breaking changes / deprecations notes..."); + BreakingChangesGenerator.update(this.breakingChangesTemplate.get().getAsFile(), this.breakingChangesFile.get().getAsFile(), entries); + } + + @InputFiles + public FileCollection getChangelogs() { + return changelogs; + } + + public void setChangelogs(FileCollection files) { + this.changelogs.setFrom(files); + } + + @InputFile + public RegularFileProperty getReleaseNotesIndexTemplate() { + return releaseNotesIndexTemplate; + } + + public void setReleaseNotesIndexTemplate(RegularFile file) { + this.releaseNotesIndexTemplate.set(file); + } + + @InputFile + public RegularFileProperty getReleaseNotesTemplate() { + return releaseNotesTemplate; + } + + public void setReleaseNotesTemplate(RegularFile file) { + this.releaseNotesTemplate.set(file); + } + + @InputFile + public RegularFileProperty getReleaseHighlightsTemplate() { + return releaseHighlightsTemplate; + } + + public void setReleaseHighlightsTemplate(RegularFile file) { + this.releaseHighlightsTemplate.set(file); + } + + @InputFile + public RegularFileProperty getBreakingChangesTemplate() { + return breakingChangesTemplate; + } + + public void setBreakingChangesTemplate(RegularFile file) { + this.breakingChangesTemplate.set(file); + } + + @OutputFile + public RegularFileProperty getReleaseNotesIndexFile() { + return releaseNotesIndexFile; + } + + public void setReleaseNotesIndexFile(RegularFile file) { + this.releaseNotesIndexFile.set(file); + } + + @OutputFile + public RegularFileProperty getReleaseNotesFile() { + return releaseNotesFile; + } + + public void setReleaseNotesFile(RegularFile file) { + this.releaseNotesFile.set(file); + } + + @OutputFile + public RegularFileProperty getReleaseHighlightsFile() { + return releaseHighlightsFile; + } + + public void setReleaseHighlightsFile(RegularFile file) { + this.releaseHighlightsFile.set(file); + } + + @OutputFile + public RegularFileProperty getBreakingChangesFile() { + return breakingChangesFile; + } + + public void setBreakingChangesFile(RegularFile file) { + this.breakingChangesFile.set(file); + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGenerator.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGenerator.java new file mode 100644 index 0000000000000..02b450aa22eea --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGenerator.java @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.release; + +import groovy.text.SimpleTemplateEngine; + +import com.google.common.annotations.VisibleForTesting; + +import org.elasticsearch.gradle.Version; +import org.elasticsearch.gradle.VersionProperties; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * Generates the release highlights notes, for changelog files that contain the highlight field. + */ +public class ReleaseHighlightsGenerator { + static void update(File templateFile, File outputFile, List entries) throws IOException { + try (FileWriter output = new FileWriter(outputFile)) { + generateFile(VersionProperties.getElasticsearchVersion(), Files.readString(templateFile.toPath()), entries, output); + } + } + + @VisibleForTesting + static void generateFile(Version version, String templateFile, List entries, FileWriter outputWriter) + throws IOException { + final List priorVersions = new ArrayList<>(); + + if (version.getMinor() > 0) { + final int major = version.getMajor(); + for (int minor = version.getMinor(); minor >= 0; minor--) { + String majorMinor = major + "." + minor; + String fileSuffix = ""; + if (major == 7 && minor < 7) { + fileSuffix = "-" + majorMinor + ".0"; + } + priorVersions.add("{ref-bare}/" + majorMinor + "/release-highlights" + fileSuffix + ".html[" + majorMinor + "]"); + } + } + + final Map> groupedHighlights = entries.stream() + .map(ChangelogEntry::getHighlight) + .filter(Objects::nonNull) + .collect(Collectors.groupingBy(ChangelogEntry.Highlight::isNotable, Collectors.toList())); + + final List notableHighlights = groupedHighlights.getOrDefault(true, List.of()); + final List nonNotableHighlights = groupedHighlights.getOrDefault(false, List.of()); + + final Map bindings = new HashMap<>(); + bindings.put("priorVersions", priorVersions); + bindings.put("notableHighlights", notableHighlights); + bindings.put("nonNotableHighlights", nonNotableHighlights); + + try { + final SimpleTemplateEngine engine = new SimpleTemplateEngine(); + engine.createTemplate(templateFile).make(bindings).writeTo(outputWriter); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGenerator.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGenerator.java new file mode 100644 index 0000000000000..52995717a435a --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGenerator.java @@ -0,0 +1,127 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.release; + +import com.google.common.annotations.VisibleForTesting; +import groovy.text.SimpleTemplateEngine; +import org.elasticsearch.gradle.Version; +import org.elasticsearch.gradle.VersionProperties; +import org.gradle.api.GradleException; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.Writer; +import java.nio.file.Files; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +/** + * Generates the release notes i.e. list of changes that have gone into this release. They are grouped by the + * type of change, then by team area. + */ +public class ReleaseNotesGenerator { + /** + * These mappings translate change types into the headings as they should appears in the release notes. + */ + private static final Map TYPE_LABELS = new HashMap<>(); + + static { + TYPE_LABELS.put("breaking", "Breaking changes"); + TYPE_LABELS.put("breaking-java", "Breaking Java changes"); + TYPE_LABELS.put("bug", "Bug fixes"); + TYPE_LABELS.put("deprecation", "Deprecations"); + TYPE_LABELS.put("enhancement", "Enhancements"); + TYPE_LABELS.put("feature", "New features"); + TYPE_LABELS.put("regression", "Regressions"); + TYPE_LABELS.put("upgrade", "Upgrades"); + } + + static void update(File templateFile, File outputFile, List changelogs) throws IOException { + final String templateString = Files.readString(templateFile.toPath()); + + try (FileWriter output = new FileWriter(outputFile)) { + generateFile(VersionProperties.getElasticsearchVersion(), templateString, changelogs, output); + } + } + + @VisibleForTesting + static void generateFile(Version version, String template, List changelogs, Writer outputWriter) throws IOException { + final var changelogsByVersionByTypeByArea = buildChangelogBreakdown(version, changelogs); + + final Map bindings = new HashMap<>(); + bindings.put("changelogsByVersionByTypeByArea", changelogsByVersionByTypeByArea); + bindings.put("TYPE_LABELS", TYPE_LABELS); + + try { + final SimpleTemplateEngine engine = new SimpleTemplateEngine(); + engine.createTemplate(template).make(bindings).writeTo(outputWriter); + } catch (ClassNotFoundException e) { + throw new GradleException("Failed to generate file from template", e); + } + } + + private static Map>>> buildChangelogBreakdown( + Version elasticsearchVersion, + List changelogs + ) { + final Predicate includedInSameMinor = v -> v.getMajor() == elasticsearchVersion.getMajor() + && v.getMinor() == elasticsearchVersion.getMinor(); + + final Map>>> changelogsByVersionByTypeByArea = changelogs.stream() + .collect( + Collectors.groupingBy( + // Key changelog entries by the earlier version in which they were released + entry -> entry.getVersions() + .stream() + .map(v -> Version.fromString(v.replaceFirst("^v", ""))) + .filter(includedInSameMinor) + .sorted() + .findFirst() + .get(), + + // Generate a reverse-ordered map. Despite the IDE saying the type can be inferred, removing it + // causes the compiler to complain. + () -> new TreeMap>>>(Comparator.reverseOrder()), + + // Group changelogs entries by their change type + Collectors.groupingBy( + // Entries with breaking info are always put in the breaking section + entry -> entry.getBreaking() == null ? entry.getType() : "breaking", + TreeMap::new, + // Group changelogs for each type by their team area + Collectors.groupingBy( + // `security` and `known-issue` areas don't need to supply an area + entry -> entry.getType().equals("known-issue") || entry.getType().equals("security") + ? "_all_" + : entry.getArea(), + TreeMap::new, + Collectors.toList() + ) + ) + ) + ); + + // Sort per-area changelogs by their summary text. Assumes that the underlying list is sortable + changelogsByVersionByTypeByArea.forEach( + (_version, byVersion) -> byVersion.forEach( + (_type, byTeam) -> byTeam.forEach( + (_team, changelogsForTeam) -> changelogsForTeam.sort(Comparator.comparing(ChangelogEntry::getSummary)) + ) + ) + ); + + return changelogsByVersionByTypeByArea; + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesIndexUpdater.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesIndexUpdater.java new file mode 100644 index 0000000000000..5403d1e03f303 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseNotesIndexUpdater.java @@ -0,0 +1,97 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.release; + +import groovy.text.SimpleTemplateEngine; + +import com.google.common.annotations.VisibleForTesting; + +import org.elasticsearch.gradle.Version; +import org.elasticsearch.gradle.VersionProperties; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.Writer; +import java.nio.file.Files; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * This class ensures that the release notes index page has the appropriate anchors and include directives + * for the current repository version. It achieves this by parsing out the existing entries and writing + * out the file again. + */ +public class ReleaseNotesIndexUpdater { + + static void update(File indexTemplate, File indexFile) throws IOException { + final List existingIndexLines = Files.readAllLines(indexFile.toPath()); + try (FileWriter indexFileWriter = new FileWriter(indexFile)) { + generateFile( + VersionProperties.getElasticsearchVersion(), + existingIndexLines, + Files.readString(indexTemplate.toPath()), + indexFileWriter + ); + } + } + + @VisibleForTesting + static void generateFile(Version version, List existingIndexLines, String indexTemplate, Writer outputWriter) + throws IOException { + final List existingVersions = existingIndexLines.stream() + .filter(line -> line.startsWith("* < line.replace("* <>", "")) + .distinct() + .collect(Collectors.toList()); + + final List existingIncludes = existingIndexLines.stream() + .filter(line -> line.startsWith("include::")) + .map(line -> line.replace("include::release-notes/", "").replace(".asciidoc[]", "")) + .distinct() + .collect(Collectors.toList()); + + final String versionString = version.toString(); + + if (existingVersions.contains(versionString) == false) { + int insertionIndex = existingVersions.size() - 1; + while (insertionIndex > 0 && Version.fromString(existingVersions.get(insertionIndex)).before(version)) { + insertionIndex -= 1; + } + existingVersions.add(insertionIndex, versionString); + } + + final String includeString = version.getMajor() + "." + version.getMinor(); + + if (existingIncludes.contains(includeString) == false) { + int insertionIndex = existingIncludes.size() - 1; + while (insertionIndex > 0 && Version.fromString(ensurePatchVersion(existingIncludes.get(insertionIndex))).before(version)) { + insertionIndex -= 1; + } + existingIncludes.add(insertionIndex, includeString); + } + + final Map bindings = new HashMap<>(); + bindings.put("existingVersions", existingVersions); + bindings.put("existingIncludes", existingIncludes); + + try { + final SimpleTemplateEngine engine = new SimpleTemplateEngine(); + engine.createTemplate(indexTemplate).make(bindings).writeTo(outputWriter); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + } + + private static String ensurePatchVersion(String version) { + return version.matches("^\\d+\\.\\d+\\.\\d+.*$") ? version : version + ".0"; + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java new file mode 100644 index 0000000000000..d7d85504a0178 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java @@ -0,0 +1,97 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.release; + +import org.elasticsearch.gradle.Version; +import org.elasticsearch.gradle.VersionProperties; +import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitTaskPlugin; +import org.elasticsearch.gradle.internal.precommit.ValidateYamlAgainstSchemaTask; +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.file.Directory; +import org.gradle.api.file.FileTree; +import org.gradle.api.file.ProjectLayout; +import org.gradle.api.provider.Provider; +import org.gradle.api.tasks.TaskProvider; +import org.gradle.api.tasks.util.PatternSet; + +import java.io.File; +import javax.inject.Inject; + +/** + * This plugin defines tasks related to releasing Elasticsearch. + */ +public class ReleaseToolsPlugin implements Plugin { + + private static final String RESOURCES = "build-tools-internal/src/main/resources/"; + + private final ProjectLayout projectLayout; + + @Inject + public ReleaseToolsPlugin(ProjectLayout projectLayout) { + this.projectLayout = projectLayout; + } + + @Override + public void apply(Project project) { + project.getPluginManager().apply(PrecommitTaskPlugin.class); + final Directory projectDirectory = projectLayout.getProjectDirectory(); + + final FileTree yamlFiles = projectDirectory.dir("docs/changelog") + .getAsFileTree() + .matching(new PatternSet().include("**/*.yml", "**/*.yaml")); + + final Provider validateChangelogsAgainstYamlTask = project.getTasks() + .register("validateChangelogsAgainstSchema", ValidateYamlAgainstSchemaTask.class, task -> { + task.setGroup("Documentation"); + task.setDescription("Validate that the changelog YAML files comply with the changelog schema"); + task.setInputFiles(yamlFiles); + task.setJsonSchema(new File(project.getRootDir(), RESOURCES + "changelog-schema.json")); + task.setReport(new File(project.getBuildDir(), "reports/validateYaml.txt")); + }); + + final TaskProvider validateChangelogsTask = project.getTasks() + .register("validateChangelogs", ValidateChangelogEntryTask.class, task -> { + task.setGroup("Documentation"); + task.setDescription("Validate that all changelog YAML files are well-formed"); + task.setChangelogs(yamlFiles); + task.dependsOn(validateChangelogsAgainstYamlTask); + }); + + project.getTasks().register("generateReleaseNotes", GenerateReleaseNotesTask.class).configure(task -> { + final Version version = VersionProperties.getElasticsearchVersion(); + + task.setGroup("Documentation"); + task.setDescription("Generates release notes from changelog files held in this checkout"); + task.setChangelogs(yamlFiles); + + task.setReleaseNotesIndexTemplate(projectDirectory.file(RESOURCES + "templates/release-notes-index.asciidoc")); + task.setReleaseNotesIndexFile(projectDirectory.file("docs/reference/release-notes.asciidoc")); + + task.setReleaseNotesTemplate(projectDirectory.file(RESOURCES + "templates/release-notes.asciidoc")); + task.setReleaseNotesFile( + projectDirectory.file(String.format("docs/reference/release-notes/%d.%d.asciidoc", version.getMajor(), version.getMinor())) + ); + + task.setReleaseHighlightsTemplate(projectDirectory.file(RESOURCES + "templates/release-highlights.asciidoc")); + task.setReleaseHighlightsFile(projectDirectory.file("docs/reference/release-notes/highlights.asciidoc")); + + task.setBreakingChangesTemplate(projectDirectory.file(RESOURCES + "templates/breaking-changes.asciidoc")); + task.setBreakingChangesFile( + projectDirectory.file( + String.format("docs/reference/migration/migrate_%d_%d.asciidoc", version.getMajor(), version.getMinor()) + ) + ); + + task.dependsOn(validateChangelogsTask); + }); + + project.getTasks().named("precommit").configure(task -> task.dependsOn(validateChangelogsTask)); + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ValidateChangelogEntryTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ValidateChangelogEntryTask.java new file mode 100644 index 0000000000000..5f030eb074653 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ValidateChangelogEntryTask.java @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.release; + +import org.gradle.api.DefaultTask; +import org.gradle.api.GradleException; +import org.gradle.api.file.ConfigurableFileCollection; +import org.gradle.api.file.FileCollection; +import org.gradle.api.file.ProjectLayout; +import org.gradle.api.model.ObjectFactory; +import org.gradle.api.tasks.InputFiles; +import org.gradle.api.tasks.TaskAction; + +import javax.inject.Inject; +import java.net.URI; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Performs additional checks on changelog files, beyond whether they conform to the schema. + */ +public class ValidateChangelogEntryTask extends DefaultTask { + private final ConfigurableFileCollection changelogs; + private final ProjectLayout projectLayout; + + @Inject + public ValidateChangelogEntryTask(ObjectFactory objectFactory, ProjectLayout projectLayout) { + this.changelogs = objectFactory.fileCollection(); + this.projectLayout = projectLayout; + } + + @TaskAction + public void executeTask() { + final URI rootDir = projectLayout.getProjectDirectory().getAsFile().toURI(); + final Map changelogs = this.changelogs.getFiles() + .stream() + .collect(Collectors.toMap(file -> rootDir.relativize(file.toURI()).toString(), ChangelogEntry::parse)); + + // We don't try to find all such errors, because we expect them to be rare e.g. only + // when a new file is added. + changelogs.forEach((path, entry) -> { + final String type = entry.getType(); + + if (type.equals("known-issue") == false && type.equals("security") == false) { + if (entry.getPr() == null) { + throw new GradleException("[" + path + "] must provide a [pr] number (only 'known-issue' and " + + "'security' entries can omit this"); + } + + if (entry.getArea() == null) { + throw new GradleException("[" + path + "] must provide an [area] (only 'known-issue' and " + + "'security' entries can omit this"); + } + } + + if ((type.equals("breaking") || type.equals("breaking-java")) && entry.getBreaking() == null) { + throw new GradleException( + "[" + path + "] has type [" + type + "] and must supply a [breaking] section with further information" + ); + } + + if (type.equals("deprecation") && entry.getDeprecation() == null) { + throw new GradleException( + "[" + path + "] has type [deprecation] and must supply a [deprecation] section with further information" + ); + } + }); + } + + @InputFiles + public FileCollection getChangelogs() { + return changelogs; + } + + public void setChangelogs(FileCollection files) { + this.changelogs.setFrom(files); + } +} diff --git a/build-tools-internal/src/main/resources/changelog-schema.json b/build-tools-internal/src/main/resources/changelog-schema.json new file mode 100644 index 0000000000000..a2dfc5ecd306f --- /dev/null +++ b/build-tools-internal/src/main/resources/changelog-schema.json @@ -0,0 +1,234 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/elastic/elasticsearch/tree/master/docs/changelog", + "$ref": "#/definitions/Changelog", + "definitions": { + "Changelog": { + "type": "object", + "properties": { + "pr": { + "type": "integer" + }, + "issues": { + "type": "array", + "items": { + "type": "integer" + } + }, + "area": { + "type": "string", + "enum": [ + "Aggregations", + "Allocation", + "Analysis", + "Audit", + "Authentication", + "Authorization", + "Autoscaling", + "CCR", + "CRUD", + "Client", + "Cluster Coordination", + "Discovery-Plugins", + "Distributed", + "EQL", + "Engine", + "FIPS", + "Features/CAT APIs", + "Features/Data streams", + "Features/Features", + "Features/ILM+SLM", + "Features/Indices APIs", + "Features/Ingest", + "Features/Java High Level REST Client", + "Features/Java Low Level REST Client", + "Features/Monitoring", + "Features/Stats", + "Features/Watcher", + "Geo", + "Graph", + "Highlighting", + "IdentityProvider", + "Infra/CLI", + "Infra/Circuit Breakers", + "Infra/Core", + "Infra/Logging", + "Infra/Node Lifecycle", + "Infra/Plugins", + "Infra/REST API", + "Infra/Resiliency", + "Infra/Scripting", + "Infra/Settings", + "Infra/Transport API", + "License", + "Machine Learning", + "Mapping", + "Network", + "Packaging", + "Percolator", + "Performance", + "Query Languages", + "Ranking", + "Recovery", + "Reindex", + "Rollup", + "SQL", + "Search", + "Security", + "Snapshot/Restore", + "Store", + "Suggesters", + "TLS", + "Task Management", + "Transform" + ] + }, + "type": { + "type": "string", + "enum": [ + "breaking", + "breaking-java", + "bug", + "deprecation", + "enhancement", + "feature", + "known-issue", + "new-aggregation", + "regression", + "security", + "upgrade" + ] + }, + "summary": { + "type": "string", + "minLength": 1 + }, + "versions": { + "type": "array", + "items": { + "type": "string", + "pattern": "^v?\\d+\\.\\d+\\.\\d+$", + "minItems": 1 + } + }, + "highlight": { + "$ref": "#/definitions/Highlight" + }, + "breaking": { + "$ref": "#/definitions/Breaking" + }, + "deprecation": { + "$ref": "#/definitions/Deprecation" + } + }, + "required": [ + "type", + "summary", + "versions" + ] + }, + "Highlight": { + "properties": { + "notable": { + "type": "boolean" + }, + "title": { + "type": "string", + "minLength": 1 + }, + "body": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "title", + "body" + ], + "additionalProperties": false + }, + "Breaking": { + "properties": { + "area": { + "$ref": "#/definitions/breakingArea" + }, + "title": { + "type": "string", + "minLength": 1 + }, + "details": { + "type": "string", + "minLength": 1 + }, + "impact": { + "type": "string", + "minLength": 1 + }, + "notable": { + "type": "boolean" + } + }, + "required": [ + "area", + "title", + "details", + "impact" + ], + "additionalProperties": false + }, + "Deprecation": { + "properties": { + "area": { + "$ref": "#/definitions/breakingArea" + }, + "title": { + "type": "string", + "minLength": 1 + }, + "body": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "area", + "title", + "body" + ], + "additionalProperties": false + }, + "breakingArea": { + "type": "string", + "enum": [ + "API", + "Aggregation", + "Allocation", + "Authentication", + "CCR", + "Cluster", + "Discovery", + "Engine", + "HTTP", + "Highlighters", + "Indices", + "Java", + "License Information", + "Logging", + "Machine Learning", + "Mappings", + "Networking", + "Packaging", + "Plugins", + "Script Cache", + "Search Changes", + "Search", + "Security", + "Settings", + "Snapshot and Restore", + "Transform", + "Transport" + ] + }, + "additionalProperties": false + } +} diff --git a/build-tools-internal/src/main/resources/templates/breaking-changes.asciidoc b/build-tools-internal/src/main/resources/templates/breaking-changes.asciidoc new file mode 100644 index 0000000000000..38573747863e9 --- /dev/null +++ b/build-tools-internal/src/main/resources/templates/breaking-changes.asciidoc @@ -0,0 +1,102 @@ +[[migrating-${majorDotMinor}]] +== Migrating to ${majorDotMinor} +++++ +${majorDotMinor} +++++ + +This section discusses the changes that you need to be aware of when migrating +your application to {es} ${majorDotMinor}. + +See also <> and <>. +<% if (isElasticsearchSnapshot) { %> +coming[${version}] +<% } %> +//NOTE: The notable-breaking-changes tagged regions are re-used in the +//Installation and Upgrade Guide +<% if (breakingChangesByNotabilityByArea.isEmpty() == false) { %> +[discrete] +[[breaking-changes-${majorDotMinor}]] +=== Breaking changes + +The following changes in {es} ${majorDotMinor} might affect your applications +and prevent them from operating normally. +Before upgrading to ${majorDotMinor} review these changes and take the described steps +to mitigate the impact. + +NOTE: Breaking changes introduced in minor versions are +normally limited to security and bug fixes. +Significant changes in behavior are deprecated in a minor release and +the old behavior is supported until the next major release. +To find out if you are using any deprecated functionality, +enable <>. +<% +[true, false].each { isNotable -> + def breakingChangesByArea = breakingChangesByNotabilityByArea.getOrDefault(isNotable, []) + + breakingChangesByArea.eachWithIndex { area, breakingChanges, i -> + print "\n" + + if (isNotable) { + print "// tag::notable-breaking-changes[]\n" + } + + print "[discrete]\n" + print "[[breaking_${majorMinor}_${ area.toLowerCase().replaceAll("[^a-z0-9]+", "_") }]]\n" + print "==== ${area}\n" + + for (breaking in breakingChanges) { %> +[[${ breaking.anchor }]] +.${breaking.title} +[%collapsible] +==== +*Details* + +${breaking.details.trim()} + +*Impact* + +${breaking.impact.trim()} +==== +<% + } + + if (isNotable) { + print "// end::notable-breaking-changes[]\n" + } + } +} +} +if (deprecationsByArea.empty == false) { %> + +[discrete] +[[deprecated-${majorDotMinor}]] +=== Deprecations + +The following functionality has been deprecated in {es} ${majorDotMinor} +and will be removed in ${nextMajor}. +While this won't have an immediate impact on your applications, +we strongly encourage you take the described steps to update your code +after upgrading to ${majorDotMinor}. + +NOTE: Significant changes in behavior are deprecated in a minor release and +the old behavior is supported until the next major release. +To find out if you are using any deprecated functionality, +enable <>." + +<% +deprecationsByArea.eachWithIndex { area, deprecations, i -> + print "\n[discrete]\n" + print "[[deprecations_${majorMinor}_${ area.toLowerCase().replaceAll("[^a-z0-9]+", "_") }]]" + print "==== ${area} deprecations" + + for (deprecation in deprecations) { %> + +[[${ deprecation.anchor }]] +.${deprecation.title} +[%collapsible] +==== +*Details* + +${deprecation.body.trim()} +==== +<% +} +} +} %> diff --git a/build-tools-internal/src/main/resources/templates/release-highlights.asciidoc b/build-tools-internal/src/main/resources/templates/release-highlights.asciidoc new file mode 100644 index 0000000000000..40b828d609745 --- /dev/null +++ b/build-tools-internal/src/main/resources/templates/release-highlights.asciidoc @@ -0,0 +1,35 @@ +[[release-highlights]] +== What's new in {minor-version} + +coming::[{minor-version}] + +Here are the highlights of what's new and improved in {es} {minor-version}! +ifeval::[\\{release-state}\\"!=\\"unreleased\\"] +For detailed information about this release, see the <> and +<>. +endif::[] +<% if (priorVersions.size > 0) { %> +// Add previous release to the list +Other versions: + +<% +print priorVersions.join("\n| ") +print "\n" +} + +if (notableHighlights.empty == false) { %> +// tag::notable-highlights[] +<% for (highlight in notableHighlights) { %> +[discrete] +[[${ highlight.anchor }]] +=== ${highlight.title} +${highlight.body.trim()} +<% } %> +// end::notable-highlights[] +<% } %> +<% for (highlight in nonNotableHighlights) { %> +[discrete] +[[${ highlight.anchor }]] +=== ${highlight.title} +${highlight.body.trim()} +<% } %> diff --git a/build-tools-internal/src/main/resources/templates/release-notes-index.asciidoc b/build-tools-internal/src/main/resources/templates/release-notes-index.asciidoc new file mode 100644 index 0000000000000..0b62b9b3f1e01 --- /dev/null +++ b/build-tools-internal/src/main/resources/templates/release-notes-index.asciidoc @@ -0,0 +1,12 @@ +[[es-release-notes]] += Release notes + +[partintro] +-- + +This section summarizes the changes in each release. + +<% existingVersions.each { print "* <>\n" } %> +-- + +<% existingIncludes.each { print "include::release-notes/${ it }.asciidoc[]\n" } %> diff --git a/build-tools-internal/src/main/resources/templates/release-notes.asciidoc b/build-tools-internal/src/main/resources/templates/release-notes.asciidoc new file mode 100644 index 0000000000000..35384c8f4ce66 --- /dev/null +++ b/build-tools-internal/src/main/resources/templates/release-notes.asciidoc @@ -0,0 +1,45 @@ +<% for (version in changelogsByVersionByTypeByArea.keySet()) { +%>[[release-notes-$version]] +== {es} version $version +<% if (version.qualifier == "SNAPSHOT") { %> +coming[$version] +<% } %> +Also see <>. +<% if (changelogsByVersionByTypeByArea[version]["security"] != null) { %> +[discrete] +[[security-updates-${version}]] +=== Security updates + +<% for (change in changelogsByVersionByTypeByArea[version].remove("security").remove("_all_")) { + print "* ${change.summary}\n" +} +} +if (changelogsByVersionByTypeByArea[version]["known-issue"] != null) { %> +[discrete] +[[known-issues-${version}]] +=== Known issues + +<% for (change in changelogsByVersionByTypeByArea[version].remove("known-issue").remove("_all_")) { + print "* ${change.summary}\n" +} +} +for (changeType in changelogsByVersionByTypeByArea[version].keySet()) { %> +[[${ changeType }-${ version }]] +[float] +=== ${ TYPE_LABELS[changeType] } +<% for (team in changelogsByVersionByTypeByArea[version][changeType].keySet()) { + print "\n${team}::\n"; + + for (change in changelogsByVersionByTypeByArea[version][changeType][team]) { + print "* ${change.summary} {es-pull}${change.pr}[#${change.pr}]" + if (change.issues != null && change.issues.empty == false) { + print change.issues.size() == 1 ? " (issue: " : " (issues: " + print change.issues.collect { "{es-issue}${it}[#${it}]" }.join(", ") + print ")" + } + print "\n" + } +} +} +} +%> diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/VersionTests.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/VersionTests.java deleted file mode 100644 index 37aa5cf9d21da..0000000000000 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/VersionTests.java +++ /dev/null @@ -1,115 +0,0 @@ -package org.elasticsearch.gradle; - -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import org.elasticsearch.gradle.internal.test.GradleUnitTestCase; -import org.junit.Rule; -import org.junit.rules.ExpectedException; - -import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; - -public class VersionTests extends GradleUnitTestCase { - - @Rule - public ExpectedException expectedEx = ExpectedException.none(); - - public void testVersionParsing() { - assertVersionEquals("7.0.1", 7, 0, 1); - assertVersionEquals("7.0.1-alpha2", 7, 0, 1); - assertVersionEquals("5.1.2-rc3", 5, 1, 2); - assertVersionEquals("6.1.2-SNAPSHOT", 6, 1, 2); - assertVersionEquals("6.1.2-beta1-SNAPSHOT", 6, 1, 2); - assertVersionEquals("17.03.11", 17, 3, 11); - } - - public void testRelaxedVersionParsing() { - assertVersionEquals("6.1.2", 6, 1, 2, Version.Mode.RELAXED); - assertVersionEquals("6.1.2-SNAPSHOT", 6, 1, 2, Version.Mode.RELAXED); - assertVersionEquals("6.1.2-beta1-SNAPSHOT", 6, 1, 2, Version.Mode.RELAXED); - assertVersionEquals("6.1.2-foo", 6, 1, 2, Version.Mode.RELAXED); - assertVersionEquals("6.1.2-foo-bar", 6, 1, 2, Version.Mode.RELAXED); - assertVersionEquals("16.01.22", 16, 1, 22, Version.Mode.RELAXED); - } - - public void testCompareWithStringVersions() { - assertTrue("1.10.20 is not interpreted as before 2.0.0", Version.fromString("1.10.20").before("2.0.0")); - assertTrue( - "7.0.0-alpha1 should be equal to 7.0.0-alpha1", - Version.fromString("7.0.0-alpha1").equals(Version.fromString("7.0.0-alpha1")) - ); - assertTrue( - "7.0.0-SNAPSHOT should be equal to 7.0.0-SNAPSHOT", - Version.fromString("7.0.0-SNAPSHOT").equals(Version.fromString("7.0.0-SNAPSHOT")) - ); - } - - public void testCollections() { - assertTrue( - Arrays.asList( - Version.fromString("5.2.0"), - Version.fromString("5.2.1-SNAPSHOT"), - Version.fromString("6.0.0"), - Version.fromString("6.0.1"), - Version.fromString("6.1.0") - ).containsAll(Arrays.asList(Version.fromString("6.0.1"), Version.fromString("5.2.1-SNAPSHOT"))) - ); - Set versions = new HashSet<>(); - versions.addAll( - Arrays.asList( - Version.fromString("5.2.0"), - Version.fromString("5.2.1-SNAPSHOT"), - Version.fromString("6.0.0"), - Version.fromString("6.0.1"), - Version.fromString("6.1.0") - ) - ); - Set subset = new HashSet<>(); - subset.addAll(Arrays.asList(Version.fromString("6.0.1"), Version.fromString("5.2.1-SNAPSHOT"))); - assertTrue(versions.containsAll(subset)); - } - - public void testToString() { - assertEquals("7.0.1", new Version(7, 0, 1).toString()); - } - - public void testCompareVersions() { - assertEquals(0, new Version(7, 0, 0).compareTo(new Version(7, 0, 0))); - assertOrder(Version.fromString("19.0.1"), Version.fromString("20.0.3")); - } - - public void testExceptionEmpty() { - expectedEx.expect(IllegalArgumentException.class); - expectedEx.expectMessage("Invalid version format"); - Version.fromString(""); - } - - public void testExceptionSyntax() { - expectedEx.expect(IllegalArgumentException.class); - expectedEx.expectMessage("Invalid version format"); - Version.fromString("foo.bar.baz"); - } - - private void assertOrder(Version smaller, Version bigger) { - assertEquals(smaller + " should be smaller than " + bigger, -1, smaller.compareTo(bigger)); - } - - private void assertVersionEquals(String stringVersion, int major, int minor, int revision) { - assertVersionEquals(stringVersion, major, minor, revision, Version.Mode.STRICT); - } - - private void assertVersionEquals(String stringVersion, int major, int minor, int revision, Version.Mode mode) { - Version version = Version.fromString(stringVersion, mode); - assertEquals(major, version.getMajor()); - assertEquals(minor, version.getMinor()); - assertEquals(revision, version.getRevision()); - } - -} diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/Version.java b/build-tools/src/main/java/org/elasticsearch/gradle/Version.java index 8ac6193bdc3f9..a86e16ad740fd 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/Version.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/Version.java @@ -19,6 +19,7 @@ public final class Version implements Comparable { private final int minor; private final int revision; private final int id; + private final String qualifier; /** * Specifies how a version string should be parsed. @@ -36,27 +37,23 @@ public enum Mode { RELAXED } - private static final Pattern pattern = Pattern.compile("(\\d+)\\.(\\d+)\\.(\\d+)(-alpha\\d+|-beta\\d+|-rc\\d+)?(-SNAPSHOT)?"); + private static final Pattern pattern = Pattern.compile("(\\d+)\\.(\\d+)\\.(\\d+)(?:-(alpha\\d+|beta\\d+|rc\\d+|SNAPSHOT))?"); - private static final Pattern relaxedPattern = Pattern.compile("(\\d+)\\.(\\d+)\\.(\\d+)(-[a-zA-Z0-9_]+)*?"); + private static final Pattern relaxedPattern = Pattern.compile("v?(\\d+)\\.(\\d+)\\.(\\d+)(?:-([a-zA-Z0-9_]+(?:-[a-zA-Z0-9]+)*))?"); public Version(int major, int minor, int revision) { - Objects.requireNonNull(major, "major version can't be null"); - Objects.requireNonNull(minor, "minor version can't be null"); - Objects.requireNonNull(revision, "revision version can't be null"); + this(major, minor, revision, null); + } + + public Version(int major, int minor, int revision, String qualifier) { this.major = major; this.minor = minor; this.revision = revision; - // currently snapshot is not taken into account + // currently qualifier is not taken into account this.id = major * 10000000 + minor * 100000 + revision * 1000; - } - private static int parseSuffixNumber(String substring) { - if (substring.isEmpty()) { - throw new IllegalArgumentException("Invalid suffix, must contain a number e.x. alpha2"); - } - return Integer.parseInt(substring); + this.qualifier = qualifier; } public static Version fromString(final String s) { @@ -68,17 +65,24 @@ public static Version fromString(final String s, final Mode mode) { Matcher matcher = mode == Mode.STRICT ? pattern.matcher(s) : relaxedPattern.matcher(s); if (matcher.matches() == false) { String expected = mode == Mode.STRICT - ? "major.minor.revision[-(alpha|beta|rc)Number][-SNAPSHOT]" + ? "major.minor.revision[-(alpha|beta|rc)Number|-SNAPSHOT]" : "major.minor.revision[-extra]"; throw new IllegalArgumentException("Invalid version format: '" + s + "'. Should be " + expected); } - return new Version(Integer.parseInt(matcher.group(1)), parseSuffixNumber(matcher.group(2)), parseSuffixNumber(matcher.group(3))); + String qualifier = matcher.group(4); + + return new Version( + Integer.parseInt(matcher.group(1)), + Integer.parseInt(matcher.group(2)), + Integer.parseInt(matcher.group(3)), + qualifier + ); } @Override public String toString() { - return String.valueOf(getMajor()) + "." + String.valueOf(getMinor()) + "." + String.valueOf(getRevision()); + return getMajor() + "." + getMinor() + "." + getRevision(); } public boolean before(Version compareTo) { @@ -146,6 +150,10 @@ protected int getId() { return id; } + public String getQualifier() { + return qualifier; + } + @Override public int compareTo(Version other) { return Integer.compare(getId(), other.getId()); diff --git a/build-tools/src/test/java/org/elasticsearch/gradle/VersionTests.java b/build-tools/src/test/java/org/elasticsearch/gradle/VersionTests.java index 37aa5cf9d21da..2dae3d9f70900 100644 --- a/build-tools/src/test/java/org/elasticsearch/gradle/VersionTests.java +++ b/build-tools/src/test/java/org/elasticsearch/gradle/VersionTests.java @@ -12,10 +12,15 @@ import org.junit.Rule; import org.junit.rules.ExpectedException; -import java.util.Arrays; import java.util.HashSet; +import java.util.List; import java.util.Set; +import static java.util.Arrays.asList; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasItems; +import static org.hamcrest.Matchers.nullValue; + public class VersionTests extends GradleUnitTestCase { @Rule @@ -26,7 +31,6 @@ public void testVersionParsing() { assertVersionEquals("7.0.1-alpha2", 7, 0, 1); assertVersionEquals("5.1.2-rc3", 5, 1, 2); assertVersionEquals("6.1.2-SNAPSHOT", 6, 1, 2); - assertVersionEquals("6.1.2-beta1-SNAPSHOT", 6, 1, 2); assertVersionEquals("17.03.11", 17, 3, 11); } @@ -41,29 +45,30 @@ public void testRelaxedVersionParsing() { public void testCompareWithStringVersions() { assertTrue("1.10.20 is not interpreted as before 2.0.0", Version.fromString("1.10.20").before("2.0.0")); - assertTrue( + assertEquals( "7.0.0-alpha1 should be equal to 7.0.0-alpha1", - Version.fromString("7.0.0-alpha1").equals(Version.fromString("7.0.0-alpha1")) + Version.fromString("7.0.0-alpha1"), + Version.fromString("7.0.0-alpha1") ); - assertTrue( + assertEquals( "7.0.0-SNAPSHOT should be equal to 7.0.0-SNAPSHOT", - Version.fromString("7.0.0-SNAPSHOT").equals(Version.fromString("7.0.0-SNAPSHOT")) + Version.fromString("7.0.0-SNAPSHOT"), + Version.fromString("7.0.0-SNAPSHOT") ); } public void testCollections() { - assertTrue( - Arrays.asList( - Version.fromString("5.2.0"), - Version.fromString("5.2.1-SNAPSHOT"), - Version.fromString("6.0.0"), - Version.fromString("6.0.1"), - Version.fromString("6.1.0") - ).containsAll(Arrays.asList(Version.fromString("6.0.1"), Version.fromString("5.2.1-SNAPSHOT"))) + List aList = asList( + Version.fromString("5.2.0"), + Version.fromString("5.2.1-SNAPSHOT"), + Version.fromString("6.0.0"), + Version.fromString("6.0.1"), + Version.fromString("6.1.0") ); - Set versions = new HashSet<>(); - versions.addAll( - Arrays.asList( + assertThat(aList, hasItems(Version.fromString("6.0.1"), Version.fromString("5.2.1-SNAPSHOT"))); + + Set aSet = new HashSet<>( + asList( Version.fromString("5.2.0"), Version.fromString("5.2.1-SNAPSHOT"), Version.fromString("6.0.0"), @@ -71,9 +76,7 @@ public void testCollections() { Version.fromString("6.1.0") ) ); - Set subset = new HashSet<>(); - subset.addAll(Arrays.asList(Version.fromString("6.0.1"), Version.fromString("5.2.1-SNAPSHOT"))); - assertTrue(versions.containsAll(subset)); + assertThat(aSet, hasItems(Version.fromString("6.0.1"), Version.fromString("5.2.1-SNAPSHOT"))); } public void testToString() { @@ -97,6 +100,20 @@ public void testExceptionSyntax() { Version.fromString("foo.bar.baz"); } + public void testQualifiers() { + Version v = Version.fromString("1.2.3"); + assertThat(v.getQualifier(), nullValue()); + + v = Version.fromString("1.2.3-rc1"); + assertThat(v.getQualifier(), equalTo("rc1")); + + v = Version.fromString("1.2.3-SNAPSHOT"); + assertThat(v.getQualifier(), equalTo("SNAPSHOT")); + + v = Version.fromString("1.2.3-SNAPSHOT-EXTRA", Version.Mode.RELAXED); + assertThat(v.getQualifier(), equalTo("SNAPSHOT-EXTRA")); + } + private void assertOrder(Version smaller, Version bigger) { assertEquals(smaller + " should be smaller than " + bigger, -1, smaller.compareTo(bigger)); } diff --git a/build.gradle b/build.gradle index 8350fddb1e59b..7dd5e75b188b7 100644 --- a/build.gradle +++ b/build.gradle @@ -43,6 +43,7 @@ plugins { id 'elasticsearch.fips' id 'elasticsearch.internal-testclusters' id 'elasticsearch.run' + id 'elasticsearch.release-tools' id "com.diffplug.spotless" version "5.12.5" apply false } @@ -410,7 +411,7 @@ gradle.projectsEvaluated { } } -tasks.register("precommit") { +tasks.named("precommit") { dependsOn gradle.includedBuild('build-tools').task(':precommit') dependsOn gradle.includedBuild('build-tools-internal').task(':precommit') }