Simplify plugin configuration

CHANGELOG.md

@@ -2,6 +2,20 @@
 
 ## [Unreleased]
 
+- Simplifies plugin configuration.
+  The following block represents the new minimal configuration:
+
+  ```kotlin
+  grammarKit {
+      lexerSource = layout.projectDirectory.file("<path to *.flex file>")
+      parserSource = layout.projectDirectory.file("<path to *.bnf file>")
+  }
+  ```
+- Introduces `GenerateLexerTask.targetRootOutputDir` as a replacement of `targetOutputDir`.
+  Automatically create a subdirectory matching the package when using this new property.
+- Deprecates `pathToParser` and `pathToPsiRoot` of `GenerateParserTask`.
+- Purge stale files by default (as soon as you migrated away from the old properties)
+
 ## [2022.3.2.2] - 2024-02-21
 
 - Support for IntelliJ Platform `2024.1` — added `opentelementry` library.

src/main/kotlin/org/jetbrains/grammarkit/GrammarKitConstants.kt

@@ -6,7 +6,9 @@ package org.jetbrains.grammarkit
 object GrammarKitConstants {
     const val GROUP_NAME = "grammarKit"
     const val GENERATE_LEXER_TASK_NAME = "generateLexer"
+    const val GENERATE_LEXER_OUT_DIR = "generated/sources/grammarkit-lexer/java/main"
     const val GENERATE_PARSER_TASK_NAME = "generateParser"
+    const val GENERATE_PARSER_OUT_DIR = "generated/sources/grammarkit-parser/java/main"
     const val GRAMMAR_KIT_DEFAULT_VERSION = "2022.3.2"
     const val JFLEX_DEFAULT_VERSION = "1.9.2"
     const val MINIMAL_SUPPORTED_GRADLE_VERSION = "7.4"

src/main/kotlin/org/jetbrains/grammarkit/GrammarKitPlugin.kt

@@ -11,12 +11,17 @@ import org.gradle.api.plugins.JavaPlugin.COMPILE_CLASSPATH_CONFIGURATION_NAME
 import org.gradle.api.plugins.JavaPlugin.COMPILE_ONLY_CONFIGURATION_NAME
 import org.gradle.api.plugins.PluginInstantiationException
 import org.gradle.api.provider.Provider
+import org.gradle.api.tasks.SourceSet
+import org.gradle.api.tasks.SourceSetContainer
 import org.gradle.kotlin.dsl.create
 import org.gradle.kotlin.dsl.maven
 import org.gradle.kotlin.dsl.register
+import org.gradle.kotlin.dsl.the
 import org.gradle.kotlin.dsl.withType
 import org.gradle.util.GradleVersion
+import org.jetbrains.grammarkit.GrammarKitConstants.GENERATE_LEXER_OUT_DIR
 import org.jetbrains.grammarkit.GrammarKitConstants.GENERATE_LEXER_TASK_NAME
+import org.jetbrains.grammarkit.GrammarKitConstants.GENERATE_PARSER_OUT_DIR
 import org.jetbrains.grammarkit.GrammarKitConstants.GENERATE_PARSER_TASK_NAME
 import org.jetbrains.grammarkit.GrammarKitConstants.GRAMMAR_KIT_CLASS_PATH_CONFIGURATION_NAME
 import org.jetbrains.grammarkit.GrammarKitConstants.MINIMAL_SUPPORTED_GRADLE_VERSION
@@ -36,16 +41,33 @@ abstract class GrammarKitPlugin : Plugin<Project> {
             val compileClasspathConfiguration = project.configurations.named(COMPILE_CLASSPATH_CONFIGURATION_NAME)
             val compileOnlyConfiguration = project.configurations.named(COMPILE_ONLY_CONFIGURATION_NAME)
 
-            project.tasks.register<GenerateLexerTask>(GENERATE_LEXER_TASK_NAME)
+            val generateLexerTask = project.tasks.register<GenerateLexerTask>(GENERATE_LEXER_TASK_NAME) {
+                sourceFile.convention(extension.lexerSource)
+                targetRootOutputDir.convention(project.layout.buildDirectory.dir(GENERATE_LEXER_OUT_DIR))
+                // Further configuration of this task is performed below together with all other tasks of type GenerateLexerTask
+            }
+            val generateParserTask = project.tasks.register<GenerateParserTask>(GENERATE_PARSER_TASK_NAME) {
+                sourceFile.convention(extension.parserSource)
+                targetRootOutputDir.convention(project.layout.buildDirectory.dir(GENERATE_PARSER_OUT_DIR))
+                // Further configuration of this task is performed below together with all other tasks of type GenerateParserTask
+            }
+
+            project.the(SourceSetContainer::class).named(SourceSet.MAIN_SOURCE_SET_NAME) {
+                java {
+                    srcDirs(
+                        // Using `.orElse(listOf())` as a workaround because Gradle doesn't allow empty providers in `SourceDirectorySet.srcDirs`
+                        extension.lexerSource.zip(generateLexerTask) { _, task -> listOf(task) }.orElse(listOf()),
+                        extension.parserSource.zip(generateParserTask) { _, task -> listOf(task) }.orElse(listOf()),
+                    )
+                }
+            }
 
             project.tasks.withType<GenerateLexerTask>().configureEach {
                 classpath(getClasspath(grammarKitClassPathConfiguration, compileClasspathConfiguration) { file ->
                     file.name.startsWith("jflex")
                 })
             }
 
-            project.tasks.register<GenerateParserTask>(GENERATE_PARSER_TASK_NAME)
-
             project.tasks.withType<GenerateParserTask>().configureEach {
                 val requiredLibs = listOf(
                     "app", "lib", "jdom", "trove4j", "junit", "guava", "asm-all", "automaton", "platform-api", "platform-impl",

src/main/kotlin/org/jetbrains/grammarkit/GrammarKitPluginExtension.kt

@@ -2,9 +2,13 @@
 
 package org.jetbrains.grammarkit
 
+import org.gradle.api.file.RegularFileProperty
 import org.gradle.api.provider.Property
 import org.gradle.api.tasks.Input
+import org.gradle.api.tasks.InputFile
 import org.gradle.api.tasks.Optional
+import org.jetbrains.grammarkit.tasks.GenerateLexerTask
+import org.jetbrains.grammarkit.tasks.GenerateParserTask
 
 abstract class GrammarKitPluginExtension {
 
@@ -28,8 +32,7 @@
     abstract val jflexRelease: Property<String>
 
     /**
-     * Version of the IntelliJ to build the classpath for [org.jetbrains.grammarkit.tasks.GenerateParserTask]
-     * and [org.jetbrains.grammarkit.tasks.GenerateLexerTask] tasks.
+     * Version of the IntelliJ to build the classpath for [GenerateParserTask] and [GenerateLexerTask] tasks.
      * If provided, [grammarKitRelease] and [jflexRelease] properties are ignored as both dependencies will be provided
      * from the given IntelliJ IDEA release.
      *
@@ -39,9 +42,37 @@
     @get:Optional
     abstract val intellijRelease: Property<String>
 
+    /**
+     * The source file for `:generateLexer`.
+     * When this property is used, the output directory of the task will automatically be added to the main source set.
+     * If this property is set, the plugin ...
+     *
+     * - uses the value as the default for [`tasks.generateLexer.sourceFile`][GenerateLexerTask.sourceFile].
+     * - calls `sourceSets.main.java.srcDir(tasks.generateLexer)`
+     *
+     * Default value: `null`
+     */
+    @get:InputFile
+    @get:Optional
+    abstract val lexerSource: RegularFileProperty
+
+    /**
+     * The source file for `:generateParser`.
+     * When this property is used, the output directory of the task will automatically be added to the main source set.
+     * If this property is set, the plugin ...
+     *
+     * - uses the value as the default for [`tasks.generateParser.sourceFile`][GenerateParserTask.sourceFile].
+     * - calls `sourceSets.main.java.srcDir(tasks.generateParser)`
+     *
+     * Default value: `null`
+     */
+    @get:InputFile
+    @get:Optional
+    abstract val parserSource: RegularFileProperty
+
     init {
         grammarKitRelease.convention(GrammarKitConstants.GRAMMAR_KIT_DEFAULT_VERSION)
         jflexRelease.convention(GrammarKitConstants.JFLEX_DEFAULT_VERSION)
         intellijRelease.convention("")
     }
 }

src/main/kotlin/org/jetbrains/grammarkit/tasks/GenerateLexerTask.kt

@@ -4,6 +4,7 @@
 
 import org.apache.tools.ant.util.TeeOutputStream
 import org.gradle.api.GradleException
+import org.gradle.api.file.Directory
 import org.gradle.api.file.DirectoryProperty
 import org.gradle.api.file.RegularFile
 import org.gradle.api.file.RegularFileProperty
@@ -13,6 +14,8 @@
 import org.jetbrains.grammarkit.GrammarKitConstants
 import org.jetbrains.grammarkit.path
 import java.io.ByteArrayOutputStream
+import java.io.File
+import java.util.regex.Pattern
 
 /**
  * The `generateLexer` task generates a lexer for the given grammar.
@@ -40,12 +43,45 @@
     abstract val targetDir: Property<String>
 
     /**
-     * Required.
      * The output directory for the generated lexer.
+     * The Java file is created directly below the given directory.
+     * The value of [packageName] is ignored.
+     * Stale files in the given directory are *not* deleted during task execution,
+     * unless [purgeOldFiles] is explicitly set to `true`.
      */
+    @Deprecated(
+        message = "Use targetRootOutputDir instead. " +
+                "There is also a new default for the `generateLexer` task which should be sufficient for most use cases." +
+                "When using the new property, stale files in `targetRootOutputDir` are deleted by default " +
+                "and the Java file is created in a subdirectory matching the package of the file. " +
+                "You can restore the previous behavior by adding `purgeOldFiles = false` and `packageName = \"\"`. ",
+        replaceWith = ReplaceWith("targetRootOutputDir"),
+        level = DeprecationLevel.WARNING,
+    )
     @get:OutputDirectory
+    @get:Optional
     abstract val targetOutputDir: DirectoryProperty
 
+    /**
+     * The output directory for the generated lexer.
+     * The Java file for the lexer is created in a subdirectory matching the [packageName].
+     * Stale files in the given directory are deleted during task execution,
+     * unless [purgeOldFiles] is explicitly set to `false`.
+     */
+    @get:OutputDirectory
+    @get:Optional
+    abstract val targetRootOutputDir: DirectoryProperty
+
+    /**
+     * The name of the package where the lexer shall be created.
+     * By default, the task tries to detect the package from the content of [sourceFile].
+     * You may set the value to the empty string (`""`), if no subdirectory shall be created.
+     * The empty string represents the unnamed package.
+     */
+    @get:Input
+    @get:Optional
+    abstract val packageName: Property<String>
+
     /**
      * The Java file name where the generated lexer will be written.
      */
@@ -66,19 +102,29 @@
         level = DeprecationLevel.ERROR
     )
     @get:Internal
     abstract val targetFile: RegularFileProperty
 
     /**
      * The location of the lexer class computed from the [targetOutputDir].
      * Because the lexer class is defined in the flex file, it needs to be passed here too.
      */
+    @Deprecated(
+        message = "You may specify the expected output directory relative to targetRootOutputDir instead.",
+        replaceWith = ReplaceWith("targetRootOutputDir.file(/* add package if necessary */ \"\${lexerClass}.java\")"),
+        level = DeprecationLevel.WARNING,
+    )
     fun targetFile(lexerClass: String): Provider<RegularFile> = targetOutputDir.file("${lexerClass}.java")
 
     /**
      * The location of the lexer class computed from the [targetOutputDir].
      * Because the lexer class is defined in the flex file, it needs to be passed here too.
      */
+    @Deprecated(
+        message = "You may specify the expected output directory relative to targetRootOutputDir instead.",
+        replaceWith = ReplaceWith("targetRootOutputDir.file(lexerClass.map { /* add package if necessary */ \"\${it}.java\"})"),
+        level = DeprecationLevel.WARNING,
+    )
     fun targetFile(lexerClass: Provider<String>): Provider<RegularFile> = lexerClass.flatMap(::targetFile)
 
     @Deprecated(
         message = "The `source` was removed in favour of `sourceFile`.",
@@ -108,6 +154,11 @@
 
     /**
      * Purge old files from the target directory before generating the lexer.
+     * By default, old files are purged unless [targetOutputDir] has been specified.
+     * If you want to disable this option because the output directory is shared with another task,
+     * note that you may run into issues with stale files. Also note that
+     * [overlapping task outputs are discouraged by Gradle](https://docs.gradle.org/current/userguide/organizing_gradle_projects.html#avoid_overlapping_task_outputs)
+     * and may cause issues when using the build cache.
      */
     @get:Input
     @get:Optional
@@ -116,7 +167,13 @@
     @TaskAction
     override fun exec() {
         if (purgeOldFiles.orNull == true) {
-            targetOutputDir.asFile.get().deleteRecursively()
+            targetOutputDir.asFile.orNull?.deleteRecursively()
+        }
+        if (purgeOldFiles.orNull != false) {
+            // Delete targetRootOutputDir even if the directory is overridden by `targetOutputDir`.
+            // The directory may still contain stale files from a previous execution,
+            // and would still be added to the source set when using `srcDir(task)`.
+            targetRootOutputDir.asFile.orNull?.deleteRecursively()
         }
         ByteArrayOutputStream().use { os ->
             try {
@@ -132,7 +189,7 @@
 
     private fun getArguments(): List<String> {
         val args = mutableListOf(
-            "-d", targetOutputDir.path,
+            "-d", getOutputDirectory().path,
         )
 
         if (skeleton.isPresent) {
@@ -144,4 +201,51 @@
 
         return args
     }
+
+    /**
+     * Resolves the correct output directory, considering the package of the lexer file.
+     */
+    private fun getOutputDirectory(): Provider<Directory> {
+        // `targetOutputDir` takes precedence for backwards compatibility
+        return if (targetOutputDir.isPresent) {
+            targetOutputDir
+        } else if (targetRootOutputDir.isPresent) {
+            val packageProvider = packageName.orElse(sourceFile.map(::readPackageDeclaration))
+            targetRootOutputDir.zip(packageProvider) { rootDir, pkg ->
+                if (pkg.isEmpty()) {
+                    rootDir
+                } else {
+                    rootDir.dir(pkg.replace('.', File.separatorChar))
+                }
+            }
+        } else {
+            throw GradleException("""
+                Neither of the properties 'targetOutputDir' and 'targetRootOutputDir' have a configured value.
+            """.trimIndent())
+        }
+    }
+
+    /**
+     * Try to find and read the package declaration in the given source file.
+     * @return the package name or the empty string
+     */
+    private fun readPackageDeclaration(source: RegularFile): String {
+        // The Maven plugin of JFlex 1.9.1 uses a similar approach to detect the package name:
+        // https://github.com/jflex-de/jflex/blob/7b36a83dfb4502c69a7da09cfd15db9c8dd5b701/jflex-maven-plugin/src/main/java/jflex/maven/plugin/jflex/LexSimpleAnalyzerUtils.java#L163-L176
+        // https://github.com/jflex-de/jflex/blob/7b36a83dfb4502c69a7da09cfd15db9c8dd5b701/jflex-maven-plugin/src/main/java/jflex/maven/plugin/jflex/SpecInfo.java#L63-L70
+        // As well as the Ant Task:
+        // https://github.com/jflex-de/jflex/blob/7b36a83dfb4502c69a7da09cfd15db9c8dd5b701/jflex/src/main/java/jflex/anttask/JFlexTask.java#L111-L116
+        // https://github.com/jflex-de/jflex/blob/7b36a83dfb4502c69a7da09cfd15db9c8dd5b701/jflex/src/main/java/jflex/anttask/JFlexTask.java#L150-L155
+        val packagePattern = Pattern.compile("\\s*package\\s+(\\S+)\\s*;.*")
+        source.asFile.useLines { lines ->
+            lines.forEach { line ->
+                val matcher = packagePattern.matcher(line)
+                if (matcher.matches()) {
+                    return matcher.group(1)
+                }
+            }
+        }
+        logger.warn("Could not detect `packageName` for $path, the lexer will be generated in the directory of the unnamed package")
+        return ""
+    }
 }