From a06dacae96825a66d45bd398e03cb7964ba56750 Mon Sep 17 00:00:00 2001 From: Jamie Thompson Date: Mon, 9 Sep 2024 19:03:25 +0200 Subject: [PATCH] WIP 49 - correct positions of build script files --- .../compile-error/src/CompileErrorTests.scala | 27 +- scalalib/src/mill/scalalib/Assembly.scala | 9 +- .../mill/scalalib/worker/ZincWorkerImpl.scala | 377 +++++++++++++++--- 3 files changed, 338 insertions(+), 75 deletions(-) diff --git a/integration/failure/compile-error/src/CompileErrorTests.scala b/integration/failure/compile-error/src/CompileErrorTests.scala index 6c718d80ae4..d5ec469d407 100644 --- a/integration/failure/compile-error/src/CompileErrorTests.scala +++ b/integration/failure/compile-error/src/CompileErrorTests.scala @@ -11,13 +11,26 @@ object CompileErrorTests extends IntegrationTestSuite { test { val res = eval("foo.scalaVersion") - assert(res.isSuccess == false) - assert(res.err.contains("""bar.mill:15:9: not found: value doesntExist""")) - assert(res.err.contains("""println(doesntExist)""")) - assert(res.err.contains("""qux.mill:4:34: type mismatch;""")) - assert(res.err.contains( - """build.mill:9:5: value noSuchMethod is not a member""" - )) + assert(!res.isSuccess) + + locally { + assert(res.err.contains("""bar.mill:15:9""")) + assert(res.err.contains("""println(doesntExist)""")) + assert(res.err.contains("""Not found: doesntExist""")) + } + + locally { + assert(res.err.contains("""qux.mill:4:34""")) + assert(res.err.contains("""myMsg.substring("0")""")) + assert(res.err.contains("""Found: ("0" : String)""")) + assert(res.err.contains("""Required: Int""")) + } + + locally { + assert(res.err.contains("""build.mill:9:5""")) + assert(res.err.contains("""foo.noSuchMethod""")) + assert(res.err.contains("""value noSuchMethod is not a member""")) + } } } } diff --git a/scalalib/src/mill/scalalib/Assembly.scala b/scalalib/src/mill/scalalib/Assembly.scala index c32b86abde4..e285b7c4476 100644 --- a/scalalib/src/mill/scalalib/Assembly.scala +++ b/scalalib/src/mill/scalalib/Assembly.scala @@ -21,15 +21,8 @@ object Assembly { private object Streamable { def bytes(is: InputStream): Array[Byte] = { - val buffer = new Array[Byte](8192) val out = new java.io.ByteArrayOutputStream - var read = 0 - while ({ - read = is.read(buffer) - read != -1 - }) { - out.write(buffer, 0, read) - } + IO.stream(is, out) out.close() out.toByteArray } diff --git a/scalalib/worker/src/mill/scalalib/worker/ZincWorkerImpl.scala b/scalalib/worker/src/mill/scalalib/worker/ZincWorkerImpl.scala index 3ed5153ea83..0864ad2eb53 100644 --- a/scalalib/worker/src/mill/scalalib/worker/ZincWorkerImpl.scala +++ b/scalalib/worker/src/mill/scalalib/worker/ZincWorkerImpl.scala @@ -1,47 +1,19 @@ package mill.scalalib.worker import mill.api.Loose.Agg -import mill.api.{ - CompileProblemReporter, - DummyOutputStream, - KeyedLockedCache, - PathRef, - Result, - internal -} +import mill.api.{CompileProblemReporter, DummyOutputStream, KeyedLockedCache, PathRef, Result, internal} import mill.scalalib.api.{CompilationResult, Versions, ZincWorkerApi, ZincWorkerUtil} -import sbt.internal.inc.{ - Analysis, - CompileFailed, - FreshCompilerCache, - ManagedLoggedReporter, - MappedFileConverter, - ScalaInstance, - Stamps, - ZincUtil, - javac -} +import sbt.internal.inc.{Analysis, CompileFailed, FreshCompilerCache, ManagedLoggedReporter, MappedFileConverter, ScalaInstance, Stamps, ZincUtil, javac} import sbt.internal.inc.classpath.ClasspathUtil import sbt.internal.inc.consistent.ConsistentFileAnalysisStore import sbt.internal.util.{ConsoleAppender, ConsoleOut} import sbt.mill.SbtLoggerUtils import xsbti.compile.analysis.ReadWriteMappers -import xsbti.compile.{ - AnalysisContents, - AnalysisStore, - AuxiliaryClassFileExtension, - ClasspathOptions, - CompileAnalysis, - CompileOrder, - Compilers, - IncOptions, - JavaTools, - MiniSetup, - PreviousResult -} +import xsbti.compile.{AnalysisContents, AnalysisStore, AuxiliaryClassFileExtension, ClasspathOptions, CompileAnalysis, CompileOrder, Compilers, IncOptions, JavaTools, MiniSetup, PreviousResult} import xsbti.{PathBasedFile, VirtualFile} import java.io.{File, PrintWriter} +import java.nio.charset.StandardCharsets import java.util.Optional import scala.annotation.tailrec import scala.collection.mutable @@ -494,8 +466,14 @@ class ZincWorkerImpl( auxiliaryClassFileExtensions: Seq[String], zincCache: os.SubPath = os.sub / "zinc" )(implicit ctx: ZincWorkerApi.Ctx): Result[CompilationResult] = { + import ZincWorkerImpl.{ForwardingReporter, TransformingReporter, PositionMapper} + os.makeDir.all(ctx.dest) + val classesDir = + if (compileToJar) ctx.dest / "classes.jar" + else ctx.dest / "classes" + reporter.foreach(_.start()) val consoleAppender = ConsoleAppender( @@ -508,31 +486,14 @@ class ZincWorkerImpl( val loggerId = Thread.currentThread().getId.toString val logger = SbtLoggerUtils.createLogger(loggerId, consoleAppender, zincLogLevel) - val newReporter = reporter match { - case None => new ManagedLoggedReporter(10, logger) with RecordingReporter + def mkNewReporter(mapper: (xsbti.Position => xsbti.Position) | Null) = reporter match { + case None => + new ManagedLoggedReporter(10, logger) with RecordingReporter with TransformingReporter(mapper) {} case Some(forwarder) => - new ManagedLoggedReporter(10, logger) with RecordingReporter { - - override def logError(problem: xsbti.Problem): Unit = { - forwarder.logError(new ZincProblem(problem)) - super.logError(problem) - } - - override def logWarning(problem: xsbti.Problem): Unit = { - forwarder.logWarning(new ZincProblem(problem)) - super.logWarning(problem) - } - - override def logInfo(problem: xsbti.Problem): Unit = { - forwarder.logInfo(new ZincProblem(problem)) - super.logInfo(problem) - } - - override def printSummary(): Unit = { - forwarder.printSummary() - super.printSummary() - } - } + new ManagedLoggedReporter(10, logger) + with ForwardingReporter(forwarder) + with RecordingReporter + with TransformingReporter(mapper) {} } val analysisMap0 = upstreamCompileOutput.map(c => c.classes.path -> c.analysisFile).toMap @@ -549,10 +510,6 @@ class ZincWorkerImpl( val lookup = MockedLookup(analysisMap) - val classesDir = - if (compileToJar) ctx.dest / "classes.jar" - else ctx.dest / "classes" - val store = fileAnalysisStore(ctx.dest / zincCache) // Fix jdk classes marked as binary dependencies, see https://github.com/com-lihaoyi/mill/pull/1904 @@ -568,6 +525,10 @@ class ZincWorkerImpl( auxiliaryClassFileExtensions.map(new AuxiliaryClassFileExtension(_)).toArray ) + val newReporter = mkNewReporter( + PositionMapper.create(sources.iterator.zip(virtualSources.iterator)) + ) + val inputs = ic.inputs( classpath = classpath, sources = virtualSources, @@ -647,6 +608,302 @@ class ZincWorkerImpl( } object ZincWorkerImpl { + + /** + * TODO: copied from mill.scalalib.Assembly + */ + private object Streamable { + def bytes(is: java.io.InputStream): Array[Byte] = { + val out = new java.io.ByteArrayOutputStream + mill.api.IO.stream(is, out) + out.close() + out.toByteArray + } + } + + private def intValue(oi: java.util.Optional[Integer], default: Int): Int = { + if oi.isPresent then oi.get().intValue() + else default + } + + private object PositionMapper { + import sbt.util.InterfaceUtil + + private val userCodeStartMarker = "//MILL_USER_CODE_START_MARKER" + + /** Transforms positions of problems if coming from a build file. */ + private def lookup(buildSources: Map[String, xsbti.Position => xsbti.Position])( + oldPos: xsbti.Position): xsbti.Position = { + val src = oldPos.sourcePath() + if src.isPresent then { + buildSources.get(src.get()) match { + case Some(f) => f(oldPos) + case _ => oldPos + } + } else { + oldPos + } + } + + def create(sources: Iterator[(os.Path, VirtualFile)]): (xsbti.Position => xsbti.Position) | Null = { + val buildSources0 = { + def isBuild(p: os.Path) = + mill.main.client.CodeGenConstants.buildFileExtensions.contains(p.ext) + + sources.collect({ case (p, vf) if isBuild(p) => + val str = new String(Streamable.bytes(vf.input()), StandardCharsets.UTF_8) + + val lines = str.linesWithSeparators.toVector + val adjustedFile = lines + .collectFirst { case s"//MILL_ORIGINAL_FILE_PATH=$rest" => rest.trim } + .getOrElse(sys.error(p.toString)) + + vf.id() -> remap(lines, adjustedFile) + }) + } + + if buildSources0.nonEmpty then lookup(buildSources0.toMap) else null + } + + private def remap(lines: Vector[String], adjustedFile: String): xsbti.Position => xsbti.Position = { + val markerLine = lines.indexWhere(_.startsWith(userCodeStartMarker)) + + val topWrapperLen = lines.take(markerLine + 1).map(_.length).sum + + val originPath = Some(adjustedFile) + val originFile = Some(java.nio.file.Paths.get(adjustedFile).toFile) + + def userCode(offset: java.util.Optional[Integer]): Boolean = + intValue(offset, -1) > topWrapperLen + + def inner(pos0: xsbti.Position): xsbti.Position = { + if userCode(pos0.startOffset()) || userCode(pos0.offset()) then { + val IArray(line, offset, startOffset, endOffset, startLine, endLine) = + IArray(pos0.line(), pos0.offset(), pos0.startOffset(), pos0.endOffset(), pos0.startLine(), pos0.endLine()) + .map(intValue(_, 1) - 1) + + InterfaceUtil.position( + line0 = Some(line - markerLine), + content = pos0.lineContent(), + offset0 = Some(offset - topWrapperLen), + pointer0 = InterfaceUtil.jo2o(pos0.pointer()), + pointerSpace0 = InterfaceUtil.jo2o(pos0.pointerSpace()), + sourcePath0 = originPath, + sourceFile0 = originFile, + startOffset0 = Some(startOffset - topWrapperLen), + endOffset0 = Some(endOffset - topWrapperLen), + startLine0 = Some(startLine - markerLine), + startColumn0 = InterfaceUtil.jo2o(pos0.startColumn()), + endLine0 = Some(endLine - markerLine), + endColumn0 = InterfaceUtil.jo2o(pos0.endColumn()), + ) + } + else { + pos0 + } + } + + inner + } + } + + private trait ForwardingReporter(forwarder: CompileProblemReporter) extends ManagedLoggedReporter { + override def logError(problem: xsbti.Problem): Unit = { + forwarder.logError(new ZincProblem(problem)) + super.logError(problem) + } + + override def logWarning(problem: xsbti.Problem): Unit = { + forwarder.logWarning(new ZincProblem(problem)) + super.logWarning(problem) + } + + override def logInfo(problem: xsbti.Problem): Unit = { + forwarder.logInfo(new ZincProblem(problem)) + super.logInfo(problem) + } + + override def printSummary(): Unit = { + forwarder.printSummary() + super.printSummary() + } + } + + private trait TransformingReporter( + optPositionMapper: (xsbti.Position => xsbti.Position) | Null) extends xsbti.Reporter { + + // Overriding this is necessary because for some reason the LoggedReporter doesn't transform positions + // of Actions and DiagnosticRelatedInformation + abstract override def log(problem0: xsbti.Problem): Unit = { + val localMapper = optPositionMapper + val problem = { + if localMapper == null then problem0 + else TransformingReporter.transformProblem(problem0, localMapper) + } + super.log(problem) + } + } + + private object TransformingReporter { + + import scala.jdk.CollectionConverters.given + import sbt.util.InterfaceUtil + + /** implements a transformation that returns the same object if the mapper has no effect. */ + private def transformProblem(problem0: xsbti.Problem, mapper: xsbti.Position => xsbti.Position): xsbti.Problem = { + val pos0 = problem0.position() + val related0 = problem0.diagnosticRelatedInformation() + val actions0 = problem0.actions() + val pos = mapper(pos0) + val related = transformRelateds(related0, mapper) + val actions = transformActions(actions0, mapper) + val posIsNew = pos ne pos0 + if posIsNew || (related ne related0) || (actions ne actions0) then + val rendered = { + // if we transformed the position, then we must re-render the message + if posIsNew then Some(dottyStyleMessage(problem0, pos)) + else InterfaceUtil.jo2o(problem0.rendered()) + } + InterfaceUtil.problem( + cat = problem0.category(), + pos = pos, + msg = problem0.message(), + sev = problem0.severity(), + rendered = rendered, + diagnosticCode = InterfaceUtil.jo2o(problem0.diagnosticCode()), + diagnosticRelatedInformation = anyToList(related), + actions = anyToList(actions), + ) + else + problem0 + } + + private type JOrSList[T] = java.util.List[T] | List[T] + + private def anyToList[T](ts: JOrSList[T]): List[T] = ts match { + case ts: List[T] => ts + case ts: java.util.List[T] => ts.asScala.toList + } + + /** Render the message in the style of dotty */ + private def dottyStyleMessage(problem0: xsbti.Problem, pos: xsbti.Position): String = { + val base = problem0.message().trim + + val normCode = { + problem0.diagnosticCode().map({ inner => + val prefix = s"[E${inner.code()}] " + inner.explanation().map(e => + s"$prefix$e: " + ).orElse(prefix) + }).orElse("") + } + + val optPath = InterfaceUtil.jo2o(pos.sourcePath()).map { path => + val line0 = intValue(pos.line(), -1) + val pointer0 = intValue(pos.pointer(), -1) + if line0 >= 0 && pointer0 >= 0 then + s"$path:$line0:${pointer0 + 1}" + else + path + } + + val normHeader = optPath.map(path => + s"-- $normCode$path\n" + ).getOrElse("") + + val optSnippet = { + val snip = pos.lineContent() + val space = pos.pointerSpace().orElse("") + val pointer = intValue(pos.pointer(), -99) + val endCol = intValue(pos.endColumn(), pointer + 1) + if snip.nonEmpty && space.nonEmpty && pointer >= 0 && endCol >= 0 then + Some( + s"""$snip + |$space${"^" * math.min(endCol - pointer, snip.length - space.length)}""".stripMargin + ) + else + None + } + + val content = optSnippet.match { + case Some(snippet) => + val initial = { + s"""$snippet + |$base + |""".stripMargin + } + val snippetLine = intValue(pos.line(), -1) + if snippetLine >= 0 then { + // add margin with line number + val lines = initial.linesWithSeparators.toVector + val pre = snippetLine.toString + val rest0 = " " * pre.length + val rest = pre +: Vector.fill(lines.size - 1)(rest0) + rest.lazyZip(lines).map((pre, line) => s"$pre │ $line").mkString + } else { + initial + } + case None => + base + } + + normHeader + content + } + + /** Implements a transformation that returns the same list if the mapper has no effect */ + private def transformActions( + actions0: java.util.List[xsbti.Action], + mapper: xsbti.Position => xsbti.Position): JOrSList[xsbti.Action] = { + if actions0.iterator().asScala.exists(a => + a.edit().changes().iterator().asScala.exists(e => + mapper(e.position()) ne e.position() + ) + ) then { + actions0.iterator().asScala.map(transformAction(_, mapper)).toList + } else { + actions0 + } + } + + /** Implements a transformation that returns the same list if the mapper has no effect */ + private def transformRelateds( + related0: java.util.List[xsbti.DiagnosticRelatedInformation], + mapper: xsbti.Position => xsbti.Position): JOrSList[xsbti.DiagnosticRelatedInformation] = { + + if related0.iterator().asScala.exists(r => mapper(r.position()) ne r.position()) then + related0.iterator().asScala.map(transformRelated(_, mapper)).toList + else + related0 + } + + private def transformRelated( + related0: xsbti.DiagnosticRelatedInformation, + mapper: xsbti.Position => xsbti.Position): xsbti.DiagnosticRelatedInformation = { + InterfaceUtil.diagnosticRelatedInformation(mapper(related0.position()), related0.message()) + } + + private def transformAction(action0: xsbti.Action, mapper: xsbti.Position => xsbti.Position): xsbti.Action = { + InterfaceUtil.action( + title = action0.title(), + description = InterfaceUtil.jo2o(action0.description()), + edit = transformEdit(action0.edit(), mapper) + ) + } + + private def transformEdit(edit0: xsbti.WorkspaceEdit, mapper: xsbti.Position => xsbti.Position): xsbti.WorkspaceEdit = { + InterfaceUtil.workspaceEdit( + edit0.changes().iterator().asScala.map(transformTEdit(_, mapper)).toList + ) + } + + private def transformTEdit(edit0: xsbti.TextEdit, mapper: xsbti.Position => xsbti.Position): xsbti.TextEdit = { + InterfaceUtil.textEdit( + position = mapper(edit0.position()), + newText = edit0.newText() + ) + } + } + // copied from ModuleUtils private def recursive[T <: String](start: T, deps: T => Seq[T]): Seq[T] = {