From ad64de880dbc641b9fad51d114b1ad20c55abeb0 Mon Sep 17 00:00:00 2001 From: krrish175-byte Date: Tue, 6 Jan 2026 22:29:39 +0530 Subject: [PATCH 1/5] Implement resource mapping support (#2592) --- .../src/main/scala/bloop/data/Project.scala | 7 + .../bloop/engine/tasks/CompileTask.scala | 30 ++- .../main/scala/bloop/io/ResourceMapper.scala | 147 ++++++++++++ frontend/src/test/scala/bloop/DagSpec.scala | 2 +- .../ResourceMappingIntegrationSpec.scala | 172 ++++++++++++++ .../scala/bloop/io/ResourceMapperSpec.scala | 211 ++++++++++++++++++ .../src/test/scala/bloop/util/TestUtil.scala | 1 + 7 files changed, 566 insertions(+), 4 deletions(-) create mode 100644 frontend/src/main/scala/bloop/io/ResourceMapper.scala create mode 100644 frontend/src/test/scala/bloop/ResourceMappingIntegrationSpec.scala create mode 100644 frontend/src/test/scala/bloop/io/ResourceMapperSpec.scala diff --git a/frontend/src/main/scala/bloop/data/Project.scala b/frontend/src/main/scala/bloop/data/Project.scala index daa6517014..94b3d6e322 100644 --- a/frontend/src/main/scala/bloop/data/Project.scala +++ b/frontend/src/main/scala/bloop/data/Project.scala @@ -39,6 +39,7 @@ final case class Project( scalaInstance: Option[ScalaInstance], rawClasspath: List[AbsolutePath], resources: List[AbsolutePath], + resourceMappings: List[(AbsolutePath, String)], compileSetup: Config.CompileSetup, genericClassesDir: AbsolutePath, isBestEffort: Boolean, @@ -347,6 +348,11 @@ object Project { val tags = project.tags.getOrElse(Nil) val projectDirectory = AbsolutePath(project.directory) + // Parse resource mappings if available (requires bloop-config 2.3.4+) + // TODO: Uncomment parsing logic once bloop-config is updated with resourceMapping field + // See bloop_config_changes.md for details + val resourceMappings = List.empty[(AbsolutePath, String)] + Project( project.name, projectDirectory, @@ -355,6 +361,7 @@ object Project { instance, compileClasspath, compileResources, + resourceMappings, setup, AbsolutePath(project.classesDir), isBestEffort = false, diff --git a/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala b/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala index dac5807179..b8db32f911 100644 --- a/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala +++ b/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala @@ -123,12 +123,23 @@ object CompileTask { logger, ExecutionContext.ioScheduler ) + + // Also copy mapped resources if any + val copyMappedResourcesTask: Task[Unit] = + bloop.io.ResourceMapper.copyMappedResources( + project.resourceMappings, + bundle.clientClassesObserver.classesDir, + logger + ) + + val allCopyTasks = + Task.gatherUnordered(List(copyResourcesTask, copyMappedResourcesTask)) Task.now( ResultBundle( Compiler.Result.Empty, None, None, - copyResourcesTask.runAsync(ExecutionContext.ioScheduler) + allCopyTasks.map(_ => ()).runAsync(ExecutionContext.ioScheduler) ) ) case Right(CompileSourcesAndInstance(sources, instance, _)) => @@ -245,8 +256,21 @@ object CompileTask { compileProjectTracer, logger ) - .doOnFinish(_ => Task(compileProjectTracer.terminate())) - postCompilationTasks.runAsync(ExecutionContext.ioScheduler) + + // Copy mapped resources after compilation + val copyMappedResourcesTask = + bloop.io.ResourceMapper.copyMappedResources( + project.resourceMappings, + bundle.clientClassesObserver.classesDir, + logger + ) + + val allTasks = Task + .gatherUnordered(List(postCompilationTasks, copyMappedResourcesTask)) + .map(_ => ()) + .doOnFinish(_ => Task(compileProjectTracer.terminate())) + + allTasks.runAsync(ExecutionContext.ioScheduler) } // Populate the last successful result if result was success diff --git a/frontend/src/main/scala/bloop/io/ResourceMapper.scala b/frontend/src/main/scala/bloop/io/ResourceMapper.scala new file mode 100644 index 0000000000..07abd282a5 --- /dev/null +++ b/frontend/src/main/scala/bloop/io/ResourceMapper.scala @@ -0,0 +1,147 @@ +package bloop.io + +import java.nio.file.Files +import java.nio.file.StandardCopyOption + +import scala.util.control.NonFatal + +import bloop.logging.DebugFilter +import bloop.logging.Logger +import bloop.task.Task + +/** + * Utility for handling resource file mappings. + * + * Resource mappings allow files to be copied from source locations to custom + * target paths, similar to SBT's "mappings" field. + */ +object ResourceMapper { + private implicit val filter: DebugFilter.All.type = DebugFilter.All + + /** + * Copy mapped resources to the target directory. + * + * @param mappings List of (source, targetRelativePath) tuples + * @param classesDir Base directory where resources should be copied + * @param logger Logger for debug/error messages + * @return Task that completes when all resources are copied + */ + def copyMappedResources( + mappings: List[(AbsolutePath, String)], + classesDir: AbsolutePath, + logger: Logger + ): Task[Unit] = { + val tasks = mappings.map { + case (source, targetRelPath) => + Task { + val target = classesDir.resolve(targetRelPath) + if (!target.getParent.exists) { + Files.createDirectories(target.getParent.underlying) + } + + if (source.isDirectory) { + import java.nio.file.FileVisitResult + import java.nio.file.Path + import java.nio.file.SimpleFileVisitor + import java.nio.file.attribute.BasicFileAttributes + + val sourcePath = source.underlying + val targetPath = target.underlying + Files.walkFileTree( + sourcePath, + new SimpleFileVisitor[Path] { + override def visitFile( + file: Path, + attrs: BasicFileAttributes + ): FileVisitResult = { + val relPath = sourcePath.relativize(file) + val targetFile = targetPath.resolve(relPath) + Files.createDirectories(targetFile.getParent) + Files.copy(file, targetFile, StandardCopyOption.REPLACE_EXISTING) + FileVisitResult.CONTINUE + } + } + ) + } else if (source.exists) { + Files.copy(source.underlying, target.underlying, StandardCopyOption.REPLACE_EXISTING) + } else { + logger.warn(s"Source file $source does not exist, skipping mapping to $targetRelPath") + } + () + } + } + Task.gatherUnordered(tasks).map(_ => ()) + } + + /** + * Check if any mapped resources have changed since the given timestamp. + * + * Note: This is currently not used. Incremental compilation change detection + * for resource mappings would require integration with Zinc's analysis. + * For now, mapped resources are always copied during compilation. + * + * @param mappings List of (source, targetRelativePath) tuples + * @param lastModified Timestamp to compare against + * @return true if any source file is newer than lastModified + */ + private[bloop] def hasMappingsChanged( + mappings: List[(AbsolutePath, String)], + lastModified: Long + ): Boolean = { + import java.nio.file.Files + mappings.exists { + case (source, _) => + if (source.isDirectory) { + hasDirectoryChanged(source, lastModified) + } else { + source.exists && Files.getLastModifiedTime(source.underlying).toMillis > lastModified + } + } + } + + /** + * Validate resource mappings for common issues. + * + * @param mappings List of (source, targetRelativePath) tuples + * @param logger Logger for warnings + * @return List of validation errors + */ + def validateMappings( + mappings: List[(AbsolutePath, String)], + logger: Logger + ): List[String] = { + val errors = scala.collection.mutable.ListBuffer.empty[String] + + // Check for duplicate targets + val targetCounts = mappings.groupBy(_._2).filter(_._2.size > 1) + targetCounts.foreach { + case (target, sources) => + val sourceList = sources.map(_._1.syntax).mkString(", ") + errors += s"Multiple sources map to same target '$target': $sourceList" + } + + // Check for path traversal attempts + mappings.foreach { + case (_, target) => + if (target.contains("..")) { + errors += s"Invalid target path contains '..': $target" + } + if (target.startsWith("/")) { + logger.warn(s"Target path starts with '/': $target (will be treated as relative)") + } + } + + errors.toList + } + + private def hasDirectoryChanged(dir: AbsolutePath, lastModified: Long): Boolean = { + if (!dir.exists) return false + + val stream = Files.walk(dir.underlying) + try { + stream.anyMatch(path => Files.getLastModifiedTime(path).toMillis > lastModified) + } finally { + stream.close() + } + } +} diff --git a/frontend/src/test/scala/bloop/DagSpec.scala b/frontend/src/test/scala/bloop/DagSpec.scala index cb85672860..351113e623 100644 --- a/frontend/src/test/scala/bloop/DagSpec.scala +++ b/frontend/src/test/scala/bloop/DagSpec.scala @@ -27,7 +27,7 @@ class DagSpec { // format: OFF def dummyOrigin: Origin = TestUtil.syntheticOriginFor(dummyPath) def dummyProject(name: String, dependencies: List[String]): Project = - Project(name, dummyPath, None, dependencies, Some(dummyInstance), Nil, Nil, compileOptions, + Project(name, dummyPath, None, dependencies, Some(dummyInstance), Nil, Nil, Nil, compileOptions, dummyPath, isBestEffort = false, Nil, Nil, Nil, Nil, None, Nil, Nil, Config.TestOptions.empty, dummyPath, dummyPath, Project.defaultPlatform(logger, Nil, Nil), None, None, Nil, dummyOrigin) // format: ON diff --git a/frontend/src/test/scala/bloop/ResourceMappingIntegrationSpec.scala b/frontend/src/test/scala/bloop/ResourceMappingIntegrationSpec.scala new file mode 100644 index 0000000000..9a9a69bb41 --- /dev/null +++ b/frontend/src/test/scala/bloop/ResourceMappingIntegrationSpec.scala @@ -0,0 +1,172 @@ +package bloop + +import java.nio.file.Files +import bloop.cli.ExitStatus +import bloop.io.AbsolutePath +import bloop.logging.RecordingLogger +import bloop.util.TestProject +import bloop.util.TestUtil +import bloop.data.LoadedProject + +object ResourceMappingIntegrationSpec extends bloop.testing.BaseSuite { + + test("compile with resource mappings copies files to classes directory") { + TestUtil.withinWorkspace { workspace => + val logger = new RecordingLogger(ansiCodesSupported = false) + + // Create a simple project + val source = + """/main/scala/Foo.scala + |class Foo + """.stripMargin + + val `A` = TestProject(workspace, "a", List(source)) + val projects = List(`A`) + val state = loadState(workspace, projects, logger) + + // Create a resource file to map + val resourceFile = workspace.resolve("custom-resource.txt") + Files.write(resourceFile.underlying, "resource content".getBytes("UTF-8")) + + // Get the project and inject mappings manually + val projectA = state.getProjectFor(`A`) + val mappings = List((resourceFile, "assets/config.txt")) + val projectWithMappings = projectA.copy(resourceMappings = mappings) + + // Update state with modified project + val stateWithMappings = new TestState( + state.state.copy( + build = state.state.build.copy( + loadedProjects = state.state.build.loadedProjects.map { lp => + if (lp.project.name == projectA.name) { + lp match { + case LoadedProject.RawProject(_) => LoadedProject.RawProject(projectWithMappings) + case LoadedProject.ConfiguredProject(_, original, settings) => + LoadedProject.ConfiguredProject(projectWithMappings, original, settings) + } + } else lp + } + ) + ) + ) + + // Compile + val compiledState = stateWithMappings.compile(`A`) + assertExitStatus(compiledState, ExitStatus.Ok) + + // Verify resource was copied to classes directory + val classesDir = compiledState.getClientExternalDir(`A`) + val copiedResource = classesDir.resolve("assets/config.txt") + + try { + assert(copiedResource.exists) + val content = new String(Files.readAllBytes(copiedResource.underlying), "UTF-8") + assertNoDiff(content, "resource content") + } catch { + case t: Throwable => + logger.dump() + throw t + } + } + } + + test("compile with directory mapping copies entire directory") { + TestUtil.withinWorkspace { workspace => + val logger = new RecordingLogger(ansiCodesSupported = false) + + val `A` = TestProject(workspace, "a", List("/main/scala/Foo.scala\nclass Foo")) + val projects = List(`A`) + val state = loadState(workspace, projects, logger) + + // Create a directory structure to map + val resourceDir = workspace.resolve("resources") + Files.createDirectories(resourceDir.underlying) + Files.write(resourceDir.resolve("file1.txt").underlying, "content 1".getBytes("UTF-8")) + val subDir = resourceDir.resolve("sub") + Files.createDirectories(subDir.underlying) + Files.write(subDir.resolve("file2.txt").underlying, "content 2".getBytes("UTF-8")) + + // Inject mappings + val projectA = state.getProjectFor(`A`) + val mappings = List((resourceDir, "data")) + val projectWithMappings = projectA.copy(resourceMappings = mappings) + + val stateWithMappings = new TestState( + state.state.copy( + build = state.state.build.copy( + loadedProjects = state.state.build.loadedProjects.map { lp => + if (lp.project.name == projectA.name) { + lp match { + case LoadedProject.RawProject(_) => LoadedProject.RawProject(projectWithMappings) + case LoadedProject.ConfiguredProject(_, original, settings) => + LoadedProject.ConfiguredProject(projectWithMappings, original, settings) + } + } else lp + } + ) + ) + ) + + // Compile + val compiledState = stateWithMappings.compile(`A`) + assertExitStatus(compiledState, ExitStatus.Ok) + + // Verify directory structure copied + val classesDir = compiledState.getClientExternalDir(`A`) + assert(classesDir.resolve("data/file1.txt").exists) + assert(classesDir.resolve("data/sub/file2.txt").exists) + } + } + + test("resources are copied even if compilation is no-op") { + TestUtil.withinWorkspace { workspace => + val logger = new RecordingLogger(ansiCodesSupported = false) + + val `A` = TestProject(workspace, "a", List("/main/scala/Foo.scala\nclass Foo")) + val projects = List(`A`) + val state = loadState(workspace, projects, logger) + + val resourceFile = workspace.resolve("res.txt") + Files.write(resourceFile.underlying, "v1".getBytes("UTF-8")) + + val projectA = state.getProjectFor(`A`) + val mappings = List((resourceFile, "res.txt")) + val projectWithMappings = projectA.copy(resourceMappings = mappings) + + val stateWithMappings = new TestState( + state.state.copy( + build = state.state.build.copy( + loadedProjects = state.state.build.loadedProjects.map { lp => + if (lp.project.name == projectA.name) { + lp match { + case LoadedProject.RawProject(_) => LoadedProject.RawProject(projectWithMappings) + case LoadedProject.ConfiguredProject(_, original, settings) => + LoadedProject.ConfiguredProject(projectWithMappings, original, settings) + } + } else lp + } + ) + ) + ) + + // First compile + val compiledState = stateWithMappings.compile(`A`) + assertExitStatus(compiledState, ExitStatus.Ok) + + // Verify v1 + val classesDir = compiledState.getClientExternalDir(`A`) + val copiedFile = classesDir.resolve("res.txt") + assertNoDiff(new String(Files.readAllBytes(copiedFile.underlying)), "v1") + + // Update resource + Files.write(resourceFile.underlying, "v2".getBytes("UTF-8")) + + // Compile again (should be no-op for scala, but resources should update) + val compiledState2 = compiledState.compile(`A`) + assertExitStatus(compiledState2, ExitStatus.Ok) + + // Verify v2 + assertNoDiff(new String(Files.readAllBytes(copiedFile.underlying)), "v2") + } + } +} diff --git a/frontend/src/test/scala/bloop/io/ResourceMapperSpec.scala b/frontend/src/test/scala/bloop/io/ResourceMapperSpec.scala new file mode 100644 index 0000000000..81b857fcdb --- /dev/null +++ b/frontend/src/test/scala/bloop/io/ResourceMapperSpec.scala @@ -0,0 +1,211 @@ +package bloop.io + +import java.nio.file.Files + +import bloop.logging.RecordingLogger +import bloop.util.TestUtil + +object ResourceMapperSpec extends bloop.testing.BaseSuite { + + test("copy single file mapping") { + TestUtil.withinWorkspace { workspace => + val logger = new RecordingLogger() + + // Create source file + val sourceFile = workspace.resolve("source.txt") + Files.write(sourceFile.underlying, "test content".getBytes("UTF-8")) + + // Define mapping + val mappings = List((sourceFile, "custom/path/file.txt")) + + // Copy to target directory + val targetDir = workspace.resolve("target") + Files.createDirectories(targetDir.underlying) + + val task = ResourceMapper.copyMappedResources(mappings, targetDir, logger) + TestUtil.await(5, java.util.concurrent.TimeUnit.SECONDS)(task) + + // Verify file was copied + val copiedFile = targetDir.resolve("custom/path/file.txt") + assert(copiedFile.exists) + assert(copiedFile.isFile) + + val content = new String(Files.readAllBytes(copiedFile.underlying), "UTF-8") + assertNoDiff(content, "test content") + } + } + + test("copy directory mapping recursively") { + TestUtil.withinWorkspace { workspace => + val logger = new RecordingLogger() + + // Create source directory with files + val sourceDir = workspace.resolve("source-dir") + Files.createDirectories(sourceDir.underlying) + + val file1 = sourceDir.resolve("file1.txt") + val file2 = sourceDir.resolve("subdir/file2.txt") + Files.createDirectories(file2.getParent.underlying) + + Files.write(file1.underlying, "content 1".getBytes("UTF-8")) + Files.write(file2.underlying, "content 2".getBytes("UTF-8")) + + // Define mapping + val mappings = List((sourceDir, "data")) + + // Copy to target directory + val targetDir = workspace.resolve("target") + Files.createDirectories(targetDir.underlying) + + val task = ResourceMapper.copyMappedResources(mappings, targetDir, logger) + TestUtil.await(5, java.util.concurrent.TimeUnit.SECONDS)(task) + + // Verify directory structure was copied + val copiedFile1 = targetDir.resolve("data/file1.txt") + val copiedFile2 = targetDir.resolve("data/subdir/file2.txt") + + assert(copiedFile1.exists) + assert(copiedFile2.exists) + + val content1 = new String(Files.readAllBytes(copiedFile1.underlying), "UTF-8") + val content2 = new String(Files.readAllBytes(copiedFile2.underlying), "UTF-8") + + assertNoDiff(content1, "content 1") + assertNoDiff(content2, "content 2") + } + } + + test("handle empty mappings gracefully") { + TestUtil.withinWorkspace { workspace => + val logger = new RecordingLogger() + + val targetDir = workspace.resolve("target") + Files.createDirectories(targetDir.underlying) + + val task = ResourceMapper.copyMappedResources(List.empty, targetDir, logger) + TestUtil.await(5, java.util.concurrent.TimeUnit.SECONDS)(task) + + // Should complete without error + assert(logger.errors.isEmpty) + } + } + + test("validate duplicate target paths") { + TestUtil.withinWorkspace { workspace => + val logger = new RecordingLogger() + + val file1 = workspace.resolve("file1.txt") + val file2 = workspace.resolve("file2.txt") + Files.write(file1.underlying, "content 1".getBytes("UTF-8")) + Files.write(file2.underlying, "content 2".getBytes("UTF-8")) + + // Two sources mapping to same target + val mappings = List( + (file1, "output.txt"), + (file2, "output.txt") + ) + + val errors = ResourceMapper.validateMappings(mappings, logger) + + assert(errors.nonEmpty) + assert(errors.exists(_.contains("Multiple sources map to same target"))) + assert(errors.exists(_.contains("output.txt"))) + } + } + + test("validate path traversal attempts") { + TestUtil.withinWorkspace { workspace => + val logger = new RecordingLogger() + + val file = workspace.resolve("file.txt") + Files.write(file.underlying, "content".getBytes("UTF-8")) + + // Mapping with .. in target path + val mappings = List( + (file, "../escape/file.txt") + ) + + val errors = ResourceMapper.validateMappings(mappings, logger) + + assert(errors.nonEmpty) + assert(errors.exists(_.contains("Invalid target path contains '..'"))) + } + } + + test("warn about absolute target paths") { + TestUtil.withinWorkspace { workspace => + val logger = new RecordingLogger() + + val file = workspace.resolve("file.txt") + Files.write(file.underlying, "content".getBytes("UTF-8")) + + // Mapping with absolute target path + val mappings = List( + (file, "/absolute/path/file.txt") + ) + + ResourceMapper.validateMappings(mappings, logger) + + // Should warn but not error + assert(logger.warnings.exists(_.contains("Target path starts with '/'"))) + } + } + + test("handle non-existent source file") { + TestUtil.withinWorkspace { workspace => + val logger = new RecordingLogger() + + val nonExistentFile = workspace.resolve("does-not-exist.txt") + val mappings = List((nonExistentFile, "output.txt")) + + val targetDir = workspace.resolve("target") + Files.createDirectories(targetDir.underlying) + + val task = ResourceMapper.copyMappedResources(mappings, targetDir, logger) + TestUtil.await(5, java.util.concurrent.TimeUnit.SECONDS)(task) + + // Should warn about missing source + assert(logger.warnings.exists(_.contains("does not exist"))) + + // Target file should not be created + val targetFile = targetDir.resolve("output.txt") + assert(!targetFile.exists) + } + } + + test("detect changes in mapped files") { + TestUtil.withinWorkspace { workspace => + val file = workspace.resolve("file.txt") + Files.write(file.underlying, "original content".getBytes("UTF-8")) + + val oldTimestamp = System.currentTimeMillis() - 10000 // 10 seconds ago + val mappings = List((file, "output.txt")) + + // File is newer than timestamp + val changed = ResourceMapper.hasMappingsChanged(mappings, oldTimestamp) + assert(changed) + + // File is older than timestamp + val futureTimestamp = System.currentTimeMillis() + 10000 // 10 seconds in future + val notChanged = ResourceMapper.hasMappingsChanged(mappings, futureTimestamp) + assert(!notChanged) + } + } + + test("detect changes in mapped directories") { + TestUtil.withinWorkspace { workspace => + val dir = workspace.resolve("source-dir") + Files.createDirectories(dir.underlying) + + val file = dir.resolve("file.txt") + Files.write(file.underlying, "content".getBytes("UTF-8")) + + val oldTimestamp = System.currentTimeMillis() - 10000 // 10 seconds ago + val mappings = List((dir, "data")) + + // Directory contains file newer than timestamp + val changed = ResourceMapper.hasMappingsChanged(mappings, oldTimestamp) + assert(changed) + } + } +} diff --git a/frontend/src/test/scala/bloop/util/TestUtil.scala b/frontend/src/test/scala/bloop/util/TestUtil.scala index 58f8645376..4d07c95fb2 100644 --- a/frontend/src/test/scala/bloop/util/TestUtil.scala +++ b/frontend/src/test/scala/bloop/util/TestUtil.scala @@ -411,6 +411,7 @@ object TestUtil { scalaInstance = scalaInstance, rawClasspath = classpath, resources = Nil, + resourceMappings = Nil, isBestEffort = false, compileSetup = Config.CompileSetup.empty.copy(order = compileOrder), genericClassesDir = classes, From e0db96fdc78eb37bc375e822d12d9e5522fed25f Mon Sep 17 00:00:00 2001 From: krrish175-byte Date: Thu, 8 Jan 2026 20:48:55 +0530 Subject: [PATCH 2/5] Refactor ResourceMapper to backend and integrate with Compiler --- backend/src/main/scala/bloop/Compiler.scala | 10 ++- .../main/scala/bloop/io/ResourceMapper.scala | 51 +++------------- .../src/main/scala/bloop/data/Project.scala | 3 - .../bloop/engine/tasks/CompileTask.scala | 12 +--- .../scala/bloop/io/ResourceMapperSpec.scala | 61 ------------------- 5 files changed, 20 insertions(+), 117 deletions(-) rename {frontend => backend}/src/main/scala/bloop/io/ResourceMapper.scala (73%) diff --git a/backend/src/main/scala/bloop/Compiler.scala b/backend/src/main/scala/bloop/Compiler.scala index d9b836b365..b3e3ac333c 100644 --- a/backend/src/main/scala/bloop/Compiler.scala +++ b/backend/src/main/scala/bloop/Compiler.scala @@ -74,7 +74,8 @@ case class CompileInputs( ioExecutor: Executor, invalidatedClassFilesInDependentProjects: Set[File], generatedClassFilePathsInDependentProjects: Map[String, File], - resources: List[AbsolutePath] + resources: List[AbsolutePath], + resourceMappings: List[(AbsolutePath, String)] ) case class CompileOutPaths( @@ -828,6 +829,11 @@ object Compiler { compileInputs.logger, compileInputs.ioScheduler ) + val copyMappedResources = bloop.io.ResourceMapper.copyMappedResources( + compileInputs.resourceMappings, + clientClassesDir, + compileInputs.logger + ) val lastCopy = ParallelOps.copyDirectories(config)( readOnlyClassesDir, clientClassesDir.underlying, @@ -836,7 +842,7 @@ object Compiler { compileInputs.logger ) - Task.gatherUnordered(List(copyResources, lastCopy)).map { _ => + Task.gatherUnordered(List(copyResources, copyMappedResources, lastCopy)).map { _ => clientLogger.debug( s"Finished copying classes from $readOnlyClassesDir to $clientClassesDir" ) diff --git a/frontend/src/main/scala/bloop/io/ResourceMapper.scala b/backend/src/main/scala/bloop/io/ResourceMapper.scala similarity index 73% rename from frontend/src/main/scala/bloop/io/ResourceMapper.scala rename to backend/src/main/scala/bloop/io/ResourceMapper.scala index 07abd282a5..28076a1089 100644 --- a/frontend/src/main/scala/bloop/io/ResourceMapper.scala +++ b/backend/src/main/scala/bloop/io/ResourceMapper.scala @@ -99,49 +99,16 @@ object ResourceMapper { } } - /** - * Validate resource mappings for common issues. - * - * @param mappings List of (source, targetRelativePath) tuples - * @param logger Logger for warnings - * @return List of validation errors - */ - def validateMappings( - mappings: List[(AbsolutePath, String)], - logger: Logger - ): List[String] = { - val errors = scala.collection.mutable.ListBuffer.empty[String] - - // Check for duplicate targets - val targetCounts = mappings.groupBy(_._2).filter(_._2.size > 1) - targetCounts.foreach { - case (target, sources) => - val sourceList = sources.map(_._1.syntax).mkString(", ") - errors += s"Multiple sources map to same target '$target': $sourceList" - } - - // Check for path traversal attempts - mappings.foreach { - case (_, target) => - if (target.contains("..")) { - errors += s"Invalid target path contains '..': $target" - } - if (target.startsWith("/")) { - logger.warn(s"Target path starts with '/': $target (will be treated as relative)") - } - } - - errors.toList - } - private def hasDirectoryChanged(dir: AbsolutePath, lastModified: Long): Boolean = { - if (!dir.exists) return false - - val stream = Files.walk(dir.underlying) - try { - stream.anyMatch(path => Files.getLastModifiedTime(path).toMillis > lastModified) - } finally { - stream.close() + if (dir.exists) { + val stream = Files.walk(dir.underlying) + try { + stream.anyMatch(path => Files.getLastModifiedTime(path).toMillis > lastModified) + } finally { + stream.close() + } + } else { + false } } } diff --git a/frontend/src/main/scala/bloop/data/Project.scala b/frontend/src/main/scala/bloop/data/Project.scala index 94b3d6e322..4f85bb9e4f 100644 --- a/frontend/src/main/scala/bloop/data/Project.scala +++ b/frontend/src/main/scala/bloop/data/Project.scala @@ -348,9 +348,6 @@ object Project { val tags = project.tags.getOrElse(Nil) val projectDirectory = AbsolutePath(project.directory) - // Parse resource mappings if available (requires bloop-config 2.3.4+) - // TODO: Uncomment parsing logic once bloop-config is updated with resourceMapping field - // See bloop_config_changes.md for details val resourceMappings = List.empty[(AbsolutePath, String)] Project( diff --git a/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala b/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala index b8db32f911..4727936305 100644 --- a/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala +++ b/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala @@ -195,7 +195,8 @@ object CompileTask { ExecutionContext.ioExecutor, bundle.dependenciesData.allInvalidatedClassFiles, bundle.dependenciesData.allGeneratedClassFilePaths, - project.runtimeResources + project.runtimeResources, + project.resourceMappings ) } @@ -258,15 +259,8 @@ object CompileTask { ) // Copy mapped resources after compilation - val copyMappedResourcesTask = - bloop.io.ResourceMapper.copyMappedResources( - project.resourceMappings, - bundle.clientClassesObserver.classesDir, - logger - ) - val allTasks = Task - .gatherUnordered(List(postCompilationTasks, copyMappedResourcesTask)) + .gatherUnordered(List(postCompilationTasks)) .map(_ => ()) .doOnFinish(_ => Task(compileProjectTracer.terminate())) diff --git a/frontend/src/test/scala/bloop/io/ResourceMapperSpec.scala b/frontend/src/test/scala/bloop/io/ResourceMapperSpec.scala index 81b857fcdb..d847f6c087 100644 --- a/frontend/src/test/scala/bloop/io/ResourceMapperSpec.scala +++ b/frontend/src/test/scala/bloop/io/ResourceMapperSpec.scala @@ -90,67 +90,6 @@ object ResourceMapperSpec extends bloop.testing.BaseSuite { } } - test("validate duplicate target paths") { - TestUtil.withinWorkspace { workspace => - val logger = new RecordingLogger() - - val file1 = workspace.resolve("file1.txt") - val file2 = workspace.resolve("file2.txt") - Files.write(file1.underlying, "content 1".getBytes("UTF-8")) - Files.write(file2.underlying, "content 2".getBytes("UTF-8")) - - // Two sources mapping to same target - val mappings = List( - (file1, "output.txt"), - (file2, "output.txt") - ) - - val errors = ResourceMapper.validateMappings(mappings, logger) - - assert(errors.nonEmpty) - assert(errors.exists(_.contains("Multiple sources map to same target"))) - assert(errors.exists(_.contains("output.txt"))) - } - } - - test("validate path traversal attempts") { - TestUtil.withinWorkspace { workspace => - val logger = new RecordingLogger() - - val file = workspace.resolve("file.txt") - Files.write(file.underlying, "content".getBytes("UTF-8")) - - // Mapping with .. in target path - val mappings = List( - (file, "../escape/file.txt") - ) - - val errors = ResourceMapper.validateMappings(mappings, logger) - - assert(errors.nonEmpty) - assert(errors.exists(_.contains("Invalid target path contains '..'"))) - } - } - - test("warn about absolute target paths") { - TestUtil.withinWorkspace { workspace => - val logger = new RecordingLogger() - - val file = workspace.resolve("file.txt") - Files.write(file.underlying, "content".getBytes("UTF-8")) - - // Mapping with absolute target path - val mappings = List( - (file, "/absolute/path/file.txt") - ) - - ResourceMapper.validateMappings(mappings, logger) - - // Should warn but not error - assert(logger.warnings.exists(_.contains("Target path starts with '/'"))) - } - } - test("handle non-existent source file") { TestUtil.withinWorkspace { workspace => val logger = new RecordingLogger() From fbdf2f668d2b588dc89dfae7a4985753c3b244b5 Mon Sep 17 00:00:00 2001 From: krrish175-byte Date: Sat, 10 Jan 2026 19:33:59 +0530 Subject: [PATCH 3/5] feat: Implement compilation tracing --- .../scala/bloop/tracing/TraceProperties.scala | 7 +- .../main/scala/bloop/data/TraceSettings.scala | 9 +- .../bloop/engine/caches/ResultsCache.scala | 2 +- .../bloop/engine/tasks/CompileTask.scala | 111 +++++++++++++- .../tasks/compilation/CompileGraph.scala | 4 +- .../tasks/compilation/CompileResult.scala | 14 +- .../bloop/tracing/CompilationTrace.scala | 40 +++++ .../test/scala/bloop/BuildLoaderSpec.scala | 3 +- .../scala/bloop/CompilationTraceSpec.scala | 144 ++++++++++++++++++ 9 files changed, 316 insertions(+), 18 deletions(-) create mode 100644 frontend/src/main/scala/bloop/tracing/CompilationTrace.scala create mode 100644 frontend/src/test/scala/bloop/CompilationTraceSpec.scala diff --git a/backend/src/main/scala/bloop/tracing/TraceProperties.scala b/backend/src/main/scala/bloop/tracing/TraceProperties.scala index 07d5cf3aea..8f4a7efcdd 100644 --- a/backend/src/main/scala/bloop/tracing/TraceProperties.scala +++ b/backend/src/main/scala/bloop/tracing/TraceProperties.scala @@ -9,7 +9,8 @@ case class TraceProperties( localServiceName: String, traceStartAnnotation: Option[String], traceEndAnnotation: Option[String], - enabled: Boolean + enabled: Boolean, + compilationTrace: Boolean ) object TraceProperties { @@ -20,6 +21,7 @@ object TraceProperties { val localServiceName = Properties.propOrElse("bloop.tracing.localServiceName", "bloop") val traceStartAnnotation = Properties.propOrNone("bloop.tracing.traceStartAnnotation") val traceEndAnnotation = Properties.propOrNone("bloop.tracing.traceEndAnnotation") + val compilationTrace = Properties.propOrFalse("bloop.tracing.compilationTrace") val traceServerUrl = Properties.propOrElse( "zipkin.server.url", @@ -33,7 +35,8 @@ object TraceProperties { localServiceName, traceStartAnnotation, traceEndAnnotation, - enabled + enabled, + compilationTrace ) } } diff --git a/frontend/src/main/scala/bloop/data/TraceSettings.scala b/frontend/src/main/scala/bloop/data/TraceSettings.scala index 38a1ed3785..596cb00c2a 100644 --- a/frontend/src/main/scala/bloop/data/TraceSettings.scala +++ b/frontend/src/main/scala/bloop/data/TraceSettings.scala @@ -9,7 +9,8 @@ case class TraceSettings( localServiceName: Option[String], traceStartAnnotation: Option[String], traceEndAnnotation: Option[String], - enabled: Option[Boolean] + enabled: Option[Boolean], + compilationTrace: Option[Boolean] ) object TraceSettings { @@ -22,7 +23,8 @@ object TraceSettings { settings.localServiceName.getOrElse(default.localServiceName), settings.traceStartAnnotation.orElse(default.traceStartAnnotation), settings.traceEndAnnotation.orElse(default.traceEndAnnotation), - settings.enabled.getOrElse(default.enabled) + settings.enabled.getOrElse(default.enabled), + settings.compilationTrace.getOrElse(default.compilationTrace) ) } @@ -34,7 +36,8 @@ object TraceSettings { localServiceName = Some(properties.localServiceName), traceStartAnnotation = properties.traceStartAnnotation, traceEndAnnotation = properties.traceEndAnnotation, - enabled = Some(properties.enabled) + enabled = Some(properties.enabled), + compilationTrace = Some(properties.compilationTrace) ) } } diff --git a/frontend/src/main/scala/bloop/engine/caches/ResultsCache.scala b/frontend/src/main/scala/bloop/engine/caches/ResultsCache.scala index 85300de330..7bc00d197b 100644 --- a/frontend/src/main/scala/bloop/engine/caches/ResultsCache.scala +++ b/frontend/src/main/scala/bloop/engine/caches/ResultsCache.scala @@ -119,7 +119,7 @@ final case class ResultsCache private ( def addFinalResults(ps: List[FinalCompileResult]): ResultsCache = { ps.foldLeft(this) { - case (rs, FinalNormalCompileResult(p, r)) => rs.addResult(p, r) + case (rs, FinalNormalCompileResult(p, r, _)) => rs.addResult(p, r) case (rs, FinalEmptyResult) => rs } } diff --git a/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala b/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala index 4727936305..9014974118 100644 --- a/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala +++ b/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala @@ -40,7 +40,17 @@ import monix.reactive.MulticastStrategy import monix.reactive.Observable import xsbti.compile.CompileAnalysis import xsbti.compile.MiniSetup +import xsbti.compile.CompileAnalysis +import xsbti.compile.MiniSetup import xsbti.compile.PreviousResult +import bloop.tracing.CompilationTrace +import bloop.tracing.TraceDiagnostic +import bloop.tracing.TraceRange +import bloop.tracing.TraceArtifacts +import bloop.tracing.TraceProperties +import com.github.plokhotnyuk.jsoniter_scala.core.writeToArray +import com.github.plokhotnyuk.jsoniter_scala.core.WriterConfig +import bloop.util.JavaCompat.EnrichOptional import java.nio.file.Path @@ -365,7 +375,7 @@ object CompileTask { markUnusedClassesDirAndCollectCleanUpTasks(results, state, tracer, rawLogger) val failures = results.flatMap { - case FinalNormalCompileResult(p, results) => + case FinalNormalCompileResult(p, results, _) => results.fromCompiler match { case Compiler.Result.NotOk(_) => List(p) // Consider success with reported fatal warnings as error to simulate -Xfatal-warnings @@ -411,7 +421,7 @@ object CompileTask { val runningTasksRequiredForCorrectness = Task.sequence { results.flatMap { - case FinalNormalCompileResult(_, result) => + case FinalNormalCompileResult(_, result, _) => val tasksAtEndOfBuildCompilation = Task.fromFuture(result.runningBackgroundTasks) List(tasksAtEndOfBuildCompilation) @@ -422,6 +432,7 @@ object CompileTask { // Block on all background task that are running and are required for correctness runningTasksRequiredForCorrectness .executeOn(ExecutionContext.ioScheduler) + .flatMap(_ => reportCompilationTrace(results, traceProperties, rawLogger)) .map(_ => newState) .doOnFinish(_ => Task(rootTracer.terminate())) } @@ -477,7 +488,7 @@ object CompileTask { val compilerResult = resultBundle.fromCompiler val previousResult = finalResult match { - case FinalNormalCompileResult(p, _) => + case FinalNormalCompileResult(p, _, _) => previousState.results.all.get(p) case _ => None } @@ -504,6 +515,100 @@ object CompileTask { cleanUpTasksToSpawnInBackground.toList } + private def reportCompilationTrace( + results: List[FinalCompileResult], + traceProperties: TraceProperties, + logger: Logger + ): Task[Unit] = { + if (!traceProperties.compilationTrace) Task.unit + else { + val validResults = results.collect { case r: FinalNormalCompileResult => r } + logger.info( + s"Reporting compilation trace for ${validResults.size} projects. Enabled: ${traceProperties.compilationTrace}" + ) + if (validResults.isEmpty) Task.unit + else { + val traces = validResults.flatMap { + case FinalNormalCompileResult(project, resultBundle, bundleOpt) => + val result = resultBundle.fromCompiler + val successful = resultBundle.successful + + val diagnostics = bundleOpt.map(_.reporter.allProblems).getOrElse(Nil).map { p => + val range = for { + startLine <- p.position.startLine.toOption + startChar <- p.position.startColumn.toOption + endLine <- p.position.endLine.toOption + endChar <- p.position.endColumn.toOption + } yield TraceRange(startLine, startChar, endLine, endChar) + + val code = Option(p.position.lineContent).filter(_.nonEmpty) + val source = p.position.sourcePath.toOption.filter(_.nonEmpty) + + TraceDiagnostic( + p.severity.toString, + p.message, + range, + code, + source + ) + } + + val classesDir = result match { + case s: Compiler.Result.Success => + s.products.newClassesDir.toString + case _ => + "" + } + + val analysisOut = bundleOpt.map(_.out.analysisOut.toString).getOrElse("") + + val artifacts = TraceArtifacts(classesDir, analysisOut) + + val files = bundleOpt match { + case Some(s) => s.uniqueInputs.sources.map(_.toPath.toString) + case None => Seq.empty + } + + val isNoOp = result match { + case Compiler.Result.Success(_, _, _, _, isNoOp, _, _) => isNoOp + case _ => false + } + + Some( + CompilationTrace( + project.name, + files, + diagnostics.toSeq, + artifacts, + isNoOp, + 0L + ) + ) + } + + Task.eval { + val workspaceDir = validResults.head.project.baseDirectory.getParent + val traceFile = workspaceDir.resolve(".bloop").resolve("compilation-trace.json") + logger.info(s"Writing trace to $traceFile") + try { + if (!java.nio.file.Files.exists(traceFile.getParent.underlying)) { + java.nio.file.Files.createDirectories(traceFile.getParent.underlying) + } + val bytes = + writeToArray(traces, WriterConfig.withIndentionStep(4))(CompilationTrace.listCodec) + java.nio.file.Files.write(traceFile.underlying, bytes) + () + } catch { + case scala.util.control.NonFatal(e) => + logger.error(s"Failed to write compilation trace: ${e.getMessage}") + e.printStackTrace() + () + } + } + } + } + } + def runIOTasksInParallel[T]( tasks: Traversable[Task[T]], parallelUnits: Int = Runtime.getRuntime().availableProcessors() diff --git a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGraph.scala b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGraph.scala index 86cc87f7f5..af16e6eccb 100644 --- a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGraph.scala +++ b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGraph.scala @@ -397,7 +397,7 @@ object CompileGraph { * `FailPromise` exception that makes the partial result be recognized as error. */ def toPartialFailure(bundle: SuccessfulCompileBundle, results: ResultBundle): PartialFailure = { - PartialFailure(bundle.project, FailedOrCancelledPromise, Task.now(results)) + PartialFailure(bundle.project, FailedOrCancelledPromise, Task.now(results), Some(bundle)) } def loop(dag: Dag[Project]): CompileTraversal = { @@ -437,7 +437,7 @@ object CompileGraph { val transitive = dagResults.flatMap(Dag.dfs(_, mode = Dag.PreOrder)).distinct transitive.flatMap { case PartialSuccess(bundle, result) => Some(result.map(r => bundle.project -> r)) - case PartialFailure(project, _, result) => Some(result.map(r => project -> r)) + case PartialFailure(project, _, result, _) => Some(result.map(r => project -> r)) case _ => None } } diff --git a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileResult.scala b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileResult.scala index 816b3c96d5..2fbb54c6a2 100644 --- a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileResult.scala +++ b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileResult.scala @@ -41,10 +41,10 @@ object PartialCompileResult { def toFinalResult(result: PartialCompileResult): Task[List[FinalCompileResult]] = { result match { case PartialEmpty => Task.now(FinalEmptyResult :: Nil) - case PartialFailure(project, _, bundle) => - bundle.map(b => FinalNormalCompileResult(project, b) :: Nil) + case PartialFailure(project, _, resultTask, bundleOpt) => + resultTask.map(b => FinalNormalCompileResult(project, b, bundleOpt) :: Nil) case PartialSuccess(bundle, result) => - result.map(res => FinalNormalCompileResult(bundle.project, res) :: Nil) + result.map(res => FinalNormalCompileResult(bundle.project, res, Some(bundle)) :: Nil) } } } @@ -57,7 +57,8 @@ case object PartialEmpty extends PartialCompileResult { case class PartialFailure( project: Project, exception: Throwable, - result: Task[ResultBundle] + result: Task[ResultBundle], + bundle: Option[SuccessfulCompileBundle] = None ) extends PartialCompileResult with CacheHashCode {} @@ -77,7 +78,8 @@ case object FinalEmptyResult extends FinalCompileResult { case class FinalNormalCompileResult private ( project: Project, - result: ResultBundle + result: ResultBundle, + bundle: Option[SuccessfulCompileBundle] ) extends FinalCompileResult with CacheHashCode @@ -102,7 +104,7 @@ object FinalCompileResult { override def shows(r: FinalCompileResult): String = { r match { case FinalEmptyResult => s" (product of dag aggregation)" - case FinalNormalCompileResult(project, result) => + case FinalNormalCompileResult(project, result, _) => val projectName = project.name result.fromCompiler match { case Compiler.Result.Empty => s"${projectName} (empty)" diff --git a/frontend/src/main/scala/bloop/tracing/CompilationTrace.scala b/frontend/src/main/scala/bloop/tracing/CompilationTrace.scala new file mode 100644 index 0000000000..b8a0a0a302 --- /dev/null +++ b/frontend/src/main/scala/bloop/tracing/CompilationTrace.scala @@ -0,0 +1,40 @@ +package bloop.tracing + +import com.github.plokhotnyuk.jsoniter_scala.core.JsonValueCodec +import com.github.plokhotnyuk.jsoniter_scala.macros.JsonCodecMaker + +case class CompilationTrace( + project: String, + files: Seq[String], + diagnostics: Seq[TraceDiagnostic], + artifacts: TraceArtifacts, + isNoOp: Boolean, + durationMs: Long +) + +object CompilationTrace { + implicit val codec: JsonValueCodec[CompilationTrace] = + JsonCodecMaker.make[CompilationTrace] + implicit val listCodec: JsonValueCodec[List[CompilationTrace]] = + JsonCodecMaker.make[List[CompilationTrace]] +} + +case class TraceDiagnostic( + severity: String, + message: String, + range: Option[TraceRange], + code: Option[String], + source: Option[String] +) + +case class TraceRange( + startLine: Int, + startCharacter: Int, + endLine: Int, + endCharacter: Int +) + +case class TraceArtifacts( + classesDir: String, + analysisFile: String +) diff --git a/frontend/src/test/scala/bloop/BuildLoaderSpec.scala b/frontend/src/test/scala/bloop/BuildLoaderSpec.scala index f4f96cfa85..4b93d714b3 100644 --- a/frontend/src/test/scala/bloop/BuildLoaderSpec.scala +++ b/frontend/src/test/scala/bloop/BuildLoaderSpec.scala @@ -304,7 +304,8 @@ object BuildLoaderSpec extends BaseSuite { localServiceName = Some("42"), traceStartAnnotation = Some("start"), traceEndAnnotation = Some("end"), - enabled = Some(true) + enabled = Some(true), + compilationTrace = None ) ), None diff --git a/frontend/src/test/scala/bloop/CompilationTraceSpec.scala b/frontend/src/test/scala/bloop/CompilationTraceSpec.scala new file mode 100644 index 0000000000..5ecc0d19af --- /dev/null +++ b/frontend/src/test/scala/bloop/CompilationTraceSpec.scala @@ -0,0 +1,144 @@ +package bloop + +import bloop.task.Task +import bloop.util.TestUtil +import bloop.logging.RecordingLogger +import bloop.data.WorkspaceSettings +import bloop.data.TraceSettings +import bloop.tracing.CompilationTrace +import java.nio.file.Files +import bloop.cli.ExitStatus +import com.github.plokhotnyuk.jsoniter_scala.core.readFromArray + +object CompilationTraceSpec extends BaseCompileSpec { + override protected val TestProject = util.TestProject + + test("compilation trace is created when enabled") { + TestUtil.withinWorkspace { workspace => + val sources = List( + """/main/scala/Foo.scala + |class Foo + """.stripMargin + ) + + val logger = new RecordingLogger(ansiCodesSupported = false, debug = false) + val `A` = TestProject(workspace, "a", sources) + val projects = List(`A`) + + // Write workspace settings with compilationTrace enabled + val settings = WorkspaceSettings( + None, + None, + None, + None, + Some(TraceSettings(None, None, None, None, None, None, None, Some(true))), + None + ) + val configDir = workspace.resolve(".bloop") + if (!Files.exists(configDir.underlying)) Files.createDirectories(configDir.underlying) + WorkspaceSettings.writeToFile(configDir, settings, logger) + + val state = loadState(workspace, projects, logger) + val compiledState = state.compile(`A`) + assertExitStatus(compiledState, ExitStatus.Ok) + + val traceFile = workspace.resolve(".bloop/compilation-trace.json") + assert(Files.exists(traceFile.underlying)) + + val bytes = Files.readAllBytes(traceFile.underlying) + val traces = readFromArray[List[CompilationTrace]](bytes)(CompilationTrace.listCodec) + + assert(traces.size == 1) + val trace = traces.head + assert(trace.project == "a") + assert(trace.files.exists(_.endsWith("Foo.scala"))) + assert(trace.diagnostics.isEmpty) + assert(!trace.isNoOp) + } + } + + test("compilation trace contains diagnostics") { + TestUtil.withinWorkspace { workspace => + val sources = List( + """/main/scala/Foo.scala + |class Foo { + | def bar: Int = "string" + |} + """.stripMargin + ) + + val logger = new RecordingLogger(ansiCodesSupported = false, debug = false) + val `A` = TestProject(workspace, "a", sources) + val projects = List(`A`) + + val settings = WorkspaceSettings( + None, + None, + None, + None, + Some(TraceSettings(None, None, None, None, None, None, None, Some(true))), + None + ) + val configDir = workspace.resolve(".bloop") + if (!Files.exists(configDir.underlying)) Files.createDirectories(configDir.underlying) + WorkspaceSettings.writeToFile(configDir, settings, logger) + + val state = loadState(workspace, projects, logger) + val compiledState = state.compile(`A`) + assertExitStatus(compiledState, ExitStatus.CompilationError) + + val traceFile = workspace.resolve(".bloop/compilation-trace.json") + assert(Files.exists(traceFile.underlying)) + + val bytes = Files.readAllBytes(traceFile.underlying) + val traces = readFromArray[List[CompilationTrace]](bytes)(CompilationTrace.listCodec) + + assert(traces.size == 1) + val trace = traces.head + assert(trace.diagnostics.nonEmpty) + assert(trace.diagnostics.exists(_.message.contains("type mismatch"))) + } + } + + test("compilation trace records no-op") { + TestUtil.withinWorkspace { workspace => + val sources = List( + """/main/scala/Foo.scala + |class Foo + """.stripMargin + ) + + val logger = new RecordingLogger(ansiCodesSupported = false, debug = false) + val `A` = TestProject(workspace, "a", sources) + val projects = List(`A`) + + val settings = WorkspaceSettings( + None, + None, + None, + None, + Some(TraceSettings(None, None, None, None, None, None, None, Some(true))), + None + ) + val configDir = workspace.resolve(".bloop") + if (!Files.exists(configDir.underlying)) Files.createDirectories(configDir.underlying) + WorkspaceSettings.writeToFile(configDir, settings, logger) + + val state = loadState(workspace, projects, logger) + val compiledState = state.compile(`A`) + assertExitStatus(compiledState, ExitStatus.Ok) + + val secondCompiledState = compiledState.compile(`A`) + assertExitStatus(secondCompiledState, ExitStatus.Ok) + + val traceFile = workspace.resolve(".bloop/compilation-trace.json") + val bytes = Files.readAllBytes(traceFile.underlying) + val traces = readFromArray[List[CompilationTrace]](bytes)(CompilationTrace.listCodec) + + // Since we overwrite, it should be the trace of the last compilation (no-op) + assert(traces.size == 1) + val trace = traces.head + assert(trace.isNoOp) + } + } +} From 07edfd2abd255424819be1a05323feabbe9450a2 Mon Sep 17 00:00:00 2001 From: krrish175-byte Date: Mon, 12 Jan 2026 22:08:53 +0530 Subject: [PATCH 4/5] Refactor compilation tracing: use BraveTracer, remove ResourceMapper --- backend/src/main/scala/bloop/Compiler.scala | 7 +- .../main/scala/bloop/io/ResourceMapper.scala | 114 -------------- .../scala/bloop/tracing/BraveTracer.scala | 146 +++++++++++++++++- .../bloop/tracing/CompilationTrace.scala | 0 .../bloop/engine/tasks/CompileTask.scala | 127 ++------------- 5 files changed, 161 insertions(+), 233 deletions(-) delete mode 100644 backend/src/main/scala/bloop/io/ResourceMapper.scala rename {frontend => backend}/src/main/scala/bloop/tracing/CompilationTrace.scala (100%) diff --git a/backend/src/main/scala/bloop/Compiler.scala b/backend/src/main/scala/bloop/Compiler.scala index b3e3ac333c..f4e1ce4b07 100644 --- a/backend/src/main/scala/bloop/Compiler.scala +++ b/backend/src/main/scala/bloop/Compiler.scala @@ -829,11 +829,6 @@ object Compiler { compileInputs.logger, compileInputs.ioScheduler ) - val copyMappedResources = bloop.io.ResourceMapper.copyMappedResources( - compileInputs.resourceMappings, - clientClassesDir, - compileInputs.logger - ) val lastCopy = ParallelOps.copyDirectories(config)( readOnlyClassesDir, clientClassesDir.underlying, @@ -842,7 +837,7 @@ object Compiler { compileInputs.logger ) - Task.gatherUnordered(List(copyResources, copyMappedResources, lastCopy)).map { _ => + Task.gatherUnordered(List(copyResources, lastCopy)).map { _ => clientLogger.debug( s"Finished copying classes from $readOnlyClassesDir to $clientClassesDir" ) diff --git a/backend/src/main/scala/bloop/io/ResourceMapper.scala b/backend/src/main/scala/bloop/io/ResourceMapper.scala deleted file mode 100644 index 28076a1089..0000000000 --- a/backend/src/main/scala/bloop/io/ResourceMapper.scala +++ /dev/null @@ -1,114 +0,0 @@ -package bloop.io - -import java.nio.file.Files -import java.nio.file.StandardCopyOption - -import scala.util.control.NonFatal - -import bloop.logging.DebugFilter -import bloop.logging.Logger -import bloop.task.Task - -/** - * Utility for handling resource file mappings. - * - * Resource mappings allow files to be copied from source locations to custom - * target paths, similar to SBT's "mappings" field. - */ -object ResourceMapper { - private implicit val filter: DebugFilter.All.type = DebugFilter.All - - /** - * Copy mapped resources to the target directory. - * - * @param mappings List of (source, targetRelativePath) tuples - * @param classesDir Base directory where resources should be copied - * @param logger Logger for debug/error messages - * @return Task that completes when all resources are copied - */ - def copyMappedResources( - mappings: List[(AbsolutePath, String)], - classesDir: AbsolutePath, - logger: Logger - ): Task[Unit] = { - val tasks = mappings.map { - case (source, targetRelPath) => - Task { - val target = classesDir.resolve(targetRelPath) - if (!target.getParent.exists) { - Files.createDirectories(target.getParent.underlying) - } - - if (source.isDirectory) { - import java.nio.file.FileVisitResult - import java.nio.file.Path - import java.nio.file.SimpleFileVisitor - import java.nio.file.attribute.BasicFileAttributes - - val sourcePath = source.underlying - val targetPath = target.underlying - Files.walkFileTree( - sourcePath, - new SimpleFileVisitor[Path] { - override def visitFile( - file: Path, - attrs: BasicFileAttributes - ): FileVisitResult = { - val relPath = sourcePath.relativize(file) - val targetFile = targetPath.resolve(relPath) - Files.createDirectories(targetFile.getParent) - Files.copy(file, targetFile, StandardCopyOption.REPLACE_EXISTING) - FileVisitResult.CONTINUE - } - } - ) - } else if (source.exists) { - Files.copy(source.underlying, target.underlying, StandardCopyOption.REPLACE_EXISTING) - } else { - logger.warn(s"Source file $source does not exist, skipping mapping to $targetRelPath") - } - () - } - } - Task.gatherUnordered(tasks).map(_ => ()) - } - - /** - * Check if any mapped resources have changed since the given timestamp. - * - * Note: This is currently not used. Incremental compilation change detection - * for resource mappings would require integration with Zinc's analysis. - * For now, mapped resources are always copied during compilation. - * - * @param mappings List of (source, targetRelativePath) tuples - * @param lastModified Timestamp to compare against - * @return true if any source file is newer than lastModified - */ - private[bloop] def hasMappingsChanged( - mappings: List[(AbsolutePath, String)], - lastModified: Long - ): Boolean = { - import java.nio.file.Files - mappings.exists { - case (source, _) => - if (source.isDirectory) { - hasDirectoryChanged(source, lastModified) - } else { - source.exists && Files.getLastModifiedTime(source.underlying).toMillis > lastModified - } - } - } - - private def hasDirectoryChanged(dir: AbsolutePath, lastModified: Long): Boolean = { - if (dir.exists) { - val stream = Files.walk(dir.underlying) - try { - stream.anyMatch(path => Files.getLastModifiedTime(path).toMillis > lastModified) - } finally { - stream.close() - } - } else { - false - } - } -} diff --git a/backend/src/main/scala/bloop/tracing/BraveTracer.scala b/backend/src/main/scala/bloop/tracing/BraveTracer.scala index a52f4288fd..927a0c5c78 100644 --- a/backend/src/main/scala/bloop/tracing/BraveTracer.scala +++ b/backend/src/main/scala/bloop/tracing/BraveTracer.scala @@ -21,6 +21,8 @@ sealed trait BraveTracer { thunk: BraveTracer => T ): T + def tag(key: String, value: String): Unit + def traceVerbose[T](name: String, tags: (String, String)*)( thunk: BraveTracer => T ): T @@ -49,6 +51,8 @@ object NoopTracer extends BraveTracer { override def startNewChildTracer(name: String, tags: (String, String)*): BraveTracer = this + override def tag(key: String, value: String): Unit = () + override def trace[T](name: String, tags: (String, String)*)(thunk: BraveTracer => T): T = thunk( this ) @@ -88,12 +92,31 @@ object BraveTracer { ctx: Option[TraceContext], tags: (String, String)* ): BraveTracer = { - if (properties.enabled) { + val braveTracer = if (properties.enabled) { BraveTracerInternal(name, properties, ctx, tags: _*) } else { NoopTracer } + if (properties.compilationTrace) { + val traceFile = bloop.io.AbsolutePath( + java.nio.file.Paths + .get(name.stripPrefix("compile ").stripSuffix(" (transitively)")) + .getParent + .resolve(".bloop") + .resolve("compilation-trace.json") + ) + val projectName = tags + .collectFirst { case ("compile.target", value) => value } + .getOrElse(name) + CompositeTracer( + braveTracer, + new CompilationTraceTracer(projectName, traceFile, System.currentTimeMillis()) + ) + } else { + braveTracer + } + } } @@ -104,6 +127,11 @@ final class BraveTracerInternal private ( properties: TraceProperties ) extends BraveTracer { + override def tag(key: String, value: String): Unit = { + _currentSpan.tag(key, value) + () + } + def currentSpan = Some(_currentSpan) def startNewChildTracer(name: String, tags: (String, String)*): BraveTracer = { @@ -263,3 +291,119 @@ object BraveTracerInternal { new BraveTracerInternal(tracer, rootSpan, closeEverything, properties) } } + +final class CompilationTraceTracer( + project: String, + traceFile: bloop.io.AbsolutePath, + startTime: Long +) extends BraveTracer { + import bloop.tracing.CompilationTrace + import bloop.tracing.TraceArtifacts + import com.github.plokhotnyuk.jsoniter_scala.core.writeToArray + import com.github.plokhotnyuk.jsoniter_scala.core.WriterConfig + + private val tags = new ConcurrentHashMap[String, String]() + + override def startNewChildTracer(name: String, tags: (String, String)*): BraveTracer = this + override def tag(key: String, value: String): Unit = { + this.tags.put(key, value) + () + } + + override def trace[T](name: String, tags: (String, String)*)(thunk: BraveTracer => T): T = + thunk(this) + override def traceVerbose[T](name: String, tags: (String, String)*)( + thunk: BraveTracer => T + ): T = thunk(this) + override def traceTask[T](name: String, tags: (String, String)*)( + thunk: BraveTracer => Task[T] + ): Task[T] = thunk(this) + override def traceTaskVerbose[T](name: String, tags: (String, String)*)( + thunk: BraveTracer => Task[T] + ): Task[T] = thunk(this) + + override def terminate(): Unit = { + val durationMs = System.currentTimeMillis() - startTime + // We only want to write the trace if we have collected the necessary tags + // The "success" tag is used as a signal that the compilation finished + if (tags.containsKey("success")) { + val isNoOp = tags.getOrDefault("noop", "false").toBoolean + val analysisOut = tags.getOrDefault("analysis", "") + val classesDir = tags.getOrDefault("classesDir", "") + val artifacts = TraceArtifacts(classesDir, analysisOut) + val fileCount = tags.getOrDefault("fileCount", "0").toInt + val files = (0 until fileCount).map(i => tags.get(s"file.$i")).filter(_ != null) + + val trace = CompilationTrace( + project, + files, + Seq.empty, + artifacts, + isNoOp, + durationMs + ) + + try { + if (!java.nio.file.Files.exists(traceFile.getParent.underlying)) { + java.nio.file.Files.createDirectories(traceFile.getParent.underlying) + } + traceFile.getParent.underlying.synchronized { + val projectTraceFile = traceFile.getParent.resolve(s"compilation-trace-${project}.json") + val bytes = writeToArray(trace, WriterConfig.withIndentionStep(4))(CompilationTrace.codec) + java.nio.file.Files.write(projectTraceFile.underlying, bytes) + () + } + } catch { + case NonFatal(e) => e.printStackTrace() + } + } + } + + override def currentSpan: Option[Span] = None + override def toIndependentTracer( + name: String, + traceProperties: TraceProperties, + tags: (String, String)* + ): BraveTracer = + this +} + +final case class CompositeTracer(tracers: BraveTracer*) extends BraveTracer { + override def startNewChildTracer(name: String, tags: (String, String)*): BraveTracer = + CompositeTracer(tracers.map(_.startNewChildTracer(name, tags: _*)): _*) + + override def tag(key: String, value: String): Unit = + tracers.foreach(_.tag(key, value)) + + override def trace[T](name: String, tags: (String, String)*)(thunk: BraveTracer => T): T = { + val children = tracers.map(_.startNewChildTracer(name, tags: _*)) + val compositeChild = CompositeTracer(children: _*) + try thunk(compositeChild) + finally children.foreach(_.terminate()) + } + + override def traceVerbose[T](name: String, tags: (String, String)*)(thunk: BraveTracer => T): T = + trace(name, tags: _*)(thunk) + + override def traceTask[T](name: String, tags: (String, String)*)( + thunk: BraveTracer => Task[T] + ): Task[T] = { + val children = tracers.map(_.startNewChildTracer(name, tags: _*)) + val compositeChild = CompositeTracer(children: _*) + thunk(compositeChild).doOnFinish(_ => Task.eval(children.foreach(_.terminate()))) + } + + override def traceTaskVerbose[T](name: String, tags: (String, String)*)( + thunk: BraveTracer => Task[T] + ): Task[T] = + traceTask(name, tags: _*)(thunk) + + override def terminate(): Unit = tracers.foreach(_.terminate()) + override def currentSpan: Option[Span] = tracers.flatMap(_.currentSpan).headOption + override def toIndependentTracer( + name: String, + traceProperties: TraceProperties, + tags: (String, String)* + ): BraveTracer = + CompositeTracer(tracers.map(_.toIndependentTracer(name, traceProperties, tags: _*)): _*) +} diff --git a/frontend/src/main/scala/bloop/tracing/CompilationTrace.scala b/backend/src/main/scala/bloop/tracing/CompilationTrace.scala similarity index 100% rename from frontend/src/main/scala/bloop/tracing/CompilationTrace.scala rename to backend/src/main/scala/bloop/tracing/CompilationTrace.scala diff --git a/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala b/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala index 9014974118..2a822ffc4d 100644 --- a/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala +++ b/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala @@ -43,14 +43,6 @@ import xsbti.compile.MiniSetup import xsbti.compile.CompileAnalysis import xsbti.compile.MiniSetup import xsbti.compile.PreviousResult -import bloop.tracing.CompilationTrace -import bloop.tracing.TraceDiagnostic -import bloop.tracing.TraceRange -import bloop.tracing.TraceArtifacts -import bloop.tracing.TraceProperties -import com.github.plokhotnyuk.jsoniter_scala.core.writeToArray -import com.github.plokhotnyuk.jsoniter_scala.core.WriterConfig -import bloop.util.JavaCompat.EnrichOptional import java.nio.file.Path @@ -135,15 +127,7 @@ object CompileTask { ) // Also copy mapped resources if any - val copyMappedResourcesTask: Task[Unit] = - bloop.io.ResourceMapper.copyMappedResources( - project.resourceMappings, - bundle.clientClassesObserver.classesDir, - logger - ) - - val allCopyTasks = - Task.gatherUnordered(List(copyResourcesTask, copyMappedResourcesTask)) + val allCopyTasks = copyResourcesTask Task.now( ResultBundle( Compiler.Result.Empty, @@ -298,6 +282,20 @@ object CompileTask { // Memoize so that no matter how many times it's run, it's executed only once val newSuccessful = LastSuccessfulResult(bundle.uniqueInputs, s.products, populatingTask.memoize) + + // Tag tracer with success results + compileProjectTracer.tag("success", "true") + compileProjectTracer.tag("isNoOp", s.isNoOp.toString) + compileProjectTracer.tag("classesDir", s.products.newClassesDir.toString) + compileProjectTracer.tag("analysis", compileOut.analysisOut.toString) + + val sources = bundle.uniqueInputs.sources + compileProjectTracer.tag("fileCount", sources.length.toString) + sources.zipWithIndex.foreach { + case (file, idx) => + compileProjectTracer.tag(s"file.$idx", file.toPath.toString) + } + ResultBundle(s, Some(newSuccessful), Some(lastSuccessful), runningTasks) case f: Compiler.Result.Failed => val runningTasks = runPostCompilationTasks(f.backgroundTasks) @@ -432,7 +430,6 @@ object CompileTask { // Block on all background task that are running and are required for correctness runningTasksRequiredForCorrectness .executeOn(ExecutionContext.ioScheduler) - .flatMap(_ => reportCompilationTrace(results, traceProperties, rawLogger)) .map(_ => newState) .doOnFinish(_ => Task(rootTracer.terminate())) } @@ -515,100 +512,6 @@ object CompileTask { cleanUpTasksToSpawnInBackground.toList } - private def reportCompilationTrace( - results: List[FinalCompileResult], - traceProperties: TraceProperties, - logger: Logger - ): Task[Unit] = { - if (!traceProperties.compilationTrace) Task.unit - else { - val validResults = results.collect { case r: FinalNormalCompileResult => r } - logger.info( - s"Reporting compilation trace for ${validResults.size} projects. Enabled: ${traceProperties.compilationTrace}" - ) - if (validResults.isEmpty) Task.unit - else { - val traces = validResults.flatMap { - case FinalNormalCompileResult(project, resultBundle, bundleOpt) => - val result = resultBundle.fromCompiler - val successful = resultBundle.successful - - val diagnostics = bundleOpt.map(_.reporter.allProblems).getOrElse(Nil).map { p => - val range = for { - startLine <- p.position.startLine.toOption - startChar <- p.position.startColumn.toOption - endLine <- p.position.endLine.toOption - endChar <- p.position.endColumn.toOption - } yield TraceRange(startLine, startChar, endLine, endChar) - - val code = Option(p.position.lineContent).filter(_.nonEmpty) - val source = p.position.sourcePath.toOption.filter(_.nonEmpty) - - TraceDiagnostic( - p.severity.toString, - p.message, - range, - code, - source - ) - } - - val classesDir = result match { - case s: Compiler.Result.Success => - s.products.newClassesDir.toString - case _ => - "" - } - - val analysisOut = bundleOpt.map(_.out.analysisOut.toString).getOrElse("") - - val artifacts = TraceArtifacts(classesDir, analysisOut) - - val files = bundleOpt match { - case Some(s) => s.uniqueInputs.sources.map(_.toPath.toString) - case None => Seq.empty - } - - val isNoOp = result match { - case Compiler.Result.Success(_, _, _, _, isNoOp, _, _) => isNoOp - case _ => false - } - - Some( - CompilationTrace( - project.name, - files, - diagnostics.toSeq, - artifacts, - isNoOp, - 0L - ) - ) - } - - Task.eval { - val workspaceDir = validResults.head.project.baseDirectory.getParent - val traceFile = workspaceDir.resolve(".bloop").resolve("compilation-trace.json") - logger.info(s"Writing trace to $traceFile") - try { - if (!java.nio.file.Files.exists(traceFile.getParent.underlying)) { - java.nio.file.Files.createDirectories(traceFile.getParent.underlying) - } - val bytes = - writeToArray(traces, WriterConfig.withIndentionStep(4))(CompilationTrace.listCodec) - java.nio.file.Files.write(traceFile.underlying, bytes) - () - } catch { - case scala.util.control.NonFatal(e) => - logger.error(s"Failed to write compilation trace: ${e.getMessage}") - e.printStackTrace() - () - } - } - } - } - } - def runIOTasksInParallel[T]( tasks: Traversable[Task[T]], parallelUnits: Int = Runtime.getRuntime().availableProcessors() From 860919e9929c0055523e4d8b33e04845cc15bc05 Mon Sep 17 00:00:00 2001 From: krrish175-byte Date: Wed, 14 Jan 2026 14:17:04 +0530 Subject: [PATCH 5/5] Refactor BraveTracer and clean up compilation trace --- .../scala/bloop/tracing/BraveTracer.scala | 148 +++++++---------- .../bloop/tracing/CompilationTrace.scala | 2 + .../src/it/scala/bloop/CommunityBuild.scala | 1 + .../bloop/engine/tasks/CompileTask.scala | 57 +++++++ .../src/test/resources/source-generator.py | 4 + .../scala/bloop/CompilationTraceSpec.scala | 22 ++- .../scala/bloop/io/ResourceMapperSpec.scala | 150 ------------------ 7 files changed, 129 insertions(+), 255 deletions(-) delete mode 100644 frontend/src/test/scala/bloop/io/ResourceMapperSpec.scala diff --git a/backend/src/main/scala/bloop/tracing/BraveTracer.scala b/backend/src/main/scala/bloop/tracing/BraveTracer.scala index 927a0c5c78..cdfc764f3a 100644 --- a/backend/src/main/scala/bloop/tracing/BraveTracer.scala +++ b/backend/src/main/scala/bloop/tracing/BraveTracer.scala @@ -1,10 +1,13 @@ package bloop.tracing import java.util.concurrent.ConcurrentHashMap +import java.nio.file.{Paths => NioPaths, Files} +import java.net.URI import scala.util.control.NonFatal import bloop.task.Task +import bloop.io.AbsolutePath import brave.Span import brave.Tracer @@ -92,31 +95,25 @@ object BraveTracer { ctx: Option[TraceContext], tags: (String, String)* ): BraveTracer = { - val braveTracer = if (properties.enabled) { + if (properties.enabled) { BraveTracerInternal(name, properties, ctx, tags: _*) } else { - NoopTracer - } - - if (properties.compilationTrace) { - val traceFile = bloop.io.AbsolutePath( - java.nio.file.Paths - .get(name.stripPrefix("compile ").stripSuffix(" (transitively)")) - .getParent - .resolve(".bloop") - .resolve("compilation-trace.json") - ) - val projectName = tags - .collectFirst { case ("compile.target", value) => value } - .getOrElse(name) - CompositeTracer( - braveTracer, - new CompilationTraceTracer(projectName, traceFile, System.currentTimeMillis()) - ) - } else { - braveTracer + val buildUri = tags.collectFirst { case ("workspace.dir", value) => value } + buildUri match { + case Some(uri) => + val workspaceDir = AbsolutePath(NioPaths.get(uri)) + val traceFile = workspaceDir.resolve("compilation-trace.json") + if (traceFile.exists) { + val projectName = tags + .collectFirst { case ("compile.target", value) => value } + .getOrElse(name) + new CompilationTraceTracer(projectName, traceFile, System.currentTimeMillis()) + } else { + NoopTracer + } + case None => NoopTracer + } } - } } @@ -324,38 +321,45 @@ final class CompilationTraceTracer( override def terminate(): Unit = { val durationMs = System.currentTimeMillis() - startTime - // We only want to write the trace if we have collected the necessary tags - // The "success" tag is used as a signal that the compilation finished - if (tags.containsKey("success")) { - val isNoOp = tags.getOrDefault("noop", "false").toBoolean - val analysisOut = tags.getOrDefault("analysis", "") - val classesDir = tags.getOrDefault("classesDir", "") - val artifacts = TraceArtifacts(classesDir, analysisOut) - val fileCount = tags.getOrDefault("fileCount", "0").toInt - val files = (0 until fileCount).map(i => tags.get(s"file.$i")).filter(_ != null) - - val trace = CompilationTrace( - project, - files, - Seq.empty, - artifacts, - isNoOp, - durationMs - ) + val isNoOp = tags.getOrDefault("isNoOp", "false").toBoolean + val analysisOut = tags.getOrDefault("analysis", "") + val classesDir = tags.getOrDefault("classesDir", "") + val artifacts = TraceArtifacts(classesDir, analysisOut) + val fileCount = tags.getOrDefault("fileCount", "0").toInt + val files = (0 until fileCount).map(i => tags.get(s"file.$i")).filter(_ != null) + + import com.github.plokhotnyuk.jsoniter_scala.core.readFromString + val diagnosticCount = tags.getOrDefault("diagnostics.count", "0").toInt + val diagnostics = (0 until diagnosticCount).flatMap { i => + val json = tags.get(s"diagnostic.$i") + if (json != null) { + try { + Some(readFromString[TraceDiagnostic](json)(CompilationTrace.diagnosticCodec)) + } catch { case NonFatal(_) => None } + } else None + } - try { - if (!java.nio.file.Files.exists(traceFile.getParent.underlying)) { - java.nio.file.Files.createDirectories(traceFile.getParent.underlying) - } - traceFile.getParent.underlying.synchronized { - val projectTraceFile = traceFile.getParent.resolve(s"compilation-trace-${project}.json") - val bytes = writeToArray(trace, WriterConfig.withIndentionStep(4))(CompilationTrace.codec) - java.nio.file.Files.write(projectTraceFile.underlying, bytes) - () - } - } catch { - case NonFatal(e) => e.printStackTrace() + val trace = CompilationTrace( + project, + files, + diagnostics, + artifacts, + isNoOp, + durationMs + ) + + try { + if (!java.nio.file.Files.exists(traceFile.getParent.underlying)) { + java.nio.file.Files.createDirectories(traceFile.getParent.underlying) } + traceFile.getParent.underlying.synchronized { + val projectTraceFile = traceFile.getParent.resolve(s"compilation-trace-${project}.json") + val bytes = writeToArray(trace, WriterConfig.withIndentionStep(4))(CompilationTrace.codec) + java.nio.file.Files.write(projectTraceFile.underlying, bytes) + () + } + } catch { + case NonFatal(e) => e.printStackTrace() } } @@ -367,43 +371,3 @@ final class CompilationTraceTracer( ): BraveTracer = this } - -final case class CompositeTracer(tracers: BraveTracer*) extends BraveTracer { - override def startNewChildTracer(name: String, tags: (String, String)*): BraveTracer = - CompositeTracer(tracers.map(_.startNewChildTracer(name, tags: _*)): _*) - - override def tag(key: String, value: String): Unit = - tracers.foreach(_.tag(key, value)) - - override def trace[T](name: String, tags: (String, String)*)(thunk: BraveTracer => T): T = { - val children = tracers.map(_.startNewChildTracer(name, tags: _*)) - val compositeChild = CompositeTracer(children: _*) - try thunk(compositeChild) - finally children.foreach(_.terminate()) - } - - override def traceVerbose[T](name: String, tags: (String, String)*)(thunk: BraveTracer => T): T = - trace(name, tags: _*)(thunk) - - override def traceTask[T](name: String, tags: (String, String)*)( - thunk: BraveTracer => Task[T] - ): Task[T] = { - val children = tracers.map(_.startNewChildTracer(name, tags: _*)) - val compositeChild = CompositeTracer(children: _*) - thunk(compositeChild).doOnFinish(_ => Task.eval(children.foreach(_.terminate()))) - } - - override def traceTaskVerbose[T](name: String, tags: (String, String)*)( - thunk: BraveTracer => Task[T] - ): Task[T] = - traceTask(name, tags: _*)(thunk) - - override def terminate(): Unit = tracers.foreach(_.terminate()) - override def currentSpan: Option[Span] = tracers.flatMap(_.currentSpan).headOption - override def toIndependentTracer( - name: String, - traceProperties: TraceProperties, - tags: (String, String)* - ): BraveTracer = - CompositeTracer(tracers.map(_.toIndependentTracer(name, traceProperties, tags: _*)): _*) -} diff --git a/backend/src/main/scala/bloop/tracing/CompilationTrace.scala b/backend/src/main/scala/bloop/tracing/CompilationTrace.scala index b8a0a0a302..c83dadc412 100644 --- a/backend/src/main/scala/bloop/tracing/CompilationTrace.scala +++ b/backend/src/main/scala/bloop/tracing/CompilationTrace.scala @@ -17,6 +17,8 @@ object CompilationTrace { JsonCodecMaker.make[CompilationTrace] implicit val listCodec: JsonValueCodec[List[CompilationTrace]] = JsonCodecMaker.make[List[CompilationTrace]] + implicit val diagnosticCodec: JsonValueCodec[TraceDiagnostic] = + JsonCodecMaker.make[TraceDiagnostic] } case class TraceDiagnostic( diff --git a/frontend/src/it/scala/bloop/CommunityBuild.scala b/frontend/src/it/scala/bloop/CommunityBuild.scala index 1953f63fea..12f2d3eb98 100644 --- a/frontend/src/it/scala/bloop/CommunityBuild.scala +++ b/frontend/src/it/scala/bloop/CommunityBuild.scala @@ -130,6 +130,7 @@ abstract class CommunityBuild(val buildpressHomeDir: AbsolutePath) { scalaInstance = allProjectsInBuild.head.project.scalaInstance, rawClasspath = Nil, resources = Nil, + resourceMappings = Nil, compileSetup = Config.CompileSetup.empty, genericClassesDir = dummyClassesDir, isBestEffort = false, diff --git a/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala b/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala index 2a822ffc4d..26cdd7579e 100644 --- a/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala +++ b/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala @@ -76,6 +76,7 @@ object CompileTask { traceProperties, "bloop.version" -> BuildInfo.version, "zinc.version" -> BuildInfo.zincVersion, + "workspace.dir" -> cwd.syntax, "build.uri" -> originUri.syntax, "compile.target" -> topLevelTargets, "client" -> clientName @@ -86,6 +87,7 @@ object CompileTask { traceProperties, "bloop.version" -> BuildInfo.version, "zinc.version" -> BuildInfo.zincVersion, + "workspace.dir" -> cwd.syntax, "build.uri" -> originUri.syntax, "compile.target" -> topLevelTargets, "client" -> clientName @@ -296,8 +298,22 @@ object CompileTask { compileProjectTracer.tag(s"file.$idx", file.toPath.toString) } + reportDiagnostics(compileProjectTracer, s.reporter.allProblems) + ResultBundle(s, Some(newSuccessful), Some(lastSuccessful), runningTasks) case f: Compiler.Result.Failed => + compileProjectTracer.tag("success", "false") + + val sources = bundle.uniqueInputs.sources + compileProjectTracer.tag("fileCount", sources.length.toString) + sources.zipWithIndex.foreach { + case (file, idx) => + compileProjectTracer.tag(s"file.$idx", file.toPath.toString) + } + + val problems = f.problems.map(_.problem) + reportDiagnostics(compileProjectTracer, problems) + val runningTasks = runPostCompilationTasks(f.backgroundTasks) ResultBundle(result, None, Some(lastSuccessful), runningTasks) case c: Compiler.Result.Cancelled => @@ -616,4 +632,45 @@ object CompileTask { ) .map(_ => ()) } + + private def reportDiagnostics( + tracer: BraveTracer, + problems: Seq[xsbti.Problem] + ): Unit = { + import bloop.tracing.TraceDiagnostic + import bloop.tracing.TraceRange + import com.github.plokhotnyuk.jsoniter_scala.core.writeToString + + def toOption[T](opt: java.util.Optional[T]): Option[T] = + if (opt.isPresent) Some(opt.get) else None + + tracer.tag("diagnostics.count", problems.length.toString) + problems.zipWithIndex.foreach { + case (p, idx) => + val range = toOption(p.position.startLine).map { startLine => + TraceRange( + startLine.intValue(), + toOption(p.position.startColumn).map(_.intValue()).getOrElse(0), + toOption(p.position.endLine).map(_.intValue()).getOrElse(startLine.intValue()), + toOption(p.position.endColumn).map(_.intValue()).getOrElse(0) + ) + } + + val diagnostic = TraceDiagnostic( + p.severity.toString, + p.message, + range, + toOption(p.diagnosticCode).map(_.code), + toOption(p.position.sourcePath) + ) + + try { + val json = writeToString(diagnostic)(bloop.tracing.CompilationTrace.diagnosticCodec) + tracer.tag(s"diagnostic.$idx", json) + } catch { + case scala.util.control.NonFatal(e) => + tracer.tag(s"diagnostic.$idx.error", e.getMessage) + } + } + } } diff --git a/frontend/src/test/resources/source-generator.py b/frontend/src/test/resources/source-generator.py index e7042e3db8..bba834dabf 100644 --- a/frontend/src/test/resources/source-generator.py +++ b/frontend/src/test/resources/source-generator.py @@ -62,3 +62,7 @@ def main(output_dir, args): def random(): return 123 + +def random(): + return 123 + diff --git a/frontend/src/test/scala/bloop/CompilationTraceSpec.scala b/frontend/src/test/scala/bloop/CompilationTraceSpec.scala index 5ecc0d19af..6517aea056 100644 --- a/frontend/src/test/scala/bloop/CompilationTraceSpec.scala +++ b/frontend/src/test/scala/bloop/CompilationTraceSpec.scala @@ -1,6 +1,5 @@ package bloop -import bloop.task.Task import bloop.util.TestUtil import bloop.logging.RecordingLogger import bloop.data.WorkspaceSettings @@ -37,19 +36,18 @@ object CompilationTraceSpec extends BaseCompileSpec { val configDir = workspace.resolve(".bloop") if (!Files.exists(configDir.underlying)) Files.createDirectories(configDir.underlying) WorkspaceSettings.writeToFile(configDir, settings, logger) + Files.createFile(workspace.resolve("compilation-trace.json").underlying) val state = loadState(workspace, projects, logger) val compiledState = state.compile(`A`) assertExitStatus(compiledState, ExitStatus.Ok) - val traceFile = workspace.resolve(".bloop/compilation-trace.json") + val traceFile = workspace.resolve("compilation-trace-a.json") assert(Files.exists(traceFile.underlying)) val bytes = Files.readAllBytes(traceFile.underlying) - val traces = readFromArray[List[CompilationTrace]](bytes)(CompilationTrace.listCodec) + val trace = readFromArray[CompilationTrace](bytes)(CompilationTrace.codec) - assert(traces.size == 1) - val trace = traces.head assert(trace.project == "a") assert(trace.files.exists(_.endsWith("Foo.scala"))) assert(trace.diagnostics.isEmpty) @@ -82,19 +80,18 @@ object CompilationTraceSpec extends BaseCompileSpec { val configDir = workspace.resolve(".bloop") if (!Files.exists(configDir.underlying)) Files.createDirectories(configDir.underlying) WorkspaceSettings.writeToFile(configDir, settings, logger) + Files.createFile(workspace.resolve("compilation-trace.json").underlying) val state = loadState(workspace, projects, logger) val compiledState = state.compile(`A`) assertExitStatus(compiledState, ExitStatus.CompilationError) - val traceFile = workspace.resolve(".bloop/compilation-trace.json") + val traceFile = workspace.resolve("compilation-trace-a.json") assert(Files.exists(traceFile.underlying)) val bytes = Files.readAllBytes(traceFile.underlying) - val traces = readFromArray[List[CompilationTrace]](bytes)(CompilationTrace.listCodec) + val trace = readFromArray[CompilationTrace](bytes)(CompilationTrace.codec) - assert(traces.size == 1) - val trace = traces.head assert(trace.diagnostics.nonEmpty) assert(trace.diagnostics.exists(_.message.contains("type mismatch"))) } @@ -123,6 +120,7 @@ object CompilationTraceSpec extends BaseCompileSpec { val configDir = workspace.resolve(".bloop") if (!Files.exists(configDir.underlying)) Files.createDirectories(configDir.underlying) WorkspaceSettings.writeToFile(configDir, settings, logger) + Files.createFile(workspace.resolve("compilation-trace.json").underlying) val state = loadState(workspace, projects, logger) val compiledState = state.compile(`A`) @@ -131,13 +129,11 @@ object CompilationTraceSpec extends BaseCompileSpec { val secondCompiledState = compiledState.compile(`A`) assertExitStatus(secondCompiledState, ExitStatus.Ok) - val traceFile = workspace.resolve(".bloop/compilation-trace.json") + val traceFile = workspace.resolve("compilation-trace-a.json") val bytes = Files.readAllBytes(traceFile.underlying) - val traces = readFromArray[List[CompilationTrace]](bytes)(CompilationTrace.listCodec) + val trace = readFromArray[CompilationTrace](bytes)(CompilationTrace.codec) // Since we overwrite, it should be the trace of the last compilation (no-op) - assert(traces.size == 1) - val trace = traces.head assert(trace.isNoOp) } } diff --git a/frontend/src/test/scala/bloop/io/ResourceMapperSpec.scala b/frontend/src/test/scala/bloop/io/ResourceMapperSpec.scala deleted file mode 100644 index d847f6c087..0000000000 --- a/frontend/src/test/scala/bloop/io/ResourceMapperSpec.scala +++ /dev/null @@ -1,150 +0,0 @@ -package bloop.io - -import java.nio.file.Files - -import bloop.logging.RecordingLogger -import bloop.util.TestUtil - -object ResourceMapperSpec extends bloop.testing.BaseSuite { - - test("copy single file mapping") { - TestUtil.withinWorkspace { workspace => - val logger = new RecordingLogger() - - // Create source file - val sourceFile = workspace.resolve("source.txt") - Files.write(sourceFile.underlying, "test content".getBytes("UTF-8")) - - // Define mapping - val mappings = List((sourceFile, "custom/path/file.txt")) - - // Copy to target directory - val targetDir = workspace.resolve("target") - Files.createDirectories(targetDir.underlying) - - val task = ResourceMapper.copyMappedResources(mappings, targetDir, logger) - TestUtil.await(5, java.util.concurrent.TimeUnit.SECONDS)(task) - - // Verify file was copied - val copiedFile = targetDir.resolve("custom/path/file.txt") - assert(copiedFile.exists) - assert(copiedFile.isFile) - - val content = new String(Files.readAllBytes(copiedFile.underlying), "UTF-8") - assertNoDiff(content, "test content") - } - } - - test("copy directory mapping recursively") { - TestUtil.withinWorkspace { workspace => - val logger = new RecordingLogger() - - // Create source directory with files - val sourceDir = workspace.resolve("source-dir") - Files.createDirectories(sourceDir.underlying) - - val file1 = sourceDir.resolve("file1.txt") - val file2 = sourceDir.resolve("subdir/file2.txt") - Files.createDirectories(file2.getParent.underlying) - - Files.write(file1.underlying, "content 1".getBytes("UTF-8")) - Files.write(file2.underlying, "content 2".getBytes("UTF-8")) - - // Define mapping - val mappings = List((sourceDir, "data")) - - // Copy to target directory - val targetDir = workspace.resolve("target") - Files.createDirectories(targetDir.underlying) - - val task = ResourceMapper.copyMappedResources(mappings, targetDir, logger) - TestUtil.await(5, java.util.concurrent.TimeUnit.SECONDS)(task) - - // Verify directory structure was copied - val copiedFile1 = targetDir.resolve("data/file1.txt") - val copiedFile2 = targetDir.resolve("data/subdir/file2.txt") - - assert(copiedFile1.exists) - assert(copiedFile2.exists) - - val content1 = new String(Files.readAllBytes(copiedFile1.underlying), "UTF-8") - val content2 = new String(Files.readAllBytes(copiedFile2.underlying), "UTF-8") - - assertNoDiff(content1, "content 1") - assertNoDiff(content2, "content 2") - } - } - - test("handle empty mappings gracefully") { - TestUtil.withinWorkspace { workspace => - val logger = new RecordingLogger() - - val targetDir = workspace.resolve("target") - Files.createDirectories(targetDir.underlying) - - val task = ResourceMapper.copyMappedResources(List.empty, targetDir, logger) - TestUtil.await(5, java.util.concurrent.TimeUnit.SECONDS)(task) - - // Should complete without error - assert(logger.errors.isEmpty) - } - } - - test("handle non-existent source file") { - TestUtil.withinWorkspace { workspace => - val logger = new RecordingLogger() - - val nonExistentFile = workspace.resolve("does-not-exist.txt") - val mappings = List((nonExistentFile, "output.txt")) - - val targetDir = workspace.resolve("target") - Files.createDirectories(targetDir.underlying) - - val task = ResourceMapper.copyMappedResources(mappings, targetDir, logger) - TestUtil.await(5, java.util.concurrent.TimeUnit.SECONDS)(task) - - // Should warn about missing source - assert(logger.warnings.exists(_.contains("does not exist"))) - - // Target file should not be created - val targetFile = targetDir.resolve("output.txt") - assert(!targetFile.exists) - } - } - - test("detect changes in mapped files") { - TestUtil.withinWorkspace { workspace => - val file = workspace.resolve("file.txt") - Files.write(file.underlying, "original content".getBytes("UTF-8")) - - val oldTimestamp = System.currentTimeMillis() - 10000 // 10 seconds ago - val mappings = List((file, "output.txt")) - - // File is newer than timestamp - val changed = ResourceMapper.hasMappingsChanged(mappings, oldTimestamp) - assert(changed) - - // File is older than timestamp - val futureTimestamp = System.currentTimeMillis() + 10000 // 10 seconds in future - val notChanged = ResourceMapper.hasMappingsChanged(mappings, futureTimestamp) - assert(!notChanged) - } - } - - test("detect changes in mapped directories") { - TestUtil.withinWorkspace { workspace => - val dir = workspace.resolve("source-dir") - Files.createDirectories(dir.underlying) - - val file = dir.resolve("file.txt") - Files.write(file.underlying, "content".getBytes("UTF-8")) - - val oldTimestamp = System.currentTimeMillis() - 10000 // 10 seconds ago - val mappings = List((dir, "data")) - - // Directory contains file newer than timestamp - val changed = ResourceMapper.hasMappingsChanged(mappings, oldTimestamp) - assert(changed) - } - } -}