diff --git a/backend/src/main/scala/bloop/Compiler.scala b/backend/src/main/scala/bloop/Compiler.scala index d9b836b365..b3e3ac333c 100644 --- a/backend/src/main/scala/bloop/Compiler.scala +++ b/backend/src/main/scala/bloop/Compiler.scala @@ -74,7 +74,8 @@ case class CompileInputs( ioExecutor: Executor, invalidatedClassFilesInDependentProjects: Set[File], generatedClassFilePathsInDependentProjects: Map[String, File], - resources: List[AbsolutePath] + resources: List[AbsolutePath], + resourceMappings: List[(AbsolutePath, String)] ) case class CompileOutPaths( @@ -828,6 +829,11 @@ object Compiler { compileInputs.logger, compileInputs.ioScheduler ) + val copyMappedResources = bloop.io.ResourceMapper.copyMappedResources( + compileInputs.resourceMappings, + clientClassesDir, + compileInputs.logger + ) val lastCopy = ParallelOps.copyDirectories(config)( readOnlyClassesDir, clientClassesDir.underlying, @@ -836,7 +842,7 @@ object Compiler { compileInputs.logger ) - Task.gatherUnordered(List(copyResources, lastCopy)).map { _ => + Task.gatherUnordered(List(copyResources, copyMappedResources, lastCopy)).map { _ => clientLogger.debug( s"Finished copying classes from $readOnlyClassesDir to $clientClassesDir" ) diff --git a/backend/src/main/scala/bloop/io/ResourceMapper.scala b/backend/src/main/scala/bloop/io/ResourceMapper.scala new file mode 100644 index 0000000000..28076a1089 --- /dev/null +++ b/backend/src/main/scala/bloop/io/ResourceMapper.scala @@ -0,0 +1,114 @@ +package bloop.io + +import java.nio.file.Files +import java.nio.file.StandardCopyOption + +import scala.util.control.NonFatal + +import bloop.logging.DebugFilter +import bloop.logging.Logger +import bloop.task.Task + +/** + * Utility for handling resource file mappings. + * + * Resource mappings allow files to be copied from source locations to custom + * target paths, similar to SBT's "mappings" field. + */ +object ResourceMapper { + private implicit val filter: DebugFilter.All.type = DebugFilter.All + + /** + * Copy mapped resources to the target directory. + * + * @param mappings List of (source, targetRelativePath) tuples + * @param classesDir Base directory where resources should be copied + * @param logger Logger for debug/error messages + * @return Task that completes when all resources are copied + */ + def copyMappedResources( + mappings: List[(AbsolutePath, String)], + classesDir: AbsolutePath, + logger: Logger + ): Task[Unit] = { + val tasks = mappings.map { + case (source, targetRelPath) => + Task { + val target = classesDir.resolve(targetRelPath) + if (!target.getParent.exists) { + Files.createDirectories(target.getParent.underlying) + } + + if (source.isDirectory) { + import java.nio.file.FileVisitResult + import java.nio.file.Path + import java.nio.file.SimpleFileVisitor + import java.nio.file.attribute.BasicFileAttributes + + val sourcePath = source.underlying + val targetPath = target.underlying + Files.walkFileTree( + sourcePath, + new SimpleFileVisitor[Path] { + override def visitFile( + file: Path, + attrs: BasicFileAttributes + ): FileVisitResult = { + val relPath = sourcePath.relativize(file) + val targetFile = targetPath.resolve(relPath) + Files.createDirectories(targetFile.getParent) + Files.copy(file, targetFile, StandardCopyOption.REPLACE_EXISTING) + FileVisitResult.CONTINUE + } + } + ) + } else if (source.exists) { + Files.copy(source.underlying, target.underlying, StandardCopyOption.REPLACE_EXISTING) + } else { + logger.warn(s"Source file $source does not exist, skipping mapping to $targetRelPath") + } + () + } + } + Task.gatherUnordered(tasks).map(_ => ()) + } + + /** + * Check if any mapped resources have changed since the given timestamp. + * + * Note: This is currently not used. Incremental compilation change detection + * for resource mappings would require integration with Zinc's analysis. + * For now, mapped resources are always copied during compilation. + * + * @param mappings List of (source, targetRelativePath) tuples + * @param lastModified Timestamp to compare against + * @return true if any source file is newer than lastModified + */ + private[bloop] def hasMappingsChanged( + mappings: List[(AbsolutePath, String)], + lastModified: Long + ): Boolean = { + import java.nio.file.Files + mappings.exists { + case (source, _) => + if (source.isDirectory) { + hasDirectoryChanged(source, lastModified) + } else { + source.exists && Files.getLastModifiedTime(source.underlying).toMillis > lastModified + } + } + } + + private def hasDirectoryChanged(dir: AbsolutePath, lastModified: Long): Boolean = { + if (dir.exists) { + val stream = Files.walk(dir.underlying) + try { + stream.anyMatch(path => Files.getLastModifiedTime(path).toMillis > lastModified) + } finally { + stream.close() + } + } else { + false + } + } +} diff --git a/frontend/src/main/scala/bloop/data/Project.scala b/frontend/src/main/scala/bloop/data/Project.scala index daa6517014..4f85bb9e4f 100644 --- a/frontend/src/main/scala/bloop/data/Project.scala +++ b/frontend/src/main/scala/bloop/data/Project.scala @@ -39,6 +39,7 @@ final case class Project( scalaInstance: Option[ScalaInstance], rawClasspath: List[AbsolutePath], resources: List[AbsolutePath], + resourceMappings: List[(AbsolutePath, String)], compileSetup: Config.CompileSetup, genericClassesDir: AbsolutePath, isBestEffort: Boolean, @@ -347,6 +348,8 @@ object Project { val tags = project.tags.getOrElse(Nil) val projectDirectory = AbsolutePath(project.directory) + val resourceMappings = List.empty[(AbsolutePath, String)] + Project( project.name, projectDirectory, @@ -355,6 +358,7 @@ object Project { instance, compileClasspath, compileResources, + resourceMappings, setup, AbsolutePath(project.classesDir), isBestEffort = false, diff --git a/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala b/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala index dac5807179..4727936305 100644 --- a/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala +++ b/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala @@ -123,12 +123,23 @@ object CompileTask { logger, ExecutionContext.ioScheduler ) + + // Also copy mapped resources if any + val copyMappedResourcesTask: Task[Unit] = + bloop.io.ResourceMapper.copyMappedResources( + project.resourceMappings, + bundle.clientClassesObserver.classesDir, + logger + ) + + val allCopyTasks = + Task.gatherUnordered(List(copyResourcesTask, copyMappedResourcesTask)) Task.now( ResultBundle( Compiler.Result.Empty, None, None, - copyResourcesTask.runAsync(ExecutionContext.ioScheduler) + allCopyTasks.map(_ => ()).runAsync(ExecutionContext.ioScheduler) ) ) case Right(CompileSourcesAndInstance(sources, instance, _)) => @@ -184,7 +195,8 @@ object CompileTask { ExecutionContext.ioExecutor, bundle.dependenciesData.allInvalidatedClassFiles, bundle.dependenciesData.allGeneratedClassFilePaths, - project.runtimeResources + project.runtimeResources, + project.resourceMappings ) } @@ -245,8 +257,14 @@ object CompileTask { compileProjectTracer, logger ) - .doOnFinish(_ => Task(compileProjectTracer.terminate())) - postCompilationTasks.runAsync(ExecutionContext.ioScheduler) + + // Copy mapped resources after compilation + val allTasks = Task + .gatherUnordered(List(postCompilationTasks)) + .map(_ => ()) + .doOnFinish(_ => Task(compileProjectTracer.terminate())) + + allTasks.runAsync(ExecutionContext.ioScheduler) } // Populate the last successful result if result was success diff --git a/frontend/src/test/scala/bloop/DagSpec.scala b/frontend/src/test/scala/bloop/DagSpec.scala index cb85672860..351113e623 100644 --- a/frontend/src/test/scala/bloop/DagSpec.scala +++ b/frontend/src/test/scala/bloop/DagSpec.scala @@ -27,7 +27,7 @@ class DagSpec { // format: OFF def dummyOrigin: Origin = TestUtil.syntheticOriginFor(dummyPath) def dummyProject(name: String, dependencies: List[String]): Project = - Project(name, dummyPath, None, dependencies, Some(dummyInstance), Nil, Nil, compileOptions, + Project(name, dummyPath, None, dependencies, Some(dummyInstance), Nil, Nil, Nil, compileOptions, dummyPath, isBestEffort = false, Nil, Nil, Nil, Nil, None, Nil, Nil, Config.TestOptions.empty, dummyPath, dummyPath, Project.defaultPlatform(logger, Nil, Nil), None, None, Nil, dummyOrigin) // format: ON diff --git a/frontend/src/test/scala/bloop/ResourceMappingIntegrationSpec.scala b/frontend/src/test/scala/bloop/ResourceMappingIntegrationSpec.scala new file mode 100644 index 0000000000..9a9a69bb41 --- /dev/null +++ b/frontend/src/test/scala/bloop/ResourceMappingIntegrationSpec.scala @@ -0,0 +1,172 @@ +package bloop + +import java.nio.file.Files +import bloop.cli.ExitStatus +import bloop.io.AbsolutePath +import bloop.logging.RecordingLogger +import bloop.util.TestProject +import bloop.util.TestUtil +import bloop.data.LoadedProject + +object ResourceMappingIntegrationSpec extends bloop.testing.BaseSuite { + + test("compile with resource mappings copies files to classes directory") { + TestUtil.withinWorkspace { workspace => + val logger = new RecordingLogger(ansiCodesSupported = false) + + // Create a simple project + val source = + """/main/scala/Foo.scala + |class Foo + """.stripMargin + + val `A` = TestProject(workspace, "a", List(source)) + val projects = List(`A`) + val state = loadState(workspace, projects, logger) + + // Create a resource file to map + val resourceFile = workspace.resolve("custom-resource.txt") + Files.write(resourceFile.underlying, "resource content".getBytes("UTF-8")) + + // Get the project and inject mappings manually + val projectA = state.getProjectFor(`A`) + val mappings = List((resourceFile, "assets/config.txt")) + val projectWithMappings = projectA.copy(resourceMappings = mappings) + + // Update state with modified project + val stateWithMappings = new TestState( + state.state.copy( + build = state.state.build.copy( + loadedProjects = state.state.build.loadedProjects.map { lp => + if (lp.project.name == projectA.name) { + lp match { + case LoadedProject.RawProject(_) => LoadedProject.RawProject(projectWithMappings) + case LoadedProject.ConfiguredProject(_, original, settings) => + LoadedProject.ConfiguredProject(projectWithMappings, original, settings) + } + } else lp + } + ) + ) + ) + + // Compile + val compiledState = stateWithMappings.compile(`A`) + assertExitStatus(compiledState, ExitStatus.Ok) + + // Verify resource was copied to classes directory + val classesDir = compiledState.getClientExternalDir(`A`) + val copiedResource = classesDir.resolve("assets/config.txt") + + try { + assert(copiedResource.exists) + val content = new String(Files.readAllBytes(copiedResource.underlying), "UTF-8") + assertNoDiff(content, "resource content") + } catch { + case t: Throwable => + logger.dump() + throw t + } + } + } + + test("compile with directory mapping copies entire directory") { + TestUtil.withinWorkspace { workspace => + val logger = new RecordingLogger(ansiCodesSupported = false) + + val `A` = TestProject(workspace, "a", List("/main/scala/Foo.scala\nclass Foo")) + val projects = List(`A`) + val state = loadState(workspace, projects, logger) + + // Create a directory structure to map + val resourceDir = workspace.resolve("resources") + Files.createDirectories(resourceDir.underlying) + Files.write(resourceDir.resolve("file1.txt").underlying, "content 1".getBytes("UTF-8")) + val subDir = resourceDir.resolve("sub") + Files.createDirectories(subDir.underlying) + Files.write(subDir.resolve("file2.txt").underlying, "content 2".getBytes("UTF-8")) + + // Inject mappings + val projectA = state.getProjectFor(`A`) + val mappings = List((resourceDir, "data")) + val projectWithMappings = projectA.copy(resourceMappings = mappings) + + val stateWithMappings = new TestState( + state.state.copy( + build = state.state.build.copy( + loadedProjects = state.state.build.loadedProjects.map { lp => + if (lp.project.name == projectA.name) { + lp match { + case LoadedProject.RawProject(_) => LoadedProject.RawProject(projectWithMappings) + case LoadedProject.ConfiguredProject(_, original, settings) => + LoadedProject.ConfiguredProject(projectWithMappings, original, settings) + } + } else lp + } + ) + ) + ) + + // Compile + val compiledState = stateWithMappings.compile(`A`) + assertExitStatus(compiledState, ExitStatus.Ok) + + // Verify directory structure copied + val classesDir = compiledState.getClientExternalDir(`A`) + assert(classesDir.resolve("data/file1.txt").exists) + assert(classesDir.resolve("data/sub/file2.txt").exists) + } + } + + test("resources are copied even if compilation is no-op") { + TestUtil.withinWorkspace { workspace => + val logger = new RecordingLogger(ansiCodesSupported = false) + + val `A` = TestProject(workspace, "a", List("/main/scala/Foo.scala\nclass Foo")) + val projects = List(`A`) + val state = loadState(workspace, projects, logger) + + val resourceFile = workspace.resolve("res.txt") + Files.write(resourceFile.underlying, "v1".getBytes("UTF-8")) + + val projectA = state.getProjectFor(`A`) + val mappings = List((resourceFile, "res.txt")) + val projectWithMappings = projectA.copy(resourceMappings = mappings) + + val stateWithMappings = new TestState( + state.state.copy( + build = state.state.build.copy( + loadedProjects = state.state.build.loadedProjects.map { lp => + if (lp.project.name == projectA.name) { + lp match { + case LoadedProject.RawProject(_) => LoadedProject.RawProject(projectWithMappings) + case LoadedProject.ConfiguredProject(_, original, settings) => + LoadedProject.ConfiguredProject(projectWithMappings, original, settings) + } + } else lp + } + ) + ) + ) + + // First compile + val compiledState = stateWithMappings.compile(`A`) + assertExitStatus(compiledState, ExitStatus.Ok) + + // Verify v1 + val classesDir = compiledState.getClientExternalDir(`A`) + val copiedFile = classesDir.resolve("res.txt") + assertNoDiff(new String(Files.readAllBytes(copiedFile.underlying)), "v1") + + // Update resource + Files.write(resourceFile.underlying, "v2".getBytes("UTF-8")) + + // Compile again (should be no-op for scala, but resources should update) + val compiledState2 = compiledState.compile(`A`) + assertExitStatus(compiledState2, ExitStatus.Ok) + + // Verify v2 + assertNoDiff(new String(Files.readAllBytes(copiedFile.underlying)), "v2") + } + } +} diff --git a/frontend/src/test/scala/bloop/io/ResourceMapperSpec.scala b/frontend/src/test/scala/bloop/io/ResourceMapperSpec.scala new file mode 100644 index 0000000000..d847f6c087 --- /dev/null +++ b/frontend/src/test/scala/bloop/io/ResourceMapperSpec.scala @@ -0,0 +1,150 @@ +package bloop.io + +import java.nio.file.Files + +import bloop.logging.RecordingLogger +import bloop.util.TestUtil + +object ResourceMapperSpec extends bloop.testing.BaseSuite { + + test("copy single file mapping") { + TestUtil.withinWorkspace { workspace => + val logger = new RecordingLogger() + + // Create source file + val sourceFile = workspace.resolve("source.txt") + Files.write(sourceFile.underlying, "test content".getBytes("UTF-8")) + + // Define mapping + val mappings = List((sourceFile, "custom/path/file.txt")) + + // Copy to target directory + val targetDir = workspace.resolve("target") + Files.createDirectories(targetDir.underlying) + + val task = ResourceMapper.copyMappedResources(mappings, targetDir, logger) + TestUtil.await(5, java.util.concurrent.TimeUnit.SECONDS)(task) + + // Verify file was copied + val copiedFile = targetDir.resolve("custom/path/file.txt") + assert(copiedFile.exists) + assert(copiedFile.isFile) + + val content = new String(Files.readAllBytes(copiedFile.underlying), "UTF-8") + assertNoDiff(content, "test content") + } + } + + test("copy directory mapping recursively") { + TestUtil.withinWorkspace { workspace => + val logger = new RecordingLogger() + + // Create source directory with files + val sourceDir = workspace.resolve("source-dir") + Files.createDirectories(sourceDir.underlying) + + val file1 = sourceDir.resolve("file1.txt") + val file2 = sourceDir.resolve("subdir/file2.txt") + Files.createDirectories(file2.getParent.underlying) + + Files.write(file1.underlying, "content 1".getBytes("UTF-8")) + Files.write(file2.underlying, "content 2".getBytes("UTF-8")) + + // Define mapping + val mappings = List((sourceDir, "data")) + + // Copy to target directory + val targetDir = workspace.resolve("target") + Files.createDirectories(targetDir.underlying) + + val task = ResourceMapper.copyMappedResources(mappings, targetDir, logger) + TestUtil.await(5, java.util.concurrent.TimeUnit.SECONDS)(task) + + // Verify directory structure was copied + val copiedFile1 = targetDir.resolve("data/file1.txt") + val copiedFile2 = targetDir.resolve("data/subdir/file2.txt") + + assert(copiedFile1.exists) + assert(copiedFile2.exists) + + val content1 = new String(Files.readAllBytes(copiedFile1.underlying), "UTF-8") + val content2 = new String(Files.readAllBytes(copiedFile2.underlying), "UTF-8") + + assertNoDiff(content1, "content 1") + assertNoDiff(content2, "content 2") + } + } + + test("handle empty mappings gracefully") { + TestUtil.withinWorkspace { workspace => + val logger = new RecordingLogger() + + val targetDir = workspace.resolve("target") + Files.createDirectories(targetDir.underlying) + + val task = ResourceMapper.copyMappedResources(List.empty, targetDir, logger) + TestUtil.await(5, java.util.concurrent.TimeUnit.SECONDS)(task) + + // Should complete without error + assert(logger.errors.isEmpty) + } + } + + test("handle non-existent source file") { + TestUtil.withinWorkspace { workspace => + val logger = new RecordingLogger() + + val nonExistentFile = workspace.resolve("does-not-exist.txt") + val mappings = List((nonExistentFile, "output.txt")) + + val targetDir = workspace.resolve("target") + Files.createDirectories(targetDir.underlying) + + val task = ResourceMapper.copyMappedResources(mappings, targetDir, logger) + TestUtil.await(5, java.util.concurrent.TimeUnit.SECONDS)(task) + + // Should warn about missing source + assert(logger.warnings.exists(_.contains("does not exist"))) + + // Target file should not be created + val targetFile = targetDir.resolve("output.txt") + assert(!targetFile.exists) + } + } + + test("detect changes in mapped files") { + TestUtil.withinWorkspace { workspace => + val file = workspace.resolve("file.txt") + Files.write(file.underlying, "original content".getBytes("UTF-8")) + + val oldTimestamp = System.currentTimeMillis() - 10000 // 10 seconds ago + val mappings = List((file, "output.txt")) + + // File is newer than timestamp + val changed = ResourceMapper.hasMappingsChanged(mappings, oldTimestamp) + assert(changed) + + // File is older than timestamp + val futureTimestamp = System.currentTimeMillis() + 10000 // 10 seconds in future + val notChanged = ResourceMapper.hasMappingsChanged(mappings, futureTimestamp) + assert(!notChanged) + } + } + + test("detect changes in mapped directories") { + TestUtil.withinWorkspace { workspace => + val dir = workspace.resolve("source-dir") + Files.createDirectories(dir.underlying) + + val file = dir.resolve("file.txt") + Files.write(file.underlying, "content".getBytes("UTF-8")) + + val oldTimestamp = System.currentTimeMillis() - 10000 // 10 seconds ago + val mappings = List((dir, "data")) + + // Directory contains file newer than timestamp + val changed = ResourceMapper.hasMappingsChanged(mappings, oldTimestamp) + assert(changed) + } + } +} diff --git a/frontend/src/test/scala/bloop/util/TestUtil.scala b/frontend/src/test/scala/bloop/util/TestUtil.scala index 58f8645376..4d07c95fb2 100644 --- a/frontend/src/test/scala/bloop/util/TestUtil.scala +++ b/frontend/src/test/scala/bloop/util/TestUtil.scala @@ -411,6 +411,7 @@ object TestUtil { scalaInstance = scalaInstance, rawClasspath = classpath, resources = Nil, + resourceMappings = Nil, isBestEffort = false, compileSetup = Config.CompileSetup.empty.copy(order = compileOrder), genericClassesDir = classes,