From c8c38351f900bd63b7cfbc04c88f092485383e90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Wi=C4=85cek?= Date: Fri, 30 May 2025 17:23:52 +0200 Subject: [PATCH 01/10] Try get rid of CompletableFuture start using TaskApp preliminary changes to get rid of CompletableFuture from Cli and use Deffered isntead and start using TaskApp instead of App One compile test is failing, not sure if end conditions are ok. It would be good to add some tests --- .../scala/bloop/task/ObservableDeferred.scala | 39 ++ build.sbt | 2 + frontend/src/main/scala/bloop/Bloop.scala | 12 +- frontend/src/main/scala/bloop/Cli.scala | 251 +++++--- .../src/test/resources/source-generator.py | 20 + .../test/scala/bloop/BaseCompileSpec.scala | 52 -- .../scala/bloop/MonixBaseCompileSpec.scala | 75 +++ .../test/scala/bloop/MonixCompileSpec.scala | 5 + .../scala/bloop/testing/MonixBaseSuite.scala | 561 ++++++++++++++++++ .../src/test/scala/bloop/util/TestUtil.scala | 13 +- project/Dependencies.scala | 2 + 11 files changed, 870 insertions(+), 162 deletions(-) create mode 100644 backend/src/main/scala/bloop/task/ObservableDeferred.scala create mode 100644 frontend/src/test/scala/bloop/MonixBaseCompileSpec.scala create mode 100644 frontend/src/test/scala/bloop/MonixCompileSpec.scala create mode 100644 frontend/src/test/scala/bloop/testing/MonixBaseSuite.scala diff --git a/backend/src/main/scala/bloop/task/ObservableDeferred.scala b/backend/src/main/scala/bloop/task/ObservableDeferred.scala new file mode 100644 index 0000000000..665fb46182 --- /dev/null +++ b/backend/src/main/scala/bloop/task/ObservableDeferred.scala @@ -0,0 +1,39 @@ +package bloop.task + +import cats.effect.Concurrent +import cats.effect.concurrent.{Deferred, Ref} +import monix.eval.{Task => MonixTask} +import monix.execution.atomic.AtomicBoolean + +class ObservableDeferred[A]( + ref: Deferred[MonixTask, A], + state: Ref[MonixTask, Boolean], + syncFlag: AtomicBoolean +) { + def complete(value: A): MonixTask[Unit] = { + MonixTask.defer { + ref + .complete(value) + .flatMap { _ => + syncFlag.set(true) + state.set(true) + } + }.uncancelable + } + + def isCompletedUnsafe: Boolean = syncFlag.get + + def isCompleted: MonixTask[Boolean] = state.get + + def get: MonixTask[A] = ref.get +} + +object ObservableDeferred { + + def apply[A](implicit F: Concurrent[MonixTask]): MonixTask[ObservableDeferred[A]] = + for { + ref <- Deferred[MonixTask, A] + state <- Ref[MonixTask].of(false) + } yield new ObservableDeferred(ref, state, AtomicBoolean(false)) + +} diff --git a/build.sbt b/build.sbt index effb961215..300bbe277b 100644 --- a/build.sbt +++ b/build.sbt @@ -105,6 +105,7 @@ lazy val backend = project Dependencies.libraryManagement, Dependencies.sourcecode, Dependencies.monix, + Dependencies.monixTest, Dependencies.directoryWatcher, Dependencies.zt, Dependencies.brave, @@ -156,6 +157,7 @@ lazy val frontend: Project = project Dependencies.jsoniterMacros % Provided, Dependencies.scalazCore, Dependencies.monix, + Dependencies.monixTest, Dependencies.caseApp, Dependencies.scalaDebugAdapter, Dependencies.bloopConfig, diff --git a/frontend/src/main/scala/bloop/Bloop.scala b/frontend/src/main/scala/bloop/Bloop.scala index fb687c0451..c378690f02 100644 --- a/frontend/src/main/scala/bloop/Bloop.scala +++ b/frontend/src/main/scala/bloop/Bloop.scala @@ -1,22 +1,14 @@ package bloop import scala.annotation.tailrec - import bloop.cli.CliOptions import bloop.cli.Commands import bloop.cli.ExitStatus import bloop.data.ClientInfo import bloop.data.WorkspaceSettings -import bloop.engine.Build -import bloop.engine.BuildLoader -import bloop.engine.Exit -import bloop.engine.Interpreter -import bloop.engine.NoPool -import bloop.engine.Run -import bloop.engine.State +import bloop.engine.{Build, BuildLoader, ExecutionContext, Exit, Interpreter, NoPool, Run, State} import bloop.io.AbsolutePath import bloop.logging.BloopLogger - import _root_.bloop.task.Task import caseapp.CaseApp import caseapp.RemainingArgs @@ -50,7 +42,7 @@ object Bloop extends CaseApp[CliOptions] { // Ignore the exit status here, all we want is the task to finish execution or fail. Cli.waitUntilEndOfWorld(options, state.pool, config, state.logger) { t.map(s => { State.stateCache.updateBuild(s.copy(status = ExitStatus.Ok)); s.status }) - } + }(ExecutionContext.ioScheduler) // Recover the state if the previous task has been successful. State.stateCache diff --git a/frontend/src/main/scala/bloop/Cli.scala b/frontend/src/main/scala/bloop/Cli.scala index b20b9e7f35..7f9bdeb14d 100644 --- a/frontend/src/main/scala/bloop/Cli.scala +++ b/frontend/src/main/scala/bloop/Cli.scala @@ -3,11 +3,8 @@ package bloop import java.io.InputStream import java.io.PrintStream import java.nio.file.Path -import java.util.concurrent.CompletableFuture import java.util.concurrent.ConcurrentHashMap - import scala.util.control.NonFatal - import bloop.cli.CliOptions import bloop.cli.Commands import bloop.cli.CommonOptions @@ -21,20 +18,26 @@ import bloop.logging.BloopLogger import bloop.logging.DebugFilter import bloop.logging.Logger import bloop.task.Task +import monix.eval.{Task => MonixTask} +import monix.eval.TaskApp import bloop.util.JavaRuntime - import caseapp.core.help.Help +import cats.effect.ExitCode +import cats.effect.concurrent.Deferred import com.martiansoftware.nailgun.NGContext +import monix.execution.Scheduler import monix.execution.atomic.AtomicBoolean +import scala.concurrent.Await +import scala.concurrent.duration.Duration + class Cli -object Cli { +object Cli extends TaskApp { implicit private val filter: DebugFilter.All.type = DebugFilter.All - def main(args: Array[String]): Unit = { - val action = parse(args, CommonOptions.default) - val exitStatus = run(action, NoPool) - sys.exit(exitStatus.code) + def run(args: List[String]): MonixTask[ExitCode] = { + val action = parse(args.toArray, CommonOptions.default) + run(action, NoPool).map(exitStatus => ExitCode(exitStatus.code)) } def reflectMain( @@ -44,8 +47,8 @@ object Cli { out: PrintStream, err: PrintStream, props: java.util.Properties, - cancel: CompletableFuture[java.lang.Boolean] - ): Int = { + cancel: Deferred[MonixTask, Boolean] + ): MonixTask[Int] = { val env = CommonOptions.PrettyProperties.from(props) val nailgunOptions = CommonOptions( in = in, @@ -59,11 +62,11 @@ object Cli { val cmd = parse(args, nailgunOptions) val exitStatus = run(cmd, NoPool, cancel) - exitStatus.code + exitStatus.map(_.code) } def nailMain(ngContext: NGContext): Unit = { - val env = CommonOptions.PrettyProperties.from(ngContext.getEnv()) + val env = CommonOptions.PrettyProperties.from(ngContext.getEnv) val nailgunOptions = CommonOptions( in = ngContext.in, out = ngContext.out, @@ -87,18 +90,21 @@ object Cli { else parse(args, nailgunOptions) } println(nailgunOptions.workingDirectory) + println(nailgunOptions.workingPath) - try { - val exitStatus = run(cmd, NailgunPool(ngContext)) - ngContext.exit(exitStatus.code) - } catch { - case x: java.util.concurrent.ExecutionException => - // print stack trace of fatal errors thrown in asynchronous code, see https://stackoverflow.com/questions/17265022/what-is-a-boxed-error-in-scala - // the stack trace is somehow propagated all the way to the client when printing this - x.getCause.printStackTrace(ngContext.out) - ngContext.exit(ExitStatus.UnexpectedError.code) - } + val handle = run(cmd, NailgunPool(ngContext)) + .onErrorHandle { + case x: java.util.concurrent.ExecutionException => + // print stack trace of fatal errors thrown in asynchronous code, see https://stackoverflow.com/questions/17265022/what-is-a-boxed-error-in-scala + // the stack trace is somehow propagated all the way to the client when printing this + x.getCause.printStackTrace(ngContext.out) + ExitStatus.UnexpectedError.code + } + .runToFuture(ExecutionContext.ioScheduler) + + Await.result(handle, Duration.Inf) + () } val commands: Seq[String] = Commands.RawCommand.help.messages.flatMap(_._1.headOption.toSeq) @@ -298,8 +304,13 @@ object Cli { } } - def run(action: Action, pool: ClientPool): ExitStatus = { - run(action, pool, FalseCancellation) + def run(action: Action, pool: ClientPool): MonixTask[ExitStatus] = { + + for { + baseCancellation <- Deferred[MonixTask, Boolean] + _ <- baseCancellation.complete(false) + result <- run(action, pool, baseCancellation) + } yield result } // Attempt to load JDI when we initialize the CLI class @@ -307,8 +318,8 @@ object Cli { private def run( action: Action, pool: ClientPool, - cancel: CompletableFuture[java.lang.Boolean] - ): ExitStatus = { + cancel: Deferred[MonixTask, Boolean] + ): MonixTask[ExitStatus] = { import bloop.io.AbsolutePath def getConfigDir(cliOptions: CliOptions): AbsolutePath = { val cwd = AbsolutePath(cliOptions.common.workingDirectory) @@ -343,21 +354,29 @@ object Cli { action match { case Print(msg, _, Exit(exitStatus)) => logger.info(msg) - exitStatus + MonixTask.now(exitStatus) case _ => - runWithState(action, pool, cancel, configDirectory, cliOptions, commonOpts, logger) + runWithState( + action, + pool, + cancel, + configDirectory, + cliOptions, + commonOpts, + logger + ) } } private def runWithState( action: Action, pool: ClientPool, - cancel: CompletableFuture[java.lang.Boolean], + cancel: Deferred[MonixTask, Boolean], configDirectory: AbsolutePath, cliOptions: CliOptions, commonOpts: CommonOptions, logger: Logger - ): ExitStatus = { + ): MonixTask[ExitStatus] = { // Set the proxy settings right before loading the state of the build bloop.util.ProxySetup.updateProxySettings(commonOpts.env.toMap, logger) @@ -366,7 +385,7 @@ object Cli { waitUntilEndOfWorld(cliOptions, pool, configDir, logger, cancel) { val taskToInterpret = { (cli: CliClientInfo) => val state = State.loadActiveStateFor(configDirectory, cli, pool, cliOptions.common, logger) - Interpreter.execute(action, state).map { newState => + val interpret = Interpreter.execute(action, state).map { newState => action match { case Run(_: Commands.ValidatedBsp, _) => () // Ignore, BSP services auto-update the build @@ -375,10 +394,11 @@ object Cli { newState } + interpret.toMonixTask(ExecutionContext.scheduler) } val session = runTaskWithCliClient(configDirectory, action, taskToInterpret, pool, logger) - val exitSession = Task.defer { + val exitSession = MonixTask.defer { cleanUpNonStableCliDirectories(session.client) } @@ -388,16 +408,13 @@ object Cli { } } - private final val FalseCancellation = - CompletableFuture.completedFuture[java.lang.Boolean](false) - private val activeCliSessions = new ConcurrentHashMap[Path, List[CliSession]]() - case class CliSession(client: CliClientInfo, task: Task[ExitStatus]) + case class CliSession(client: CliClientInfo, task: MonixTask[ExitStatus]) def runTaskWithCliClient( configDir: AbsolutePath, action: Action, - processCliTask: CliClientInfo => Task[State], + processCliTask: CliClientInfo => MonixTask[State], pool: ClientPool, logger: Logger ): CliSession = { @@ -439,37 +456,55 @@ object Cli { def cleanUpNonStableCliDirectories( client: CliClientInfo - ): Task[Unit] = { - if (client.useStableCliDirs) Task.unit + ): MonixTask[Unit] = { + if (client.useStableCliDirs) MonixTask.unit else { val deleteTasks = client.getCreatedCliDirectories.map { freshDir => - if (!freshDir.exists) Task.unit + if (!freshDir.exists) MonixTask.unit else { - Task.eval(Paths.delete(freshDir)).asyncBoundary + MonixTask.eval(Paths.delete(freshDir)).asyncBoundary } } val groups = deleteTasks .grouped(4) - .map(group => Task.gatherUnordered(group).map(_ => ())) + .map(group => MonixTask.parSequenceUnordered(group).map(_ => ())) .toList - Task + MonixTask .sequence(groups) .map(_ => ()) .executeOn(ExecutionContext.ioScheduler) } } - import scala.concurrent.Await - import scala.concurrent.duration.Duration + private[bloop] def waitUntilEndOfWorld( + cliOptions: CliOptions, + pool: ClientPool, + configDirectory: Path, + logger: Logger + )(task: Task[ExitStatus])(implicit s: Scheduler): ExitStatus = { + val handler = + for { + cancel <- Deferred[MonixTask, Boolean] + exitStatus <- waitUntilEndOfWorld(cliOptions, pool, configDirectory, logger, cancel)( + task.toMonixTask + ) + } yield exitStatus + Await.result( + handler.runToFuture(ExecutionContext.ioScheduler), + Duration.Inf + ) + } + private[bloop] def waitUntilEndOfWorld( cliOptions: CliOptions, pool: ClientPool, configDirectory: Path, logger: Logger, - cancel: CompletableFuture[java.lang.Boolean] = FalseCancellation - )(task: Task[ExitStatus]): ExitStatus = { + cancel: Deferred[MonixTask, Boolean] + )(task: MonixTask[ExitStatus]): MonixTask[ExitStatus] = { + val ngout = cliOptions.common.ngout def logElapsed(since: Long): Unit = { val elapsed = (System.nanoTime() - since).toDouble / 1e6 @@ -477,58 +512,80 @@ object Cli { } // Simulate try-catch-finally with monix tasks to time the task execution - val handle = - Task - .now(System.nanoTime()) - .flatMap(start => task.materialize.map(s => (s, start))) - .map { case (state, start) => logElapsed(start); state } - .dematerialize - .runAsync(ExecutionContext.scheduler) - - if (!cancel.isDone) { - // Add support for a client to cancel bloop via Java's completable future - import bloop.util.Java8Compat.JavaCompletableFutureUtils - Task - .deferFutureAction(cancel.asScala(_)) - .map { cancel => - if (cancel) { - cliOptions.common.out.println( - s"Client in $configDirectory triggered cancellation. Cancelling tasks..." - ) - handle.cancel() - } + val handle: MonixTask[ExitStatus] = + for { + start <- MonixTask.now(System.nanoTime()) + tryState <- task.materialize + _ = logElapsed(start) + state <- MonixTask.fromTry(tryState) + } yield state + + def waitForCanceled: MonixTask[ExitStatus] = + for { + isCanceled <- cancel.get + status <- + if (isCanceled) + MonixTask + .now { + cliOptions.common.out.println( + s"Client in $configDirectory triggered cancellation. Cancelling tasks..." + ) + } + .map(_ => ExitStatus.UnexpectedError) + else MonixTask.now(ExitStatus.Ok) + } yield status + + def handleException(t: Throwable, completed: Boolean): MonixTask[ExitStatus] = + for { + _ <- if (completed) cancel.complete(completed) else MonixTask.unit + } yield { + logger.error(s"Caught $t") + logger.trace(t) + ExitStatus.UnexpectedError + } + // Let's cancel tasks (if supported by the underlying implementation) when clients disconnect + def registerListener( + cancelSignal: Deferred[MonixTask, Boolean] + )(implicit s: Scheduler): MonixTask[Unit] = { + MonixTask { + pool.addListener { e: CloseEvent => + MonixTask + .defer { + cancelSignal + .complete(true) + .attempt + .map { + case Right(()) => + ngout.println( + s"Client in $configDirectory disconnected with a '$e' event. Cancelling tasks..." + ) + case Left(_) => + ngout.println( + s"Client in $configDirectory disconnected with a '$e' event. Cancelling tasks..." + ) + } + } + .uncancelable // TODO: do we need it? + .runAsyncAndForget } - .runAsync(ExecutionContext.ioScheduler) - } - - def handleException(t: Throwable) = { - handle.cancel() - if (!cancel.isDone) - cancel.complete(false) - logger.error(s"Caught $t") - logger.trace(t) - ExitStatus.UnexpectedError + } } - try { - // Let's cancel tasks (if supported by the underlying implementation) when clients disconnect - pool.addListener { - case e: CloseEvent => - if (!handle.isCompleted) { - ngout.println( - s"Client in $configDirectory disconnected with a '$e' event. Cancelling tasks..." - ) - handle.cancel() - if (!cancel.isDone) - cancel.complete(false) - () - } + MonixTask + .racePair( + registerListener(cancel)(Scheduler.singleThread("cancel-signal")) *> waitForCanceled, + handle + ) + .flatMap { + case Left((result, fiber)) => + if (result.isOk) fiber.join else fiber.cancel *> MonixTask.now(result) + case Right((fiber, _)) => fiber.join + } + .onErrorHandleWith { + case i: InterruptedException => handleException(i, completed = true) + } + .onErrorRecoverWith { + case NonFatal(t) => handleException(t, completed = false) } - - Await.result(handle, Duration.Inf) - } catch { - case i: InterruptedException => handleException(i) - case NonFatal(t) => handleException(t) - } } } diff --git a/frontend/src/test/resources/source-generator.py b/frontend/src/test/resources/source-generator.py index e7042e3db8..43e31ef7eb 100644 --- a/frontend/src/test/resources/source-generator.py +++ b/frontend/src/test/resources/source-generator.py @@ -62,3 +62,23 @@ def main(output_dir, args): def random(): return 123 + +def random(): + return 123 + + +def random(): + return 123 + + +def random(): + return 123 + + +def random(): + return 123 + + +def random(): + return 123 + diff --git a/frontend/src/test/scala/bloop/BaseCompileSpec.scala b/frontend/src/test/scala/bloop/BaseCompileSpec.scala index 173eea1c42..6812062663 100644 --- a/frontend/src/test/scala/bloop/BaseCompileSpec.scala +++ b/frontend/src/test/scala/bloop/BaseCompileSpec.scala @@ -1664,58 +1664,6 @@ abstract class BaseCompileSpec extends bloop.testing.BaseSuite { } } - test("don't compile build in two concurrent CLI clients") { - TestUtil.withinWorkspace { workspace => - val sources = List( - """/main/scala/Foo.scala - |class Foo - """.stripMargin - ) - val testOut = new ByteArrayOutputStream() - val options = CommonOptions.default.copy(out = new PrintStream(testOut)) - val `A` = TestProject(workspace, "a", sources) - val configDir = TestProject.populateWorkspace(workspace, List(`A`)) - val compileArgs = Array("compile", "a", "--config-dir", configDir.syntax) - val compileAction = Cli.parse(compileArgs, options) - def runCompileAsync = Task.eval(Cli.run(compileAction, NoPool)).executeAsync - val runCompile = Task.gatherUnordered(List(runCompileAsync, runCompileAsync)).map(_ => ()) - Await.result(runCompile.runAsync(ExecutionContext.ioScheduler), FiniteDuration(10, "s")) - - val actionsOutput = new String(testOut.toByteArray(), StandardCharsets.UTF_8) - def removeAsciiColorCodes(line: String): String = line.replaceAll("\u001B\\[[;\\d]*m", "") - - val obtained = actionsOutput.splitLines - .filterNot(_.startsWith("Compiled")) - .map(removeAsciiColorCodes) - .map(msg => RecordingLogger.replaceTimingInfo(msg)) - .mkString(lineSeparator) - .replaceAll("'(bloop-cli-.*)'", "'bloop-cli'") - .replaceAll("'bloop-cli'", "???") - - try { - assertNoDiff( - processOutput(obtained), - s"""Compiling a (1 Scala source) - |Deduplicating compilation of a from cli client ??? (since ??? - |Compiling a (1 Scala source) - |$extraCompilationMessageOutput - |""".stripMargin - ) - } catch { - case _: DiffAssertions.TestFailedException => - assertNoDiff( - processOutput(obtained), - s""" - |Deduplicating compilation of a from cli client ??? (since ??? - |Compiling a (1 Scala source) - |Compiling a (1 Scala source) - |$extraCompilationMessageOutput - |""".stripMargin - ) - } - } - } - test("compile a project that redundantly lists an exact file as well as parent directory") { TestUtil.withinWorkspace { workspace => val filename = "/main/scala/Foo.scala" diff --git a/frontend/src/test/scala/bloop/MonixBaseCompileSpec.scala b/frontend/src/test/scala/bloop/MonixBaseCompileSpec.scala new file mode 100644 index 0000000000..8ccae576d7 --- /dev/null +++ b/frontend/src/test/scala/bloop/MonixBaseCompileSpec.scala @@ -0,0 +1,75 @@ +package bloop + +import bloop.cli.{CommonOptions, ExitStatus} +import bloop.engine.NoPool +import bloop.io.Environment.{LineSplitter, lineSeparator} +import bloop.logging.RecordingLogger +import monix.eval.Task +import bloop.testing.DiffAssertions +import bloop.util.{BaseTestProject, TestUtil} + +import java.io.{ByteArrayOutputStream, PrintStream} +import java.nio.charset.StandardCharsets + +abstract class MonixBaseCompileSpec extends bloop.testing.MonixBaseSuite { + protected def TestProject: BaseTestProject + + protected def extraCompilationMessageOutput: String = "" + protected def processOutput(output: String) = output + + test("don't compile build in two concurrent CLI clients") { + TestUtil.withinWorkspaceV2 { workspace => + val sources = List( + """/main/scala/Foo.scala + |class Foo + """.stripMargin + ) + val testOut = new ByteArrayOutputStream() + val options = CommonOptions.default.copy(out = new PrintStream(testOut)) + val `A` = TestProject(workspace, "a", sources) + val configDir = TestProject.populateWorkspace(workspace, List(`A`)) + val compileArgs = + Array("compile", "a", "--config-dir", configDir.syntax) + val compileAction = Cli.parse(compileArgs, options) + def runCompileAsync: Task[ExitStatus] = Cli.run(compileAction, NoPool) + for { + runCompile <- Task.parSequenceUnordered(List(runCompileAsync, runCompileAsync)) + } yield { + + val actionsOutput = new String(testOut.toByteArray, StandardCharsets.UTF_8) + def removeAsciiColorCodes(line: String): String = line.replaceAll("\u001B\\[[;\\d]*m", "") + + val obtained = actionsOutput.splitLines + .filterNot(_.startsWith("Compiled")) + .map(removeAsciiColorCodes) + .map(msg => RecordingLogger.replaceTimingInfo(msg)) + .mkString(lineSeparator) + .replaceAll("'(bloop-cli-.*)'", "'bloop-cli'") + .replaceAll("'bloop-cli'", "???") + + try { + assertNoDiff( + processOutput(obtained), + s"""Compiling a (1 Scala source) + |Deduplicating compilation of a from cli client ??? (since ??? + |Compiling a (1 Scala source) + |$extraCompilationMessageOutput + |""".stripMargin + ) + } catch { + case _: DiffAssertions.TestFailedException => + assertNoDiff( + processOutput(obtained), + s""" + |Deduplicating compilation of a from cli client ??? (since ??? + |Compiling a (1 Scala source) + |Compiling a (1 Scala source) + |$extraCompilationMessageOutput + |""".stripMargin + ) + } + } + } + } + +} diff --git a/frontend/src/test/scala/bloop/MonixCompileSpec.scala b/frontend/src/test/scala/bloop/MonixCompileSpec.scala new file mode 100644 index 0000000000..0a9351bd1d --- /dev/null +++ b/frontend/src/test/scala/bloop/MonixCompileSpec.scala @@ -0,0 +1,5 @@ +package bloop + +object MonixCompileSpec extends MonixBaseCompileSpec { + override protected val TestProject = util.TestProject +} diff --git a/frontend/src/test/scala/bloop/testing/MonixBaseSuite.scala b/frontend/src/test/scala/bloop/testing/MonixBaseSuite.scala new file mode 100644 index 0000000000..5841e5e2ee --- /dev/null +++ b/frontend/src/test/scala/bloop/testing/MonixBaseSuite.scala @@ -0,0 +1,561 @@ +package bloop.testing + +import bloop.Compiler +import bloop.cli.ExitStatus +import bloop.engine.caches.LastSuccessfulResult +import bloop.io.AbsolutePath +import bloop.io.Paths.AttributedPath +import bloop.logging.{BspServerLogger, RecordingLogger} +import bloop.reporter.Problem +import bloop.util.{TestProject, TestUtil} +import monix.eval.Task +import monix.execution.{ExecutionModel, Scheduler} +import monix.testing.utest.MonixTaskTest +import utest.Tests +import utest.asserts.Asserts +import utest.framework.{Formatter, TestCallTree, Tree} +import utest.ufansi.{Attrs, Color, Str} + +import scala.annotation.nowarn +import scala.collection.JavaConverters._ +import scala.concurrent.duration._ +import scala.language.experimental.macros +import scala.reflect.ClassTag +import scala.util.control.NonFatal + +abstract class MonixBaseSuite extends MonixTaskTest with BloopHelpers { + override implicit val scheduler: Scheduler = + Scheduler.global + override val timeout: FiniteDuration = 300.second + val pprint = _root_.pprint.PPrinter.BlackWhite + def isWindows: Boolean = bloop.util.CrossPlatform.isWindows + def isMac: Boolean = bloop.util.CrossPlatform.isMac + def isAppveyor: Boolean = "True" == System.getenv("APPVEYOR") + def beforeAll(): Unit = () + def afterAll(): Unit = () + def intercept[T: ClassTag](exprs: Unit): T = macro Asserts.interceptProxy[T] + + def assertNotEmpty(string: String): Unit = { + if (string.isEmpty) { + fail( + s"expected non-empty string, obtained empty string." + ) + } + } + + def assertEmpty(string: String): Unit = { + if (!string.isEmpty) { + fail( + s"expected empty string, obtained: $string" + ) + } + } + + def assertContains(string: String, substring: String): Unit = { + assert(string.contains(substring)) + } + + def assertNotContains(string: String, substring: String): Unit = { + assert(!string.contains(substring)) + } + + def assert(exprs: Boolean*): Unit = macro Asserts.assertProxy + + def assertNotEquals[T](obtained: T, expected: T, hint: String = ""): Unit = { + if (obtained == expected) { + val hintMsg = if (hint.isEmpty) "" else s" (hint: $hint)" + assertNoDiff(obtained.toString, expected.toString, hint) + fail(s"obtained=<$obtained> == expected=<$expected>$hintMsg") + } + } + + def assertEquals[T](obtained: T, expected: T, hint: String = ""): Unit = { + if (obtained != expected) { + val hintMsg = if (hint.isEmpty) "" else s" (hint: $hint)" + assertNoDiff(obtained.toString, expected.toString, hint) + fail(s"obtained=<$obtained> != expected=<$expected>$hintMsg") + } + } + + def assertNotFile(path: AbsolutePath): Unit = { + if (path.isFile) { + fail(s"file exists: $path", stackBump = 1) + } + } + + def assertIsFile(path: AbsolutePath): Unit = { + if (!path.isFile) { + fail(s"no such file: $path", stackBump = 1) + } + } + + def assertIsDirectory(path: AbsolutePath): Unit = { + if (!path.isDirectory) { + fail(s"directory doesn't exist: $path", stackBump = 1) + } + } + + def assertIsNotDirectory(path: AbsolutePath): Unit = { + if (path.isDirectory) { + fail(s"directory exists: $path", stackBump = 1) + } + } + + def assertExitStatus(obtainedState: TestState, expected: ExitStatus): Unit = { + val obtained = obtainedState.status + try assert(obtained == expected) + catch { + case NonFatal(t) => + obtainedState.state.logger match { + case logger: RecordingLogger => logger.dump(); throw t + case logger: BspServerLogger => + logger.underlying match { + case logger: RecordingLogger => logger.dump(); throw t + case _ => throw t + } + case _ => throw t + } + } + } + + def assertSameResult( + a: LastSuccessfulResult, + b: LastSuccessfulResult + ): Unit = { + assert(a.previous == b.previous) + assert(a.classesDir == b.classesDir) + } + + import bloop.logging.NoopLogger + def takeDirectorySnapshot( + dir: AbsolutePath + ): List[AttributedPath] = { + import java.io.File + val files = bloop.io.Paths + .attributedPathFilesUnder(dir, "glob:**.*", NoopLogger) + .map { ap => + val prefixPath = dir.syntax.stripSuffix("/") + val osInsensitivePath = ap.path.syntax.replace(prefixPath, "").replace(File.separator, "/") + val maskedRelativePath = AbsolutePath(osInsensitivePath) + if (!maskedRelativePath.syntax.startsWith("/classes-")) { + ap.withPath(maskedRelativePath) + } else { + // Remove '/classes-*' from path + val newPath = maskedRelativePath.syntax.split(File.separatorChar).tail.tail.mkString("/") + ap.withPath(AbsolutePath("/" + newPath)) + } + } + + files.sortBy(_.toString) + } + + def assertDifferentExternalClassesDirs( + s1: TestState, + s2: TestState, + projects: List[TestProject] + )(implicit filename: sourcecode.File, line: sourcecode.Line): Unit = { + projects.foreach { project => + assertDifferentExternalClassesDirs(s1, s2, project) + } + } + + def assertDifferentExternalClassesDirs( + s1: TestState, + s2: TestState, + project: TestProject + )(implicit filename: sourcecode.File, line: sourcecode.Line): Unit = { + try { + assertSameExternalClassesDirs(s1, s2, project, printDiff = false) + fail("External classes dirs of ${project.config.name} in both states is the same!") + } catch { + case _: DiffAssertions.TestFailedException => () + } + } + + def assertSameExternalClassesDirs( + s1: TestState, + s2: TestState, + project: TestProject, + printDiff: Boolean = true + )(implicit filename: sourcecode.File, line: sourcecode.Line): Unit = { + val projectName = project.config.name + val a = s1.client + .getUniqueClassesDirFor(s1.build.getProjectFor(projectName).get, forceGeneration = true) + val b = s2.client + .getUniqueClassesDirFor(s2.build.getProjectFor(projectName).get, forceGeneration = true) + val filesA = takeDirectorySnapshot(a) + val filesB = takeDirectorySnapshot(b) + assertNoDiff( + pprint.apply(filesA, height = Int.MaxValue).render, + pprint.apply(filesB, height = Int.MaxValue).render, + print = printDiff + ) + } + + def assertSameExternalClassesDirs( + s1: TestState, + s2: TestState, + projects: List[TestProject] + )(implicit filename: sourcecode.File, line: sourcecode.Line): Unit = { + projects.foreach { project => + assertSameExternalClassesDirs(s1, s2, project) + } + } + + def assertSameFilesIn( + a: AbsolutePath, + b: AbsolutePath + )(implicit filename: sourcecode.File, line: sourcecode.Line): Unit = { + val filesA = takeDirectorySnapshot(a) + val filesB = takeDirectorySnapshot(b) + assertNoDiff( + pprint.apply(filesA, height = Int.MaxValue).render, + pprint.apply(filesB, height = Int.MaxValue).render, + a.syntax, + b.syntax + ) + } + + def assertCancelledCompilation(state: TestState, projects: List[TestProject]): Unit = { + projects.foreach { project => + state.getLastResultFor(project) match { + case _: Compiler.Result.Cancelled => () + case result => fail(s"Result ${result} is not cancelled!") + } + } + } + + def assertSuccessfulCompilation( + state: TestState, + projects: List[TestProject], + isNoOp: Boolean + ): Unit = { + projects.foreach { project => + state.getLastResultFor(project) match { + case s: Compiler.Result.Success => if (isNoOp) assert(s.isNoOp) + case result => fail(s"Result ${result} is not cancelled!") + } + } + } + + def assertDiagnosticsResult( + result: Compiler.Result, + errors: Int, + warnings: Int = 0, + expectFatalWarnings: Boolean = false + ): Unit = { + if (errors > 0) { + result match { + case Compiler.Result.Failed(problems, t, _, _, _) => + val count = Problem.count(problems) + if (count.errors == 0 && errors != 0) { + // If there's an exception count it as one error + val errors = t match { + case Some(_) => 1 + case None => 0 + } + + assert(count.errors == errors) + } else { + assert(count.errors == errors) + } + assert(count.warnings == warnings) + case _ => fail(s"Result ${result} != Failed") + } + } else { + result match { + case Compiler.Result.Success(_, reporter, _, _, _, _, reportedFatalWarnings) => + val count = Problem.count(reporter.allProblemsPerPhase.toList) + assert(count.errors == 0) + assert(expectFatalWarnings == reportedFatalWarnings) + assert(count.warnings == warnings) + case _ => fail("Result ${result} != Success, but expected errors == 0") + } + } + } + + def assertInvalidCompilationState( + state: TestState, + projects: List[TestProject], + existsAnalysisFile: Boolean, + hasPreviousSuccessful: Boolean, + hasSameContentsInClassesDir: Boolean + )(implicit filename: sourcecode.File, line: sourcecode.Line): Unit = { + val buildProjects = projects.flatMap(p => state.build.getProjectFor(p.config.name).toList) + assert(projects.size == buildProjects.size) + projects.zip(buildProjects).foreach { + case (testProject, buildProject) => + if (!existsAnalysisFile) assertNotFile(buildProject.analysisOut) + else assertIsFile(buildProject.analysisOut) + val latestResult = state.getLastSuccessfulResultFor(testProject) + if (hasPreviousSuccessful) { + val result = latestResult.get + val analysis = result.previous.analysis().get() + val stamps = analysis.readStamps + assert(stamps.getAllProductStamps.asScala.nonEmpty) + assert(stamps.getAllSourceStamps.asScala.nonEmpty) + + if (hasSameContentsInClassesDir) { + val projectClassesDir = + state.client.getUniqueClassesDirFor(buildProject, forceGeneration = true) + assert(takeDirectorySnapshot(result.classesDir).nonEmpty) + assertSameFilesIn(projectClassesDir, result.classesDir) + } + } else { + assert(latestResult.isEmpty) + val lastResult = state.getLastResultFor(testProject) + assert(lastResult != Compiler.Result.Empty) + lastResult match { + case Compiler.Result.NotOk(_) => () + case r => fail(s"Result $r is considered a success!") + } + } + } + } + + import bloop.io.RelativePath + def assertCompileProduct( + state: TestState, + project: TestProject, + classFile: RelativePath, + existing: Boolean + ): Unit = { + val buildProject = state.build.getProjectFor(project.config.name).get + val externalClassesDir = + state.client.getUniqueClassesDirFor(buildProject, forceGeneration = true) + if (existing) assert(externalClassesDir.resolve(classFile).exists) + else assert(!externalClassesDir.resolve(classFile).exists) + } + + def assertNonExistingCompileProduct( + state: TestState, + project: TestProject, + classFile: RelativePath + ): Unit = { + assertCompileProduct(state, project, classFile, false) + } + + def assertExistingCompileProduct( + state: TestState, + project: TestProject, + classFile: RelativePath + ): Unit = { + assertCompileProduct(state, project, classFile, true) + } + + def assertExistingInternalClassesDir(lastState: TestState)( + stateToCheck: TestState, + projects: List[TestProject] + ): Unit = { + assertExistenceOfInternalClassesDir(lastState, stateToCheck, projects, checkExists = true) + } + + def assertNonExistingInternalClassesDir(lastState: TestState)( + stateToCheck: TestState, + projects: List[TestProject] + ): Unit = { + assertExistenceOfInternalClassesDir(lastState, stateToCheck, projects, checkExists = false) + } + + private def assertExistenceOfInternalClassesDir( + lastState: TestState, + stateToCheck: TestState, + projects: List[TestProject], + checkExists: Boolean + ): Unit = { + val buildProjects = + projects.flatMap(p => stateToCheck.build.getProjectFor(p.config.name).toList) + assert(projects.size == buildProjects.size) + projects.zip(buildProjects).foreach { + case (testProject, buildProject) => + // Force the execution of the next last successful to delete the directory + lastState.results.lastSuccessfulResult(buildProject).foreach { s => + import scala.concurrent.duration.FiniteDuration + val _ = TestUtil.await(FiniteDuration(5, "s"))(s.populatingProducts) + } + + val last = stateToCheck.getLastSuccessfulResultFor(testProject).get + val classesDir = last.classesDir + // Sleep for 20 ms to give time to classes dir to be deleted (runs inside `doOnFinish`) + Thread.sleep(20) + if (checkExists) assert(classesDir.exists) + else assert(!classesDir.exists) + } + } + + def assertEmptyCompilationState( + state: TestState, + projects: List[TestProject] + ): Unit = { + val buildProjects = projects.flatMap(p => state.build.getProjectFor(p.config.name).toList) + assert(projects.size == buildProjects.size) + projects.zip(buildProjects).foreach { + case (testProject, buildProject) => + assertNotFile(buildProject.analysisOut) + assert(state.getLastSuccessfulResultFor(testProject).isEmpty) + assert(state.getLastResultFor(testProject) == Compiler.Result.Empty) + val classesDir = state.client.getUniqueClassesDirFor(buildProject, forceGeneration = true) + assert(takeDirectorySnapshot(classesDir).isEmpty) + } + } + + def assertValidCompilationState( + state: TestState, + projects: List[TestProject] + )(implicit filename: sourcecode.File, line: sourcecode.Line): Unit = { + val buildProjects = projects.flatMap(p => state.build.getProjectFor(p.config.name).toList) + assert(projects.size == buildProjects.size) + projects.zip(buildProjects).foreach { + case (testProject, buildProject) => + assertIsFile(buildProject.analysisOut) + val latestResult = state + .getLastSuccessfulResultFor(testProject) + .getOrElse( + sys.error(s"No latest result for ${testProject.config.name}, results: ${state.results}") + ) + + val analysis = latestResult.previous.analysis().get() + val stamps = analysis.readStamps + assert(stamps.getAllProductStamps.asScala.nonEmpty) + assert(stamps.getAllSourceStamps.asScala.nonEmpty) + + assert(takeDirectorySnapshot(latestResult.classesDir).nonEmpty) + val projectClassesDir = + state.client.getUniqueClassesDirFor(buildProject, forceGeneration = true) + assertSameFilesIn(projectClassesDir, latestResult.classesDir) + } + } + + def assertNoDiff( + obtained: String, + expected: String, + obtainedTitle: String, + expectedTitle: String + )(implicit filename: sourcecode.File, line: sourcecode.Line): Unit = { + colored { + DiffAssertions.assertNoDiffOrPrintExpected( + obtained, + expected, + obtainedTitle, + expectedTitle, + true + ) + () + } + } + + def assertNoDiff( + obtained: String, + expected: String, + title: String = "", + print: Boolean = true + )(implicit filename: sourcecode.File, line: sourcecode.Line): Unit = { + colored { + DiffAssertions.assertNoDiffOrPrintExpected(obtained, expected, title, title, print) + () + } + } + + def colored[T]( + thunk: => T + )(implicit filename: sourcecode.File, line: sourcecode.Line): T = { + try { + thunk + } catch { + case scala.util.control.NonFatal(e) => + val message = e.getMessage.linesIterator + .map { line => + if (line.startsWith("+")) Color.Green(line) + else if (line.startsWith("-")) Color.LightRed(line) + else Color.Reset(line) + } + .mkString("\n") + val location = s"failed assertion at ${filename.value}:${line.value}\n" + throw new DiffAssertions.TestFailedException(location + message) + } + } + + def unsafeGetResource(resourceId: String): AbsolutePath = { + val resource = this.getClass.getClassLoader().getResource(resourceId) + if (resource == null) sys.error(s"Missing resource $resourceId") + else AbsolutePath(resource.toURI()) + } + + override def utestAfterAll(): Unit = afterAll() + override def utestFormatter: Formatter = new Formatter { + override def exceptionMsgColor: Attrs = Attrs.Empty + override def exceptionStackFrameHighlighter( + s: StackTraceElement + ): Boolean = { + s.getClassName.startsWith("bloop.") && + !(s.getClassName.startsWith("bloop.util") || + s.getClassName.startsWith("bloop.testing")) + } + + override def formatWrapWidth: Int = 3000 + override def formatException(x: Throwable, leftIndent: String): Str = + super.formatException(x, "") + } + + case class FlatTest(name: String, thunk: () => Task[Any]) + private val myTests = IndexedSeq.newBuilder[FlatTest] + + @nowarn("msg=parameter value fun in method ignore is never used") + def ignore(name: String, label: String = "IGNORED")(fun: => Task[Any]): Unit = { + test(utest.ufansi.Color.LightRed(s"$label - $name").toString())(Task.unit) + } + + def testOnlyOnJava8(name: String)(fun: => Task[Any]): Unit = { + if (TestUtil.isJdk8) test(name)(fun) + else ignore(name, label = s"IGNORED ON JAVA v${TestUtil.jdkVersion}")(fun) + } + +// def flakyTest(name: String, attempts: Int)(fun: => Task[Any]): Unit = { +// assert(attempts >= 0) +// def retry(fun: => Task[Any], attempts: Int): Task[Any] = { +// try { +// fun; () +// } catch { +// case NonFatal(t) => +// if (attempts == 0) throw t +// else { +// System.err.println( +// s"Caught exception on flaky test run (remaining $attempts), restarting..." +// ) +// t.printStackTrace(System.err) +// Thread.sleep(10000) +// retry(fun, attempts - 1) +// } +// } +// } +// +// myTests += FlatTest(name, () => retry(fun, attempts)) +// } + + def test(name: String)(fun: => Task[Any]): Unit = { + myTests += FlatTest(name, () => fun) + } + +// def testTask(name: String, maxDuration: Duration = Duration("20s"))(fun: => Task[Unit]): Unit = { +// myTests += FlatTest( +// name, +// () => { TestUtil.await(maxDuration, ExecutionContext.ioScheduler)(fun) } +// ) +// } + + def fail(msg: String, stackBump: Int = 0): Nothing = { + val ex = new DiffAssertions.TestFailedException(msg) + ex.setStackTrace(ex.getStackTrace.slice(1 + stackBump, 5 + stackBump)) + throw ex + } + + override def tests: Tests = { + val ts = myTests.result() + val names = Tree("", ts.map(x => Tree(x.name)): _*) + val thunks = new TestCallTree({ + this.beforeAll() + Right(ts.map(x => new TestCallTree(Left(x.thunk())))) + }) + Tests(names, thunks) + } +} diff --git a/frontend/src/test/scala/bloop/util/TestUtil.scala b/frontend/src/test/scala/bloop/util/TestUtil.scala index 58f8645376..203c56553c 100644 --- a/frontend/src/test/scala/bloop/util/TestUtil.scala +++ b/frontend/src/test/scala/bloop/util/TestUtil.scala @@ -7,7 +7,6 @@ import java.nio.file.Path import java.nio.file.attribute.FileTime import java.util.concurrent.TimeUnit import java.util.concurrent.TimeoutException - import scala.concurrent.Await import scala.concurrent.ExecutionException import scala.concurrent.duration.Duration @@ -15,7 +14,6 @@ import scala.concurrent.duration.FiniteDuration import scala.concurrent.duration.TimeUnit import scala.tools.nsc.Properties import scala.util.control.NonFatal - import bloop.CompilerCache import bloop.DependencyResolution import bloop.ScalaInstance @@ -50,8 +48,8 @@ import bloop.logging.BufferedLogger import bloop.logging.DebugFilter import bloop.logging.Logger import bloop.logging.RecordingLogger - import _root_.bloop.task.Task +import _root_.monix.eval.{Task => MonixTask} import _root_.monix.execution.Scheduler import org.junit.Assert import sbt.internal.inc.BloopComponentCompiler @@ -483,6 +481,15 @@ object TestUtil { finally delete(AbsolutePath(temp)) } + /** Creates an empty workspace where operations can happen. */ + def withinWorkspaceV2[T](op: AbsolutePath => MonixTask[T]): MonixTask[T] = { + MonixTask + .now(Files.createTempDirectory("bloop-test-workspace").toRealPath()) + .flatMap { temp => + op(AbsolutePath(temp)).guarantee(MonixTask(delete(AbsolutePath(temp)))) + } + } + /** Creates an empty workspace where operations can happen. */ def withinWorkspace[T](op: AbsolutePath => Task[T]): Task[T] = { val temp = Files.createTempDirectory("bloop-test-workspace").toRealPath() diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 7445357699..6a3c08792f 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -27,6 +27,7 @@ object Dependencies { val junitVersion = "0.13.3" val directoryWatcherVersion = "0.8.0+6-f651bd93" val monixVersion = "3.2.0" + val monixTestVersion = "0.3.0" val jsoniterVersion = "2.13.3.2" val shapelessVersion = "2.3.4" val scalaNative04Version = "0.4.17" @@ -88,6 +89,7 @@ object Dependencies { val difflib = "com.googlecode.java-diff-utils" % "diffutils" % difflibVersion val monix = "io.monix" %% "monix" % monixVersion + val monixTest = "io.monix" %% "monix-testing-utest" % monixTestVersion % "test" val jsoniterCore = "com.github.plokhotnyuk.jsoniter-scala" %% "jsoniter-scala-core" % jsoniterVersion val jsoniterMacros = From 69709d1fcb28bf49b9e448ef7ff7c8aa023933b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Wi=C4=85cek?= Date: Sat, 31 May 2025 15:24:14 +0200 Subject: [PATCH 02/10] stop using concurrenthash map and replace it with cats-effect Ref There is still problem with `reflectMain` I am not sure where it is used. At the moment I have changed its signature --- frontend/src/main/scala/bloop/Cli.scala | 70 +++++++++++++++---------- 1 file changed, 41 insertions(+), 29 deletions(-) diff --git a/frontend/src/main/scala/bloop/Cli.scala b/frontend/src/main/scala/bloop/Cli.scala index 7f9bdeb14d..b00901a0cb 100644 --- a/frontend/src/main/scala/bloop/Cli.scala +++ b/frontend/src/main/scala/bloop/Cli.scala @@ -3,7 +3,6 @@ package bloop import java.io.InputStream import java.io.PrintStream import java.nio.file.Path -import java.util.concurrent.ConcurrentHashMap import scala.util.control.NonFatal import bloop.cli.CliOptions import bloop.cli.Commands @@ -23,7 +22,7 @@ import monix.eval.TaskApp import bloop.util.JavaRuntime import caseapp.core.help.Help import cats.effect.ExitCode -import cats.effect.concurrent.Deferred +import cats.effect.concurrent.{Deferred, Ref} import com.martiansoftware.nailgun.NGContext import monix.execution.Scheduler import monix.execution.atomic.AtomicBoolean @@ -47,7 +46,8 @@ object Cli extends TaskApp { out: PrintStream, err: PrintStream, props: java.util.Properties, - cancel: Deferred[MonixTask, Boolean] + cancel: Deferred[MonixTask, Boolean], + activeCliSessions: Ref[MonixTask, Map[Path, List[CliSession]]] ): MonixTask[Int] = { val env = CommonOptions.PrettyProperties.from(props) val nailgunOptions = CommonOptions( @@ -61,7 +61,7 @@ object Cli extends TaskApp { ) val cmd = parse(args, nailgunOptions) - val exitStatus = run(cmd, NoPool, cancel) + val exitStatus = run(cmd, NoPool, cancel, activeCliSessions) exitStatus.map(_.code) } @@ -305,11 +305,11 @@ object Cli extends TaskApp { } def run(action: Action, pool: ClientPool): MonixTask[ExitStatus] = { - for { baseCancellation <- Deferred[MonixTask, Boolean] + activeCliSessions <- Ref.of[MonixTask, Map[Path, List[CliSession]]](Map.empty) _ <- baseCancellation.complete(false) - result <- run(action, pool, baseCancellation) + result <- run(action, pool, baseCancellation, activeCliSessions) } yield result } @@ -318,7 +318,8 @@ object Cli extends TaskApp { private def run( action: Action, pool: ClientPool, - cancel: Deferred[MonixTask, Boolean] + cancel: Deferred[MonixTask, Boolean], + activeCliSessions: Ref[MonixTask, Map[Path, List[CliSession]]] ): MonixTask[ExitStatus] = { import bloop.io.AbsolutePath def getConfigDir(cliOptions: CliOptions): AbsolutePath = { @@ -360,6 +361,7 @@ object Cli extends TaskApp { action, pool, cancel, + activeCliSessions, configDirectory, cliOptions, commonOpts, @@ -372,6 +374,7 @@ object Cli extends TaskApp { action: Action, pool: ClientPool, cancel: Deferred[MonixTask, Boolean], + activeCliSessions: Ref[MonixTask, Map[Path, List[CliSession]]], configDirectory: AbsolutePath, cliOptions: CliOptions, commonOpts: CommonOptions, @@ -397,27 +400,34 @@ object Cli extends TaskApp { interpret.toMonixTask(ExecutionContext.scheduler) } - val session = runTaskWithCliClient(configDirectory, action, taskToInterpret, pool, logger) + val session = runTaskWithCliClient( + configDirectory, + action, + taskToInterpret, + activeCliSessions, + pool, + logger + ) val exitSession = MonixTask.defer { - cleanUpNonStableCliDirectories(session.client) + session.flatMap(s => cleanUpNonStableCliDirectories(s.client)) } - session.task + session + .flatMap(_.task) .doOnCancel(exitSession) .doOnFinish(_ => exitSession) } } - private val activeCliSessions = new ConcurrentHashMap[Path, List[CliSession]]() - case class CliSession(client: CliClientInfo, task: MonixTask[ExitStatus]) def runTaskWithCliClient( configDir: AbsolutePath, action: Action, processCliTask: CliClientInfo => MonixTask[State], + activeCliSessions2: Ref[MonixTask, Map[Path, List[CliSession]]], pool: ClientPool, logger: Logger - ): CliSession = { + ): MonixTask[CliSession] = { val isClientConnected = AtomicBoolean(true) pool.addListener(_ => isClientConnected.set(false)) val defaultClient = CliClientInfo(useStableCliDirs = true, () => isClientConnected.get) @@ -429,28 +439,30 @@ object Cli extends TaskApp { val defaultClientSession = sessionFor(defaultClient) action match { - case Exit(_) => defaultClientSession + case Exit(_) => MonixTask.now(defaultClientSession) // Don't synchronize on commands that don't use compilation products and can run concurrently - case Run(_: Commands.About, _) => defaultClientSession - case Run(_: Commands.Projects, _) => defaultClientSession - case Run(_: Commands.Autocomplete, _) => defaultClientSession - case Run(_: Commands.Bsp, _) => defaultClientSession - case Run(_: Commands.ValidatedBsp, _) => defaultClientSession - case _ => - val activeSessions = activeCliSessions.compute( - configDir.underlying, - (_: Path, sessions: List[CliSession]) => { - if (sessions == null || sessions.isEmpty) List(defaultClientSession) - else { + case Run(_: Commands.About, _) => MonixTask.now(defaultClientSession) + case Run(_: Commands.Projects, _) => MonixTask.now(defaultClientSession) + case Run(_: Commands.Autocomplete, _) => MonixTask.now(defaultClientSession) + case Run(_: Commands.Bsp, _) => MonixTask.now(defaultClientSession) + case Run(_: Commands.ValidatedBsp, _) => MonixTask.now(defaultClientSession) + case a @ _ => + activeCliSessions2.modify { sessionsMap => + val currentSessions = sessionsMap.getOrElse(configDir.underlying, Nil) + + val updatedSessions = + if (currentSessions.isEmpty) { + List(defaultClientSession) + } else { logger.debug("Detected connected cli clients, starting CLI with unique dirs...") val newClient = CliClientInfo(useStableCliDirs = false, () => isClientConnected.get) val newClientSession = sessionFor(newClient) - newClientSession :: sessions + newClientSession :: currentSessions } - } - ) - activeSessions.head + val updatedMap = sessionsMap.updated(configDir.underlying, updatedSessions) + (updatedMap, updatedSessions.head) + } } } From 0ec7bcd610b3179fdaf7040acbd98ea17891816a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Wi=C4=85cek?= Date: Mon, 2 Jun 2025 12:30:54 +0200 Subject: [PATCH 03/10] remove unused file and some invalid changes --- .../scala/bloop/task/ObservableDeferred.scala | 39 ------------------- .../src/test/resources/source-generator.py | 20 ---------- 2 files changed, 59 deletions(-) delete mode 100644 backend/src/main/scala/bloop/task/ObservableDeferred.scala diff --git a/backend/src/main/scala/bloop/task/ObservableDeferred.scala b/backend/src/main/scala/bloop/task/ObservableDeferred.scala deleted file mode 100644 index 665fb46182..0000000000 --- a/backend/src/main/scala/bloop/task/ObservableDeferred.scala +++ /dev/null @@ -1,39 +0,0 @@ -package bloop.task - -import cats.effect.Concurrent -import cats.effect.concurrent.{Deferred, Ref} -import monix.eval.{Task => MonixTask} -import monix.execution.atomic.AtomicBoolean - -class ObservableDeferred[A]( - ref: Deferred[MonixTask, A], - state: Ref[MonixTask, Boolean], - syncFlag: AtomicBoolean -) { - def complete(value: A): MonixTask[Unit] = { - MonixTask.defer { - ref - .complete(value) - .flatMap { _ => - syncFlag.set(true) - state.set(true) - } - }.uncancelable - } - - def isCompletedUnsafe: Boolean = syncFlag.get - - def isCompleted: MonixTask[Boolean] = state.get - - def get: MonixTask[A] = ref.get -} - -object ObservableDeferred { - - def apply[A](implicit F: Concurrent[MonixTask]): MonixTask[ObservableDeferred[A]] = - for { - ref <- Deferred[MonixTask, A] - state <- Ref[MonixTask].of(false) - } yield new ObservableDeferred(ref, state, AtomicBoolean(false)) - -} diff --git a/frontend/src/test/resources/source-generator.py b/frontend/src/test/resources/source-generator.py index 43e31ef7eb..e7042e3db8 100644 --- a/frontend/src/test/resources/source-generator.py +++ b/frontend/src/test/resources/source-generator.py @@ -62,23 +62,3 @@ def main(output_dir, args): def random(): return 123 - -def random(): - return 123 - - -def random(): - return 123 - - -def random(): - return 123 - - -def random(): - return 123 - - -def random(): - return 123 - From 981ca87894ebd93a16eda2e1b082c9e50525e37e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Wi=C4=85cek?= Date: Tue, 3 Jun 2025 14:21:42 +0200 Subject: [PATCH 04/10] non working rewrite to Ref. --- .../src/it/scala/bloop/CommunityBuild.scala | 2 +- .../tasks/compilation/CompileGatekeeper.scala | 303 +-- .../tasks/compilation/CompileGraph.scala | 410 ++-- .../tasks/compilation/CompileResult.scala | 10 + .../test/scala/bloop/DeduplicationSpec.scala | 1736 ++++++++--------- 5 files changed, 1255 insertions(+), 1206 deletions(-) diff --git a/frontend/src/it/scala/bloop/CommunityBuild.scala b/frontend/src/it/scala/bloop/CommunityBuild.scala index 1953f63fea..4937f38e4f 100644 --- a/frontend/src/it/scala/bloop/CommunityBuild.scala +++ b/frontend/src/it/scala/bloop/CommunityBuild.scala @@ -108,7 +108,7 @@ abstract class CommunityBuild(val buildpressHomeDir: AbsolutePath) { } // First thing to do: clear cache of successful results between project runs to free up space - CompileGatekeeper.clearSuccessfulResults() +// CompileGatekeeper.clearSuccessfulResults() // After reporting the state of the execution, compile the projects accordingly. val logger = BloopLogger.default("community-build-logger") diff --git a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGatekeeper.scala b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGatekeeper.scala index 339135343c..b40f163a02 100644 --- a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGatekeeper.scala +++ b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGatekeeper.scala @@ -1,7 +1,5 @@ package bloop.engine.tasks.compilation -import java.util.concurrent.ConcurrentHashMap - import bloop.Compiler import bloop.UniqueCompileInputs import bloop.data.ClientInfo @@ -14,7 +12,8 @@ import bloop.logging.Logger import bloop.logging.LoggerAction import bloop.reporter.ReporterAction import bloop.task.Task - +import monix.eval.{Task => MonixTask} +import cats.effect.concurrent.{Deferred, Ref} import monix.execution.atomic.AtomicBoolean import monix.execution.atomic.AtomicInt import monix.reactive.Observable @@ -33,9 +32,16 @@ object CompileGatekeeper { /* -------------------------------------------------------------------------------------------- */ - private val currentlyUsedClassesDirs = new ConcurrentHashMap[AbsolutePath, AtomicInt]() - private val runningCompilations = new ConcurrentHashMap[UniqueCompileInputs, RunningCompilation]() - private val lastSuccessfulResults = new ConcurrentHashMap[ProjectId, LastSuccessfulResult]() + case class CompilerState( + currentlyUsedClassesDirs: Map[AbsolutePath, AtomicInt], + runningCompilations: Map[UniqueCompileInputs, RunningCompilation], + lastSuccessfulResults: Map[ProjectId, LastSuccessfulResult] + ) + object CompilerState { + def empty: CompilerState = CompilerState(Map.empty, Map.empty, Map.empty) + } + + private val compilerStateRef: Ref[MonixTask, CompilerState] = Ref.unsafe(CompilerState.empty) /* -------------------------------------------------------------------------------------------- */ @@ -44,48 +50,68 @@ object CompileGatekeeper { bundle: SuccessfulCompileBundle, client: ClientInfo, compile: SuccessfulCompileBundle => CompileTraversal - ): (RunningCompilation, CanBeDeduplicated) = { - var deduplicate = true - - val running = runningCompilations.compute( - bundle.uniqueInputs, - (_: UniqueCompileInputs, running: RunningCompilation) => { - if (running == null) { - deduplicate = false - scheduleCompilation(inputs, bundle, client, compile) - } else { - val usedClassesDir = running.usedLastSuccessful.classesDir - val usedClassesDirCounter = running.usedLastSuccessful.counterForClassesDir - - usedClassesDirCounter.getAndTransform { count => - if (count == 0) { - // Abort deduplication, dir is scheduled to be deleted in background - deduplicate = false - // Remove from map of used classes dirs in case it hasn't already been - currentlyUsedClassesDirs.remove(usedClassesDir, usedClassesDirCounter) - // Return previous count, this counter will soon be deallocated - count - } else { - // Increase count to prevent other compiles to schedule its deletion - count + 1 + ): Task[(RunningCompilation, CanBeDeduplicated)] = + scheduleCompilation(inputs, bundle, client, compile) + .flatMap { orCompilation => + Task.liftMonixTaskUncancellable { + compilerStateRef + .modify { state => + val currentCompilation = state.runningCompilations.get(bundle.uniqueInputs) + val (compilation, deduplicate, classesDirs) = currentCompilation + .fold( + ( + orCompilation, + false, + state.currentlyUsedClassesDirs + ) + ) { running => + val usedClassesDir = running.usedLastSuccessful.classesDir + val usedClassesDirCounter = running.usedLastSuccessful.counterForClassesDir + val deduplicate = usedClassesDirCounter.transformAndExtract { + case count if count == 0 => (false -> count) + case count => true -> (count + 1) + } + if (deduplicate) (running, deduplicate, state.currentlyUsedClassesDirs) + else { + val classesDirs = + if ( + state.currentlyUsedClassesDirs + .get(usedClassesDir) + .contains(usedClassesDirCounter) + ) + state.currentlyUsedClassesDirs - usedClassesDir + else + state.currentlyUsedClassesDirs + ( + orCompilation, + deduplicate, + classesDirs + ) + } + } + val newState = + state.copy( + currentlyUsedClassesDirs = classesDirs, + runningCompilations = + state.runningCompilations + (bundle.uniqueInputs -> compilation) + ) + (state, (compilation, deduplicate)) } - } - - if (deduplicate) running - else scheduleCompilation(inputs, bundle, client, compile) } } - ) - - (running, deduplicate) - } def disconnectDeduplicationFromRunning( inputs: UniqueCompileInputs, runningCompilation: RunningCompilation - ): Unit = { + ): Task[Unit] = Task.liftMonixTaskUncancellable { runningCompilation.isUnsubscribed.compareAndSet(false, true) - runningCompilations.remove(inputs, runningCompilation); () + compilerStateRef.modify { state => + val updated = + if (state.runningCompilations.get(inputs).contains(runningCompilation)) + state.runningCompilations - inputs + else state.runningCompilations + (state.copy(runningCompilations = updated), ()) + } } /** @@ -100,15 +126,15 @@ object CompileGatekeeper { bundle: SuccessfulCompileBundle, client: ClientInfo, compile: SuccessfulCompileBundle => CompileTraversal - ): RunningCompilation = { + ): Task[RunningCompilation] = { import inputs.project import bundle.logger import logger.debug - var counterForUsedClassesDir: AtomicInt = null - - def initializeLastSuccessful(previousOrNull: LastSuccessfulResult): LastSuccessfulResult = { - val result = Option(previousOrNull).getOrElse(bundle.lastSuccessful) + def initializeLastSuccessful( + maybePreviousResult: Option[LastSuccessfulResult] + ): LastSuccessfulResult = { + val result = maybePreviousResult.getOrElse(bundle.lastSuccessful) if (!result.classesDir.exists) { debug(s"Ignoring analysis for ${project.name}, directory ${result.classesDir} is missing") LastSuccessfulResult.empty(inputs.project) @@ -128,65 +154,58 @@ object CompileGatekeeper { } } - def getMostRecentSuccessfulResultAtomically = { - lastSuccessfulResults.compute( - project.uniqueId, - (_: String, previousResultOrNull: LastSuccessfulResult) => { - // Return previous result or the initial last successful coming from the bundle - val previousResult = initializeLastSuccessful(previousResultOrNull) - - currentlyUsedClassesDirs.compute( - previousResult.classesDir, - (_: AbsolutePath, counter: AtomicInt) => { - // Set counter for used classes dir when init or incrementing - if (counter == null) { - val initialCounter = AtomicInt(1) - counterForUsedClassesDir = initialCounter - initialCounter - } else { - counterForUsedClassesDir = counter - val newCount = counter.incrementAndGet(1) - logger.debug(s"Increasing counter for ${previousResult.classesDir} to $newCount") - counter - } - } - ) - - previousResult.copy(counterForClassesDir = counterForUsedClassesDir) - } - ) + def getMostRecentSuccessfulResultAtomically = Task.liftMonixTaskUncancellable { + compilerStateRef.modify { state => + val previousResult = + initializeLastSuccessful(state.lastSuccessfulResults.get(project.uniqueId)) + val counter = state.currentlyUsedClassesDirs + .get(previousResult.classesDir) + .fold { + val initialCounter = AtomicInt(1) + initialCounter + } { counter => + val newCount = counter.incrementAndGet(1) + logger.debug(s"Increasing counter for ${previousResult.classesDir} to $newCount") + counter + } + val newUserClassesDir = (previousResult.classesDir, counter) + val newResults = (project.uniqueId, previousResult) + state.copy( + lastSuccessfulResults = state.lastSuccessfulResults + newResults, + currentlyUsedClassesDirs = state.currentlyUsedClassesDirs + newUserClassesDir + ) -> previousResult + } } logger.debug(s"Scheduling compilation for ${project.name}...") - // Replace client-specific last successful with the most recent result - val mostRecentSuccessful = getMostRecentSuccessfulResultAtomically - - val isUnsubscribed = AtomicBoolean(false) - val newBundle = bundle.copy(lastSuccessful = mostRecentSuccessful) - val compileAndUnsubscribe = { - compile(newBundle) - .doOnFinish(_ => Task(logger.observer.onComplete())) - .map { result => - // Unregister deduplication atomically and register last successful if any - processResultAtomically( - result, - project, - bundle.uniqueInputs, - isUnsubscribed, - logger - ) - } - .memoize // Without memoization, there is no deduplication - } + getMostRecentSuccessfulResultAtomically + .map { mostRecentSuccessful => + val isUnsubscribed = AtomicBoolean(false) + val newBundle = bundle.copy(lastSuccessful = mostRecentSuccessful) + val compileAndUnsubscribe = compile(newBundle) + .doOnFinish(_ => Task(logger.observer.onComplete())) + .flatMap { result => + // Unregister deduplication atomically and register last successful if any + processResultAtomically( + result, + project, + bundle.uniqueInputs, + isUnsubscribed, + logger + ) + } + .memoize + + RunningCompilation( + compileAndUnsubscribe, + mostRecentSuccessful, + isUnsubscribed, + bundle.mirror, + client + ) // Without memoization, there is no deduplication + } - RunningCompilation( - compileAndUnsubscribe, - mostRecentSuccessful, - isUnsubscribed, - bundle.mirror, - client - ) } private def processResultAtomically( @@ -195,27 +214,36 @@ object CompileGatekeeper { oinputs: UniqueCompileInputs, isAlreadyUnsubscribed: AtomicBoolean, logger: Logger - ): Dag[PartialCompileResult] = { - - def cleanUpAfterCompilationError[T](result: T): T = { - if (!isAlreadyUnsubscribed.get) { - // Remove running compilation if host compilation hasn't unsubscribed (maybe it's blocked) - runningCompilations.remove(oinputs) - } - - result + ): Task[Dag[PartialCompileResult]] = { + + def cleanUpAfterCompilationError[T](result: T): Task[T] = { + Task { + if (!isAlreadyUnsubscribed.get) { + // Remove running compilation if host compilation hasn't unsubscribed (maybe it's blocked) + Task.liftMonixTaskUncancellable { + compilerStateRef.update { state => + state.copy(runningCompilations = state.runningCompilations - oinputs) + } + } + } else + Task.unit + }.flatten.map(_ => result) } // Unregister deduplication atomically and register last successful if any - PartialCompileResult.mapEveryResult(resultDag) { + PartialCompileResult.mapEveryResultTask(resultDag) { case s: PartialSuccess => - val processedResult = s.result.map { (result: ResultBundle) => - result.successful match { - case None => cleanUpAfterCompilationError(result) - case Some(res) => - unregisterDeduplicationAndRegisterSuccessful(project, oinputs, res, logger) - } - result + val processedResult = s.result.flatMap { (result: ResultBundle) => + result.successful + .fold(cleanUpAfterCompilationError(result)) { res => + unregisterDeduplicationAndRegisterSuccessful( + project, + oinputs, + res, + logger + ) + .map(_ => result) + } } /** @@ -223,7 +251,7 @@ object CompileGatekeeper { * memoized for correctness reasons. The result task can be called * several times by the compilation engine driving the execution. */ - s.copy(result = processedResult.memoize) + Task(s.copy(result = processedResult.memoize)) case result => cleanUpAfterCompilationError(result) } @@ -240,26 +268,33 @@ object CompileGatekeeper { oracleInputs: UniqueCompileInputs, successful: LastSuccessfulResult, logger: Logger - ): Unit = { - runningCompilations.compute( - oracleInputs, - (_: UniqueCompileInputs, _: RunningCompilation) => { - lastSuccessfulResults.compute(project.uniqueId, (_, _) => successful) - null + ): Task[Unit] = Task.liftMonixTaskUncancellable { + compilerStateRef + .update { state => + val newSuccessfulResults = (project.uniqueId, successful) + if (state.runningCompilations.contains(oracleInputs)) { + state + .copy( + lastSuccessfulResults = state.lastSuccessfulResults + newSuccessfulResults, + runningCompilations = (state.runningCompilations - oracleInputs) + ) + } else { + state + } } - ) - - logger.debug( - s"Recording new last successful request for ${project.name} associated with ${successful.classesDir}" - ) + .map { _ => + logger.debug( + s"Recording new last successful request for ${project.name} associated with ${successful.classesDir}" + ) - () + () + } } // Expose clearing mechanism so that it can be invoked in the tests and community build runner - private[bloop] def clearSuccessfulResults(): Unit = { - lastSuccessfulResults.synchronized { - lastSuccessfulResults.clear() - } - } +// private[bloop] def clearSuccessfulResults(): Unit = { +// lastSuccessfulResults.synchronized { +// lastSuccessfulResults.clear() +// } +// } } diff --git a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGraph.scala b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGraph.scala index 1aab0af1ef..8d7aef5b21 100644 --- a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGraph.scala +++ b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGraph.scala @@ -133,223 +133,227 @@ object CompileGraph { withBundle { bundle0 => val logger = bundle0.logger - val (runningCompilation, deduplicate) = - CompileGatekeeper.findRunningCompilationAtomically(inputs, bundle0, client, compile) - val bundle = bundle0.copy(lastSuccessful = runningCompilation.usedLastSuccessful) - - if (!deduplicate) { - runningCompilation.traversal - } else { - val rawLogger = logger.underlying - rawLogger.info( - s"Deduplicating compilation of ${bundle.project.name} from ${runningCompilation.client}" - ) - val reporter = bundle.reporter.underlying - // Don't use `bundle.lastSuccessful`, it's not the final input to `compile` - val analysis = runningCompilation.usedLastSuccessful.previous.analysis().toOption - val previousSuccessfulProblems = - Compiler.previousProblemsFromSuccessfulCompilation(analysis) - val wasPreviousSuccessful = bundle.latestResult match { - case Compiler.Result.Ok(_) => true - case _ => false - } - val previousProblems = - Compiler.previousProblemsFromResult(bundle.latestResult, previousSuccessfulProblems) - - val clientClassesObserver = client.getClassesObserverFor(bundle.project) - - // Replay events asynchronously to waiting for the compilation result - import scala.concurrent.duration.FiniteDuration - import monix.execution.exceptions.UpstreamTimeoutException - val disconnectionTime = SystemProperties.getCompileDisconnectionTime(rawLogger) - val replayEventsTask = runningCompilation.mirror - .timeoutOnSlowUpstream(disconnectionTime) - .foreachL { - case Left(action) => - action match { - case ReporterAction.EnableFatalWarnings => - reporter.enableFatalWarnings() - case ReporterAction.ReportStartCompilation => - reporter.reportStartCompilation(previousProblems, wasPreviousSuccessful) - case a: ReporterAction.ReportStartIncrementalCycle => - reporter.reportStartIncrementalCycle(a.sources, a.outputDirs) - case a: ReporterAction.ReportProblem => reporter.log(a.problem) - case ReporterAction.PublishDiagnosticsSummary => - reporter.printSummary() - case a: ReporterAction.ReportNextPhase => - reporter.reportNextPhase(a.phase, a.sourceFile) - case a: ReporterAction.ReportCompilationProgress => - reporter.reportCompilationProgress(a.progress, a.total) - case a: ReporterAction.ReportEndIncrementalCycle => - reporter.reportEndIncrementalCycle(a.durationMs, a.result) - case ReporterAction.ReportCancelledCompilation => - reporter.reportCancelledCompilation() - case a: ReporterAction.ProcessEndCompilation => - a.code match { - case BspStatusCode.Cancelled | BspStatusCode.Error => - reporter.processEndCompilation(previousProblems, a.code, None, None) - reporter.reportEndCompilation() - case _ => - /* - * Only process the end, don't report it. It's only safe to - * report when all the client tasks have been run and the - * analysis/classes dirs are fully populated so that clients - * can use `taskFinish` notifications as a signal to process them. - */ - reporter.processEndCompilation( - previousProblems, - a.code, - Some(clientClassesObserver.classesDir), - Some(bundle.out.analysisOut) - ) + + CompileGatekeeper + .findRunningCompilationAtomically(inputs, bundle0, client, compile) + .flatMap { a => + val (runningCompilation, deduplicate) = a + val bundle = bundle0.copy(lastSuccessful = runningCompilation.usedLastSuccessful) + + if (!deduplicate) { + runningCompilation.traversal + } else { + val rawLogger = logger.underlying + rawLogger.info( + s"Deduplicating compilation of ${bundle.project.name} from ${runningCompilation.client}" + ) + val reporter = bundle.reporter.underlying + // Don't use `bundle.lastSuccessful`, it's not the final input to `compile` + val analysis = runningCompilation.usedLastSuccessful.previous.analysis().toOption + val previousSuccessfulProblems = + Compiler.previousProblemsFromSuccessfulCompilation(analysis) + val wasPreviousSuccessful = bundle.latestResult match { + case Compiler.Result.Ok(_) => true + case _ => false + } + val previousProblems = + Compiler.previousProblemsFromResult(bundle.latestResult, previousSuccessfulProblems) + + val clientClassesObserver = client.getClassesObserverFor(bundle.project) + + // Replay events asynchronously to waiting for the compilation result + import scala.concurrent.duration.FiniteDuration + import monix.execution.exceptions.UpstreamTimeoutException + val disconnectionTime = SystemProperties.getCompileDisconnectionTime(rawLogger) + val replayEventsTask = runningCompilation.mirror + .timeoutOnSlowUpstream(disconnectionTime) + .foreachL { + case Left(action) => + action match { + case ReporterAction.EnableFatalWarnings => + reporter.enableFatalWarnings() + case ReporterAction.ReportStartCompilation => + reporter.reportStartCompilation(previousProblems, wasPreviousSuccessful) + case a: ReporterAction.ReportStartIncrementalCycle => + reporter.reportStartIncrementalCycle(a.sources, a.outputDirs) + case a: ReporterAction.ReportProblem => reporter.log(a.problem) + case ReporterAction.PublishDiagnosticsSummary => + reporter.printSummary() + case a: ReporterAction.ReportNextPhase => + reporter.reportNextPhase(a.phase, a.sourceFile) + case a: ReporterAction.ReportCompilationProgress => + reporter.reportCompilationProgress(a.progress, a.total) + case a: ReporterAction.ReportEndIncrementalCycle => + reporter.reportEndIncrementalCycle(a.durationMs, a.result) + case ReporterAction.ReportCancelledCompilation => + reporter.reportCancelledCompilation() + case a: ReporterAction.ProcessEndCompilation => + a.code match { + case BspStatusCode.Cancelled | BspStatusCode.Error => + reporter.processEndCompilation(previousProblems, a.code, None, None) + reporter.reportEndCompilation() + case _ => + /* + * Only process the end, don't report it. It's only safe to + * report when all the client tasks have been run and the + * analysis/classes dirs are fully populated so that clients + * can use `taskFinish` notifications as a signal to process them. + */ + reporter.processEndCompilation( + previousProblems, + a.code, + Some(clientClassesObserver.classesDir), + Some(bundle.out.analysisOut) + ) + } + } + case Right(action) => + action match { + case LoggerAction.LogErrorMessage(msg) => rawLogger.error(msg) + case LoggerAction.LogWarnMessage(msg) => rawLogger.warn(msg) + case LoggerAction.LogInfoMessage(msg) => rawLogger.info(msg) + case LoggerAction.LogDebugMessage(msg) => + rawLogger.debug(msg) + case LoggerAction.LogTraceMessage(msg) => + rawLogger.debug(msg) } } - case Right(action) => - action match { - case LoggerAction.LogErrorMessage(msg) => rawLogger.error(msg) - case LoggerAction.LogWarnMessage(msg) => rawLogger.warn(msg) - case LoggerAction.LogInfoMessage(msg) => rawLogger.info(msg) - case LoggerAction.LogDebugMessage(msg) => - rawLogger.debug(msg) - case LoggerAction.LogTraceMessage(msg) => - rawLogger.debug(msg) + .materialize + .map { + case Success(_) => DeduplicationResult.Ok + case Failure(_: UpstreamTimeoutException) => + DeduplicationResult.DisconnectFromDeduplication + case Failure(t) => DeduplicationResult.DeduplicationError(t) } - } - .materialize - .map { - case Success(_) => DeduplicationResult.Ok - case Failure(_: UpstreamTimeoutException) => - DeduplicationResult.DisconnectFromDeduplication - case Failure(t) => DeduplicationResult.DeduplicationError(t) - } - /* The task set up by another process whose memoized result we're going to - * reuse. To prevent blocking compilations, we execute this task (which will - * block until its completion is done) in the IO thread pool, which is - * unbounded. This makes sure that the blocking threads *never* block - * the computation pool, which could produce a hang in the build server. - */ - val runningCompilationTask = - runningCompilation.traversal.executeOn(ExecutionContext.ioScheduler) - - val deduplicateStreamSideEffectsHandle = - replayEventsTask.runToFuture(ExecutionContext.ioScheduler) - - /** - * Deduplicate and change the implementation of the task returning the - * deduplicate compiler result to trigger a syncing process to keep the - * client external classes directory up-to-date with the new classes - * directory. This copying process blocks until the background IO work - * of the deduplicated compilation result has been finished. Note that - * this mechanism allows pipelined compilations to perform this IO only - * when the full compilation of a module is finished. - */ - val obtainResultFromDeduplication = runningCompilationTask.map { results => - PartialCompileResult.mapEveryResult(results) { - case s @ PartialSuccess(bundle, compilerResult) => - val newCompilerResult = compilerResult.flatMap { results => - results.fromCompiler match { - case s: Compiler.Result.Success => - // Wait on new classes to be populated for correctness - val runningBackgroundTasks = s.backgroundTasks - .trigger(clientClassesObserver, reporter, bundle.tracer, logger) - .runAsync(ExecutionContext.ioScheduler) - Task.now(results.copy(runningBackgroundTasks = runningBackgroundTasks)) - case _: Compiler.Result.Cancelled => - // Make sure to cancel the deduplicating task if compilation is cancelled - deduplicateStreamSideEffectsHandle.cancel() - Task.now(results) - case _ => Task.now(results) - } + /* The task set up by another process whose memoized result we're going to + * reuse. To prevent blocking compilations, we execute this task (which will + * block until its completion is done) in the IO thread pool, which is + * unbounded. This makes sure that the blocking threads *never* block + * the computation pool, which could produce a hang in the build server. + */ + val runningCompilationTask = + runningCompilation.traversal.executeOn(ExecutionContext.ioScheduler) + + val deduplicateStreamSideEffectsHandle = + replayEventsTask.runToFuture(ExecutionContext.ioScheduler) + + /** + * Deduplicate and change the implementation of the task returning the + * deduplicate compiler result to trigger a syncing process to keep the + * client external classes directory up-to-date with the new classes + * directory. This copying process blocks until the background IO work + * of the deduplicated compilation result has been finished. Note that + * this mechanism allows pipelined compilations to perform this IO only + * when the full compilation of a module is finished. + */ + val obtainResultFromDeduplication = runningCompilationTask.map { results => + PartialCompileResult.mapEveryResult(results) { + case s @ PartialSuccess(bundle, compilerResult) => + val newCompilerResult = compilerResult.flatMap { results => + results.fromCompiler match { + case s: Compiler.Result.Success => + // Wait on new classes to be populated for correctness + val runningBackgroundTasks = s.backgroundTasks + .trigger(clientClassesObserver, reporter, bundle.tracer, logger) + .runAsync(ExecutionContext.ioScheduler) + Task.now(results.copy(runningBackgroundTasks = runningBackgroundTasks)) + case _: Compiler.Result.Cancelled => + // Make sure to cancel the deduplicating task if compilation is cancelled + deduplicateStreamSideEffectsHandle.cancel() + Task.now(results) + case _ => Task.now(results) + } + } + s.copy(result = newCompilerResult) + case result => result } - s.copy(result = newCompilerResult) - case result => result - } - } + } - val compileAndDeduplicate = Task - .chooseFirstOf( - obtainResultFromDeduplication, - Task.fromFuture(deduplicateStreamSideEffectsHandle) - ) - .executeOn(ExecutionContext.ioScheduler) - - val finalCompileTask = compileAndDeduplicate.flatMap { - case Left((result, deduplicationFuture)) => - Task.fromFuture(deduplicationFuture).map(_ => result) - case Right((compilationFuture, deduplicationResult)) => - deduplicationResult match { - case DeduplicationResult.Ok => Task.fromFuture(compilationFuture) - case DeduplicationResult.DeduplicationError(t) => - rawLogger.trace(t) - val failedDeduplicationResult = Compiler.Result.GlobalError( - s"Unexpected error while deduplicating compilation for ${inputs.project.name}: ${t.getMessage}", - Some(t) - ) - - /* - * When an error happens while replaying all events of the - * deduplicated compilation, we keep track of the error, wait - * until the deduplicated compilation finishes and then we - * replace the result by a failed result that informs the - * client compilation was not successfully deduplicated. - */ - Task.fromFuture(compilationFuture).map { results => - PartialCompileResult.mapEveryResult(results) { (p: PartialCompileResult) => - p match { - case s: PartialSuccess => - val failedBundle = ResultBundle(failedDeduplicationResult, None, None) - s.copy(result = s.result.map(_ => failedBundle)) - case result => result + val compileAndDeduplicate = Task + .chooseFirstOf( + obtainResultFromDeduplication, + Task.fromFuture(deduplicateStreamSideEffectsHandle) + ) + .executeOn(ExecutionContext.ioScheduler) + + val finalCompileTask = compileAndDeduplicate.flatMap { + case Left((result, deduplicationFuture)) => + Task.fromFuture(deduplicationFuture).map(_ => result) + case Right((compilationFuture, deduplicationResult)) => + deduplicationResult match { + case DeduplicationResult.Ok => Task.fromFuture(compilationFuture) + case DeduplicationResult.DeduplicationError(t) => + rawLogger.trace(t) + val failedDeduplicationResult = Compiler.Result.GlobalError( + s"Unexpected error while deduplicating compilation for ${inputs.project.name}: ${t.getMessage}", + Some(t) + ) + + /* + * When an error happens while replaying all events of the + * deduplicated compilation, we keep track of the error, wait + * until the deduplicated compilation finishes and then we + * replace the result by a failed result that informs the + * client compilation was not successfully deduplicated. + */ + Task.fromFuture(compilationFuture).map { results => + PartialCompileResult.mapEveryResult(results) { (p: PartialCompileResult) => + p match { + case s: PartialSuccess => + val failedBundle = ResultBundle(failedDeduplicationResult, None, None) + s.copy(result = s.result.map(_ => failedBundle)) + case result => result + } + } } - } - } - case DeduplicationResult.DisconnectFromDeduplication => - /* - * Deduplication timed out after no compilation updates were - * recorded. In theory, this could happen because a rogue - * compilation process has stalled or is blocked. To ensure - * deduplicated clients always make progress, we now proceed - * with: - * + case DeduplicationResult.DisconnectFromDeduplication => + /* + * Deduplication timed out after no compilation updates were + * recorded. In theory, this could happen because a rogue + * compilation process has stalled or is blocked. To ensure + * deduplicated clients always make progress, we now proceed + * with: + * * 1. Cancelling the dead-looking compilation, hoping that the - * process will wake up at some point and stop running. - * 2. Shutting down the deduplication and triggering a new - * compilation. If there are several clients deduplicating this - * compilation, they will compete to start the compilation again - * with new compile inputs, as they could have already changed. - * 3. Reporting the end of compilation in case it hasn't been - * reported. Clients must handle two end compilation notifications - * gracefully. - * 4. Display the user that the deduplication was cancelled and a - * new compilation was scheduled. - */ - - CompileGatekeeper.disconnectDeduplicationFromRunning( - bundle.uniqueInputs, - runningCompilation - ) - - compilationFuture.cancel() - reporter.processEndCompilation(Nil, StatusCode.Cancelled, None, None) - reporter.reportEndCompilation() - - logger.displayWarningToUser( - s"""Disconnecting from deduplication of ongoing compilation for '${inputs.project.name}' - |No progress update for ${(disconnectionTime: FiniteDuration) - .toString()} caused bloop to cancel compilation and schedule a new compile. + * process will wake up at some point and stop running. + * 2. Shutting down the deduplication and triggering a new + * compilation. If there are several clients deduplicating this + * compilation, they will compete to start the compilation again + * with new compile inputs, as they could have already changed. + * 3. Reporting the end of compilation in case it hasn't been + * reported. Clients must handle two end compilation notifications + * gracefully. + * 4. Display the user that the deduplication was cancelled and a + * new compilation was scheduled. + */ + + CompileGatekeeper.disconnectDeduplicationFromRunning( + bundle.uniqueInputs, + runningCompilation + ) + + compilationFuture.cancel() + reporter.processEndCompilation(Nil, StatusCode.Cancelled, None, None) + reporter.reportEndCompilation() + + logger.displayWarningToUser( + s"""Disconnecting from deduplication of ongoing compilation for '${inputs.project.name}' + |No progress update for ${(disconnectionTime: FiniteDuration) + .toString()} caused bloop to cancel compilation and schedule a new compile. """.stripMargin - ) + ) - setupAndDeduplicate(client, inputs, setup)(compile) + setupAndDeduplicate(client, inputs, setup)(compile) + } } - } - bundle.tracer.traceTask(s"deduplicating ${bundle.project.name}") { _ => - finalCompileTask.executeOn(ExecutionContext.ioScheduler) + bundle.tracer.traceTask(s"deduplicating ${bundle.project.name}") { _ => + finalCompileTask.executeOn(ExecutionContext.ioScheduler) + } + } } - } } } diff --git a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileResult.scala b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileResult.scala index 1ed33a9510..f99344b0f2 100644 --- a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileResult.scala +++ b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileResult.scala @@ -32,6 +32,16 @@ object PartialCompileResult { } } + def mapEveryResultTask( + results: Dag[PartialCompileResult] + )(f: PartialCompileResult => Task[PartialCompileResult]): Task[Dag[PartialCompileResult]] = { + results match { + case Leaf(result) => f(result).map(Leaf(_)) + case Parent(result, children) => f(result).map(Parent(_, children)) + case Aggregate(_) => sys.error("Unexpected aggregate node in compile result!") + } + } + /** * Turns a partial compile result to a full one. In the case of normal * compilation, this is an instant operation since the task returning the diff --git a/frontend/src/test/scala/bloop/DeduplicationSpec.scala b/frontend/src/test/scala/bloop/DeduplicationSpec.scala index c4015a7a1d..e0a77a3609 100644 --- a/frontend/src/test/scala/bloop/DeduplicationSpec.scala +++ b/frontend/src/test/scala/bloop/DeduplicationSpec.scala @@ -37,874 +37,874 @@ object DeduplicationSpec extends bloop.bsp.BspBaseSuite { val deduplicated = logger.infos.exists(_.startsWith("Deduplicating compilation")) if (isDeduplicated) assert(deduplicated) else assert(!deduplicated) } - - test("three concurrent clients deduplicate compilation") { - val logger = new RecordingLogger(ansiCodesSupported = false) - val logger1 = new RecordingLogger(ansiCodesSupported = false) - val logger2 = new RecordingLogger(ansiCodesSupported = false) - val logger3 = new RecordingLogger(ansiCodesSupported = false) - BuildUtil.testSlowBuild(logger) { build => - val state = new TestState(build.state) - val compiledMacrosState = state.compile(build.macroProject) - assert(compiledMacrosState.status == ExitStatus.Ok) - assertValidCompilationState(compiledMacrosState, List(build.macroProject)) - assertNoDiff( - logger.compilingInfos.mkString(lineSeparator), - s""" - |Compiling macros (1 Scala source) - """.stripMargin - ) - - val compileStartPromises = - new mutable.HashMap[scalabsp.BuildTargetIdentifier, Promise[Unit]]() - val startedProjectCompilation = Promise[Unit]() - compileStartPromises.put(build.userProject.bspId, startedProjectCompilation) - - val projects = List(build.macroProject, build.userProject) - loadBspState( - build.workspace, - projects, - logger1, - compileStartPromises = Some(compileStartPromises) - ) { bspState => - val firstCompilation = bspState.compileHandle(build.userProject) - - val secondCompilation = waitUntilStartAndCompile( - compiledMacrosState, - build.userProject, - startedProjectCompilation, - logger2 - ) - - val thirdCompilation = waitUntilStartAndCompile( - compiledMacrosState, - build.userProject, - startedProjectCompilation, - logger3 - ) - - val firstCompiledState = waitInSeconds(firstCompilation, 10)(logger1.writeToFile("1")) - val (secondCompiledState, thirdCompiledState) = - TestUtil.blockOnTask(mapBoth(secondCompilation, thirdCompilation), 3) - - assert(firstCompiledState.status == ExitStatus.Ok) - assert(secondCompiledState.status == ExitStatus.Ok) - assert(thirdCompiledState.status == ExitStatus.Ok) - - // We get the same class files in all their external directories - assertValidCompilationState(firstCompiledState, projects) - assertValidCompilationState(secondCompiledState, projects) - assertValidCompilationState(thirdCompiledState, projects) - assertSameExternalClassesDirs( - secondCompiledState, - firstCompiledState.toTestState, - projects - ) - assertSameExternalClassesDirs( - thirdCompiledState, - firstCompiledState.toTestState, - projects - ) - - checkDeduplication(logger2, isDeduplicated = true) - checkDeduplication(logger3, isDeduplicated = true) - - assertNoDiff( - firstCompiledState.lastDiagnostics(build.userProject), - """#1: task start 1 - | -> Msg: Compiling user (2 Scala sources) - | -> Data kind: compile-task - |#1: task finish 1 - | -> errors 0, warnings 0 - | -> Msg: Compiled 'user' - | -> Data kind: compile-report - """.stripMargin - ) - - assertNoDiff( - logger2.compilingInfos.mkString(lineSeparator), - s""" - |Compiling user (2 Scala sources) - """.stripMargin - ) - - assertNoDiff( - logger3.compilingInfos.mkString(lineSeparator), - s""" - |Compiling user (2 Scala sources) - """.stripMargin - ) - - val delayFirstNoop = Some(random(0, 20)) - val delaySecondNoop = Some(random(0, 20)) - val noopCompiles = mapBoth( - firstCompiledState.compileHandle(build.userProject, delayFirstNoop), - secondCompiledState.compileHandle(build.userProject, delaySecondNoop) - ) - - val (firstNoopState, secondNoopState) = TestUtil.blockOnTask(noopCompiles, 5) - - assert(firstNoopState.status == ExitStatus.Ok) - assert(secondNoopState.status == ExitStatus.Ok) - assertValidCompilationState(firstNoopState, projects) - assertValidCompilationState(secondNoopState, projects) - assertSameExternalClassesDirs(firstNoopState.toTestState, secondNoopState, projects) - assertSameExternalClassesDirs(firstNoopState.toTestState, thirdCompiledState, projects) - - assertNoDiff( - firstCompiledState.lastDiagnostics(build.userProject), - "" // expect None here since it's a no-op which turns into "" - ) - - // Same check as before because no-op should not show any more input - assertNoDiff( - logger2.compilingInfos.mkString(lineSeparator), - s""" - |Compiling user (2 Scala sources) - """.stripMargin - ) - } - } - } - - test("deduplication removes invalidated class files from all external classes dirs") { - val logger = new RecordingLogger(ansiCodesSupported = false) - BuildUtil.testSlowBuild(logger) { build => - val state = new TestState(build.state) - val compiledMacrosState = state.compile(build.macroProject) - assert(compiledMacrosState.status == ExitStatus.Ok) - assertValidCompilationState(compiledMacrosState, List(build.macroProject)) - assertNoDiff( - logger.compilingInfos.mkString(lineSeparator), - s""" - |Compiling macros (1 Scala source) - """.stripMargin - ) - - val bspLogger = new RecordingLogger(ansiCodesSupported = false) - val cliLogger = new RecordingLogger(ansiCodesSupported = false) - - val compileStartPromises = - new mutable.HashMap[scalabsp.BuildTargetIdentifier, Promise[Unit]]() - val startedFirstCompilation = Promise[Unit]() - compileStartPromises.put(build.userProject.bspId, startedFirstCompilation) - - val projects = List(build.macroProject, build.userProject) - loadBspState( - build.workspace, - projects, - bspLogger, - compileStartPromises = Some(compileStartPromises) - ) { bspState => - val firstCompilation = bspState.compileHandle(build.userProject) - val firstCliCompilation = - waitUntilStartAndCompile( - compiledMacrosState, - build.userProject, - startedFirstCompilation, - cliLogger - ) - - val firstCompiledState = - Await.result(firstCompilation, FiniteDuration(10, TimeUnit.SECONDS)) - val firstCliCompiledState = - Await.result(firstCliCompilation, FiniteDuration(1, TimeUnit.SECONDS)) - - assert(firstCompiledState.status == ExitStatus.Ok) - assert(firstCliCompiledState.status == ExitStatus.Ok) - - // We get the same class files in all their external directories - assertValidCompilationState(firstCompiledState, projects) - assertValidCompilationState(firstCliCompiledState, projects) - assertSameExternalClassesDirs( - firstCliCompiledState, - firstCompiledState.toTestState, - projects - ) - - checkDeduplication(cliLogger, isDeduplicated = true) - - assertNoDiff( - firstCompiledState.lastDiagnostics(build.userProject), - """#1: task start 1 - | -> Msg: Compiling user (2 Scala sources) - | -> Data kind: compile-task - |#1: task finish 1 - | -> errors 0, warnings 0 - | -> Msg: Compiled 'user' - | -> Data kind: compile-report - """.stripMargin - ) - - assertNoDiff( - cliLogger.compilingInfos.mkString(lineSeparator), - s""" - |Compiling user (2 Scala sources) - """.stripMargin - ) - - object Sources { - // A modified version of `User2` that instead renames to `User3` - val `User2.scala` = - """/main/scala/User2.scala - |package user - | - |object User3 extends App { - | macros.SleepMacro.sleep() - |} - """.stripMargin - } - - val startedSecondCompilation = Promise[Unit]() - compileStartPromises.put(build.userProject.bspId, startedSecondCompilation) - - val `User2.scala` = build.userProject.srcFor("/main/scala/User2.scala") - assertIsFile(writeFile(`User2.scala`, Sources.`User2.scala`)) - val secondCompilation = firstCompiledState.compileHandle(build.userProject) - val secondCliCompilation = - waitUntilStartAndCompile( - firstCliCompiledState, - build.userProject, - startedSecondCompilation, - cliLogger - ) - - val secondCompiledState = - Await.result(secondCompilation, FiniteDuration(5, TimeUnit.SECONDS)) - val secondCliCompiledState = - Await.result(secondCliCompilation, FiniteDuration(500, TimeUnit.MILLISECONDS)) - - assert(secondCompiledState.status == ExitStatus.Ok) - assert(secondCliCompiledState.status == ExitStatus.Ok) - assertValidCompilationState(secondCompiledState, projects) - assertValidCompilationState(secondCliCompiledState, projects) - - assertNonExistingCompileProduct( - secondCompiledState.toTestState, - build.userProject, - RelativePath("User2.class") - ) - - assertNonExistingCompileProduct( - secondCompiledState.toTestState, - build.userProject, - RelativePath("User2$.class") - ) - - assertNonExistingCompileProduct( - secondCliCompiledState, - build.userProject, - RelativePath("User2.class") - ) - - assertNonExistingCompileProduct( - secondCliCompiledState, - build.userProject, - RelativePath("User2$.class") - ) - - assertSameExternalClassesDirs( - secondCompiledState.toTestState, - secondCliCompiledState, - projects - ) - - assertNoDiff( - cliLogger.compilingInfos.mkString(lineSeparator), - s""" - |Compiling user (2 Scala sources) - |Compiling user (1 Scala source) - """.stripMargin - ) - - assertNoDiff( - secondCompiledState.lastDiagnostics(build.userProject), - """#2: task start 2 - | -> Msg: Compiling user (1 Scala source) - | -> Data kind: compile-task - |#2: task finish 2 - | -> errors 0, warnings 0 - | -> Msg: Compiled 'user' - | -> Data kind: compile-report - """.stripMargin - ) - } - } - } - - test("deduplication doesn't work if project definition changes") { - val logger = new RecordingLogger(ansiCodesSupported = false) - BuildUtil.testSlowBuild(logger) { build => - val state = new TestState(build.state) - val projects = List(build.macroProject, build.userProject) - val compiledMacrosState = state.compile(build.macroProject) - - assert(compiledMacrosState.status == ExitStatus.Ok) - assertValidCompilationState(compiledMacrosState, List(build.macroProject)) - assertNoDiff( - logger.compilingInfos.mkString(lineSeparator), - s""" - |Compiling macros (1 Scala source) - """.stripMargin - ) - - val bspLogger = new RecordingLogger(ansiCodesSupported = false) - val cliLogger = new RecordingLogger(ansiCodesSupported = false) - - object Sources { - val `User2.scala` = - """/main/scala/User2.scala - |package user - | - |object User2 extends App { - | // Should report warning with -Ywarn-numeric-widen - | val i: Long = 1.toInt - | macros.SleepMacro.sleep() - |} - """.stripMargin - } - - writeFile(build.userProject.srcFor("/main/scala/User2.scala"), Sources.`User2.scala`) - loadBspState(build.workspace, projects, bspLogger) { bspState => - val firstCompilation = bspState.compileHandle(build.userProject) - - val changeOpts = (s: Config.Scala) => s.copy(options = "-Ywarn-numeric-widen" :: s.options) - val newFutureProject = build.userProject.rewriteProject(changeOpts) - - val firstCliCompilation = { - compiledMacrosState - .withLogger(cliLogger) - .compileHandle( - build.userProject, - Some(FiniteDuration(2, TimeUnit.SECONDS)), - beforeTask = Task { - // Write config file before forcing second compilation - reloadWithNewProject(newFutureProject, compiledMacrosState).withLogger(cliLogger) - } - ) - } - - val firstCompiledState = - Await.result(firstCompilation, FiniteDuration(10, TimeUnit.SECONDS)) - val firstCliCompiledState = - Await.result(firstCliCompilation, FiniteDuration(10, TimeUnit.SECONDS)) - - assert(firstCompiledState.status == ExitStatus.Ok) - assert(firstCliCompiledState.status == ExitStatus.Ok) - - // assertValidCompilationState(firstCompiledState, projects) - assertValidCompilationState(firstCliCompiledState, projects) - assertDifferentExternalClassesDirs( - firstCliCompiledState, - firstCompiledState.toTestState, - projects - ) - - checkDeduplication(cliLogger, isDeduplicated = false) - - // First compilation should not report warning - assertNoDiff( - firstCompiledState.lastDiagnostics(build.userProject), - """#1: task start 1 - | -> Msg: Compiling user (2 Scala sources) - | -> Data kind: compile-task - |#1: task finish 1 - | -> errors 0, warnings 0 - | -> Msg: Compiled 'user' - | -> Data kind: compile-report - """.stripMargin - ) - - assertNoDiff( - cliLogger.compilingInfos.mkString(lineSeparator), - s""" - |Compiling user (2 Scala sources) - """.stripMargin - ) - - // Second compilation should be independent from 1 and report warning - assertNoDiff( - cliLogger.warnings.mkString(lineSeparator), - s"""| [E1] ${TestUtil.universalPath("user/src/main/scala/User2.scala")}:5:19 - | implicit numeric widening - | L5: val i: Long = 1.toInt - | ^ - |${TestUtil.universalPath("user/src/main/scala/User2.scala")}: L5 [E1] - |""".stripMargin - ) - } - } - } - - test("three concurrent clients receive error diagnostics appropriately") { - TestUtil.withinWorkspace { workspace => - object Sources { - val `A.scala` = - """/A.scala - |package macros - | - |import scala.reflect.macros.blackbox.Context - |import scala.language.experimental.macros - | - |object SleepMacro { - | def sleep(): Unit = macro sleepImpl - | def sleepImpl(c: Context)(): c.Expr[Unit] = { - | import c.universe._ - | Thread.sleep(1000) - | reify { () } - | } - |}""".stripMargin - - val `B.scala` = - """/B.scala - |object B { - | def foo(s: String): String = s.toString - |} - """.stripMargin - - // Second (non-compiling) version of `B` - val `B2.scala` = - """/B.scala - |object B { - | macros.SleepMacro.sleep() - | def foo(i: Int): String = i - |} - """.stripMargin - - // Third (compiling) slow version of `B` - val `B3.scala` = - """/B.scala - |object B { - | macros.SleepMacro.sleep() - | def foo(s: String): String = s.toString - |} - """.stripMargin - } - - val cliLogger1 = new RecordingLogger(ansiCodesSupported = false) - val cliLogger2 = new RecordingLogger(ansiCodesSupported = false) - val bspLogger = new RecordingLogger(ansiCodesSupported = false) - - val `A` = TestProject(workspace, "a", List(Sources.`A.scala`)) - val `B` = TestProject(workspace, "b", List(Sources.`B.scala`), List(`A`)) - - val projects = List(`A`, `B`) - val state = loadState(workspace, projects, cliLogger1) - val compiledState = state.compile(`B`) - - writeFile(`B`.srcFor("B.scala"), Sources.`B2.scala`) - - val compileStartPromises = - new mutable.HashMap[scalabsp.BuildTargetIdentifier, Promise[Unit]]() - val startedFirstCompilation = Promise[Unit]() - compileStartPromises.put(`B`.bspId, startedFirstCompilation) - - loadBspState( - workspace, - projects, - bspLogger, - compileStartPromises = Some(compileStartPromises) - ) { bspState => - // val firstDelay = Some(random(400, 100)) - val firstCompilation = bspState.compileHandle(`B`) - val secondCompilation = - waitUntilStartAndCompile(compiledState, `B`, startedFirstCompilation, cliLogger1) - val thirdCompilation = - waitUntilStartAndCompile(compiledState, `B`, startedFirstCompilation, cliLogger2) - - val firstCompiledState = - Await.result(firstCompilation, FiniteDuration(15, TimeUnit.SECONDS)) - val (secondCompiledState, thirdCompiledState) = - TestUtil.blockOnTask(mapBoth(secondCompilation, thirdCompilation), 3) - - assert(firstCompiledState.status == ExitStatus.CompilationError) - assert(secondCompiledState.status == ExitStatus.CompilationError) - assert(thirdCompiledState.status == ExitStatus.CompilationError) - - // Check we get the same class files in all their external directories - assertInvalidCompilationState( - firstCompiledState, - projects, - existsAnalysisFile = true, - hasPreviousSuccessful = true, - // Classes dir of BSP session is empty because no successful compilation happened - hasSameContentsInClassesDir = false - ) - - assertInvalidCompilationState( - secondCompiledState, - projects, - existsAnalysisFile = true, - hasPreviousSuccessful = true, - hasSameContentsInClassesDir = true - ) - - assertInvalidCompilationState( - thirdCompiledState, - projects, - existsAnalysisFile = true, - hasPreviousSuccessful = true, - hasSameContentsInClassesDir = true - ) - - assertSameExternalClassesDirs(secondCompiledState, thirdCompiledState, projects) - - checkDeduplication(bspLogger, isDeduplicated = false) - checkDeduplication(cliLogger1, isDeduplicated = true) - checkDeduplication(cliLogger2, isDeduplicated = true) - - // We reproduce the same streaming side effects during compilation - assertNoDiff( - firstCompiledState.lastDiagnostics(`B`), - """#1: task start 1 - | -> Msg: Compiling b (1 Scala source) - | -> Data kind: compile-task - |#1: b/src/B.scala - | -> List(Diagnostic(Range(Position(2,28),Position(2,28)),Some(Error),Some(_),Some(_),type mismatch; found : Int required: String,None,None,Some({"actions":[]}))) - | -> reset = true - |#1: task finish 1 - | -> errors 1, warnings 0 - | -> Msg: Compiled 'b' - | -> Data kind: compile-report - """.stripMargin - ) - - assertNoDiff( - cliLogger1.compilingInfos.mkString(lineSeparator), - s""" - |Compiling a (1 Scala source) - |Compiling b (1 Scala source) - |Compiling b (1 Scala source) - """.stripMargin - ) - - assertNoDiff( - cliLogger1.errors.mkString(lineSeparator), - s""" - |[E1] ${TestUtil.universalPath("b/src/B.scala")}:3:29 - | type mismatch; - | found : Int - | required: String - | L3: def foo(i: Int): String = i - | ^ - |${TestUtil.universalPath("b/src/B.scala")}: L3 [E1] - |Failed to compile 'b'""".stripMargin - ) - - assertNoDiff( - cliLogger2.compilingInfos.mkString(lineSeparator), - s""" - |Compiling b (1 Scala source) - """.stripMargin - ) - - val targetB = TestUtil.universalPath("b/src/B.scala") - assertNoDiff( - cliLogger2.errors.mkString(lineSeparator), - s""" - |[E1] ${targetB}:3:29 - | type mismatch; - | found : Int - | required: String - | L3: def foo(i: Int): String = i - | ^ - |${targetB}: L3 [E1] - |Failed to compile 'b'""".stripMargin - ) - - /* Repeat the same but this time the CLI client runs the compilation first */ - - val startedSecondCompilation = Promise[Unit]() - compileStartPromises.put(`B`.bspId, startedSecondCompilation) - - val cliLogger4 = new RecordingLogger(ansiCodesSupported = false) - val cliLogger5 = new RecordingLogger(ansiCodesSupported = false) - - val fourthCompilation = bspState.compileHandle(`B`) - val fifthCompilation = - waitUntilStartAndCompile(thirdCompiledState, `B`, startedSecondCompilation, cliLogger4) - val sixthCompilation = - waitUntilStartAndCompile(thirdCompiledState, `B`, startedSecondCompilation, cliLogger5) - - val secondBspState = - Await.result(fourthCompilation, FiniteDuration(4, TimeUnit.SECONDS)) - val (_, fifthCompiledState) = - TestUtil.blockOnTask(mapBoth(fifthCompilation, sixthCompilation), 2) - - checkDeduplication(bspLogger, isDeduplicated = false) - checkDeduplication(cliLogger4, isDeduplicated = true) - checkDeduplication(cliLogger5, isDeduplicated = true) - - assertNoDiff( - secondBspState.lastDiagnostics(`B`), - """#2: task start 2 - | -> Msg: Compiling b (1 Scala source) - | -> Data kind: compile-task - |#2: b/src/B.scala - | -> List(Diagnostic(Range(Position(2,28),Position(2,28)),Some(Error),Some(_),Some(_),type mismatch; found : Int required: String,None,None,Some({"actions":[]}))) - | -> reset = true - |#2: task finish 2 - | -> errors 1, warnings 0 - | -> Msg: Compiled 'b' - | -> Data kind: compile-report - """.stripMargin - ) - - assertNoDiff( - cliLogger4.compilingInfos.mkString(lineSeparator), - s""" - |Compiling b (1 Scala source) - """.stripMargin - ) - - assertNoDiff( - cliLogger4.errors.mkString(lineSeparator), - s""" - |[E1] ${targetB}:3:29 - | type mismatch; - | found : Int - | required: String - | L3: def foo(i: Int): String = i - | ^ - |${targetB}: L3 [E1] - |Failed to compile 'b'""".stripMargin - ) - - assertNoDiff( - cliLogger5.compilingInfos.mkString(lineSeparator), - s""" - |Compiling b (1 Scala source) - """.stripMargin - ) - - assertNoDiff( - cliLogger5.errors.mkString(lineSeparator), - s""" - |[E1] ${targetB}:3:29 - | type mismatch; - | found : Int - | required: String - | L3: def foo(i: Int): String = i - | ^ - |${targetB}: L3 [E1] - |Failed to compile 'b'""".stripMargin - ) - - writeFile(`B`.srcFor("B.scala"), Sources.`B3.scala`) - - val cliLogger7 = new RecordingLogger(ansiCodesSupported = false) - - val startedThirdCompilation = Promise[Unit]() - compileStartPromises.put(`B`.bspId, startedThirdCompilation) - - /* - * Repeat the same (the CLI client drives the compilation first) but - * this time the latest result in cli client differs with that of the - * BSP client. This test proves that our deduplication logic uses - * client-specific data to populate reporters that then consume the - * stream of compilation events. - */ - - val newCliCompiledState = { - val underlyingState = fifthCompiledState.withLogger(cliLogger7).state - val newPreviousResults = Map( - // Use empty, remember we don't change last successful so incrementality works - fifthCompiledState.getProjectFor(`B`) -> Compiler.Result.Empty - ) - val newResults = underlyingState.results.replacePreviousResults(newPreviousResults) - new TestState(underlyingState.copy(results = newResults)) - } - - val seventhCompilation = secondBspState.compileHandle(`B`) - val eighthCompilation = - waitUntilStartAndCompile(newCliCompiledState, `B`, startedThirdCompilation, cliLogger7) - - val thirdBspState = Await.result(seventhCompilation, FiniteDuration(10, TimeUnit.SECONDS)) - - Await.result(eighthCompilation, FiniteDuration(5, TimeUnit.SECONDS)) - - checkDeduplication(bspLogger, isDeduplicated = false) - checkDeduplication(cliLogger7, isDeduplicated = true) - - assertNoDiff( - cliLogger7.compilingInfos.mkString(lineSeparator), - s""" - |Compiling b (1 Scala source) - """.stripMargin - ) - - assert(cliLogger7.errors.size == 0) - - assertNoDiff( - thirdBspState.lastDiagnostics(`B`), - """#3: task start 3 - | -> Msg: Compiling b (1 Scala source) - | -> Data kind: compile-task - |#3: b/src/B.scala - | -> List() - | -> reset = true - |#3: task finish 3 - | -> errors 0, warnings 0 - | -> Msg: Compiled 'b' - | -> Data kind: compile-report - """.stripMargin - ) - } - } - } - - // TODO(jvican): Compile project of cancelled compilation to ensure no deduplication trace is left - test("cancel deduplicated compilation finishes all clients") { - TestUtil.withinWorkspace { workspace => - object Sources { - val `A.scala` = - """/A.scala - |package macros - | - |import scala.reflect.macros.blackbox.Context - |import scala.language.experimental.macros - | - |object SleepMacro { - | def sleep(): Unit = macro sleepImpl - | def sleepImpl(c: Context)(): c.Expr[Unit] = { - | import c.universe._ - | Thread.sleep(500) - | Thread.sleep(500) - | Thread.sleep(500) - | Thread.sleep(500) - | reify { () } - | } - |}""".stripMargin - - val `B.scala` = - """/B.scala - |object B { - | def foo(s: String): String = s.toString - |} - """.stripMargin - - // Sleep in independent files to force compiler to check for cancelled status - val `B2.scala` = - """/B.scala - |object B { - | def foo(s: String): String = s.toString - | macros.SleepMacro.sleep() - |} - """.stripMargin - - val `Extra.scala` = - """/Extra.scala - |object Extra { def foo(s: String): String = s.toString; macros.SleepMacro.sleep() } - """.stripMargin - - val `Extra2.scala` = - """/Extra2.scala - |object Extra2 { def foo(s: String): String = s.toString; macros.SleepMacro.sleep() } - """.stripMargin - - val `Extra3.scala` = - """/Extra3.scala - |object Extra3 { def foo(s: String): String = s.toString; macros.SleepMacro.sleep() } - """.stripMargin - - val `Extra4.scala` = - """/Extra4.scala - |object Extra4 { def foo(s: String): String = s.toString; macros.SleepMacro.sleep() } - """.stripMargin - - val `Extra5.scala` = - """/Extra5.scala - |object Extra5 { def foo(s: String): String = s.toString; macros.SleepMacro.sleep() } - """.stripMargin - } - - val cliLogger = new RecordingLogger(ansiCodesSupported = false) - val bspLogger = new RecordingLogger(ansiCodesSupported = false) - - val `A` = TestProject(workspace, "a", List(Sources.`A.scala`)) - val `B` = TestProject(workspace, "b", List(Sources.`B.scala`), List(`A`)) - - val projects = List(`A`, `B`) - val state = loadState(workspace, projects, cliLogger) - val compiledState = state.compile(`B`) - - writeFile(`B`.srcFor("B.scala"), Sources.`B2.scala`) - writeFile(`B`.srcFor("Extra.scala", exists = false), Sources.`Extra.scala`) - writeFile(`B`.srcFor("Extra2.scala", exists = false), Sources.`Extra2.scala`) - writeFile(`B`.srcFor("Extra3.scala", exists = false), Sources.`Extra3.scala`) - writeFile(`B`.srcFor("Extra4.scala", exists = false), Sources.`Extra4.scala`) - writeFile(`B`.srcFor("Extra5.scala", exists = false), Sources.`Extra5.scala`) - - val compileStartPromises = - new mutable.HashMap[scalabsp.BuildTargetIdentifier, Promise[Unit]]() - val startedFirstCompilation = Promise[Unit]() - compileStartPromises.put(`B`.bspId, startedFirstCompilation) - - loadBspState( - workspace, - projects, - bspLogger, - compileStartPromises = Some(compileStartPromises) - ) { bspState => - val firstCompilation = - bspState.compileHandle(`B`, userScheduler = Some(ExecutionContext.ioScheduler)) - val secondCompilation = - waitUntilStartAndCompile(compiledState, `B`, startedFirstCompilation, cliLogger) - - val _ = Task - .fromFuture(startedFirstCompilation.future) - .map(_ => { firstCompilation.cancel() }) - .runAsync(ExecutionContext.ioScheduler) - - val (firstCompiledState, secondCompiledState) = - TestUtil.blockOnTask( - mapBoth(firstCompilation, secondCompilation), - 15, - loggers = List(cliLogger, bspLogger), - userScheduler = Some(ExecutionContext.ioScheduler) - ) - - if (firstCompiledState.status == ExitStatus.CompilationError && CrossPlatform.isWindows) { - System.err.println("Ignoring failed cancellation with deduplication on Windows") - } else { - assert(firstCompiledState.status == ExitStatus.CompilationError) - assertCancelledCompilation(firstCompiledState.toTestState, List(`B`)) - assertNoDiff( - bspLogger.infos.filterNot(_.contains("tcp")).mkString(lineSeparator), - """ - |request received: build/initialize - |BSP initialization handshake complete. - |Cancelling request Number(5.0) - """.stripMargin - ) - - assert(secondCompiledState.status == ExitStatus.CompilationError) - assertCancelledCompilation(secondCompiledState, List(`B`)) - - checkDeduplication(bspLogger, isDeduplicated = false) - checkDeduplication(cliLogger, isDeduplicated = true) - - assertNoDiff( - firstCompiledState.lastDiagnostics(`B`), - s""" - |#1: task start 1 - | -> Msg: Compiling b (6 Scala sources) - | -> Data kind: compile-task - |#1: task finish 1 - | -> errors 0, warnings 0 - | -> Msg: Compiled 'b' - | -> Data kind: compile-report - """.stripMargin - ) - - assertNoDiff( - cliLogger.warnings.mkString(lineSeparator), - "Cancelling compilation of b" - ) - } - } - } - } +// +// test("three concurrent clients deduplicate compilation") { +// val logger = new RecordingLogger(ansiCodesSupported = false) +// val logger1 = new RecordingLogger(ansiCodesSupported = false) +// val logger2 = new RecordingLogger(ansiCodesSupported = false) +// val logger3 = new RecordingLogger(ansiCodesSupported = false) +// BuildUtil.testSlowBuild(logger) { build => +// val state = new TestState(build.state) +// val compiledMacrosState = state.compile(build.macroProject) +// assert(compiledMacrosState.status == ExitStatus.Ok) +// assertValidCompilationState(compiledMacrosState, List(build.macroProject)) +// assertNoDiff( +// logger.compilingInfos.mkString(lineSeparator), +// s""" +// |Compiling macros (1 Scala source) +// """.stripMargin +// ) +// +// val compileStartPromises = +// new mutable.HashMap[scalabsp.BuildTargetIdentifier, Promise[Unit]]() +// val startedProjectCompilation = Promise[Unit]() +// compileStartPromises.put(build.userProject.bspId, startedProjectCompilation) +// +// val projects = List(build.macroProject, build.userProject) +// loadBspState( +// build.workspace, +// projects, +// logger1, +// compileStartPromises = Some(compileStartPromises) +// ) { bspState => +// val firstCompilation = bspState.compileHandle(build.userProject) +// +// val secondCompilation = waitUntilStartAndCompile( +// compiledMacrosState, +// build.userProject, +// startedProjectCompilation, +// logger2 +// ) +// +// val thirdCompilation = waitUntilStartAndCompile( +// compiledMacrosState, +// build.userProject, +// startedProjectCompilation, +// logger3 +// ) +// +// val firstCompiledState = waitInSeconds(firstCompilation, 10)(logger1.writeToFile("1")) +// val (secondCompiledState, thirdCompiledState) = +// TestUtil.blockOnTask(mapBoth(secondCompilation, thirdCompilation), 3) +// +// assert(firstCompiledState.status == ExitStatus.Ok) +// assert(secondCompiledState.status == ExitStatus.Ok) +// assert(thirdCompiledState.status == ExitStatus.Ok) +// +// // We get the same class files in all their external directories +// assertValidCompilationState(firstCompiledState, projects) +// assertValidCompilationState(secondCompiledState, projects) +// assertValidCompilationState(thirdCompiledState, projects) +// assertSameExternalClassesDirs( +// secondCompiledState, +// firstCompiledState.toTestState, +// projects +// ) +// assertSameExternalClassesDirs( +// thirdCompiledState, +// firstCompiledState.toTestState, +// projects +// ) +// +// checkDeduplication(logger2, isDeduplicated = true) +// checkDeduplication(logger3, isDeduplicated = true) +// +// assertNoDiff( +// firstCompiledState.lastDiagnostics(build.userProject), +// """#1: task start 1 +// | -> Msg: Compiling user (2 Scala sources) +// | -> Data kind: compile-task +// |#1: task finish 1 +// | -> errors 0, warnings 0 +// | -> Msg: Compiled 'user' +// | -> Data kind: compile-report +// """.stripMargin +// ) +// +// assertNoDiff( +// logger2.compilingInfos.mkString(lineSeparator), +// s""" +// |Compiling user (2 Scala sources) +// """.stripMargin +// ) +// +// assertNoDiff( +// logger3.compilingInfos.mkString(lineSeparator), +// s""" +// |Compiling user (2 Scala sources) +// """.stripMargin +// ) +// +// val delayFirstNoop = Some(random(0, 20)) +// val delaySecondNoop = Some(random(0, 20)) +// val noopCompiles = mapBoth( +// firstCompiledState.compileHandle(build.userProject, delayFirstNoop), +// secondCompiledState.compileHandle(build.userProject, delaySecondNoop) +// ) +// +// val (firstNoopState, secondNoopState) = TestUtil.blockOnTask(noopCompiles, 5) +// +// assert(firstNoopState.status == ExitStatus.Ok) +// assert(secondNoopState.status == ExitStatus.Ok) +// assertValidCompilationState(firstNoopState, projects) +// assertValidCompilationState(secondNoopState, projects) +// assertSameExternalClassesDirs(firstNoopState.toTestState, secondNoopState, projects) +// assertSameExternalClassesDirs(firstNoopState.toTestState, thirdCompiledState, projects) +// +// assertNoDiff( +// firstCompiledState.lastDiagnostics(build.userProject), +// "" // expect None here since it's a no-op which turns into "" +// ) +// +// // Same check as before because no-op should not show any more input +// assertNoDiff( +// logger2.compilingInfos.mkString(lineSeparator), +// s""" +// |Compiling user (2 Scala sources) +// """.stripMargin +// ) +// } +// } +// } +// +// test("deduplication removes invalidated class files from all external classes dirs") { +// val logger = new RecordingLogger(ansiCodesSupported = false) +// BuildUtil.testSlowBuild(logger) { build => +// val state = new TestState(build.state) +// val compiledMacrosState = state.compile(build.macroProject) +// assert(compiledMacrosState.status == ExitStatus.Ok) +// assertValidCompilationState(compiledMacrosState, List(build.macroProject)) +// assertNoDiff( +// logger.compilingInfos.mkString(lineSeparator), +// s""" +// |Compiling macros (1 Scala source) +// """.stripMargin +// ) +// +// val bspLogger = new RecordingLogger(ansiCodesSupported = false) +// val cliLogger = new RecordingLogger(ansiCodesSupported = false) +// +// val compileStartPromises = +// new mutable.HashMap[scalabsp.BuildTargetIdentifier, Promise[Unit]]() +// val startedFirstCompilation = Promise[Unit]() +// compileStartPromises.put(build.userProject.bspId, startedFirstCompilation) +// +// val projects = List(build.macroProject, build.userProject) +// loadBspState( +// build.workspace, +// projects, +// bspLogger, +// compileStartPromises = Some(compileStartPromises) +// ) { bspState => +// val firstCompilation = bspState.compileHandle(build.userProject) +// val firstCliCompilation = +// waitUntilStartAndCompile( +// compiledMacrosState, +// build.userProject, +// startedFirstCompilation, +// cliLogger +// ) +// +// val firstCompiledState = +// Await.result(firstCompilation, FiniteDuration(10, TimeUnit.SECONDS)) +// val firstCliCompiledState = +// Await.result(firstCliCompilation, FiniteDuration(1, TimeUnit.SECONDS)) +// +// assert(firstCompiledState.status == ExitStatus.Ok) +// assert(firstCliCompiledState.status == ExitStatus.Ok) +// +// // We get the same class files in all their external directories +// assertValidCompilationState(firstCompiledState, projects) +// assertValidCompilationState(firstCliCompiledState, projects) +// assertSameExternalClassesDirs( +// firstCliCompiledState, +// firstCompiledState.toTestState, +// projects +// ) +// +// checkDeduplication(cliLogger, isDeduplicated = true) +// +// assertNoDiff( +// firstCompiledState.lastDiagnostics(build.userProject), +// """#1: task start 1 +// | -> Msg: Compiling user (2 Scala sources) +// | -> Data kind: compile-task +// |#1: task finish 1 +// | -> errors 0, warnings 0 +// | -> Msg: Compiled 'user' +// | -> Data kind: compile-report +// """.stripMargin +// ) +// +// assertNoDiff( +// cliLogger.compilingInfos.mkString(lineSeparator), +// s""" +// |Compiling user (2 Scala sources) +// """.stripMargin +// ) +// +// object Sources { +// // A modified version of `User2` that instead renames to `User3` +// val `User2.scala` = +// """/main/scala/User2.scala +// |package user +// | +// |object User3 extends App { +// | macros.SleepMacro.sleep() +// |} +// """.stripMargin +// } +// +// val startedSecondCompilation = Promise[Unit]() +// compileStartPromises.put(build.userProject.bspId, startedSecondCompilation) +// +// val `User2.scala` = build.userProject.srcFor("/main/scala/User2.scala") +// assertIsFile(writeFile(`User2.scala`, Sources.`User2.scala`)) +// val secondCompilation = firstCompiledState.compileHandle(build.userProject) +// val secondCliCompilation = +// waitUntilStartAndCompile( +// firstCliCompiledState, +// build.userProject, +// startedSecondCompilation, +// cliLogger +// ) +// +// val secondCompiledState = +// Await.result(secondCompilation, FiniteDuration(5, TimeUnit.SECONDS)) +// val secondCliCompiledState = +// Await.result(secondCliCompilation, FiniteDuration(500, TimeUnit.MILLISECONDS)) +// +// assert(secondCompiledState.status == ExitStatus.Ok) +// assert(secondCliCompiledState.status == ExitStatus.Ok) +// assertValidCompilationState(secondCompiledState, projects) +// assertValidCompilationState(secondCliCompiledState, projects) +// +// assertNonExistingCompileProduct( +// secondCompiledState.toTestState, +// build.userProject, +// RelativePath("User2.class") +// ) +// +// assertNonExistingCompileProduct( +// secondCompiledState.toTestState, +// build.userProject, +// RelativePath("User2$.class") +// ) +// +// assertNonExistingCompileProduct( +// secondCliCompiledState, +// build.userProject, +// RelativePath("User2.class") +// ) +// +// assertNonExistingCompileProduct( +// secondCliCompiledState, +// build.userProject, +// RelativePath("User2$.class") +// ) +// +// assertSameExternalClassesDirs( +// secondCompiledState.toTestState, +// secondCliCompiledState, +// projects +// ) +// +// assertNoDiff( +// cliLogger.compilingInfos.mkString(lineSeparator), +// s""" +// |Compiling user (2 Scala sources) +// |Compiling user (1 Scala source) +// """.stripMargin +// ) +// +// assertNoDiff( +// secondCompiledState.lastDiagnostics(build.userProject), +// """#2: task start 2 +// | -> Msg: Compiling user (1 Scala source) +// | -> Data kind: compile-task +// |#2: task finish 2 +// | -> errors 0, warnings 0 +// | -> Msg: Compiled 'user' +// | -> Data kind: compile-report +// """.stripMargin +// ) +// } +// } +// } +// +// test("deduplication doesn't work if project definition changes") { +// val logger = new RecordingLogger(ansiCodesSupported = false) +// BuildUtil.testSlowBuild(logger) { build => +// val state = new TestState(build.state) +// val projects = List(build.macroProject, build.userProject) +// val compiledMacrosState = state.compile(build.macroProject) +// +// assert(compiledMacrosState.status == ExitStatus.Ok) +// assertValidCompilationState(compiledMacrosState, List(build.macroProject)) +// assertNoDiff( +// logger.compilingInfos.mkString(lineSeparator), +// s""" +// |Compiling macros (1 Scala source) +// """.stripMargin +// ) +// +// val bspLogger = new RecordingLogger(ansiCodesSupported = false) +// val cliLogger = new RecordingLogger(ansiCodesSupported = false) +// +// object Sources { +// val `User2.scala` = +// """/main/scala/User2.scala +// |package user +// | +// |object User2 extends App { +// | // Should report warning with -Ywarn-numeric-widen +// | val i: Long = 1.toInt +// | macros.SleepMacro.sleep() +// |} +// """.stripMargin +// } +// +// writeFile(build.userProject.srcFor("/main/scala/User2.scala"), Sources.`User2.scala`) +// loadBspState(build.workspace, projects, bspLogger) { bspState => +// val firstCompilation = bspState.compileHandle(build.userProject) +// +// val changeOpts = (s: Config.Scala) => s.copy(options = "-Ywarn-numeric-widen" :: s.options) +// val newFutureProject = build.userProject.rewriteProject(changeOpts) +// +// val firstCliCompilation = { +// compiledMacrosState +// .withLogger(cliLogger) +// .compileHandle( +// build.userProject, +// Some(FiniteDuration(2, TimeUnit.SECONDS)), +// beforeTask = Task { +// // Write config file before forcing second compilation +// reloadWithNewProject(newFutureProject, compiledMacrosState).withLogger(cliLogger) +// } +// ) +// } +// +// val firstCompiledState = +// Await.result(firstCompilation, FiniteDuration(10, TimeUnit.SECONDS)) +// val firstCliCompiledState = +// Await.result(firstCliCompilation, FiniteDuration(10, TimeUnit.SECONDS)) +// +// assert(firstCompiledState.status == ExitStatus.Ok) +// assert(firstCliCompiledState.status == ExitStatus.Ok) +// +// // assertValidCompilationState(firstCompiledState, projects) +// assertValidCompilationState(firstCliCompiledState, projects) +// assertDifferentExternalClassesDirs( +// firstCliCompiledState, +// firstCompiledState.toTestState, +// projects +// ) +// +// checkDeduplication(cliLogger, isDeduplicated = false) +// +// // First compilation should not report warning +// assertNoDiff( +// firstCompiledState.lastDiagnostics(build.userProject), +// """#1: task start 1 +// | -> Msg: Compiling user (2 Scala sources) +// | -> Data kind: compile-task +// |#1: task finish 1 +// | -> errors 0, warnings 0 +// | -> Msg: Compiled 'user' +// | -> Data kind: compile-report +// """.stripMargin +// ) +// +// assertNoDiff( +// cliLogger.compilingInfos.mkString(lineSeparator), +// s""" +// |Compiling user (2 Scala sources) +// """.stripMargin +// ) +// +// // Second compilation should be independent from 1 and report warning +// assertNoDiff( +// cliLogger.warnings.mkString(lineSeparator), +// s"""| [E1] ${TestUtil.universalPath("user/src/main/scala/User2.scala")}:5:19 +// | implicit numeric widening +// | L5: val i: Long = 1.toInt +// | ^ +// |${TestUtil.universalPath("user/src/main/scala/User2.scala")}: L5 [E1] +// |""".stripMargin +// ) +// } +// } +// } +// +// test("three concurrent clients receive error diagnostics appropriately") { +// TestUtil.withinWorkspace { workspace => +// object Sources { +// val `A.scala` = +// """/A.scala +// |package macros +// | +// |import scala.reflect.macros.blackbox.Context +// |import scala.language.experimental.macros +// | +// |object SleepMacro { +// | def sleep(): Unit = macro sleepImpl +// | def sleepImpl(c: Context)(): c.Expr[Unit] = { +// | import c.universe._ +// | Thread.sleep(1000) +// | reify { () } +// | } +// |}""".stripMargin +// +// val `B.scala` = +// """/B.scala +// |object B { +// | def foo(s: String): String = s.toString +// |} +// """.stripMargin +// +// // Second (non-compiling) version of `B` +// val `B2.scala` = +// """/B.scala +// |object B { +// | macros.SleepMacro.sleep() +// | def foo(i: Int): String = i +// |} +// """.stripMargin +// +// // Third (compiling) slow version of `B` +// val `B3.scala` = +// """/B.scala +// |object B { +// | macros.SleepMacro.sleep() +// | def foo(s: String): String = s.toString +// |} +// """.stripMargin +// } +// +// val cliLogger1 = new RecordingLogger(ansiCodesSupported = false) +// val cliLogger2 = new RecordingLogger(ansiCodesSupported = false) +// val bspLogger = new RecordingLogger(ansiCodesSupported = false) +// +// val `A` = TestProject(workspace, "a", List(Sources.`A.scala`)) +// val `B` = TestProject(workspace, "b", List(Sources.`B.scala`), List(`A`)) +// +// val projects = List(`A`, `B`) +// val state = loadState(workspace, projects, cliLogger1) +// val compiledState = state.compile(`B`) +// +// writeFile(`B`.srcFor("B.scala"), Sources.`B2.scala`) +// +// val compileStartPromises = +// new mutable.HashMap[scalabsp.BuildTargetIdentifier, Promise[Unit]]() +// val startedFirstCompilation = Promise[Unit]() +// compileStartPromises.put(`B`.bspId, startedFirstCompilation) +// +// loadBspState( +// workspace, +// projects, +// bspLogger, +// compileStartPromises = Some(compileStartPromises) +// ) { bspState => +// // val firstDelay = Some(random(400, 100)) +// val firstCompilation = bspState.compileHandle(`B`) +// val secondCompilation = +// waitUntilStartAndCompile(compiledState, `B`, startedFirstCompilation, cliLogger1) +// val thirdCompilation = +// waitUntilStartAndCompile(compiledState, `B`, startedFirstCompilation, cliLogger2) +// +// val firstCompiledState = +// Await.result(firstCompilation, FiniteDuration(15, TimeUnit.SECONDS)) +// val (secondCompiledState, thirdCompiledState) = +// TestUtil.blockOnTask(mapBoth(secondCompilation, thirdCompilation), 3) +// +// assert(firstCompiledState.status == ExitStatus.CompilationError) +// assert(secondCompiledState.status == ExitStatus.CompilationError) +// assert(thirdCompiledState.status == ExitStatus.CompilationError) +// +// // Check we get the same class files in all their external directories +// assertInvalidCompilationState( +// firstCompiledState, +// projects, +// existsAnalysisFile = true, +// hasPreviousSuccessful = true, +// // Classes dir of BSP session is empty because no successful compilation happened +// hasSameContentsInClassesDir = false +// ) +// +// assertInvalidCompilationState( +// secondCompiledState, +// projects, +// existsAnalysisFile = true, +// hasPreviousSuccessful = true, +// hasSameContentsInClassesDir = true +// ) +// +// assertInvalidCompilationState( +// thirdCompiledState, +// projects, +// existsAnalysisFile = true, +// hasPreviousSuccessful = true, +// hasSameContentsInClassesDir = true +// ) +// +// assertSameExternalClassesDirs(secondCompiledState, thirdCompiledState, projects) +// +// checkDeduplication(bspLogger, isDeduplicated = false) +// checkDeduplication(cliLogger1, isDeduplicated = true) +// checkDeduplication(cliLogger2, isDeduplicated = true) +// +// // We reproduce the same streaming side effects during compilation +// assertNoDiff( +// firstCompiledState.lastDiagnostics(`B`), +// """#1: task start 1 +// | -> Msg: Compiling b (1 Scala source) +// | -> Data kind: compile-task +// |#1: b/src/B.scala +// | -> List(Diagnostic(Range(Position(2,28),Position(2,28)),Some(Error),Some(_),Some(_),type mismatch; found : Int required: String,None,None,Some({"actions":[]}))) +// | -> reset = true +// |#1: task finish 1 +// | -> errors 1, warnings 0 +// | -> Msg: Compiled 'b' +// | -> Data kind: compile-report +// """.stripMargin +// ) +// +// assertNoDiff( +// cliLogger1.compilingInfos.mkString(lineSeparator), +// s""" +// |Compiling a (1 Scala source) +// |Compiling b (1 Scala source) +// |Compiling b (1 Scala source) +// """.stripMargin +// ) +// +// assertNoDiff( +// cliLogger1.errors.mkString(lineSeparator), +// s""" +// |[E1] ${TestUtil.universalPath("b/src/B.scala")}:3:29 +// | type mismatch; +// | found : Int +// | required: String +// | L3: def foo(i: Int): String = i +// | ^ +// |${TestUtil.universalPath("b/src/B.scala")}: L3 [E1] +// |Failed to compile 'b'""".stripMargin +// ) +// +// assertNoDiff( +// cliLogger2.compilingInfos.mkString(lineSeparator), +// s""" +// |Compiling b (1 Scala source) +// """.stripMargin +// ) +// +// val targetB = TestUtil.universalPath("b/src/B.scala") +// assertNoDiff( +// cliLogger2.errors.mkString(lineSeparator), +// s""" +// |[E1] ${targetB}:3:29 +// | type mismatch; +// | found : Int +// | required: String +// | L3: def foo(i: Int): String = i +// | ^ +// |${targetB}: L3 [E1] +// |Failed to compile 'b'""".stripMargin +// ) +// +// /* Repeat the same but this time the CLI client runs the compilation first */ +// +// val startedSecondCompilation = Promise[Unit]() +// compileStartPromises.put(`B`.bspId, startedSecondCompilation) +// +// val cliLogger4 = new RecordingLogger(ansiCodesSupported = false) +// val cliLogger5 = new RecordingLogger(ansiCodesSupported = false) +// +// val fourthCompilation = bspState.compileHandle(`B`) +// val fifthCompilation = +// waitUntilStartAndCompile(thirdCompiledState, `B`, startedSecondCompilation, cliLogger4) +// val sixthCompilation = +// waitUntilStartAndCompile(thirdCompiledState, `B`, startedSecondCompilation, cliLogger5) +// +// val secondBspState = +// Await.result(fourthCompilation, FiniteDuration(4, TimeUnit.SECONDS)) +// val (_, fifthCompiledState) = +// TestUtil.blockOnTask(mapBoth(fifthCompilation, sixthCompilation), 2) +// +// checkDeduplication(bspLogger, isDeduplicated = false) +// checkDeduplication(cliLogger4, isDeduplicated = true) +// checkDeduplication(cliLogger5, isDeduplicated = true) +// +// assertNoDiff( +// secondBspState.lastDiagnostics(`B`), +// """#2: task start 2 +// | -> Msg: Compiling b (1 Scala source) +// | -> Data kind: compile-task +// |#2: b/src/B.scala +// | -> List(Diagnostic(Range(Position(2,28),Position(2,28)),Some(Error),Some(_),Some(_),type mismatch; found : Int required: String,None,None,Some({"actions":[]}))) +// | -> reset = true +// |#2: task finish 2 +// | -> errors 1, warnings 0 +// | -> Msg: Compiled 'b' +// | -> Data kind: compile-report +// """.stripMargin +// ) +// +// assertNoDiff( +// cliLogger4.compilingInfos.mkString(lineSeparator), +// s""" +// |Compiling b (1 Scala source) +// """.stripMargin +// ) +// +// assertNoDiff( +// cliLogger4.errors.mkString(lineSeparator), +// s""" +// |[E1] ${targetB}:3:29 +// | type mismatch; +// | found : Int +// | required: String +// | L3: def foo(i: Int): String = i +// | ^ +// |${targetB}: L3 [E1] +// |Failed to compile 'b'""".stripMargin +// ) +// +// assertNoDiff( +// cliLogger5.compilingInfos.mkString(lineSeparator), +// s""" +// |Compiling b (1 Scala source) +// """.stripMargin +// ) +// +// assertNoDiff( +// cliLogger5.errors.mkString(lineSeparator), +// s""" +// |[E1] ${targetB}:3:29 +// | type mismatch; +// | found : Int +// | required: String +// | L3: def foo(i: Int): String = i +// | ^ +// |${targetB}: L3 [E1] +// |Failed to compile 'b'""".stripMargin +// ) +// +// writeFile(`B`.srcFor("B.scala"), Sources.`B3.scala`) +// +// val cliLogger7 = new RecordingLogger(ansiCodesSupported = false) +// +// val startedThirdCompilation = Promise[Unit]() +// compileStartPromises.put(`B`.bspId, startedThirdCompilation) +// +// /* +// * Repeat the same (the CLI client drives the compilation first) but +// * this time the latest result in cli client differs with that of the +// * BSP client. This test proves that our deduplication logic uses +// * client-specific data to populate reporters that then consume the +// * stream of compilation events. +// */ +// +// val newCliCompiledState = { +// val underlyingState = fifthCompiledState.withLogger(cliLogger7).state +// val newPreviousResults = Map( +// // Use empty, remember we don't change last successful so incrementality works +// fifthCompiledState.getProjectFor(`B`) -> Compiler.Result.Empty +// ) +// val newResults = underlyingState.results.replacePreviousResults(newPreviousResults) +// new TestState(underlyingState.copy(results = newResults)) +// } +// +// val seventhCompilation = secondBspState.compileHandle(`B`) +// val eighthCompilation = +// waitUntilStartAndCompile(newCliCompiledState, `B`, startedThirdCompilation, cliLogger7) +// +// val thirdBspState = Await.result(seventhCompilation, FiniteDuration(10, TimeUnit.SECONDS)) +// +// Await.result(eighthCompilation, FiniteDuration(5, TimeUnit.SECONDS)) +// +// checkDeduplication(bspLogger, isDeduplicated = false) +// checkDeduplication(cliLogger7, isDeduplicated = true) +// +// assertNoDiff( +// cliLogger7.compilingInfos.mkString(lineSeparator), +// s""" +// |Compiling b (1 Scala source) +// """.stripMargin +// ) +// +// assert(cliLogger7.errors.size == 0) +// +// assertNoDiff( +// thirdBspState.lastDiagnostics(`B`), +// """#3: task start 3 +// | -> Msg: Compiling b (1 Scala source) +// | -> Data kind: compile-task +// |#3: b/src/B.scala +// | -> List() +// | -> reset = true +// |#3: task finish 3 +// | -> errors 0, warnings 0 +// | -> Msg: Compiled 'b' +// | -> Data kind: compile-report +// """.stripMargin +// ) +// } +// } +// } +// +// // TODO(jvican): Compile project of cancelled compilation to ensure no deduplication trace is left +// test("cancel deduplicated compilation finishes all clients") { +// TestUtil.withinWorkspace { workspace => +// object Sources { +// val `A.scala` = +// """/A.scala +// |package macros +// | +// |import scala.reflect.macros.blackbox.Context +// |import scala.language.experimental.macros +// | +// |object SleepMacro { +// | def sleep(): Unit = macro sleepImpl +// | def sleepImpl(c: Context)(): c.Expr[Unit] = { +// | import c.universe._ +// | Thread.sleep(500) +// | Thread.sleep(500) +// | Thread.sleep(500) +// | Thread.sleep(500) +// | reify { () } +// | } +// |}""".stripMargin +// +// val `B.scala` = +// """/B.scala +// |object B { +// | def foo(s: String): String = s.toString +// |} +// """.stripMargin +// +// // Sleep in independent files to force compiler to check for cancelled status +// val `B2.scala` = +// """/B.scala +// |object B { +// | def foo(s: String): String = s.toString +// | macros.SleepMacro.sleep() +// |} +// """.stripMargin +// +// val `Extra.scala` = +// """/Extra.scala +// |object Extra { def foo(s: String): String = s.toString; macros.SleepMacro.sleep() } +// """.stripMargin +// +// val `Extra2.scala` = +// """/Extra2.scala +// |object Extra2 { def foo(s: String): String = s.toString; macros.SleepMacro.sleep() } +// """.stripMargin +// +// val `Extra3.scala` = +// """/Extra3.scala +// |object Extra3 { def foo(s: String): String = s.toString; macros.SleepMacro.sleep() } +// """.stripMargin +// +// val `Extra4.scala` = +// """/Extra4.scala +// |object Extra4 { def foo(s: String): String = s.toString; macros.SleepMacro.sleep() } +// """.stripMargin +// +// val `Extra5.scala` = +// """/Extra5.scala +// |object Extra5 { def foo(s: String): String = s.toString; macros.SleepMacro.sleep() } +// """.stripMargin +// } +// +// val cliLogger = new RecordingLogger(ansiCodesSupported = false) +// val bspLogger = new RecordingLogger(ansiCodesSupported = false) +// +// val `A` = TestProject(workspace, "a", List(Sources.`A.scala`)) +// val `B` = TestProject(workspace, "b", List(Sources.`B.scala`), List(`A`)) +// +// val projects = List(`A`, `B`) +// val state = loadState(workspace, projects, cliLogger) +// val compiledState = state.compile(`B`) +// +// writeFile(`B`.srcFor("B.scala"), Sources.`B2.scala`) +// writeFile(`B`.srcFor("Extra.scala", exists = false), Sources.`Extra.scala`) +// writeFile(`B`.srcFor("Extra2.scala", exists = false), Sources.`Extra2.scala`) +// writeFile(`B`.srcFor("Extra3.scala", exists = false), Sources.`Extra3.scala`) +// writeFile(`B`.srcFor("Extra4.scala", exists = false), Sources.`Extra4.scala`) +// writeFile(`B`.srcFor("Extra5.scala", exists = false), Sources.`Extra5.scala`) +// +// val compileStartPromises = +// new mutable.HashMap[scalabsp.BuildTargetIdentifier, Promise[Unit]]() +// val startedFirstCompilation = Promise[Unit]() +// compileStartPromises.put(`B`.bspId, startedFirstCompilation) +// +// loadBspState( +// workspace, +// projects, +// bspLogger, +// compileStartPromises = Some(compileStartPromises) +// ) { bspState => +// val firstCompilation = +// bspState.compileHandle(`B`, userScheduler = Some(ExecutionContext.ioScheduler)) +// val secondCompilation = +// waitUntilStartAndCompile(compiledState, `B`, startedFirstCompilation, cliLogger) +// +// val _ = Task +// .fromFuture(startedFirstCompilation.future) +// .map(_ => { firstCompilation.cancel() }) +// .runAsync(ExecutionContext.ioScheduler) +// +// val (firstCompiledState, secondCompiledState) = +// TestUtil.blockOnTask( +// mapBoth(firstCompilation, secondCompilation), +// 15, +// loggers = List(cliLogger, bspLogger), +// userScheduler = Some(ExecutionContext.ioScheduler) +// ) +// +// if (firstCompiledState.status == ExitStatus.CompilationError && CrossPlatform.isWindows) { +// System.err.println("Ignoring failed cancellation with deduplication on Windows") +// } else { +// assert(firstCompiledState.status == ExitStatus.CompilationError) +// assertCancelledCompilation(firstCompiledState.toTestState, List(`B`)) +// assertNoDiff( +// bspLogger.infos.filterNot(_.contains("tcp")).mkString(lineSeparator), +// """ +// |request received: build/initialize +// |BSP initialization handshake complete. +// |Cancelling request Number(5.0) +// """.stripMargin +// ) +// +// assert(secondCompiledState.status == ExitStatus.CompilationError) +// assertCancelledCompilation(secondCompiledState, List(`B`)) +// +// checkDeduplication(bspLogger, isDeduplicated = false) +// checkDeduplication(cliLogger, isDeduplicated = true) +// +// assertNoDiff( +// firstCompiledState.lastDiagnostics(`B`), +// s""" +// |#1: task start 1 +// | -> Msg: Compiling b (6 Scala sources) +// | -> Data kind: compile-task +// |#1: task finish 1 +// | -> errors 0, warnings 0 +// | -> Msg: Compiled 'b' +// | -> Data kind: compile-report +// """.stripMargin +// ) +// +// assertNoDiff( +// cliLogger.warnings.mkString(lineSeparator), +// "Cancelling compilation of b" +// ) +// } +// } +// } +// } test("cancel deduplication on blocked compilation") { // Change default value to speed up test and only wait for 2 seconds From 06216f6c78dc76291313db675b4537993bf4c66a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Wi=C4=85cek?= Date: Tue, 3 Jun 2025 16:06:14 +0200 Subject: [PATCH 05/10] Non working with Ref and deferred --- .../tasks/compilation/CompileGatekeeper.scala | 162 +- .../test/scala/bloop/DeduplicationSpec.scala | 1738 ++++++++--------- 2 files changed, 944 insertions(+), 956 deletions(-) diff --git a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGatekeeper.scala b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGatekeeper.scala index b40f163a02..fa8251db00 100644 --- a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGatekeeper.scala +++ b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGatekeeper.scala @@ -31,17 +31,13 @@ object CompileGatekeeper { ) /* -------------------------------------------------------------------------------------------- */ - - case class CompilerState( - currentlyUsedClassesDirs: Map[AbsolutePath, AtomicInt], - runningCompilations: Map[UniqueCompileInputs, RunningCompilation], - lastSuccessfulResults: Map[ProjectId, LastSuccessfulResult] - ) - object CompilerState { - def empty: CompilerState = CompilerState(Map.empty, Map.empty, Map.empty) - } - - private val compilerStateRef: Ref[MonixTask, CompilerState] = Ref.unsafe(CompilerState.empty) + private val runningCompilations + : Ref[MonixTask, Map[UniqueCompileInputs, Deferred[MonixTask, RunningCompilation]]] = + Ref.unsafe(Map.empty) + private val currentlyUsedClassesDirs: Ref[MonixTask, Map[AbsolutePath, AtomicInt]] = + Ref.unsafe(Map.empty) + private val lastSuccessfulResults: Ref[MonixTask, Map[ProjectId, LastSuccessfulResult]] = + Ref.unsafe(Map.empty) /* -------------------------------------------------------------------------------------------- */ @@ -50,67 +46,67 @@ object CompileGatekeeper { bundle: SuccessfulCompileBundle, client: ClientInfo, compile: SuccessfulCompileBundle => CompileTraversal - ): Task[(RunningCompilation, CanBeDeduplicated)] = - scheduleCompilation(inputs, bundle, client, compile) - .flatMap { orCompilation => - Task.liftMonixTaskUncancellable { - compilerStateRef - .modify { state => - val currentCompilation = state.runningCompilations.get(bundle.uniqueInputs) - val (compilation, deduplicate, classesDirs) = currentCompilation - .fold( - ( - orCompilation, - false, - state.currentlyUsedClassesDirs - ) - ) { running => + ): Task[(RunningCompilation, CanBeDeduplicated)] = Task.liftMonixTaskUncancellable { + Deferred[MonixTask, RunningCompilation].flatMap { deferred => + runningCompilations.modify { state => + state.get(bundle.uniqueInputs) match { + case Some(existingDeferred) => + val output = + existingDeferred.get + .flatMap { running => val usedClassesDir = running.usedLastSuccessful.classesDir val usedClassesDirCounter = running.usedLastSuccessful.counterForClassesDir val deduplicate = usedClassesDirCounter.transformAndExtract { case count if count == 0 => (false -> count) case count => true -> (count + 1) } - if (deduplicate) (running, deduplicate, state.currentlyUsedClassesDirs) + if (deduplicate) MonixTask.now((running, deduplicate)) else { val classesDirs = - if ( - state.currentlyUsedClassesDirs - .get(usedClassesDir) - .contains(usedClassesDirCounter) + currentlyUsedClassesDirs + .update { classesDirs => + if ( + classesDirs + .get(usedClassesDir) + .contains(usedClassesDirCounter) + ) + classesDirs - usedClassesDir + else + classesDirs + } + scheduleCompilation(inputs, bundle, client, compile) + .flatMap(compilation => + deferred + .complete(compilation) + .flatMap(_ => classesDirs.map(_ => (compilation, false))) ) - state.currentlyUsedClassesDirs - usedClassesDir - else - state.currentlyUsedClassesDirs - ( - orCompilation, - deduplicate, - classesDirs - ) } } - val newState = - state.copy( - currentlyUsedClassesDirs = classesDirs, - runningCompilations = - state.runningCompilations + (bundle.uniqueInputs -> compilation) - ) - (state, (compilation, deduplicate)) - } + state -> output + case None => + val newState: Map[UniqueCompileInputs, Deferred[MonixTask, RunningCompilation]] = + state + (bundle.uniqueInputs -> deferred) + newState -> scheduleCompilation(inputs, bundle, client, compile).flatMap(compilation => + deferred.complete(compilation).map(_ => compilation -> false) + ) } - } + }.flatten + } + } def disconnectDeduplicationFromRunning( inputs: UniqueCompileInputs, runningCompilation: RunningCompilation - ): Task[Unit] = Task.liftMonixTaskUncancellable { + ): MonixTask[Unit] = { runningCompilation.isUnsubscribed.compareAndSet(false, true) - compilerStateRef.modify { state => + runningCompilations.modify { state => val updated = - if (state.runningCompilations.get(inputs).contains(runningCompilation)) - state.runningCompilations - inputs - else state.runningCompilations - (state.copy(runningCompilations = updated), ()) + if ( + state.contains(inputs) + ) // .contains(runningCompilationDeferred)) // TODO: no way to verfiy if it is the same + state - inputs + else state + (updated, ()) } } @@ -126,7 +122,7 @@ object CompileGatekeeper { bundle: SuccessfulCompileBundle, client: ClientInfo, compile: SuccessfulCompileBundle => CompileTraversal - ): Task[RunningCompilation] = { + ): MonixTask[RunningCompilation] = { import inputs.project import bundle.logger import logger.debug @@ -154,27 +150,24 @@ object CompileGatekeeper { } } - def getMostRecentSuccessfulResultAtomically = Task.liftMonixTaskUncancellable { - compilerStateRef.modify { state => + def getMostRecentSuccessfulResultAtomically: MonixTask[LastSuccessfulResult] = { + lastSuccessfulResults.modify { state => val previousResult = - initializeLastSuccessful(state.lastSuccessfulResults.get(project.uniqueId)) - val counter = state.currentlyUsedClassesDirs - .get(previousResult.classesDir) - .fold { - val initialCounter = AtomicInt(1) - initialCounter - } { counter => - val newCount = counter.incrementAndGet(1) - logger.debug(s"Increasing counter for ${previousResult.classesDir} to $newCount") - counter + initializeLastSuccessful(state.get(project.uniqueId)) + state -> currentlyUsedClassesDirs + .modify { counters => + counters.get(previousResult.classesDir) match { + case None => + val initialCounter = AtomicInt(1) + (counters + (previousResult.classesDir -> initialCounter), initialCounter) + case Some(counter) => + val newCount = counter.incrementAndGet(1) + logger.debug(s"Increasing counter for ${previousResult.classesDir} to $newCount") + counters -> counter + } } - val newUserClassesDir = (previousResult.classesDir, counter) - val newResults = (project.uniqueId, previousResult) - state.copy( - lastSuccessfulResults = state.lastSuccessfulResults + newResults, - currentlyUsedClassesDirs = state.currentlyUsedClassesDirs + newUserClassesDir - ) -> previousResult - } + .map(_ => previousResult) + }.flatten } logger.debug(s"Scheduling compilation for ${project.name}...") @@ -221,9 +214,7 @@ object CompileGatekeeper { if (!isAlreadyUnsubscribed.get) { // Remove running compilation if host compilation hasn't unsubscribed (maybe it's blocked) Task.liftMonixTaskUncancellable { - compilerStateRef.update { state => - state.copy(runningCompilations = state.runningCompilations - oinputs) - } + runningCompilations.update(_ - oinputs) } } else Task.unit @@ -269,24 +260,21 @@ object CompileGatekeeper { successful: LastSuccessfulResult, logger: Logger ): Task[Unit] = Task.liftMonixTaskUncancellable { - compilerStateRef - .update { state => + runningCompilations + .modify { state => val newSuccessfulResults = (project.uniqueId, successful) - if (state.runningCompilations.contains(oracleInputs)) { - state - .copy( - lastSuccessfulResults = state.lastSuccessfulResults + newSuccessfulResults, - runningCompilations = (state.runningCompilations - oracleInputs) - ) + if (state.contains(oracleInputs)) { + val newState = state - oracleInputs + (newState, lastSuccessfulResults.update { _ + newSuccessfulResults }) } else { - state + (state, MonixTask.unit) } } + .flatten .map { _ => logger.debug( s"Recording new last successful request for ${project.name} associated with ${successful.classesDir}" ) - () } } diff --git a/frontend/src/test/scala/bloop/DeduplicationSpec.scala b/frontend/src/test/scala/bloop/DeduplicationSpec.scala index e0a77a3609..41dc161db0 100644 --- a/frontend/src/test/scala/bloop/DeduplicationSpec.scala +++ b/frontend/src/test/scala/bloop/DeduplicationSpec.scala @@ -37,874 +37,874 @@ object DeduplicationSpec extends bloop.bsp.BspBaseSuite { val deduplicated = logger.infos.exists(_.startsWith("Deduplicating compilation")) if (isDeduplicated) assert(deduplicated) else assert(!deduplicated) } -// -// test("three concurrent clients deduplicate compilation") { -// val logger = new RecordingLogger(ansiCodesSupported = false) -// val logger1 = new RecordingLogger(ansiCodesSupported = false) -// val logger2 = new RecordingLogger(ansiCodesSupported = false) -// val logger3 = new RecordingLogger(ansiCodesSupported = false) -// BuildUtil.testSlowBuild(logger) { build => -// val state = new TestState(build.state) -// val compiledMacrosState = state.compile(build.macroProject) -// assert(compiledMacrosState.status == ExitStatus.Ok) -// assertValidCompilationState(compiledMacrosState, List(build.macroProject)) -// assertNoDiff( -// logger.compilingInfos.mkString(lineSeparator), -// s""" -// |Compiling macros (1 Scala source) -// """.stripMargin -// ) -// -// val compileStartPromises = -// new mutable.HashMap[scalabsp.BuildTargetIdentifier, Promise[Unit]]() -// val startedProjectCompilation = Promise[Unit]() -// compileStartPromises.put(build.userProject.bspId, startedProjectCompilation) -// -// val projects = List(build.macroProject, build.userProject) -// loadBspState( -// build.workspace, -// projects, -// logger1, -// compileStartPromises = Some(compileStartPromises) -// ) { bspState => -// val firstCompilation = bspState.compileHandle(build.userProject) -// -// val secondCompilation = waitUntilStartAndCompile( -// compiledMacrosState, -// build.userProject, -// startedProjectCompilation, -// logger2 -// ) -// -// val thirdCompilation = waitUntilStartAndCompile( -// compiledMacrosState, -// build.userProject, -// startedProjectCompilation, -// logger3 -// ) -// -// val firstCompiledState = waitInSeconds(firstCompilation, 10)(logger1.writeToFile("1")) -// val (secondCompiledState, thirdCompiledState) = -// TestUtil.blockOnTask(mapBoth(secondCompilation, thirdCompilation), 3) -// -// assert(firstCompiledState.status == ExitStatus.Ok) -// assert(secondCompiledState.status == ExitStatus.Ok) -// assert(thirdCompiledState.status == ExitStatus.Ok) -// -// // We get the same class files in all their external directories -// assertValidCompilationState(firstCompiledState, projects) -// assertValidCompilationState(secondCompiledState, projects) -// assertValidCompilationState(thirdCompiledState, projects) -// assertSameExternalClassesDirs( -// secondCompiledState, -// firstCompiledState.toTestState, -// projects -// ) -// assertSameExternalClassesDirs( -// thirdCompiledState, -// firstCompiledState.toTestState, -// projects -// ) -// -// checkDeduplication(logger2, isDeduplicated = true) -// checkDeduplication(logger3, isDeduplicated = true) -// -// assertNoDiff( -// firstCompiledState.lastDiagnostics(build.userProject), -// """#1: task start 1 -// | -> Msg: Compiling user (2 Scala sources) -// | -> Data kind: compile-task -// |#1: task finish 1 -// | -> errors 0, warnings 0 -// | -> Msg: Compiled 'user' -// | -> Data kind: compile-report -// """.stripMargin -// ) -// -// assertNoDiff( -// logger2.compilingInfos.mkString(lineSeparator), -// s""" -// |Compiling user (2 Scala sources) -// """.stripMargin -// ) -// -// assertNoDiff( -// logger3.compilingInfos.mkString(lineSeparator), -// s""" -// |Compiling user (2 Scala sources) -// """.stripMargin -// ) -// -// val delayFirstNoop = Some(random(0, 20)) -// val delaySecondNoop = Some(random(0, 20)) -// val noopCompiles = mapBoth( -// firstCompiledState.compileHandle(build.userProject, delayFirstNoop), -// secondCompiledState.compileHandle(build.userProject, delaySecondNoop) -// ) -// -// val (firstNoopState, secondNoopState) = TestUtil.blockOnTask(noopCompiles, 5) -// -// assert(firstNoopState.status == ExitStatus.Ok) -// assert(secondNoopState.status == ExitStatus.Ok) -// assertValidCompilationState(firstNoopState, projects) -// assertValidCompilationState(secondNoopState, projects) -// assertSameExternalClassesDirs(firstNoopState.toTestState, secondNoopState, projects) -// assertSameExternalClassesDirs(firstNoopState.toTestState, thirdCompiledState, projects) -// -// assertNoDiff( -// firstCompiledState.lastDiagnostics(build.userProject), -// "" // expect None here since it's a no-op which turns into "" -// ) -// -// // Same check as before because no-op should not show any more input -// assertNoDiff( -// logger2.compilingInfos.mkString(lineSeparator), -// s""" -// |Compiling user (2 Scala sources) -// """.stripMargin -// ) -// } -// } -// } -// -// test("deduplication removes invalidated class files from all external classes dirs") { -// val logger = new RecordingLogger(ansiCodesSupported = false) -// BuildUtil.testSlowBuild(logger) { build => -// val state = new TestState(build.state) -// val compiledMacrosState = state.compile(build.macroProject) -// assert(compiledMacrosState.status == ExitStatus.Ok) -// assertValidCompilationState(compiledMacrosState, List(build.macroProject)) -// assertNoDiff( -// logger.compilingInfos.mkString(lineSeparator), -// s""" -// |Compiling macros (1 Scala source) -// """.stripMargin -// ) -// -// val bspLogger = new RecordingLogger(ansiCodesSupported = false) -// val cliLogger = new RecordingLogger(ansiCodesSupported = false) -// -// val compileStartPromises = -// new mutable.HashMap[scalabsp.BuildTargetIdentifier, Promise[Unit]]() -// val startedFirstCompilation = Promise[Unit]() -// compileStartPromises.put(build.userProject.bspId, startedFirstCompilation) -// -// val projects = List(build.macroProject, build.userProject) -// loadBspState( -// build.workspace, -// projects, -// bspLogger, -// compileStartPromises = Some(compileStartPromises) -// ) { bspState => -// val firstCompilation = bspState.compileHandle(build.userProject) -// val firstCliCompilation = -// waitUntilStartAndCompile( -// compiledMacrosState, -// build.userProject, -// startedFirstCompilation, -// cliLogger -// ) -// -// val firstCompiledState = -// Await.result(firstCompilation, FiniteDuration(10, TimeUnit.SECONDS)) -// val firstCliCompiledState = -// Await.result(firstCliCompilation, FiniteDuration(1, TimeUnit.SECONDS)) -// -// assert(firstCompiledState.status == ExitStatus.Ok) -// assert(firstCliCompiledState.status == ExitStatus.Ok) -// -// // We get the same class files in all their external directories -// assertValidCompilationState(firstCompiledState, projects) -// assertValidCompilationState(firstCliCompiledState, projects) -// assertSameExternalClassesDirs( -// firstCliCompiledState, -// firstCompiledState.toTestState, -// projects -// ) -// -// checkDeduplication(cliLogger, isDeduplicated = true) -// -// assertNoDiff( -// firstCompiledState.lastDiagnostics(build.userProject), -// """#1: task start 1 -// | -> Msg: Compiling user (2 Scala sources) -// | -> Data kind: compile-task -// |#1: task finish 1 -// | -> errors 0, warnings 0 -// | -> Msg: Compiled 'user' -// | -> Data kind: compile-report -// """.stripMargin -// ) -// -// assertNoDiff( -// cliLogger.compilingInfos.mkString(lineSeparator), -// s""" -// |Compiling user (2 Scala sources) -// """.stripMargin -// ) -// -// object Sources { -// // A modified version of `User2` that instead renames to `User3` -// val `User2.scala` = -// """/main/scala/User2.scala -// |package user -// | -// |object User3 extends App { -// | macros.SleepMacro.sleep() -// |} -// """.stripMargin -// } -// -// val startedSecondCompilation = Promise[Unit]() -// compileStartPromises.put(build.userProject.bspId, startedSecondCompilation) -// -// val `User2.scala` = build.userProject.srcFor("/main/scala/User2.scala") -// assertIsFile(writeFile(`User2.scala`, Sources.`User2.scala`)) -// val secondCompilation = firstCompiledState.compileHandle(build.userProject) -// val secondCliCompilation = -// waitUntilStartAndCompile( -// firstCliCompiledState, -// build.userProject, -// startedSecondCompilation, -// cliLogger -// ) -// -// val secondCompiledState = -// Await.result(secondCompilation, FiniteDuration(5, TimeUnit.SECONDS)) -// val secondCliCompiledState = -// Await.result(secondCliCompilation, FiniteDuration(500, TimeUnit.MILLISECONDS)) -// -// assert(secondCompiledState.status == ExitStatus.Ok) -// assert(secondCliCompiledState.status == ExitStatus.Ok) -// assertValidCompilationState(secondCompiledState, projects) -// assertValidCompilationState(secondCliCompiledState, projects) -// -// assertNonExistingCompileProduct( -// secondCompiledState.toTestState, -// build.userProject, -// RelativePath("User2.class") -// ) -// -// assertNonExistingCompileProduct( -// secondCompiledState.toTestState, -// build.userProject, -// RelativePath("User2$.class") -// ) -// -// assertNonExistingCompileProduct( -// secondCliCompiledState, -// build.userProject, -// RelativePath("User2.class") -// ) -// -// assertNonExistingCompileProduct( -// secondCliCompiledState, -// build.userProject, -// RelativePath("User2$.class") -// ) -// -// assertSameExternalClassesDirs( -// secondCompiledState.toTestState, -// secondCliCompiledState, -// projects -// ) -// -// assertNoDiff( -// cliLogger.compilingInfos.mkString(lineSeparator), -// s""" -// |Compiling user (2 Scala sources) -// |Compiling user (1 Scala source) -// """.stripMargin -// ) -// -// assertNoDiff( -// secondCompiledState.lastDiagnostics(build.userProject), -// """#2: task start 2 -// | -> Msg: Compiling user (1 Scala source) -// | -> Data kind: compile-task -// |#2: task finish 2 -// | -> errors 0, warnings 0 -// | -> Msg: Compiled 'user' -// | -> Data kind: compile-report -// """.stripMargin -// ) -// } -// } -// } -// -// test("deduplication doesn't work if project definition changes") { -// val logger = new RecordingLogger(ansiCodesSupported = false) -// BuildUtil.testSlowBuild(logger) { build => -// val state = new TestState(build.state) -// val projects = List(build.macroProject, build.userProject) -// val compiledMacrosState = state.compile(build.macroProject) -// -// assert(compiledMacrosState.status == ExitStatus.Ok) -// assertValidCompilationState(compiledMacrosState, List(build.macroProject)) -// assertNoDiff( -// logger.compilingInfos.mkString(lineSeparator), -// s""" -// |Compiling macros (1 Scala source) -// """.stripMargin -// ) -// -// val bspLogger = new RecordingLogger(ansiCodesSupported = false) -// val cliLogger = new RecordingLogger(ansiCodesSupported = false) -// -// object Sources { -// val `User2.scala` = -// """/main/scala/User2.scala -// |package user -// | -// |object User2 extends App { -// | // Should report warning with -Ywarn-numeric-widen -// | val i: Long = 1.toInt -// | macros.SleepMacro.sleep() -// |} -// """.stripMargin -// } -// -// writeFile(build.userProject.srcFor("/main/scala/User2.scala"), Sources.`User2.scala`) -// loadBspState(build.workspace, projects, bspLogger) { bspState => -// val firstCompilation = bspState.compileHandle(build.userProject) -// -// val changeOpts = (s: Config.Scala) => s.copy(options = "-Ywarn-numeric-widen" :: s.options) -// val newFutureProject = build.userProject.rewriteProject(changeOpts) -// -// val firstCliCompilation = { -// compiledMacrosState -// .withLogger(cliLogger) -// .compileHandle( -// build.userProject, -// Some(FiniteDuration(2, TimeUnit.SECONDS)), -// beforeTask = Task { -// // Write config file before forcing second compilation -// reloadWithNewProject(newFutureProject, compiledMacrosState).withLogger(cliLogger) -// } -// ) -// } -// -// val firstCompiledState = -// Await.result(firstCompilation, FiniteDuration(10, TimeUnit.SECONDS)) -// val firstCliCompiledState = -// Await.result(firstCliCompilation, FiniteDuration(10, TimeUnit.SECONDS)) -// -// assert(firstCompiledState.status == ExitStatus.Ok) -// assert(firstCliCompiledState.status == ExitStatus.Ok) -// -// // assertValidCompilationState(firstCompiledState, projects) -// assertValidCompilationState(firstCliCompiledState, projects) -// assertDifferentExternalClassesDirs( -// firstCliCompiledState, -// firstCompiledState.toTestState, -// projects -// ) -// -// checkDeduplication(cliLogger, isDeduplicated = false) -// -// // First compilation should not report warning -// assertNoDiff( -// firstCompiledState.lastDiagnostics(build.userProject), -// """#1: task start 1 -// | -> Msg: Compiling user (2 Scala sources) -// | -> Data kind: compile-task -// |#1: task finish 1 -// | -> errors 0, warnings 0 -// | -> Msg: Compiled 'user' -// | -> Data kind: compile-report -// """.stripMargin -// ) -// -// assertNoDiff( -// cliLogger.compilingInfos.mkString(lineSeparator), -// s""" -// |Compiling user (2 Scala sources) -// """.stripMargin -// ) -// -// // Second compilation should be independent from 1 and report warning -// assertNoDiff( -// cliLogger.warnings.mkString(lineSeparator), -// s"""| [E1] ${TestUtil.universalPath("user/src/main/scala/User2.scala")}:5:19 -// | implicit numeric widening -// | L5: val i: Long = 1.toInt -// | ^ -// |${TestUtil.universalPath("user/src/main/scala/User2.scala")}: L5 [E1] -// |""".stripMargin -// ) -// } -// } -// } -// -// test("three concurrent clients receive error diagnostics appropriately") { -// TestUtil.withinWorkspace { workspace => -// object Sources { -// val `A.scala` = -// """/A.scala -// |package macros -// | -// |import scala.reflect.macros.blackbox.Context -// |import scala.language.experimental.macros -// | -// |object SleepMacro { -// | def sleep(): Unit = macro sleepImpl -// | def sleepImpl(c: Context)(): c.Expr[Unit] = { -// | import c.universe._ -// | Thread.sleep(1000) -// | reify { () } -// | } -// |}""".stripMargin -// -// val `B.scala` = -// """/B.scala -// |object B { -// | def foo(s: String): String = s.toString -// |} -// """.stripMargin -// -// // Second (non-compiling) version of `B` -// val `B2.scala` = -// """/B.scala -// |object B { -// | macros.SleepMacro.sleep() -// | def foo(i: Int): String = i -// |} -// """.stripMargin -// -// // Third (compiling) slow version of `B` -// val `B3.scala` = -// """/B.scala -// |object B { -// | macros.SleepMacro.sleep() -// | def foo(s: String): String = s.toString -// |} -// """.stripMargin -// } -// -// val cliLogger1 = new RecordingLogger(ansiCodesSupported = false) -// val cliLogger2 = new RecordingLogger(ansiCodesSupported = false) -// val bspLogger = new RecordingLogger(ansiCodesSupported = false) -// -// val `A` = TestProject(workspace, "a", List(Sources.`A.scala`)) -// val `B` = TestProject(workspace, "b", List(Sources.`B.scala`), List(`A`)) -// -// val projects = List(`A`, `B`) -// val state = loadState(workspace, projects, cliLogger1) -// val compiledState = state.compile(`B`) -// -// writeFile(`B`.srcFor("B.scala"), Sources.`B2.scala`) -// -// val compileStartPromises = -// new mutable.HashMap[scalabsp.BuildTargetIdentifier, Promise[Unit]]() -// val startedFirstCompilation = Promise[Unit]() -// compileStartPromises.put(`B`.bspId, startedFirstCompilation) -// -// loadBspState( -// workspace, -// projects, -// bspLogger, -// compileStartPromises = Some(compileStartPromises) -// ) { bspState => -// // val firstDelay = Some(random(400, 100)) -// val firstCompilation = bspState.compileHandle(`B`) -// val secondCompilation = -// waitUntilStartAndCompile(compiledState, `B`, startedFirstCompilation, cliLogger1) -// val thirdCompilation = -// waitUntilStartAndCompile(compiledState, `B`, startedFirstCompilation, cliLogger2) -// -// val firstCompiledState = -// Await.result(firstCompilation, FiniteDuration(15, TimeUnit.SECONDS)) -// val (secondCompiledState, thirdCompiledState) = -// TestUtil.blockOnTask(mapBoth(secondCompilation, thirdCompilation), 3) -// -// assert(firstCompiledState.status == ExitStatus.CompilationError) -// assert(secondCompiledState.status == ExitStatus.CompilationError) -// assert(thirdCompiledState.status == ExitStatus.CompilationError) -// -// // Check we get the same class files in all their external directories -// assertInvalidCompilationState( -// firstCompiledState, -// projects, -// existsAnalysisFile = true, -// hasPreviousSuccessful = true, -// // Classes dir of BSP session is empty because no successful compilation happened -// hasSameContentsInClassesDir = false -// ) -// -// assertInvalidCompilationState( -// secondCompiledState, -// projects, -// existsAnalysisFile = true, -// hasPreviousSuccessful = true, -// hasSameContentsInClassesDir = true -// ) -// -// assertInvalidCompilationState( -// thirdCompiledState, -// projects, -// existsAnalysisFile = true, -// hasPreviousSuccessful = true, -// hasSameContentsInClassesDir = true -// ) -// -// assertSameExternalClassesDirs(secondCompiledState, thirdCompiledState, projects) -// -// checkDeduplication(bspLogger, isDeduplicated = false) -// checkDeduplication(cliLogger1, isDeduplicated = true) -// checkDeduplication(cliLogger2, isDeduplicated = true) -// -// // We reproduce the same streaming side effects during compilation -// assertNoDiff( -// firstCompiledState.lastDiagnostics(`B`), -// """#1: task start 1 -// | -> Msg: Compiling b (1 Scala source) -// | -> Data kind: compile-task -// |#1: b/src/B.scala -// | -> List(Diagnostic(Range(Position(2,28),Position(2,28)),Some(Error),Some(_),Some(_),type mismatch; found : Int required: String,None,None,Some({"actions":[]}))) -// | -> reset = true -// |#1: task finish 1 -// | -> errors 1, warnings 0 -// | -> Msg: Compiled 'b' -// | -> Data kind: compile-report -// """.stripMargin -// ) -// -// assertNoDiff( -// cliLogger1.compilingInfos.mkString(lineSeparator), -// s""" -// |Compiling a (1 Scala source) -// |Compiling b (1 Scala source) -// |Compiling b (1 Scala source) -// """.stripMargin -// ) -// -// assertNoDiff( -// cliLogger1.errors.mkString(lineSeparator), -// s""" -// |[E1] ${TestUtil.universalPath("b/src/B.scala")}:3:29 -// | type mismatch; -// | found : Int -// | required: String -// | L3: def foo(i: Int): String = i -// | ^ -// |${TestUtil.universalPath("b/src/B.scala")}: L3 [E1] -// |Failed to compile 'b'""".stripMargin -// ) -// -// assertNoDiff( -// cliLogger2.compilingInfos.mkString(lineSeparator), -// s""" -// |Compiling b (1 Scala source) -// """.stripMargin -// ) -// -// val targetB = TestUtil.universalPath("b/src/B.scala") -// assertNoDiff( -// cliLogger2.errors.mkString(lineSeparator), -// s""" -// |[E1] ${targetB}:3:29 -// | type mismatch; -// | found : Int -// | required: String -// | L3: def foo(i: Int): String = i -// | ^ -// |${targetB}: L3 [E1] -// |Failed to compile 'b'""".stripMargin -// ) -// -// /* Repeat the same but this time the CLI client runs the compilation first */ -// -// val startedSecondCompilation = Promise[Unit]() -// compileStartPromises.put(`B`.bspId, startedSecondCompilation) -// -// val cliLogger4 = new RecordingLogger(ansiCodesSupported = false) -// val cliLogger5 = new RecordingLogger(ansiCodesSupported = false) -// -// val fourthCompilation = bspState.compileHandle(`B`) -// val fifthCompilation = -// waitUntilStartAndCompile(thirdCompiledState, `B`, startedSecondCompilation, cliLogger4) -// val sixthCompilation = -// waitUntilStartAndCompile(thirdCompiledState, `B`, startedSecondCompilation, cliLogger5) -// -// val secondBspState = -// Await.result(fourthCompilation, FiniteDuration(4, TimeUnit.SECONDS)) -// val (_, fifthCompiledState) = -// TestUtil.blockOnTask(mapBoth(fifthCompilation, sixthCompilation), 2) -// -// checkDeduplication(bspLogger, isDeduplicated = false) -// checkDeduplication(cliLogger4, isDeduplicated = true) -// checkDeduplication(cliLogger5, isDeduplicated = true) -// -// assertNoDiff( -// secondBspState.lastDiagnostics(`B`), -// """#2: task start 2 -// | -> Msg: Compiling b (1 Scala source) -// | -> Data kind: compile-task -// |#2: b/src/B.scala -// | -> List(Diagnostic(Range(Position(2,28),Position(2,28)),Some(Error),Some(_),Some(_),type mismatch; found : Int required: String,None,None,Some({"actions":[]}))) -// | -> reset = true -// |#2: task finish 2 -// | -> errors 1, warnings 0 -// | -> Msg: Compiled 'b' -// | -> Data kind: compile-report -// """.stripMargin -// ) -// -// assertNoDiff( -// cliLogger4.compilingInfos.mkString(lineSeparator), -// s""" -// |Compiling b (1 Scala source) -// """.stripMargin -// ) -// -// assertNoDiff( -// cliLogger4.errors.mkString(lineSeparator), -// s""" -// |[E1] ${targetB}:3:29 -// | type mismatch; -// | found : Int -// | required: String -// | L3: def foo(i: Int): String = i -// | ^ -// |${targetB}: L3 [E1] -// |Failed to compile 'b'""".stripMargin -// ) -// -// assertNoDiff( -// cliLogger5.compilingInfos.mkString(lineSeparator), -// s""" -// |Compiling b (1 Scala source) -// """.stripMargin -// ) -// -// assertNoDiff( -// cliLogger5.errors.mkString(lineSeparator), -// s""" -// |[E1] ${targetB}:3:29 -// | type mismatch; -// | found : Int -// | required: String -// | L3: def foo(i: Int): String = i -// | ^ -// |${targetB}: L3 [E1] -// |Failed to compile 'b'""".stripMargin -// ) -// -// writeFile(`B`.srcFor("B.scala"), Sources.`B3.scala`) -// -// val cliLogger7 = new RecordingLogger(ansiCodesSupported = false) -// -// val startedThirdCompilation = Promise[Unit]() -// compileStartPromises.put(`B`.bspId, startedThirdCompilation) -// -// /* -// * Repeat the same (the CLI client drives the compilation first) but -// * this time the latest result in cli client differs with that of the -// * BSP client. This test proves that our deduplication logic uses -// * client-specific data to populate reporters that then consume the -// * stream of compilation events. -// */ -// -// val newCliCompiledState = { -// val underlyingState = fifthCompiledState.withLogger(cliLogger7).state -// val newPreviousResults = Map( -// // Use empty, remember we don't change last successful so incrementality works -// fifthCompiledState.getProjectFor(`B`) -> Compiler.Result.Empty -// ) -// val newResults = underlyingState.results.replacePreviousResults(newPreviousResults) -// new TestState(underlyingState.copy(results = newResults)) -// } -// -// val seventhCompilation = secondBspState.compileHandle(`B`) -// val eighthCompilation = -// waitUntilStartAndCompile(newCliCompiledState, `B`, startedThirdCompilation, cliLogger7) -// -// val thirdBspState = Await.result(seventhCompilation, FiniteDuration(10, TimeUnit.SECONDS)) -// -// Await.result(eighthCompilation, FiniteDuration(5, TimeUnit.SECONDS)) -// -// checkDeduplication(bspLogger, isDeduplicated = false) -// checkDeduplication(cliLogger7, isDeduplicated = true) -// -// assertNoDiff( -// cliLogger7.compilingInfos.mkString(lineSeparator), -// s""" -// |Compiling b (1 Scala source) -// """.stripMargin -// ) -// -// assert(cliLogger7.errors.size == 0) -// -// assertNoDiff( -// thirdBspState.lastDiagnostics(`B`), -// """#3: task start 3 -// | -> Msg: Compiling b (1 Scala source) -// | -> Data kind: compile-task -// |#3: b/src/B.scala -// | -> List() -// | -> reset = true -// |#3: task finish 3 -// | -> errors 0, warnings 0 -// | -> Msg: Compiled 'b' -// | -> Data kind: compile-report -// """.stripMargin -// ) -// } -// } -// } -// -// // TODO(jvican): Compile project of cancelled compilation to ensure no deduplication trace is left -// test("cancel deduplicated compilation finishes all clients") { -// TestUtil.withinWorkspace { workspace => -// object Sources { -// val `A.scala` = -// """/A.scala -// |package macros -// | -// |import scala.reflect.macros.blackbox.Context -// |import scala.language.experimental.macros -// | -// |object SleepMacro { -// | def sleep(): Unit = macro sleepImpl -// | def sleepImpl(c: Context)(): c.Expr[Unit] = { -// | import c.universe._ -// | Thread.sleep(500) -// | Thread.sleep(500) -// | Thread.sleep(500) -// | Thread.sleep(500) -// | reify { () } -// | } -// |}""".stripMargin -// -// val `B.scala` = -// """/B.scala -// |object B { -// | def foo(s: String): String = s.toString -// |} -// """.stripMargin -// -// // Sleep in independent files to force compiler to check for cancelled status -// val `B2.scala` = -// """/B.scala -// |object B { -// | def foo(s: String): String = s.toString -// | macros.SleepMacro.sleep() -// |} -// """.stripMargin -// -// val `Extra.scala` = -// """/Extra.scala -// |object Extra { def foo(s: String): String = s.toString; macros.SleepMacro.sleep() } -// """.stripMargin -// -// val `Extra2.scala` = -// """/Extra2.scala -// |object Extra2 { def foo(s: String): String = s.toString; macros.SleepMacro.sleep() } -// """.stripMargin -// -// val `Extra3.scala` = -// """/Extra3.scala -// |object Extra3 { def foo(s: String): String = s.toString; macros.SleepMacro.sleep() } -// """.stripMargin -// -// val `Extra4.scala` = -// """/Extra4.scala -// |object Extra4 { def foo(s: String): String = s.toString; macros.SleepMacro.sleep() } -// """.stripMargin -// -// val `Extra5.scala` = -// """/Extra5.scala -// |object Extra5 { def foo(s: String): String = s.toString; macros.SleepMacro.sleep() } -// """.stripMargin -// } -// -// val cliLogger = new RecordingLogger(ansiCodesSupported = false) -// val bspLogger = new RecordingLogger(ansiCodesSupported = false) -// -// val `A` = TestProject(workspace, "a", List(Sources.`A.scala`)) -// val `B` = TestProject(workspace, "b", List(Sources.`B.scala`), List(`A`)) -// -// val projects = List(`A`, `B`) -// val state = loadState(workspace, projects, cliLogger) -// val compiledState = state.compile(`B`) -// -// writeFile(`B`.srcFor("B.scala"), Sources.`B2.scala`) -// writeFile(`B`.srcFor("Extra.scala", exists = false), Sources.`Extra.scala`) -// writeFile(`B`.srcFor("Extra2.scala", exists = false), Sources.`Extra2.scala`) -// writeFile(`B`.srcFor("Extra3.scala", exists = false), Sources.`Extra3.scala`) -// writeFile(`B`.srcFor("Extra4.scala", exists = false), Sources.`Extra4.scala`) -// writeFile(`B`.srcFor("Extra5.scala", exists = false), Sources.`Extra5.scala`) -// -// val compileStartPromises = -// new mutable.HashMap[scalabsp.BuildTargetIdentifier, Promise[Unit]]() -// val startedFirstCompilation = Promise[Unit]() -// compileStartPromises.put(`B`.bspId, startedFirstCompilation) -// -// loadBspState( -// workspace, -// projects, -// bspLogger, -// compileStartPromises = Some(compileStartPromises) -// ) { bspState => -// val firstCompilation = -// bspState.compileHandle(`B`, userScheduler = Some(ExecutionContext.ioScheduler)) -// val secondCompilation = -// waitUntilStartAndCompile(compiledState, `B`, startedFirstCompilation, cliLogger) -// -// val _ = Task -// .fromFuture(startedFirstCompilation.future) -// .map(_ => { firstCompilation.cancel() }) -// .runAsync(ExecutionContext.ioScheduler) -// -// val (firstCompiledState, secondCompiledState) = -// TestUtil.blockOnTask( -// mapBoth(firstCompilation, secondCompilation), -// 15, -// loggers = List(cliLogger, bspLogger), -// userScheduler = Some(ExecutionContext.ioScheduler) -// ) -// -// if (firstCompiledState.status == ExitStatus.CompilationError && CrossPlatform.isWindows) { -// System.err.println("Ignoring failed cancellation with deduplication on Windows") -// } else { -// assert(firstCompiledState.status == ExitStatus.CompilationError) -// assertCancelledCompilation(firstCompiledState.toTestState, List(`B`)) -// assertNoDiff( -// bspLogger.infos.filterNot(_.contains("tcp")).mkString(lineSeparator), -// """ -// |request received: build/initialize -// |BSP initialization handshake complete. -// |Cancelling request Number(5.0) -// """.stripMargin -// ) -// -// assert(secondCompiledState.status == ExitStatus.CompilationError) -// assertCancelledCompilation(secondCompiledState, List(`B`)) -// -// checkDeduplication(bspLogger, isDeduplicated = false) -// checkDeduplication(cliLogger, isDeduplicated = true) -// -// assertNoDiff( -// firstCompiledState.lastDiagnostics(`B`), -// s""" -// |#1: task start 1 -// | -> Msg: Compiling b (6 Scala sources) -// | -> Data kind: compile-task -// |#1: task finish 1 -// | -> errors 0, warnings 0 -// | -> Msg: Compiled 'b' -// | -> Data kind: compile-report -// """.stripMargin -// ) -// -// assertNoDiff( -// cliLogger.warnings.mkString(lineSeparator), -// "Cancelling compilation of b" -// ) -// } -// } -// } -// } + + test("three concurrent clients deduplicate compilation") { + val logger = new RecordingLogger(ansiCodesSupported = false) + val logger1 = new RecordingLogger(ansiCodesSupported = false) + val logger2 = new RecordingLogger(ansiCodesSupported = false) + val logger3 = new RecordingLogger(ansiCodesSupported = false) + BuildUtil.testSlowBuild(logger) { build => + val state = new TestState(build.state) + val compiledMacrosState = state.compile(build.macroProject) + assert(compiledMacrosState.status == ExitStatus.Ok) + assertValidCompilationState(compiledMacrosState, List(build.macroProject)) + assertNoDiff( + logger.compilingInfos.mkString(lineSeparator), + s""" + |Compiling macros (1 Scala source) + """.stripMargin + ) + + val compileStartPromises = + new mutable.HashMap[scalabsp.BuildTargetIdentifier, Promise[Unit]]() + val startedProjectCompilation = Promise[Unit]() + compileStartPromises.put(build.userProject.bspId, startedProjectCompilation) + + val projects = List(build.macroProject, build.userProject) + loadBspState( + build.workspace, + projects, + logger1, + compileStartPromises = Some(compileStartPromises) + ) { bspState => + val firstCompilation = bspState.compileHandle(build.userProject) + + val secondCompilation = waitUntilStartAndCompile( + compiledMacrosState, + build.userProject, + startedProjectCompilation, + logger2 + ) + + val thirdCompilation = waitUntilStartAndCompile( + compiledMacrosState, + build.userProject, + startedProjectCompilation, + logger3 + ) + + val firstCompiledState = waitInSeconds(firstCompilation, 10)(logger1.writeToFile("1")) + val (secondCompiledState, thirdCompiledState) = + TestUtil.blockOnTask(mapBoth(secondCompilation, thirdCompilation), 3) + + assert(firstCompiledState.status == ExitStatus.Ok) + assert(secondCompiledState.status == ExitStatus.Ok) + assert(thirdCompiledState.status == ExitStatus.Ok) + + // We get the same class files in all their external directories + assertValidCompilationState(firstCompiledState, projects) + assertValidCompilationState(secondCompiledState, projects) + assertValidCompilationState(thirdCompiledState, projects) + assertSameExternalClassesDirs( + secondCompiledState, + firstCompiledState.toTestState, + projects + ) + assertSameExternalClassesDirs( + thirdCompiledState, + firstCompiledState.toTestState, + projects + ) + + checkDeduplication(logger2, isDeduplicated = true) + checkDeduplication(logger3, isDeduplicated = true) + + assertNoDiff( + firstCompiledState.lastDiagnostics(build.userProject), + """#1: task start 1 + | -> Msg: Compiling user (2 Scala sources) + | -> Data kind: compile-task + |#1: task finish 1 + | -> errors 0, warnings 0 + | -> Msg: Compiled 'user' + | -> Data kind: compile-report + """.stripMargin + ) + + assertNoDiff( + logger2.compilingInfos.mkString(lineSeparator), + s""" + |Compiling user (2 Scala sources) + """.stripMargin + ) + + assertNoDiff( + logger3.compilingInfos.mkString(lineSeparator), + s""" + |Compiling user (2 Scala sources) + """.stripMargin + ) + + val delayFirstNoop = Some(random(0, 20)) + val delaySecondNoop = Some(random(0, 20)) + val noopCompiles = mapBoth( + firstCompiledState.compileHandle(build.userProject, delayFirstNoop), + secondCompiledState.compileHandle(build.userProject, delaySecondNoop) + ) + + val (firstNoopState, secondNoopState) = TestUtil.blockOnTask(noopCompiles, 5) + + assert(firstNoopState.status == ExitStatus.Ok) + assert(secondNoopState.status == ExitStatus.Ok) + assertValidCompilationState(firstNoopState, projects) + assertValidCompilationState(secondNoopState, projects) + assertSameExternalClassesDirs(firstNoopState.toTestState, secondNoopState, projects) + assertSameExternalClassesDirs(firstNoopState.toTestState, thirdCompiledState, projects) + + assertNoDiff( + firstCompiledState.lastDiagnostics(build.userProject), + "" // expect None here since it's a no-op which turns into "" + ) + + // Same check as before because no-op should not show any more input + assertNoDiff( + logger2.compilingInfos.mkString(lineSeparator), + s""" + |Compiling user (2 Scala sources) + """.stripMargin + ) + } + } + } + + test("deduplication removes invalidated class files from all external classes dirs") { + val logger = new RecordingLogger(ansiCodesSupported = false) + BuildUtil.testSlowBuild(logger) { build => + val state = new TestState(build.state) + val compiledMacrosState = state.compile(build.macroProject) + assert(compiledMacrosState.status == ExitStatus.Ok) + assertValidCompilationState(compiledMacrosState, List(build.macroProject)) + assertNoDiff( + logger.compilingInfos.mkString(lineSeparator), + s""" + |Compiling macros (1 Scala source) + """.stripMargin + ) + + val bspLogger = new RecordingLogger(ansiCodesSupported = false) + val cliLogger = new RecordingLogger(ansiCodesSupported = false) + + val compileStartPromises = + new mutable.HashMap[scalabsp.BuildTargetIdentifier, Promise[Unit]]() + val startedFirstCompilation = Promise[Unit]() + compileStartPromises.put(build.userProject.bspId, startedFirstCompilation) + + val projects = List(build.macroProject, build.userProject) + loadBspState( + build.workspace, + projects, + bspLogger, + compileStartPromises = Some(compileStartPromises) + ) { bspState => + val firstCompilation = bspState.compileHandle(build.userProject) + val firstCliCompilation = + waitUntilStartAndCompile( + compiledMacrosState, + build.userProject, + startedFirstCompilation, + cliLogger + ) + + val firstCompiledState = + Await.result(firstCompilation, FiniteDuration(10, TimeUnit.SECONDS)) + val firstCliCompiledState = + Await.result(firstCliCompilation, FiniteDuration(1, TimeUnit.SECONDS)) + + assert(firstCompiledState.status == ExitStatus.Ok) + assert(firstCliCompiledState.status == ExitStatus.Ok) + + // We get the same class files in all their external directories + assertValidCompilationState(firstCompiledState, projects) + assertValidCompilationState(firstCliCompiledState, projects) + assertSameExternalClassesDirs( + firstCliCompiledState, + firstCompiledState.toTestState, + projects + ) + + checkDeduplication(cliLogger, isDeduplicated = true) + + assertNoDiff( + firstCompiledState.lastDiagnostics(build.userProject), + """#1: task start 1 + | -> Msg: Compiling user (2 Scala sources) + | -> Data kind: compile-task + |#1: task finish 1 + | -> errors 0, warnings 0 + | -> Msg: Compiled 'user' + | -> Data kind: compile-report + """.stripMargin + ) + + assertNoDiff( + cliLogger.compilingInfos.mkString(lineSeparator), + s""" + |Compiling user (2 Scala sources) + """.stripMargin + ) + + object Sources { + // A modified version of `User2` that instead renames to `User3` + val `User2.scala` = + """/main/scala/User2.scala + |package user + | + |object User3 extends App { + | macros.SleepMacro.sleep() + |} + """.stripMargin + } + + val startedSecondCompilation = Promise[Unit]() + compileStartPromises.put(build.userProject.bspId, startedSecondCompilation) + + val `User2.scala` = build.userProject.srcFor("/main/scala/User2.scala") + assertIsFile(writeFile(`User2.scala`, Sources.`User2.scala`)) + val secondCompilation = firstCompiledState.compileHandle(build.userProject) + val secondCliCompilation = + waitUntilStartAndCompile( + firstCliCompiledState, + build.userProject, + startedSecondCompilation, + cliLogger + ) + + val secondCompiledState = + Await.result(secondCompilation, FiniteDuration(5, TimeUnit.SECONDS)) + val secondCliCompiledState = + Await.result(secondCliCompilation, FiniteDuration(500, TimeUnit.MILLISECONDS)) + + assert(secondCompiledState.status == ExitStatus.Ok) + assert(secondCliCompiledState.status == ExitStatus.Ok) + assertValidCompilationState(secondCompiledState, projects) + assertValidCompilationState(secondCliCompiledState, projects) + + assertNonExistingCompileProduct( + secondCompiledState.toTestState, + build.userProject, + RelativePath("User2.class") + ) + + assertNonExistingCompileProduct( + secondCompiledState.toTestState, + build.userProject, + RelativePath("User2$.class") + ) + + assertNonExistingCompileProduct( + secondCliCompiledState, + build.userProject, + RelativePath("User2.class") + ) + + assertNonExistingCompileProduct( + secondCliCompiledState, + build.userProject, + RelativePath("User2$.class") + ) + + assertSameExternalClassesDirs( + secondCompiledState.toTestState, + secondCliCompiledState, + projects + ) + + assertNoDiff( + cliLogger.compilingInfos.mkString(lineSeparator), + s""" + |Compiling user (2 Scala sources) + |Compiling user (1 Scala source) + """.stripMargin + ) + + assertNoDiff( + secondCompiledState.lastDiagnostics(build.userProject), + """#2: task start 2 + | -> Msg: Compiling user (1 Scala source) + | -> Data kind: compile-task + |#2: task finish 2 + | -> errors 0, warnings 0 + | -> Msg: Compiled 'user' + | -> Data kind: compile-report + """.stripMargin + ) + } + } + } + + test("deduplication doesn't work if project definition changes") { + val logger = new RecordingLogger(ansiCodesSupported = false) + BuildUtil.testSlowBuild(logger) { build => + val state = new TestState(build.state) + val projects = List(build.macroProject, build.userProject) + val compiledMacrosState = state.compile(build.macroProject) + + assert(compiledMacrosState.status == ExitStatus.Ok) + assertValidCompilationState(compiledMacrosState, List(build.macroProject)) + assertNoDiff( + logger.compilingInfos.mkString(lineSeparator), + s""" + |Compiling macros (1 Scala source) + """.stripMargin + ) + + val bspLogger = new RecordingLogger(ansiCodesSupported = false) + val cliLogger = new RecordingLogger(ansiCodesSupported = false) + + object Sources { + val `User2.scala` = + """/main/scala/User2.scala + |package user + | + |object User2 extends App { + | // Should report warning with -Ywarn-numeric-widen + | val i: Long = 1.toInt + | macros.SleepMacro.sleep() + |} + """.stripMargin + } + + writeFile(build.userProject.srcFor("/main/scala/User2.scala"), Sources.`User2.scala`) + loadBspState(build.workspace, projects, bspLogger) { bspState => + val firstCompilation = bspState.compileHandle(build.userProject) + + val changeOpts = (s: Config.Scala) => s.copy(options = "-Ywarn-numeric-widen" :: s.options) + val newFutureProject = build.userProject.rewriteProject(changeOpts) + + val firstCliCompilation = { + compiledMacrosState + .withLogger(cliLogger) + .compileHandle( + build.userProject, + Some(FiniteDuration(2, TimeUnit.SECONDS)), + beforeTask = Task { + // Write config file before forcing second compilation + reloadWithNewProject(newFutureProject, compiledMacrosState).withLogger(cliLogger) + } + ) + } + + val firstCompiledState = + Await.result(firstCompilation, FiniteDuration(10, TimeUnit.SECONDS)) + val firstCliCompiledState = + Await.result(firstCliCompilation, FiniteDuration(10, TimeUnit.SECONDS)) + + assert(firstCompiledState.status == ExitStatus.Ok) + assert(firstCliCompiledState.status == ExitStatus.Ok) + + // assertValidCompilationState(firstCompiledState, projects) + assertValidCompilationState(firstCliCompiledState, projects) + assertDifferentExternalClassesDirs( + firstCliCompiledState, + firstCompiledState.toTestState, + projects + ) + + checkDeduplication(cliLogger, isDeduplicated = false) + + // First compilation should not report warning + assertNoDiff( + firstCompiledState.lastDiagnostics(build.userProject), + """#1: task start 1 + | -> Msg: Compiling user (2 Scala sources) + | -> Data kind: compile-task + |#1: task finish 1 + | -> errors 0, warnings 0 + | -> Msg: Compiled 'user' + | -> Data kind: compile-report + """.stripMargin + ) + + assertNoDiff( + cliLogger.compilingInfos.mkString(lineSeparator), + s""" + |Compiling user (2 Scala sources) + """.stripMargin + ) + + // Second compilation should be independent from 1 and report warning + assertNoDiff( + cliLogger.warnings.mkString(lineSeparator), + s"""| [E1] ${TestUtil.universalPath("user/src/main/scala/User2.scala")}:5:19 + | implicit numeric widening + | L5: val i: Long = 1.toInt + | ^ + |${TestUtil.universalPath("user/src/main/scala/User2.scala")}: L5 [E1] + |""".stripMargin + ) + } + } + } + + test("three concurrent clients receive error diagnostics appropriately") { + TestUtil.withinWorkspace { workspace => + object Sources { + val `A.scala` = + """/A.scala + |package macros + | + |import scala.reflect.macros.blackbox.Context + |import scala.language.experimental.macros + | + |object SleepMacro { + | def sleep(): Unit = macro sleepImpl + | def sleepImpl(c: Context)(): c.Expr[Unit] = { + | import c.universe._ + | Thread.sleep(1000) + | reify { () } + | } + |}""".stripMargin + + val `B.scala` = + """/B.scala + |object B { + | def foo(s: String): String = s.toString + |} + """.stripMargin + + // Second (non-compiling) version of `B` + val `B2.scala` = + """/B.scala + |object B { + | macros.SleepMacro.sleep() + | def foo(i: Int): String = i + |} + """.stripMargin + + // Third (compiling) slow version of `B` + val `B3.scala` = + """/B.scala + |object B { + | macros.SleepMacro.sleep() + | def foo(s: String): String = s.toString + |} + """.stripMargin + } + + val cliLogger1 = new RecordingLogger(ansiCodesSupported = false) + val cliLogger2 = new RecordingLogger(ansiCodesSupported = false) + val bspLogger = new RecordingLogger(ansiCodesSupported = false) + + val `A` = TestProject(workspace, "a", List(Sources.`A.scala`)) + val `B` = TestProject(workspace, "b", List(Sources.`B.scala`), List(`A`)) + + val projects = List(`A`, `B`) + val state = loadState(workspace, projects, cliLogger1) + val compiledState = state.compile(`B`) + + writeFile(`B`.srcFor("B.scala"), Sources.`B2.scala`) + + val compileStartPromises = + new mutable.HashMap[scalabsp.BuildTargetIdentifier, Promise[Unit]]() + val startedFirstCompilation = Promise[Unit]() + compileStartPromises.put(`B`.bspId, startedFirstCompilation) + + loadBspState( + workspace, + projects, + bspLogger, + compileStartPromises = Some(compileStartPromises) + ) { bspState => + // val firstDelay = Some(random(400, 100)) + val firstCompilation = bspState.compileHandle(`B`) + val secondCompilation = + waitUntilStartAndCompile(compiledState, `B`, startedFirstCompilation, cliLogger1) + val thirdCompilation = + waitUntilStartAndCompile(compiledState, `B`, startedFirstCompilation, cliLogger2) + + val firstCompiledState = + Await.result(firstCompilation, FiniteDuration(15, TimeUnit.SECONDS)) + val (secondCompiledState, thirdCompiledState) = + TestUtil.blockOnTask(mapBoth(secondCompilation, thirdCompilation), 3) + + assert(firstCompiledState.status == ExitStatus.CompilationError) + assert(secondCompiledState.status == ExitStatus.CompilationError) + assert(thirdCompiledState.status == ExitStatus.CompilationError) + + // Check we get the same class files in all their external directories + assertInvalidCompilationState( + firstCompiledState, + projects, + existsAnalysisFile = true, + hasPreviousSuccessful = true, + // Classes dir of BSP session is empty because no successful compilation happened + hasSameContentsInClassesDir = false + ) + + assertInvalidCompilationState( + secondCompiledState, + projects, + existsAnalysisFile = true, + hasPreviousSuccessful = true, + hasSameContentsInClassesDir = true + ) + + assertInvalidCompilationState( + thirdCompiledState, + projects, + existsAnalysisFile = true, + hasPreviousSuccessful = true, + hasSameContentsInClassesDir = true + ) + + assertSameExternalClassesDirs(secondCompiledState, thirdCompiledState, projects) + + checkDeduplication(bspLogger, isDeduplicated = false) + checkDeduplication(cliLogger1, isDeduplicated = true) + checkDeduplication(cliLogger2, isDeduplicated = true) + + // We reproduce the same streaming side effects during compilation + assertNoDiff( + firstCompiledState.lastDiagnostics(`B`), + """#1: task start 1 + | -> Msg: Compiling b (1 Scala source) + | -> Data kind: compile-task + |#1: b/src/B.scala + | -> List(Diagnostic(Range(Position(2,28),Position(2,28)),Some(Error),Some(_),Some(_),type mismatch; found : Int required: String,None,None,Some({"actions":[]}))) + | -> reset = true + |#1: task finish 1 + | -> errors 1, warnings 0 + | -> Msg: Compiled 'b' + | -> Data kind: compile-report + """.stripMargin + ) + + assertNoDiff( + cliLogger1.compilingInfos.mkString(lineSeparator), + s""" + |Compiling a (1 Scala source) + |Compiling b (1 Scala source) + |Compiling b (1 Scala source) + """.stripMargin + ) + + assertNoDiff( + cliLogger1.errors.mkString(lineSeparator), + s""" + |[E1] ${TestUtil.universalPath("b/src/B.scala")}:3:29 + | type mismatch; + | found : Int + | required: String + | L3: def foo(i: Int): String = i + | ^ + |${TestUtil.universalPath("b/src/B.scala")}: L3 [E1] + |Failed to compile 'b'""".stripMargin + ) + + assertNoDiff( + cliLogger2.compilingInfos.mkString(lineSeparator), + s""" + |Compiling b (1 Scala source) + """.stripMargin + ) + + val targetB = TestUtil.universalPath("b/src/B.scala") + assertNoDiff( + cliLogger2.errors.mkString(lineSeparator), + s""" + |[E1] ${targetB}:3:29 + | type mismatch; + | found : Int + | required: String + | L3: def foo(i: Int): String = i + | ^ + |${targetB}: L3 [E1] + |Failed to compile 'b'""".stripMargin + ) + + /* Repeat the same but this time the CLI client runs the compilation first */ + + val startedSecondCompilation = Promise[Unit]() + compileStartPromises.put(`B`.bspId, startedSecondCompilation) + + val cliLogger4 = new RecordingLogger(ansiCodesSupported = false) + val cliLogger5 = new RecordingLogger(ansiCodesSupported = false) + + val fourthCompilation = bspState.compileHandle(`B`) + val fifthCompilation = + waitUntilStartAndCompile(thirdCompiledState, `B`, startedSecondCompilation, cliLogger4) + val sixthCompilation = + waitUntilStartAndCompile(thirdCompiledState, `B`, startedSecondCompilation, cliLogger5) + + val secondBspState = + Await.result(fourthCompilation, FiniteDuration(4, TimeUnit.SECONDS)) + val (_, fifthCompiledState) = + TestUtil.blockOnTask(mapBoth(fifthCompilation, sixthCompilation), 2) + + checkDeduplication(bspLogger, isDeduplicated = false) + checkDeduplication(cliLogger4, isDeduplicated = true) + checkDeduplication(cliLogger5, isDeduplicated = true) + + assertNoDiff( + secondBspState.lastDiagnostics(`B`), + """#2: task start 2 + | -> Msg: Compiling b (1 Scala source) + | -> Data kind: compile-task + |#2: b/src/B.scala + | -> List(Diagnostic(Range(Position(2,28),Position(2,28)),Some(Error),Some(_),Some(_),type mismatch; found : Int required: String,None,None,Some({"actions":[]}))) + | -> reset = true + |#2: task finish 2 + | -> errors 1, warnings 0 + | -> Msg: Compiled 'b' + | -> Data kind: compile-report + """.stripMargin + ) + + assertNoDiff( + cliLogger4.compilingInfos.mkString(lineSeparator), + s""" + |Compiling b (1 Scala source) + """.stripMargin + ) + + assertNoDiff( + cliLogger4.errors.mkString(lineSeparator), + s""" + |[E1] ${targetB}:3:29 + | type mismatch; + | found : Int + | required: String + | L3: def foo(i: Int): String = i + | ^ + |${targetB}: L3 [E1] + |Failed to compile 'b'""".stripMargin + ) + + assertNoDiff( + cliLogger5.compilingInfos.mkString(lineSeparator), + s""" + |Compiling b (1 Scala source) + """.stripMargin + ) + + assertNoDiff( + cliLogger5.errors.mkString(lineSeparator), + s""" + |[E1] ${targetB}:3:29 + | type mismatch; + | found : Int + | required: String + | L3: def foo(i: Int): String = i + | ^ + |${targetB}: L3 [E1] + |Failed to compile 'b'""".stripMargin + ) + + writeFile(`B`.srcFor("B.scala"), Sources.`B3.scala`) + + val cliLogger7 = new RecordingLogger(ansiCodesSupported = false) + + val startedThirdCompilation = Promise[Unit]() + compileStartPromises.put(`B`.bspId, startedThirdCompilation) + + /* + * Repeat the same (the CLI client drives the compilation first) but + * this time the latest result in cli client differs with that of the + * BSP client. This test proves that our deduplication logic uses + * client-specific data to populate reporters that then consume the + * stream of compilation events. + */ + + val newCliCompiledState = { + val underlyingState = fifthCompiledState.withLogger(cliLogger7).state + val newPreviousResults = Map( + // Use empty, remember we don't change last successful so incrementality works + fifthCompiledState.getProjectFor(`B`) -> Compiler.Result.Empty + ) + val newResults = underlyingState.results.replacePreviousResults(newPreviousResults) + new TestState(underlyingState.copy(results = newResults)) + } + + val seventhCompilation = secondBspState.compileHandle(`B`) + val eighthCompilation = + waitUntilStartAndCompile(newCliCompiledState, `B`, startedThirdCompilation, cliLogger7) + + val thirdBspState = Await.result(seventhCompilation, FiniteDuration(10, TimeUnit.SECONDS)) + + Await.result(eighthCompilation, FiniteDuration(5, TimeUnit.SECONDS)) + + checkDeduplication(bspLogger, isDeduplicated = false) + checkDeduplication(cliLogger7, isDeduplicated = true) + + assertNoDiff( + cliLogger7.compilingInfos.mkString(lineSeparator), + s""" + |Compiling b (1 Scala source) + """.stripMargin + ) + + assert(cliLogger7.errors.size == 0) + + assertNoDiff( + thirdBspState.lastDiagnostics(`B`), + """#3: task start 3 + | -> Msg: Compiling b (1 Scala source) + | -> Data kind: compile-task + |#3: b/src/B.scala + | -> List() + | -> reset = true + |#3: task finish 3 + | -> errors 0, warnings 0 + | -> Msg: Compiled 'b' + | -> Data kind: compile-report + """.stripMargin + ) + } + } + } + + // TODO(jvican): Compile project of cancelled compilation to ensure no deduplication trace is left + test("cancel deduplicated compilation finishes all clients") { + TestUtil.withinWorkspace { workspace => + object Sources { + val `A.scala` = + """/A.scala + |package macros + | + |import scala.reflect.macros.blackbox.Context + |import scala.language.experimental.macros + | + |object SleepMacro { + | def sleep(): Unit = macro sleepImpl + | def sleepImpl(c: Context)(): c.Expr[Unit] = { + | import c.universe._ + | Thread.sleep(500) + | Thread.sleep(500) + | Thread.sleep(500) + | Thread.sleep(500) + | reify { () } + | } + |}""".stripMargin + + val `B.scala` = + """/B.scala + |object B { + | def foo(s: String): String = s.toString + |} + """.stripMargin + + // Sleep in independent files to force compiler to check for cancelled status + val `B2.scala` = + """/B.scala + |object B { + | def foo(s: String): String = s.toString + | macros.SleepMacro.sleep() + |} + """.stripMargin + + val `Extra.scala` = + """/Extra.scala + |object Extra { def foo(s: String): String = s.toString; macros.SleepMacro.sleep() } + """.stripMargin + + val `Extra2.scala` = + """/Extra2.scala + |object Extra2 { def foo(s: String): String = s.toString; macros.SleepMacro.sleep() } + """.stripMargin + + val `Extra3.scala` = + """/Extra3.scala + |object Extra3 { def foo(s: String): String = s.toString; macros.SleepMacro.sleep() } + """.stripMargin + + val `Extra4.scala` = + """/Extra4.scala + |object Extra4 { def foo(s: String): String = s.toString; macros.SleepMacro.sleep() } + """.stripMargin + + val `Extra5.scala` = + """/Extra5.scala + |object Extra5 { def foo(s: String): String = s.toString; macros.SleepMacro.sleep() } + """.stripMargin + } + + val cliLogger = new RecordingLogger(ansiCodesSupported = false) + val bspLogger = new RecordingLogger(ansiCodesSupported = false) + + val `A` = TestProject(workspace, "a", List(Sources.`A.scala`)) + val `B` = TestProject(workspace, "b", List(Sources.`B.scala`), List(`A`)) + + val projects = List(`A`, `B`) + val state = loadState(workspace, projects, cliLogger) + val compiledState = state.compile(`B`) + + writeFile(`B`.srcFor("B.scala"), Sources.`B2.scala`) + writeFile(`B`.srcFor("Extra.scala", exists = false), Sources.`Extra.scala`) + writeFile(`B`.srcFor("Extra2.scala", exists = false), Sources.`Extra2.scala`) + writeFile(`B`.srcFor("Extra3.scala", exists = false), Sources.`Extra3.scala`) + writeFile(`B`.srcFor("Extra4.scala", exists = false), Sources.`Extra4.scala`) + writeFile(`B`.srcFor("Extra5.scala", exists = false), Sources.`Extra5.scala`) + + val compileStartPromises = + new mutable.HashMap[scalabsp.BuildTargetIdentifier, Promise[Unit]]() + val startedFirstCompilation = Promise[Unit]() + compileStartPromises.put(`B`.bspId, startedFirstCompilation) + + loadBspState( + workspace, + projects, + bspLogger, + compileStartPromises = Some(compileStartPromises) + ) { bspState => + val firstCompilation = + bspState.compileHandle(`B`, userScheduler = Some(ExecutionContext.ioScheduler)) + val secondCompilation = + waitUntilStartAndCompile(compiledState, `B`, startedFirstCompilation, cliLogger) + + val _ = Task + .fromFuture(startedFirstCompilation.future) + .map(_ => { firstCompilation.cancel() }) + .runAsync(ExecutionContext.ioScheduler) + + val (firstCompiledState, secondCompiledState) = + TestUtil.blockOnTask( + mapBoth(firstCompilation, secondCompilation), + 15, + loggers = List(cliLogger, bspLogger), + userScheduler = Some(ExecutionContext.ioScheduler) + ) + + if (firstCompiledState.status == ExitStatus.CompilationError && CrossPlatform.isWindows) { + System.err.println("Ignoring failed cancellation with deduplication on Windows") + } else { + assert(firstCompiledState.status == ExitStatus.CompilationError) + assertCancelledCompilation(firstCompiledState.toTestState, List(`B`)) + assertNoDiff( + bspLogger.infos.filterNot(_.contains("tcp")).mkString(lineSeparator), + """ + |request received: build/initialize + |BSP initialization handshake complete. + |Cancelling request Number(5.0) + """.stripMargin + ) + + assert(secondCompiledState.status == ExitStatus.CompilationError) + assertCancelledCompilation(secondCompiledState, List(`B`)) + + checkDeduplication(bspLogger, isDeduplicated = false) + checkDeduplication(cliLogger, isDeduplicated = true) + + assertNoDiff( + firstCompiledState.lastDiagnostics(`B`), + s""" + |#1: task start 1 + | -> Msg: Compiling b (6 Scala sources) + | -> Data kind: compile-task + |#1: task finish 1 + | -> errors 0, warnings 0 + | -> Msg: Compiled 'b' + | -> Data kind: compile-report + """.stripMargin + ) + + assertNoDiff( + cliLogger.warnings.mkString(lineSeparator), + "Cancelling compilation of b" + ) + } + } + } + } test("cancel deduplication on blocked compilation") { // Change default value to speed up test and only wait for 2 seconds @@ -1034,7 +1034,7 @@ object DeduplicationSpec extends bloop.bsp.BspBaseSuite { } } - ignore("two concurrent clients deduplicate compilation and run at the same time") { + test("two concurrent clients deduplicate compilation and run at the same time") { val logger = new RecordingLogger(ansiCodesSupported = false) val logger1 = new RecordingLogger(ansiCodesSupported = false) val logger2 = new RecordingLogger(ansiCodesSupported = false) From 7c581191d7ccfa9524393386acebea5feb1640c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Wi=C4=85cek?= Date: Tue, 3 Jun 2025 16:16:33 +0200 Subject: [PATCH 06/10] revert changes --- .../tasks/compilation/CompileGatekeeper.scala | 291 ++++++------- .../tasks/compilation/CompileGraph.scala | 410 +++++++++--------- 2 files changed, 337 insertions(+), 364 deletions(-) diff --git a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGatekeeper.scala b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGatekeeper.scala index fa8251db00..6028b4e837 100644 --- a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGatekeeper.scala +++ b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGatekeeper.scala @@ -1,5 +1,7 @@ package bloop.engine.tasks.compilation +import java.util.concurrent.ConcurrentHashMap + import bloop.Compiler import bloop.UniqueCompileInputs import bloop.data.ClientInfo @@ -12,8 +14,7 @@ import bloop.logging.Logger import bloop.logging.LoggerAction import bloop.reporter.ReporterAction import bloop.task.Task -import monix.eval.{Task => MonixTask} -import cats.effect.concurrent.{Deferred, Ref} + import monix.execution.atomic.AtomicBoolean import monix.execution.atomic.AtomicInt import monix.reactive.Observable @@ -31,13 +32,10 @@ object CompileGatekeeper { ) /* -------------------------------------------------------------------------------------------- */ - private val runningCompilations - : Ref[MonixTask, Map[UniqueCompileInputs, Deferred[MonixTask, RunningCompilation]]] = - Ref.unsafe(Map.empty) - private val currentlyUsedClassesDirs: Ref[MonixTask, Map[AbsolutePath, AtomicInt]] = - Ref.unsafe(Map.empty) - private val lastSuccessfulResults: Ref[MonixTask, Map[ProjectId, LastSuccessfulResult]] = - Ref.unsafe(Map.empty) + + private val currentlyUsedClassesDirs = new ConcurrentHashMap[AbsolutePath, AtomicInt]() + private val runningCompilations = new ConcurrentHashMap[UniqueCompileInputs, RunningCompilation]() + private val lastSuccessfulResults = new ConcurrentHashMap[ProjectId, LastSuccessfulResult]() /* -------------------------------------------------------------------------------------------- */ @@ -46,68 +44,48 @@ object CompileGatekeeper { bundle: SuccessfulCompileBundle, client: ClientInfo, compile: SuccessfulCompileBundle => CompileTraversal - ): Task[(RunningCompilation, CanBeDeduplicated)] = Task.liftMonixTaskUncancellable { - Deferred[MonixTask, RunningCompilation].flatMap { deferred => - runningCompilations.modify { state => - state.get(bundle.uniqueInputs) match { - case Some(existingDeferred) => - val output = - existingDeferred.get - .flatMap { running => - val usedClassesDir = running.usedLastSuccessful.classesDir - val usedClassesDirCounter = running.usedLastSuccessful.counterForClassesDir - val deduplicate = usedClassesDirCounter.transformAndExtract { - case count if count == 0 => (false -> count) - case count => true -> (count + 1) - } - if (deduplicate) MonixTask.now((running, deduplicate)) - else { - val classesDirs = - currentlyUsedClassesDirs - .update { classesDirs => - if ( - classesDirs - .get(usedClassesDir) - .contains(usedClassesDirCounter) - ) - classesDirs - usedClassesDir - else - classesDirs - } - scheduleCompilation(inputs, bundle, client, compile) - .flatMap(compilation => - deferred - .complete(compilation) - .flatMap(_ => classesDirs.map(_ => (compilation, false))) - ) - } - } - state -> output - case None => - val newState: Map[UniqueCompileInputs, Deferred[MonixTask, RunningCompilation]] = - state + (bundle.uniqueInputs -> deferred) - newState -> scheduleCompilation(inputs, bundle, client, compile).flatMap(compilation => - deferred.complete(compilation).map(_ => compilation -> false) - ) + ): (RunningCompilation, CanBeDeduplicated) = { + var deduplicate = true + + val running = runningCompilations.compute( + bundle.uniqueInputs, + (_: UniqueCompileInputs, running: RunningCompilation) => { + if (running == null) { + deduplicate = false + scheduleCompilation(inputs, bundle, client, compile) + } else { + val usedClassesDir = running.usedLastSuccessful.classesDir + val usedClassesDirCounter = running.usedLastSuccessful.counterForClassesDir + + usedClassesDirCounter.getAndTransform { count => + if (count == 0) { + // Abort deduplication, dir is scheduled to be deleted in background + deduplicate = false + // Remove from map of used classes dirs in case it hasn't already been + currentlyUsedClassesDirs.remove(usedClassesDir, usedClassesDirCounter) + // Return previous count, this counter will soon be deallocated + count + } else { + // Increase count to prevent other compiles to schedule its deletion + count + 1 + } + } + + if (deduplicate) running + else scheduleCompilation(inputs, bundle, client, compile) } - }.flatten - } + } + ) + + (running, deduplicate) } def disconnectDeduplicationFromRunning( inputs: UniqueCompileInputs, runningCompilation: RunningCompilation - ): MonixTask[Unit] = { + ): Unit = { runningCompilation.isUnsubscribed.compareAndSet(false, true) - runningCompilations.modify { state => - val updated = - if ( - state.contains(inputs) - ) // .contains(runningCompilationDeferred)) // TODO: no way to verfiy if it is the same - state - inputs - else state - (updated, ()) - } + runningCompilations.remove(inputs, runningCompilation); () } /** @@ -117,20 +95,20 @@ object CompileGatekeeper { * inputs. The call-site ensures that only one compilation can exist for the * same inputs for a period of time. */ - def scheduleCompilation( + private def scheduleCompilation( inputs: BundleInputs, bundle: SuccessfulCompileBundle, client: ClientInfo, compile: SuccessfulCompileBundle => CompileTraversal - ): MonixTask[RunningCompilation] = { + ): RunningCompilation = { import inputs.project import bundle.logger import logger.debug - def initializeLastSuccessful( - maybePreviousResult: Option[LastSuccessfulResult] - ): LastSuccessfulResult = { - val result = maybePreviousResult.getOrElse(bundle.lastSuccessful) + var counterForUsedClassesDir: AtomicInt = null + + def initializeLastSuccessful(previousOrNull: LastSuccessfulResult): LastSuccessfulResult = { + val result = Option(previousOrNull).getOrElse(bundle.lastSuccessful) if (!result.classesDir.exists) { debug(s"Ignoring analysis for ${project.name}, directory ${result.classesDir} is missing") LastSuccessfulResult.empty(inputs.project) @@ -150,55 +128,65 @@ object CompileGatekeeper { } } - def getMostRecentSuccessfulResultAtomically: MonixTask[LastSuccessfulResult] = { - lastSuccessfulResults.modify { state => - val previousResult = - initializeLastSuccessful(state.get(project.uniqueId)) - state -> currentlyUsedClassesDirs - .modify { counters => - counters.get(previousResult.classesDir) match { - case None => + def getMostRecentSuccessfulResultAtomically = { + lastSuccessfulResults.compute( + project.uniqueId, + (_: String, previousResultOrNull: LastSuccessfulResult) => { + // Return previous result or the initial last successful coming from the bundle + val previousResult = initializeLastSuccessful(previousResultOrNull) + + currentlyUsedClassesDirs.compute( + previousResult.classesDir, + (_: AbsolutePath, counter: AtomicInt) => { + // Set counter for used classes dir when init or incrementing + if (counter == null) { val initialCounter = AtomicInt(1) - (counters + (previousResult.classesDir -> initialCounter), initialCounter) - case Some(counter) => + counterForUsedClassesDir = initialCounter + initialCounter + } else { + counterForUsedClassesDir = counter val newCount = counter.incrementAndGet(1) logger.debug(s"Increasing counter for ${previousResult.classesDir} to $newCount") - counters -> counter + counter + } } - } - .map(_ => previousResult) - }.flatten + ) + + previousResult.copy(counterForClassesDir = counterForUsedClassesDir) + } + ) } logger.debug(s"Scheduling compilation for ${project.name}...") - getMostRecentSuccessfulResultAtomically - .map { mostRecentSuccessful => - val isUnsubscribed = AtomicBoolean(false) - val newBundle = bundle.copy(lastSuccessful = mostRecentSuccessful) - val compileAndUnsubscribe = compile(newBundle) - .doOnFinish(_ => Task(logger.observer.onComplete())) - .flatMap { result => - // Unregister deduplication atomically and register last successful if any - processResultAtomically( - result, - project, - bundle.uniqueInputs, - isUnsubscribed, - logger - ) - } - .memoize - - RunningCompilation( - compileAndUnsubscribe, - mostRecentSuccessful, - isUnsubscribed, - bundle.mirror, - client - ) // Without memoization, there is no deduplication - } + // Replace client-specific last successful with the most recent result + val mostRecentSuccessful = getMostRecentSuccessfulResultAtomically + + val isUnsubscribed = AtomicBoolean(false) + val newBundle = bundle.copy(lastSuccessful = mostRecentSuccessful) + val compileAndUnsubscribe = { + compile(newBundle) + .doOnFinish(_ => Task(logger.observer.onComplete())) + .map { result => + // Unregister deduplication atomically and register last successful if any + processResultAtomically( + result, + project, + bundle.uniqueInputs, + isUnsubscribed, + logger + ) + } + .memoize // Without memoization, there is no deduplication + } + RunningCompilation( + compileAndUnsubscribe, + mostRecentSuccessful, + isUnsubscribed, + bundle.mirror, + client + ) } private def processResultAtomically( @@ -207,34 +195,27 @@ object CompileGatekeeper { oinputs: UniqueCompileInputs, isAlreadyUnsubscribed: AtomicBoolean, logger: Logger - ): Task[Dag[PartialCompileResult]] = { - - def cleanUpAfterCompilationError[T](result: T): Task[T] = { - Task { - if (!isAlreadyUnsubscribed.get) { - // Remove running compilation if host compilation hasn't unsubscribed (maybe it's blocked) - Task.liftMonixTaskUncancellable { - runningCompilations.update(_ - oinputs) - } - } else - Task.unit - }.flatten.map(_ => result) + ): Dag[PartialCompileResult] = { + + def cleanUpAfterCompilationError[T](result: T): T = { + if (!isAlreadyUnsubscribed.get) { + // Remove running compilation if host compilation hasn't unsubscribed (maybe it's blocked) + runningCompilations.remove(oinputs) + } + + result } // Unregister deduplication atomically and register last successful if any - PartialCompileResult.mapEveryResultTask(resultDag) { + PartialCompileResult.mapEveryResult(resultDag) { case s: PartialSuccess => - val processedResult = s.result.flatMap { (result: ResultBundle) => - result.successful - .fold(cleanUpAfterCompilationError(result)) { res => - unregisterDeduplicationAndRegisterSuccessful( - project, - oinputs, - res, - logger - ) - .map(_ => result) - } + val processedResult = s.result.map { (result: ResultBundle) => + result.successful match { + case None => cleanUpAfterCompilationError(result) + case Some(res) => + unregisterDeduplicationAndRegisterSuccessful(project, oinputs, res, logger) + } + result } /** @@ -242,7 +223,7 @@ object CompileGatekeeper { * memoized for correctness reasons. The result task can be called * several times by the compilation engine driving the execution. */ - Task(s.copy(result = processedResult.memoize)) + s.copy(result = processedResult.memoize) case result => cleanUpAfterCompilationError(result) } @@ -259,30 +240,26 @@ object CompileGatekeeper { oracleInputs: UniqueCompileInputs, successful: LastSuccessfulResult, logger: Logger - ): Task[Unit] = Task.liftMonixTaskUncancellable { - runningCompilations - .modify { state => - val newSuccessfulResults = (project.uniqueId, successful) - if (state.contains(oracleInputs)) { - val newState = state - oracleInputs - (newState, lastSuccessfulResults.update { _ + newSuccessfulResults }) - } else { - (state, MonixTask.unit) - } - } - .flatten - .map { _ => - logger.debug( - s"Recording new last successful request for ${project.name} associated with ${successful.classesDir}" - ) - () + ): Unit = { + runningCompilations.compute( + oracleInputs, + (_: UniqueCompileInputs, _: RunningCompilation) => { + lastSuccessfulResults.compute(project.uniqueId, (_, _) => successful) + null } + ) + + logger.debug( + s"Recording new last successful request for ${project.name} associated with ${successful.classesDir}" + ) + + () } // Expose clearing mechanism so that it can be invoked in the tests and community build runner -// private[bloop] def clearSuccessfulResults(): Unit = { -// lastSuccessfulResults.synchronized { -// lastSuccessfulResults.clear() -// } -// } + private[bloop] def clearSuccessfulResults(): Unit = { + lastSuccessfulResults.synchronized { + lastSuccessfulResults.clear() + } + } } diff --git a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGraph.scala b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGraph.scala index 8d7aef5b21..1aab0af1ef 100644 --- a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGraph.scala +++ b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGraph.scala @@ -133,227 +133,223 @@ object CompileGraph { withBundle { bundle0 => val logger = bundle0.logger - - CompileGatekeeper - .findRunningCompilationAtomically(inputs, bundle0, client, compile) - .flatMap { a => - val (runningCompilation, deduplicate) = a - val bundle = bundle0.copy(lastSuccessful = runningCompilation.usedLastSuccessful) - - if (!deduplicate) { - runningCompilation.traversal - } else { - val rawLogger = logger.underlying - rawLogger.info( - s"Deduplicating compilation of ${bundle.project.name} from ${runningCompilation.client}" - ) - val reporter = bundle.reporter.underlying - // Don't use `bundle.lastSuccessful`, it's not the final input to `compile` - val analysis = runningCompilation.usedLastSuccessful.previous.analysis().toOption - val previousSuccessfulProblems = - Compiler.previousProblemsFromSuccessfulCompilation(analysis) - val wasPreviousSuccessful = bundle.latestResult match { - case Compiler.Result.Ok(_) => true - case _ => false - } - val previousProblems = - Compiler.previousProblemsFromResult(bundle.latestResult, previousSuccessfulProblems) - - val clientClassesObserver = client.getClassesObserverFor(bundle.project) - - // Replay events asynchronously to waiting for the compilation result - import scala.concurrent.duration.FiniteDuration - import monix.execution.exceptions.UpstreamTimeoutException - val disconnectionTime = SystemProperties.getCompileDisconnectionTime(rawLogger) - val replayEventsTask = runningCompilation.mirror - .timeoutOnSlowUpstream(disconnectionTime) - .foreachL { - case Left(action) => - action match { - case ReporterAction.EnableFatalWarnings => - reporter.enableFatalWarnings() - case ReporterAction.ReportStartCompilation => - reporter.reportStartCompilation(previousProblems, wasPreviousSuccessful) - case a: ReporterAction.ReportStartIncrementalCycle => - reporter.reportStartIncrementalCycle(a.sources, a.outputDirs) - case a: ReporterAction.ReportProblem => reporter.log(a.problem) - case ReporterAction.PublishDiagnosticsSummary => - reporter.printSummary() - case a: ReporterAction.ReportNextPhase => - reporter.reportNextPhase(a.phase, a.sourceFile) - case a: ReporterAction.ReportCompilationProgress => - reporter.reportCompilationProgress(a.progress, a.total) - case a: ReporterAction.ReportEndIncrementalCycle => - reporter.reportEndIncrementalCycle(a.durationMs, a.result) - case ReporterAction.ReportCancelledCompilation => - reporter.reportCancelledCompilation() - case a: ReporterAction.ProcessEndCompilation => - a.code match { - case BspStatusCode.Cancelled | BspStatusCode.Error => - reporter.processEndCompilation(previousProblems, a.code, None, None) - reporter.reportEndCompilation() - case _ => - /* - * Only process the end, don't report it. It's only safe to - * report when all the client tasks have been run and the - * analysis/classes dirs are fully populated so that clients - * can use `taskFinish` notifications as a signal to process them. - */ - reporter.processEndCompilation( - previousProblems, - a.code, - Some(clientClassesObserver.classesDir), - Some(bundle.out.analysisOut) - ) - } - } - case Right(action) => - action match { - case LoggerAction.LogErrorMessage(msg) => rawLogger.error(msg) - case LoggerAction.LogWarnMessage(msg) => rawLogger.warn(msg) - case LoggerAction.LogInfoMessage(msg) => rawLogger.info(msg) - case LoggerAction.LogDebugMessage(msg) => - rawLogger.debug(msg) - case LoggerAction.LogTraceMessage(msg) => - rawLogger.debug(msg) + val (runningCompilation, deduplicate) = + CompileGatekeeper.findRunningCompilationAtomically(inputs, bundle0, client, compile) + val bundle = bundle0.copy(lastSuccessful = runningCompilation.usedLastSuccessful) + + if (!deduplicate) { + runningCompilation.traversal + } else { + val rawLogger = logger.underlying + rawLogger.info( + s"Deduplicating compilation of ${bundle.project.name} from ${runningCompilation.client}" + ) + val reporter = bundle.reporter.underlying + // Don't use `bundle.lastSuccessful`, it's not the final input to `compile` + val analysis = runningCompilation.usedLastSuccessful.previous.analysis().toOption + val previousSuccessfulProblems = + Compiler.previousProblemsFromSuccessfulCompilation(analysis) + val wasPreviousSuccessful = bundle.latestResult match { + case Compiler.Result.Ok(_) => true + case _ => false + } + val previousProblems = + Compiler.previousProblemsFromResult(bundle.latestResult, previousSuccessfulProblems) + + val clientClassesObserver = client.getClassesObserverFor(bundle.project) + + // Replay events asynchronously to waiting for the compilation result + import scala.concurrent.duration.FiniteDuration + import monix.execution.exceptions.UpstreamTimeoutException + val disconnectionTime = SystemProperties.getCompileDisconnectionTime(rawLogger) + val replayEventsTask = runningCompilation.mirror + .timeoutOnSlowUpstream(disconnectionTime) + .foreachL { + case Left(action) => + action match { + case ReporterAction.EnableFatalWarnings => + reporter.enableFatalWarnings() + case ReporterAction.ReportStartCompilation => + reporter.reportStartCompilation(previousProblems, wasPreviousSuccessful) + case a: ReporterAction.ReportStartIncrementalCycle => + reporter.reportStartIncrementalCycle(a.sources, a.outputDirs) + case a: ReporterAction.ReportProblem => reporter.log(a.problem) + case ReporterAction.PublishDiagnosticsSummary => + reporter.printSummary() + case a: ReporterAction.ReportNextPhase => + reporter.reportNextPhase(a.phase, a.sourceFile) + case a: ReporterAction.ReportCompilationProgress => + reporter.reportCompilationProgress(a.progress, a.total) + case a: ReporterAction.ReportEndIncrementalCycle => + reporter.reportEndIncrementalCycle(a.durationMs, a.result) + case ReporterAction.ReportCancelledCompilation => + reporter.reportCancelledCompilation() + case a: ReporterAction.ProcessEndCompilation => + a.code match { + case BspStatusCode.Cancelled | BspStatusCode.Error => + reporter.processEndCompilation(previousProblems, a.code, None, None) + reporter.reportEndCompilation() + case _ => + /* + * Only process the end, don't report it. It's only safe to + * report when all the client tasks have been run and the + * analysis/classes dirs are fully populated so that clients + * can use `taskFinish` notifications as a signal to process them. + */ + reporter.processEndCompilation( + previousProblems, + a.code, + Some(clientClassesObserver.classesDir), + Some(bundle.out.analysisOut) + ) } } - .materialize - .map { - case Success(_) => DeduplicationResult.Ok - case Failure(_: UpstreamTimeoutException) => - DeduplicationResult.DisconnectFromDeduplication - case Failure(t) => DeduplicationResult.DeduplicationError(t) + case Right(action) => + action match { + case LoggerAction.LogErrorMessage(msg) => rawLogger.error(msg) + case LoggerAction.LogWarnMessage(msg) => rawLogger.warn(msg) + case LoggerAction.LogInfoMessage(msg) => rawLogger.info(msg) + case LoggerAction.LogDebugMessage(msg) => + rawLogger.debug(msg) + case LoggerAction.LogTraceMessage(msg) => + rawLogger.debug(msg) } + } + .materialize + .map { + case Success(_) => DeduplicationResult.Ok + case Failure(_: UpstreamTimeoutException) => + DeduplicationResult.DisconnectFromDeduplication + case Failure(t) => DeduplicationResult.DeduplicationError(t) + } - /* The task set up by another process whose memoized result we're going to - * reuse. To prevent blocking compilations, we execute this task (which will - * block until its completion is done) in the IO thread pool, which is - * unbounded. This makes sure that the blocking threads *never* block - * the computation pool, which could produce a hang in the build server. - */ - val runningCompilationTask = - runningCompilation.traversal.executeOn(ExecutionContext.ioScheduler) - - val deduplicateStreamSideEffectsHandle = - replayEventsTask.runToFuture(ExecutionContext.ioScheduler) - - /** - * Deduplicate and change the implementation of the task returning the - * deduplicate compiler result to trigger a syncing process to keep the - * client external classes directory up-to-date with the new classes - * directory. This copying process blocks until the background IO work - * of the deduplicated compilation result has been finished. Note that - * this mechanism allows pipelined compilations to perform this IO only - * when the full compilation of a module is finished. - */ - val obtainResultFromDeduplication = runningCompilationTask.map { results => - PartialCompileResult.mapEveryResult(results) { - case s @ PartialSuccess(bundle, compilerResult) => - val newCompilerResult = compilerResult.flatMap { results => - results.fromCompiler match { - case s: Compiler.Result.Success => - // Wait on new classes to be populated for correctness - val runningBackgroundTasks = s.backgroundTasks - .trigger(clientClassesObserver, reporter, bundle.tracer, logger) - .runAsync(ExecutionContext.ioScheduler) - Task.now(results.copy(runningBackgroundTasks = runningBackgroundTasks)) - case _: Compiler.Result.Cancelled => - // Make sure to cancel the deduplicating task if compilation is cancelled - deduplicateStreamSideEffectsHandle.cancel() - Task.now(results) - case _ => Task.now(results) - } - } - s.copy(result = newCompilerResult) - case result => result + /* The task set up by another process whose memoized result we're going to + * reuse. To prevent blocking compilations, we execute this task (which will + * block until its completion is done) in the IO thread pool, which is + * unbounded. This makes sure that the blocking threads *never* block + * the computation pool, which could produce a hang in the build server. + */ + val runningCompilationTask = + runningCompilation.traversal.executeOn(ExecutionContext.ioScheduler) + + val deduplicateStreamSideEffectsHandle = + replayEventsTask.runToFuture(ExecutionContext.ioScheduler) + + /** + * Deduplicate and change the implementation of the task returning the + * deduplicate compiler result to trigger a syncing process to keep the + * client external classes directory up-to-date with the new classes + * directory. This copying process blocks until the background IO work + * of the deduplicated compilation result has been finished. Note that + * this mechanism allows pipelined compilations to perform this IO only + * when the full compilation of a module is finished. + */ + val obtainResultFromDeduplication = runningCompilationTask.map { results => + PartialCompileResult.mapEveryResult(results) { + case s @ PartialSuccess(bundle, compilerResult) => + val newCompilerResult = compilerResult.flatMap { results => + results.fromCompiler match { + case s: Compiler.Result.Success => + // Wait on new classes to be populated for correctness + val runningBackgroundTasks = s.backgroundTasks + .trigger(clientClassesObserver, reporter, bundle.tracer, logger) + .runAsync(ExecutionContext.ioScheduler) + Task.now(results.copy(runningBackgroundTasks = runningBackgroundTasks)) + case _: Compiler.Result.Cancelled => + // Make sure to cancel the deduplicating task if compilation is cancelled + deduplicateStreamSideEffectsHandle.cancel() + Task.now(results) + case _ => Task.now(results) + } } - } + s.copy(result = newCompilerResult) + case result => result + } + } - val compileAndDeduplicate = Task - .chooseFirstOf( - obtainResultFromDeduplication, - Task.fromFuture(deduplicateStreamSideEffectsHandle) - ) - .executeOn(ExecutionContext.ioScheduler) - - val finalCompileTask = compileAndDeduplicate.flatMap { - case Left((result, deduplicationFuture)) => - Task.fromFuture(deduplicationFuture).map(_ => result) - case Right((compilationFuture, deduplicationResult)) => - deduplicationResult match { - case DeduplicationResult.Ok => Task.fromFuture(compilationFuture) - case DeduplicationResult.DeduplicationError(t) => - rawLogger.trace(t) - val failedDeduplicationResult = Compiler.Result.GlobalError( - s"Unexpected error while deduplicating compilation for ${inputs.project.name}: ${t.getMessage}", - Some(t) - ) - - /* - * When an error happens while replaying all events of the - * deduplicated compilation, we keep track of the error, wait - * until the deduplicated compilation finishes and then we - * replace the result by a failed result that informs the - * client compilation was not successfully deduplicated. - */ - Task.fromFuture(compilationFuture).map { results => - PartialCompileResult.mapEveryResult(results) { (p: PartialCompileResult) => - p match { - case s: PartialSuccess => - val failedBundle = ResultBundle(failedDeduplicationResult, None, None) - s.copy(result = s.result.map(_ => failedBundle)) - case result => result - } - } + val compileAndDeduplicate = Task + .chooseFirstOf( + obtainResultFromDeduplication, + Task.fromFuture(deduplicateStreamSideEffectsHandle) + ) + .executeOn(ExecutionContext.ioScheduler) + + val finalCompileTask = compileAndDeduplicate.flatMap { + case Left((result, deduplicationFuture)) => + Task.fromFuture(deduplicationFuture).map(_ => result) + case Right((compilationFuture, deduplicationResult)) => + deduplicationResult match { + case DeduplicationResult.Ok => Task.fromFuture(compilationFuture) + case DeduplicationResult.DeduplicationError(t) => + rawLogger.trace(t) + val failedDeduplicationResult = Compiler.Result.GlobalError( + s"Unexpected error while deduplicating compilation for ${inputs.project.name}: ${t.getMessage}", + Some(t) + ) + + /* + * When an error happens while replaying all events of the + * deduplicated compilation, we keep track of the error, wait + * until the deduplicated compilation finishes and then we + * replace the result by a failed result that informs the + * client compilation was not successfully deduplicated. + */ + Task.fromFuture(compilationFuture).map { results => + PartialCompileResult.mapEveryResult(results) { (p: PartialCompileResult) => + p match { + case s: PartialSuccess => + val failedBundle = ResultBundle(failedDeduplicationResult, None, None) + s.copy(result = s.result.map(_ => failedBundle)) + case result => result } + } + } - case DeduplicationResult.DisconnectFromDeduplication => - /* - * Deduplication timed out after no compilation updates were - * recorded. In theory, this could happen because a rogue - * compilation process has stalled or is blocked. To ensure - * deduplicated clients always make progress, we now proceed - * with: - * + case DeduplicationResult.DisconnectFromDeduplication => + /* + * Deduplication timed out after no compilation updates were + * recorded. In theory, this could happen because a rogue + * compilation process has stalled or is blocked. To ensure + * deduplicated clients always make progress, we now proceed + * with: + * * 1. Cancelling the dead-looking compilation, hoping that the - * process will wake up at some point and stop running. - * 2. Shutting down the deduplication and triggering a new - * compilation. If there are several clients deduplicating this - * compilation, they will compete to start the compilation again - * with new compile inputs, as they could have already changed. - * 3. Reporting the end of compilation in case it hasn't been - * reported. Clients must handle two end compilation notifications - * gracefully. - * 4. Display the user that the deduplication was cancelled and a - * new compilation was scheduled. - */ - - CompileGatekeeper.disconnectDeduplicationFromRunning( - bundle.uniqueInputs, - runningCompilation - ) - - compilationFuture.cancel() - reporter.processEndCompilation(Nil, StatusCode.Cancelled, None, None) - reporter.reportEndCompilation() - - logger.displayWarningToUser( - s"""Disconnecting from deduplication of ongoing compilation for '${inputs.project.name}' - |No progress update for ${(disconnectionTime: FiniteDuration) - .toString()} caused bloop to cancel compilation and schedule a new compile. + * process will wake up at some point and stop running. + * 2. Shutting down the deduplication and triggering a new + * compilation. If there are several clients deduplicating this + * compilation, they will compete to start the compilation again + * with new compile inputs, as they could have already changed. + * 3. Reporting the end of compilation in case it hasn't been + * reported. Clients must handle two end compilation notifications + * gracefully. + * 4. Display the user that the deduplication was cancelled and a + * new compilation was scheduled. + */ + + CompileGatekeeper.disconnectDeduplicationFromRunning( + bundle.uniqueInputs, + runningCompilation + ) + + compilationFuture.cancel() + reporter.processEndCompilation(Nil, StatusCode.Cancelled, None, None) + reporter.reportEndCompilation() + + logger.displayWarningToUser( + s"""Disconnecting from deduplication of ongoing compilation for '${inputs.project.name}' + |No progress update for ${(disconnectionTime: FiniteDuration) + .toString()} caused bloop to cancel compilation and schedule a new compile. """.stripMargin - ) + ) - setupAndDeduplicate(client, inputs, setup)(compile) - } + setupAndDeduplicate(client, inputs, setup)(compile) } + } - bundle.tracer.traceTask(s"deduplicating ${bundle.project.name}") { _ => - finalCompileTask.executeOn(ExecutionContext.ioScheduler) - } - } + bundle.tracer.traceTask(s"deduplicating ${bundle.project.name}") { _ => + finalCompileTask.executeOn(ExecutionContext.ioScheduler) } + } } } From 9514e52c9a6092d8f80b90642d9aa57d6f494ba7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Wi=C4=85cek?= Date: Tue, 3 Jun 2025 18:20:24 +0200 Subject: [PATCH 07/10] addtional logging --- .../scala/bloop/engine/tasks/CompileTask.scala | 1 + .../tasks/compilation/CompileGatekeeper.scala | 15 ++++++++++++++- .../engine/tasks/compilation/CompileGraph.scala | 3 ++- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala b/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala index daeb365bb0..725655d488 100644 --- a/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala +++ b/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala @@ -484,6 +484,7 @@ object CompileTask { compilerResult: Compiler.Result, logger: Logger ): Task[Unit] = { + logger.debug("cleaning up previous results") val previousClassesDir = previousSuccessful.classesDir val currentlyUsedCounter = previousSuccessful.counterForClassesDir.decrementAndGet(1) diff --git a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGatekeeper.scala b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGatekeeper.scala index 6028b4e837..17ac435e56 100644 --- a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGatekeeper.scala +++ b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGatekeeper.scala @@ -45,12 +45,14 @@ object CompileGatekeeper { client: ClientInfo, compile: SuccessfulCompileBundle => CompileTraversal ): (RunningCompilation, CanBeDeduplicated) = { + import bundle.logger var deduplicate = true val running = runningCompilations.compute( bundle.uniqueInputs, (_: UniqueCompileInputs, running: RunningCompilation) => { if (running == null) { + logger.debug(s"no running compilation found starting new one:${bundle.uniqueInputs}") deduplicate = false scheduleCompilation(inputs, bundle, client, compile) } else { @@ -59,6 +61,7 @@ object CompileGatekeeper { usedClassesDirCounter.getAndTransform { count => if (count == 0) { + logger.debug(s"Abort deduplication, dir is scheduled to be deleted in background:${bundle.uniqueInputs}") // Abort deduplication, dir is scheduled to be deleted in background deduplicate = false // Remove from map of used classes dirs in case it hasn't already been @@ -66,6 +69,7 @@ object CompileGatekeeper { // Return previous count, this counter will soon be deallocated count } else { + logger.debug(s"Increase count to prevent other compiles to schedule its deletion:${bundle.uniqueInputs}") // Increase count to prevent other compiles to schedule its deletion count + 1 } @@ -82,8 +86,10 @@ object CompileGatekeeper { def disconnectDeduplicationFromRunning( inputs: UniqueCompileInputs, - runningCompilation: RunningCompilation + runningCompilation: RunningCompilation, + logger: Logger ): Unit = { + logger.debug(s"Disconnected deduplication from running compilation:${inputs}") runningCompilation.isUnsubscribed.compareAndSet(false, true) runningCompilations.remove(inputs, runningCompilation); () } @@ -132,14 +138,19 @@ object CompileGatekeeper { lastSuccessfulResults.compute( project.uniqueId, (_: String, previousResultOrNull: LastSuccessfulResult) => { + logger.debug( + s"Return previous result or the initial last successful coming from the bundle:${project.uniqueId}" + ) // Return previous result or the initial last successful coming from the bundle val previousResult = initializeLastSuccessful(previousResultOrNull) currentlyUsedClassesDirs.compute( previousResult.classesDir, (_: AbsolutePath, counter: AtomicInt) => { + logger.debug(s"Set counter for used classes dir when init or incrementing:${previousResult.classesDir}") // Set counter for used classes dir when init or incrementing if (counter == null) { + logger.debug(s"Create new counter:${previousResult.classesDir}") val initialCounter = AtomicInt(1) counterForUsedClassesDir = initialCounter initialCounter @@ -200,6 +211,7 @@ object CompileGatekeeper { def cleanUpAfterCompilationError[T](result: T): T = { if (!isAlreadyUnsubscribed.get) { // Remove running compilation if host compilation hasn't unsubscribed (maybe it's blocked) + logger.debug(s"Remove running compilation if host compilation hasn't unsubscribed (maybe it's blocked):${oinputs}") runningCompilations.remove(oinputs) } @@ -244,6 +256,7 @@ object CompileGatekeeper { runningCompilations.compute( oracleInputs, (_: UniqueCompileInputs, _: RunningCompilation) => { + logger.debug("Unregister deduplication and registered successfully") lastSuccessfulResults.compute(project.uniqueId, (_, _) => successful) null } diff --git a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGraph.scala b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGraph.scala index 1aab0af1ef..7b69d2247f 100644 --- a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGraph.scala +++ b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGraph.scala @@ -328,7 +328,8 @@ object CompileGraph { CompileGatekeeper.disconnectDeduplicationFromRunning( bundle.uniqueInputs, - runningCompilation + runningCompilation, + logger ) compilationFuture.cancel() From 25fcdebd247abca33cba7f122c1ac1e6ce93c20b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Wi=C4=85cek?= Date: Tue, 3 Jun 2025 18:22:40 +0200 Subject: [PATCH 08/10] reformat code --- .../tasks/compilation/CompileGatekeeper.scala | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGatekeeper.scala b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGatekeeper.scala index 17ac435e56..9dc34d4ad1 100644 --- a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGatekeeper.scala +++ b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGatekeeper.scala @@ -61,7 +61,9 @@ object CompileGatekeeper { usedClassesDirCounter.getAndTransform { count => if (count == 0) { - logger.debug(s"Abort deduplication, dir is scheduled to be deleted in background:${bundle.uniqueInputs}") + logger.debug( + s"Abort deduplication, dir is scheduled to be deleted in background:${bundle.uniqueInputs}" + ) // Abort deduplication, dir is scheduled to be deleted in background deduplicate = false // Remove from map of used classes dirs in case it hasn't already been @@ -69,7 +71,9 @@ object CompileGatekeeper { // Return previous count, this counter will soon be deallocated count } else { - logger.debug(s"Increase count to prevent other compiles to schedule its deletion:${bundle.uniqueInputs}") + logger.debug( + s"Increase count to prevent other compiles to schedule its deletion:${bundle.uniqueInputs}" + ) // Increase count to prevent other compiles to schedule its deletion count + 1 } @@ -147,7 +151,9 @@ object CompileGatekeeper { currentlyUsedClassesDirs.compute( previousResult.classesDir, (_: AbsolutePath, counter: AtomicInt) => { - logger.debug(s"Set counter for used classes dir when init or incrementing:${previousResult.classesDir}") + logger.debug( + s"Set counter for used classes dir when init or incrementing:${previousResult.classesDir}" + ) // Set counter for used classes dir when init or incrementing if (counter == null) { logger.debug(s"Create new counter:${previousResult.classesDir}") @@ -211,7 +217,9 @@ object CompileGatekeeper { def cleanUpAfterCompilationError[T](result: T): T = { if (!isAlreadyUnsubscribed.get) { // Remove running compilation if host compilation hasn't unsubscribed (maybe it's blocked) - logger.debug(s"Remove running compilation if host compilation hasn't unsubscribed (maybe it's blocked):${oinputs}") + logger.debug( + s"Remove running compilation if host compilation hasn't unsubscribed (maybe it's blocked):${oinputs}" + ) runningCompilations.remove(oinputs) } From 9711a04f73062c7bfdd18721705d64c39008a96b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Wi=C4=85cek?= Date: Wed, 11 Jun 2025 13:12:05 +0200 Subject: [PATCH 09/10] addtional logging and traces --- .../scala/bloop/tracing/TraceProperties.scala | 2 +- .../src/it/scala/bloop/CommunityBuild.scala | 2 +- frontend/src/main/scala/bloop/Cli.scala | 125 ++-- .../scala/bloop/bsp/BloopBspServices.scala | 3 +- .../main/scala/bloop/engine/Interpreter.scala | 51 +- .../bloop/engine/tasks/CompileTask.scala | 537 +++++++-------- .../tasks/compilation/CompileGatekeeper.scala | 359 +++++----- .../compilation/CompileGatekeeperNew.scala | 288 ++++++++ .../tasks/compilation/CompileGraph.scala | 624 +++++++++--------- .../src/test/resources/source-generator.py | 32 + .../scala/bloop/MonixBaseCompileSpec.scala | 30 +- .../test/scala/bloop/MonixCompile2Spec.scala | 5 + .../scala/bloop/dap/DebugServerSpec.scala | 2 +- 13 files changed, 1256 insertions(+), 804 deletions(-) create mode 100644 frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGatekeeperNew.scala create mode 100644 frontend/src/test/scala/bloop/MonixCompile2Spec.scala diff --git a/backend/src/main/scala/bloop/tracing/TraceProperties.scala b/backend/src/main/scala/bloop/tracing/TraceProperties.scala index 07d5cf3aea..3788457d6b 100644 --- a/backend/src/main/scala/bloop/tracing/TraceProperties.scala +++ b/backend/src/main/scala/bloop/tracing/TraceProperties.scala @@ -33,7 +33,7 @@ object TraceProperties { localServiceName, traceStartAnnotation, traceEndAnnotation, - enabled + true ) } } diff --git a/frontend/src/it/scala/bloop/CommunityBuild.scala b/frontend/src/it/scala/bloop/CommunityBuild.scala index 4937f38e4f..7d4a37202b 100644 --- a/frontend/src/it/scala/bloop/CommunityBuild.scala +++ b/frontend/src/it/scala/bloop/CommunityBuild.scala @@ -108,7 +108,7 @@ abstract class CommunityBuild(val buildpressHomeDir: AbsolutePath) { } // First thing to do: clear cache of successful results between project runs to free up space -// CompileGatekeeper.clearSuccessfulResults() +// CompileGatekeeperOld.clearSuccessfulResults() // After reporting the state of the execution, compile the projects accordingly. val logger = BloopLogger.default("community-build-logger") diff --git a/frontend/src/main/scala/bloop/Cli.scala b/frontend/src/main/scala/bloop/Cli.scala index b00901a0cb..fd860ef6db 100644 --- a/frontend/src/main/scala/bloop/Cli.scala +++ b/frontend/src/main/scala/bloop/Cli.scala @@ -36,7 +36,10 @@ object Cli extends TaskApp { implicit private val filter: DebugFilter.All.type = DebugFilter.All def run(args: List[String]): MonixTask[ExitCode] = { val action = parse(args.toArray, CommonOptions.default) - run(action, NoPool).map(exitStatus => ExitCode(exitStatus.code)) + for { + activeCliSessions <- Ref.of[MonixTask, Map[Path, List[CliSession]]](Map.empty) + exitStatus <- run(action, NoPool, activeCliSessions, None) + } yield ExitCode(exitStatus.code) } def reflectMain( @@ -61,7 +64,7 @@ object Cli extends TaskApp { ) val cmd = parse(args, nailgunOptions) - val exitStatus = run(cmd, NoPool, cancel, activeCliSessions) + val exitStatus = run(cmd, NoPool, cancel, activeCliSessions, None) exitStatus.map(_.code) } @@ -93,15 +96,18 @@ object Cli extends TaskApp { println(nailgunOptions.workingPath) - val handle = run(cmd, NailgunPool(ngContext)) - .onErrorHandle { - case x: java.util.concurrent.ExecutionException => - // print stack trace of fatal errors thrown in asynchronous code, see https://stackoverflow.com/questions/17265022/what-is-a-boxed-error-in-scala - // the stack trace is somehow propagated all the way to the client when printing this - x.getCause.printStackTrace(ngContext.out) - ExitStatus.UnexpectedError.code - } - .runToFuture(ExecutionContext.ioScheduler) + val handle = + Ref + .of[MonixTask, Map[Path, List[CliSession]]](Map.empty) + .flatMap(run(cmd, NailgunPool(ngContext), _, None)) + .onErrorHandle { + case x: java.util.concurrent.ExecutionException => + // print stack trace of fatal errors thrown in asynchronous code, see https://stackoverflow.com/questions/17265022/what-is-a-boxed-error-in-scala + // the stack trace is somehow propagated all the way to the client when printing this + x.getCause.printStackTrace(ngContext.out) + ExitStatus.UnexpectedError.code + } + .runToFuture(ExecutionContext.ioScheduler) Await.result(handle, Duration.Inf) () @@ -304,12 +310,16 @@ object Cli extends TaskApp { } } - def run(action: Action, pool: ClientPool): MonixTask[ExitStatus] = { + def run( + action: Action, + pool: ClientPool, + activeCliSessions: Ref[MonixTask, Map[Path, List[CliSession]]], + postfix: Option[String] + ): MonixTask[ExitStatus] = { for { baseCancellation <- Deferred[MonixTask, Boolean] - activeCliSessions <- Ref.of[MonixTask, Map[Path, List[CliSession]]](Map.empty) _ <- baseCancellation.complete(false) - result <- run(action, pool, baseCancellation, activeCliSessions) + result <- run(action, pool, baseCancellation, activeCliSessions, postfix) } yield result } @@ -319,7 +329,8 @@ object Cli extends TaskApp { action: Action, pool: ClientPool, cancel: Deferred[MonixTask, Boolean], - activeCliSessions: Ref[MonixTask, Map[Path, List[CliSession]]] + activeCliSessions: Ref[MonixTask, Map[Path, List[CliSession]]], + postfix: Option[String] ): MonixTask[ExitStatus] = { import bloop.io.AbsolutePath def getConfigDir(cliOptions: CliOptions): AbsolutePath = { @@ -351,12 +362,13 @@ object Cli extends TaskApp { !(cliOptions.noColor || commonOpts.env.containsKey("NO_COLOR")), debugFilter ) - + logger.debug("Running cli" + postfix.fold("")(":" + _) + "...") action match { case Print(msg, _, Exit(exitStatus)) => logger.info(msg) MonixTask.now(exitStatus) case _ => + logger.debug("running action" + postfix.fold("")(":" + _) + "...") runWithState( action, pool, @@ -365,7 +377,8 @@ object Cli extends TaskApp { configDirectory, cliOptions, commonOpts, - logger + logger, + postfix ) } } @@ -378,17 +391,19 @@ object Cli extends TaskApp { configDirectory: AbsolutePath, cliOptions: CliOptions, commonOpts: CommonOptions, - logger: Logger + logger: Logger, + postfix: Option[String] ): MonixTask[ExitStatus] = { - + logger.debug("running with state" + postfix.fold("")(":" + _) + "...") // Set the proxy settings right before loading the state of the build bloop.util.ProxySetup.updateProxySettings(commonOpts.env.toMap, logger) val configDir = configDirectory.underlying waitUntilEndOfWorld(cliOptions, pool, configDir, logger, cancel) { val taskToInterpret = { (cli: CliClientInfo) => + logger.debug("Running command interpretation" + postfix.fold("")(":" + _) + "...") val state = State.loadActiveStateFor(configDirectory, cli, pool, cliOptions.common, logger) - val interpret = Interpreter.execute(action, state).map { newState => + val interpret = Interpreter.execute(action, state, postfix).map { newState => action match { case Run(_: Commands.ValidatedBsp, _) => () // Ignore, BSP services auto-update the build @@ -406,16 +421,25 @@ object Cli extends TaskApp { taskToInterpret, activeCliSessions, pool, - logger + logger, + postfix ) val exitSession = MonixTask.defer { - session.flatMap(s => cleanUpNonStableCliDirectories(s.client)) + session.flatMap { session => + cleanUpNonStableCliDirectories(session.client, logger).flatMap { _ => + logger.debug( + s"Cleaning up non-stable cli directories: $configDirectory" + postfix.fold("")( + ":" + _ + ) + "..." + ) + activeCliSessions.update(_ - configDirectory.underlying) + } + } } session .flatMap(_.task) - .doOnCancel(exitSession) - .doOnFinish(_ => exitSession) + .guarantee(exitSession) } } @@ -424,9 +448,10 @@ object Cli extends TaskApp { configDir: AbsolutePath, action: Action, processCliTask: CliClientInfo => MonixTask[State], - activeCliSessions2: Ref[MonixTask, Map[Path, List[CliSession]]], + activeCliSessions: Ref[MonixTask, Map[Path, List[CliSession]]], pool: ClientPool, - logger: Logger + logger: Logger, + postfix: Option[String] ): MonixTask[CliSession] = { val isClientConnected = AtomicBoolean(true) pool.addListener(_ => isClientConnected.set(false)) @@ -447,30 +472,48 @@ object Cli extends TaskApp { case Run(_: Commands.Bsp, _) => MonixTask.now(defaultClientSession) case Run(_: Commands.ValidatedBsp, _) => MonixTask.now(defaultClientSession) case a @ _ => - activeCliSessions2.modify { sessionsMap => - val currentSessions = sessionsMap.getOrElse(configDir.underlying, Nil) - - val updatedSessions = - if (currentSessions.isEmpty) { - List(defaultClientSession) - } else { - logger.debug("Detected connected cli clients, starting CLI with unique dirs...") - val newClient = CliClientInfo(useStableCliDirs = false, () => isClientConnected.get) - val newClientSession = sessionFor(newClient) - newClientSession :: currentSessions + logger.debug( + "command: " + action.asInstanceOf[Run].command.getClass.toString + " " + postfix.fold("")( + ":" + _ + ) + "..." + ) + activeCliSessions + .modify { sessionsMap => + sessionsMap.get(configDir.underlying) match { + case Some(sessions) => + val newClient = CliClientInfo(useStableCliDirs = false, () => isClientConnected.get) + val newClientSession = sessionFor(newClient) + ( + sessionsMap.updated(configDir.underlying, newClientSession :: sessions), + newClientSession + ) + case None => + ( + sessionsMap.updated(configDir.underlying, List(defaultClientSession)), + defaultClientSession + ) } + } + .flatMap { sessions => + activeCliSessions.get + .map { sessionsMap => + logger.debug("currentActiveSessions:" + postfix.fold("")(_ + ": ") + sessionsMap) + sessions + } - val updatedMap = sessionsMap.updated(configDir.underlying, updatedSessions) - (updatedMap, updatedSessions.head) - } + } } } def cleanUpNonStableCliDirectories( - client: CliClientInfo + client: CliClientInfo, + logger: Logger ): MonixTask[Unit] = { if (client.useStableCliDirs) MonixTask.unit else { + logger.debug( + s"Cleaning up non-stable CLI directories ${client.getCreatedCliDirectories.mkString(",")}" + ) val deleteTasks = client.getCreatedCliDirectories.map { freshDir => if (!freshDir.exists) MonixTask.unit else { diff --git a/frontend/src/main/scala/bloop/bsp/BloopBspServices.scala b/frontend/src/main/scala/bloop/bsp/BloopBspServices.scala index a47b207b2e..f97feb95f2 100644 --- a/frontend/src/main/scala/bloop/bsp/BloopBspServices.scala +++ b/frontend/src/main/scala/bloop/bsp/BloopBspServices.scala @@ -579,7 +579,8 @@ final class BloopBspServices( bestEffortAllowed, cancelCompilation, store, - logger + logger, + Option("bloopBsp") ) } diff --git a/frontend/src/main/scala/bloop/engine/Interpreter.scala b/frontend/src/main/scala/bloop/engine/Interpreter.scala index 5342eaccf5..8305044d5f 100644 --- a/frontend/src/main/scala/bloop/engine/Interpreter.scala +++ b/frontend/src/main/scala/bloop/engine/Interpreter.scala @@ -37,8 +37,8 @@ import bloop.testing.TestInternals object Interpreter { // This is stack-safe because of Monix's trampolined execution - def execute(action: Action, stateTask: Task[State]): Task[State] = { - def execute(action: Action, stateTask: Task[State]): Task[State] = { + def execute(action: Action, stateTask: Task[State], postfix: Option[String] = None): Task[State] = { + def execute(action: Action, stateTask: Task[State], postfix: Option[String]): Task[State] = { stateTask.flatMap { state => action match { // We keep it case because there is a 'match may not be exhaustive' false positive by scalac @@ -48,13 +48,13 @@ object Interpreter { case Exit(exitStatus: ExitStatus) => Task.now(state.mergeStatus(exitStatus)) case Print(msg, _, next) => state.logger.info(msg) - execute(next, Task.now(state)) + execute(next, Task.now(state), postfix) case Run(cmd: Commands.Bsp, _) => val msg = "Internal error: The `bsp` command must be validated before use" val printAction = Print(msg, cmd.cliOptions.common, Exit(ExitStatus.UnexpectedError)) - execute(printAction, Task.now(state)) + execute(printAction, Task.now(state), postfix) case Run(cmd: Commands.ValidatedBsp, next) => - execute(next, runBsp(cmd, state)) + execute(next, runBsp(cmd, state), postfix) case Run(cmd: Commands.About, _) => notHandled("about", cmd.cliOptions, state) case Run(cmd: Commands.Help, _) => @@ -67,23 +67,23 @@ object Interpreter { else { cmd match { case cmd: Commands.Clean => - execute(next, clean(cmd, state)) + execute(next, clean(cmd, state), postfix) case cmd: Commands.Compile => - execute(next, compile(cmd, state)) + execute(next, compile(cmd, state, postfix), postfix) case cmd: Commands.Console => - execute(next, console(cmd, state)) + execute(next, console(cmd, state), postfix) case cmd: Commands.Projects => - execute(next, showProjects(cmd, state)) + execute(next, showProjects(cmd, state), postfix) case cmd: Commands.Test => - execute(next, test(cmd, state)) + execute(next, test(cmd, state), postfix) case cmd: Commands.Run => - execute(next, run(cmd, state)) + execute(next, run(cmd, state), postfix) case cmd: Commands.Configure => - execute(next, configure(cmd, state)) + execute(next, configure(cmd, state), postfix) case cmd: Commands.Autocomplete => - execute(next, autocomplete(cmd, state)) + execute(next, autocomplete(cmd, state), postfix) case cmd: Commands.Link => - execute(next, link(cmd, state)) + execute(next, link(cmd, state), postfix) } } } @@ -91,7 +91,7 @@ object Interpreter { } } - execute(action, stateTask) + execute(action, stateTask, postfix) } private def notHandled(command: String, cliOptions: CliOptions, state: State): Task[State] = { @@ -153,7 +153,8 @@ object Interpreter { cmd: CompilingCommand, state0: State, projects: List[Project], - noColor: Boolean + noColor: Boolean, + postfix: Option[String] = None ): Task[State] = { // Make new state cleaned of all compilation products if compilation is not incremental val state: Task[State] = { @@ -178,14 +179,19 @@ object Interpreter { bestEffortAllowed = false, Promise[Unit](), CompileClientStore.NoStore, - state.logger + state.logger, + postfix ) } compileTask.map(_.mergeStatus(ExitStatus.Ok)) } - private def compile(cmd: Commands.Compile, state: State): Task[State] = { + private def compile( + cmd: Commands.Compile, + state: State, + postfix: Option[String] = None + ): Task[State] = { val lookup = lookupProjects(cmd.projects, state.build.getProjectFor(_)) if (lookup.missing.nonEmpty) Task.now(reportMissing(lookup.missing, state)) else { @@ -194,8 +200,8 @@ object Interpreter { else Dag.inverseDependencies(state.build.dags, lookup.found).reduced } - if (!cmd.watch) runCompile(cmd, state, projects, cmd.cliOptions.noColor) - else watch(projects, state)(runCompile(cmd, _, projects, cmd.cliOptions.noColor)) + if (!cmd.watch) runCompile(cmd, state, projects, cmd.cliOptions.noColor, postfix) + else watch(projects, state)(runCompile(cmd, _, projects, cmd.cliOptions.noColor, postfix.map(_ + cmd + ":watch"))) } } @@ -218,9 +224,10 @@ object Interpreter { state: State, projects: List[Project], noColor: Boolean, - nextAction: String + nextAction: String, + postfix: Option[String] = None )(next: State => Task[State]): Task[State] = { - runCompile(cmd, state, projects, noColor).flatMap { compiled => + runCompile(cmd, state, projects, noColor, postfix.orElse(Some(nextAction))).flatMap { compiled => if (compiled.status != ExitStatus.CompilationError) next(compiled) else { val projectsString = projects.mkString(", ") diff --git a/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala b/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala index 725655d488..8e1e3b69cd 100644 --- a/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala +++ b/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala @@ -47,6 +47,7 @@ import xsbti.compile.PreviousResult object CompileTask { private implicit val logContext: DebugFilter = DebugFilter.Compilation + def compile[UseSiteLogger <: Logger]( state: State, dag: Dag[Project], @@ -55,7 +56,8 @@ object CompileTask { bestEffortAllowed: Boolean, cancelCompilation: Promise[Unit], store: CompileClientStore, - rawLogger: UseSiteLogger + rawLogger: UseSiteLogger, + postfix: Option[String] = None ): Task[State] = Task.defer { import bloop.data.ClientInfo import bloop.internal.build.BuildInfo @@ -70,7 +72,7 @@ object CompileTask { val traceProperties = WorkspaceSettings.tracePropertiesFrom(state.build.workspaceSettings) val rootTracer = BraveTracer( - s"compile $topLevelTargets (transitively)", + s"compile $topLevelTargets (transitively)${postfix.fold("")(":" + _)}", traceProperties, "bloop.version" -> BuildInfo.version, "zinc.version" -> BuildInfo.zincVersion, @@ -89,296 +91,305 @@ object CompileTask { "client" -> clientName ) - def compile( - graphInputs: CompileGraph.Inputs, - isBestEffort: Boolean, - isBestEffortDep: Boolean - ): Task[ResultBundle] = { - val bundle = graphInputs.bundle - val project = bundle.project - val logger = bundle.logger - val reporter = bundle.reporter - val previousResult = bundle.latestResult - val compileOut = bundle.out - val lastSuccessful = bundle.lastSuccessful - val compileProjectTracer = rootTracer.startNewChildTracer( - s"compile ${project.name}", - "compile.target" -> project.name - ) + rootTracer.trace("CompileTask.compile") { tracer => + def compile( + graphInputs: CompileGraph.Inputs, + isBestEffort: Boolean, + isBestEffortDep: Boolean, + tracer: BraveTracer + ): Task[ResultBundle] = tracer.trace("CompileTask.compile - inner") { tracer => + val bundle = graphInputs.bundle + val project = bundle.project + val logger = bundle.logger + val reporter = bundle.reporter + val previousResult = bundle.latestResult + val compileOut = bundle.out + val lastSuccessful = bundle.lastSuccessful + val compileProjectTracer = tracer.startNewChildTracer( + s"compile ${project.name}", + "compile.target" -> project.name + ) - bundle.prepareSourcesAndInstance match { - case Left(earlyResultBundle) => - compileProjectTracer.terminate() - Task.now(earlyResultBundle) - case Right(CompileSourcesAndInstance(sources, instance, _)) => - val readOnlyClassesDir = lastSuccessful.classesDir - val newClassesDir = compileOut.internalNewClassesDir - val classpath = bundle.dependenciesData.buildFullCompileClasspathFor( - project, - readOnlyClassesDir, - newClassesDir - ) - - // Warn user if detected missing dep, see https://github.com/scalacenter/bloop/issues/708 - state.build.hasMissingDependencies(project).foreach { missing => - Feedback - .detectMissingDependencies(project.name, missing) - .foreach(msg => logger.warn(msg)) - } + bundle.prepareSourcesAndInstance match { + case Left(earlyResultBundle) => + compileProjectTracer.terminate() + Task.now(earlyResultBundle) + case Right(CompileSourcesAndInstance(sources, instance, _)) => + val readOnlyClassesDir = lastSuccessful.classesDir + val newClassesDir = compileOut.internalNewClassesDir + val classpath = bundle.dependenciesData.buildFullCompileClasspathFor( + project, + readOnlyClassesDir, + newClassesDir + ) - val configuration = configureCompilation(project) - val newScalacOptions = { - CompilerPluginAllowlist - .enableCachingInScalacOptions( - instance.version, - configuration.scalacOptions, + // Warn user if detected missing dep, see https://github.com/scalacenter/bloop/issues/708 + state.build.hasMissingDependencies(project).foreach { missing => + Feedback + .detectMissingDependencies(project.name, missing) + .foreach(msg => logger.warn(msg)) + } + + val configuration = configureCompilation(project) + val newScalacOptions = { + CompilerPluginAllowlist + .enableCachingInScalacOptions( + instance.version, + configuration.scalacOptions, + logger, + compileProjectTracer, + 5 + ) + } + + val inputs = newScalacOptions.map { newScalacOptions => + CompileInputs( + instance, + state.compilerCache, + sources.toArray, + classpath, + bundle.uniqueInputs, + compileOut, + project.out, + newScalacOptions.toArray, + project.javacOptions.toArray, + project.compileJdkConfig.flatMap(_.javacBin), + project.compileOrder, + project.classpathOptions, + lastSuccessful.previous, + previousResult, + reporter, logger, + graphInputs.dependentResults, + cancelCompilation, compileProjectTracer, - 5 + ExecutionContext.ioScheduler, + ExecutionContext.ioExecutor, + bundle.dependenciesData.allInvalidatedClassFiles, + bundle.dependenciesData.allGeneratedClassFilePaths, + project.runtimeResources ) - } - - val inputs = newScalacOptions.map { newScalacOptions => - CompileInputs( - instance, - state.compilerCache, - sources.toArray, - classpath, - bundle.uniqueInputs, - compileOut, - project.out, - newScalacOptions.toArray, - project.javacOptions.toArray, - project.compileJdkConfig.flatMap(_.javacBin), - project.compileOrder, - project.classpathOptions, - lastSuccessful.previous, - previousResult, - reporter, - logger, - graphInputs.dependentResults, - cancelCompilation, - compileProjectTracer, - ExecutionContext.ioScheduler, - ExecutionContext.ioExecutor, - bundle.dependenciesData.allInvalidatedClassFiles, - bundle.dependenciesData.allGeneratedClassFilePaths, - project.runtimeResources - ) - } - - val waitOnReadClassesDir = { - compileProjectTracer.traceTaskVerbose("wait on populating products") { _ => - // This task is memoized and started by the compilation that created - // it, so this execution blocks until it's run or completes right away - lastSuccessful.populatingProducts } - } - // Block on the task associated with this result that sets up the read-only classes dir - waitOnReadClassesDir.flatMap { _ => - // Only when the task is finished, we kickstart the compilation - def compile(inputs: CompileInputs) = { - val firstResult = - Compiler.compile(inputs, isBestEffort, isBestEffortDep, firstCompilation = true) - firstResult.flatMap { - case result @ Compiler.Result.Failed( - _, - _, - _, - _, - Some(BestEffortProducts(_, _, recompile)) - ) if recompile => - // we restart the compilation, starting from scratch (without any previous artifacts) - inputs.reporter.reset() - val emptyResult = - PreviousResult.of(Optional.empty[CompileAnalysis], Optional.empty[MiniSetup]) - val nonIncrementalClasspath = - inputs.classpath.filter(_ != inputs.out.internalReadOnlyClassesDir) - val newInputs = inputs.copy( - sources = inputs.sources, - classpath = nonIncrementalClasspath, - previousCompilerResult = result, - previousResult = emptyResult - ) - Compiler.compile( - newInputs, - isBestEffort, - isBestEffortDep, - firstCompilation = false - ) - case result => Task(result) + val waitOnReadClassesDir = { + compileProjectTracer.traceTaskVerbose("wait on populating products") { _ => + // This task is memoized and started by the compilation that created + // it, so this execution blocks until it's run or completes right away + lastSuccessful.populatingProducts } } - inputs.flatMap(inputs => compile(inputs)).map { result => - def runPostCompilationTasks( - backgroundTasks: CompileBackgroundTasks - ): CancelableFuture[Unit] = { - // Post compilation tasks use tracer, so terminate right after they have - val postCompilationTasks = - backgroundTasks - .trigger( - bundle.clientClassesObserver, - reporter.underlying, - compileProjectTracer, - logger + + // Block on the task associated with this result that sets up the read-only classes dir + waitOnReadClassesDir.flatMap { _ => + // Only when the task is finished, we kickstart the compilation + def compile(inputs: CompileInputs) = { + val firstResult = + Compiler.compile(inputs, isBestEffort, isBestEffortDep, firstCompilation = true) + firstResult.flatMap { + case result @ Compiler.Result.Failed( + _, + _, + _, + _, + Some(BestEffortProducts(_, _, recompile)) + ) if recompile => + // we restart the compilation, starting from scratch (without any previous artifacts) + inputs.reporter.reset() + val emptyResult = + PreviousResult.of(Optional.empty[CompileAnalysis], Optional.empty[MiniSetup]) + val nonIncrementalClasspath = + inputs.classpath.filter(_ != inputs.out.internalReadOnlyClassesDir) + val newInputs = inputs.copy( + sources = inputs.sources, + classpath = nonIncrementalClasspath, + previousCompilerResult = result, + previousResult = emptyResult + ) + Compiler.compile( + newInputs, + isBestEffort, + isBestEffortDep, + firstCompilation = false ) - .doOnFinish(_ => Task(compileProjectTracer.terminate())) - postCompilationTasks.runAsync(ExecutionContext.ioScheduler) + case result => Task(result) + } } - // Populate the last successful result if result was success - result match { - case s: Compiler.Result.Success => - val runningTasks = runPostCompilationTasks(s.backgroundTasks) - val blockingOnRunningTasks = Task - .fromFuture(runningTasks) - .executeOn(ExecutionContext.ioScheduler) - val populatingTask = { - if (s.isNoOp) blockingOnRunningTasks // Task.unit - else { - for { - _ <- blockingOnRunningTasks - _ <- populateNewReadOnlyClassesDir(s.products, bgTracer, rawLogger) - .doOnFinish(_ => Task(bgTracer.terminate())) - } yield () + inputs.flatMap(inputs => compile(inputs)).map { result => + def runPostCompilationTasks( + backgroundTasks: CompileBackgroundTasks + ): CancelableFuture[Unit] = { + // Post compilation tasks use tracer, so terminate right after they have + val postCompilationTasks = + backgroundTasks + .trigger( + bundle.clientClassesObserver, + reporter.underlying, + compileProjectTracer, + logger + ) + .doOnFinish(_ => Task(compileProjectTracer.terminate())) + postCompilationTasks.runAsync(ExecutionContext.ioScheduler) + } + + // Populate the last successful result if result was success + result match { + case s: Compiler.Result.Success => + val runningTasks = runPostCompilationTasks(s.backgroundTasks) + val blockingOnRunningTasks = Task + .fromFuture(runningTasks) + .executeOn(ExecutionContext.ioScheduler) + val populatingTask = { + if (s.isNoOp) blockingOnRunningTasks // Task.unit + else { + for { + _ <- blockingOnRunningTasks + _ <- populateNewReadOnlyClassesDir(s.products, bgTracer, rawLogger) + .doOnFinish(_ => Task(bgTracer.terminate())) + } yield () + } } - } - - // Memoize so that no matter how many times it's run, it's executed only once - val newSuccessful = - LastSuccessfulResult(bundle.uniqueInputs, s.products, populatingTask.memoize) - ResultBundle(s, Some(newSuccessful), Some(lastSuccessful), runningTasks) - case f: Compiler.Result.Failed => - val runningTasks = runPostCompilationTasks(f.backgroundTasks) - ResultBundle(result, None, Some(lastSuccessful), runningTasks) - case c: Compiler.Result.Cancelled => - val runningTasks = runPostCompilationTasks(c.backgroundTasks) - ResultBundle(result, None, Some(lastSuccessful), runningTasks) - case _: Compiler.Result.Blocked | Compiler.Result.Empty | - _: Compiler.Result.GlobalError => - ResultBundle(result, None, None, CancelableFuture.unit) + + // Memoize so that no matter how many times it's run, it's executed only once + val newSuccessful = + LastSuccessfulResult(bundle.uniqueInputs, s.products, populatingTask.memoize) + ResultBundle(s, Some(newSuccessful), Some(lastSuccessful), runningTasks) + case f: Compiler.Result.Failed => + val runningTasks = runPostCompilationTasks(f.backgroundTasks) + ResultBundle(result, None, Some(lastSuccessful), runningTasks) + case c: Compiler.Result.Cancelled => + val runningTasks = runPostCompilationTasks(c.backgroundTasks) + ResultBundle(result, None, Some(lastSuccessful), runningTasks) + case _: Compiler.Result.Blocked | Compiler.Result.Empty | + _: Compiler.Result.GlobalError => + ResultBundle(result, None, None, CancelableFuture.unit) + } } } - } - } - } - - def setup(inputs: CompileDefinitions.BundleInputs): Task[CompileBundle] = { - // Create a multicast observable stream to allow multiple mirrors of loggers - val (observer, obs) = { - Observable.multicast[Either[ReporterAction, LoggerAction]]( - MulticastStrategy.replay - )(ExecutionContext.ioScheduler) - } - - // Compute the previous and last successful results from the results cache - import inputs.project - val (prev, last) = { - if (pipeline) { - val emptySuccessful = LastSuccessfulResult.empty(project) - // Disable incremental compilation if pipelining is enabled - Compiler.Result.Empty -> emptySuccessful - } else { - // Use last successful from user cache, only useful if this is its first - // compilation, otherwise we use the last successful from [[CompileGraph]] - val latestResult = state.results.latestResult(project) - val lastSuccessful = state.results.lastSuccessfulResultOrEmpty(project) - latestResult -> lastSuccessful } } - val t = rootTracer - val o = state.commonOptions - val cancel = cancelCompilation - val logger = ObservedLogger(rawLogger, observer) - val clientClassesObserver = state.client.getClassesObserverFor(inputs.project) - val underlying = createReporter(ReporterInputs(inputs.project, cwd, rawLogger)) - val reporter = new ObservedReporter(logger, underlying) - val sourceGeneratorCache = state.sourceGeneratorCache - CompileBundle.computeFrom( - inputs, - sourceGeneratorCache, - clientClassesObserver, - reporter, - last, - prev, - cancel, - logger, - obs, - t, - o - ) - } + def setup( + inputs: CompileDefinitions.BundleInputs, + tracer: BraveTracer + ): Task[CompileBundle] = { + // Create a multicast observable stream to allow multiple mirrors of loggers + val (observer, obs) = { + Observable.multicast[Either[ReporterAction, LoggerAction]]( + MulticastStrategy.replay + )(ExecutionContext.ioScheduler) + } - val client = state.client - CompileGraph.traverse(dag, client, store, bestEffortAllowed, setup(_), compile).flatMap { - pdag => - val partialResults = Dag.dfs(pdag, mode = Dag.PreOrder) - val finalResults = partialResults.map(r => PartialCompileResult.toFinalResult(r)) - Task.gatherUnordered(finalResults).map(_.flatten).flatMap { results => - val cleanUpTasksToRunInBackground = - markUnusedClassesDirAndCollectCleanUpTasks(results, state, rawLogger) - - val failures = results.flatMap { - case FinalNormalCompileResult(p, results) => - results.fromCompiler match { - case Compiler.Result.NotOk(_) => List(p) - // Consider success with reported fatal warnings as error to simulate -Xfatal-warnings - case s: Compiler.Result.Success if s.reportedFatalWarnings => List(p) - case _ => Nil - } - case _ => Nil + // Compute the previous and last successful results from the results cache + import inputs.project + val (prev, last) = { + if (pipeline) { + val emptySuccessful = LastSuccessfulResult.empty(project) + // Disable incremental compilation if pipelining is enabled + Compiler.Result.Empty -> emptySuccessful + } else { + // Use last successful from user cache, only useful if this is its first + // compilation, otherwise we use the last successful from [[CompileGraph]] + val latestResult = state.results.latestResult(project) + val lastSuccessful = state.results.lastSuccessfulResultOrEmpty(project) + latestResult -> lastSuccessful } + } - val newState: State = { - val stateWithResults = state.copy(results = state.results.addFinalResults(results)) - if (failures.isEmpty) { - stateWithResults.copy(status = ExitStatus.Ok) - } else { - results.foreach { - case FinalNormalCompileResult.HasException(project, err) => - val errMsg = err.fold(identity, Logger.prettyPrintException) - rawLogger.error(s"Unexpected error when compiling ${project.name}: $errMsg") - case _ => () // Do nothing when the final compilation result is not an actual error - } + val t = tracer + val o = state.commonOptions + val cancel = cancelCompilation + val logger = ObservedLogger(rawLogger, observer) + val clientClassesObserver = state.client.getClassesObserverFor(inputs.project) + val underlying = createReporter(ReporterInputs(inputs.project, cwd, rawLogger)) + val reporter = new ObservedReporter(logger, underlying) + val sourceGeneratorCache = state.sourceGeneratorCache + CompileBundle.computeFrom( + inputs, + sourceGeneratorCache, + clientClassesObserver, + reporter, + last, + prev, + cancel, + logger, + obs, + t, + o + ) + } - client match { - case _: ClientInfo.CliClientInfo => - // Reverse list of failed projects to get ~correct order of failure - val projectsFailedToCompile = failures.map(p => s"'${p.name}'").reverse - val failureMessage = - if (failures.size <= 2) projectsFailedToCompile.mkString(",") - else { - s"${projectsFailedToCompile.take(2).mkString(", ")} and ${projectsFailedToCompile.size - 2} more projects" - } + val client = state.client + CompileGraph + .traverse(dag, client, store, bestEffortAllowed, setup, compile, tracer) + .flatMap { pdag => + val partialResults = Dag.dfs(pdag, mode = Dag.PreOrder) + val finalResults = partialResults.map(r => PartialCompileResult.toFinalResult(r)) + Task.gatherUnordered(finalResults).map(_.flatten).flatMap { results => + val cleanUpTasksToRunInBackground = + markUnusedClassesDirAndCollectCleanUpTasks(results, state, rawLogger) + + val failures = results.flatMap { + case FinalNormalCompileResult(p, results) => + results.fromCompiler match { + case Compiler.Result.NotOk(_) => List(p) + // Consider success with reported fatal warnings as error to simulate -Xfatal-warnings + case s: Compiler.Result.Success if s.reportedFatalWarnings => List(p) + case _ => Nil + } + case _ => Nil + } - rawLogger.error("Failed to compile " + failureMessage) - case _: ClientInfo.BspClientInfo => () // Don't report if bsp client + val newState: State = { + val stateWithResults = state.copy(results = state.results.addFinalResults(results)) + if (failures.isEmpty) { + stateWithResults.copy(status = ExitStatus.Ok) + } else { + results.foreach { + case FinalNormalCompileResult.HasException(project, err) => + val errMsg = err.fold(identity, Logger.prettyPrintException) + rawLogger.error(s"Unexpected error when compiling ${project.name}: $errMsg") + case _ => + () // Do nothing when the final compilation result is not an actual error + } + + client match { + case _: ClientInfo.CliClientInfo => + // Reverse list of failed projects to get ~correct order of failure + val projectsFailedToCompile = failures.map(p => s"'${p.name}'").reverse + val failureMessage = + if (failures.size <= 2) projectsFailedToCompile.mkString(",") + else { + s"${projectsFailedToCompile.take(2).mkString(", ")} and ${projectsFailedToCompile.size - 2} more projects" + } + + rawLogger.error("Failed to compile " + failureMessage) + case _: ClientInfo.BspClientInfo => () // Don't report if bsp client + } + + stateWithResults.copy(status = ExitStatus.CompilationError) } - - stateWithResults.copy(status = ExitStatus.CompilationError) } - } - // Schedule to run clean-up tasks in the background - runIOTasksInParallel(cleanUpTasksToRunInBackground) + // Schedule to run clean-up tasks in the background + runIOTasksInParallel(cleanUpTasksToRunInBackground) - val runningTasksRequiredForCorrectness = Task.sequence { - results.flatMap { - case FinalNormalCompileResult(_, result) => - val tasksAtEndOfBuildCompilation = - Task.fromFuture(result.runningBackgroundTasks) - List(tasksAtEndOfBuildCompilation) - case _ => Nil + val runningTasksRequiredForCorrectness = Task.sequence { + results.flatMap { + case FinalNormalCompileResult(_, result) => + val tasksAtEndOfBuildCompilation = + Task.fromFuture(result.runningBackgroundTasks) + List(tasksAtEndOfBuildCompilation) + case _ => Nil + } } - } - // Block on all background task that are running and are required for correctness - runningTasksRequiredForCorrectness - .executeOn(ExecutionContext.ioScheduler) - .map(_ => newState) - .doOnFinish(_ => Task(rootTracer.terminate())) + // Block on all background task that are running and are required for correctness + runningTasksRequiredForCorrectness + .executeOn(ExecutionContext.ioScheduler) + .map(_ => newState) + .doOnFinish(_ => Task(rootTracer.terminate())) + } } } } diff --git a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGatekeeper.scala b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGatekeeper.scala index 9dc34d4ad1..ac682d6ba6 100644 --- a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGatekeeper.scala +++ b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGatekeeper.scala @@ -1,24 +1,19 @@ package bloop.engine.tasks.compilation -import java.util.concurrent.ConcurrentHashMap - -import bloop.Compiler -import bloop.UniqueCompileInputs -import bloop.data.ClientInfo -import bloop.data.Project +import bloop.{Compiler, UniqueCompileInputs} +import bloop.data.{ClientInfo, Project} import bloop.engine.Dag import bloop.engine.caches.LastSuccessfulResult import bloop.io.AbsolutePath -import bloop.logging.DebugFilter -import bloop.logging.Logger -import bloop.logging.LoggerAction +import bloop.logging.{DebugFilter, Logger, LoggerAction} import bloop.reporter.ReporterAction import bloop.task.Task - -import monix.execution.atomic.AtomicBoolean -import monix.execution.atomic.AtomicInt +import bloop.tracing.BraveTracer +import monix.execution.atomic.{AtomicBoolean, AtomicInt} import monix.reactive.Observable +import java.util.concurrent.ConcurrentHashMap + object CompileGatekeeper { private implicit val filter: DebugFilter = DebugFilter.Compilation import bloop.engine.tasks.compilation.CompileDefinitions._ @@ -43,59 +38,87 @@ object CompileGatekeeper { inputs: BundleInputs, bundle: SuccessfulCompileBundle, client: ClientInfo, - compile: SuccessfulCompileBundle => CompileTraversal - ): (RunningCompilation, CanBeDeduplicated) = { - import bundle.logger - var deduplicate = true - - val running = runningCompilations.compute( - bundle.uniqueInputs, - (_: UniqueCompileInputs, running: RunningCompilation) => { - if (running == null) { - logger.debug(s"no running compilation found starting new one:${bundle.uniqueInputs}") - deduplicate = false - scheduleCompilation(inputs, bundle, client, compile) - } else { - val usedClassesDir = running.usedLastSuccessful.classesDir - val usedClassesDirCounter = running.usedLastSuccessful.counterForClassesDir - - usedClassesDirCounter.getAndTransform { count => - if (count == 0) { - logger.debug( - s"Abort deduplication, dir is scheduled to be deleted in background:${bundle.uniqueInputs}" - ) - // Abort deduplication, dir is scheduled to be deleted in background + compile: SuccessfulCompileBundle => CompileTraversal, + tracer: BraveTracer + ): (RunningCompilation, CanBeDeduplicated) = + tracer.trace("Finding compilation atomically.") { tracer => + import bundle.logger + var deduplicate = true + + val running = runningCompilations.compute( + bundle.uniqueInputs, + (_: UniqueCompileInputs, running: RunningCompilation) => { + if (running == null) { + tracer.trace( + "no running compilation found starting new one", + ("uniqueInputs", bundle.uniqueInputs.toString) + ) { tracer => + logger.debug(s"no running compilation found starting new one:${bundle.uniqueInputs}") deduplicate = false - // Remove from map of used classes dirs in case it hasn't already been - currentlyUsedClassesDirs.remove(usedClassesDir, usedClassesDirCounter) - // Return previous count, this counter will soon be deallocated - count - } else { - logger.debug( - s"Increase count to prevent other compiles to schedule its deletion:${bundle.uniqueInputs}" - ) - // Increase count to prevent other compiles to schedule its deletion - count + 1 + scheduleCompilation(inputs, bundle, client, compile, tracer) } - } + } else { + tracer.trace( + "Found matching compilation", + ("uniqueInputs", bundle.uniqueInputs.toString) + ) { tracer => + val usedClassesDir = running.usedLastSuccessful.classesDir + val usedClassesDirCounter = running.usedLastSuccessful.counterForClassesDir + + usedClassesDirCounter.getAndTransform { count => + if (count == 0) { + tracer.trace( + "Aborting deduplication", + ("uniqueInputs", bundle.uniqueInputs.toString) + ) { tracer => + logger.debug( + s"Abort deduplication, dir is scheduled to be deleted in background:${bundle.uniqueInputs}" + ) + // Abort deduplication, dir is scheduled to be deleted in background + deduplicate = false + // Remove from map of used classes dirs in case it hasn't already been + currentlyUsedClassesDirs.remove(usedClassesDir, usedClassesDirCounter) + // Return previous count, this counter will soon be deallocated + count + } + } else { + tracer.trace( + "Increasing compilation counter", + ("uniqueInputs", bundle.uniqueInputs.toString) + ) { tracer => + logger.debug( + s"Increase count to prevent other compiles to schedule its deletion:${bundle.uniqueInputs}" + ) + // Increase count to prevent other compiles to schedule its deletion + count + 1 + } + } + } - if (deduplicate) running - else scheduleCompilation(inputs, bundle, client, compile) + if (deduplicate) running + else scheduleCompilation(inputs, bundle, client, compile, tracer) + } + } } - } - ) + ) - (running, deduplicate) - } + (running, deduplicate) + } def disconnectDeduplicationFromRunning( inputs: UniqueCompileInputs, runningCompilation: RunningCompilation, - logger: Logger + logger: Logger, + tracer: BraveTracer ): Unit = { - logger.debug(s"Disconnected deduplication from running compilation:${inputs}") - runningCompilation.isUnsubscribed.compareAndSet(false, true) - runningCompilations.remove(inputs, runningCompilation); () + tracer.trace( + "disconnectDeduplicationFromRunning", + ("uniqueInputs", inputs.toString) + ) { _ => + logger.debug(s"Disconnected deduplication from running compilation:${inputs}") + runningCompilation.isUnsubscribed.compareAndSet(false, true) + runningCompilations.remove(inputs, runningCompilation); () + } } /** @@ -109,131 +132,150 @@ object CompileGatekeeper { inputs: BundleInputs, bundle: SuccessfulCompileBundle, client: ClientInfo, - compile: SuccessfulCompileBundle => CompileTraversal - ): RunningCompilation = { - import inputs.project - import bundle.logger - import logger.debug - - var counterForUsedClassesDir: AtomicInt = null - - def initializeLastSuccessful(previousOrNull: LastSuccessfulResult): LastSuccessfulResult = { - val result = Option(previousOrNull).getOrElse(bundle.lastSuccessful) - if (!result.classesDir.exists) { - debug(s"Ignoring analysis for ${project.name}, directory ${result.classesDir} is missing") - LastSuccessfulResult.empty(inputs.project) - } else if (bundle.latestResult == Compiler.Result.Empty) { - debug(s"Ignoring existing analysis for ${project.name}, last result was empty") - LastSuccessfulResult - .empty(inputs.project) - // Replace classes dir, counter and populating with values from previous for correctness - .copy( - classesDir = result.classesDir, - counterForClassesDir = result.counterForClassesDir, - populatingProducts = result.populatingProducts - ) - } else { - debug(s"Using successful result for ${project.name} associated with ${result.classesDir}") - result - } - } - - def getMostRecentSuccessfulResultAtomically = { - lastSuccessfulResults.compute( - project.uniqueId, - (_: String, previousResultOrNull: LastSuccessfulResult) => { - logger.debug( - s"Return previous result or the initial last successful coming from the bundle:${project.uniqueId}" - ) - // Return previous result or the initial last successful coming from the bundle - val previousResult = initializeLastSuccessful(previousResultOrNull) + compile: SuccessfulCompileBundle => CompileTraversal, + tracer: BraveTracer + ): RunningCompilation = + tracer.trace("schedule compilation") { _ => + import bundle.logger + import inputs.project + + var counterForUsedClassesDir: AtomicInt = null + + def initializeLastSuccessful(previousOrNull: LastSuccessfulResult): LastSuccessfulResult = + tracer.trace(s"initialize last successful") { _ => + val result = Option(previousOrNull).getOrElse(bundle.lastSuccessful) + if (!result.classesDir.exists) { + logger.debug( + s"Ignoring analysis for ${project.name}, directory ${result.classesDir} is missing" + ) + LastSuccessfulResult.empty(inputs.project) + } else if (bundle.latestResult == Compiler.Result.Empty) { + logger.debug(s"Ignoring existing analysis for ${project.name}, last result was empty") + LastSuccessfulResult + .empty(inputs.project) + // Replace classes dir, counter and populating with values from previous for correctness + .copy( + classesDir = result.classesDir, + counterForClassesDir = result.counterForClassesDir, + populatingProducts = result.populatingProducts + ) + } else { + logger.debug( + s"Using successful result for ${project.name} associated with ${result.classesDir}" + ) + result + } + } - currentlyUsedClassesDirs.compute( - previousResult.classesDir, - (_: AbsolutePath, counter: AtomicInt) => { + def getMostRecentSuccessfulResultAtomically = + tracer.trace("get most recent successful result atomically") { _ => + lastSuccessfulResults.compute( + project.uniqueId, + (_: String, previousResultOrNull: LastSuccessfulResult) => { logger.debug( - s"Set counter for used classes dir when init or incrementing:${previousResult.classesDir}" + s"Return previous result or the initial last successful coming from the bundle:${project.uniqueId}" ) - // Set counter for used classes dir when init or incrementing - if (counter == null) { - logger.debug(s"Create new counter:${previousResult.classesDir}") - val initialCounter = AtomicInt(1) - counterForUsedClassesDir = initialCounter - initialCounter - } else { - counterForUsedClassesDir = counter - val newCount = counter.incrementAndGet(1) - logger.debug(s"Increasing counter for ${previousResult.classesDir} to $newCount") - counter - } + // Return previous result or the initial last successful coming from the bundle + val previousResult = initializeLastSuccessful(previousResultOrNull) + + currentlyUsedClassesDirs.compute( + previousResult.classesDir, + (_: AbsolutePath, counter: AtomicInt) => { + logger.debug( + s"Set counter for used classes dir when init or incrementing:${previousResult.classesDir}" + ) + // Set counter for used classes dir when init or incrementing + if (counter == null) { + logger.debug(s"Create new counter:${previousResult.classesDir}") + val initialCounter = AtomicInt(1) + counterForUsedClassesDir = initialCounter + initialCounter + } else { + counterForUsedClassesDir = counter + val newCount = counter.incrementAndGet(1) + logger.debug( + s"Increasing counter for ${previousResult.classesDir} to $newCount" + ) + counter + } + } + ) + + previousResult.copy(counterForClassesDir = counterForUsedClassesDir) } ) - - previousResult.copy(counterForClassesDir = counterForUsedClassesDir) } - ) - } - logger.debug(s"Scheduling compilation for ${project.name}...") - - // Replace client-specific last successful with the most recent result - val mostRecentSuccessful = getMostRecentSuccessfulResultAtomically - - val isUnsubscribed = AtomicBoolean(false) - val newBundle = bundle.copy(lastSuccessful = mostRecentSuccessful) - val compileAndUnsubscribe = { - compile(newBundle) - .doOnFinish(_ => Task(logger.observer.onComplete())) - .map { result => - // Unregister deduplication atomically and register last successful if any - processResultAtomically( - result, - project, - bundle.uniqueInputs, - isUnsubscribed, - logger - ) - } - .memoize // Without memoization, there is no deduplication - } + logger.debug(s"Scheduling compilation for ${project.name}...") + + // Replace client-specific last successful with the most recent result + val mostRecentSuccessful = getMostRecentSuccessfulResultAtomically + + val isUnsubscribed = AtomicBoolean(false) + val newBundle = bundle.copy(lastSuccessful = mostRecentSuccessful) + val compileAndUnsubscribe = tracer.trace("compile and unsubscribe") { _ => + compile(newBundle) + .doOnFinish(_ => Task(logger.observer.onComplete())) + .map { result => + // Unregister deduplication atomically and register last successful if any + tracer.trace("process result atomically") { _ => + processResultAtomically( + result, + project, + bundle.uniqueInputs, + isUnsubscribed, + logger, + tracer + ) + } // Without memoization, there is no deduplication + } + .memoize + } - RunningCompilation( - compileAndUnsubscribe, - mostRecentSuccessful, - isUnsubscribed, - bundle.mirror, - client - ) - } + RunningCompilation( + compileAndUnsubscribe, + mostRecentSuccessful, + isUnsubscribed, + bundle.mirror, + client + ) + } private def processResultAtomically( resultDag: Dag[PartialCompileResult], project: Project, oinputs: UniqueCompileInputs, isAlreadyUnsubscribed: AtomicBoolean, - logger: Logger + logger: Logger, + tracer: BraveTracer ): Dag[PartialCompileResult] = { - def cleanUpAfterCompilationError[T](result: T): T = { - if (!isAlreadyUnsubscribed.get) { - // Remove running compilation if host compilation hasn't unsubscribed (maybe it's blocked) - logger.debug( - s"Remove running compilation if host compilation hasn't unsubscribed (maybe it's blocked):${oinputs}" - ) - runningCompilations.remove(oinputs) - } + def cleanUpAfterCompilationError[T](result: T): T = + tracer.trace("cleaning after compilation error") { _ => + if (!isAlreadyUnsubscribed.get) { + // Remove running compilation if host compilation hasn't unsubscribed (maybe it's blocked) + logger.debug( + s"Remove running compilation if host compilation hasn't unsubscribed (maybe it's blocked):${oinputs}" + ) + runningCompilations.remove(oinputs) + } - result - } + result + } // Unregister deduplication atomically and register last successful if any PartialCompileResult.mapEveryResult(resultDag) { case s: PartialSuccess => val processedResult = s.result.map { (result: ResultBundle) => result.successful match { - case None => cleanUpAfterCompilationError(result) + case None => + tracer.trace("cleaning after compilation error") { _ => + cleanUpAfterCompilationError(result) + } case Some(res) => - unregisterDeduplicationAndRegisterSuccessful(project, oinputs, res, logger) + tracer.trace("unregister deduplication and register successful") { _ => + unregisterDeduplicationAndRegisterSuccessful(project, oinputs, res, logger) + } } result } @@ -245,7 +287,10 @@ object CompileGatekeeper { */ s.copy(result = processedResult.memoize) - case result => cleanUpAfterCompilationError(result) + case result => + tracer.trace("cleaning after compilation error") { _ => + cleanUpAfterCompilationError(result) + } } } diff --git a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGatekeeperNew.scala b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGatekeeperNew.scala new file mode 100644 index 0000000000..95e134fe34 --- /dev/null +++ b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGatekeeperNew.scala @@ -0,0 +1,288 @@ +package bloop.engine.tasks.compilation + +import bloop.Compiler +import bloop.UniqueCompileInputs +import bloop.data.ClientInfo +import bloop.data.Project +import bloop.engine.Dag +import bloop.engine.caches.LastSuccessfulResult +import bloop.io.AbsolutePath +import bloop.logging.DebugFilter +import bloop.logging.Logger +import bloop.logging.LoggerAction +import bloop.reporter.ReporterAction +import bloop.task.Task +import monix.eval.{Task => MonixTask} +import cats.effect.concurrent.{Deferred, Ref} +import monix.execution.atomic.AtomicBoolean +import monix.execution.atomic.AtomicInt +import monix.reactive.Observable + +object CompileGatekeeperNew { + private implicit val filter: DebugFilter = DebugFilter.Compilation + import bloop.engine.tasks.compilation.CompileDefinitions._ + + private[bloop] final case class RunningCompilation( + traversal: CompileTraversal, + usedLastSuccessful: LastSuccessfulResult, + isUnsubscribed: AtomicBoolean, + mirror: Observable[Either[ReporterAction, LoggerAction]], + client: ClientInfo + ) + + /* -------------------------------------------------------------------------------------------- */ + private val runningCompilations + : Ref[MonixTask, Map[UniqueCompileInputs, Deferred[MonixTask, RunningCompilation]]] = + Ref.unsafe(Map.empty) + private val currentlyUsedClassesDirs: Ref[MonixTask, Map[AbsolutePath, AtomicInt]] = + Ref.unsafe(Map.empty) + private val lastSuccessfulResults: Ref[MonixTask, Map[ProjectId, LastSuccessfulResult]] = + Ref.unsafe(Map.empty) + + /* -------------------------------------------------------------------------------------------- */ + + def findRunningCompilationAtomically( + inputs: BundleInputs, + bundle: SuccessfulCompileBundle, + client: ClientInfo, + compile: SuccessfulCompileBundle => CompileTraversal + ): Task[(RunningCompilation, CanBeDeduplicated)] = Task.liftMonixTaskUncancellable { + Deferred[MonixTask, RunningCompilation].flatMap { deferred => + runningCompilations.modify { state => + state.get(bundle.uniqueInputs) match { + case Some(existingDeferred) => + val output = + existingDeferred.get + .flatMap { running => + val usedClassesDir = running.usedLastSuccessful.classesDir + val usedClassesDirCounter = running.usedLastSuccessful.counterForClassesDir + val deduplicate = usedClassesDirCounter.transformAndExtract { + case count if count == 0 => (false -> count) + case count => true -> (count + 1) + } + if (deduplicate) MonixTask.now((running, deduplicate)) + else { + val classesDirs = + currentlyUsedClassesDirs + .update { classesDirs => + if ( + classesDirs + .get(usedClassesDir) + .contains(usedClassesDirCounter) + ) + classesDirs - usedClassesDir + else + classesDirs + } + scheduleCompilation(inputs, bundle, client, compile) + .flatMap(compilation => + deferred + .complete(compilation) + .flatMap(_ => classesDirs.map(_ => (compilation, false))) + ) + } + } + state -> output + case None => + val newState: Map[UniqueCompileInputs, Deferred[MonixTask, RunningCompilation]] = + state + (bundle.uniqueInputs -> deferred) + newState -> scheduleCompilation(inputs, bundle, client, compile).flatMap(compilation => + deferred.complete(compilation).map(_ => compilation -> false) + ) + } + }.flatten + } + } + + def disconnectDeduplicationFromRunning( + inputs: UniqueCompileInputs, + runningCompilation: RunningCompilation + ): MonixTask[Unit] = { + runningCompilation.isUnsubscribed.compareAndSet(false, true) + runningCompilations.modify { state => + val updated = + if ( + state.contains(inputs) + ) // .contains(runningCompilationDeferred)) // TODO: no way to verfiy if it is the same + state - inputs + else state + (updated, ()) + } + } + + /** + * Schedules a unique compilation for the given inputs. + * + * This compilation can be deduplicated by other clients that have the same + * inputs. The call-site ensures that only one compilation can exist for the + * same inputs for a period of time. + */ + def scheduleCompilation( + inputs: BundleInputs, + bundle: SuccessfulCompileBundle, + client: ClientInfo, + compile: SuccessfulCompileBundle => CompileTraversal + ): MonixTask[RunningCompilation] = { + import inputs.project + import bundle.logger + import logger.debug + + def initializeLastSuccessful( + maybePreviousResult: Option[LastSuccessfulResult] + ): LastSuccessfulResult = { + val result = maybePreviousResult.getOrElse(bundle.lastSuccessful) + if (!result.classesDir.exists) { + debug(s"Ignoring analysis for ${project.name}, directory ${result.classesDir} is missing") + LastSuccessfulResult.empty(inputs.project) + } else if (bundle.latestResult == Compiler.Result.Empty) { + debug(s"Ignoring existing analysis for ${project.name}, last result was empty") + LastSuccessfulResult + .empty(inputs.project) + // Replace classes dir, counter and populating with values from previous for correctness + .copy( + classesDir = result.classesDir, + counterForClassesDir = result.counterForClassesDir, + populatingProducts = result.populatingProducts + ) + } else { + debug(s"Using successful result for ${project.name} associated with ${result.classesDir}") + result + } + } + + def getMostRecentSuccessfulResultAtomically: MonixTask[LastSuccessfulResult] = { + lastSuccessfulResults.modify { state => + val previousResult = + initializeLastSuccessful(state.get(project.uniqueId)) + state -> currentlyUsedClassesDirs + .modify { counters => + counters.get(previousResult.classesDir) match { + case None => + val initialCounter = AtomicInt(1) + (counters + (previousResult.classesDir -> initialCounter), initialCounter) + case Some(counter) => + val newCount = counter.incrementAndGet(1) + logger.debug(s"Increasing counter for ${previousResult.classesDir} to $newCount") + counters -> counter + } + } + .map(_ => previousResult) + }.flatten + } + + logger.debug(s"Scheduling compilation for ${project.name}...") + + getMostRecentSuccessfulResultAtomically + .map { mostRecentSuccessful => + val isUnsubscribed = AtomicBoolean(false) + val newBundle = bundle.copy(lastSuccessful = mostRecentSuccessful) + val compileAndUnsubscribe = compile(newBundle) + .doOnFinish(_ => Task(logger.observer.onComplete())) + .flatMap { result => + // Unregister deduplication atomically and register last successful if any + processResultAtomically( + result, + project, + bundle.uniqueInputs, + isUnsubscribed, + logger + ) + } + .memoize + + RunningCompilation( + compileAndUnsubscribe, + mostRecentSuccessful, + isUnsubscribed, + bundle.mirror, + client + ) // Without memoization, there is no deduplication + } + + } + + private def processResultAtomically( + resultDag: Dag[PartialCompileResult], + project: Project, + oinputs: UniqueCompileInputs, + isAlreadyUnsubscribed: AtomicBoolean, + logger: Logger + ): Task[Dag[PartialCompileResult]] = { + + def cleanUpAfterCompilationError[T](result: T): Task[T] = { + Task { + if (!isAlreadyUnsubscribed.get) { + // Remove running compilation if host compilation hasn't unsubscribed (maybe it's blocked) + Task.liftMonixTaskUncancellable { + runningCompilations.update(_ - oinputs) + } + } else + Task.unit + }.flatten.map(_ => result) + } + + // Unregister deduplication atomically and register last successful if any + PartialCompileResult.mapEveryResultTask(resultDag) { + case s: PartialSuccess => + val processedResult = s.result.flatMap { (result: ResultBundle) => + result.successful + .fold(cleanUpAfterCompilationError(result)) { res => + unregisterDeduplicationAndRegisterSuccessful( + project, + oinputs, + res, + logger + ) + .map(_ => result) + } + } + + /** + * This result task must only be run once and thus needs to be + * memoized for correctness reasons. The result task can be called + * several times by the compilation engine driving the execution. + */ + Task(s.copy(result = processedResult.memoize)) + + case result => cleanUpAfterCompilationError(result) + } + } + + /** + * Removes the deduplication and registers the last successful compilation + * atomically. When registering the last successful compilation, we make sure + * that the old last successful result is deleted if its count is 0, which + * means it's not being used by anyone. + */ + private def unregisterDeduplicationAndRegisterSuccessful( + project: Project, + oracleInputs: UniqueCompileInputs, + successful: LastSuccessfulResult, + logger: Logger + ): Task[Unit] = Task.liftMonixTaskUncancellable { + runningCompilations + .modify { state => + val newSuccessfulResults = (project.uniqueId, successful) + if (state.contains(oracleInputs)) { + val newState = state - oracleInputs + (newState, lastSuccessfulResults.update { _ + newSuccessfulResults }) + } else { + (state, MonixTask.unit) + } + } + .flatten + .map { _ => + logger.debug( + s"Recording new last successful request for ${project.name} associated with ${successful.classesDir}" + ) + () + } + } + + // Expose clearing mechanism so that it can be invoked in the tests and community build runner +// private[bloop] def clearSuccessfulResults(): Unit = { +// lastSuccessfulResults.synchronized { +// lastSuccessfulResults.clear() +// } +// } +} diff --git a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGraph.scala b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGraph.scala index 7b69d2247f..668dc6ffe7 100644 --- a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGraph.scala +++ b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGraph.scala @@ -4,13 +4,10 @@ package bloop.engine.tasks.compilation import java.io.File import java.io.PrintWriter import java.io.StringWriter - import scala.util.Failure import scala.util.Success - import ch.epfl.scala.bsp.StatusCode import ch.epfl.scala.bsp.{StatusCode => BspStatusCode} - import bloop.CompileBackgroundTasks import bloop.CompileExceptions.BlockURI import bloop.CompileExceptions.FailedOrCancelledPromise @@ -27,10 +24,10 @@ import bloop.logging.DebugFilter import bloop.logging.LoggerAction import bloop.reporter.ReporterAction import bloop.task.Task +import bloop.tracing.BraveTracer import bloop.util.BestEffortUtils.BestEffortProducts import bloop.util.JavaCompat.EnrichOptional import bloop.util.SystemProperties - import xsbti.compile.PreviousResult object CompileGraph { @@ -94,10 +91,11 @@ object CompileGraph { def setupAndDeduplicate( client: ClientInfo, inputs: BundleInputs, - setup: BundleInputs => Task[CompileBundle] + setup: (BundleInputs, BraveTracer) => Task[CompileBundle], + tracer: BraveTracer )( compile: SuccessfulCompileBundle => CompileTraversal - ): CompileTraversal = { + ): CompileTraversal = tracer.trace("setupAndDeduplicate") { tracer => def partialFailure( errorMsg: String, err: Option[Throwable] @@ -110,8 +108,8 @@ object CompileGraph { } implicit val filter = DebugFilter.Compilation - def withBundle(f: SuccessfulCompileBundle => CompileTraversal): CompileTraversal = { - setup(inputs).materialize.flatMap { + def withBundle(f: SuccessfulCompileBundle => CompileTraversal): CompileTraversal = tracer.trace("withBundle") { tracer => + setup(inputs, tracer).materialize.flatMap { case Success(bundle: SuccessfulCompileBundle) => f(bundle).materialize.flatMap { case Success(result) => Task.now(result) @@ -134,221 +132,232 @@ object CompileGraph { withBundle { bundle0 => val logger = bundle0.logger val (runningCompilation, deduplicate) = - CompileGatekeeper.findRunningCompilationAtomically(inputs, bundle0, client, compile) + CompileGatekeeper.findRunningCompilationAtomically(inputs, bundle0, client, compile, tracer) val bundle = bundle0.copy(lastSuccessful = runningCompilation.usedLastSuccessful) if (!deduplicate) { - runningCompilation.traversal - } else { - val rawLogger = logger.underlying - rawLogger.info( - s"Deduplicating compilation of ${bundle.project.name} from ${runningCompilation.client}" - ) - val reporter = bundle.reporter.underlying - // Don't use `bundle.lastSuccessful`, it's not the final input to `compile` - val analysis = runningCompilation.usedLastSuccessful.previous.analysis().toOption - val previousSuccessfulProblems = - Compiler.previousProblemsFromSuccessfulCompilation(analysis) - val wasPreviousSuccessful = bundle.latestResult match { - case Compiler.Result.Ok(_) => true - case _ => false + tracer.trace("traversing compilation") { tracer => + runningCompilation.traversal } - val previousProblems = - Compiler.previousProblemsFromResult(bundle.latestResult, previousSuccessfulProblems) - - val clientClassesObserver = client.getClassesObserverFor(bundle.project) - - // Replay events asynchronously to waiting for the compilation result - import scala.concurrent.duration.FiniteDuration - import monix.execution.exceptions.UpstreamTimeoutException - val disconnectionTime = SystemProperties.getCompileDisconnectionTime(rawLogger) - val replayEventsTask = runningCompilation.mirror - .timeoutOnSlowUpstream(disconnectionTime) - .foreachL { - case Left(action) => - action match { - case ReporterAction.EnableFatalWarnings => - reporter.enableFatalWarnings() - case ReporterAction.ReportStartCompilation => - reporter.reportStartCompilation(previousProblems, wasPreviousSuccessful) - case a: ReporterAction.ReportStartIncrementalCycle => - reporter.reportStartIncrementalCycle(a.sources, a.outputDirs) - case a: ReporterAction.ReportProblem => reporter.log(a.problem) - case ReporterAction.PublishDiagnosticsSummary => - reporter.printSummary() - case a: ReporterAction.ReportNextPhase => - reporter.reportNextPhase(a.phase, a.sourceFile) - case a: ReporterAction.ReportCompilationProgress => - reporter.reportCompilationProgress(a.progress, a.total) - case a: ReporterAction.ReportEndIncrementalCycle => - reporter.reportEndIncrementalCycle(a.durationMs, a.result) - case ReporterAction.ReportCancelledCompilation => - reporter.reportCancelledCompilation() - case a: ReporterAction.ProcessEndCompilation => - a.code match { - case BspStatusCode.Cancelled | BspStatusCode.Error => - reporter.processEndCompilation(previousProblems, a.code, None, None) - reporter.reportEndCompilation() - case _ => - /* - * Only process the end, don't report it. It's only safe to - * report when all the client tasks have been run and the - * analysis/classes dirs are fully populated so that clients - * can use `taskFinish` notifications as a signal to process them. - */ - reporter.processEndCompilation( - previousProblems, - a.code, - Some(clientClassesObserver.classesDir), - Some(bundle.out.analysisOut) - ) - } - } - case Right(action) => - action match { - case LoggerAction.LogErrorMessage(msg) => rawLogger.error(msg) - case LoggerAction.LogWarnMessage(msg) => rawLogger.warn(msg) - case LoggerAction.LogInfoMessage(msg) => rawLogger.info(msg) - case LoggerAction.LogDebugMessage(msg) => - rawLogger.debug(msg) - case LoggerAction.LogTraceMessage(msg) => - rawLogger.debug(msg) - } - } - .materialize - .map { - case Success(_) => DeduplicationResult.Ok - case Failure(_: UpstreamTimeoutException) => - DeduplicationResult.DisconnectFromDeduplication - case Failure(t) => DeduplicationResult.DeduplicationError(t) + } else { + tracer.trace("deduplication required") { tracer => + val rawLogger = logger.underlying + rawLogger.info( + s"Deduplicating compilation of ${bundle.project.name} from ${runningCompilation.client}" + ) + val reporter = bundle.reporter.underlying + // Don't use `bundle.lastSuccessful`, it's not the final input to `compile` + val analysis = runningCompilation.usedLastSuccessful.previous.analysis().toOption + val previousSuccessfulProblems = + Compiler.previousProblemsFromSuccessfulCompilation(analysis) + val wasPreviousSuccessful = bundle.latestResult match { + case Compiler.Result.Ok(_) => true + case _ => false } + val previousProblems = + Compiler.previousProblemsFromResult(bundle.latestResult, previousSuccessfulProblems) + + val clientClassesObserver = client.getClassesObserverFor(bundle.project) + + // Replay events asynchronously to waiting for the compilation result + import scala.concurrent.duration.FiniteDuration + import monix.execution.exceptions.UpstreamTimeoutException + val disconnectionTime = SystemProperties.getCompileDisconnectionTime(rawLogger) + val replayEventsTask = runningCompilation.mirror + .timeoutOnSlowUpstream(disconnectionTime) + .foreachL { + case Left(action) => + action match { + case ReporterAction.EnableFatalWarnings => + reporter.enableFatalWarnings() + case ReporterAction.ReportStartCompilation => + reporter.reportStartCompilation(previousProblems, wasPreviousSuccessful) + case a: ReporterAction.ReportStartIncrementalCycle => + reporter.reportStartIncrementalCycle(a.sources, a.outputDirs) + case a: ReporterAction.ReportProblem => reporter.log(a.problem) + case ReporterAction.PublishDiagnosticsSummary => + reporter.printSummary() + case a: ReporterAction.ReportNextPhase => + reporter.reportNextPhase(a.phase, a.sourceFile) + case a: ReporterAction.ReportCompilationProgress => + reporter.reportCompilationProgress(a.progress, a.total) + case a: ReporterAction.ReportEndIncrementalCycle => + reporter.reportEndIncrementalCycle(a.durationMs, a.result) + case ReporterAction.ReportCancelledCompilation => + reporter.reportCancelledCompilation() + case a: ReporterAction.ProcessEndCompilation => + a.code match { + case BspStatusCode.Cancelled | BspStatusCode.Error => + reporter.processEndCompilation(previousProblems, a.code, None, None) + reporter.reportEndCompilation() + case _ => + /* + * Only process the end, don't report it. It's only safe to + * report when all the client tasks have been run and the + * analysis/classes dirs are fully populated so that clients + * can use `taskFinish` notifications as a signal to process them. + */ + reporter.processEndCompilation( + previousProblems, + a.code, + Some(clientClassesObserver.classesDir), + Some(bundle.out.analysisOut) + ) + } + } + case Right(action) => + action match { + case LoggerAction.LogErrorMessage(msg) => rawLogger.error(msg) + case LoggerAction.LogWarnMessage(msg) => rawLogger.warn(msg) + case LoggerAction.LogInfoMessage(msg) => rawLogger.info(msg) + case LoggerAction.LogDebugMessage(msg) => + rawLogger.debug(msg) + case LoggerAction.LogTraceMessage(msg) => + rawLogger.debug(msg) + } + } + .materialize + .map { + case Success(_) => DeduplicationResult.Ok + case Failure(_: UpstreamTimeoutException) => + DeduplicationResult.DisconnectFromDeduplication + case Failure(t) => DeduplicationResult.DeduplicationError(t) + } - /* The task set up by another process whose memoized result we're going to - * reuse. To prevent blocking compilations, we execute this task (which will - * block until its completion is done) in the IO thread pool, which is - * unbounded. This makes sure that the blocking threads *never* block - * the computation pool, which could produce a hang in the build server. - */ - val runningCompilationTask = - runningCompilation.traversal.executeOn(ExecutionContext.ioScheduler) - - val deduplicateStreamSideEffectsHandle = - replayEventsTask.runToFuture(ExecutionContext.ioScheduler) - - /** - * Deduplicate and change the implementation of the task returning the - * deduplicate compiler result to trigger a syncing process to keep the - * client external classes directory up-to-date with the new classes - * directory. This copying process blocks until the background IO work - * of the deduplicated compilation result has been finished. Note that - * this mechanism allows pipelined compilations to perform this IO only - * when the full compilation of a module is finished. - */ - val obtainResultFromDeduplication = runningCompilationTask.map { results => - PartialCompileResult.mapEveryResult(results) { - case s @ PartialSuccess(bundle, compilerResult) => - val newCompilerResult = compilerResult.flatMap { results => - results.fromCompiler match { - case s: Compiler.Result.Success => - // Wait on new classes to be populated for correctness - val runningBackgroundTasks = s.backgroundTasks - .trigger(clientClassesObserver, reporter, bundle.tracer, logger) - .runAsync(ExecutionContext.ioScheduler) - Task.now(results.copy(runningBackgroundTasks = runningBackgroundTasks)) - case _: Compiler.Result.Cancelled => - // Make sure to cancel the deduplicating task if compilation is cancelled - deduplicateStreamSideEffectsHandle.cancel() - Task.now(results) - case _ => Task.now(results) + /* The task set up by another process whose memoized result we're going to + * reuse. To prevent blocking compilations, we execute this task (which will + * block until its completion is done) in the IO thread pool, which is + * unbounded. This makes sure that the blocking threads *never* block + * the computation pool, which could produce a hang in the build server. + */ + val runningCompilationTask = + runningCompilation.traversal.executeOn(ExecutionContext.ioScheduler) + + val deduplicateStreamSideEffectsHandle = + replayEventsTask.runToFuture(ExecutionContext.ioScheduler) + + /** + * Deduplicate and change the implementation of the task returning the + * deduplicate compiler result to trigger a syncing process to keep the + * client external classes directory up-to-date with the new classes + * directory. This copying process blocks until the background IO work + * of the deduplicated compilation result has been finished. Note that + * this mechanism allows pipelined compilations to perform this IO only + * when the full compilation of a module is finished. + */ + val obtainResultFromDeduplication = runningCompilationTask.map { results => + PartialCompileResult.mapEveryResult(results) { + case s @ PartialSuccess(bundle, compilerResult) => + val newCompilerResult = compilerResult.flatMap { results => + results.fromCompiler match { + case s: Compiler.Result.Success => + // Wait on new classes to be populated for correctness + val runningBackgroundTasks = s.backgroundTasks + .trigger(clientClassesObserver, reporter, tracer, logger) + .runAsync(ExecutionContext.ioScheduler) + Task.now(results.copy(runningBackgroundTasks = runningBackgroundTasks)) + case _: Compiler.Result.Cancelled => + // Make sure to cancel the deduplicating task if compilation is cancelled + deduplicateStreamSideEffectsHandle.cancel() + Task.now(results) + case _ => Task.now(results) + } } - } - s.copy(result = newCompilerResult) - case result => result + s.copy(result = newCompilerResult) + case result => result + } } - } - val compileAndDeduplicate = Task - .chooseFirstOf( - obtainResultFromDeduplication, - Task.fromFuture(deduplicateStreamSideEffectsHandle) - ) - .executeOn(ExecutionContext.ioScheduler) - - val finalCompileTask = compileAndDeduplicate.flatMap { - case Left((result, deduplicationFuture)) => - Task.fromFuture(deduplicationFuture).map(_ => result) - case Right((compilationFuture, deduplicationResult)) => - deduplicationResult match { - case DeduplicationResult.Ok => Task.fromFuture(compilationFuture) - case DeduplicationResult.DeduplicationError(t) => - rawLogger.trace(t) - val failedDeduplicationResult = Compiler.Result.GlobalError( - s"Unexpected error while deduplicating compilation for ${inputs.project.name}: ${t.getMessage}", - Some(t) - ) - - /* - * When an error happens while replaying all events of the - * deduplicated compilation, we keep track of the error, wait - * until the deduplicated compilation finishes and then we - * replace the result by a failed result that informs the - * client compilation was not successfully deduplicated. - */ - Task.fromFuture(compilationFuture).map { results => - PartialCompileResult.mapEveryResult(results) { (p: PartialCompileResult) => - p match { - case s: PartialSuccess => - val failedBundle = ResultBundle(failedDeduplicationResult, None, None) - s.copy(result = s.result.map(_ => failedBundle)) - case result => result + val compileAndDeduplicate = Task + .chooseFirstOf( + tracer.trace("obtainResultFromDeduplication - obtainResultFromDeduplication") { _ => + obtainResultFromDeduplication + }, + tracer.trace("obtainResultFromDeduplication - deduplicateStreamSideEffectsHandle") { _ => + Task.fromFuture(deduplicateStreamSideEffectsHandle) + } + ) + .executeOn(ExecutionContext.ioScheduler) + + val finalCompileTask = compileAndDeduplicate.flatMap { + case Left((result, deduplicationFuture)) => + Task.fromFuture(deduplicationFuture).map(_ => result) + case Right((compilationFuture, deduplicationResult)) => + deduplicationResult match { + case DeduplicationResult.Ok => Task.fromFuture(compilationFuture) + case DeduplicationResult.DeduplicationError(t) => + rawLogger.trace(t) + val failedDeduplicationResult = Compiler.Result.GlobalError( + s"Unexpected error while deduplicating compilation for ${inputs.project.name}: ${t.getMessage}", + Some(t) + ) + + /* + * When an error happens while replaying all events of the + * deduplicated compilation, we keep track of the error, wait + * until the deduplicated compilation finishes and then we + * replace the result by a failed result that informs the + * client compilation was not successfully deduplicated. + */ + Task.fromFuture(compilationFuture).map { results => + PartialCompileResult.mapEveryResult(results) { (p: PartialCompileResult) => + p match { + case s: PartialSuccess => + val failedBundle = ResultBundle(failedDeduplicationResult, None, None) + s.copy(result = s.result.map(_ => failedBundle)) + case result => result + } } } - } - case DeduplicationResult.DisconnectFromDeduplication => - /* - * Deduplication timed out after no compilation updates were - * recorded. In theory, this could happen because a rogue - * compilation process has stalled or is blocked. To ensure - * deduplicated clients always make progress, we now proceed - * with: - * + case DeduplicationResult.DisconnectFromDeduplication => + /* + * Deduplication timed out after no compilation updates were + * recorded. In theory, this could happen because a rogue + * compilation process has stalled or is blocked. To ensure + * deduplicated clients always make progress, we now proceed + * with: + * * 1. Cancelling the dead-looking compilation, hoping that the - * process will wake up at some point and stop running. - * 2. Shutting down the deduplication and triggering a new - * compilation. If there are several clients deduplicating this - * compilation, they will compete to start the compilation again - * with new compile inputs, as they could have already changed. - * 3. Reporting the end of compilation in case it hasn't been - * reported. Clients must handle two end compilation notifications - * gracefully. - * 4. Display the user that the deduplication was cancelled and a - * new compilation was scheduled. - */ - - CompileGatekeeper.disconnectDeduplicationFromRunning( - bundle.uniqueInputs, - runningCompilation, - logger - ) - - compilationFuture.cancel() - reporter.processEndCompilation(Nil, StatusCode.Cancelled, None, None) - reporter.reportEndCompilation() - - logger.displayWarningToUser( - s"""Disconnecting from deduplication of ongoing compilation for '${inputs.project.name}' - |No progress update for ${(disconnectionTime: FiniteDuration) - .toString()} caused bloop to cancel compilation and schedule a new compile. + * process will wake up at some point and stop running. + * 2. Shutting down the deduplication and triggering a new + * compilation. If there are several clients deduplicating this + * compilation, they will compete to start the compilation again + * with new compile inputs, as they could have already changed. + * 3. Reporting the end of compilation in case it hasn't been + * reported. Clients must handle two end compilation notifications + * gracefully. + * 4. Display the user that the deduplication was cancelled and a + * new compilation was scheduled. + */ + + CompileGatekeeper.disconnectDeduplicationFromRunning( + bundle.uniqueInputs, + runningCompilation, + logger, + tracer + ) + + compilationFuture.cancel() + reporter.processEndCompilation(Nil, StatusCode.Cancelled, None, None) + reporter.reportEndCompilation() + + logger.displayWarningToUser( + s"""Disconnecting from deduplication of ongoing compilation for '${inputs.project.name}' + |No progress update for ${(disconnectionTime: FiniteDuration) + .toString()} caused bloop to cancel compilation and schedule a new compile. """.stripMargin - ) + ) - setupAndDeduplicate(client, inputs, setup)(compile) - } - } + tracer.trace("setupAndDeduplicate - disconnected from deduplication") { tracer => + setupAndDeduplicate(client, inputs, setup, tracer)(compile) + } + } + } - bundle.tracer.traceTask(s"deduplicating ${bundle.project.name}") { _ => - finalCompileTask.executeOn(ExecutionContext.ioScheduler) + tracer.traceTask(s"deduplicating ${bundle.project.name}") { _ => + finalCompileTask.executeOn(ExecutionContext.ioScheduler) + } } } } @@ -369,120 +378,127 @@ object CompileGraph { client: ClientInfo, store: CompileClientStore, bestEffortAllowed: Boolean, - computeBundle: BundleInputs => Task[CompileBundle], - compile: (Inputs, Boolean, Boolean) => Task[ResultBundle] - ): CompileTraversal = { - val tasks = new mutable.HashMap[Dag[Project], CompileTraversal]() - def register(k: Dag[Project], v: CompileTraversal): CompileTraversal = { - val toCache = store.findPreviousTraversalOrAddNew(k, v).getOrElse(v) - tasks.put(k, toCache) - toCache - } + computeBundle: (BundleInputs, BraveTracer) => Task[CompileBundle], + compile: (Inputs, Boolean, Boolean, BraveTracer) => Task[ResultBundle], + tracer: BraveTracer + ): CompileTraversal = + tracer.trace("traversing ") { _ => + val tasks = new mutable.HashMap[Dag[Project], CompileTraversal]() + def register(k: Dag[Project], v: CompileTraversal): CompileTraversal = { + val toCache = store.findPreviousTraversalOrAddNew(k, v).getOrElse(v) + tasks.put(k, toCache) + toCache + } - /* - * [[PartialCompileResult]] is our way to represent errors at the build graph - * so that we can block the compilation of downstream projects. As we have to - * abide by this contract because it's used by the pipeline traversal too, we - * turn an actual compiler failure into a partial failure with a dummy - * `FailPromise` exception that makes the partial result be recognized as error. - */ - def toPartialFailure(bundle: SuccessfulCompileBundle, results: ResultBundle): PartialFailure = { - PartialFailure(bundle.project, FailedOrCancelledPromise, Task.now(results)) - } + /* + * [[PartialCompileResult]] is our way to represent errors at the build graph + * so that we can block the compilation of downstream projects. As we have to + * abide by this contract because it's used by the pipeline traversal too, we + * turn an actual compiler failure into a partial failure with a dummy + * `FailPromise` exception that makes the partial result be recognized as error. + */ + def toPartialFailure(bundle: SuccessfulCompileBundle, results: ResultBundle): PartialFailure = { + PartialFailure(bundle.project, FailedOrCancelledPromise, Task.now(results)) + } - def loop(dag: Dag[Project]): CompileTraversal = { - tasks.get(dag) match { - case Some(task) => task - case None => - val task: Task[Dag[PartialCompileResult]] = dag match { - case Leaf(project) => - val bundleInputs = BundleInputs(project, dag, Map.empty) - setupAndDeduplicate(client, bundleInputs, computeBundle) { bundle => - val isBestEffortDep = false - compile(Inputs(bundle, Map.empty), bestEffortAllowed && project.isBestEffort, isBestEffortDep).map { results => - results.fromCompiler match { - case Compiler.Result.Ok(_) => Leaf(partialSuccess(bundle, results)) - case _ => Leaf(toPartialFailure(bundle, results)) + def loop(dag: Dag[Project]): CompileTraversal = { + tasks.get(dag) match { + case Some(task) => task + case None => + val task: Task[Dag[PartialCompileResult]] = dag match { + case Leaf(project) => + val bundleInputs = BundleInputs(project, dag, Map.empty) + tracer.trace("setupAndDeduplicate - leaf project") { tracer => + setupAndDeduplicate(client, bundleInputs, computeBundle, tracer) { bundle => + val isBestEffortDep = false + compile(Inputs(bundle, Map.empty), bestEffortAllowed && project.isBestEffort, isBestEffortDep, tracer).map { + results => + results.fromCompiler match { + case Compiler.Result.Ok(_) => Leaf(partialSuccess(bundle, results)) + case _ => Leaf(toPartialFailure(bundle, results)) + } + } } } - } - - case Aggregate(dags) => - val downstream = dags.map(loop(_)) - Task.gatherUnordered(downstream).flatMap { dagResults => - Task.now(Parent(PartialEmpty, dagResults)) - } - case Parent(project, dependencies) => - val downstream = dependencies.map(loop(_)) - Task.gatherUnordered(downstream).flatMap { dagResults => - val depsSupportBestEffort = - dependencies.map(Dag.dfs(_, mode = Dag.PreOrder)).flatten.forall(_.isBestEffort) - val failed = dagResults.flatMap(dag => blockedBy(dag).toList) - - val allResults = Task.gatherUnordered { - val transitive = dagResults.flatMap(Dag.dfs(_, mode = Dag.PreOrder)).distinct - transitive.flatMap { - case PartialSuccess(bundle, result) => Some(result.map(r => bundle.project -> r)) - case PartialFailure(project, _, result) => Some(result.map(r => project -> r)) - case _ => None - } + case Aggregate(dags) => + val downstream = dags.map(loop(_)) + Task.gatherUnordered(downstream).flatMap { dagResults => + Task.now(Parent(PartialEmpty, dagResults)) } - allResults.flatMap { results => - val successfulBestEffort = !results.exists { - case (_, ResultBundle(f: Compiler.Result.Failed, _, _, _)) => f.bestEffortProducts.isEmpty - case _ => false + case Parent(project, dependencies) => + val downstream = dependencies.map(loop(_)) + Task.gatherUnordered(downstream).flatMap { dagResults => + val depsSupportBestEffort = + dependencies.map(Dag.dfs(_, mode = Dag.PreOrder)).flatten.forall(_.isBestEffort) + val failed = dagResults.flatMap(dag => blockedBy(dag).toList) + + val allResults = Task.gatherUnordered { + val transitive = dagResults.flatMap(Dag.dfs(_, mode = Dag.PreOrder)).distinct + transitive.flatMap { + case PartialSuccess(bundle, result) => Some(result.map(r => bundle.project -> r)) + case PartialFailure(project, _, result) => Some(result.map(r => project -> r)) + case _ => None + } } - val continue = bestEffortAllowed && depsSupportBestEffort && successfulBestEffort || failed.isEmpty - val dependsOnBestEffort = failed.nonEmpty && bestEffortAllowed && depsSupportBestEffort - - if (!continue) { - // Register the name of the projects we're blocked on (intransitively) - val blockedResult = Compiler.Result.Blocked(failed.map(_.name)) - val blocked = Task.now(ResultBundle(blockedResult, None, None)) - Task.now(Parent(PartialFailure(project, BlockURI, blocked), dagResults)) - } else { - val dependentProducts = new mutable.ListBuffer[(Project, BundleProducts)]() - val dependentResults = new mutable.ListBuffer[(File, PreviousResult)]() - results.foreach { - case (p, ResultBundle(s: Compiler.Result.Success, _, _, _)) => - val newProducts = s.products - dependentProducts.+=(p -> Right(newProducts)) - val newResult = newProducts.resultForDependentCompilationsInSameRun - dependentResults - .+=(newProducts.newClassesDir.toFile -> newResult) - .+=(newProducts.readOnlyClassesDir.toFile -> newResult) - case (p, ResultBundle(f: Compiler.Result.Failed, _, _, _)) => - f.bestEffortProducts.foreach { - case BestEffortProducts(products, _, _) => - dependentProducts += (p -> Right(products)) - } - case _ => () + + allResults.flatMap { results => + val successfulBestEffort = !results.exists { + case (_, ResultBundle(f: Compiler.Result.Failed, _, _, _)) => f.bestEffortProducts.isEmpty + case _ => false } + val continue = bestEffortAllowed && depsSupportBestEffort && successfulBestEffort || failed.isEmpty + val dependsOnBestEffort = failed.nonEmpty && bestEffortAllowed && depsSupportBestEffort + + if (!continue) { + // Register the name of the projects we're blocked on (intransitively) + val blockedResult = Compiler.Result.Blocked(failed.map(_.name)) + val blocked = Task.now(ResultBundle(blockedResult, None, None)) + Task.now(Parent(PartialFailure(project, BlockURI, blocked), dagResults)) + } else { + val dependentProducts = new mutable.ListBuffer[(Project, BundleProducts)]() + val dependentResults = new mutable.ListBuffer[(File, PreviousResult)]() + results.foreach { + case (p, ResultBundle(s: Compiler.Result.Success, _, _, _)) => + val newProducts = s.products + dependentProducts.+=(p -> Right(newProducts)) + val newResult = newProducts.resultForDependentCompilationsInSameRun + dependentResults + .+=(newProducts.newClassesDir.toFile -> newResult) + .+=(newProducts.readOnlyClassesDir.toFile -> newResult) + case (p, ResultBundle(f: Compiler.Result.Failed, _, _, _)) => + f.bestEffortProducts.foreach { + case BestEffortProducts(products, _, _) => + dependentProducts += (p -> Right(products)) + } + case _ => () + } - val resultsMap = dependentResults.toMap - val bundleInputs = BundleInputs(project, dag, dependentProducts.toMap) - setupAndDeduplicate(client, bundleInputs, computeBundle) { bundle => - val inputs = Inputs(bundle, resultsMap) - compile(inputs, bestEffortAllowed && project.isBestEffort, dependsOnBestEffort).map { results => - results.fromCompiler match { - case Compiler.Result.Ok(_) if failed.isEmpty => - Parent(partialSuccess(bundle, results), dagResults) - case _ => Parent(toPartialFailure(bundle, results), dagResults) + val resultsMap = dependentResults.toMap + val bundleInputs = BundleInputs(project, dag, dependentProducts.toMap) + tracer.trace("setupAndDeduplicate - parent project") { tracer => + setupAndDeduplicate(client, bundleInputs, computeBundle, tracer) { bundle => + val inputs = Inputs(bundle, resultsMap) + compile(inputs, bestEffortAllowed && project.isBestEffort, dependsOnBestEffort, tracer).map { results => + results.fromCompiler match { + case Compiler.Result.Ok(_) if failed.isEmpty => + Parent(partialSuccess(bundle, results), dagResults) + case _ => Parent(toPartialFailure(bundle, results), dagResults) + } + } } } } } } - } - } - register(dag, task.memoize) + } + register(dag, task.memoize) + } } - } - loop(dag) - } + loop(dag) + } private def errorToString(err: Throwable): String = { val sw = new StringWriter() diff --git a/frontend/src/test/resources/source-generator.py b/frontend/src/test/resources/source-generator.py index e7042e3db8..a691c2dade 100644 --- a/frontend/src/test/resources/source-generator.py +++ b/frontend/src/test/resources/source-generator.py @@ -62,3 +62,35 @@ def main(output_dir, args): def random(): return 123 + +def random(): + return 123 + + +def random(): + return 123 + + +def random(): + return 123 + + +def random(): + return 123 + + +def random(): + return 123 + + +def random(): + return 123 + + +def random(): + return 123 + + +def random(): + return 123 + diff --git a/frontend/src/test/scala/bloop/MonixBaseCompileSpec.scala b/frontend/src/test/scala/bloop/MonixBaseCompileSpec.scala index 8ccae576d7..71c17c0b3f 100644 --- a/frontend/src/test/scala/bloop/MonixBaseCompileSpec.scala +++ b/frontend/src/test/scala/bloop/MonixBaseCompileSpec.scala @@ -1,5 +1,6 @@ package bloop +import bloop.Cli.CliSession import bloop.cli.{CommonOptions, ExitStatus} import bloop.engine.NoPool import bloop.io.Environment.{LineSplitter, lineSeparator} @@ -7,9 +8,11 @@ import bloop.logging.RecordingLogger import monix.eval.Task import bloop.testing.DiffAssertions import bloop.util.{BaseTestProject, TestUtil} +import cats.effect.concurrent.Ref import java.io.{ByteArrayOutputStream, PrintStream} import java.nio.charset.StandardCharsets +import java.nio.file.Path abstract class MonixBaseCompileSpec extends bloop.testing.MonixBaseSuite { protected def TestProject: BaseTestProject @@ -20,22 +23,23 @@ abstract class MonixBaseCompileSpec extends bloop.testing.MonixBaseSuite { test("don't compile build in two concurrent CLI clients") { TestUtil.withinWorkspaceV2 { workspace => val sources = List( - """/main/scala/Foo.scala - |class Foo + """/main/scala/Faa.scala + |class Faa """.stripMargin ) val testOut = new ByteArrayOutputStream() val options = CommonOptions.default.copy(out = new PrintStream(testOut)) - val `A` = TestProject(workspace, "a", sources) + val `A` = TestProject(workspace, "z", sources) val configDir = TestProject.populateWorkspace(workspace, List(`A`)) val compileArgs = - Array("compile", "a", "--config-dir", configDir.syntax) + Array("compile", "z", "--config-dir", configDir.syntax, "--verbose") val compileAction = Cli.parse(compileArgs, options) - def runCompileAsync: Task[ExitStatus] = Cli.run(compileAction, NoPool) + def runCompileAsync(activeSessions: Ref[Task, Map[Path, List[CliSession]]], postfix: String): Task[ExitStatus] = + Cli.run(compileAction, NoPool, activeSessions, Option(postfix)) for { - runCompile <- Task.parSequenceUnordered(List(runCompileAsync, runCompileAsync)) + activeSessions <- Ref.of[Task, Map[Path, List[CliSession]]](Map.empty) + _ <- Task.parSequenceUnordered(List(runCompileAsync(activeSessions, "left"), runCompileAsync(activeSessions, "right"))) } yield { - val actionsOutput = new String(testOut.toByteArray, StandardCharsets.UTF_8) def removeAsciiColorCodes(line: String): String = line.replaceAll("\u001B\\[[;\\d]*m", "") @@ -50,9 +54,9 @@ abstract class MonixBaseCompileSpec extends bloop.testing.MonixBaseSuite { try { assertNoDiff( processOutput(obtained), - s"""Compiling a (1 Scala source) - |Deduplicating compilation of a from cli client ??? (since ??? - |Compiling a (1 Scala source) + s"""Compiling z (1 Scala source) + |Deduplicating compilation of z from cli client ??? (since ??? + |Compiling z (1 Scala source) |$extraCompilationMessageOutput |""".stripMargin ) @@ -61,9 +65,9 @@ abstract class MonixBaseCompileSpec extends bloop.testing.MonixBaseSuite { assertNoDiff( processOutput(obtained), s""" - |Deduplicating compilation of a from cli client ??? (since ??? - |Compiling a (1 Scala source) - |Compiling a (1 Scala source) + |Deduplicating compilation of z from cli client ??? (since ??? + |Compiling z (1 Scala source) + |Compiling z (1 Scala source) |$extraCompilationMessageOutput |""".stripMargin ) diff --git a/frontend/src/test/scala/bloop/MonixCompile2Spec.scala b/frontend/src/test/scala/bloop/MonixCompile2Spec.scala new file mode 100644 index 0000000000..02ad71ef6f --- /dev/null +++ b/frontend/src/test/scala/bloop/MonixCompile2Spec.scala @@ -0,0 +1,5 @@ +package bloop + +object MonixCompile2Spec extends MonixBaseCompileSpec { + override protected val TestProject = util.TestProject +} diff --git a/frontend/src/test/scala/bloop/dap/DebugServerSpec.scala b/frontend/src/test/scala/bloop/dap/DebugServerSpec.scala index 2e3ec92a42..93dd5165c3 100644 --- a/frontend/src/test/scala/bloop/dap/DebugServerSpec.scala +++ b/frontend/src/test/scala/bloop/dap/DebugServerSpec.scala @@ -1005,7 +1005,7 @@ object DebugServerSpec extends DebugBspBaseSuite { def cliCompile(project: TestProject) = { val compileArgs = Array("compile", project.config.name, "--config-dir", configDir.syntax) val compileAction = Cli.parse(compileArgs, CommonOptions.default) - Task.eval(Cli.run(compileAction, NoPool)).executeAsync + Task.eval(Cli.run(compileAction, NoPool, ???, None)).executeAsync } def bspCommand() = createBspCommand(configDir) From eb0acff36ba10a3c432e6d317f88b121a42f6a9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Wi=C4=85cek?= Date: Wed, 11 Jun 2025 15:02:59 +0200 Subject: [PATCH 10/10] fix for failing tests --- .../scala/bloop/tracing/TraceProperties.scala | 2 +- .../src/it/scala/bloop/CommunityBuild.scala | 2 +- frontend/src/main/scala/bloop/Cli.scala | 72 +- .../scala/bloop/bsp/BloopBspServices.scala | 3 +- .../main/scala/bloop/engine/Interpreter.scala | 51 +- .../bloop/engine/tasks/CompileTask.scala | 537 ++++++++------- .../tasks/compilation/CompileGatekeeper.scala | 346 ++++------ .../compilation/CompileGatekeeperNew.scala | 288 -------- .../tasks/compilation/CompileGraph.scala | 623 +++++++++--------- .../tasks/compilation/CompileResult.scala | 10 - .../src/test/resources/source-generator.py | 32 - .../test/scala/bloop/DeduplicationSpec.scala | 2 +- .../scala/bloop/MonixBaseCompileSpec.scala | 30 +- .../test/scala/bloop/MonixCompile2Spec.scala | 5 - .../scala/bloop/dap/DebugServerSpec.scala | 15 +- 15 files changed, 780 insertions(+), 1238 deletions(-) delete mode 100644 frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGatekeeperNew.scala delete mode 100644 frontend/src/test/scala/bloop/MonixCompile2Spec.scala diff --git a/backend/src/main/scala/bloop/tracing/TraceProperties.scala b/backend/src/main/scala/bloop/tracing/TraceProperties.scala index 3788457d6b..07d5cf3aea 100644 --- a/backend/src/main/scala/bloop/tracing/TraceProperties.scala +++ b/backend/src/main/scala/bloop/tracing/TraceProperties.scala @@ -33,7 +33,7 @@ object TraceProperties { localServiceName, traceStartAnnotation, traceEndAnnotation, - true + enabled ) } } diff --git a/frontend/src/it/scala/bloop/CommunityBuild.scala b/frontend/src/it/scala/bloop/CommunityBuild.scala index 7d4a37202b..1953f63fea 100644 --- a/frontend/src/it/scala/bloop/CommunityBuild.scala +++ b/frontend/src/it/scala/bloop/CommunityBuild.scala @@ -108,7 +108,7 @@ abstract class CommunityBuild(val buildpressHomeDir: AbsolutePath) { } // First thing to do: clear cache of successful results between project runs to free up space -// CompileGatekeeperOld.clearSuccessfulResults() + CompileGatekeeper.clearSuccessfulResults() // After reporting the state of the execution, compile the projects accordingly. val logger = BloopLogger.default("community-build-logger") diff --git a/frontend/src/main/scala/bloop/Cli.scala b/frontend/src/main/scala/bloop/Cli.scala index fd860ef6db..116c3b33c1 100644 --- a/frontend/src/main/scala/bloop/Cli.scala +++ b/frontend/src/main/scala/bloop/Cli.scala @@ -38,7 +38,7 @@ object Cli extends TaskApp { val action = parse(args.toArray, CommonOptions.default) for { activeCliSessions <- Ref.of[MonixTask, Map[Path, List[CliSession]]](Map.empty) - exitStatus <- run(action, NoPool, activeCliSessions, None) + exitStatus <- run(action, NoPool, activeCliSessions) } yield ExitCode(exitStatus.code) } @@ -64,7 +64,7 @@ object Cli extends TaskApp { ) val cmd = parse(args, nailgunOptions) - val exitStatus = run(cmd, NoPool, cancel, activeCliSessions, None) + val exitStatus = run(cmd, NoPool, cancel, activeCliSessions) exitStatus.map(_.code) } @@ -99,7 +99,7 @@ object Cli extends TaskApp { val handle = Ref .of[MonixTask, Map[Path, List[CliSession]]](Map.empty) - .flatMap(run(cmd, NailgunPool(ngContext), _, None)) + .flatMap(run(cmd, NailgunPool(ngContext), _)) .onErrorHandle { case x: java.util.concurrent.ExecutionException => // print stack trace of fatal errors thrown in asynchronous code, see https://stackoverflow.com/questions/17265022/what-is-a-boxed-error-in-scala @@ -313,13 +313,12 @@ object Cli extends TaskApp { def run( action: Action, pool: ClientPool, - activeCliSessions: Ref[MonixTask, Map[Path, List[CliSession]]], - postfix: Option[String] + activeCliSessions: Ref[MonixTask, Map[Path, List[CliSession]]] ): MonixTask[ExitStatus] = { for { baseCancellation <- Deferred[MonixTask, Boolean] _ <- baseCancellation.complete(false) - result <- run(action, pool, baseCancellation, activeCliSessions, postfix) + result <- run(action, pool, baseCancellation, activeCliSessions) } yield result } @@ -329,8 +328,7 @@ object Cli extends TaskApp { action: Action, pool: ClientPool, cancel: Deferred[MonixTask, Boolean], - activeCliSessions: Ref[MonixTask, Map[Path, List[CliSession]]], - postfix: Option[String] + activeCliSessions: Ref[MonixTask, Map[Path, List[CliSession]]] ): MonixTask[ExitStatus] = { import bloop.io.AbsolutePath def getConfigDir(cliOptions: CliOptions): AbsolutePath = { @@ -362,13 +360,11 @@ object Cli extends TaskApp { !(cliOptions.noColor || commonOpts.env.containsKey("NO_COLOR")), debugFilter ) - logger.debug("Running cli" + postfix.fold("")(":" + _) + "...") action match { case Print(msg, _, Exit(exitStatus)) => logger.info(msg) MonixTask.now(exitStatus) case _ => - logger.debug("running action" + postfix.fold("")(":" + _) + "...") runWithState( action, pool, @@ -377,8 +373,7 @@ object Cli extends TaskApp { configDirectory, cliOptions, commonOpts, - logger, - postfix + logger ) } } @@ -391,19 +386,16 @@ object Cli extends TaskApp { configDirectory: AbsolutePath, cliOptions: CliOptions, commonOpts: CommonOptions, - logger: Logger, - postfix: Option[String] + logger: Logger ): MonixTask[ExitStatus] = { - logger.debug("running with state" + postfix.fold("")(":" + _) + "...") // Set the proxy settings right before loading the state of the build bloop.util.ProxySetup.updateProxySettings(commonOpts.env.toMap, logger) val configDir = configDirectory.underlying waitUntilEndOfWorld(cliOptions, pool, configDir, logger, cancel) { val taskToInterpret = { (cli: CliClientInfo) => - logger.debug("Running command interpretation" + postfix.fold("")(":" + _) + "...") val state = State.loadActiveStateFor(configDirectory, cli, pool, cliOptions.common, logger) - val interpret = Interpreter.execute(action, state, postfix).map { newState => + val interpret = Interpreter.execute(action, state).map { newState => action match { case Run(_: Commands.ValidatedBsp, _) => () // Ignore, BSP services auto-update the build @@ -412,7 +404,7 @@ object Cli extends TaskApp { newState } - interpret.toMonixTask(ExecutionContext.scheduler) + MonixTask.defer(interpret.toMonixTask(ExecutionContext.scheduler)) } val session = runTaskWithCliClient( @@ -420,18 +412,11 @@ object Cli extends TaskApp { action, taskToInterpret, activeCliSessions, - pool, - logger, - postfix + pool ) val exitSession = MonixTask.defer { session.flatMap { session => cleanUpNonStableCliDirectories(session.client, logger).flatMap { _ => - logger.debug( - s"Cleaning up non-stable cli directories: $configDirectory" + postfix.fold("")( - ":" + _ - ) + "..." - ) activeCliSessions.update(_ - configDirectory.underlying) } } @@ -449,9 +434,7 @@ object Cli extends TaskApp { action: Action, processCliTask: CliClientInfo => MonixTask[State], activeCliSessions: Ref[MonixTask, Map[Path, List[CliSession]]], - pool: ClientPool, - logger: Logger, - postfix: Option[String] + pool: ClientPool ): MonixTask[CliSession] = { val isClientConnected = AtomicBoolean(true) pool.addListener(_ => isClientConnected.set(false)) @@ -462,21 +445,15 @@ object Cli extends TaskApp { CliSession(client, cliTask) } - val defaultClientSession = sessionFor(defaultClient) action match { - case Exit(_) => MonixTask.now(defaultClientSession) + case Exit(_) => MonixTask.now(sessionFor(defaultClient)) // Don't synchronize on commands that don't use compilation products and can run concurrently - case Run(_: Commands.About, _) => MonixTask.now(defaultClientSession) - case Run(_: Commands.Projects, _) => MonixTask.now(defaultClientSession) - case Run(_: Commands.Autocomplete, _) => MonixTask.now(defaultClientSession) - case Run(_: Commands.Bsp, _) => MonixTask.now(defaultClientSession) - case Run(_: Commands.ValidatedBsp, _) => MonixTask.now(defaultClientSession) + case Run(_: Commands.About, _) => MonixTask.now(sessionFor(defaultClient)) + case Run(_: Commands.Projects, _) => MonixTask.now(sessionFor(defaultClient)) + case Run(_: Commands.Autocomplete, _) => MonixTask.now(sessionFor(defaultClient)) + case Run(_: Commands.Bsp, _) => MonixTask.now(sessionFor(defaultClient)) + case Run(_: Commands.ValidatedBsp, _) => MonixTask.now(sessionFor(defaultClient)) case a @ _ => - logger.debug( - "command: " + action.asInstanceOf[Run].command.getClass.toString + " " + postfix.fold("")( - ":" + _ - ) + "..." - ) activeCliSessions .modify { sessionsMap => sessionsMap.get(configDir.underlying) match { @@ -488,20 +465,13 @@ object Cli extends TaskApp { newClientSession ) case None => + val newSession = sessionFor(defaultClient) ( - sessionsMap.updated(configDir.underlying, List(defaultClientSession)), - defaultClientSession + sessionsMap.updated(configDir.underlying, List(newSession)), + newSession ) } } - .flatMap { sessions => - activeCliSessions.get - .map { sessionsMap => - logger.debug("currentActiveSessions:" + postfix.fold("")(_ + ": ") + sessionsMap) - sessions - } - - } } } diff --git a/frontend/src/main/scala/bloop/bsp/BloopBspServices.scala b/frontend/src/main/scala/bloop/bsp/BloopBspServices.scala index f97feb95f2..a47b207b2e 100644 --- a/frontend/src/main/scala/bloop/bsp/BloopBspServices.scala +++ b/frontend/src/main/scala/bloop/bsp/BloopBspServices.scala @@ -579,8 +579,7 @@ final class BloopBspServices( bestEffortAllowed, cancelCompilation, store, - logger, - Option("bloopBsp") + logger ) } diff --git a/frontend/src/main/scala/bloop/engine/Interpreter.scala b/frontend/src/main/scala/bloop/engine/Interpreter.scala index 8305044d5f..5342eaccf5 100644 --- a/frontend/src/main/scala/bloop/engine/Interpreter.scala +++ b/frontend/src/main/scala/bloop/engine/Interpreter.scala @@ -37,8 +37,8 @@ import bloop.testing.TestInternals object Interpreter { // This is stack-safe because of Monix's trampolined execution - def execute(action: Action, stateTask: Task[State], postfix: Option[String] = None): Task[State] = { - def execute(action: Action, stateTask: Task[State], postfix: Option[String]): Task[State] = { + def execute(action: Action, stateTask: Task[State]): Task[State] = { + def execute(action: Action, stateTask: Task[State]): Task[State] = { stateTask.flatMap { state => action match { // We keep it case because there is a 'match may not be exhaustive' false positive by scalac @@ -48,13 +48,13 @@ object Interpreter { case Exit(exitStatus: ExitStatus) => Task.now(state.mergeStatus(exitStatus)) case Print(msg, _, next) => state.logger.info(msg) - execute(next, Task.now(state), postfix) + execute(next, Task.now(state)) case Run(cmd: Commands.Bsp, _) => val msg = "Internal error: The `bsp` command must be validated before use" val printAction = Print(msg, cmd.cliOptions.common, Exit(ExitStatus.UnexpectedError)) - execute(printAction, Task.now(state), postfix) + execute(printAction, Task.now(state)) case Run(cmd: Commands.ValidatedBsp, next) => - execute(next, runBsp(cmd, state), postfix) + execute(next, runBsp(cmd, state)) case Run(cmd: Commands.About, _) => notHandled("about", cmd.cliOptions, state) case Run(cmd: Commands.Help, _) => @@ -67,23 +67,23 @@ object Interpreter { else { cmd match { case cmd: Commands.Clean => - execute(next, clean(cmd, state), postfix) + execute(next, clean(cmd, state)) case cmd: Commands.Compile => - execute(next, compile(cmd, state, postfix), postfix) + execute(next, compile(cmd, state)) case cmd: Commands.Console => - execute(next, console(cmd, state), postfix) + execute(next, console(cmd, state)) case cmd: Commands.Projects => - execute(next, showProjects(cmd, state), postfix) + execute(next, showProjects(cmd, state)) case cmd: Commands.Test => - execute(next, test(cmd, state), postfix) + execute(next, test(cmd, state)) case cmd: Commands.Run => - execute(next, run(cmd, state), postfix) + execute(next, run(cmd, state)) case cmd: Commands.Configure => - execute(next, configure(cmd, state), postfix) + execute(next, configure(cmd, state)) case cmd: Commands.Autocomplete => - execute(next, autocomplete(cmd, state), postfix) + execute(next, autocomplete(cmd, state)) case cmd: Commands.Link => - execute(next, link(cmd, state), postfix) + execute(next, link(cmd, state)) } } } @@ -91,7 +91,7 @@ object Interpreter { } } - execute(action, stateTask, postfix) + execute(action, stateTask) } private def notHandled(command: String, cliOptions: CliOptions, state: State): Task[State] = { @@ -153,8 +153,7 @@ object Interpreter { cmd: CompilingCommand, state0: State, projects: List[Project], - noColor: Boolean, - postfix: Option[String] = None + noColor: Boolean ): Task[State] = { // Make new state cleaned of all compilation products if compilation is not incremental val state: Task[State] = { @@ -179,19 +178,14 @@ object Interpreter { bestEffortAllowed = false, Promise[Unit](), CompileClientStore.NoStore, - state.logger, - postfix + state.logger ) } compileTask.map(_.mergeStatus(ExitStatus.Ok)) } - private def compile( - cmd: Commands.Compile, - state: State, - postfix: Option[String] = None - ): Task[State] = { + private def compile(cmd: Commands.Compile, state: State): Task[State] = { val lookup = lookupProjects(cmd.projects, state.build.getProjectFor(_)) if (lookup.missing.nonEmpty) Task.now(reportMissing(lookup.missing, state)) else { @@ -200,8 +194,8 @@ object Interpreter { else Dag.inverseDependencies(state.build.dags, lookup.found).reduced } - if (!cmd.watch) runCompile(cmd, state, projects, cmd.cliOptions.noColor, postfix) - else watch(projects, state)(runCompile(cmd, _, projects, cmd.cliOptions.noColor, postfix.map(_ + cmd + ":watch"))) + if (!cmd.watch) runCompile(cmd, state, projects, cmd.cliOptions.noColor) + else watch(projects, state)(runCompile(cmd, _, projects, cmd.cliOptions.noColor)) } } @@ -224,10 +218,9 @@ object Interpreter { state: State, projects: List[Project], noColor: Boolean, - nextAction: String, - postfix: Option[String] = None + nextAction: String )(next: State => Task[State]): Task[State] = { - runCompile(cmd, state, projects, noColor, postfix.orElse(Some(nextAction))).flatMap { compiled => + runCompile(cmd, state, projects, noColor).flatMap { compiled => if (compiled.status != ExitStatus.CompilationError) next(compiled) else { val projectsString = projects.mkString(", ") diff --git a/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala b/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala index 8e1e3b69cd..725655d488 100644 --- a/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala +++ b/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala @@ -47,7 +47,6 @@ import xsbti.compile.PreviousResult object CompileTask { private implicit val logContext: DebugFilter = DebugFilter.Compilation - def compile[UseSiteLogger <: Logger]( state: State, dag: Dag[Project], @@ -56,8 +55,7 @@ object CompileTask { bestEffortAllowed: Boolean, cancelCompilation: Promise[Unit], store: CompileClientStore, - rawLogger: UseSiteLogger, - postfix: Option[String] = None + rawLogger: UseSiteLogger ): Task[State] = Task.defer { import bloop.data.ClientInfo import bloop.internal.build.BuildInfo @@ -72,7 +70,7 @@ object CompileTask { val traceProperties = WorkspaceSettings.tracePropertiesFrom(state.build.workspaceSettings) val rootTracer = BraveTracer( - s"compile $topLevelTargets (transitively)${postfix.fold("")(":" + _)}", + s"compile $topLevelTargets (transitively)", traceProperties, "bloop.version" -> BuildInfo.version, "zinc.version" -> BuildInfo.zincVersion, @@ -91,305 +89,296 @@ object CompileTask { "client" -> clientName ) - rootTracer.trace("CompileTask.compile") { tracer => - def compile( - graphInputs: CompileGraph.Inputs, - isBestEffort: Boolean, - isBestEffortDep: Boolean, - tracer: BraveTracer - ): Task[ResultBundle] = tracer.trace("CompileTask.compile - inner") { tracer => - val bundle = graphInputs.bundle - val project = bundle.project - val logger = bundle.logger - val reporter = bundle.reporter - val previousResult = bundle.latestResult - val compileOut = bundle.out - val lastSuccessful = bundle.lastSuccessful - val compileProjectTracer = tracer.startNewChildTracer( - s"compile ${project.name}", - "compile.target" -> project.name - ) - - bundle.prepareSourcesAndInstance match { - case Left(earlyResultBundle) => - compileProjectTracer.terminate() - Task.now(earlyResultBundle) - case Right(CompileSourcesAndInstance(sources, instance, _)) => - val readOnlyClassesDir = lastSuccessful.classesDir - val newClassesDir = compileOut.internalNewClassesDir - val classpath = bundle.dependenciesData.buildFullCompileClasspathFor( - project, - readOnlyClassesDir, - newClassesDir - ) - - // Warn user if detected missing dep, see https://github.com/scalacenter/bloop/issues/708 - state.build.hasMissingDependencies(project).foreach { missing => - Feedback - .detectMissingDependencies(project.name, missing) - .foreach(msg => logger.warn(msg)) - } + def compile( + graphInputs: CompileGraph.Inputs, + isBestEffort: Boolean, + isBestEffortDep: Boolean + ): Task[ResultBundle] = { + val bundle = graphInputs.bundle + val project = bundle.project + val logger = bundle.logger + val reporter = bundle.reporter + val previousResult = bundle.latestResult + val compileOut = bundle.out + val lastSuccessful = bundle.lastSuccessful + val compileProjectTracer = rootTracer.startNewChildTracer( + s"compile ${project.name}", + "compile.target" -> project.name + ) - val configuration = configureCompilation(project) - val newScalacOptions = { - CompilerPluginAllowlist - .enableCachingInScalacOptions( - instance.version, - configuration.scalacOptions, - logger, - compileProjectTracer, - 5 - ) - } + bundle.prepareSourcesAndInstance match { + case Left(earlyResultBundle) => + compileProjectTracer.terminate() + Task.now(earlyResultBundle) + case Right(CompileSourcesAndInstance(sources, instance, _)) => + val readOnlyClassesDir = lastSuccessful.classesDir + val newClassesDir = compileOut.internalNewClassesDir + val classpath = bundle.dependenciesData.buildFullCompileClasspathFor( + project, + readOnlyClassesDir, + newClassesDir + ) + + // Warn user if detected missing dep, see https://github.com/scalacenter/bloop/issues/708 + state.build.hasMissingDependencies(project).foreach { missing => + Feedback + .detectMissingDependencies(project.name, missing) + .foreach(msg => logger.warn(msg)) + } - val inputs = newScalacOptions.map { newScalacOptions => - CompileInputs( - instance, - state.compilerCache, - sources.toArray, - classpath, - bundle.uniqueInputs, - compileOut, - project.out, - newScalacOptions.toArray, - project.javacOptions.toArray, - project.compileJdkConfig.flatMap(_.javacBin), - project.compileOrder, - project.classpathOptions, - lastSuccessful.previous, - previousResult, - reporter, + val configuration = configureCompilation(project) + val newScalacOptions = { + CompilerPluginAllowlist + .enableCachingInScalacOptions( + instance.version, + configuration.scalacOptions, logger, - graphInputs.dependentResults, - cancelCompilation, compileProjectTracer, - ExecutionContext.ioScheduler, - ExecutionContext.ioExecutor, - bundle.dependenciesData.allInvalidatedClassFiles, - bundle.dependenciesData.allGeneratedClassFilePaths, - project.runtimeResources + 5 ) + } + + val inputs = newScalacOptions.map { newScalacOptions => + CompileInputs( + instance, + state.compilerCache, + sources.toArray, + classpath, + bundle.uniqueInputs, + compileOut, + project.out, + newScalacOptions.toArray, + project.javacOptions.toArray, + project.compileJdkConfig.flatMap(_.javacBin), + project.compileOrder, + project.classpathOptions, + lastSuccessful.previous, + previousResult, + reporter, + logger, + graphInputs.dependentResults, + cancelCompilation, + compileProjectTracer, + ExecutionContext.ioScheduler, + ExecutionContext.ioExecutor, + bundle.dependenciesData.allInvalidatedClassFiles, + bundle.dependenciesData.allGeneratedClassFilePaths, + project.runtimeResources + ) + } + + val waitOnReadClassesDir = { + compileProjectTracer.traceTaskVerbose("wait on populating products") { _ => + // This task is memoized and started by the compilation that created + // it, so this execution blocks until it's run or completes right away + lastSuccessful.populatingProducts } + } - val waitOnReadClassesDir = { - compileProjectTracer.traceTaskVerbose("wait on populating products") { _ => - // This task is memoized and started by the compilation that created - // it, so this execution blocks until it's run or completes right away - lastSuccessful.populatingProducts + // Block on the task associated with this result that sets up the read-only classes dir + waitOnReadClassesDir.flatMap { _ => + // Only when the task is finished, we kickstart the compilation + def compile(inputs: CompileInputs) = { + val firstResult = + Compiler.compile(inputs, isBestEffort, isBestEffortDep, firstCompilation = true) + firstResult.flatMap { + case result @ Compiler.Result.Failed( + _, + _, + _, + _, + Some(BestEffortProducts(_, _, recompile)) + ) if recompile => + // we restart the compilation, starting from scratch (without any previous artifacts) + inputs.reporter.reset() + val emptyResult = + PreviousResult.of(Optional.empty[CompileAnalysis], Optional.empty[MiniSetup]) + val nonIncrementalClasspath = + inputs.classpath.filter(_ != inputs.out.internalReadOnlyClassesDir) + val newInputs = inputs.copy( + sources = inputs.sources, + classpath = nonIncrementalClasspath, + previousCompilerResult = result, + previousResult = emptyResult + ) + Compiler.compile( + newInputs, + isBestEffort, + isBestEffortDep, + firstCompilation = false + ) + case result => Task(result) } } - - // Block on the task associated with this result that sets up the read-only classes dir - waitOnReadClassesDir.flatMap { _ => - // Only when the task is finished, we kickstart the compilation - def compile(inputs: CompileInputs) = { - val firstResult = - Compiler.compile(inputs, isBestEffort, isBestEffortDep, firstCompilation = true) - firstResult.flatMap { - case result @ Compiler.Result.Failed( - _, - _, - _, - _, - Some(BestEffortProducts(_, _, recompile)) - ) if recompile => - // we restart the compilation, starting from scratch (without any previous artifacts) - inputs.reporter.reset() - val emptyResult = - PreviousResult.of(Optional.empty[CompileAnalysis], Optional.empty[MiniSetup]) - val nonIncrementalClasspath = - inputs.classpath.filter(_ != inputs.out.internalReadOnlyClassesDir) - val newInputs = inputs.copy( - sources = inputs.sources, - classpath = nonIncrementalClasspath, - previousCompilerResult = result, - previousResult = emptyResult - ) - Compiler.compile( - newInputs, - isBestEffort, - isBestEffortDep, - firstCompilation = false + inputs.flatMap(inputs => compile(inputs)).map { result => + def runPostCompilationTasks( + backgroundTasks: CompileBackgroundTasks + ): CancelableFuture[Unit] = { + // Post compilation tasks use tracer, so terminate right after they have + val postCompilationTasks = + backgroundTasks + .trigger( + bundle.clientClassesObserver, + reporter.underlying, + compileProjectTracer, + logger ) - case result => Task(result) - } + .doOnFinish(_ => Task(compileProjectTracer.terminate())) + postCompilationTasks.runAsync(ExecutionContext.ioScheduler) } - inputs.flatMap(inputs => compile(inputs)).map { result => - def runPostCompilationTasks( - backgroundTasks: CompileBackgroundTasks - ): CancelableFuture[Unit] = { - // Post compilation tasks use tracer, so terminate right after they have - val postCompilationTasks = - backgroundTasks - .trigger( - bundle.clientClassesObserver, - reporter.underlying, - compileProjectTracer, - logger - ) - .doOnFinish(_ => Task(compileProjectTracer.terminate())) - postCompilationTasks.runAsync(ExecutionContext.ioScheduler) - } - - // Populate the last successful result if result was success - result match { - case s: Compiler.Result.Success => - val runningTasks = runPostCompilationTasks(s.backgroundTasks) - val blockingOnRunningTasks = Task - .fromFuture(runningTasks) - .executeOn(ExecutionContext.ioScheduler) - val populatingTask = { - if (s.isNoOp) blockingOnRunningTasks // Task.unit - else { - for { - _ <- blockingOnRunningTasks - _ <- populateNewReadOnlyClassesDir(s.products, bgTracer, rawLogger) - .doOnFinish(_ => Task(bgTracer.terminate())) - } yield () - } + // Populate the last successful result if result was success + result match { + case s: Compiler.Result.Success => + val runningTasks = runPostCompilationTasks(s.backgroundTasks) + val blockingOnRunningTasks = Task + .fromFuture(runningTasks) + .executeOn(ExecutionContext.ioScheduler) + val populatingTask = { + if (s.isNoOp) blockingOnRunningTasks // Task.unit + else { + for { + _ <- blockingOnRunningTasks + _ <- populateNewReadOnlyClassesDir(s.products, bgTracer, rawLogger) + .doOnFinish(_ => Task(bgTracer.terminate())) + } yield () } - - // Memoize so that no matter how many times it's run, it's executed only once - val newSuccessful = - LastSuccessfulResult(bundle.uniqueInputs, s.products, populatingTask.memoize) - ResultBundle(s, Some(newSuccessful), Some(lastSuccessful), runningTasks) - case f: Compiler.Result.Failed => - val runningTasks = runPostCompilationTasks(f.backgroundTasks) - ResultBundle(result, None, Some(lastSuccessful), runningTasks) - case c: Compiler.Result.Cancelled => - val runningTasks = runPostCompilationTasks(c.backgroundTasks) - ResultBundle(result, None, Some(lastSuccessful), runningTasks) - case _: Compiler.Result.Blocked | Compiler.Result.Empty | - _: Compiler.Result.GlobalError => - ResultBundle(result, None, None, CancelableFuture.unit) - } + } + + // Memoize so that no matter how many times it's run, it's executed only once + val newSuccessful = + LastSuccessfulResult(bundle.uniqueInputs, s.products, populatingTask.memoize) + ResultBundle(s, Some(newSuccessful), Some(lastSuccessful), runningTasks) + case f: Compiler.Result.Failed => + val runningTasks = runPostCompilationTasks(f.backgroundTasks) + ResultBundle(result, None, Some(lastSuccessful), runningTasks) + case c: Compiler.Result.Cancelled => + val runningTasks = runPostCompilationTasks(c.backgroundTasks) + ResultBundle(result, None, Some(lastSuccessful), runningTasks) + case _: Compiler.Result.Blocked | Compiler.Result.Empty | + _: Compiler.Result.GlobalError => + ResultBundle(result, None, None, CancelableFuture.unit) } } - } + } } + } - def setup( - inputs: CompileDefinitions.BundleInputs, - tracer: BraveTracer - ): Task[CompileBundle] = { - // Create a multicast observable stream to allow multiple mirrors of loggers - val (observer, obs) = { - Observable.multicast[Either[ReporterAction, LoggerAction]]( - MulticastStrategy.replay - )(ExecutionContext.ioScheduler) - } + def setup(inputs: CompileDefinitions.BundleInputs): Task[CompileBundle] = { + // Create a multicast observable stream to allow multiple mirrors of loggers + val (observer, obs) = { + Observable.multicast[Either[ReporterAction, LoggerAction]]( + MulticastStrategy.replay + )(ExecutionContext.ioScheduler) + } - // Compute the previous and last successful results from the results cache - import inputs.project - val (prev, last) = { - if (pipeline) { - val emptySuccessful = LastSuccessfulResult.empty(project) - // Disable incremental compilation if pipelining is enabled - Compiler.Result.Empty -> emptySuccessful - } else { - // Use last successful from user cache, only useful if this is its first - // compilation, otherwise we use the last successful from [[CompileGraph]] - val latestResult = state.results.latestResult(project) - val lastSuccessful = state.results.lastSuccessfulResultOrEmpty(project) - latestResult -> lastSuccessful - } + // Compute the previous and last successful results from the results cache + import inputs.project + val (prev, last) = { + if (pipeline) { + val emptySuccessful = LastSuccessfulResult.empty(project) + // Disable incremental compilation if pipelining is enabled + Compiler.Result.Empty -> emptySuccessful + } else { + // Use last successful from user cache, only useful if this is its first + // compilation, otherwise we use the last successful from [[CompileGraph]] + val latestResult = state.results.latestResult(project) + val lastSuccessful = state.results.lastSuccessfulResultOrEmpty(project) + latestResult -> lastSuccessful } - - val t = tracer - val o = state.commonOptions - val cancel = cancelCompilation - val logger = ObservedLogger(rawLogger, observer) - val clientClassesObserver = state.client.getClassesObserverFor(inputs.project) - val underlying = createReporter(ReporterInputs(inputs.project, cwd, rawLogger)) - val reporter = new ObservedReporter(logger, underlying) - val sourceGeneratorCache = state.sourceGeneratorCache - CompileBundle.computeFrom( - inputs, - sourceGeneratorCache, - clientClassesObserver, - reporter, - last, - prev, - cancel, - logger, - obs, - t, - o - ) } - val client = state.client - CompileGraph - .traverse(dag, client, store, bestEffortAllowed, setup, compile, tracer) - .flatMap { pdag => - val partialResults = Dag.dfs(pdag, mode = Dag.PreOrder) - val finalResults = partialResults.map(r => PartialCompileResult.toFinalResult(r)) - Task.gatherUnordered(finalResults).map(_.flatten).flatMap { results => - val cleanUpTasksToRunInBackground = - markUnusedClassesDirAndCollectCleanUpTasks(results, state, rawLogger) - - val failures = results.flatMap { - case FinalNormalCompileResult(p, results) => - results.fromCompiler match { - case Compiler.Result.NotOk(_) => List(p) - // Consider success with reported fatal warnings as error to simulate -Xfatal-warnings - case s: Compiler.Result.Success if s.reportedFatalWarnings => List(p) - case _ => Nil - } - case _ => Nil - } + val t = rootTracer + val o = state.commonOptions + val cancel = cancelCompilation + val logger = ObservedLogger(rawLogger, observer) + val clientClassesObserver = state.client.getClassesObserverFor(inputs.project) + val underlying = createReporter(ReporterInputs(inputs.project, cwd, rawLogger)) + val reporter = new ObservedReporter(logger, underlying) + val sourceGeneratorCache = state.sourceGeneratorCache + CompileBundle.computeFrom( + inputs, + sourceGeneratorCache, + clientClassesObserver, + reporter, + last, + prev, + cancel, + logger, + obs, + t, + o + ) + } - val newState: State = { - val stateWithResults = state.copy(results = state.results.addFinalResults(results)) - if (failures.isEmpty) { - stateWithResults.copy(status = ExitStatus.Ok) - } else { - results.foreach { - case FinalNormalCompileResult.HasException(project, err) => - val errMsg = err.fold(identity, Logger.prettyPrintException) - rawLogger.error(s"Unexpected error when compiling ${project.name}: $errMsg") - case _ => - () // Do nothing when the final compilation result is not an actual error - } - - client match { - case _: ClientInfo.CliClientInfo => - // Reverse list of failed projects to get ~correct order of failure - val projectsFailedToCompile = failures.map(p => s"'${p.name}'").reverse - val failureMessage = - if (failures.size <= 2) projectsFailedToCompile.mkString(",") - else { - s"${projectsFailedToCompile.take(2).mkString(", ")} and ${projectsFailedToCompile.size - 2} more projects" - } - - rawLogger.error("Failed to compile " + failureMessage) - case _: ClientInfo.BspClientInfo => () // Don't report if bsp client - } - - stateWithResults.copy(status = ExitStatus.CompilationError) + val client = state.client + CompileGraph.traverse(dag, client, store, bestEffortAllowed, setup(_), compile).flatMap { + pdag => + val partialResults = Dag.dfs(pdag, mode = Dag.PreOrder) + val finalResults = partialResults.map(r => PartialCompileResult.toFinalResult(r)) + Task.gatherUnordered(finalResults).map(_.flatten).flatMap { results => + val cleanUpTasksToRunInBackground = + markUnusedClassesDirAndCollectCleanUpTasks(results, state, rawLogger) + + val failures = results.flatMap { + case FinalNormalCompileResult(p, results) => + results.fromCompiler match { + case Compiler.Result.NotOk(_) => List(p) + // Consider success with reported fatal warnings as error to simulate -Xfatal-warnings + case s: Compiler.Result.Success if s.reportedFatalWarnings => List(p) + case _ => Nil } - } + case _ => Nil + } - // Schedule to run clean-up tasks in the background - runIOTasksInParallel(cleanUpTasksToRunInBackground) + val newState: State = { + val stateWithResults = state.copy(results = state.results.addFinalResults(results)) + if (failures.isEmpty) { + stateWithResults.copy(status = ExitStatus.Ok) + } else { + results.foreach { + case FinalNormalCompileResult.HasException(project, err) => + val errMsg = err.fold(identity, Logger.prettyPrintException) + rawLogger.error(s"Unexpected error when compiling ${project.name}: $errMsg") + case _ => () // Do nothing when the final compilation result is not an actual error + } - val runningTasksRequiredForCorrectness = Task.sequence { - results.flatMap { - case FinalNormalCompileResult(_, result) => - val tasksAtEndOfBuildCompilation = - Task.fromFuture(result.runningBackgroundTasks) - List(tasksAtEndOfBuildCompilation) - case _ => Nil + client match { + case _: ClientInfo.CliClientInfo => + // Reverse list of failed projects to get ~correct order of failure + val projectsFailedToCompile = failures.map(p => s"'${p.name}'").reverse + val failureMessage = + if (failures.size <= 2) projectsFailedToCompile.mkString(",") + else { + s"${projectsFailedToCompile.take(2).mkString(", ")} and ${projectsFailedToCompile.size - 2} more projects" + } + + rawLogger.error("Failed to compile " + failureMessage) + case _: ClientInfo.BspClientInfo => () // Don't report if bsp client } + + stateWithResults.copy(status = ExitStatus.CompilationError) } + } + + // Schedule to run clean-up tasks in the background + runIOTasksInParallel(cleanUpTasksToRunInBackground) - // Block on all background task that are running and are required for correctness - runningTasksRequiredForCorrectness - .executeOn(ExecutionContext.ioScheduler) - .map(_ => newState) - .doOnFinish(_ => Task(rootTracer.terminate())) + val runningTasksRequiredForCorrectness = Task.sequence { + results.flatMap { + case FinalNormalCompileResult(_, result) => + val tasksAtEndOfBuildCompilation = + Task.fromFuture(result.runningBackgroundTasks) + List(tasksAtEndOfBuildCompilation) + case _ => Nil + } } + + // Block on all background task that are running and are required for correctness + runningTasksRequiredForCorrectness + .executeOn(ExecutionContext.ioScheduler) + .map(_ => newState) + .doOnFinish(_ => Task(rootTracer.terminate())) } } } diff --git a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGatekeeper.scala b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGatekeeper.scala index ac682d6ba6..6028b4e837 100644 --- a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGatekeeper.scala +++ b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGatekeeper.scala @@ -1,18 +1,23 @@ package bloop.engine.tasks.compilation -import bloop.{Compiler, UniqueCompileInputs} -import bloop.data.{ClientInfo, Project} +import java.util.concurrent.ConcurrentHashMap + +import bloop.Compiler +import bloop.UniqueCompileInputs +import bloop.data.ClientInfo +import bloop.data.Project import bloop.engine.Dag import bloop.engine.caches.LastSuccessfulResult import bloop.io.AbsolutePath -import bloop.logging.{DebugFilter, Logger, LoggerAction} +import bloop.logging.DebugFilter +import bloop.logging.Logger +import bloop.logging.LoggerAction import bloop.reporter.ReporterAction import bloop.task.Task -import bloop.tracing.BraveTracer -import monix.execution.atomic.{AtomicBoolean, AtomicInt} -import monix.reactive.Observable -import java.util.concurrent.ConcurrentHashMap +import monix.execution.atomic.AtomicBoolean +import monix.execution.atomic.AtomicInt +import monix.reactive.Observable object CompileGatekeeper { private implicit val filter: DebugFilter = DebugFilter.Compilation @@ -38,87 +43,49 @@ object CompileGatekeeper { inputs: BundleInputs, bundle: SuccessfulCompileBundle, client: ClientInfo, - compile: SuccessfulCompileBundle => CompileTraversal, - tracer: BraveTracer - ): (RunningCompilation, CanBeDeduplicated) = - tracer.trace("Finding compilation atomically.") { tracer => - import bundle.logger - var deduplicate = true - - val running = runningCompilations.compute( - bundle.uniqueInputs, - (_: UniqueCompileInputs, running: RunningCompilation) => { - if (running == null) { - tracer.trace( - "no running compilation found starting new one", - ("uniqueInputs", bundle.uniqueInputs.toString) - ) { tracer => - logger.debug(s"no running compilation found starting new one:${bundle.uniqueInputs}") + compile: SuccessfulCompileBundle => CompileTraversal + ): (RunningCompilation, CanBeDeduplicated) = { + var deduplicate = true + + val running = runningCompilations.compute( + bundle.uniqueInputs, + (_: UniqueCompileInputs, running: RunningCompilation) => { + if (running == null) { + deduplicate = false + scheduleCompilation(inputs, bundle, client, compile) + } else { + val usedClassesDir = running.usedLastSuccessful.classesDir + val usedClassesDirCounter = running.usedLastSuccessful.counterForClassesDir + + usedClassesDirCounter.getAndTransform { count => + if (count == 0) { + // Abort deduplication, dir is scheduled to be deleted in background deduplicate = false - scheduleCompilation(inputs, bundle, client, compile, tracer) - } - } else { - tracer.trace( - "Found matching compilation", - ("uniqueInputs", bundle.uniqueInputs.toString) - ) { tracer => - val usedClassesDir = running.usedLastSuccessful.classesDir - val usedClassesDirCounter = running.usedLastSuccessful.counterForClassesDir - - usedClassesDirCounter.getAndTransform { count => - if (count == 0) { - tracer.trace( - "Aborting deduplication", - ("uniqueInputs", bundle.uniqueInputs.toString) - ) { tracer => - logger.debug( - s"Abort deduplication, dir is scheduled to be deleted in background:${bundle.uniqueInputs}" - ) - // Abort deduplication, dir is scheduled to be deleted in background - deduplicate = false - // Remove from map of used classes dirs in case it hasn't already been - currentlyUsedClassesDirs.remove(usedClassesDir, usedClassesDirCounter) - // Return previous count, this counter will soon be deallocated - count - } - } else { - tracer.trace( - "Increasing compilation counter", - ("uniqueInputs", bundle.uniqueInputs.toString) - ) { tracer => - logger.debug( - s"Increase count to prevent other compiles to schedule its deletion:${bundle.uniqueInputs}" - ) - // Increase count to prevent other compiles to schedule its deletion - count + 1 - } - } - } - - if (deduplicate) running - else scheduleCompilation(inputs, bundle, client, compile, tracer) + // Remove from map of used classes dirs in case it hasn't already been + currentlyUsedClassesDirs.remove(usedClassesDir, usedClassesDirCounter) + // Return previous count, this counter will soon be deallocated + count + } else { + // Increase count to prevent other compiles to schedule its deletion + count + 1 } } + + if (deduplicate) running + else scheduleCompilation(inputs, bundle, client, compile) } - ) + } + ) - (running, deduplicate) - } + (running, deduplicate) + } def disconnectDeduplicationFromRunning( inputs: UniqueCompileInputs, - runningCompilation: RunningCompilation, - logger: Logger, - tracer: BraveTracer + runningCompilation: RunningCompilation ): Unit = { - tracer.trace( - "disconnectDeduplicationFromRunning", - ("uniqueInputs", inputs.toString) - ) { _ => - logger.debug(s"Disconnected deduplication from running compilation:${inputs}") - runningCompilation.isUnsubscribed.compareAndSet(false, true) - runningCompilations.remove(inputs, runningCompilation); () - } + runningCompilation.isUnsubscribed.compareAndSet(false, true) + runningCompilations.remove(inputs, runningCompilation); () } /** @@ -132,150 +99,121 @@ object CompileGatekeeper { inputs: BundleInputs, bundle: SuccessfulCompileBundle, client: ClientInfo, - compile: SuccessfulCompileBundle => CompileTraversal, - tracer: BraveTracer - ): RunningCompilation = - tracer.trace("schedule compilation") { _ => - import bundle.logger - import inputs.project - - var counterForUsedClassesDir: AtomicInt = null - - def initializeLastSuccessful(previousOrNull: LastSuccessfulResult): LastSuccessfulResult = - tracer.trace(s"initialize last successful") { _ => - val result = Option(previousOrNull).getOrElse(bundle.lastSuccessful) - if (!result.classesDir.exists) { - logger.debug( - s"Ignoring analysis for ${project.name}, directory ${result.classesDir} is missing" - ) - LastSuccessfulResult.empty(inputs.project) - } else if (bundle.latestResult == Compiler.Result.Empty) { - logger.debug(s"Ignoring existing analysis for ${project.name}, last result was empty") - LastSuccessfulResult - .empty(inputs.project) - // Replace classes dir, counter and populating with values from previous for correctness - .copy( - classesDir = result.classesDir, - counterForClassesDir = result.counterForClassesDir, - populatingProducts = result.populatingProducts - ) - } else { - logger.debug( - s"Using successful result for ${project.name} associated with ${result.classesDir}" - ) - result - } - } + compile: SuccessfulCompileBundle => CompileTraversal + ): RunningCompilation = { + import inputs.project + import bundle.logger + import logger.debug + + var counterForUsedClassesDir: AtomicInt = null + + def initializeLastSuccessful(previousOrNull: LastSuccessfulResult): LastSuccessfulResult = { + val result = Option(previousOrNull).getOrElse(bundle.lastSuccessful) + if (!result.classesDir.exists) { + debug(s"Ignoring analysis for ${project.name}, directory ${result.classesDir} is missing") + LastSuccessfulResult.empty(inputs.project) + } else if (bundle.latestResult == Compiler.Result.Empty) { + debug(s"Ignoring existing analysis for ${project.name}, last result was empty") + LastSuccessfulResult + .empty(inputs.project) + // Replace classes dir, counter and populating with values from previous for correctness + .copy( + classesDir = result.classesDir, + counterForClassesDir = result.counterForClassesDir, + populatingProducts = result.populatingProducts + ) + } else { + debug(s"Using successful result for ${project.name} associated with ${result.classesDir}") + result + } + } - def getMostRecentSuccessfulResultAtomically = - tracer.trace("get most recent successful result atomically") { _ => - lastSuccessfulResults.compute( - project.uniqueId, - (_: String, previousResultOrNull: LastSuccessfulResult) => { - logger.debug( - s"Return previous result or the initial last successful coming from the bundle:${project.uniqueId}" - ) - // Return previous result or the initial last successful coming from the bundle - val previousResult = initializeLastSuccessful(previousResultOrNull) - - currentlyUsedClassesDirs.compute( - previousResult.classesDir, - (_: AbsolutePath, counter: AtomicInt) => { - logger.debug( - s"Set counter for used classes dir when init or incrementing:${previousResult.classesDir}" - ) - // Set counter for used classes dir when init or incrementing - if (counter == null) { - logger.debug(s"Create new counter:${previousResult.classesDir}") - val initialCounter = AtomicInt(1) - counterForUsedClassesDir = initialCounter - initialCounter - } else { - counterForUsedClassesDir = counter - val newCount = counter.incrementAndGet(1) - logger.debug( - s"Increasing counter for ${previousResult.classesDir} to $newCount" - ) - counter - } - } - ) - - previousResult.copy(counterForClassesDir = counterForUsedClassesDir) + def getMostRecentSuccessfulResultAtomically = { + lastSuccessfulResults.compute( + project.uniqueId, + (_: String, previousResultOrNull: LastSuccessfulResult) => { + // Return previous result or the initial last successful coming from the bundle + val previousResult = initializeLastSuccessful(previousResultOrNull) + + currentlyUsedClassesDirs.compute( + previousResult.classesDir, + (_: AbsolutePath, counter: AtomicInt) => { + // Set counter for used classes dir when init or incrementing + if (counter == null) { + val initialCounter = AtomicInt(1) + counterForUsedClassesDir = initialCounter + initialCounter + } else { + counterForUsedClassesDir = counter + val newCount = counter.incrementAndGet(1) + logger.debug(s"Increasing counter for ${previousResult.classesDir} to $newCount") + counter + } } ) - } - - logger.debug(s"Scheduling compilation for ${project.name}...") - - // Replace client-specific last successful with the most recent result - val mostRecentSuccessful = getMostRecentSuccessfulResultAtomically - - val isUnsubscribed = AtomicBoolean(false) - val newBundle = bundle.copy(lastSuccessful = mostRecentSuccessful) - val compileAndUnsubscribe = tracer.trace("compile and unsubscribe") { _ => - compile(newBundle) - .doOnFinish(_ => Task(logger.observer.onComplete())) - .map { result => - // Unregister deduplication atomically and register last successful if any - tracer.trace("process result atomically") { _ => - processResultAtomically( - result, - project, - bundle.uniqueInputs, - isUnsubscribed, - logger, - tracer - ) - } // Without memoization, there is no deduplication - } - .memoize - } - RunningCompilation( - compileAndUnsubscribe, - mostRecentSuccessful, - isUnsubscribed, - bundle.mirror, - client + previousResult.copy(counterForClassesDir = counterForUsedClassesDir) + } ) } + logger.debug(s"Scheduling compilation for ${project.name}...") + + // Replace client-specific last successful with the most recent result + val mostRecentSuccessful = getMostRecentSuccessfulResultAtomically + + val isUnsubscribed = AtomicBoolean(false) + val newBundle = bundle.copy(lastSuccessful = mostRecentSuccessful) + val compileAndUnsubscribe = { + compile(newBundle) + .doOnFinish(_ => Task(logger.observer.onComplete())) + .map { result => + // Unregister deduplication atomically and register last successful if any + processResultAtomically( + result, + project, + bundle.uniqueInputs, + isUnsubscribed, + logger + ) + } + .memoize // Without memoization, there is no deduplication + } + + RunningCompilation( + compileAndUnsubscribe, + mostRecentSuccessful, + isUnsubscribed, + bundle.mirror, + client + ) + } + private def processResultAtomically( resultDag: Dag[PartialCompileResult], project: Project, oinputs: UniqueCompileInputs, isAlreadyUnsubscribed: AtomicBoolean, - logger: Logger, - tracer: BraveTracer + logger: Logger ): Dag[PartialCompileResult] = { - def cleanUpAfterCompilationError[T](result: T): T = - tracer.trace("cleaning after compilation error") { _ => - if (!isAlreadyUnsubscribed.get) { - // Remove running compilation if host compilation hasn't unsubscribed (maybe it's blocked) - logger.debug( - s"Remove running compilation if host compilation hasn't unsubscribed (maybe it's blocked):${oinputs}" - ) - runningCompilations.remove(oinputs) - } - - result + def cleanUpAfterCompilationError[T](result: T): T = { + if (!isAlreadyUnsubscribed.get) { + // Remove running compilation if host compilation hasn't unsubscribed (maybe it's blocked) + runningCompilations.remove(oinputs) } + result + } + // Unregister deduplication atomically and register last successful if any PartialCompileResult.mapEveryResult(resultDag) { case s: PartialSuccess => val processedResult = s.result.map { (result: ResultBundle) => result.successful match { - case None => - tracer.trace("cleaning after compilation error") { _ => - cleanUpAfterCompilationError(result) - } + case None => cleanUpAfterCompilationError(result) case Some(res) => - tracer.trace("unregister deduplication and register successful") { _ => - unregisterDeduplicationAndRegisterSuccessful(project, oinputs, res, logger) - } + unregisterDeduplicationAndRegisterSuccessful(project, oinputs, res, logger) } result } @@ -287,10 +225,7 @@ object CompileGatekeeper { */ s.copy(result = processedResult.memoize) - case result => - tracer.trace("cleaning after compilation error") { _ => - cleanUpAfterCompilationError(result) - } + case result => cleanUpAfterCompilationError(result) } } @@ -309,7 +244,6 @@ object CompileGatekeeper { runningCompilations.compute( oracleInputs, (_: UniqueCompileInputs, _: RunningCompilation) => { - logger.debug("Unregister deduplication and registered successfully") lastSuccessfulResults.compute(project.uniqueId, (_, _) => successful) null } diff --git a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGatekeeperNew.scala b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGatekeeperNew.scala deleted file mode 100644 index 95e134fe34..0000000000 --- a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGatekeeperNew.scala +++ /dev/null @@ -1,288 +0,0 @@ -package bloop.engine.tasks.compilation - -import bloop.Compiler -import bloop.UniqueCompileInputs -import bloop.data.ClientInfo -import bloop.data.Project -import bloop.engine.Dag -import bloop.engine.caches.LastSuccessfulResult -import bloop.io.AbsolutePath -import bloop.logging.DebugFilter -import bloop.logging.Logger -import bloop.logging.LoggerAction -import bloop.reporter.ReporterAction -import bloop.task.Task -import monix.eval.{Task => MonixTask} -import cats.effect.concurrent.{Deferred, Ref} -import monix.execution.atomic.AtomicBoolean -import monix.execution.atomic.AtomicInt -import monix.reactive.Observable - -object CompileGatekeeperNew { - private implicit val filter: DebugFilter = DebugFilter.Compilation - import bloop.engine.tasks.compilation.CompileDefinitions._ - - private[bloop] final case class RunningCompilation( - traversal: CompileTraversal, - usedLastSuccessful: LastSuccessfulResult, - isUnsubscribed: AtomicBoolean, - mirror: Observable[Either[ReporterAction, LoggerAction]], - client: ClientInfo - ) - - /* -------------------------------------------------------------------------------------------- */ - private val runningCompilations - : Ref[MonixTask, Map[UniqueCompileInputs, Deferred[MonixTask, RunningCompilation]]] = - Ref.unsafe(Map.empty) - private val currentlyUsedClassesDirs: Ref[MonixTask, Map[AbsolutePath, AtomicInt]] = - Ref.unsafe(Map.empty) - private val lastSuccessfulResults: Ref[MonixTask, Map[ProjectId, LastSuccessfulResult]] = - Ref.unsafe(Map.empty) - - /* -------------------------------------------------------------------------------------------- */ - - def findRunningCompilationAtomically( - inputs: BundleInputs, - bundle: SuccessfulCompileBundle, - client: ClientInfo, - compile: SuccessfulCompileBundle => CompileTraversal - ): Task[(RunningCompilation, CanBeDeduplicated)] = Task.liftMonixTaskUncancellable { - Deferred[MonixTask, RunningCompilation].flatMap { deferred => - runningCompilations.modify { state => - state.get(bundle.uniqueInputs) match { - case Some(existingDeferred) => - val output = - existingDeferred.get - .flatMap { running => - val usedClassesDir = running.usedLastSuccessful.classesDir - val usedClassesDirCounter = running.usedLastSuccessful.counterForClassesDir - val deduplicate = usedClassesDirCounter.transformAndExtract { - case count if count == 0 => (false -> count) - case count => true -> (count + 1) - } - if (deduplicate) MonixTask.now((running, deduplicate)) - else { - val classesDirs = - currentlyUsedClassesDirs - .update { classesDirs => - if ( - classesDirs - .get(usedClassesDir) - .contains(usedClassesDirCounter) - ) - classesDirs - usedClassesDir - else - classesDirs - } - scheduleCompilation(inputs, bundle, client, compile) - .flatMap(compilation => - deferred - .complete(compilation) - .flatMap(_ => classesDirs.map(_ => (compilation, false))) - ) - } - } - state -> output - case None => - val newState: Map[UniqueCompileInputs, Deferred[MonixTask, RunningCompilation]] = - state + (bundle.uniqueInputs -> deferred) - newState -> scheduleCompilation(inputs, bundle, client, compile).flatMap(compilation => - deferred.complete(compilation).map(_ => compilation -> false) - ) - } - }.flatten - } - } - - def disconnectDeduplicationFromRunning( - inputs: UniqueCompileInputs, - runningCompilation: RunningCompilation - ): MonixTask[Unit] = { - runningCompilation.isUnsubscribed.compareAndSet(false, true) - runningCompilations.modify { state => - val updated = - if ( - state.contains(inputs) - ) // .contains(runningCompilationDeferred)) // TODO: no way to verfiy if it is the same - state - inputs - else state - (updated, ()) - } - } - - /** - * Schedules a unique compilation for the given inputs. - * - * This compilation can be deduplicated by other clients that have the same - * inputs. The call-site ensures that only one compilation can exist for the - * same inputs for a period of time. - */ - def scheduleCompilation( - inputs: BundleInputs, - bundle: SuccessfulCompileBundle, - client: ClientInfo, - compile: SuccessfulCompileBundle => CompileTraversal - ): MonixTask[RunningCompilation] = { - import inputs.project - import bundle.logger - import logger.debug - - def initializeLastSuccessful( - maybePreviousResult: Option[LastSuccessfulResult] - ): LastSuccessfulResult = { - val result = maybePreviousResult.getOrElse(bundle.lastSuccessful) - if (!result.classesDir.exists) { - debug(s"Ignoring analysis for ${project.name}, directory ${result.classesDir} is missing") - LastSuccessfulResult.empty(inputs.project) - } else if (bundle.latestResult == Compiler.Result.Empty) { - debug(s"Ignoring existing analysis for ${project.name}, last result was empty") - LastSuccessfulResult - .empty(inputs.project) - // Replace classes dir, counter and populating with values from previous for correctness - .copy( - classesDir = result.classesDir, - counterForClassesDir = result.counterForClassesDir, - populatingProducts = result.populatingProducts - ) - } else { - debug(s"Using successful result for ${project.name} associated with ${result.classesDir}") - result - } - } - - def getMostRecentSuccessfulResultAtomically: MonixTask[LastSuccessfulResult] = { - lastSuccessfulResults.modify { state => - val previousResult = - initializeLastSuccessful(state.get(project.uniqueId)) - state -> currentlyUsedClassesDirs - .modify { counters => - counters.get(previousResult.classesDir) match { - case None => - val initialCounter = AtomicInt(1) - (counters + (previousResult.classesDir -> initialCounter), initialCounter) - case Some(counter) => - val newCount = counter.incrementAndGet(1) - logger.debug(s"Increasing counter for ${previousResult.classesDir} to $newCount") - counters -> counter - } - } - .map(_ => previousResult) - }.flatten - } - - logger.debug(s"Scheduling compilation for ${project.name}...") - - getMostRecentSuccessfulResultAtomically - .map { mostRecentSuccessful => - val isUnsubscribed = AtomicBoolean(false) - val newBundle = bundle.copy(lastSuccessful = mostRecentSuccessful) - val compileAndUnsubscribe = compile(newBundle) - .doOnFinish(_ => Task(logger.observer.onComplete())) - .flatMap { result => - // Unregister deduplication atomically and register last successful if any - processResultAtomically( - result, - project, - bundle.uniqueInputs, - isUnsubscribed, - logger - ) - } - .memoize - - RunningCompilation( - compileAndUnsubscribe, - mostRecentSuccessful, - isUnsubscribed, - bundle.mirror, - client - ) // Without memoization, there is no deduplication - } - - } - - private def processResultAtomically( - resultDag: Dag[PartialCompileResult], - project: Project, - oinputs: UniqueCompileInputs, - isAlreadyUnsubscribed: AtomicBoolean, - logger: Logger - ): Task[Dag[PartialCompileResult]] = { - - def cleanUpAfterCompilationError[T](result: T): Task[T] = { - Task { - if (!isAlreadyUnsubscribed.get) { - // Remove running compilation if host compilation hasn't unsubscribed (maybe it's blocked) - Task.liftMonixTaskUncancellable { - runningCompilations.update(_ - oinputs) - } - } else - Task.unit - }.flatten.map(_ => result) - } - - // Unregister deduplication atomically and register last successful if any - PartialCompileResult.mapEveryResultTask(resultDag) { - case s: PartialSuccess => - val processedResult = s.result.flatMap { (result: ResultBundle) => - result.successful - .fold(cleanUpAfterCompilationError(result)) { res => - unregisterDeduplicationAndRegisterSuccessful( - project, - oinputs, - res, - logger - ) - .map(_ => result) - } - } - - /** - * This result task must only be run once and thus needs to be - * memoized for correctness reasons. The result task can be called - * several times by the compilation engine driving the execution. - */ - Task(s.copy(result = processedResult.memoize)) - - case result => cleanUpAfterCompilationError(result) - } - } - - /** - * Removes the deduplication and registers the last successful compilation - * atomically. When registering the last successful compilation, we make sure - * that the old last successful result is deleted if its count is 0, which - * means it's not being used by anyone. - */ - private def unregisterDeduplicationAndRegisterSuccessful( - project: Project, - oracleInputs: UniqueCompileInputs, - successful: LastSuccessfulResult, - logger: Logger - ): Task[Unit] = Task.liftMonixTaskUncancellable { - runningCompilations - .modify { state => - val newSuccessfulResults = (project.uniqueId, successful) - if (state.contains(oracleInputs)) { - val newState = state - oracleInputs - (newState, lastSuccessfulResults.update { _ + newSuccessfulResults }) - } else { - (state, MonixTask.unit) - } - } - .flatten - .map { _ => - logger.debug( - s"Recording new last successful request for ${project.name} associated with ${successful.classesDir}" - ) - () - } - } - - // Expose clearing mechanism so that it can be invoked in the tests and community build runner -// private[bloop] def clearSuccessfulResults(): Unit = { -// lastSuccessfulResults.synchronized { -// lastSuccessfulResults.clear() -// } -// } -} diff --git a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGraph.scala b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGraph.scala index 668dc6ffe7..1aab0af1ef 100644 --- a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGraph.scala +++ b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGraph.scala @@ -4,10 +4,13 @@ package bloop.engine.tasks.compilation import java.io.File import java.io.PrintWriter import java.io.StringWriter + import scala.util.Failure import scala.util.Success + import ch.epfl.scala.bsp.StatusCode import ch.epfl.scala.bsp.{StatusCode => BspStatusCode} + import bloop.CompileBackgroundTasks import bloop.CompileExceptions.BlockURI import bloop.CompileExceptions.FailedOrCancelledPromise @@ -24,10 +27,10 @@ import bloop.logging.DebugFilter import bloop.logging.LoggerAction import bloop.reporter.ReporterAction import bloop.task.Task -import bloop.tracing.BraveTracer import bloop.util.BestEffortUtils.BestEffortProducts import bloop.util.JavaCompat.EnrichOptional import bloop.util.SystemProperties + import xsbti.compile.PreviousResult object CompileGraph { @@ -91,11 +94,10 @@ object CompileGraph { def setupAndDeduplicate( client: ClientInfo, inputs: BundleInputs, - setup: (BundleInputs, BraveTracer) => Task[CompileBundle], - tracer: BraveTracer + setup: BundleInputs => Task[CompileBundle] )( compile: SuccessfulCompileBundle => CompileTraversal - ): CompileTraversal = tracer.trace("setupAndDeduplicate") { tracer => + ): CompileTraversal = { def partialFailure( errorMsg: String, err: Option[Throwable] @@ -108,8 +110,8 @@ object CompileGraph { } implicit val filter = DebugFilter.Compilation - def withBundle(f: SuccessfulCompileBundle => CompileTraversal): CompileTraversal = tracer.trace("withBundle") { tracer => - setup(inputs, tracer).materialize.flatMap { + def withBundle(f: SuccessfulCompileBundle => CompileTraversal): CompileTraversal = { + setup(inputs).materialize.flatMap { case Success(bundle: SuccessfulCompileBundle) => f(bundle).materialize.flatMap { case Success(result) => Task.now(result) @@ -132,232 +134,220 @@ object CompileGraph { withBundle { bundle0 => val logger = bundle0.logger val (runningCompilation, deduplicate) = - CompileGatekeeper.findRunningCompilationAtomically(inputs, bundle0, client, compile, tracer) + CompileGatekeeper.findRunningCompilationAtomically(inputs, bundle0, client, compile) val bundle = bundle0.copy(lastSuccessful = runningCompilation.usedLastSuccessful) if (!deduplicate) { - tracer.trace("traversing compilation") { tracer => - runningCompilation.traversal - } + runningCompilation.traversal } else { - tracer.trace("deduplication required") { tracer => - val rawLogger = logger.underlying - rawLogger.info( - s"Deduplicating compilation of ${bundle.project.name} from ${runningCompilation.client}" - ) - val reporter = bundle.reporter.underlying - // Don't use `bundle.lastSuccessful`, it's not the final input to `compile` - val analysis = runningCompilation.usedLastSuccessful.previous.analysis().toOption - val previousSuccessfulProblems = - Compiler.previousProblemsFromSuccessfulCompilation(analysis) - val wasPreviousSuccessful = bundle.latestResult match { - case Compiler.Result.Ok(_) => true - case _ => false + val rawLogger = logger.underlying + rawLogger.info( + s"Deduplicating compilation of ${bundle.project.name} from ${runningCompilation.client}" + ) + val reporter = bundle.reporter.underlying + // Don't use `bundle.lastSuccessful`, it's not the final input to `compile` + val analysis = runningCompilation.usedLastSuccessful.previous.analysis().toOption + val previousSuccessfulProblems = + Compiler.previousProblemsFromSuccessfulCompilation(analysis) + val wasPreviousSuccessful = bundle.latestResult match { + case Compiler.Result.Ok(_) => true + case _ => false + } + val previousProblems = + Compiler.previousProblemsFromResult(bundle.latestResult, previousSuccessfulProblems) + + val clientClassesObserver = client.getClassesObserverFor(bundle.project) + + // Replay events asynchronously to waiting for the compilation result + import scala.concurrent.duration.FiniteDuration + import monix.execution.exceptions.UpstreamTimeoutException + val disconnectionTime = SystemProperties.getCompileDisconnectionTime(rawLogger) + val replayEventsTask = runningCompilation.mirror + .timeoutOnSlowUpstream(disconnectionTime) + .foreachL { + case Left(action) => + action match { + case ReporterAction.EnableFatalWarnings => + reporter.enableFatalWarnings() + case ReporterAction.ReportStartCompilation => + reporter.reportStartCompilation(previousProblems, wasPreviousSuccessful) + case a: ReporterAction.ReportStartIncrementalCycle => + reporter.reportStartIncrementalCycle(a.sources, a.outputDirs) + case a: ReporterAction.ReportProblem => reporter.log(a.problem) + case ReporterAction.PublishDiagnosticsSummary => + reporter.printSummary() + case a: ReporterAction.ReportNextPhase => + reporter.reportNextPhase(a.phase, a.sourceFile) + case a: ReporterAction.ReportCompilationProgress => + reporter.reportCompilationProgress(a.progress, a.total) + case a: ReporterAction.ReportEndIncrementalCycle => + reporter.reportEndIncrementalCycle(a.durationMs, a.result) + case ReporterAction.ReportCancelledCompilation => + reporter.reportCancelledCompilation() + case a: ReporterAction.ProcessEndCompilation => + a.code match { + case BspStatusCode.Cancelled | BspStatusCode.Error => + reporter.processEndCompilation(previousProblems, a.code, None, None) + reporter.reportEndCompilation() + case _ => + /* + * Only process the end, don't report it. It's only safe to + * report when all the client tasks have been run and the + * analysis/classes dirs are fully populated so that clients + * can use `taskFinish` notifications as a signal to process them. + */ + reporter.processEndCompilation( + previousProblems, + a.code, + Some(clientClassesObserver.classesDir), + Some(bundle.out.analysisOut) + ) + } + } + case Right(action) => + action match { + case LoggerAction.LogErrorMessage(msg) => rawLogger.error(msg) + case LoggerAction.LogWarnMessage(msg) => rawLogger.warn(msg) + case LoggerAction.LogInfoMessage(msg) => rawLogger.info(msg) + case LoggerAction.LogDebugMessage(msg) => + rawLogger.debug(msg) + case LoggerAction.LogTraceMessage(msg) => + rawLogger.debug(msg) + } + } + .materialize + .map { + case Success(_) => DeduplicationResult.Ok + case Failure(_: UpstreamTimeoutException) => + DeduplicationResult.DisconnectFromDeduplication + case Failure(t) => DeduplicationResult.DeduplicationError(t) } - val previousProblems = - Compiler.previousProblemsFromResult(bundle.latestResult, previousSuccessfulProblems) - - val clientClassesObserver = client.getClassesObserverFor(bundle.project) - - // Replay events asynchronously to waiting for the compilation result - import scala.concurrent.duration.FiniteDuration - import monix.execution.exceptions.UpstreamTimeoutException - val disconnectionTime = SystemProperties.getCompileDisconnectionTime(rawLogger) - val replayEventsTask = runningCompilation.mirror - .timeoutOnSlowUpstream(disconnectionTime) - .foreachL { - case Left(action) => - action match { - case ReporterAction.EnableFatalWarnings => - reporter.enableFatalWarnings() - case ReporterAction.ReportStartCompilation => - reporter.reportStartCompilation(previousProblems, wasPreviousSuccessful) - case a: ReporterAction.ReportStartIncrementalCycle => - reporter.reportStartIncrementalCycle(a.sources, a.outputDirs) - case a: ReporterAction.ReportProblem => reporter.log(a.problem) - case ReporterAction.PublishDiagnosticsSummary => - reporter.printSummary() - case a: ReporterAction.ReportNextPhase => - reporter.reportNextPhase(a.phase, a.sourceFile) - case a: ReporterAction.ReportCompilationProgress => - reporter.reportCompilationProgress(a.progress, a.total) - case a: ReporterAction.ReportEndIncrementalCycle => - reporter.reportEndIncrementalCycle(a.durationMs, a.result) - case ReporterAction.ReportCancelledCompilation => - reporter.reportCancelledCompilation() - case a: ReporterAction.ProcessEndCompilation => - a.code match { - case BspStatusCode.Cancelled | BspStatusCode.Error => - reporter.processEndCompilation(previousProblems, a.code, None, None) - reporter.reportEndCompilation() - case _ => - /* - * Only process the end, don't report it. It's only safe to - * report when all the client tasks have been run and the - * analysis/classes dirs are fully populated so that clients - * can use `taskFinish` notifications as a signal to process them. - */ - reporter.processEndCompilation( - previousProblems, - a.code, - Some(clientClassesObserver.classesDir), - Some(bundle.out.analysisOut) - ) - } - } - case Right(action) => - action match { - case LoggerAction.LogErrorMessage(msg) => rawLogger.error(msg) - case LoggerAction.LogWarnMessage(msg) => rawLogger.warn(msg) - case LoggerAction.LogInfoMessage(msg) => rawLogger.info(msg) - case LoggerAction.LogDebugMessage(msg) => - rawLogger.debug(msg) - case LoggerAction.LogTraceMessage(msg) => - rawLogger.debug(msg) - } - } - .materialize - .map { - case Success(_) => DeduplicationResult.Ok - case Failure(_: UpstreamTimeoutException) => - DeduplicationResult.DisconnectFromDeduplication - case Failure(t) => DeduplicationResult.DeduplicationError(t) - } - /* The task set up by another process whose memoized result we're going to - * reuse. To prevent blocking compilations, we execute this task (which will - * block until its completion is done) in the IO thread pool, which is - * unbounded. This makes sure that the blocking threads *never* block - * the computation pool, which could produce a hang in the build server. - */ - val runningCompilationTask = - runningCompilation.traversal.executeOn(ExecutionContext.ioScheduler) - - val deduplicateStreamSideEffectsHandle = - replayEventsTask.runToFuture(ExecutionContext.ioScheduler) - - /** - * Deduplicate and change the implementation of the task returning the - * deduplicate compiler result to trigger a syncing process to keep the - * client external classes directory up-to-date with the new classes - * directory. This copying process blocks until the background IO work - * of the deduplicated compilation result has been finished. Note that - * this mechanism allows pipelined compilations to perform this IO only - * when the full compilation of a module is finished. - */ - val obtainResultFromDeduplication = runningCompilationTask.map { results => - PartialCompileResult.mapEveryResult(results) { - case s @ PartialSuccess(bundle, compilerResult) => - val newCompilerResult = compilerResult.flatMap { results => - results.fromCompiler match { - case s: Compiler.Result.Success => - // Wait on new classes to be populated for correctness - val runningBackgroundTasks = s.backgroundTasks - .trigger(clientClassesObserver, reporter, tracer, logger) - .runAsync(ExecutionContext.ioScheduler) - Task.now(results.copy(runningBackgroundTasks = runningBackgroundTasks)) - case _: Compiler.Result.Cancelled => - // Make sure to cancel the deduplicating task if compilation is cancelled - deduplicateStreamSideEffectsHandle.cancel() - Task.now(results) - case _ => Task.now(results) - } + /* The task set up by another process whose memoized result we're going to + * reuse. To prevent blocking compilations, we execute this task (which will + * block until its completion is done) in the IO thread pool, which is + * unbounded. This makes sure that the blocking threads *never* block + * the computation pool, which could produce a hang in the build server. + */ + val runningCompilationTask = + runningCompilation.traversal.executeOn(ExecutionContext.ioScheduler) + + val deduplicateStreamSideEffectsHandle = + replayEventsTask.runToFuture(ExecutionContext.ioScheduler) + + /** + * Deduplicate and change the implementation of the task returning the + * deduplicate compiler result to trigger a syncing process to keep the + * client external classes directory up-to-date with the new classes + * directory. This copying process blocks until the background IO work + * of the deduplicated compilation result has been finished. Note that + * this mechanism allows pipelined compilations to perform this IO only + * when the full compilation of a module is finished. + */ + val obtainResultFromDeduplication = runningCompilationTask.map { results => + PartialCompileResult.mapEveryResult(results) { + case s @ PartialSuccess(bundle, compilerResult) => + val newCompilerResult = compilerResult.flatMap { results => + results.fromCompiler match { + case s: Compiler.Result.Success => + // Wait on new classes to be populated for correctness + val runningBackgroundTasks = s.backgroundTasks + .trigger(clientClassesObserver, reporter, bundle.tracer, logger) + .runAsync(ExecutionContext.ioScheduler) + Task.now(results.copy(runningBackgroundTasks = runningBackgroundTasks)) + case _: Compiler.Result.Cancelled => + // Make sure to cancel the deduplicating task if compilation is cancelled + deduplicateStreamSideEffectsHandle.cancel() + Task.now(results) + case _ => Task.now(results) } - s.copy(result = newCompilerResult) - case result => result - } + } + s.copy(result = newCompilerResult) + case result => result } + } - val compileAndDeduplicate = Task - .chooseFirstOf( - tracer.trace("obtainResultFromDeduplication - obtainResultFromDeduplication") { _ => - obtainResultFromDeduplication - }, - tracer.trace("obtainResultFromDeduplication - deduplicateStreamSideEffectsHandle") { _ => - Task.fromFuture(deduplicateStreamSideEffectsHandle) - } - ) - .executeOn(ExecutionContext.ioScheduler) - - val finalCompileTask = compileAndDeduplicate.flatMap { - case Left((result, deduplicationFuture)) => - Task.fromFuture(deduplicationFuture).map(_ => result) - case Right((compilationFuture, deduplicationResult)) => - deduplicationResult match { - case DeduplicationResult.Ok => Task.fromFuture(compilationFuture) - case DeduplicationResult.DeduplicationError(t) => - rawLogger.trace(t) - val failedDeduplicationResult = Compiler.Result.GlobalError( - s"Unexpected error while deduplicating compilation for ${inputs.project.name}: ${t.getMessage}", - Some(t) - ) - - /* - * When an error happens while replaying all events of the - * deduplicated compilation, we keep track of the error, wait - * until the deduplicated compilation finishes and then we - * replace the result by a failed result that informs the - * client compilation was not successfully deduplicated. - */ - Task.fromFuture(compilationFuture).map { results => - PartialCompileResult.mapEveryResult(results) { (p: PartialCompileResult) => - p match { - case s: PartialSuccess => - val failedBundle = ResultBundle(failedDeduplicationResult, None, None) - s.copy(result = s.result.map(_ => failedBundle)) - case result => result - } + val compileAndDeduplicate = Task + .chooseFirstOf( + obtainResultFromDeduplication, + Task.fromFuture(deduplicateStreamSideEffectsHandle) + ) + .executeOn(ExecutionContext.ioScheduler) + + val finalCompileTask = compileAndDeduplicate.flatMap { + case Left((result, deduplicationFuture)) => + Task.fromFuture(deduplicationFuture).map(_ => result) + case Right((compilationFuture, deduplicationResult)) => + deduplicationResult match { + case DeduplicationResult.Ok => Task.fromFuture(compilationFuture) + case DeduplicationResult.DeduplicationError(t) => + rawLogger.trace(t) + val failedDeduplicationResult = Compiler.Result.GlobalError( + s"Unexpected error while deduplicating compilation for ${inputs.project.name}: ${t.getMessage}", + Some(t) + ) + + /* + * When an error happens while replaying all events of the + * deduplicated compilation, we keep track of the error, wait + * until the deduplicated compilation finishes and then we + * replace the result by a failed result that informs the + * client compilation was not successfully deduplicated. + */ + Task.fromFuture(compilationFuture).map { results => + PartialCompileResult.mapEveryResult(results) { (p: PartialCompileResult) => + p match { + case s: PartialSuccess => + val failedBundle = ResultBundle(failedDeduplicationResult, None, None) + s.copy(result = s.result.map(_ => failedBundle)) + case result => result } } + } - case DeduplicationResult.DisconnectFromDeduplication => - /* - * Deduplication timed out after no compilation updates were - * recorded. In theory, this could happen because a rogue - * compilation process has stalled or is blocked. To ensure - * deduplicated clients always make progress, we now proceed - * with: - * + case DeduplicationResult.DisconnectFromDeduplication => + /* + * Deduplication timed out after no compilation updates were + * recorded. In theory, this could happen because a rogue + * compilation process has stalled or is blocked. To ensure + * deduplicated clients always make progress, we now proceed + * with: + * * 1. Cancelling the dead-looking compilation, hoping that the - * process will wake up at some point and stop running. - * 2. Shutting down the deduplication and triggering a new - * compilation. If there are several clients deduplicating this - * compilation, they will compete to start the compilation again - * with new compile inputs, as they could have already changed. - * 3. Reporting the end of compilation in case it hasn't been - * reported. Clients must handle two end compilation notifications - * gracefully. - * 4. Display the user that the deduplication was cancelled and a - * new compilation was scheduled. - */ - - CompileGatekeeper.disconnectDeduplicationFromRunning( - bundle.uniqueInputs, - runningCompilation, - logger, - tracer - ) - - compilationFuture.cancel() - reporter.processEndCompilation(Nil, StatusCode.Cancelled, None, None) - reporter.reportEndCompilation() - - logger.displayWarningToUser( - s"""Disconnecting from deduplication of ongoing compilation for '${inputs.project.name}' - |No progress update for ${(disconnectionTime: FiniteDuration) - .toString()} caused bloop to cancel compilation and schedule a new compile. + * process will wake up at some point and stop running. + * 2. Shutting down the deduplication and triggering a new + * compilation. If there are several clients deduplicating this + * compilation, they will compete to start the compilation again + * with new compile inputs, as they could have already changed. + * 3. Reporting the end of compilation in case it hasn't been + * reported. Clients must handle two end compilation notifications + * gracefully. + * 4. Display the user that the deduplication was cancelled and a + * new compilation was scheduled. + */ + + CompileGatekeeper.disconnectDeduplicationFromRunning( + bundle.uniqueInputs, + runningCompilation + ) + + compilationFuture.cancel() + reporter.processEndCompilation(Nil, StatusCode.Cancelled, None, None) + reporter.reportEndCompilation() + + logger.displayWarningToUser( + s"""Disconnecting from deduplication of ongoing compilation for '${inputs.project.name}' + |No progress update for ${(disconnectionTime: FiniteDuration) + .toString()} caused bloop to cancel compilation and schedule a new compile. """.stripMargin - ) + ) - tracer.trace("setupAndDeduplicate - disconnected from deduplication") { tracer => - setupAndDeduplicate(client, inputs, setup, tracer)(compile) - } - } - } + setupAndDeduplicate(client, inputs, setup)(compile) + } + } - tracer.traceTask(s"deduplicating ${bundle.project.name}") { _ => - finalCompileTask.executeOn(ExecutionContext.ioScheduler) - } + bundle.tracer.traceTask(s"deduplicating ${bundle.project.name}") { _ => + finalCompileTask.executeOn(ExecutionContext.ioScheduler) } } } @@ -378,128 +368,121 @@ object CompileGraph { client: ClientInfo, store: CompileClientStore, bestEffortAllowed: Boolean, - computeBundle: (BundleInputs, BraveTracer) => Task[CompileBundle], - compile: (Inputs, Boolean, Boolean, BraveTracer) => Task[ResultBundle], - tracer: BraveTracer - ): CompileTraversal = - tracer.trace("traversing ") { _ => - val tasks = new mutable.HashMap[Dag[Project], CompileTraversal]() - def register(k: Dag[Project], v: CompileTraversal): CompileTraversal = { - val toCache = store.findPreviousTraversalOrAddNew(k, v).getOrElse(v) - tasks.put(k, toCache) - toCache - } + computeBundle: BundleInputs => Task[CompileBundle], + compile: (Inputs, Boolean, Boolean) => Task[ResultBundle] + ): CompileTraversal = { + val tasks = new mutable.HashMap[Dag[Project], CompileTraversal]() + def register(k: Dag[Project], v: CompileTraversal): CompileTraversal = { + val toCache = store.findPreviousTraversalOrAddNew(k, v).getOrElse(v) + tasks.put(k, toCache) + toCache + } - /* - * [[PartialCompileResult]] is our way to represent errors at the build graph - * so that we can block the compilation of downstream projects. As we have to - * abide by this contract because it's used by the pipeline traversal too, we - * turn an actual compiler failure into a partial failure with a dummy - * `FailPromise` exception that makes the partial result be recognized as error. - */ - def toPartialFailure(bundle: SuccessfulCompileBundle, results: ResultBundle): PartialFailure = { - PartialFailure(bundle.project, FailedOrCancelledPromise, Task.now(results)) - } + /* + * [[PartialCompileResult]] is our way to represent errors at the build graph + * so that we can block the compilation of downstream projects. As we have to + * abide by this contract because it's used by the pipeline traversal too, we + * turn an actual compiler failure into a partial failure with a dummy + * `FailPromise` exception that makes the partial result be recognized as error. + */ + def toPartialFailure(bundle: SuccessfulCompileBundle, results: ResultBundle): PartialFailure = { + PartialFailure(bundle.project, FailedOrCancelledPromise, Task.now(results)) + } - def loop(dag: Dag[Project]): CompileTraversal = { - tasks.get(dag) match { - case Some(task) => task - case None => - val task: Task[Dag[PartialCompileResult]] = dag match { - case Leaf(project) => - val bundleInputs = BundleInputs(project, dag, Map.empty) - tracer.trace("setupAndDeduplicate - leaf project") { tracer => - setupAndDeduplicate(client, bundleInputs, computeBundle, tracer) { bundle => - val isBestEffortDep = false - compile(Inputs(bundle, Map.empty), bestEffortAllowed && project.isBestEffort, isBestEffortDep, tracer).map { - results => - results.fromCompiler match { - case Compiler.Result.Ok(_) => Leaf(partialSuccess(bundle, results)) - case _ => Leaf(toPartialFailure(bundle, results)) - } - } + def loop(dag: Dag[Project]): CompileTraversal = { + tasks.get(dag) match { + case Some(task) => task + case None => + val task: Task[Dag[PartialCompileResult]] = dag match { + case Leaf(project) => + val bundleInputs = BundleInputs(project, dag, Map.empty) + setupAndDeduplicate(client, bundleInputs, computeBundle) { bundle => + val isBestEffortDep = false + compile(Inputs(bundle, Map.empty), bestEffortAllowed && project.isBestEffort, isBestEffortDep).map { results => + results.fromCompiler match { + case Compiler.Result.Ok(_) => Leaf(partialSuccess(bundle, results)) + case _ => Leaf(toPartialFailure(bundle, results)) } } + } - case Aggregate(dags) => - val downstream = dags.map(loop(_)) - Task.gatherUnordered(downstream).flatMap { dagResults => - Task.now(Parent(PartialEmpty, dagResults)) - } + case Aggregate(dags) => + val downstream = dags.map(loop(_)) + Task.gatherUnordered(downstream).flatMap { dagResults => + Task.now(Parent(PartialEmpty, dagResults)) + } - case Parent(project, dependencies) => - val downstream = dependencies.map(loop(_)) - Task.gatherUnordered(downstream).flatMap { dagResults => - val depsSupportBestEffort = - dependencies.map(Dag.dfs(_, mode = Dag.PreOrder)).flatten.forall(_.isBestEffort) - val failed = dagResults.flatMap(dag => blockedBy(dag).toList) - - val allResults = Task.gatherUnordered { - val transitive = dagResults.flatMap(Dag.dfs(_, mode = Dag.PreOrder)).distinct - transitive.flatMap { - case PartialSuccess(bundle, result) => Some(result.map(r => bundle.project -> r)) - case PartialFailure(project, _, result) => Some(result.map(r => project -> r)) - case _ => None - } + case Parent(project, dependencies) => + val downstream = dependencies.map(loop(_)) + Task.gatherUnordered(downstream).flatMap { dagResults => + val depsSupportBestEffort = + dependencies.map(Dag.dfs(_, mode = Dag.PreOrder)).flatten.forall(_.isBestEffort) + val failed = dagResults.flatMap(dag => blockedBy(dag).toList) + + val allResults = Task.gatherUnordered { + val transitive = dagResults.flatMap(Dag.dfs(_, mode = Dag.PreOrder)).distinct + transitive.flatMap { + case PartialSuccess(bundle, result) => Some(result.map(r => bundle.project -> r)) + case PartialFailure(project, _, result) => Some(result.map(r => project -> r)) + case _ => None } + } - allResults.flatMap { results => - val successfulBestEffort = !results.exists { - case (_, ResultBundle(f: Compiler.Result.Failed, _, _, _)) => f.bestEffortProducts.isEmpty - case _ => false + allResults.flatMap { results => + val successfulBestEffort = !results.exists { + case (_, ResultBundle(f: Compiler.Result.Failed, _, _, _)) => f.bestEffortProducts.isEmpty + case _ => false + } + val continue = bestEffortAllowed && depsSupportBestEffort && successfulBestEffort || failed.isEmpty + val dependsOnBestEffort = failed.nonEmpty && bestEffortAllowed && depsSupportBestEffort + + if (!continue) { + // Register the name of the projects we're blocked on (intransitively) + val blockedResult = Compiler.Result.Blocked(failed.map(_.name)) + val blocked = Task.now(ResultBundle(blockedResult, None, None)) + Task.now(Parent(PartialFailure(project, BlockURI, blocked), dagResults)) + } else { + val dependentProducts = new mutable.ListBuffer[(Project, BundleProducts)]() + val dependentResults = new mutable.ListBuffer[(File, PreviousResult)]() + results.foreach { + case (p, ResultBundle(s: Compiler.Result.Success, _, _, _)) => + val newProducts = s.products + dependentProducts.+=(p -> Right(newProducts)) + val newResult = newProducts.resultForDependentCompilationsInSameRun + dependentResults + .+=(newProducts.newClassesDir.toFile -> newResult) + .+=(newProducts.readOnlyClassesDir.toFile -> newResult) + case (p, ResultBundle(f: Compiler.Result.Failed, _, _, _)) => + f.bestEffortProducts.foreach { + case BestEffortProducts(products, _, _) => + dependentProducts += (p -> Right(products)) + } + case _ => () } - val continue = bestEffortAllowed && depsSupportBestEffort && successfulBestEffort || failed.isEmpty - val dependsOnBestEffort = failed.nonEmpty && bestEffortAllowed && depsSupportBestEffort - - if (!continue) { - // Register the name of the projects we're blocked on (intransitively) - val blockedResult = Compiler.Result.Blocked(failed.map(_.name)) - val blocked = Task.now(ResultBundle(blockedResult, None, None)) - Task.now(Parent(PartialFailure(project, BlockURI, blocked), dagResults)) - } else { - val dependentProducts = new mutable.ListBuffer[(Project, BundleProducts)]() - val dependentResults = new mutable.ListBuffer[(File, PreviousResult)]() - results.foreach { - case (p, ResultBundle(s: Compiler.Result.Success, _, _, _)) => - val newProducts = s.products - dependentProducts.+=(p -> Right(newProducts)) - val newResult = newProducts.resultForDependentCompilationsInSameRun - dependentResults - .+=(newProducts.newClassesDir.toFile -> newResult) - .+=(newProducts.readOnlyClassesDir.toFile -> newResult) - case (p, ResultBundle(f: Compiler.Result.Failed, _, _, _)) => - f.bestEffortProducts.foreach { - case BestEffortProducts(products, _, _) => - dependentProducts += (p -> Right(products)) - } - case _ => () - } - val resultsMap = dependentResults.toMap - val bundleInputs = BundleInputs(project, dag, dependentProducts.toMap) - tracer.trace("setupAndDeduplicate - parent project") { tracer => - setupAndDeduplicate(client, bundleInputs, computeBundle, tracer) { bundle => - val inputs = Inputs(bundle, resultsMap) - compile(inputs, bestEffortAllowed && project.isBestEffort, dependsOnBestEffort, tracer).map { results => - results.fromCompiler match { - case Compiler.Result.Ok(_) if failed.isEmpty => - Parent(partialSuccess(bundle, results), dagResults) - case _ => Parent(toPartialFailure(bundle, results), dagResults) - } - } + val resultsMap = dependentResults.toMap + val bundleInputs = BundleInputs(project, dag, dependentProducts.toMap) + setupAndDeduplicate(client, bundleInputs, computeBundle) { bundle => + val inputs = Inputs(bundle, resultsMap) + compile(inputs, bestEffortAllowed && project.isBestEffort, dependsOnBestEffort).map { results => + results.fromCompiler match { + case Compiler.Result.Ok(_) if failed.isEmpty => + Parent(partialSuccess(bundle, results), dagResults) + case _ => Parent(toPartialFailure(bundle, results), dagResults) } } } } } - } - register(dag, task.memoize) - } + } + } + register(dag, task.memoize) } - - loop(dag) } + loop(dag) + } + private def errorToString(err: Throwable): String = { val sw = new StringWriter() val pw = new PrintWriter(sw) diff --git a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileResult.scala b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileResult.scala index f99344b0f2..1ed33a9510 100644 --- a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileResult.scala +++ b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileResult.scala @@ -32,16 +32,6 @@ object PartialCompileResult { } } - def mapEveryResultTask( - results: Dag[PartialCompileResult] - )(f: PartialCompileResult => Task[PartialCompileResult]): Task[Dag[PartialCompileResult]] = { - results match { - case Leaf(result) => f(result).map(Leaf(_)) - case Parent(result, children) => f(result).map(Parent(_, children)) - case Aggregate(_) => sys.error("Unexpected aggregate node in compile result!") - } - } - /** * Turns a partial compile result to a full one. In the case of normal * compilation, this is an instant operation since the task returning the diff --git a/frontend/src/test/resources/source-generator.py b/frontend/src/test/resources/source-generator.py index a691c2dade..e7042e3db8 100644 --- a/frontend/src/test/resources/source-generator.py +++ b/frontend/src/test/resources/source-generator.py @@ -62,35 +62,3 @@ def main(output_dir, args): def random(): return 123 - -def random(): - return 123 - - -def random(): - return 123 - - -def random(): - return 123 - - -def random(): - return 123 - - -def random(): - return 123 - - -def random(): - return 123 - - -def random(): - return 123 - - -def random(): - return 123 - diff --git a/frontend/src/test/scala/bloop/DeduplicationSpec.scala b/frontend/src/test/scala/bloop/DeduplicationSpec.scala index 41dc161db0..c4015a7a1d 100644 --- a/frontend/src/test/scala/bloop/DeduplicationSpec.scala +++ b/frontend/src/test/scala/bloop/DeduplicationSpec.scala @@ -1034,7 +1034,7 @@ object DeduplicationSpec extends bloop.bsp.BspBaseSuite { } } - test("two concurrent clients deduplicate compilation and run at the same time") { + ignore("two concurrent clients deduplicate compilation and run at the same time") { val logger = new RecordingLogger(ansiCodesSupported = false) val logger1 = new RecordingLogger(ansiCodesSupported = false) val logger2 = new RecordingLogger(ansiCodesSupported = false) diff --git a/frontend/src/test/scala/bloop/MonixBaseCompileSpec.scala b/frontend/src/test/scala/bloop/MonixBaseCompileSpec.scala index 71c17c0b3f..3f8b82c169 100644 --- a/frontend/src/test/scala/bloop/MonixBaseCompileSpec.scala +++ b/frontend/src/test/scala/bloop/MonixBaseCompileSpec.scala @@ -23,22 +23,26 @@ abstract class MonixBaseCompileSpec extends bloop.testing.MonixBaseSuite { test("don't compile build in two concurrent CLI clients") { TestUtil.withinWorkspaceV2 { workspace => val sources = List( - """/main/scala/Faa.scala - |class Faa + """/main/scala/Foo.scala + |class Foo """.stripMargin ) val testOut = new ByteArrayOutputStream() val options = CommonOptions.default.copy(out = new PrintStream(testOut)) - val `A` = TestProject(workspace, "z", sources) + val `A` = TestProject(workspace, "a", sources) val configDir = TestProject.populateWorkspace(workspace, List(`A`)) val compileArgs = - Array("compile", "z", "--config-dir", configDir.syntax, "--verbose") + Array("compile", "a", "--config-dir", configDir.syntax) val compileAction = Cli.parse(compileArgs, options) - def runCompileAsync(activeSessions: Ref[Task, Map[Path, List[CliSession]]], postfix: String): Task[ExitStatus] = - Cli.run(compileAction, NoPool, activeSessions, Option(postfix)) + def runCompileAsync( + activeSessions: Ref[Task, Map[Path, List[CliSession]]] + ): Task[ExitStatus] = + Cli.run(compileAction, NoPool, activeSessions) for { activeSessions <- Ref.of[Task, Map[Path, List[CliSession]]](Map.empty) - _ <- Task.parSequenceUnordered(List(runCompileAsync(activeSessions, "left"), runCompileAsync(activeSessions, "right"))) + _ <- Task.parSequenceUnordered( + List(runCompileAsync(activeSessions), runCompileAsync(activeSessions)) + ) } yield { val actionsOutput = new String(testOut.toByteArray, StandardCharsets.UTF_8) def removeAsciiColorCodes(line: String): String = line.replaceAll("\u001B\\[[;\\d]*m", "") @@ -54,9 +58,9 @@ abstract class MonixBaseCompileSpec extends bloop.testing.MonixBaseSuite { try { assertNoDiff( processOutput(obtained), - s"""Compiling z (1 Scala source) - |Deduplicating compilation of z from cli client ??? (since ??? - |Compiling z (1 Scala source) + s"""Compiling a (1 Scala source) + |Deduplicating compilation of a from cli client ??? (since ??? + |Compiling a (1 Scala source) |$extraCompilationMessageOutput |""".stripMargin ) @@ -65,9 +69,9 @@ abstract class MonixBaseCompileSpec extends bloop.testing.MonixBaseSuite { assertNoDiff( processOutput(obtained), s""" - |Deduplicating compilation of z from cli client ??? (since ??? - |Compiling z (1 Scala source) - |Compiling z (1 Scala source) + |Deduplicating compilation of a from cli client ??? (since ??? + |Compiling a (1 Scala source) + |Compiling a (1 Scala source) |$extraCompilationMessageOutput |""".stripMargin ) diff --git a/frontend/src/test/scala/bloop/MonixCompile2Spec.scala b/frontend/src/test/scala/bloop/MonixCompile2Spec.scala deleted file mode 100644 index 02ad71ef6f..0000000000 --- a/frontend/src/test/scala/bloop/MonixCompile2Spec.scala +++ /dev/null @@ -1,5 +0,0 @@ -package bloop - -object MonixCompile2Spec extends MonixBaseCompileSpec { - override protected val TestProject = util.TestProject -} diff --git a/frontend/src/test/scala/bloop/dap/DebugServerSpec.scala b/frontend/src/test/scala/bloop/dap/DebugServerSpec.scala index 93dd5165c3..e162f5ab41 100644 --- a/frontend/src/test/scala/bloop/dap/DebugServerSpec.scala +++ b/frontend/src/test/scala/bloop/dap/DebugServerSpec.scala @@ -6,7 +6,6 @@ import java.net.SocketException import java.net.SocketTimeoutException import java.util.NoSuchElementException import java.util.concurrent.TimeUnit.MILLISECONDS - import scala.collection.JavaConverters._ import scala.collection.mutable import scala.concurrent.Future @@ -15,12 +14,11 @@ import scala.concurrent.TimeoutException import scala.concurrent.duration.Duration import scala.concurrent.duration.FiniteDuration import scala.concurrent.duration._ - import ch.epfl.scala.bsp import ch.epfl.scala.bsp.ScalaMainClass import ch.epfl.scala.debugadapter._ - import bloop.Cli +import bloop.Cli.CliSession import bloop.ScalaInstance import bloop.cli.CommonOptions import bloop.cli.ExitStatus @@ -45,15 +43,18 @@ import bloop.reporter.ReporterAction import bloop.task.Task import bloop.util.TestProject import bloop.util.TestUtil - +import cats.effect.concurrent.Ref import com.microsoft.java.debug.core.protocol.Requests.SetBreakpointArguments import com.microsoft.java.debug.core.protocol.Types import com.microsoft.java.debug.core.protocol.Types.SourceBreakpoint import coursierapi.Dependency import coursierapi.Fetch +import monix.eval.{Task => MonixTask} import monix.execution.Ack import monix.reactive.Observer +import java.nio.file.Path + object DebugServerSpec extends DebugBspBaseSuite { private val ServerNotListening = new IllegalStateException("Server is not accepting connections") private val Success: ExitStatus = ExitStatus.Ok @@ -1005,7 +1006,11 @@ object DebugServerSpec extends DebugBspBaseSuite { def cliCompile(project: TestProject) = { val compileArgs = Array("compile", project.config.name, "--config-dir", configDir.syntax) val compileAction = Cli.parse(compileArgs, CommonOptions.default) - Task.eval(Cli.run(compileAction, NoPool, ???, None)).executeAsync + Task.liftMonixTaskUncancellable { + Ref + .of[MonixTask, Map[Path, List[CliSession]]](Map.empty) + .flatMap(Cli.run(compileAction, NoPool, _)) + }.executeAsync } def bspCommand() = createBspCommand(configDir)