diff --git a/OPAL/si/src/main/scala/org/opalj/fpcf/AnalysisScenario.scala b/OPAL/si/src/main/scala/org/opalj/fpcf/AnalysisScenario.scala index c52d1c2e69..59257af202 100644 --- a/OPAL/si/src/main/scala/org/opalj/fpcf/AnalysisScenario.scala +++ b/OPAL/si/src/main/scala/org/opalj/fpcf/AnalysisScenario.scala @@ -6,6 +6,7 @@ import com.typesafe.config.Config import org.opalj.fpcf.AnalysisScenario.AnalysisAutoConfigKey import org.opalj.fpcf.AnalysisScenario.AnalysisSchedulingStrategyKey +import org.opalj.fpcf.scheduling.CleanupSpec import org.opalj.fpcf.scheduling.SchedulingStrategy import org.opalj.graphs.Graph import org.opalj.log.LogContext @@ -294,7 +295,8 @@ class AnalysisScenario[A](val ps: PropertyStore) { Schedule( scheduledBatches, - initializationData + initializationData, + Some(CleanupSpec()) ) } } diff --git a/OPAL/si/src/main/scala/org/opalj/fpcf/PhaseConfiguration.scala b/OPAL/si/src/main/scala/org/opalj/fpcf/PhaseConfiguration.scala index 62a0a2d3da..08ee5c98ca 100644 --- a/OPAL/si/src/main/scala/org/opalj/fpcf/PhaseConfiguration.scala +++ b/OPAL/si/src/main/scala/org/opalj/fpcf/PhaseConfiguration.scala @@ -4,5 +4,6 @@ package fpcf case class PhaseConfiguration[A]( propertyKinds: PropertyKindsConfiguration, - scheduled: List[ComputationSpecification[A]] + scheduled: List[ComputationSpecification[A]], + toDelete: Set[Int] = Set.empty ) diff --git a/OPAL/si/src/main/scala/org/opalj/fpcf/PropertyKey.scala b/OPAL/si/src/main/scala/org/opalj/fpcf/PropertyKey.scala index dac81ab898..93bf94a448 100644 --- a/OPAL/si/src/main/scala/org/opalj/fpcf/PropertyKey.scala +++ b/OPAL/si/src/main/scala/org/opalj/fpcf/PropertyKey.scala @@ -123,6 +123,18 @@ object PropertyKey { */ def name(id: Int): String = propertyKeyNames(id) + def idByName(name: String): Int = + getByName(name).id + + /** + * Returns a [[SomePropertyKey]] associated with the given name. To get it by id use [[key]]. Throws an [[IllegalArgumentException]] if no [[PropertyKey]] exists for the given name. + */ + def getByName(name: String): SomePropertyKey = { + propertyKeys.find(k => PropertyKey.name(k.id) == name).getOrElse( + throw new IllegalArgumentException(s"Unknown property name: $name") + ) + } + final def name(pKind: PropertyKind): String = name(pKind.id) final def name(eOptionP: SomeEOptionP): String = name(eOptionP.pk.id) diff --git a/OPAL/si/src/main/scala/org/opalj/fpcf/PropertyStore.scala b/OPAL/si/src/main/scala/org/opalj/fpcf/PropertyStore.scala index 8f5935cfee..dc46895d0e 100644 --- a/OPAL/si/src/main/scala/org/opalj/fpcf/PropertyStore.scala +++ b/OPAL/si/src/main/scala/org/opalj/fpcf/PropertyStore.scala @@ -135,6 +135,7 @@ abstract class PropertyStore { // private[this] val externalInformation = new ConcurrentHashMap[AnyRef, AnyRef]() + protected[this] var currentPhaseDeletionMask: Array[Boolean] = Array.fill(PropertyKey.maxId + 1)(false) /** * Attaches or returns some information associated with the property store using a key object. @@ -499,6 +500,26 @@ abstract class PropertyStore { protected[this] def doSet(e: Entity, p: Property): Unit + /** + * Removes [[PropertyKind]] from a [[PropertyStore]] using a pre-calculated [[currentPhaseDeletionMask]]. Calls [[clearPK]] which need to be implemented in the implementations of the [[PropertyStore]]. + */ + protected[fpcf] final def clearObsoletePropertyKinds(): Unit = { + val mask = currentPhaseDeletionMask + var index = 0 + while (index < mask.length) { + if (mask(index)) { + clearPK(index) + } + index += 1 + } + } + + /** + * Placeholder to remove a given [[PropertyKey]] from a [[PropertyStore]]. Needs to be overriden in the implementation for access to the given [[PropertyStore]]. + * @param id ID of the [[PropertyKey]] to be removed + */ + protected def clearPK(id: Int): Unit + /** * Associates the given entity with the newly computed intermediate property P. * @@ -538,6 +559,16 @@ abstract class PropertyStore { ) } + final def setupPhase(configuration: PropertyKindsConfiguration, toDelete: Set[Int]): Unit = { + setupPhase( + configuration.propertyKindsComputedInThisPhase, + configuration.propertyKindsComputedInLaterPhase, + configuration.suppressInterimUpdates, + configuration.collaborativelyComputedPropertyKindsFinalizationOrder, + toDelete + ) + } + protected[this] var subPhaseId: Int = 0 protected[this] var hasSuppressedNotifications: Boolean = false @@ -570,7 +601,8 @@ abstract class PropertyStore { propertyKindsComputedInThisPhase: Set[PropertyKind], propertyKindsComputedInLaterPhase: Set[PropertyKind] = Set.empty, suppressInterimUpdates: Map[PropertyKind, Set[PropertyKind]] = Map.empty, - finalizationOrder: List[List[PropertyKind]] = List.empty + finalizationOrder: List[List[PropertyKind]] = List.empty, + toDelete: Set[Int] = Set.empty ): Unit = handleExceptions { if (!isIdle) { throw new IllegalStateException("computations are already running"); @@ -638,14 +670,23 @@ abstract class PropertyStore { hasSuppressedNotifications = suppressInterimUpdates.nonEmpty // Step 5 + // Set up a new deletionMask by resetting it first, then filling it with the help of toDelete. + val mask = currentPhaseDeletionMask + java.util.Arrays.fill(mask, false) + + toDelete.foreach(id => mask(id) = true) + + // Step 6 // Call `newPhaseInitialized` to enable subclasses to perform custom initialization steps // when a phase was setup. newPhaseInitialized( propertyKindsComputedInThisPhase, propertyKindsComputedInLaterPhase, suppressInterimUpdates, - finalizationOrder + finalizationOrder, + currentPhaseDeletionMask ) + } /** @@ -656,7 +697,8 @@ abstract class PropertyStore { propertyKindsComputedInThisPhase: Set[PropertyKind], propertyKindsComputedInLaterPhase: Set[PropertyKind], suppressInterimUpdates: Map[PropertyKind, Set[PropertyKind]], - finalizationOrder: List[List[PropertyKind]] + finalizationOrder: List[List[PropertyKind]], + phaseDeletionMask: Array[Boolean] ): Unit = { /*nothing to do*/ } /** diff --git a/OPAL/si/src/main/scala/org/opalj/fpcf/PropertyStoreBasedCommandLineConfig.scala b/OPAL/si/src/main/scala/org/opalj/fpcf/PropertyStoreBasedCommandLineConfig.scala index 592cd2b0b6..e0bad17cea 100644 --- a/OPAL/si/src/main/scala/org/opalj/fpcf/PropertyStoreBasedCommandLineConfig.scala +++ b/OPAL/si/src/main/scala/org/opalj/fpcf/PropertyStoreBasedCommandLineConfig.scala @@ -3,6 +3,7 @@ package org.opalj package fpcf import com.typesafe.config.Config +import com.typesafe.config.ConfigValueFactory import org.rogach.scallop.ScallopConf import org.rogach.scallop.flagConverter @@ -12,6 +13,7 @@ import org.opalj.cli.Arg import org.opalj.cli.ConvertedArg import org.opalj.cli.ForwardingArg import org.opalj.cli.OPALCommandLineConfig +import org.opalj.cli.ParsedArg import org.opalj.cli.PlainArg import org.opalj.fpcf.par.SchedulingStrategyArg import org.opalj.log.LogContext @@ -68,7 +70,10 @@ trait PropertyStoreBasedCommandLineConfig extends OPALCommandLineConfig { self: generalArgs( PropertyStoreThreadsNumArg, PropertyStoreDebugArg, - SchedulingStrategyArg + SchedulingStrategyArg, + DisableCleanupArg, + KeepPropertyKeysArg, + ClearPropertyKeysArg ) def setupPropertyStore(project: Project): (PropertyStore, Seconds) = { @@ -91,3 +96,49 @@ trait PropertyStoreBasedCommandLineConfig extends OPALCommandLineConfig { self: else FPCFAnalysesRegistry.lazyFactory(analysisName) } } + +object DisableCleanupArg extends PlainArg[Boolean] { + override val name: String = "disableCleanup" + override def description: String = "Disable cleanup of the PropertyStore inbetween phases" + override val defaultValue: Option[Boolean] = Some(false) + override def apply(config: Config, value: Option[Boolean]): Config = { + config.withValue( + "org.opalj.fpcf.AnalysisScenario.DisableCleanup", + ConfigValueFactory.fromAnyRef(value.getOrElse(false)) + ) + } +} + +object KeepPropertyKeysArg extends ParsedArg[List[String], List[SomePropertyKey]] { + override val name: String = "keepPropertyKeys" + override val description: String = "List of Properties to keep at the end of the analysis" + override val defaultValue: Option[List[String]] = None + + override def apply(config: Config, value: Option[List[SomePropertyKey]]): Config = { + config.withValue( + "org.opalj.fpcf.AnalysisScenario.KeepPropertyKeys", + ConfigValueFactory.fromAnyRef(value.getOrElse("")) + ) + } + + override def parse(arg: List[String]): List[SomePropertyKey] = { + arg.flatMap(_.split(",")).map(PropertyKey.getByName) + } +} + +object ClearPropertyKeysArg extends ParsedArg[List[String], List[SomePropertyKey]] { + override val name: String = "clearPropertyKeys" + override val description: String = "List of Properties to keep at the end of the analysis" + override val defaultValue: Option[List[String]] = None + + override def apply(config: Config, value: Option[List[SomePropertyKey]]): Config = { + config.withValue( + "org.opalj.fpcf.AnalysisScenario.ClearPropertyKeys", + ConfigValueFactory.fromAnyRef(value.getOrElse("")) + ) + } + + override def parse(arg: List[String]): List[SomePropertyKey] = { + arg.flatMap(_.split(",")).map(PropertyKey.getByName) + } +} diff --git a/OPAL/si/src/main/scala/org/opalj/fpcf/Schedule.scala b/OPAL/si/src/main/scala/org/opalj/fpcf/Schedule.scala index c329afc031..d1ecda9995 100644 --- a/OPAL/si/src/main/scala/org/opalj/fpcf/Schedule.scala +++ b/OPAL/si/src/main/scala/org/opalj/fpcf/Schedule.scala @@ -2,6 +2,7 @@ package org.opalj package fpcf +import org.opalj.fpcf.scheduling.CleanupSpec import org.opalj.log.LogContext import org.opalj.log.OPALLogger.info import org.opalj.util.PerformanceEvaluation.time @@ -17,7 +18,8 @@ import org.opalj.util.PerformanceEvaluation.time */ case class Schedule[A]( batches: List[PhaseConfiguration[A]], - initializationData: Map[ComputationSpecification[A], Any] + initializationData: Map[ComputationSpecification[A], Any], + cleanupSpec: Option[CleanupSpec] ) extends ( ( PropertyStore, @@ -43,16 +45,24 @@ case class Schedule[A]( ): List[(ComputationSpecification[A], A)] = { implicit val logContext: LogContext = ps.logContext + val phases = cleanupSpec.map(scheduling.Cleanup.withPerPhaseCleanup(batches, ps, _)).getOrElse(batches) + var allExecutedAnalyses: List[(ComputationSpecification[A], A)] = Nil - batches.iterator.zipWithIndex foreach { batchId => - val (PhaseConfiguration(configuration, css), id) = batchId + phases.iterator.zipWithIndex foreach { batchId => + val (phase, id) = batchId + val configuration = phase.propertyKinds + val css = phase.scheduled if (trace) { info("analysis progress", s"setting up analysis phase $id: $configuration") + if (phase.toDelete.nonEmpty) { + info("analysis progress", s"to be deleted after this phase: " + phase.toDelete.map(PropertyKey.name)) + } } time { - ps.setupPhase(configuration) + ps.setupPhase(configuration, phase.toDelete) + afterPhaseSetup(configuration) assert(ps.isIdle, "the property store is not idle after phase setup") @@ -87,8 +97,9 @@ case class Schedule[A]( ) } } + // ... we are done now; the computed properties will no longer be computed! - ps.setupPhase(Set.empty, Set.empty) + ps.setupPhase(Set.empty, propertyKindsComputedInLaterPhase = Set.empty) allExecutedAnalyses } diff --git a/OPAL/si/src/main/scala/org/opalj/fpcf/par/PKECPropertyStore.scala b/OPAL/si/src/main/scala/org/opalj/fpcf/par/PKECPropertyStore.scala index 9f5abcffe8..3ae02bd92b 100644 --- a/OPAL/si/src/main/scala/org/opalj/fpcf/par/PKECPropertyStore.scala +++ b/OPAL/si/src/main/scala/org/opalj/fpcf/par/PKECPropertyStore.scala @@ -519,13 +519,15 @@ class PKECPropertyStore( startThreads(new PartialPropertiesFinalizerThread(_)) subPhaseId += 1 - ps(AnalysisKeyId).clear() } + clearObsoletePropertyKinds() idle = true } + override protected def clearPK(id: Int): Unit = ps(id).clear() + private[this] val interimStates: Array[ArrayBuffer[EPKState]] = Array.fill(THREAD_COUNT)(null) private[this] val successors: Array[EPKState => Iterable[EPKState]] = diff --git a/OPAL/si/src/main/scala/org/opalj/fpcf/scheduling/Cleanup.scala b/OPAL/si/src/main/scala/org/opalj/fpcf/scheduling/Cleanup.scala new file mode 100644 index 0000000000..a71e4ac9e7 --- /dev/null +++ b/OPAL/si/src/main/scala/org/opalj/fpcf/scheduling/Cleanup.scala @@ -0,0 +1,66 @@ +/* BSD 2-Clause License - see OPAL/LICENSE for details. */ +package org.opalj +package fpcf +package scheduling + +/** + * Class that allows to configure the cleanup of the PropertyStore inbetween phases programmatically + * @param keep IDs of the PropertyKeys to be kept at the end + * @param clear IDs of the PropertyKeys to be definitely removed + * @param disable Allows the cleanup to be disabled since it is on by default + */ +final case class CleanupSpec( + keep: Set[Int] = Set.empty, + clear: Set[Int] = Set.empty, + disable: Boolean = false +) + +/** + * Factory for creating [[CleanupSpec]]-objects, also handles calculation of per-phase cleanup + */ +object Cleanup { + + /** + * Creates a [[CleanupSpec]] from given [[PropertyKey]]-names to keep and/or to clear. Also allows disabling the cleanup by setting 'disable' to 'true'. + * @param keep Names of PropertyKeys to be kept after the analyses + * @param clear Names of PropertyKeys to be removed after the analyses + * @param disable Setting this to 'true' disables the cleanup inbetween the phases + * @return A new [[CleanupSpec]] + */ + def fromArgs(keep: Set[String], clear: Set[String], disable: Boolean): CleanupSpec = { + val toKeep = keep.map(PropertyKey.idByName) + val toClear = clear.map(PropertyKey.idByName) + CleanupSpec(toKeep, toClear, disable) + } + + /** + * Calculates the properties to be safely removed inbetween phases. Returns an unmodified schedule if cleanup is disabled + */ + def withPerPhaseCleanup[A]( + schedule: List[PhaseConfiguration[A]], + ps: PropertyStore, + spec: CleanupSpec + ): List[PhaseConfiguration[A]] = { + if (spec.disable) return schedule + + val producedInAnyPhase: Set[Int] = + schedule.iterator.flatMap(_.propertyKinds.propertyKindsComputedInThisPhase.map(_.id)).toSet + val neededLater = Array.fill[Set[Int]](schedule.size + 1)(Set.empty) + var index = schedule.size - 1 + var usedInAnyPhase: Set[Int] = Set.empty + while (index >= 0) { + val usedInThisPhase: Set[Int] = + schedule(index).scheduled.iterator.flatMap(_.uses(ps).iterator).map(_.pk.id).toSet + neededLater(index) = usedInAnyPhase + usedInAnyPhase = usedInAnyPhase union usedInThisPhase + index -= 1 + } + + schedule.indices.iterator.map { index => + val producedHere = schedule(index) + val toDelete = ((producedInAnyPhase -- neededLater(index)) -- spec.keep) union spec.clear + producedHere.copy(toDelete = toDelete) + }.toList + } + +} diff --git a/OPAL/si/src/main/scala/org/opalj/fpcf/scheduling/MaximumPhaseScheduling.scala b/OPAL/si/src/main/scala/org/opalj/fpcf/scheduling/MaximumPhaseScheduling.scala index 246c39e490..8ec33e234e 100644 --- a/OPAL/si/src/main/scala/org/opalj/fpcf/scheduling/MaximumPhaseScheduling.scala +++ b/OPAL/si/src/main/scala/org/opalj/fpcf/scheduling/MaximumPhaseScheduling.scala @@ -97,7 +97,6 @@ abstract class MultiplePhaseScheduling extends SchedulingStrategy { remainingAnalyses --= phaseAnalyses computePhase(ps, phaseAnalyses, remainingAnalyses) } - schedule } @@ -169,3 +168,9 @@ object MaximumPhaseScheduling extends MultiplePhaseScheduling { (initialPhaseDependencyGraph.toMap, initialPhaseIndexToAnalyses.toMap) } } + +object CleanupCalculation { + final val PropertiesToKeepKey = s"${ConfigKeyPrefix}KeepPropertyKeys" + final val PropertiesToRemoveKey = s"${ConfigKeyPrefix}ClearPropertyKeys" + final val DisableCleanupKey = s"${ConfigKeyPrefix}DisableCleanup" +} diff --git a/OPAL/si/src/main/scala/org/opalj/fpcf/seq/PKESequentialPropertyStore.scala b/OPAL/si/src/main/scala/org/opalj/fpcf/seq/PKESequentialPropertyStore.scala index f6846ee97f..e9fa96f9ac 100644 --- a/OPAL/si/src/main/scala/org/opalj/fpcf/seq/PKESequentialPropertyStore.scala +++ b/OPAL/si/src/main/scala/org/opalj/fpcf/seq/PKESequentialPropertyStore.scala @@ -825,11 +825,14 @@ final class PKESequentialPropertyStore protected ( } } while (continueComputation) + clearObsoletePropertyKinds() idle = true if (exception != null) throw exception; } + override protected def clearPK(id: Int): Unit = ps(id).clear() + def shutdown(): Unit = {} }