[GR-71299] Improve traversing compilation queue responsiveness.

PullRequest: graal/22602

Author: jchalou

compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/truffle/test/DynamicCompilationThresholdsTest.java

@@ -27,6 +27,8 @@
 import java.io.IOException;
 import java.util.LinkedList;
 import java.util.Map;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
@@ -59,7 +61,6 @@ static class SourceCompilation {
         private final int id;
         private final Source source;
         private final CountDownLatch compilationDoneLatch = new CountDownLatch(1);
-        private final CountDownLatch compilationStartedLatch = new CountDownLatch(1);
         private final CountDownLatch compilationGoLatch = new CountDownLatch(1);
 
         SourceCompilation(int id, Source source) {
@@ -75,6 +76,7 @@ public void testDynamicCompilationThreshods() throws IOException, InterruptedExc
         Assume.assumeTrue(Truffle.getRuntime() instanceof OptimizedTruffleRuntime);
         Runnable test = () -> {
             OptimizedTruffleRuntime optimizedTruffleRuntime = (OptimizedTruffleRuntime) Truffle.getRuntime();
+            BlockingQueue<SourceCompilation> startedCompilationsQueue = new ArrayBlockingQueue<>(30);
             OptimizedTruffleRuntimeListener listener = new OptimizedTruffleRuntimeListener() {
                 @Override
                 public void onCompilationSuccess(OptimizedCallTarget target, AbstractCompilationTask task, TruffleCompilerListener.GraphInfo graph,
@@ -97,7 +99,7 @@ private SourceCompilation getSourceCompilation(OptimizedCallTarget target) {
                 @Override
                 public void onCompilationStarted(OptimizedCallTarget target, AbstractCompilationTask task) {
                     if (getSourceCompilation(target) instanceof SourceCompilation compilation) {
-                        compilation.compilationStartedLatch.countDown();
+                        startedCompilationsQueue.add(compilation);
                         try {
                             compilation.compilationGoLatch.await();
                         } catch (InterruptedException ie) {
@@ -117,8 +119,9 @@ public void onCompilationStarted(OptimizedCallTarget target, AbstractCompilation
                             .option("engine.DynamicCompilationThresholdsMinNormalLoad", "10") //
                             .option("engine.DynamicCompilationThresholdsMinScale", "0.1") //
                             .option("engine.CompilerThreads", "1").build()) {
-                int firstCompilation = submitCompilation(context);
-                waitForCompilationStart(firstCompilation);
+                int firstCompilationId = submitCompilation(context);
+                SourceCompilation firstCompilation = startedCompilationsQueue.take();
+                Assert.assertEquals(firstCompilationId, firstCompilation.id);
                 LinkedList<Integer> scales = new LinkedList<>();
                 int firstScale = FixedPointMath.toFixedPoint(0.1);
                 Assert.assertEquals(firstScale, optimizedTruffleRuntime.compilationThresholdScale());
@@ -141,14 +144,14 @@ public void onCompilationStarted(OptimizedCallTarget target, AbstractCompilation
                     Assert.assertEquals(scale, optimizedTruffleRuntime.compilationThresholdScale());
                     scales.push(scale);
                 }
-                allowCompilationToProceed(firstCompilation);
-                waitForCompilationDone(firstCompilation);
+                allowCompilationToProceed(firstCompilationId);
+                waitForCompilationDone(firstCompilationId);
                 scales.pop();
-                for (int i = firstCompilation + 1; i <= 30; i++) {
-                    waitForCompilationStart(i);
+                for (int i = firstCompilationId + 1; i <= 30; i++) {
+                    SourceCompilation compilation = startedCompilationsQueue.take();
                     Assert.assertEquals((int) scales.pop(), optimizedTruffleRuntime.compilationThresholdScale());
-                    allowCompilationToProceed(i);
-                    waitForCompilationDone(i);
+                    allowCompilationToProceed(compilation.id);
+                    waitForCompilationDone(compilation.id);
                 }
                 Assert.assertTrue(scales.isEmpty());
             } catch (IOException | InterruptedException e) {
@@ -164,12 +167,6 @@ private void allowCompilationToProceed(int id) throws InterruptedException {
         compilationMap.get(id).compilationGoLatch.countDown();
     }
 
-    private void waitForCompilationStart(int id) throws InterruptedException {
-        if (!compilationMap.get(id).compilationStartedLatch.await(5, TimeUnit.MINUTES)) {
-            throw new AssertionError("Compilation of source " + id + " did not start in time");
-        }
-    }
-
     private void waitForCompilationDone(int id) throws InterruptedException {
         if (!compilationMap.get(id).compilationDoneLatch.await(5, TimeUnit.MINUTES)) {
             throw new AssertionError("Compilation of source " + id + " did not finish in time");

substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/TruffleFeature.java

@@ -126,6 +126,7 @@
 import com.oracle.svm.core.stack.JavaStackWalker;
 import com.oracle.svm.core.util.UserError;
 import com.oracle.svm.core.util.VMError;
+import com.oracle.svm.graal.TruffleRuntimeCompilationSupport;
 import com.oracle.svm.graal.hosted.runtimecompilation.CallTreeInfo;
 import com.oracle.svm.graal.hosted.runtimecompilation.RuntimeCompilationCandidate;
 import com.oracle.svm.graal.hosted.runtimecompilation.RuntimeCompilationFeature;
@@ -136,13 +137,12 @@
 import com.oracle.svm.hosted.FeatureImpl.DuringSetupAccessImpl;
 import com.oracle.svm.hosted.meta.HostedType;
 import com.oracle.svm.hosted.substitute.DeletedElementException;
-import com.oracle.svm.graal.TruffleRuntimeCompilationSupport;
+import com.oracle.svm.truffle.api.SubstrateKnownTruffleTypes;
 import com.oracle.svm.truffle.api.SubstrateThreadLocalHandshake;
 import com.oracle.svm.truffle.api.SubstrateThreadLocalHandshakeSnippets;
 import com.oracle.svm.truffle.api.SubstrateTruffleCompiler;
 import com.oracle.svm.truffle.api.SubstrateTruffleRuntime;
 import com.oracle.svm.truffle.api.SubstrateTruffleUniverseFactory;
-import com.oracle.svm.truffle.api.SubstrateKnownTruffleTypes;
 import com.oracle.svm.util.AnnotationUtil;
 import com.oracle.svm.util.LogUtils;
 import com.oracle.svm.util.ReflectionUtil;
@@ -1118,6 +1118,19 @@ final class Target_com_oracle_truffle_runtime_OptimizedCallTarget {
     @Alias @RecomputeFieldValue(kind = RecomputeFieldValue.Kind.Reset) //
     long initializedTimestamp;
 
+    /*
+     * The call count must start from zero at runtime, compilation prioritization depends on it.
+     */
+    @Alias @RecomputeFieldValue(kind = RecomputeFieldValue.Kind.Reset) //
+    int callCount;
+
+    /*
+     * The call and loop count must start from zero at runtime, compilation prioritization depends
+     * on it.
+     */
+    @Alias @RecomputeFieldValue(kind = RecomputeFieldValue.Kind.Reset) //
+    int callAndLoopCount;
+
     private static final class TransformToTrue implements FieldValueTransformer {
         @Override
         public Object transform(Object receiver, Object originalValue) {

truffle/CHANGELOG.md

@@ -37,6 +37,8 @@ This changelog summarizes major changes between Truffle versions relevant to lan
 
 * GR-70086: Added `replacementOf` and `replacementMethod` attributes to `GenerateLibrary.Abstract` annotation. They enable automatic generation of legacy delegators during message library evolution, while allowing custom conversions when needed.
 * GR-70086 Deprecated `Message.resolve(Class<?>, String)`. Use `Message.resolveExact(Class<?>, String, Class<?>...)` with argument types instead. This deprecation was necessary as library messages are no longer unique by message name, if the previous message was deprecated.
+* GR-71299 Improved the responsiveness of the Truffle compilation queue by refining the computation of the execution rate of compilation units in the queue.
+  * Added `engine.TraversingQueueRateHalfLife` to allow fine-tuning of the compilation queue responsiveness.
 
 * GR-69861: Bytecode DSL: Added a `BytecodeFrame` abstraction for capturing frame state and accessing frame data. This abstraction should be preferred over `BytecodeNode` access methods because it captures the correct interpreter location data.
 * GR-69861: Bytecode DSL: Added a `captureFramesForTrace` parameter to `@GenerateBytecode` that enables capturing of frames in `TruffleStackTraceElement`s. Previously, frame data was unreliably available in stack traces; now, it is guaranteed to be available if requested. Languages must use the `BytecodeFrame` abstraction to access frame data from `TruffleStackTraceElement`s rather than access the frame directly.

truffle/docs/Options.md

@@ -179,6 +179,7 @@ These are internal options for debugging language implementations and tools.
 - `--engine.TraversingQueueFirstTierPriority` : Traversing queue gives first tier compilations priority.
 - `--engine.TraversingQueueInvalidatedBonus=[0.0, inf)` : Controls how much of a priority should be given to compilations after invalidations (default: 1.0, no bonus).
 - `--engine.TraversingQueueOSRBonus=[0.0, inf)` : Controls how much of a priority should be given to OSR compilations (default: 1.0, no bonus).
+- `--engine.TraversingQueueRateHalfLife=[0, inf)` : Sets the time, in milliseconds, after which the impact of a compilation unit's observed execution rate is halved. (default: 300 ms)
 - `--engine.TraversingQueueWeightingBothTiers=true|false` : Traversing queue uses rate as priority for both tier. (default: true)
 - `--compiler.CompilationTimeout` : Time limit in seconds before a compilation expires and throws a bailout (0 to disable the limit). 
 - `--compiler.DeoptCycleDetectionAllowedRepeats` : Maximum allowed repeats of the same compiled code for the same compilable. Works only if the detection of repeated compilation is enabled after DeoptCycleDetectionThreshold has been reached for the compilable. (negative integer means 0, default: 0)

truffle/src/com.oracle.truffle.api/src/com/oracle/truffle/api/impl/Accessor.java

@@ -1334,6 +1334,8 @@ public AbstractFastThreadLocal getContextThreadLocal() {
         public abstract boolean isLegacyCompilerOption(String key);
 
         public abstract <T> ThreadLocal<T> createTerminatingThreadLocal(Supplier<T> initialValue, Consumer<T> onThreadTermination);
+
+        public abstract void setInitializedTimestamp(CallTarget target, long timestamp);
     }
 
     public abstract static class LanguageProviderSupport extends Support {

truffle/src/com.oracle.truffle.api/src/com/oracle/truffle/api/impl/DefaultRuntimeAccessor.java

@@ -304,6 +304,11 @@ public AbstractFastThreadLocal getContextThreadLocal() {
         public <T> ThreadLocal<T> createTerminatingThreadLocal(Supplier<T> initialValue, Consumer<T> onThreadTermination) {
             return ThreadLocal.withInitial(initialValue);
         }
+
+        @Override
+        public void setInitializedTimestamp(CallTarget target, long timestamp) {
+
+        }
     }
 
 }

truffle/src/com.oracle.truffle.polyglot/src/com/oracle/truffle/polyglot/PolyglotEngineImpl.java

@@ -774,6 +774,12 @@ boolean patch(SandboxPolicy newSandboxPolicy,
         }
         validateSandbox();
         printDeprecatedOptionsWarning(deprecatedDescriptors);
+
+        long currentTimestamp = System.nanoTime();
+        for (CallTarget loadedCallTarget : getCallTargets()) {
+            RUNTIME.setInitializedTimestamp(loadedCallTarget, currentTimestamp);
+        }
+
         return true;
     }
 

truffle/src/com.oracle.truffle.runtime/src/com/oracle/truffle/runtime/CompilationTask.java

@@ -102,18 +102,31 @@ private CompilationTask(BackgroundCompileQueue.Priority priority, WeakReference<
         // cannot compute a reliable rate yet. The rate will be computed the first time updateWeight
         // is called.
         lastWeight = -1.0;
-        lastTime = System.nanoTime();
+        long timestamp = System.nanoTime();
         OptimizedCallTarget target = targetRef.get();
         if (target == null) {
+            lastTime = timestamp;
             lastCount = Integer.MIN_VALUE;
             engineData = null;
             isOSR = false;
             cancelledPredicate = null;
         } else {
-            lastCount = target.getCallAndLoopCount();
             engineData = target.engine;
             isOSR = target.isOSR();
             cancelledPredicate = target.engine.cancelledPredicate;
+            lastCount = 0;
+            lastTime = target.getInitializedTimestamp();
+            if (target.isInitialized()) {
+                assert lastTime != 0;
+                updateWeight(timestamp);
+            } else {
+                /*
+                 * Call targets used for the initialization of the compiler are not initialized
+                 * before they are sent to the compilation queue.
+                 */
+                lastCount = target.getCallAndLoopCount();
+                lastTime = timestamp;
+            }
         }
     }
 
@@ -284,11 +297,28 @@ boolean updateWeight(long currentTime) {
         // A last weight > 0 indicates that it has been initialized. If so, only update its weight
         // if 1_000_000ns have elapsed since its last calculation. The elapsed time may be negative
         // since tasks may be added to the queue while traversing it.
-        if (lastWeight > 0 && elapsed < 1_000_000 || elapsed < 0) {
+        if (lastWeight >= 0 && elapsed < 1_000_000 || elapsed < 0) {
             return true;
         }
         int count = target.getCallAndLoopCount();
-        lastRate = rate(count, elapsed);
+        double currentRate = rate(count, elapsed);
+        if (lastWeight < 0) {
+            lastRate = currentRate;
+        } else {
+            /*
+             * "Assembles" the new rate from the last rate and the current rate by applying a decay
+             * from the interval (0, 1) to the last rate. The decay is computed using the configured
+             * rate half-life. For example, if the elapsed time is equal to the half-life, then <new
+             * rate> = <last rate> * 0.5 + <current rate> * 0.5; if the elapsed time is twice the
+             * half-life, then <new rate> = <last rate> * 0.25 + <current rate> * 0.75. The reason
+             * we do this is that the current interval, which has an unpredictable size, might not
+             * capture the current hotness of the compilation unit very well. Using the decay makes
+             * the computed rate capture a longer time interval while still giving priority to more
+             * recent executions.
+             */
+            double decay = Math.pow(2.0d, -1.0d * elapsed / engineData.traversingRateHalfLifeNs);
+            lastRate = lastRate * decay + currentRate * (1 - decay);
+        }
         lastTime = currentTime;
         lastCount = count;
         double weight = (1 + lastRate) * lastCount;
@@ -351,7 +381,8 @@ private static boolean hasEffect(double bonus) {
 
     private double rate(int count, long elapsed) {
         // Divide by a minimum of 1000 to prevent division by zero/very small numbers.
-        return lastRate = ((double) count - lastCount) / Math.max(elapsed, 1000);
+        // Multiply by 1_000_000 to get per-millisecond rate.
+        return ((double) count - lastCount) / Math.max(elapsed, 1000) * 1_000_000;
     }
 
     public int targetHighestCompiledTier() {

truffle/src/com.oracle.truffle.runtime/src/com/oracle/truffle/runtime/EngineData.java

@@ -80,6 +80,7 @@
 import static com.oracle.truffle.runtime.OptimizedRuntimeOptions.TraversingQueueFirstTierPriority;
 import static com.oracle.truffle.runtime.OptimizedRuntimeOptions.TraversingQueueInvalidatedBonus;
 import static com.oracle.truffle.runtime.OptimizedRuntimeOptions.TraversingQueueOSRBonus;
+import static com.oracle.truffle.runtime.OptimizedRuntimeOptions.TraversingQueueRateHalfLife;
 import static com.oracle.truffle.runtime.OptimizedRuntimeOptions.TraversingQueueWeightingBothTiers;
 import static com.oracle.truffle.runtime.OptimizedTruffleRuntime.getRuntime;
 
@@ -164,6 +165,7 @@ public final class EngineData {
     @CompilationFinal public double traversingFirstTierBonus;
     @CompilationFinal public double traversingInvalidatedBonus;
     @CompilationFinal public double traversingOSRBonus;
+    @CompilationFinal public long traversingRateHalfLifeNs;
     @CompilationFinal public boolean propagateCallAndLoopCount;
     @CompilationFinal public int propagateCallAndLoopCountMaxDepth;
     @CompilationFinal public int maximumCompilations;
@@ -346,6 +348,7 @@ private void loadOptions(OptionValues options, SandboxPolicy sandboxPolicy) {
         maximumCompilations = options.get(MaximumCompilations);
         traversingInvalidatedBonus = options.get(TraversingQueueInvalidatedBonus);
         traversingOSRBonus = options.get(TraversingQueueOSRBonus);
+        traversingRateHalfLifeNs = options.get(TraversingQueueRateHalfLife) * 1_000_000;
 
         this.returnTypeSpeculation = options.get(ReturnTypeSpeculation);
         this.argumentTypeSpeculation = options.get(ArgumentTypeSpeculation);

truffle/src/com.oracle.truffle.runtime/src/com/oracle/truffle/runtime/OptimizedCallTarget.java

@@ -807,7 +807,9 @@ private void propagateCallAndLoopCount() {
             }
             if (callerCallTarget.frameDescriptorEquals(parentFrameDescriptor)) {
                 callerCallNode.forceInlining();
-                callerCallTarget.callAndLoopCount += this.callAndLoopCount;
+                int oldLoopCallCount = callerCallTarget.callAndLoopCount;
+                int newLoopCallCount = oldLoopCallCount + this.callAndLoopCount;
+                callerCallTarget.callAndLoopCount = newLoopCallCount >= oldLoopCallCount ? newLoopCallCount : Integer.MAX_VALUE;
                 return;
             }
             currentSingleCallNode = callerCallTarget.singleCallNode;
@@ -882,11 +884,7 @@ private synchronized void initialize(boolean validate) {
             assert !validate || OptimizedRuntimeAccessor.NODES.getCallTargetWithoutInitialization(rootNode) == this : "Call target out of sync.";
 
             OptimizedRuntimeAccessor.INSTRUMENT.onFirstExecution(getRootNode(), validate);
-            if (engine.callTargetStatistics) {
-                this.initializedTimestamp = System.nanoTime();
-            } else {
-                this.initializedTimestamp = 0L;
-            }
+            this.initializedTimestamp = System.nanoTime();
             initialized = true;
         }
     }
@@ -1364,6 +1362,12 @@ public final long getInitializedTimestamp() {
         return initializedTimestamp;
     }
 
+    final void setInitializedTimestamp(long timestamp) {
+        if (initialized) {
+            initializedTimestamp = timestamp;
+        }
+    }
+
     public final Map<String, Object> getDebugProperties() {
         Map<String, Object> properties = new LinkedHashMap<>();
         OptimizedTruffleRuntimeListener.addASTSizeProperty(this, properties);