Skip to content

Commit d1a39d3

Browse files
[GR-71361] Fixes and cleanups for TLAB-related options.
PullRequest: graal/22620
2 parents e625817 + d8a589f commit d1a39d3

File tree

10 files changed

+89
-111
lines changed

10 files changed

+89
-111
lines changed

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -425,7 +425,7 @@ public static UnsignedWord getChunkBytes() {
425425
}
426426

427427
private static void resizeAllTlabs() {
428-
if (SubstrateGCOptions.TlabOptions.ResizeTLAB.getValue()) {
428+
if (SubstrateGCOptions.ResizeTLAB.getValue()) {
429429
for (IsolateThread thread = VMThreads.firstThread(); thread.isNonNull(); thread = VMThreads.nextThread(thread)) {
430430
TlabSupport.resize(thread);
431431
}

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GenScavengeGCFeature.java

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,11 @@ public void duringSetup(DuringSetupAccess access) {
106106
ImageSingletons.add(Heap.class, new HeapImpl());
107107
ImageSingletons.add(ImageHeapInfo.class, new ImageHeapInfo());
108108
ImageSingletons.add(GCAllocationSupport.class, new GenScavengeAllocationSupport());
109-
ImageSingletons.add(TlabOptionCache.class, new TlabOptionCache());
109+
110+
TlabOptionCache tlabOptionCache = new TlabOptionCache();
111+
ImageSingletons.add(TlabOptionCache.class, tlabOptionCache);
112+
tlabOptionCache.validateHostedOptionValues();
113+
110114
if (ImageLayerBuildingSupport.firstImageBuild()) {
111115
ImageSingletons.add(PinnedObjectSupport.class, new PinnedObjectSupportImpl());
112116
}

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SerialAndEpsilonGCOptions.java

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,6 @@
2424
*/
2525
package com.oracle.svm.core.genscavenge;
2626

27-
import static com.oracle.svm.core.option.RuntimeOptionKey.RuntimeOptionKeyFlag.RegisterForIsolateArgumentParser;
28-
2927
import com.oracle.svm.core.SubstrateOptions;
3028
import com.oracle.svm.core.metaspace.Metaspace;
3129
import com.oracle.svm.core.option.HostedOptionKey;
@@ -71,9 +69,6 @@ public final class SerialAndEpsilonGCOptions {
7169
@Option(help = "Number of bytes at the beginning of each heap chunk that are not used for payload data, i.e., can be freely used as metadata by the heap chunk provider. Serial and epsilon GC only.", type = OptionType.Debug) //
7270
public static final HostedOptionKey<Integer> HeapChunkHeaderPadding = new HostedOptionKey<>(0, SerialAndEpsilonGCOptions::validateSerialOrEpsilonHostedOption);
7371

74-
@Option(help = "Starting TLAB size (in bytes); zero means set ergonomically.", type = OptionType.Expert)//
75-
public static final RuntimeOptionKey<Long> InitialTLABSize = new RuntimeOptionKey<>(8 * 1024L, SerialAndEpsilonGCOptions::validateSerialOrEpsilonRuntimeOption, RegisterForIsolateArgumentParser);
76-
7772
@Option(help = "Print information about TLABs. Printed when The TLABs are retired before a GC, and during the resizing of the TLABs. Serial and epsilon GC only.", type = OptionType.Expert)//
7873
public static final HostedOptionKey<Boolean> PrintTLAB = new HostedOptionKey<>(false, SerialAndEpsilonGCOptions::validateSerialOrEpsilonHostedOption);
7974

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/TlabOptionCache.java

Lines changed: 67 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525
package com.oracle.svm.core.genscavenge;
2626

2727
import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE;
28-
import static com.oracle.svm.core.genscavenge.TlabSupport.maxSize;
2928

3029
import org.graalvm.nativeimage.ImageSingletons;
3130
import org.graalvm.nativeimage.Platform;
@@ -34,10 +33,14 @@
3433
import com.oracle.svm.core.IsolateArgumentParser;
3534
import com.oracle.svm.core.SubstrateGCOptions;
3635
import com.oracle.svm.core.SubstrateOptions;
36+
import com.oracle.svm.core.SubstrateUtil;
3737
import com.oracle.svm.core.Uninterruptible;
3838
import com.oracle.svm.core.config.ConfigurationValues;
3939
import com.oracle.svm.core.jdk.UninterruptibleUtils;
40+
import com.oracle.svm.core.option.RuntimeOptionKey;
4041
import com.oracle.svm.core.option.RuntimeOptionValidationSupport;
42+
import com.oracle.svm.core.option.RuntimeOptionValidationSupport.RuntimeOptionValidation;
43+
import com.oracle.svm.core.util.UserError;
4144

4245
import jdk.graal.compiler.api.replacements.Fold;
4346
import jdk.graal.compiler.core.common.NumUtil;
@@ -49,26 +52,27 @@
4952
* options and reports errors (see {@link #registerOptionValidations}).
5053
*/
5154
public class TlabOptionCache {
55+
private static final long DEFAULT_INITIAL_TLAB_SIZE = 8 * 1024;
5256

5357
private long minTlabSize;
5458
private long tlabSize;
55-
private long initialTLABSize;
5659

5760
@Platforms(Platform.HOSTED_ONLY.class)
5861
public TlabOptionCache() {
59-
minTlabSize = getAbsoluteMinTlabSize();
60-
tlabSize = SubstrateGCOptions.TlabOptions.TLABSize.getHostedValue();
61-
initialTLABSize = SerialAndEpsilonGCOptions.InitialTLABSize.getHostedValue();
6262
}
6363

6464
@Fold
6565
public static TlabOptionCache singleton() {
6666
return ImageSingletons.lookup(TlabOptionCache.class);
6767
}
6868

69-
/*
70-
* The minimum size a TLAB must always have. A smaller TLAB may lead to a VM crash.
71-
*/
69+
@Platforms(Platform.HOSTED_ONLY.class)
70+
public void validateHostedOptionValues() {
71+
validateMinTlabSize(SubstrateGCOptions.ConcealedOptions.MinTLABSize);
72+
validateTlabSize(SubstrateGCOptions.ConcealedOptions.TLABSize);
73+
}
74+
75+
/* The minimum size that a TLAB must have. Anything smaller than that could crash the VM. */
7276
@Fold
7377
static long getAbsoluteMinTlabSize() {
7478
int additionalHeaderBytes = SubstrateOptions.AdditionalHeaderBytes.getValue();
@@ -78,97 +82,87 @@ static long getAbsoluteMinTlabSize() {
7882

7983
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
8084
public long getMinTlabSize() {
85+
if (SubstrateUtil.HOSTED) {
86+
return Math.max(getAbsoluteMinTlabSize(), SubstrateGCOptions.ConcealedOptions.MinTLABSize.getHostedValue());
87+
}
88+
89+
assert minTlabSize >= getAbsoluteMinTlabSize() && ConfigurationValues.getObjectLayout().isAligned(minTlabSize) && minTlabSize <= TlabSupport.maxSize().rawValue();
8190
return minTlabSize;
8291
}
8392

8493
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
8594
public long getTlabSize() {
95+
assert tlabSize >= minTlabSize && ConfigurationValues.getObjectLayout().isAligned(tlabSize) && tlabSize <= TlabSupport.maxSize().rawValue();
8696
return tlabSize;
8797
}
8898

99+
/**
100+
* Based on the build-time and run-time option values, compute sane values that are at least
101+
* good enough for VM startup.
102+
*/
89103
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
90-
public long getInitialTLABSize() {
91-
return initialTLABSize;
104+
public void cacheOptionValues() {
105+
long maxTlabSize = TlabSupport.maxSize().rawValue();
106+
assert ConfigurationValues.getObjectLayout().isAligned(maxTlabSize) : "rounded values must not exceed max size";
107+
108+
cacheMinTlabSize(maxTlabSize);
109+
cacheTlabSize(maxTlabSize);
92110
}
93111

94112
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
95-
public void cacheOptionValues() {
96-
int minTlabSizeIdx = IsolateArgumentParser.getOptionIndex(SubstrateGCOptions.TlabOptions.MinTLABSize);
97-
long minTlabSizeValue = IsolateArgumentParser.singleton().getLongOptionValue(minTlabSizeIdx);
98-
cacheMinTlabSize(minTlabSizeValue);
99-
100-
int tlabSizeIdx = IsolateArgumentParser.getOptionIndex(SubstrateGCOptions.TlabOptions.TLABSize);
101-
long tlabSizeValue = IsolateArgumentParser.singleton().getLongOptionValue(tlabSizeIdx);
102-
cacheTlabSize(tlabSizeValue);
113+
private void cacheMinTlabSize(long maxTlabSize) {
114+
int optionIndex = IsolateArgumentParser.getOptionIndex(SubstrateGCOptions.ConcealedOptions.MinTLABSize);
115+
long optionValue = IsolateArgumentParser.singleton().getLongOptionValue(optionIndex);
116+
optionValue = UninterruptibleUtils.Math.clamp(optionValue, getAbsoluteMinTlabSize(), maxTlabSize);
117+
minTlabSize = ConfigurationValues.getObjectLayout().alignUp(optionValue);
118+
}
103119

104-
int initialTlabSizeIdx = IsolateArgumentParser.getOptionIndex(SerialAndEpsilonGCOptions.InitialTLABSize);
105-
long initialTlabSizeValue = IsolateArgumentParser.singleton().getLongOptionValue(initialTlabSizeIdx);
106-
cacheInitialTlabSize(initialTlabSizeValue, initialTLABSize != initialTlabSizeValue);
120+
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
121+
private void cacheTlabSize(long maxTlabSize) {
122+
int optionIndex = IsolateArgumentParser.getOptionIndex(SubstrateGCOptions.ConcealedOptions.TLABSize);
123+
long optionValue = IsolateArgumentParser.singleton().getLongOptionValue(optionIndex);
124+
if (optionValue == 0) {
125+
optionValue = UninterruptibleUtils.Math.clamp(DEFAULT_INITIAL_TLAB_SIZE, minTlabSize, maxTlabSize);
126+
} else {
127+
optionValue = UninterruptibleUtils.Math.clamp(optionValue, minTlabSize, maxTlabSize);
128+
}
129+
tlabSize = ConfigurationValues.getObjectLayout().alignUp(optionValue);
107130
}
108131

109132
public static void registerOptionValidations() {
110-
111-
long maxSize = maxSize().rawValue();
112-
113133
RuntimeOptionValidationSupport validationSupport = RuntimeOptionValidationSupport.singleton();
134+
validationSupport.register(new RuntimeOptionValidation<>(TlabOptionCache::validateMinTlabSize, SubstrateGCOptions.ConcealedOptions.MinTLABSize));
135+
validationSupport.register(new RuntimeOptionValidation<>(TlabOptionCache::validateTlabSize, SubstrateGCOptions.ConcealedOptions.TLABSize));
136+
}
114137

115-
validationSupport.register(new RuntimeOptionValidationSupport.RuntimeOptionValidation<>(optionKey -> {
116-
long minTlabSizeValue = optionKey.getValue();
117-
118-
if (optionKey.hasBeenSet() && minTlabSizeValue < getAbsoluteMinTlabSize()) {
119-
throw new IllegalArgumentException(String.format("MinTLABSize (%d) must be greater than or equal to reserved area in TLAB (%d).", minTlabSizeValue, getAbsoluteMinTlabSize()));
120-
}
121-
if (minTlabSizeValue > maxSize) {
122-
throw new IllegalArgumentException(String.format("MinTLABSize (%d) must be less than or equal to ergonomic TLAB maximum (%d).", minTlabSizeValue, maxSize));
123-
}
124-
}, SubstrateGCOptions.TlabOptions.MinTLABSize));
125-
126-
validationSupport.register(new RuntimeOptionValidationSupport.RuntimeOptionValidation<>(optionKey -> {
127-
// Check that TLABSize is still the default value or size >= abs min && size <= abs max.
128-
long tlabSizeValue = optionKey.getValue();
129-
if (optionKey.hasBeenSet() && tlabSizeValue < SubstrateGCOptions.TlabOptions.MinTLABSize.getValue()) {
130-
throw new IllegalArgumentException(
131-
String.format("TLABSize (%d) must be greater than or equal to MinTLABSize (%d).", tlabSizeValue, SubstrateGCOptions.TlabOptions.MinTLABSize.getValue()));
132-
}
133-
if (tlabSizeValue > maxSize) {
134-
throw new IllegalArgumentException(String.format("TLABSize (%d) must be less than or equal to ergonomic TLAB maximum size (%d).", tlabSizeValue, maxSize));
135-
}
136-
}, SubstrateGCOptions.TlabOptions.TLABSize));
137-
138-
validationSupport.register(new RuntimeOptionValidationSupport.RuntimeOptionValidation<>(optionKey -> {
139-
long initialTlabSizeValue = optionKey.getValue();
140-
if (initialTlabSizeValue < SubstrateGCOptions.TlabOptions.MinTLABSize.getValue()) {
141-
throw new IllegalArgumentException(
142-
String.format("InitialTLABSize (%d) must be greater than or equal to MinTLABSize (%d).", initialTlabSizeValue, SubstrateGCOptions.TlabOptions.MinTLABSize.getValue()));
143-
}
144-
if (initialTlabSizeValue > maxSize) {
145-
throw new IllegalArgumentException(String.format("TLABSize (%d) must be less than or equal to ergonomic TLAB maximum size (%d).", initialTlabSizeValue, maxSize));
146-
}
147-
}, SerialAndEpsilonGCOptions.InitialTLABSize));
138+
private static void validateMinTlabSize(RuntimeOptionKey<Long> optionKey) {
139+
long optionValue = optionKey.getValue();
140+
if (optionKey.hasBeenSet() && optionValue < getAbsoluteMinTlabSize()) {
141+
throw invalidOptionValue("Option 'MinTLABSize' (" + optionValue + ") must not be smaller than " + getAbsoluteMinTlabSize());
142+
}
148143

144+
long maxSize = TlabSupport.maxSize().rawValue();
145+
if (optionValue > maxSize) {
146+
throw invalidOptionValue("Option 'MinTLABSize' (" + optionValue + ") must not be larger than " + maxSize);
147+
}
149148
}
150149

151-
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
152-
private void cacheMinTlabSize(long optionValue) {
153-
if (getAbsoluteMinTlabSize() <= optionValue && optionValue <= maxSize().rawValue()) {
154-
minTlabSize = optionValue;
150+
private static void validateTlabSize(RuntimeOptionKey<Long> optionKey) {
151+
long optionValue = optionKey.getValue();
152+
if (optionKey.hasBeenSet() && optionValue < TlabOptionCache.singleton().getMinTlabSize()) {
153+
throw invalidOptionValue("Option 'TLABSize' (" + optionValue + ") must not be smaller than 'MinTLABSize' (" + TlabOptionCache.singleton().getMinTlabSize() + ").");
155154
}
156-
}
157155

158-
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
159-
private void cacheTlabSize(long optionValue) {
160-
if (getAbsoluteMinTlabSize() <= optionValue && optionValue <= maxSize().rawValue()) {
161-
tlabSize = optionValue;
156+
long maxSize = TlabSupport.maxSize().rawValue();
157+
if (optionValue > maxSize) {
158+
throw invalidOptionValue("Option 'TLABSize' (" + optionValue + ") must not be larger than " + maxSize);
162159
}
163160
}
164161

165-
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
166-
private void cacheInitialTlabSize(long optionValue, boolean hasBeenSet) {
167-
if (!hasBeenSet && minTlabSize > initialTLABSize) {
168-
initialTLABSize = minTlabSize;
169-
} else if (getAbsoluteMinTlabSize() <= optionValue && optionValue <= maxSize().rawValue()) {
170-
initialTLABSize = UninterruptibleUtils.Math.max(minTlabSize, optionValue);
162+
private static RuntimeException invalidOptionValue(String msg) {
163+
if (SubstrateUtil.HOSTED) {
164+
throw UserError.abort(msg);
171165
}
166+
throw new IllegalArgumentException(msg);
172167
}
173-
174168
}

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/TlabSupport.java

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -379,25 +379,15 @@ private static void resetStatistics(IsolateThread thread) {
379379
@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-23-ga/src/hotspot/share/gc/shared/threadLocalAllocBuffer.cpp#L270-L289")
380380
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
381381
private static UnsignedWord initialDesiredSize() {
382-
UnsignedWord initSize;
383-
384-
if (TlabOptionCache.singleton().getTlabSize() > 0) {
385-
long tlabSize = TlabOptionCache.singleton().getTlabSize();
386-
initSize = Word.unsigned(ConfigurationValues.getObjectLayout().alignUp(tlabSize));
387-
} else {
388-
long initialTLABSize = TlabOptionCache.singleton().getInitialTLABSize();
389-
initSize = Word.unsigned(ConfigurationValues.getObjectLayout().alignUp(initialTLABSize));
390-
}
391-
long minTlabSize = TlabOptionCache.singleton().getMinTlabSize();
392-
return UnsignedUtils.clamp(initSize, Word.unsigned(minTlabSize), maxSize());
382+
return Word.unsigned(TlabOptionCache.singleton().getTlabSize());
393383
}
394384

395385
/**
396386
* Compute the next tlab size using expected allocation amount.
397387
*/
398388
@BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+11/src/hotspot/share/gc/shared/threadLocalAllocBuffer.cpp#L154-L172")
399389
public static void resize(IsolateThread thread) {
400-
assert SubstrateGCOptions.TlabOptions.ResizeTLAB.getValue();
390+
assert SubstrateGCOptions.ResizeTLAB.getValue();
401391
assert VMOperation.isGCInProgress();
402392

403393
UnsignedWord allocatedAvg = Word.unsigned((long) AdaptiveWeightedAverageStruct.getAverage(allocatedBytesAvg.getAddress(thread)));

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/GenScavengeAllocationSupport.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ public ForeignCallDescriptor getNewPodInstanceStub() {
9191

9292
@Override
9393
public boolean useTLAB() {
94-
return SubstrateGCOptions.TlabOptions.UseTLAB.getValue();
94+
return SubstrateGCOptions.UseTLAB.getValue();
9595
}
9696

9797
@Override

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateGCOptions.java

Lines changed: 9 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,6 @@
4747
*/
4848
@DuplicatedInNativeCode
4949
public class SubstrateGCOptions {
50-
private static final int K = 1024;
51-
5250
@Option(help = "The minimum heap size at run-time, in bytes.", type = OptionType.User)//
5351
public static final RuntimeOptionKey<Long> MinHeapSize = new NotifyGCRuntimeOptionKey<>(0L, RegisterForIsolateArgumentParser) {
5452
@Override
@@ -106,20 +104,20 @@ protected void onValueUpdate(EconomicMap<OptionKey<?>, Object> values, Long oldV
106104
@Option(help = "Determines if references from runtime-compiled code to Java heap objects should be treated as strong or weak.", type = OptionType.Debug)//
107105
public static final HostedOptionKey<Boolean> TreatRuntimeCodeInfoReferencesAsWeak = new HostedOptionKey<>(true);
108106

109-
@DuplicatedInNativeCode
110-
public static class TlabOptions {
111-
@Option(help = "Use thread-local object allocation.", type = OptionType.Expert)//
112-
public static final HostedOptionKey<Boolean> UseTLAB = new HostedOptionKey<>(true);
107+
@Option(help = "Use thread-local object allocation.", type = OptionType.Expert)//
108+
public static final HostedOptionKey<Boolean> UseTLAB = new HostedOptionKey<>(true);
113109

114-
@Option(help = "Dynamically resize TLAB size for threads.", type = OptionType.Expert)//
115-
public static final RuntimeOptionKey<Boolean> ResizeTLAB = new RuntimeOptionKey<>(true, IsolateCreationOnly);
110+
@Option(help = "Dynamically resize TLAB size for threads.", type = OptionType.Expert)//
111+
public static final RuntimeOptionKey<Boolean> ResizeTLAB = new RuntimeOptionKey<>(true, IsolateCreationOnly);
116112

113+
@DuplicatedInNativeCode
114+
public static class ConcealedOptions {
115+
/** Use GC-specific accessors instead. */
117116
@Option(help = "Minimum allowed TLAB size (in bytes).", type = OptionType.Expert)//
118-
public static final RuntimeOptionKey<Long> MinTLABSize = new RuntimeOptionKey<>(2L * K, RegisterForIsolateArgumentParser);
117+
public static final RuntimeOptionKey<Long> MinTLABSize = new RuntimeOptionKey<>(0L, RegisterForIsolateArgumentParser);
119118

119+
/** Use GC-specific accessors instead. */
120120
@Option(help = "Starting TLAB size (in bytes); zero means set ergonomically.", type = OptionType.Expert)//
121121
public static final RuntimeOptionKey<Long> TLABSize = new RuntimeOptionKey<>(0L, RegisterForIsolateArgumentParser);
122-
123122
}
124-
125123
}

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/gc/shared/graal/NativeGCAllocationSupport.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ public ForeignCallDescriptor getNewPodInstanceStub() {
108108

109109
@Override
110110
public boolean useTLAB() {
111-
return SubstrateGCOptions.TlabOptions.UseTLAB.getValue();
111+
return SubstrateGCOptions.UseTLAB.getValue();
112112
}
113113

114114
@Override

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/RuntimeOptionValidationSupport.java

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,6 @@ public void validate() {
6666
}
6767

6868
public static class RuntimeOptionValidation<T> {
69-
7069
private final Consumer<RuntimeOptionKey<T>> validation;
7170
private final RuntimeOptionKey<T> optionKey;
7271

@@ -78,7 +77,5 @@ public RuntimeOptionValidation(Consumer<RuntimeOptionKey<T>> validation, Runtime
7877
void validate() {
7978
validation.accept(optionKey);
8079
}
81-
8280
}
83-
8481
}

0 commit comments

Comments
 (0)