From d67a38d2f3f2d15f0c0d366dc3036347910dc77d Mon Sep 17 00:00:00 2001 From: Dave Marion Date: Fri, 18 Apr 2025 21:23:03 +0000 Subject: [PATCH 01/11] Modified IteratorEnvironment to no longer throw UOE Modified the IteratorEnvironment interface such that the methods no longer have default implementation that throw UnsupportedOperationException. Created the class ClientIteratorEnvironment to serve as the default implementation for the client objects that defined their own implementation and for use in the tests. Removed all test implementations of IteratorEnvironment. Closes #4810 --- .../client/ClientSideIteratorScanner.java | 74 +------ .../core/client/rfile/RFileScanner.java | 75 ++++--- .../core/clientImpl/OfflineIterator.java | 115 +--------- .../iterators/ClientIteratorEnvironment.java | 204 ++++++++++++++++++ .../core/iterators/IteratorEnvironment.java | 52 ++--- .../accumulo/core/file/rfile/RFileTest.java | 37 +--- .../iterators/DefaultIteratorEnvironment.java | 56 ----- .../FirstEntryInRowIteratorTest.java | 3 +- .../core/iterators/SortedMapIteratorTest.java | 15 +- .../iterators/user/ColumnSliceFilterTest.java | 4 +- .../core/iterators/user/CombinerTest.java | 31 +-- .../core/iterators/user/FilterTest.java | 20 +- .../user/IndexedDocIteratorTest.java | 4 +- .../user/IntersectingIteratorTest.java | 4 +- .../iterators/user/LargeRowFilterTest.java | 3 +- .../core/iterators/user/RegExFilterTest.java | 32 +-- .../user/RowDeletingIteratorTest.java | 34 +-- .../user/RowEncodingIteratorTest.java | 22 +- .../core/iterators/user/RowFilterTest.java | 16 +- .../user/TransformingIteratorTest.java | 13 +- .../iteratorsImpl/IteratorConfigUtilTest.java | 5 +- .../iteratortest/IteratorTestInput.java | 10 +- .../SimpleIteratorEnvironment.java | 33 --- .../iterators/TabletIteratorEnvironment.java | 6 + .../replication/StatusCombinerTest.java | 12 +- .../accumulo/tserver/InMemoryMapTest.java | 80 ++++--- .../performance/scan/CollectTabletStats.java | 14 +- .../test/iterator/SummingCombinerTest.java | 12 +- 28 files changed, 423 insertions(+), 563 deletions(-) create mode 100644 core/src/main/java/org/apache/accumulo/core/iterators/ClientIteratorEnvironment.java delete mode 100644 core/src/test/java/org/apache/accumulo/core/iterators/DefaultIteratorEnvironment.java delete mode 100644 iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/environments/SimpleIteratorEnvironment.java diff --git a/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java b/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java index 217a3bb31cd..089dd54286b 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java +++ b/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java @@ -45,6 +45,7 @@ import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.dataImpl.thrift.IterInfo; +import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorAdapter; import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; @@ -52,7 +53,6 @@ import org.apache.accumulo.core.iteratorsImpl.IteratorBuilder; import org.apache.accumulo.core.iteratorsImpl.IteratorConfigUtil; import org.apache.accumulo.core.security.Authorizations; -import org.apache.accumulo.core.spi.common.ServiceEnvironment; import org.apache.hadoop.io.Text; /** @@ -87,70 +87,6 @@ public class ClientSideIteratorScanner extends ScannerOptions implements Scanner private final Supplier context; private final Supplier tableId; - private class ClientSideIteratorEnvironment implements IteratorEnvironment { - - private final SamplerConfiguration samplerConfig; - private final boolean sampleEnabled; - - ClientSideIteratorEnvironment(boolean sampleEnabled, SamplerConfiguration samplerConfig) { - this.sampleEnabled = sampleEnabled; - this.samplerConfig = samplerConfig; - } - - @Override - public IteratorScope getIteratorScope() { - return IteratorScope.scan; - } - - @Override - public boolean isFullMajorCompaction() { - // The javadocs state this method will throw an ISE when scope is not majc - throw new IllegalStateException( - "Asked about major compaction type when scope is " + getIteratorScope()); - } - - @Override - public boolean isUserCompaction() { - return false; - } - - @Override - public Authorizations getAuthorizations() { - return ClientSideIteratorScanner.this.getAuthorizations(); - } - - @Override - public IteratorEnvironment cloneWithSamplingEnabled() { - return new ClientSideIteratorEnvironment(true, samplerConfig); - } - - @Override - public boolean isSamplingEnabled() { - return sampleEnabled; - } - - @Override - public SamplerConfiguration getSamplerConfiguration() { - return samplerConfig; - } - - @Deprecated(since = "2.1.0") - @Override - public ServiceEnvironment getServiceEnv() { - return new ClientServiceEnvironmentImpl(context.get()); - } - - @Override - public PluginEnvironment getPluginEnv() { - return new ClientServiceEnvironmentImpl(context.get()); - } - - @Override - public TableId getTableId() { - return tableId.get(); - } - } - /** * A class that wraps a Scanner in a SortedKeyValueIterator so that other accumulo iterators can * use it as a source. @@ -295,9 +231,11 @@ public Iterator> iterator() { SortedKeyValueIterator skvi; try { - IteratorEnvironment iterEnv = new ClientSideIteratorEnvironment( - getSamplerConfiguration() != null, getIteratorSamplerConfigurationInternal()); - + IteratorEnvironment iterEnv = new ClientIteratorEnvironment.Builder() + .withServiceEnvironment(new ClientServiceEnvironmentImpl(context.get())) + .withAuthorizations(getAuthorizations()).withScope(IteratorScope.scan) + .withTableId(tableId.get()) + .withSamplerConfiguration(getIteratorSamplerConfigurationInternal()).build(); IteratorBuilder ib = IteratorBuilder.builder(tm.values()).opts(serverSideIteratorOptions).env(iterEnv).build(); diff --git a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java index be49a9c68ac..bfbf324ca9e 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java +++ b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java @@ -30,8 +30,8 @@ import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.rfile.RFileScannerBuilder.InputArgs; -import org.apache.accumulo.core.client.sample.SamplerConfiguration; import org.apache.accumulo.core.clientImpl.ScannerOptions; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.ConfigurationCopy; @@ -42,6 +42,7 @@ import org.apache.accumulo.core.data.Column; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.file.blockfile.cache.impl.BlockCacheConfiguration; import org.apache.accumulo.core.file.blockfile.cache.impl.BlockCacheManagerFactory; @@ -51,6 +52,7 @@ import org.apache.accumulo.core.file.blockfile.impl.CacheProvider; import org.apache.accumulo.core.file.rfile.RFile; import org.apache.accumulo.core.file.rfile.RFile.Reader; +import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorAdapter; import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; @@ -64,8 +66,10 @@ import org.apache.accumulo.core.spi.cache.BlockCache; import org.apache.accumulo.core.spi.cache.BlockCacheManager; import org.apache.accumulo.core.spi.cache.CacheType; +import org.apache.accumulo.core.spi.common.ServiceEnvironment; import org.apache.accumulo.core.spi.crypto.CryptoEnvironment; import org.apache.accumulo.core.spi.crypto.CryptoService; +import org.apache.accumulo.core.util.ConfigurationImpl; import org.apache.accumulo.core.util.LocalityGroupUtil; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.io.Text; @@ -225,33 +229,6 @@ public void updateScanIteratorOption(String iteratorName, String key, String val super.updateScanIteratorOption(iteratorName, key, value); } - private class IterEnv implements IteratorEnvironment { - @Override - public IteratorScope getIteratorScope() { - return IteratorScope.scan; - } - - @Override - public boolean isFullMajorCompaction() { - return false; - } - - @Override - public Authorizations getAuthorizations() { - return opts.auths; - } - - @Override - public boolean isSamplingEnabled() { - return RFileScanner.this.getSamplerConfiguration() != null; - } - - @Override - public SamplerConfiguration getSamplerConfiguration() { - return RFileScanner.this.getSamplerConfiguration(); - } - } - @Override public Iterator> iterator() { try { @@ -292,15 +269,53 @@ public Iterator> iterator() { EMPTY_BYTES, tableConf); } + ServiceEnvironment senv = new ServiceEnvironment() { + + @Override + public String getTableName(TableId tableId) throws TableNotFoundException { + throw new IllegalStateException("TableId not known in RFileScanner"); + } + + @Override + public T instantiate(String className, Class base) throws Exception { + return RFileScanner.class.getClassLoader().loadClass(className).asSubclass(base) + .getDeclaredConstructor().newInstance(); + } + + @Override + public T instantiate(TableId tableId, String className, Class base) + throws Exception { + return instantiate(className, base); + } + + @Override + public Configuration getConfiguration() { + return new ConfigurationImpl(new ConfigurationCopy(DefaultConfiguration.getInstance())); + } + + @Override + public Configuration getConfiguration(TableId tableId) { + return new ConfigurationImpl(new ConfigurationCopy(opts.tableConfig)); + } + + }; + + ClientIteratorEnvironment.Builder iterEnvBuilder = + new ClientIteratorEnvironment.Builder().withAuthorizations(opts.auths) + .withScope(IteratorScope.scan).withServiceEnvironment(senv); + if (getSamplerConfiguration() != null) { + iterEnvBuilder.withSamplerConfiguration(getSamplerConfiguration()); + } + IteratorEnvironment iterEnv = iterEnvBuilder.build(); try { if (opts.tableConfig != null && !opts.tableConfig.isEmpty()) { var ibEnv = IteratorConfigUtil.loadIterConf(IteratorScope.scan, serverSideIteratorList, serverSideIteratorOptions, tableConf); - var iteratorBuilder = ibEnv.env(new IterEnv()).build(); + var iteratorBuilder = ibEnv.env(iterEnv).build(); iterator = IteratorConfigUtil.loadIterators(iterator, iteratorBuilder); } else { var iteratorBuilder = IteratorBuilder.builder(serverSideIteratorList) - .opts(serverSideIteratorOptions).env(new IterEnv()).build(); + .opts(serverSideIteratorOptions).env(iterEnv).build(); iterator = IteratorConfigUtil.loadIterators(iterator, iteratorBuilder); } } catch (IOException e) { diff --git a/core/src/main/java/org/apache/accumulo/core/clientImpl/OfflineIterator.java b/core/src/main/java/org/apache/accumulo/core/clientImpl/OfflineIterator.java index a03cc811ab8..04bdb4e8311 100644 --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/OfflineIterator.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/OfflineIterator.java @@ -34,7 +34,6 @@ import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; -import org.apache.accumulo.core.client.PluginEnvironment; import org.apache.accumulo.core.client.SampleNotPresentException; import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.sample.SamplerConfiguration; @@ -51,6 +50,7 @@ import org.apache.accumulo.core.dataImpl.KeyExtent; import org.apache.accumulo.core.file.FileOperations; import org.apache.accumulo.core.file.FileSKVIterator; +import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; @@ -66,7 +66,6 @@ import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.security.ColumnVisibility; -import org.apache.accumulo.core.spi.common.ServiceEnvironment; import org.apache.accumulo.core.util.LocalityGroupUtil; import org.apache.accumulo.core.volume.VolumeConfiguration; import org.apache.hadoop.conf.Configuration; @@ -75,105 +74,6 @@ class OfflineIterator implements Iterator> { - static class OfflineIteratorEnvironment implements IteratorEnvironment { - - private final Authorizations authorizations; - private final AccumuloConfiguration conf; - private final boolean useSample; - private final SamplerConfiguration sampleConf; - private final ClientContext context; - private final TableId tableId; - - public OfflineIteratorEnvironment(ClientContext context, TableId tableId, Authorizations auths, - AccumuloConfiguration acuTableConf, boolean useSample, SamplerConfiguration samplerConf) { - this.context = context; - this.tableId = tableId; - this.authorizations = auths; - this.conf = acuTableConf; - this.useSample = useSample; - this.sampleConf = samplerConf; - } - - @Deprecated(since = "2.0.0") - @Override - public AccumuloConfiguration getConfig() { - return conf; - } - - @Override - public IteratorScope getIteratorScope() { - return IteratorScope.scan; - } - - @Override - public boolean isFullMajorCompaction() { - return false; - } - - @Override - public boolean isUserCompaction() { - return false; - } - - private final ArrayList> topLevelIterators = - new ArrayList<>(); - - @Deprecated(since = "2.0.0") - @Override - public void registerSideChannel(SortedKeyValueIterator iter) { - topLevelIterators.add(iter); - } - - @Override - public Authorizations getAuthorizations() { - return authorizations; - } - - SortedKeyValueIterator getTopLevelIterator(SortedKeyValueIterator iter) { - if (topLevelIterators.isEmpty()) { - return iter; - } - ArrayList> allIters = new ArrayList<>(topLevelIterators); - allIters.add(iter); - return new MultiIterator(allIters, false); - } - - @Override - public boolean isSamplingEnabled() { - return useSample; - } - - @Override - public SamplerConfiguration getSamplerConfiguration() { - return sampleConf; - } - - @Override - public IteratorEnvironment cloneWithSamplingEnabled() { - if (sampleConf == null) { - throw new SampleNotPresentException(); - } - return new OfflineIteratorEnvironment(context, tableId, authorizations, conf, true, - sampleConf); - } - - @Deprecated(since = "2.1.0") - @Override - public ServiceEnvironment getServiceEnv() { - return new ClientServiceEnvironmentImpl(context); - } - - @Override - public PluginEnvironment getPluginEnv() { - return new ClientServiceEnvironmentImpl(context); - } - - @Override - public TableId getTableId() { - return tableId; - } - } - private SortedKeyValueIterator iter; private Range range; private KeyExtent currentExtent; @@ -345,9 +245,13 @@ private SortedKeyValueIterator createIterator(KeyExtent extent, MultiIterator multiIter = new MultiIterator(readers, extent); - OfflineIteratorEnvironment iterEnv = - new OfflineIteratorEnvironment(context, tableId, authorizations, tableCC, false, - samplerConfImpl == null ? null : samplerConfImpl.toSamplerConfiguration()); + ClientIteratorEnvironment.Builder iterEnvBuilder = new ClientIteratorEnvironment.Builder() + .withAuthorizations(authorizations).withScope(IteratorScope.scan).withTableId(tableId) + .withServiceEnvironment(new ClientServiceEnvironmentImpl(context)); + if (scannerSamplerConfig != null) { + iterEnvBuilder.withSamplerConfiguration(scannerSamplerConfig); + } + IteratorEnvironment iterEnv = iterEnvBuilder.build(); byte[] defaultSecurityLabel; ColumnVisibility cv = @@ -360,8 +264,7 @@ private SortedKeyValueIterator createIterator(KeyExtent extent, var iteratorBuilderEnv = IteratorConfigUtil.loadIterConf(IteratorScope.scan, options.serverSideIteratorList, options.serverSideIteratorOptions, tableCC); var iteratorBuilder = iteratorBuilderEnv.env(iterEnv).build(); - return iterEnv - .getTopLevelIterator(IteratorConfigUtil.loadIterators(visFilter, iteratorBuilder)); + return IteratorConfigUtil.loadIterators(visFilter, iteratorBuilder); } @Override diff --git a/core/src/main/java/org/apache/accumulo/core/iterators/ClientIteratorEnvironment.java b/core/src/main/java/org/apache/accumulo/core/iterators/ClientIteratorEnvironment.java new file mode 100644 index 00000000000..da41f9c9cd6 --- /dev/null +++ b/core/src/main/java/org/apache/accumulo/core/iterators/ClientIteratorEnvironment.java @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.core.iterators; + +import java.io.IOException; +import java.util.Optional; + +import org.apache.accumulo.core.client.PluginEnvironment; +import org.apache.accumulo.core.client.SampleNotPresentException; +import org.apache.accumulo.core.client.sample.SamplerConfiguration; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.TableId; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; +import org.apache.accumulo.core.iteratorsImpl.system.MapFileIterator; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.core.spi.common.ServiceEnvironment; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; + +public class ClientIteratorEnvironment implements IteratorEnvironment { + + public static class Builder { + + private Optional scope = Optional.empty(); + private boolean isFullMajorCompaction = false; + private Optional auths = Optional.empty(); + private boolean isUserCompaction = false; + private Optional tableId = Optional.empty(); + private Optional samplerConfig = Optional.empty(); + private Optional env = Optional.empty(); + + public Builder withScope(IteratorScope scope) { + this.scope = Optional.of(scope); + return this; + } + + public Builder isFullMajorCompaction() { + this.isFullMajorCompaction = true; + return this; + } + + public Builder withAuthorizations(Authorizations auths) { + this.auths = Optional.of(auths); + return this; + } + + public Builder isUserCompaction() { + this.isUserCompaction = true; + return this; + } + + public Builder withTableId(TableId tableId) { + this.tableId = Optional.of(tableId); + return this; + } + + public Builder withSamplerConfiguration(SamplerConfiguration sc) { + this.samplerConfig = Optional.of(sc); + return this; + } + + public Builder withServiceEnvironment(ServiceEnvironment env) { + this.env = Optional.of(env); + return this; + } + + public ClientIteratorEnvironment build() { + return new ClientIteratorEnvironment(this); + } + + } + + private static final UnsupportedOperationException UOE = + new UnsupportedOperationException("Feature not supported"); + + public static final IteratorEnvironment DEFAULT = new Builder().build(); + + private final Optional scope; + private final boolean isFullMajorCompaction; + private final Optional auths; + private final boolean isUserCompaction; + private final Optional tableId; + private final Optional samplerConfig; + private final Optional env; + + private ClientIteratorEnvironment(Builder builder) { + this.scope = builder.scope; + this.isFullMajorCompaction = builder.isFullMajorCompaction; + this.auths = builder.auths; + this.isUserCompaction = builder.isUserCompaction; + this.tableId = builder.tableId; + this.samplerConfig = builder.samplerConfig; + this.env = builder.env; + } + + private ClientIteratorEnvironment(ClientIteratorEnvironment copy) { + this.scope = copy.scope.isPresent() ? Optional.of(copy.scope.orElseThrow()) : Optional.empty(); + this.isFullMajorCompaction = copy.isFullMajorCompaction; + this.auths = copy.auths.isPresent() ? Optional.of(copy.auths.orElseThrow()) : Optional.empty(); + this.isUserCompaction = copy.isUserCompaction; + this.tableId = + copy.tableId.isPresent() ? Optional.of(copy.tableId.orElseThrow()) : Optional.empty(); + this.samplerConfig = copy.samplerConfig.isPresent() + ? Optional.of(copy.samplerConfig.orElseThrow()) : Optional.empty(); + this.env = copy.env.isPresent() ? Optional.of(copy.env.orElseThrow()) : Optional.empty(); + } + + @Override + @Deprecated(since = "2.0.0") + public SortedKeyValueIterator reserveMapFileReader(String mapFileName) + throws IOException { + Configuration hadoopConf = new Configuration(); + FileSystem fs = FileSystem.get(hadoopConf); + return new MapFileIterator(fs, mapFileName, hadoopConf); + } + + @Override + public IteratorScope getIteratorScope() { + return scope.orElseThrow(); + } + + @Override + public boolean isFullMajorCompaction() { + if (getIteratorScope() != IteratorScope.majc) { + throw new IllegalStateException("Iterator scope is not majc"); + } + return isFullMajorCompaction; + } + + @Override + @Deprecated(since = "2.0.0") + public void registerSideChannel(SortedKeyValueIterator iter) { + throw UOE; + } + + @Override + public Authorizations getAuthorizations() { + if (getIteratorScope() != IteratorScope.scan) { + throw new IllegalStateException("Iterator scope is not majc"); + } + return auths.orElseThrow(); + } + + @Override + public IteratorEnvironment cloneWithSamplingEnabled() { + if (!isSamplingEnabled()) { + throw new SampleNotPresentException(); + } + return new ClientIteratorEnvironment(this); + } + + @Override + public boolean isSamplingEnabled() { + return this.samplerConfig.isPresent(); + } + + @Override + public SamplerConfiguration getSamplerConfiguration() { + return samplerConfig.orElse(null); + } + + @Override + public boolean isUserCompaction() { + if (getIteratorScope() == IteratorScope.scan) { + throw new IllegalStateException( + "scan iterator scope is incompatible with a possible user compaction"); + } + return this.isUserCompaction; + } + + @Override + @Deprecated(since = "2.1.0") + public ServiceEnvironment getServiceEnv() { + return env.orElseThrow(); + } + + @Override + public PluginEnvironment getPluginEnv() { + return getServiceEnv(); + } + + @Override + public TableId getTableId() { + return this.tableId.orElseThrow(); + } + +} diff --git a/core/src/main/java/org/apache/accumulo/core/iterators/IteratorEnvironment.java b/core/src/main/java/org/apache/accumulo/core/iterators/IteratorEnvironment.java index 372a0e49a30..1646a196388 100644 --- a/core/src/main/java/org/apache/accumulo/core/iterators/IteratorEnvironment.java +++ b/core/src/main/java/org/apache/accumulo/core/iterators/IteratorEnvironment.java @@ -24,6 +24,7 @@ import org.apache.accumulo.core.client.SampleNotPresentException; import org.apache.accumulo.core.client.sample.SamplerConfiguration; import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.ConfigurationCopy; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.data.Value; @@ -37,10 +38,7 @@ public interface IteratorEnvironment { * @deprecated since 2.0.0. This is a legacy method used for internal backwards compatibility. */ @Deprecated(since = "2.0.0") - default SortedKeyValueIterator reserveMapFileReader(String mapFileName) - throws IOException { - throw new UnsupportedOperationException(); - } + SortedKeyValueIterator reserveMapFileReader(String mapFileName) throws IOException; /** * @deprecated since 2.0.0. This method was using an unstable non public type. Use @@ -48,40 +46,32 @@ default SortedKeyValueIterator reserveMapFileReader(String mapFileNam */ @Deprecated(since = "2.0.0") default AccumuloConfiguration getConfig() { - throw new UnsupportedOperationException(); + return new ConfigurationCopy(getPluginEnv().getConfiguration()); } /** * Return the executed scope of the Iterator. Value will be one of the following: * {@link IteratorScope#scan}, {@link IteratorScope#minc}, {@link IteratorScope#majc} */ - default IteratorScope getIteratorScope() { - throw new UnsupportedOperationException(); - } + IteratorScope getIteratorScope(); /** * Return true if the compaction is a full major compaction. Will throw IllegalStateException if * {@link #getIteratorScope()} != {@link IteratorScope#majc}. */ - default boolean isFullMajorCompaction() { - throw new UnsupportedOperationException(); - } + boolean isFullMajorCompaction(); /** * @deprecated since 2.0.0. This was an experimental feature and was never tested or documented. */ @Deprecated(since = "2.0.0") - default void registerSideChannel(SortedKeyValueIterator iter) { - throw new UnsupportedOperationException(); - } + void registerSideChannel(SortedKeyValueIterator iter); /** * Return the Scan Authorizations used in this Iterator. Will throw UnsupportedOperationException * if {@link #getIteratorScope()} != {@link IteratorScope#scan}. */ - default Authorizations getAuthorizations() { - throw new UnsupportedOperationException(); - } + Authorizations getAuthorizations(); /** * Returns a new iterator environment object that can be used to create deep copies over sample @@ -113,9 +103,7 @@ default Authorizations getAuthorizations() { * @throws SampleNotPresentException when sampling is not configured for table. * @since 1.8.0 */ - default IteratorEnvironment cloneWithSamplingEnabled() { - throw new UnsupportedOperationException(); - } + IteratorEnvironment cloneWithSamplingEnabled(); /** * There are at least two conditions under which sampling will be enabled for an environment. One @@ -126,27 +114,21 @@ default IteratorEnvironment cloneWithSamplingEnabled() { * @return true if sampling is enabled for this environment. * @since 1.8.0 */ - default boolean isSamplingEnabled() { - throw new UnsupportedOperationException(); - } + boolean isSamplingEnabled(); /** * * @return sampling configuration is sampling is enabled for environment, otherwise returns null. * @since 1.8.0 */ - default SamplerConfiguration getSamplerConfiguration() { - throw new UnsupportedOperationException(); - } + SamplerConfiguration getSamplerConfiguration(); /** * True if compaction was user initiated. * * @since 2.0.0 */ - default boolean isUserCompaction() { - throw new UnsupportedOperationException(); - } + boolean isUserCompaction(); /** * Returns an object containing information about the server where this iterator was run. To @@ -161,9 +143,7 @@ default boolean isUserCompaction() { * {@link #getPluginEnv()} instead because it has better stability guarantees. */ @Deprecated(since = "2.1.0") - default ServiceEnvironment getServiceEnv() { - throw new UnsupportedOperationException(); - } + ServiceEnvironment getServiceEnv(); /** * Returns an object containing information about the server where this iterator was run. To @@ -175,16 +155,12 @@ default ServiceEnvironment getServiceEnv() { * * @since 2.1.0 */ - default PluginEnvironment getPluginEnv() { - return getServiceEnv(); - } + PluginEnvironment getPluginEnv(); /** * Return the table Id associated with this iterator. * * @since 2.0.0 */ - default TableId getTableId() { - throw new UnsupportedOperationException(); - } + TableId getTableId(); } diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java index 4b0c533dd1d..59c522a59f8 100644 --- a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java +++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java @@ -49,7 +49,6 @@ import org.apache.accumulo.core.client.sample.RowSampler; import org.apache.accumulo.core.client.sample.Sampler; -import org.apache.accumulo.core.client.sample.SamplerConfiguration; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.ConfigurationCopy; import org.apache.accumulo.core.conf.DefaultConfiguration; @@ -72,6 +71,7 @@ import org.apache.accumulo.core.file.blockfile.impl.CachableBlockFile.CachableBuilder; import org.apache.accumulo.core.file.rfile.RFile.Reader; import org.apache.accumulo.core.file.rfile.bcfile.BCFile; +import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; import org.apache.accumulo.core.iteratorsImpl.system.ColumnFamilySkippingIterator; @@ -107,26 +107,6 @@ public class RFileTest { private static final SecureRandom random = new SecureRandom(); - - public static class SampleIE implements IteratorEnvironment { - - private SamplerConfiguration samplerConfig; - - SampleIE(SamplerConfiguration config) { - this.samplerConfig = config; - } - - @Override - public boolean isSamplingEnabled() { - return samplerConfig != null; - } - - @Override - public SamplerConfiguration getSamplerConfiguration() { - return samplerConfig; - } - } - private static final Collection EMPTY_COL_FAMS = new ArrayList<>(); private static final Configuration hadoopConf = new Configuration(); @@ -2068,15 +2048,16 @@ public void testSample() throws IOException { trf.openReader(); - FileSKVIterator sample = - trf.reader.getSample(SamplerConfigurationImpl.newSamplerConfig(sampleConf)); + SamplerConfigurationImpl sc = SamplerConfigurationImpl.newSamplerConfig(sampleConf); + + FileSKVIterator sample = trf.reader.getSample(sc); checkSample(sample, sampleData); assertEquals(expectedDataHash, hash(trf.reader)); - SampleIE ie = new SampleIE( - SamplerConfigurationImpl.newSamplerConfig(sampleConf).toSamplerConfiguration()); + IteratorEnvironment ie = new ClientIteratorEnvironment.Builder() + .withSamplerConfiguration(sc.toSamplerConfiguration()).build(); for (int i = 0; i < 3; i++) { // test opening and closing deep copies a few times. @@ -2086,8 +2067,10 @@ public void testSample() throws IOException { SortedKeyValueIterator sampleDC1 = sample.deepCopy(ie); SortedKeyValueIterator sampleDC2 = sample.deepCopy(ie); SortedKeyValueIterator sampleDC3 = trf.reader.deepCopy(ie); - SortedKeyValueIterator allDC1 = sampleDC1.deepCopy(new SampleIE(null)); - SortedKeyValueIterator allDC2 = sample.deepCopy(new SampleIE(null)); + SortedKeyValueIterator allDC1 = + sampleDC1.deepCopy(ClientIteratorEnvironment.DEFAULT); + SortedKeyValueIterator allDC2 = + sample.deepCopy(ClientIteratorEnvironment.DEFAULT); assertEquals(expectedDataHash, hash(allDC1)); assertEquals(expectedDataHash, hash(allDC2)); diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/DefaultIteratorEnvironment.java b/core/src/test/java/org/apache/accumulo/core/iterators/DefaultIteratorEnvironment.java deleted file mode 100644 index 802dfe849cc..00000000000 --- a/core/src/test/java/org/apache/accumulo/core/iterators/DefaultIteratorEnvironment.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.accumulo.core.iterators; - -import java.io.IOException; - -import org.apache.accumulo.core.conf.AccumuloConfiguration; -import org.apache.accumulo.core.conf.DefaultConfiguration; -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iteratorsImpl.system.MapFileIterator; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; - -public class DefaultIteratorEnvironment implements IteratorEnvironment { - - AccumuloConfiguration conf; - Configuration hadoopConf = new Configuration(); - - public DefaultIteratorEnvironment(AccumuloConfiguration conf) { - this.conf = conf; - } - - public DefaultIteratorEnvironment() { - this.conf = DefaultConfiguration.getInstance(); - } - - @Deprecated(since = "2.0.0") - @Override - public SortedKeyValueIterator reserveMapFileReader(String mapFileName) - throws IOException { - FileSystem fs = FileSystem.get(hadoopConf); - return new MapFileIterator(fs, mapFileName, hadoopConf); - } - - @Override - public boolean isSamplingEnabled() { - return false; - } -} diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/FirstEntryInRowIteratorTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/FirstEntryInRowIteratorTest.java index 5a7f6dbcc07..c76a77bf0d6 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/FirstEntryInRowIteratorTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/FirstEntryInRowIteratorTest.java @@ -41,9 +41,8 @@ private static long process(TreeMap sourceMap, TreeMap res SortedMapIterator source = new SortedMapIterator(sourceMap); CountingIterator counter = new CountingIterator(source); FirstEntryInRowIterator feiri = new FirstEntryInRowIterator(); - IteratorEnvironment env = new DefaultIteratorEnvironment(); - feiri.init(counter, iteratorSetting.getOptions(), env); + feiri.init(counter, iteratorSetting.getOptions(), ClientIteratorEnvironment.DEFAULT); feiri.seek(range, Set.of(), false); while (feiri.hasTop()) { diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/SortedMapIteratorTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/SortedMapIteratorTest.java index c35cf8f1c6a..9c0441187b0 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/SortedMapIteratorTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/SortedMapIteratorTest.java @@ -33,16 +33,9 @@ public class SortedMapIteratorTest { @Test public void testSampleNotPresent() { SortedMapIterator smi = new SortedMapIterator(new TreeMap<>()); - assertThrows(SampleNotPresentException.class, () -> smi.deepCopy(new IteratorEnvironment() { - @Override - public boolean isSamplingEnabled() { - return true; - } - - @Override - public SamplerConfiguration getSamplerConfiguration() { - return new SamplerConfiguration(RowSampler.class.getName()); - } - })); + assertThrows(SampleNotPresentException.class, + () -> smi.deepCopy(new ClientIteratorEnvironment.Builder() + .withSamplerConfiguration(new SamplerConfiguration(RowSampler.class.getName())) + .build())); } } diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/ColumnSliceFilterTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/ColumnSliceFilterTest.java index 1fe7f3ac524..18225c5b1b3 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/ColumnSliceFilterTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/ColumnSliceFilterTest.java @@ -34,7 +34,7 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.DefaultIteratorEnvironment; +import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator; import org.apache.hadoop.io.Text; @@ -72,7 +72,7 @@ private static Key newKey(String row, String cf, String cq) { @BeforeEach public void setUp() { columnSliceFilter.describeOptions(); - iteratorEnvironment = new DefaultIteratorEnvironment(); + iteratorEnvironment = ClientIteratorEnvironment.DEFAULT; is = new IteratorSetting(1, ColumnSliceFilter.class); } diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java index df757966903..d6e0f7ca279 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java @@ -37,10 +37,10 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; import org.apache.accumulo.core.iterators.Combiner; import org.apache.accumulo.core.iterators.Combiner.ValueIterator; import org.apache.accumulo.core.iterators.CombinerTestUtil; -import org.apache.accumulo.core.iterators.DefaultIteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; import org.apache.accumulo.core.iterators.LongCombiner; @@ -59,29 +59,8 @@ public class CombinerTest { private static final Collection EMPTY_COL_FAMS = new ArrayList<>(); - static class CombinerIteratorEnvironment extends DefaultIteratorEnvironment { - - private IteratorScope scope; - private boolean isFullMajc; - - CombinerIteratorEnvironment(IteratorScope scope, boolean isFullMajc) { - this.scope = scope; - this.isFullMajc = isFullMajc; - } - - @Override - public IteratorScope getIteratorScope() { - return scope; - } - - @Override - public boolean isFullMajorCompaction() { - return isFullMajc; - } - } - static final IteratorEnvironment SCAN_IE = - new CombinerIteratorEnvironment(IteratorScope.scan, false); + new ClientIteratorEnvironment.Builder().withScope(IteratorScope.scan).build(); static Key newKey(int row, int colf, int colq, long ts, boolean deleted) { Key k = newKey(row, colf, colq, ts); @@ -884,8 +863,10 @@ public void testDeleteHandling() throws Exception { TreeMap input = new TreeMap<>(); - IteratorEnvironment paritalMajcIe = new CombinerIteratorEnvironment(IteratorScope.majc, false); - IteratorEnvironment fullMajcIe = new CombinerIteratorEnvironment(IteratorScope.majc, true); + IteratorEnvironment paritalMajcIe = + new ClientIteratorEnvironment.Builder().withScope(IteratorScope.majc).build(); + IteratorEnvironment fullMajcIe = new ClientIteratorEnvironment.Builder() + .withScope(IteratorScope.majc).isFullMajorCompaction().build(); // keys that aggregate newKeyValue(input, 1, 1, 1, 1, false, 4L, encoder); diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/FilterTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/FilterTest.java index 996b3df97a5..a3574cbc8b5 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/FilterTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/FilterTest.java @@ -40,7 +40,7 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.DefaultIteratorEnvironment; +import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; import org.apache.accumulo.core.iterators.Filter; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; import org.apache.accumulo.core.iteratorsImpl.system.ColumnQualifierFilter; @@ -230,19 +230,19 @@ public void test2a() throws IOException { ColumnAgeOffFilter a = new ColumnAgeOffFilter(); assertTrue(a.validateOptions(is.getOptions())); - a.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + a.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); a.overrideCurrentTime(ts); a.seek(new Range(), EMPTY_COL_FAMS, false); assertEquals(902, size(a)); ColumnAgeOffFilter.addTTL(is, new IteratorSetting.Column("a", "b"), 101L); - a.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + a.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); a.overrideCurrentTime(ts); a.seek(new Range(), EMPTY_COL_FAMS, false); assertEquals(102, size(a)); ColumnAgeOffFilter.removeTTL(is, new IteratorSetting.Column("a", "b")); - a.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + a.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); a = (ColumnAgeOffFilter) a.deepCopy(null); a.overrideCurrentTime(ts); a.seek(new Range(), EMPTY_COL_FAMS, false); @@ -271,19 +271,19 @@ public void test2aNegate() throws IOException { ColumnAgeOffFilter a = new ColumnAgeOffFilter(); assertTrue(a.validateOptions(is.getOptions())); - a.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + a.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); a.overrideCurrentTime(ts); a.seek(new Range(), EMPTY_COL_FAMS, false); assertEquals(98, size(a)); ColumnAgeOffFilter.addTTL(is, new IteratorSetting.Column("a", "b"), 101L); - a.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + a.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); a.overrideCurrentTime(ts); a.seek(new Range(), EMPTY_COL_FAMS, false); assertEquals(898, size(a)); ColumnAgeOffFilter.removeTTL(is, new IteratorSetting.Column("a", "b")); - a.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + a.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); a = (ColumnAgeOffFilter) a.deepCopy(null); a.overrideCurrentTime(ts); a.seek(new Range(), EMPTY_COL_FAMS, false); @@ -312,19 +312,19 @@ public void test2b() throws IOException { ColumnAgeOffFilter a = new ColumnAgeOffFilter(); assertTrue(a.validateOptions(is.getOptions())); - a.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + a.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); a.overrideCurrentTime(ts); a.seek(new Range(), EMPTY_COL_FAMS, false); assertEquals(902, size(a)); ColumnAgeOffFilter.addTTL(is, new IteratorSetting.Column("negate", "b"), 101L); - a.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + a.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); a.overrideCurrentTime(ts); a.seek(new Range(), EMPTY_COL_FAMS, false); assertEquals(102, size(a)); ColumnAgeOffFilter.removeTTL(is, new IteratorSetting.Column("negate", "b")); - a.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + a.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); a = (ColumnAgeOffFilter) a.deepCopy(null); a.overrideCurrentTime(ts); a.seek(new Range(), EMPTY_COL_FAMS, false); diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/IndexedDocIteratorTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/IndexedDocIteratorTest.java index fdb34e1fa36..8a1d15ed2b1 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/IndexedDocIteratorTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/IndexedDocIteratorTest.java @@ -38,7 +38,7 @@ import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.file.rfile.RFileTest; import org.apache.accumulo.core.file.rfile.RFileTest.TestRFile; -import org.apache.accumulo.core.iterators.DefaultIteratorEnvironment; +import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; import org.apache.accumulo.core.iteratorsImpl.system.MultiIterator; @@ -51,7 +51,7 @@ public class IndexedDocIteratorTest { private static final Collection EMPTY_COL_FAMS = new ArrayList<>(); private static final byte[] nullByte = {0}; - private static IteratorEnvironment env = new DefaultIteratorEnvironment(); + private static IteratorEnvironment env = ClientIteratorEnvironment.DEFAULT; Text[] columnFamilies; Text[] otherColumnFamilies; diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/IntersectingIteratorTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/IntersectingIteratorTest.java index 2c3710d9579..e2eedb454f1 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/IntersectingIteratorTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/IntersectingIteratorTest.java @@ -33,7 +33,7 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.DefaultIteratorEnvironment; +import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; import org.apache.accumulo.core.iteratorsImpl.system.MultiIterator; @@ -45,7 +45,7 @@ public class IntersectingIteratorTest { private static final SecureRandom random = new SecureRandom(); private static final Collection EMPTY_COL_FAMS = new ArrayList<>(); - private static IteratorEnvironment env = new DefaultIteratorEnvironment(); + private static IteratorEnvironment env = ClientIteratorEnvironment.DEFAULT; HashSet docs = new HashSet<>(); Text[] columnFamilies; diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/LargeRowFilterTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/LargeRowFilterTest.java index b0a83759348..2a4523e6561 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/LargeRowFilterTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/LargeRowFilterTest.java @@ -33,6 +33,7 @@ import org.apache.accumulo.core.data.PartialKey; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; import org.apache.accumulo.core.iteratorsImpl.system.ColumnFamilySkippingIterator; import org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator; @@ -67,7 +68,7 @@ private LargeRowFilter setupIterator(TreeMap testData, int maxColumns IteratorSetting is = new IteratorSetting(1, LargeRowFilter.class); LargeRowFilter.setMaxColumns(is, maxColumns); lrfi.init(new ColumnFamilySkippingIterator(smi), is.getOptions(), - new RowDeletingIteratorTest.TestIE(scope, false)); + new ClientIteratorEnvironment.Builder().withScope(scope).build()); return lrfi; } diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/RegExFilterTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/RegExFilterTest.java index 3a558374b4a..f2de0ebc9bb 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/RegExFilterTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/RegExFilterTest.java @@ -33,7 +33,7 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.DefaultIteratorEnvironment; +import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; import org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator; import org.apache.hadoop.io.Text; import org.junit.jupiter.api.Test; @@ -67,7 +67,7 @@ public void test1() throws IOException { RegExFilter.setRegexs(is, ".*2", null, null, null, false); assertTrue(rei.validateOptions(is.getOptions())); - rei.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + rei.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); rei.seek(new Range(), EMPTY_COL_FAMS, false); assertTrue(rei.hasTop()); @@ -82,7 +82,7 @@ public void test1() throws IOException { RegExFilter.setRegexs(is, null, null, null, "amst", false, true); // Should only match hamster rei.validateOptions(is.getOptions()); - rei.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + rei.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); rei.seek(new Range(), EMPTY_COL_FAMS, false); assertTrue(rei.hasTop()); @@ -95,7 +95,7 @@ public void test1() throws IOException { RegExFilter.setRegexs(is, null, "ya.*", null, null, false); assertTrue(rei.validateOptions(is.getOptions())); - rei.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + rei.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); rei.seek(new Range(), EMPTY_COL_FAMS, false); assertTrue(rei.hasTop()); @@ -108,7 +108,7 @@ public void test1() throws IOException { RegExFilter.setRegexs(is, null, null, ".*01", null, false); assertTrue(rei.validateOptions(is.getOptions())); - rei.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + rei.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); rei.seek(new Range(), EMPTY_COL_FAMS, false); assertTrue(rei.hasTop()); @@ -121,7 +121,7 @@ public void test1() throws IOException { RegExFilter.setRegexs(is, null, null, null, ".*at", false); assertTrue(rei.validateOptions(is.getOptions())); - rei.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + rei.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); rei.seek(new Range(), EMPTY_COL_FAMS, false); assertTrue(rei.hasTop()); @@ -133,7 +133,7 @@ public void test1() throws IOException { is.clearOptions(); RegExFilter.setRegexs(is, null, null, null, ".*ap", false); - rei.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + rei.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); rei.seek(new Range(), EMPTY_COL_FAMS, false); assertFalse(rei.hasTop()); @@ -142,7 +142,7 @@ public void test1() throws IOException { is.clearOptions(); RegExFilter.setRegexs(is, null, "ya.*", null, ".*at", false); - rei.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + rei.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); rei.seek(new Range(), EMPTY_COL_FAMS, false); assertTrue(rei.hasTop()); @@ -154,7 +154,7 @@ public void test1() throws IOException { is.clearOptions(); RegExFilter.setRegexs(is, null, "ya.*", null, ".*ap", false); - rei.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + rei.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); rei.seek(new Range(), EMPTY_COL_FAMS, false); assertFalse(rei.hasTop()); @@ -163,7 +163,7 @@ public void test1() throws IOException { is.clearOptions(); RegExFilter.setRegexs(is, "boo1", null, null, null, false); - rei.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + rei.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); rei.seek(new Range(), EMPTY_COL_FAMS, false); assertTrue(rei.hasTop()); @@ -177,7 +177,7 @@ public void test1() throws IOException { // ----------------------------------------------------- is.clearOptions(); - rei.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + rei.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); rei.seek(new Range(), EMPTY_COL_FAMS, false); assertTrue(rei.hasTop()); @@ -195,7 +195,7 @@ public void test1() throws IOException { is.clearOptions(); RegExFilter.setRegexs(is, "hamster", null, "hamster", "hamster", true); - rei.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + rei.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); rei.seek(new Range(), EMPTY_COL_FAMS, false); assertTrue(rei.hasTop()); @@ -207,7 +207,7 @@ public void test1() throws IOException { is.clearOptions(); RegExFilter.setRegexs(is, null, "ya.*", "hamster", null, true); - rei.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + rei.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); rei.seek(new Range(), EMPTY_COL_FAMS, false); assertTrue(rei.hasTop()); @@ -218,9 +218,9 @@ public void test1() throws IOException { is.clearOptions(); RegExFilter.setRegexs(is, null, "ya.*", "hamster", null, true); - rei.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + rei.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); rei.seek(new Range(), EMPTY_COL_FAMS, false); - rei.deepCopy(new DefaultIteratorEnvironment()); + rei.deepCopy(ClientIteratorEnvironment.DEFAULT); // ----------------------------------------------------- String multiByteText = new String("\u6d67\u6F68\u7067"); @@ -234,7 +234,7 @@ public void test1() throws IOException { is.clearOptions(); RegExFilter.setRegexs(is, null, null, null, multiByteRegex, true); - rei.init(new SortedMapIterator(tm), is.getOptions(), new DefaultIteratorEnvironment()); + rei.init(new SortedMapIterator(tm), is.getOptions(), ClientIteratorEnvironment.DEFAULT); rei.seek(new Range(), EMPTY_COL_FAMS, false); assertTrue(rei.hasTop()); diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/RowDeletingIteratorTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/RowDeletingIteratorTest.java index 39f0547cff8..c5a211a8a3a 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/RowDeletingIteratorTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/RowDeletingIteratorTest.java @@ -32,7 +32,7 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.IteratorEnvironment; +import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; import org.apache.accumulo.core.iteratorsImpl.system.ColumnFamilySkippingIterator; import org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator; @@ -41,27 +41,6 @@ public class RowDeletingIteratorTest { - public static class TestIE implements IteratorEnvironment { - - private IteratorScope scope; - private boolean fmc; - - public TestIE(IteratorScope scope, boolean fmc) { - this.scope = scope; - this.fmc = fmc; - } - - @Override - public IteratorScope getIteratorScope() { - return scope; - } - - @Override - public boolean isFullMajorCompaction() { - return fmc; - } - } - Key newKey(String row, String cf, String cq, long time) { return new Key(new Text(row), new Text(cf), new Text(cq), time); } @@ -91,7 +70,8 @@ public void test1() throws Exception { put(tm1, "r2", "cf1", "cq1", 5, "v1"); RowDeletingIterator rdi = new RowDeletingIterator(); - rdi.init(new SortedMapIterator(tm1), null, new TestIE(IteratorScope.scan, false)); + rdi.init(new SortedMapIterator(tm1), null, + new ClientIteratorEnvironment.Builder().withScope(IteratorScope.scan).build()); rdi.seek(new Range(), new ArrayList<>(), false); testAssertions(rdi, "r2", "cf1", "cq1", 5, "v1"); @@ -133,7 +113,8 @@ public void test2() throws Exception { put(tm1, "r2", "cf1", "cq1", 5, "v1"); RowDeletingIterator rdi = new RowDeletingIterator(); - rdi.init(new SortedMapIterator(tm1), null, new TestIE(IteratorScope.scan, false)); + rdi.init(new SortedMapIterator(tm1), null, + new ClientIteratorEnvironment.Builder().withScope(IteratorScope.scan).build()); rdi.seek(new Range(), new ArrayList<>(), false); testAssertions(rdi, "r1", "cf1", "cq3", 15, "v1"); @@ -175,7 +156,7 @@ public void test3() throws Exception { RowDeletingIterator rdi = new RowDeletingIterator(); rdi.init(new ColumnFamilySkippingIterator(new SortedMapIterator(tm1)), null, - new TestIE(IteratorScope.scan, false)); + new ClientIteratorEnvironment.Builder().withScope(IteratorScope.scan).build()); HashSet cols = new HashSet<>(); cols.add(new ArrayByteSequence("cf1".getBytes(UTF_8))); @@ -206,7 +187,8 @@ public void test4() throws Exception { put(tm1, "r2", "cf1", "cq1", 5, "v1"); RowDeletingIterator rdi = new RowDeletingIterator(); - rdi.init(new SortedMapIterator(tm1), null, new TestIE(IteratorScope.minc, false)); + rdi.init(new SortedMapIterator(tm1), null, + new ClientIteratorEnvironment.Builder().withScope(IteratorScope.minc).build()); rdi.seek(new Range(), new ArrayList<>(), false); testAssertions(rdi, "r1", "", "", 10, RowDeletingIterator.DELETE_ROW_VALUE.toString()); diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/RowEncodingIteratorTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/RowEncodingIteratorTest.java index 91908a7764f..4a4fc587372 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/RowEncodingIteratorTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/RowEncodingIteratorTest.java @@ -38,26 +38,14 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.IteratorEnvironment; -import org.apache.accumulo.core.iterators.IteratorUtil; +import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; +import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; import org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator; import org.apache.hadoop.io.Text; import org.junit.jupiter.api.Test; public class RowEncodingIteratorTest { - private static final class DummyIteratorEnv implements IteratorEnvironment { - @Override - public IteratorUtil.IteratorScope getIteratorScope() { - return IteratorUtil.IteratorScope.scan; - } - - @Override - public boolean isFullMajorCompaction() { - return false; - } - } - private static final class RowEncodingIteratorImpl extends RowEncodingIterator { public static SortedMap decodeRow(Value rowValue) throws IOException { @@ -139,7 +127,8 @@ public void testEncodeAll() throws IOException { RowEncodingIteratorImpl iter = new RowEncodingIteratorImpl(); Map bigBufferOpts = new HashMap<>(); bigBufferOpts.put(RowEncodingIterator.MAX_BUFFER_SIZE_OPT, "3K"); - iter.init(src, bigBufferOpts, new DummyIteratorEnv()); + iter.init(src, bigBufferOpts, + new ClientIteratorEnvironment.Builder().withScope(IteratorScope.scan).build()); iter.seek(range, new ArrayList<>(), false); assertTrue(iter.hasTop()); @@ -173,7 +162,8 @@ public void testEncodeSome() throws IOException { RowEncodingIteratorImpl iter = new RowEncodingIteratorImpl(); Map bigBufferOpts = new HashMap<>(); bigBufferOpts.put(RowEncodingIterator.MAX_BUFFER_SIZE_OPT, "1K"); - iter.init(src, bigBufferOpts, new DummyIteratorEnv()); + iter.init(src, bigBufferOpts, + new ClientIteratorEnvironment.Builder().withScope(IteratorScope.scan).build()); assertThrows(IllegalArgumentException.class, () -> iter.seek(range, new ArrayList<>(), false)); // IllegalArgumentException should be thrown as we can't fit the whole row into its buffer } diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/RowFilterTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/RowFilterTest.java index a18583efb43..2bdc7e2db31 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/RowFilterTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/RowFilterTest.java @@ -39,7 +39,7 @@ import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.DefaultIteratorEnvironment; +import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; import org.apache.accumulo.core.iteratorsImpl.system.ColumnFamilySkippingIterator; @@ -205,7 +205,7 @@ public void test1() throws Exception { new ColumnFamilySkippingIterator(new SortedMapIterator(createKeyValues())); RowFilter filter = new SummingRowFilter(); - filter.init(source, Collections.emptyMap(), new DefaultIteratorEnvironment()); + filter.init(source, Collections.emptyMap(), ClientIteratorEnvironment.DEFAULT); filter.seek(new Range(), Collections.emptySet(), false); @@ -235,10 +235,10 @@ public void testChainedRowFilters() throws Exception { SortedMapIterator source = new SortedMapIterator(createKeyValues()); RowFilter filter0 = new TrueFilter(); - filter0.init(source, Collections.emptyMap(), new DefaultIteratorEnvironment()); + filter0.init(source, Collections.emptyMap(), ClientIteratorEnvironment.DEFAULT); RowFilter filter = new TrueFilter(); - filter.init(filter0, Collections.emptyMap(), new DefaultIteratorEnvironment()); + filter.init(filter0, Collections.emptyMap(), ClientIteratorEnvironment.DEFAULT); filter.seek(new Range(), Collections.emptySet(), false); @@ -251,10 +251,10 @@ public void testFilterConjunction() throws Exception { SortedMapIterator source = new SortedMapIterator(createKeyValues()); RowFilter filter0 = new RowZeroOrOneFilter(); - filter0.init(source, Collections.emptyMap(), new DefaultIteratorEnvironment()); + filter0.init(source, Collections.emptyMap(), ClientIteratorEnvironment.DEFAULT); RowFilter filter = new RowOneOrTwoFilter(); - filter.init(filter0, Collections.emptyMap(), new DefaultIteratorEnvironment()); + filter.init(filter0, Collections.emptyMap(), ClientIteratorEnvironment.DEFAULT); filter.seek(new Range(), Collections.emptySet(), false); @@ -266,7 +266,7 @@ public void deepCopyCopiesTheSource() throws Exception { SortedMapIterator source = new SortedMapIterator(createKeyValues()); RowFilter filter = new RowZeroOrOneFilter(); - filter.init(source, Collections.emptyMap(), new DefaultIteratorEnvironment()); + filter.init(source, Collections.emptyMap(), ClientIteratorEnvironment.DEFAULT); filter.seek(new Range(), Collections.emptySet(), false); @@ -286,7 +286,7 @@ public void deepCopyCopiesTheSource() throws Exception { } // Make a copy of the original RowFilter - RowFilter copy = (RowFilter) filter.deepCopy(new DefaultIteratorEnvironment()); + RowFilter copy = (RowFilter) filter.deepCopy(ClientIteratorEnvironment.DEFAULT); // Because it's a copy, we should be able to safely seek this one without affecting the original copy.seek(new Range(), Collections.emptySet(), false); diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/TransformingIteratorTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/TransformingIteratorTest.java index 6798b6fecfb..f0c83154b98 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/TransformingIteratorTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/TransformingIteratorTest.java @@ -43,6 +43,7 @@ import org.apache.accumulo.core.data.PartialKey; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; @@ -619,7 +620,7 @@ public static class ColFamReversingCompactionKeyTransformingIterator @Override public void init(SortedKeyValueIterator source, Map options, IteratorEnvironment env) throws IOException { - env = new MajCIteratorEnvironmentAdapter(); + env = new ClientIteratorEnvironment.Builder().withScope(IteratorScope.majc).build(); super.init(source, options, env); } } @@ -665,7 +666,7 @@ public static class IllegalVisCompactionKeyTransformingIterator @Override public void init(SortedKeyValueIterator source, Map options, IteratorEnvironment env) throws IOException { - env = new MajCIteratorEnvironmentAdapter(); + env = new ClientIteratorEnvironment.Builder().withScope(IteratorScope.majc).build(); super.init(source, options, env); } } @@ -695,7 +696,7 @@ public static class BadVisCompactionKeyTransformingIterator @Override public void init(SortedKeyValueIterator source, Map options, IteratorEnvironment env) throws IOException { - env = new MajCIteratorEnvironmentAdapter(); + env = new ClientIteratorEnvironment.Builder().withScope(IteratorScope.majc).build(); super.init(source, options, env); } } @@ -742,10 +743,4 @@ private void loadTop() { } } - private static class MajCIteratorEnvironmentAdapter implements IteratorEnvironment { - @Override - public IteratorScope getIteratorScope() { - return IteratorScope.majc; - } - } } diff --git a/core/src/test/java/org/apache/accumulo/core/iteratorsImpl/IteratorConfigUtilTest.java b/core/src/test/java/org/apache/accumulo/core/iteratorsImpl/IteratorConfigUtilTest.java index de5794607c4..4b360162da6 100644 --- a/core/src/test/java/org/apache/accumulo/core/iteratorsImpl/IteratorConfigUtilTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iteratorsImpl/IteratorConfigUtilTest.java @@ -39,7 +39,7 @@ import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.dataImpl.thrift.IterInfo; -import org.apache.accumulo.core.iterators.DefaultIteratorEnvironment; +import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; @@ -139,8 +139,7 @@ public Value getTopValue() { private SortedKeyValueIterator createIter(IteratorScope scope, SortedMapIterator source, AccumuloConfiguration conf) throws IOException { var ibEnv = IteratorConfigUtil.loadIterConf(scope, EMPTY_ITERS, new HashMap<>(), conf); - var iteratorBuilder = - ibEnv.env(new DefaultIteratorEnvironment(conf)).useClassLoader(null).build(); + var iteratorBuilder = ibEnv.env(ClientIteratorEnvironment.DEFAULT).useClassLoader(null).build(); return IteratorConfigUtil.loadIterators(source, iteratorBuilder); } diff --git a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/IteratorTestInput.java b/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/IteratorTestInput.java index 408f9242b43..758165906ec 100644 --- a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/IteratorTestInput.java +++ b/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/IteratorTestInput.java @@ -31,9 +31,9 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; -import org.apache.accumulo.iteratortest.environments.SimpleIteratorEnvironment; /** * The necessary user-input to invoke a test on a {@link SortedKeyValueIterator}. @@ -49,7 +49,7 @@ public class IteratorTestInput { private final IteratorEnvironment iteratorEnvironment; /** - * Construct an instance of the test input, using {@link SimpleIteratorEnvironment}. + * Construct an instance of the test input, using {@link TestIteratorEnvironment}. * * @param iteratorClass The class for the iterator to test. * @param iteratorOptions Options, if any, to provide to the iterator ({@link IteratorSetting}'s @@ -61,7 +61,7 @@ public class IteratorTestInput { public IteratorTestInput(Class> iteratorClass, Map iteratorOptions, Range range, SortedMap input) { this(iteratorClass, iteratorOptions, range, input, Collections.emptySet(), false, - new SimpleIteratorEnvironment()); + ClientIteratorEnvironment.DEFAULT); } /** @@ -93,7 +93,7 @@ public IteratorTestInput(Class> iter * @param families Column families passed to {@link SortedKeyValueIterator#seek}. * @param inclusive Whether the families are inclusive or exclusive. * @param iterEnv An optional provided {@link IteratorEnvironment}. - * {@link SimpleIteratorEnvironment} will be used if null. + * {@link TestIteratorEnvironment} will be used if null. */ public IteratorTestInput(Class> iteratorClass, Map iteratorOptions, Range range, SortedMap input, @@ -108,7 +108,7 @@ public IteratorTestInput(Class> iter this.input = Collections.unmodifiableSortedMap(requireNonNull(input)); this.families = Collections.unmodifiableCollection(requireNonNull(families)); this.inclusive = inclusive; - this.iteratorEnvironment = iterEnv == null ? new SimpleIteratorEnvironment() : iterEnv; + this.iteratorEnvironment = iterEnv == null ? ClientIteratorEnvironment.DEFAULT : iterEnv; } public Class> getIteratorClass() { diff --git a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/environments/SimpleIteratorEnvironment.java b/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/environments/SimpleIteratorEnvironment.java deleted file mode 100644 index 0c89197aed8..00000000000 --- a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/environments/SimpleIteratorEnvironment.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.accumulo.iteratortest.environments; - -import org.apache.accumulo.core.iterators.IteratorEnvironment; - -/** - * A simple implementation of {@link IteratorEnvironment} which is unimplemented. - */ -public class SimpleIteratorEnvironment implements IteratorEnvironment { - - @Override - public boolean isSamplingEnabled() { - return false; - } - -} diff --git a/server/base/src/main/java/org/apache/accumulo/server/iterators/TabletIteratorEnvironment.java b/server/base/src/main/java/org/apache/accumulo/server/iterators/TabletIteratorEnvironment.java index 63970b2943a..2eada1bfeea 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/iterators/TabletIteratorEnvironment.java +++ b/server/base/src/main/java/org/apache/accumulo/server/iterators/TabletIteratorEnvironment.java @@ -23,6 +23,7 @@ import java.util.Collections; import java.util.Map; +import org.apache.accumulo.core.client.PluginEnvironment; import org.apache.accumulo.core.client.SampleNotPresentException; import org.apache.accumulo.core.client.sample.SamplerConfiguration; import org.apache.accumulo.core.conf.AccumuloConfiguration; @@ -236,4 +237,9 @@ public ServiceEnvironment getServiceEnv() { public TableId getTableId() { return tableId; } + + @Override + public PluginEnvironment getPluginEnv() { + return serviceEnvironment; + } } diff --git a/server/base/src/test/java/org/apache/accumulo/server/replication/StatusCombinerTest.java b/server/base/src/test/java/org/apache/accumulo/server/replication/StatusCombinerTest.java index 974617d5e84..6671e47ef99 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/replication/StatusCombinerTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/replication/StatusCombinerTest.java @@ -32,9 +32,9 @@ import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.IteratorSetting.Column; import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; import org.apache.accumulo.core.iterators.Combiner; import org.apache.accumulo.core.iterators.DevNull; -import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; import org.apache.accumulo.core.replication.ReplicationSchema.StatusSection; import org.apache.accumulo.server.replication.proto.Replication.Status; @@ -48,13 +48,6 @@ public class StatusCombinerTest { private Key key; private Status.Builder builder; - private static class TestIE implements IteratorEnvironment { - @Override - public IteratorScope getIteratorScope() { - return IteratorScope.scan; - } - } - @BeforeEach public void initCombiner() throws IOException { key = new Key(); @@ -62,7 +55,8 @@ public void initCombiner() throws IOException { builder = Status.newBuilder(); IteratorSetting cfg = new IteratorSetting(50, StatusCombiner.class); Combiner.setColumns(cfg, Collections.singletonList(new Column(StatusSection.NAME))); - combiner.init(new DevNull(), cfg.getOptions(), new TestIE()); + combiner.init(new DevNull(), cfg.getOptions(), + new ClientIteratorEnvironment.Builder().withScope(IteratorScope.scan).build()); } @Test diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/InMemoryMapTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/InMemoryMapTest.java index eefa5c1252b..fd6291acae1 100644 --- a/server/tserver/src/test/java/org/apache/accumulo/tserver/InMemoryMapTest.java +++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/InMemoryMapTest.java @@ -38,7 +38,6 @@ import org.apache.accumulo.core.client.SampleNotPresentException; import org.apache.accumulo.core.client.sample.RowSampler; import org.apache.accumulo.core.client.sample.Sampler; -import org.apache.accumulo.core.client.sample.SamplerConfiguration; import org.apache.accumulo.core.conf.ConfigurationCopy; import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.conf.Property; @@ -49,7 +48,7 @@ import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.IteratorEnvironment; +import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; import org.apache.accumulo.core.iteratorsImpl.system.ColumnFamilySkippingIterator; import org.apache.accumulo.core.iteratorsImpl.system.IterationInterruptedException; @@ -71,29 +70,6 @@ @SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", justification = "paths not set by user input") public class InMemoryMapTest extends WithTestNames { - private static class SampleIE implements IteratorEnvironment { - - private final SamplerConfiguration sampleConfig; - - public SampleIE() { - this.sampleConfig = null; - } - - public SampleIE(SamplerConfigurationImpl sampleConfig) { - this.sampleConfig = sampleConfig.toSamplerConfiguration(); - } - - @Override - public boolean isSamplingEnabled() { - return sampleConfig != null; - } - - @Override - public SamplerConfiguration getSamplerConfiguration() { - return sampleConfig; - } - } - public static ServerContext getServerContext() { Configuration hadoopConf = new Configuration(); ServerContext context = EasyMock.createMock(ServerContext.class); @@ -319,7 +295,7 @@ public void test6() throws Exception { mutate(imm, "r1", "foo:cq5", 3, "bar5"); - SortedKeyValueIterator dc = ski1.deepCopy(new SampleIE()); + SortedKeyValueIterator dc = ski1.deepCopy(ClientIteratorEnvironment.DEFAULT); ski1.seek(new Range(newKey("r1", "foo:cq1", 3), null), Set.of(), false); testAndCallNext(ski1, "r1", "foo:cq1", 3, "bar1"); @@ -371,7 +347,7 @@ private void deepCopyAndDelete(int interleaving, boolean interrupt) throws Excep } } - SortedKeyValueIterator dc = ski1.deepCopy(new SampleIE()); + SortedKeyValueIterator dc = ski1.deepCopy(ClientIteratorEnvironment.DEFAULT); if (interleaving == 2) { imm.delete(0); @@ -523,7 +499,7 @@ public void testLocalityGroups() throws Exception { MemoryIterator iter1 = imm.skvIterator(null); seekLocalityGroups(iter1); - SortedKeyValueIterator dc1 = iter1.deepCopy(new SampleIE()); + SortedKeyValueIterator dc1 = iter1.deepCopy(ClientIteratorEnvironment.DEFAULT); seekLocalityGroups(dc1); assertEquals(10, imm.getNumEntries()); @@ -576,12 +552,21 @@ public void testSample() throws Exception { MemoryIterator iter1 = imm.skvIterator(sampleConfig); MemoryIterator iter2 = imm.skvIterator(null); - SortedKeyValueIterator iter0dc1 = iter0.deepCopy(new SampleIE()); - SortedKeyValueIterator iter0dc2 = iter0.deepCopy(new SampleIE(sampleConfig)); - SortedKeyValueIterator iter1dc1 = iter1.deepCopy(new SampleIE()); - SortedKeyValueIterator iter1dc2 = iter1.deepCopy(new SampleIE(sampleConfig)); - SortedKeyValueIterator iter2dc1 = iter2.deepCopy(new SampleIE()); - SortedKeyValueIterator iter2dc2 = iter2.deepCopy(new SampleIE(sampleConfig)); + SortedKeyValueIterator iter0dc1 = + iter0.deepCopy(ClientIteratorEnvironment.DEFAULT); + SortedKeyValueIterator iter0dc2 = + iter0.deepCopy(new ClientIteratorEnvironment.Builder() + .withSamplerConfiguration(sampleConfig.toSamplerConfiguration()).build()); + SortedKeyValueIterator iter1dc1 = + iter1.deepCopy(ClientIteratorEnvironment.DEFAULT); + SortedKeyValueIterator iter1dc2 = + iter1.deepCopy(new ClientIteratorEnvironment.Builder() + .withSamplerConfiguration(sampleConfig.toSamplerConfiguration()).build()); + SortedKeyValueIterator iter2dc1 = + iter2.deepCopy(ClientIteratorEnvironment.DEFAULT); + SortedKeyValueIterator iter2dc2 = + iter2.deepCopy(new ClientIteratorEnvironment.Builder() + .withSamplerConfiguration(sampleConfig.toSamplerConfiguration()).build()); assertEquals(expectedNone, readAll(iter0)); assertEquals(expectedNone, readAll(iter0dc1)); @@ -605,12 +590,21 @@ public void testSample() throws Exception { assertEquals(expectedSample, readAll(iter1dc2)); assertEquals(expectedSample, readAll(iter2dc2)); - SortedKeyValueIterator iter0dc3 = iter0.deepCopy(new SampleIE()); - SortedKeyValueIterator iter0dc4 = iter0.deepCopy(new SampleIE(sampleConfig)); - SortedKeyValueIterator iter1dc3 = iter1.deepCopy(new SampleIE()); - SortedKeyValueIterator iter1dc4 = iter1.deepCopy(new SampleIE(sampleConfig)); - SortedKeyValueIterator iter2dc3 = iter2.deepCopy(new SampleIE()); - SortedKeyValueIterator iter2dc4 = iter2.deepCopy(new SampleIE(sampleConfig)); + SortedKeyValueIterator iter0dc3 = + iter0.deepCopy(ClientIteratorEnvironment.DEFAULT); + SortedKeyValueIterator iter0dc4 = + iter0.deepCopy(new ClientIteratorEnvironment.Builder() + .withSamplerConfiguration(sampleConfig.toSamplerConfiguration()).build()); + SortedKeyValueIterator iter1dc3 = + iter1.deepCopy(ClientIteratorEnvironment.DEFAULT); + SortedKeyValueIterator iter1dc4 = + iter1.deepCopy(new ClientIteratorEnvironment.Builder() + .withSamplerConfiguration(sampleConfig.toSamplerConfiguration()).build()); + SortedKeyValueIterator iter2dc3 = + iter2.deepCopy(ClientIteratorEnvironment.DEFAULT); + SortedKeyValueIterator iter2dc4 = + iter2.deepCopy(new ClientIteratorEnvironment.Builder() + .withSamplerConfiguration(sampleConfig.toSamplerConfiguration()).build()); assertEquals(expectedNone, readAll(iter0dc3)); assertEquals(expectedNone, readAll(iter0dc4)); @@ -667,7 +661,8 @@ private void runInterruptSampleTest(boolean deepCopy, boolean delete, boolean dc } if (deepCopy) { - iter = iter.deepCopy(new SampleIE(sampleConfig1)); + iter = iter.deepCopy(new ClientIteratorEnvironment.Builder() + .withSamplerConfiguration(sampleConfig1.toSamplerConfiguration()).build()); } if (delete && dcAfterDelete) { @@ -768,7 +763,8 @@ public void testDeferredSamplerCreation() throws Exception { iter.seek(new Range(), Set.of(), false); assertEquals(expectedSample, readAll(iter)); - SortedKeyValueIterator dc = iter.deepCopy(new SampleIE(sampleConfig2)); + SortedKeyValueIterator dc = iter.deepCopy(new ClientIteratorEnvironment.Builder() + .withSamplerConfiguration(sampleConfig2.toSamplerConfiguration()).build()); dc.seek(new Range(), Set.of(), false); assertEquals(expectedSample, readAll(dc)); diff --git a/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java b/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java index 63ec7fcf113..e34747e3aea 100644 --- a/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java +++ b/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java @@ -52,7 +52,6 @@ import org.apache.accumulo.core.dataImpl.thrift.IterInfo; import org.apache.accumulo.core.file.FileOperations; import org.apache.accumulo.core.file.FileSKVIterator; -import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; import org.apache.accumulo.core.iteratorsImpl.IteratorConfigUtil; @@ -73,6 +72,7 @@ import org.apache.accumulo.server.cli.ServerUtilOpts; import org.apache.accumulo.server.conf.TableConfiguration; import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.accumulo.server.iterators.TabletIteratorEnvironment; import org.apache.accumulo.server.util.MetadataTableUtil; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FileStatus; @@ -103,8 +103,6 @@ static class CollectOptions extends ServerUtilOpts { String columns; } - static class TestEnvironment implements IteratorEnvironment {} - public static void main(String[] args) throws Exception { final CollectOptions opts = new CollectOptions(); @@ -429,8 +427,8 @@ private static void reportHdfsBlockLocations(ServerContext context, List createScanIterator(KeyExtent ke, Collection> mapfiles, Authorizations authorizations, byte[] defaultLabels, HashSet columnSet, List ssiList, - Map> ssio, boolean useTableIterators, TableConfiguration conf) - throws IOException { + Map> ssio, boolean useTableIterators, TableConfiguration conf, + ServerContext context) throws IOException { SortedMapIterator smi = new SortedMapIterator(new TreeMap<>()); @@ -449,7 +447,9 @@ private static SortedKeyValueIterator createScanIterator(KeyExtent ke if (useTableIterators) { var ibEnv = IteratorConfigUtil.loadIterConf(IteratorScope.scan, ssiList, ssio, conf); - var iteratorBuilder = ibEnv.env(new TestEnvironment()).useClassLoader("test").build(); + TabletIteratorEnvironment iterEnv = + new TabletIteratorEnvironment(context, IteratorScope.scan, conf, ke.tableId()); + var iteratorBuilder = ibEnv.env(iterEnv).useClassLoader("test").build(); return IteratorConfigUtil.loadIterators(visFilter, iteratorBuilder); } return visFilter; @@ -506,7 +506,7 @@ private static int readFilesUsingIterStack(VolumeManager fs, ServerContext conte Map> emptySsio = Collections.emptyMap(); TableConfiguration tconf = context.getTableConfiguration(ke.tableId()); reader = createScanIterator(ke, readers, auths, new byte[] {}, new HashSet<>(), emptyIterinfo, - emptySsio, useTableIterators, tconf); + emptySsio, useTableIterators, tconf, context); HashSet columnSet = createColumnBSS(columns); diff --git a/test/src/test/java/org/apache/accumulo/test/iterator/SummingCombinerTest.java b/test/src/test/java/org/apache/accumulo/test/iterator/SummingCombinerTest.java index 7d8b21519f0..2043f992209 100644 --- a/test/src/test/java/org/apache/accumulo/test/iterator/SummingCombinerTest.java +++ b/test/src/test/java/org/apache/accumulo/test/iterator/SummingCombinerTest.java @@ -28,6 +28,7 @@ import org.apache.accumulo.core.data.PartialKey; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; import org.apache.accumulo.core.iterators.Combiner; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; import org.apache.accumulo.core.iterators.LongCombiner; @@ -36,7 +37,6 @@ import org.apache.accumulo.iteratortest.IteratorTestInput; import org.apache.accumulo.iteratortest.IteratorTestOutput; import org.apache.accumulo.iteratortest.IteratorTestParameters; -import org.apache.accumulo.iteratortest.environments.SimpleIteratorEnvironment; /** * Iterator test harness tests for SummingCombiner @@ -48,14 +48,8 @@ public class SummingCombinerTest extends IteratorTestBase { @Override protected Stream parameters() { - var env = new SimpleIteratorEnvironment() { - @Override - public IteratorScope getIteratorScope() { - return IteratorScope.majc; - } - }; - var input = - new IteratorTestInput(SummingCombiner.class, createOpts(), new Range(), INPUT_DATA, env); + var input = new IteratorTestInput(SummingCombiner.class, createOpts(), new Range(), INPUT_DATA, + new ClientIteratorEnvironment.Builder().withScope(IteratorScope.majc).build()); var expectedOutput = new IteratorTestOutput(OUTPUT_DATA); return builtinTestCases().map(test -> test.toParameters(input, expectedOutput)); } From 08d29a6db210b40fe8309545e43500b8b8e28bac Mon Sep 17 00:00:00 2001 From: Dave Marion Date: Mon, 21 Apr 2025 12:40:13 +0000 Subject: [PATCH 02/11] Resove build error Removed ServiceEnvironment from ClientSideIteratorScanner, use TabletIteratorEnvironment in server test. --- .../client/ClientSideIteratorScanner.java | 2 +- .../core/client/rfile/RFileScanner.java | 14 +-- .../core/clientImpl/OfflineIterator.java | 2 +- .../iterators/ClientIteratorEnvironment.java | 7 +- .../iterators/TabletIteratorEnvironment.java | 24 +++++ .../accumulo/tserver/InMemoryMapTest.java | 87 ++++++++++++------- 6 files changed, 91 insertions(+), 45 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java b/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java index 089dd54286b..2a74f04e87b 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java +++ b/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java @@ -232,7 +232,7 @@ public Iterator> iterator() { SortedKeyValueIterator skvi; try { IteratorEnvironment iterEnv = new ClientIteratorEnvironment.Builder() - .withServiceEnvironment(new ClientServiceEnvironmentImpl(context.get())) + .withEnvironment(new ClientServiceEnvironmentImpl(context.get())) .withAuthorizations(getAuthorizations()).withScope(IteratorScope.scan) .withTableId(tableId.get()) .withSamplerConfiguration(getIteratorSamplerConfigurationInternal()).build(); diff --git a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java index bfbf324ca9e..1541098844f 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java +++ b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java @@ -32,6 +32,7 @@ import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.rfile.RFileScannerBuilder.InputArgs; +import org.apache.accumulo.core.clientImpl.ClientServiceEnvironmentImpl; import org.apache.accumulo.core.clientImpl.ScannerOptions; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.ConfigurationCopy; @@ -66,7 +67,6 @@ import org.apache.accumulo.core.spi.cache.BlockCache; import org.apache.accumulo.core.spi.cache.BlockCacheManager; import org.apache.accumulo.core.spi.cache.CacheType; -import org.apache.accumulo.core.spi.common.ServiceEnvironment; import org.apache.accumulo.core.spi.crypto.CryptoEnvironment; import org.apache.accumulo.core.spi.crypto.CryptoService; import org.apache.accumulo.core.util.ConfigurationImpl; @@ -269,7 +269,7 @@ public Iterator> iterator() { EMPTY_BYTES, tableConf); } - ServiceEnvironment senv = new ServiceEnvironment() { + ClientServiceEnvironmentImpl senv = new ClientServiceEnvironmentImpl(null) { @Override public String getTableName(TableId tableId) throws TableNotFoundException { @@ -277,14 +277,15 @@ public String getTableName(TableId tableId) throws TableNotFoundException { } @Override - public T instantiate(String className, Class base) throws Exception { + public T instantiate(String className, Class base) + throws ReflectiveOperationException, IOException { return RFileScanner.class.getClassLoader().loadClass(className).asSubclass(base) .getDeclaredConstructor().newInstance(); } @Override public T instantiate(TableId tableId, String className, Class base) - throws Exception { + throws ReflectiveOperationException, IOException { return instantiate(className, base); } @@ -300,9 +301,8 @@ public Configuration getConfiguration(TableId tableId) { }; - ClientIteratorEnvironment.Builder iterEnvBuilder = - new ClientIteratorEnvironment.Builder().withAuthorizations(opts.auths) - .withScope(IteratorScope.scan).withServiceEnvironment(senv); + ClientIteratorEnvironment.Builder iterEnvBuilder = new ClientIteratorEnvironment.Builder() + .withAuthorizations(opts.auths).withScope(IteratorScope.scan).withEnvironment(senv); if (getSamplerConfiguration() != null) { iterEnvBuilder.withSamplerConfiguration(getSamplerConfiguration()); } diff --git a/core/src/main/java/org/apache/accumulo/core/clientImpl/OfflineIterator.java b/core/src/main/java/org/apache/accumulo/core/clientImpl/OfflineIterator.java index 04bdb4e8311..4b890ecd96d 100644 --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/OfflineIterator.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/OfflineIterator.java @@ -247,7 +247,7 @@ private SortedKeyValueIterator createIterator(KeyExtent extent, ClientIteratorEnvironment.Builder iterEnvBuilder = new ClientIteratorEnvironment.Builder() .withAuthorizations(authorizations).withScope(IteratorScope.scan).withTableId(tableId) - .withServiceEnvironment(new ClientServiceEnvironmentImpl(context)); + .withEnvironment(new ClientServiceEnvironmentImpl(context)); if (scannerSamplerConfig != null) { iterEnvBuilder.withSamplerConfiguration(scannerSamplerConfig); } diff --git a/core/src/main/java/org/apache/accumulo/core/iterators/ClientIteratorEnvironment.java b/core/src/main/java/org/apache/accumulo/core/iterators/ClientIteratorEnvironment.java index da41f9c9cd6..ca129f30139 100644 --- a/core/src/main/java/org/apache/accumulo/core/iterators/ClientIteratorEnvironment.java +++ b/core/src/main/java/org/apache/accumulo/core/iterators/ClientIteratorEnvironment.java @@ -24,6 +24,7 @@ import org.apache.accumulo.core.client.PluginEnvironment; import org.apache.accumulo.core.client.SampleNotPresentException; import org.apache.accumulo.core.client.sample.SamplerConfiguration; +import org.apache.accumulo.core.clientImpl.ClientServiceEnvironmentImpl; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.data.Value; @@ -44,7 +45,7 @@ public static class Builder { private boolean isUserCompaction = false; private Optional tableId = Optional.empty(); private Optional samplerConfig = Optional.empty(); - private Optional env = Optional.empty(); + private Optional env = Optional.empty(); public Builder withScope(IteratorScope scope) { this.scope = Optional.of(scope); @@ -76,7 +77,7 @@ public Builder withSamplerConfiguration(SamplerConfiguration sc) { return this; } - public Builder withServiceEnvironment(ServiceEnvironment env) { + public Builder withEnvironment(ClientServiceEnvironmentImpl env) { this.env = Optional.of(env); return this; } @@ -98,7 +99,7 @@ public ClientIteratorEnvironment build() { private final boolean isUserCompaction; private final Optional tableId; private final Optional samplerConfig; - private final Optional env; + private final Optional env; private ClientIteratorEnvironment(Builder builder) { this.scope = builder.scope; diff --git a/server/base/src/main/java/org/apache/accumulo/server/iterators/TabletIteratorEnvironment.java b/server/base/src/main/java/org/apache/accumulo/server/iterators/TabletIteratorEnvironment.java index 2eada1bfeea..a1b657b5628 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/iterators/TabletIteratorEnvironment.java +++ b/server/base/src/main/java/org/apache/accumulo/server/iterators/TabletIteratorEnvironment.java @@ -80,6 +80,30 @@ public TabletIteratorEnvironment(ServerContext context, IteratorScope scope, this.topLevelIterators = new ArrayList<>(); } + public TabletIteratorEnvironment(ServerContext context, IteratorScope scope, + AccumuloConfiguration tableConfig, TableId tableId, SamplerConfigurationImpl samplerConfig) { + if (scope == IteratorScope.majc) { + throw new IllegalArgumentException("must set if compaction is full"); + } + + this.context = context; + this.serviceEnvironment = new ServiceEnvironmentImpl(context); + this.scope = scope; + this.trm = null; + this.tableConfig = tableConfig; + this.tableId = tableId; + this.fullMajorCompaction = false; + this.userCompaction = false; + this.authorizations = Authorizations.EMPTY; + if (samplerConfig != null) { + enableSampleForDeepCopy = true; + this.samplerConfig = samplerConfig.toSamplerConfiguration(); + } else { + enableSampleForDeepCopy = false; + } + this.topLevelIterators = new ArrayList<>(); + } + public TabletIteratorEnvironment(ServerContext context, IteratorScope scope, AccumuloConfiguration tableConfig, TableId tableId, ScanFileManager trm, Map files, Authorizations authorizations, diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/InMemoryMapTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/InMemoryMapTest.java index fd6291acae1..5bd26d5b74f 100644 --- a/server/tserver/src/test/java/org/apache/accumulo/tserver/InMemoryMapTest.java +++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/InMemoryMapTest.java @@ -48,7 +48,7 @@ import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; +import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; import org.apache.accumulo.core.iteratorsImpl.system.ColumnFamilySkippingIterator; import org.apache.accumulo.core.iteratorsImpl.system.IterationInterruptedException; @@ -58,6 +58,7 @@ import org.apache.accumulo.core.util.LocalityGroupUtil; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.conf.TableConfiguration; +import org.apache.accumulo.server.iterators.TabletIteratorEnvironment; import org.apache.accumulo.tserver.InMemoryMap.MemoryIterator; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; @@ -295,7 +296,9 @@ public void test6() throws Exception { mutate(imm, "r1", "foo:cq5", 3, "bar5"); - SortedKeyValueIterator dc = ski1.deepCopy(ClientIteratorEnvironment.DEFAULT); + SortedKeyValueIterator dc = + ski1.deepCopy(new TabletIteratorEnvironment(getServerContext(), IteratorScope.scan, + getServerContext().getTableConfiguration(TableId.of("foo")), TableId.of("foo"))); ski1.seek(new Range(newKey("r1", "foo:cq1", 3), null), Set.of(), false); testAndCallNext(ski1, "r1", "foo:cq1", 3, "bar1"); @@ -347,8 +350,9 @@ private void deepCopyAndDelete(int interleaving, boolean interrupt) throws Excep } } - SortedKeyValueIterator dc = ski1.deepCopy(ClientIteratorEnvironment.DEFAULT); - + SortedKeyValueIterator dc = + ski1.deepCopy(new TabletIteratorEnvironment(getServerContext(), IteratorScope.scan, + getServerContext().getTableConfiguration(TableId.of("foo")), TableId.of("foo"))); if (interleaving == 2) { imm.delete(0); if (interrupt) { @@ -499,7 +503,9 @@ public void testLocalityGroups() throws Exception { MemoryIterator iter1 = imm.skvIterator(null); seekLocalityGroups(iter1); - SortedKeyValueIterator dc1 = iter1.deepCopy(ClientIteratorEnvironment.DEFAULT); + SortedKeyValueIterator dc1 = + iter1.deepCopy(new TabletIteratorEnvironment(getServerContext(), IteratorScope.scan, + getServerContext().getTableConfiguration(TableId.of("foo")), TableId.of("foo"))); seekLocalityGroups(dc1); assertEquals(10, imm.getNumEntries()); @@ -553,20 +559,26 @@ public void testSample() throws Exception { MemoryIterator iter1 = imm.skvIterator(sampleConfig); MemoryIterator iter2 = imm.skvIterator(null); SortedKeyValueIterator iter0dc1 = - iter0.deepCopy(ClientIteratorEnvironment.DEFAULT); - SortedKeyValueIterator iter0dc2 = - iter0.deepCopy(new ClientIteratorEnvironment.Builder() - .withSamplerConfiguration(sampleConfig.toSamplerConfiguration()).build()); + iter0.deepCopy(new TabletIteratorEnvironment(getServerContext(), IteratorScope.scan, + getServerContext().getTableConfiguration(TableId.of("foo")), TableId.of("foo"))); + SortedKeyValueIterator iter0dc2 = iter0.deepCopy(new TabletIteratorEnvironment(getServerContext(), + IteratorScope.scan, getServerContext().getTableConfiguration(TableId.of("foo")), + TableId.of("foo"), sampleConfig)); SortedKeyValueIterator iter1dc1 = - iter1.deepCopy(ClientIteratorEnvironment.DEFAULT); - SortedKeyValueIterator iter1dc2 = - iter1.deepCopy(new ClientIteratorEnvironment.Builder() - .withSamplerConfiguration(sampleConfig.toSamplerConfiguration()).build()); + iter1.deepCopy(new TabletIteratorEnvironment(getServerContext(), IteratorScope.scan, + getServerContext().getTableConfiguration(TableId.of("foo")), TableId.of("foo"))); + SortedKeyValueIterator iter1dc2 = iter1.deepCopy(new TabletIteratorEnvironment(getServerContext(), + IteratorScope.scan, getServerContext().getTableConfiguration(TableId.of("foo")), + TableId.of("foo"), sampleConfig)); SortedKeyValueIterator iter2dc1 = - iter2.deepCopy(ClientIteratorEnvironment.DEFAULT); - SortedKeyValueIterator iter2dc2 = - iter2.deepCopy(new ClientIteratorEnvironment.Builder() - .withSamplerConfiguration(sampleConfig.toSamplerConfiguration()).build()); + iter2.deepCopy(new TabletIteratorEnvironment(getServerContext(), IteratorScope.scan, + getServerContext().getTableConfiguration(TableId.of("foo")), TableId.of("foo"))); + SortedKeyValueIterator iter2dc2 = iter2.deepCopy(new TabletIteratorEnvironment(getServerContext(), + IteratorScope.scan, getServerContext().getTableConfiguration(TableId.of("foo")), + TableId.of("foo"), sampleConfig)); assertEquals(expectedNone, readAll(iter0)); assertEquals(expectedNone, readAll(iter0dc1)); @@ -591,20 +603,26 @@ public void testSample() throws Exception { assertEquals(expectedSample, readAll(iter2dc2)); SortedKeyValueIterator iter0dc3 = - iter0.deepCopy(ClientIteratorEnvironment.DEFAULT); - SortedKeyValueIterator iter0dc4 = - iter0.deepCopy(new ClientIteratorEnvironment.Builder() - .withSamplerConfiguration(sampleConfig.toSamplerConfiguration()).build()); + iter0.deepCopy(new TabletIteratorEnvironment(getServerContext(), IteratorScope.scan, + getServerContext().getTableConfiguration(TableId.of("foo")), TableId.of("foo"))); + SortedKeyValueIterator iter0dc4 = iter0.deepCopy(new TabletIteratorEnvironment(getServerContext(), + IteratorScope.scan, getServerContext().getTableConfiguration(TableId.of("foo")), + TableId.of("foo"), sampleConfig)); SortedKeyValueIterator iter1dc3 = - iter1.deepCopy(ClientIteratorEnvironment.DEFAULT); - SortedKeyValueIterator iter1dc4 = - iter1.deepCopy(new ClientIteratorEnvironment.Builder() - .withSamplerConfiguration(sampleConfig.toSamplerConfiguration()).build()); + iter1.deepCopy(new TabletIteratorEnvironment(getServerContext(), IteratorScope.scan, + getServerContext().getTableConfiguration(TableId.of("foo")), TableId.of("foo"))); + SortedKeyValueIterator iter1dc4 = iter1.deepCopy(new TabletIteratorEnvironment(getServerContext(), + IteratorScope.scan, getServerContext().getTableConfiguration(TableId.of("foo")), + TableId.of("foo"), sampleConfig)); SortedKeyValueIterator iter2dc3 = - iter2.deepCopy(ClientIteratorEnvironment.DEFAULT); - SortedKeyValueIterator iter2dc4 = - iter2.deepCopy(new ClientIteratorEnvironment.Builder() - .withSamplerConfiguration(sampleConfig.toSamplerConfiguration()).build()); + iter2.deepCopy(new TabletIteratorEnvironment(getServerContext(), IteratorScope.scan, + getServerContext().getTableConfiguration(TableId.of("foo")), TableId.of("foo"))); + SortedKeyValueIterator iter2dc4 = iter2.deepCopy(new TabletIteratorEnvironment(getServerContext(), + IteratorScope.scan, getServerContext().getTableConfiguration(TableId.of("foo")), + TableId.of("foo"), sampleConfig)); assertEquals(expectedNone, readAll(iter0dc3)); assertEquals(expectedNone, readAll(iter0dc4)); @@ -661,8 +679,9 @@ private void runInterruptSampleTest(boolean deepCopy, boolean delete, boolean dc } if (deepCopy) { - iter = iter.deepCopy(new ClientIteratorEnvironment.Builder() - .withSamplerConfiguration(sampleConfig1.toSamplerConfiguration()).build()); + iter = iter.deepCopy(new TabletIteratorEnvironment(getServerContext(), IteratorScope.scan, + getServerContext().getTableConfiguration(TableId.of("foo")), TableId.of("foo"), + sampleConfig1)); } if (delete && dcAfterDelete) { @@ -763,8 +782,10 @@ public void testDeferredSamplerCreation() throws Exception { iter.seek(new Range(), Set.of(), false); assertEquals(expectedSample, readAll(iter)); - SortedKeyValueIterator dc = iter.deepCopy(new ClientIteratorEnvironment.Builder() - .withSamplerConfiguration(sampleConfig2.toSamplerConfiguration()).build()); + SortedKeyValueIterator dc = iter.deepCopy(new TabletIteratorEnvironment(getServerContext(), + IteratorScope.scan, getServerContext().getTableConfiguration(TableId.of("foo")), + TableId.of("foo"), sampleConfig2)); dc.seek(new Range(), Set.of(), false); assertEquals(expectedSample, readAll(dc)); From 00a2758e6b69a4adcc665a1a5886b38b2eb4a90a Mon Sep 17 00:00:00 2001 From: Dave Marion Date: Mon, 21 Apr 2025 15:32:48 +0000 Subject: [PATCH 03/11] Resolve build issue --- .../core/client/ClientSideIteratorScanner.java | 6 ++---- .../accumulo/core/client/rfile/RFileScanner.java | 16 ++++++++++++++-- .../core/clientImpl/OfflineIterator.java | 6 +++--- .../iterators/ClientIteratorEnvironment.java | 8 +++++--- .../accumulo/iteratortest/IteratorTestInput.java | 4 ++-- 5 files changed, 26 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java b/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java index 2a74f04e87b..a8617c2178f 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java +++ b/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java @@ -34,7 +34,6 @@ import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.sample.SamplerConfiguration; import org.apache.accumulo.core.clientImpl.ClientContext; -import org.apache.accumulo.core.clientImpl.ClientServiceEnvironmentImpl; import org.apache.accumulo.core.clientImpl.ScannerImpl; import org.apache.accumulo.core.clientImpl.ScannerOptions; import org.apache.accumulo.core.data.ArrayByteSequence; @@ -232,9 +231,8 @@ public Iterator> iterator() { SortedKeyValueIterator skvi; try { IteratorEnvironment iterEnv = new ClientIteratorEnvironment.Builder() - .withEnvironment(new ClientServiceEnvironmentImpl(context.get())) - .withAuthorizations(getAuthorizations()).withScope(IteratorScope.scan) - .withTableId(tableId.get()) + .withClient(context.get()).withAuthorizations(getAuthorizations()) + .withScope(IteratorScope.scan).withTableId(tableId.get()) .withSamplerConfiguration(getIteratorSamplerConfigurationInternal()).build(); IteratorBuilder ib = IteratorBuilder.builder(tm.values()).opts(serverSideIteratorOptions).env(iterEnv).build(); diff --git a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java index 1541098844f..253443c017b 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java +++ b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java @@ -25,6 +25,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map.Entry; +import java.util.Optional; import java.util.Set; import java.util.SortedSet; @@ -78,6 +79,16 @@ class RFileScanner extends ScannerOptions implements Scanner { + private static class RFileScannerIteratorEnvironmentBuilder + extends ClientIteratorEnvironment.Builder { + + public ClientIteratorEnvironment.Builder withEnvironment(ClientServiceEnvironmentImpl env) { + this.env = Optional.of(env); + return this; + } + + } + private static final byte[] EMPTY_BYTES = new byte[0]; private static final Range EMPTY_RANGE = new Range(); @@ -301,8 +312,9 @@ public Configuration getConfiguration(TableId tableId) { }; - ClientIteratorEnvironment.Builder iterEnvBuilder = new ClientIteratorEnvironment.Builder() - .withAuthorizations(opts.auths).withScope(IteratorScope.scan).withEnvironment(senv); + ClientIteratorEnvironment.Builder iterEnvBuilder = + new RFileScannerIteratorEnvironmentBuilder().withEnvironment(senv) + .withAuthorizations(opts.auths).withScope(IteratorScope.scan); if (getSamplerConfiguration() != null) { iterEnvBuilder.withSamplerConfiguration(getSamplerConfiguration()); } diff --git a/core/src/main/java/org/apache/accumulo/core/clientImpl/OfflineIterator.java b/core/src/main/java/org/apache/accumulo/core/clientImpl/OfflineIterator.java index 4b890ecd96d..89c4e35779a 100644 --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/OfflineIterator.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/OfflineIterator.java @@ -245,9 +245,9 @@ private SortedKeyValueIterator createIterator(KeyExtent extent, MultiIterator multiIter = new MultiIterator(readers, extent); - ClientIteratorEnvironment.Builder iterEnvBuilder = new ClientIteratorEnvironment.Builder() - .withAuthorizations(authorizations).withScope(IteratorScope.scan).withTableId(tableId) - .withEnvironment(new ClientServiceEnvironmentImpl(context)); + ClientIteratorEnvironment.Builder iterEnvBuilder = + new ClientIteratorEnvironment.Builder().withAuthorizations(authorizations) + .withScope(IteratorScope.scan).withTableId(tableId).withClient(context); if (scannerSamplerConfig != null) { iterEnvBuilder.withSamplerConfiguration(scannerSamplerConfig); } diff --git a/core/src/main/java/org/apache/accumulo/core/iterators/ClientIteratorEnvironment.java b/core/src/main/java/org/apache/accumulo/core/iterators/ClientIteratorEnvironment.java index ca129f30139..18bd8b7e127 100644 --- a/core/src/main/java/org/apache/accumulo/core/iterators/ClientIteratorEnvironment.java +++ b/core/src/main/java/org/apache/accumulo/core/iterators/ClientIteratorEnvironment.java @@ -21,9 +21,11 @@ import java.io.IOException; import java.util.Optional; +import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.PluginEnvironment; import org.apache.accumulo.core.client.SampleNotPresentException; import org.apache.accumulo.core.client.sample.SamplerConfiguration; +import org.apache.accumulo.core.clientImpl.ClientContext; import org.apache.accumulo.core.clientImpl.ClientServiceEnvironmentImpl; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.TableId; @@ -45,7 +47,7 @@ public static class Builder { private boolean isUserCompaction = false; private Optional tableId = Optional.empty(); private Optional samplerConfig = Optional.empty(); - private Optional env = Optional.empty(); + protected Optional env = Optional.empty(); public Builder withScope(IteratorScope scope) { this.scope = Optional.of(scope); @@ -77,8 +79,8 @@ public Builder withSamplerConfiguration(SamplerConfiguration sc) { return this; } - public Builder withEnvironment(ClientServiceEnvironmentImpl env) { - this.env = Optional.of(env); + public Builder withClient(AccumuloClient client) { + this.env = Optional.of(new ClientServiceEnvironmentImpl((ClientContext) client)); return this; } diff --git a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/IteratorTestInput.java b/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/IteratorTestInput.java index 758165906ec..a0363e0f30d 100644 --- a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/IteratorTestInput.java +++ b/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/IteratorTestInput.java @@ -49,7 +49,7 @@ public class IteratorTestInput { private final IteratorEnvironment iteratorEnvironment; /** - * Construct an instance of the test input, using {@link TestIteratorEnvironment}. + * Construct an instance of the test input, using {@link ClientIteratorEnvironment}. * * @param iteratorClass The class for the iterator to test. * @param iteratorOptions Options, if any, to provide to the iterator ({@link IteratorSetting}'s @@ -93,7 +93,7 @@ public IteratorTestInput(Class> iter * @param families Column families passed to {@link SortedKeyValueIterator#seek}. * @param inclusive Whether the families are inclusive or exclusive. * @param iterEnv An optional provided {@link IteratorEnvironment}. - * {@link TestIteratorEnvironment} will be used if null. + * {@link ClientIteratorEnvironment} will be used if null. */ public IteratorTestInput(Class> iteratorClass, Map iteratorOptions, Range range, SortedMap input, From fdd77d0712c2b0ee7b380ecf67ab225839cf04b5 Mon Sep 17 00:00:00 2001 From: Dave Marion Date: Mon, 21 Apr 2025 15:16:17 -0400 Subject: [PATCH 04/11] Update core/src/main/java/org/apache/accumulo/core/iterators/ClientIteratorEnvironment.java Co-authored-by: Kevin Rathbun --- .../accumulo/core/iterators/ClientIteratorEnvironment.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/accumulo/core/iterators/ClientIteratorEnvironment.java b/core/src/main/java/org/apache/accumulo/core/iterators/ClientIteratorEnvironment.java index 18bd8b7e127..70f68271b08 100644 --- a/core/src/main/java/org/apache/accumulo/core/iterators/ClientIteratorEnvironment.java +++ b/core/src/main/java/org/apache/accumulo/core/iterators/ClientIteratorEnvironment.java @@ -196,7 +196,7 @@ public ServiceEnvironment getServiceEnv() { @Override public PluginEnvironment getPluginEnv() { - return getServiceEnv(); + return env.orElseThrow(); } @Override From c14c35f221a59f093619322ff2b9187d75d9264d Mon Sep 17 00:00:00 2001 From: Dave Marion Date: Mon, 21 Apr 2025 19:48:50 +0000 Subject: [PATCH 05/11] Addressed some PR comments --- .../client/ClientSideIteratorScanner.java | 9 ++- .../core/client/rfile/RFileScanner.java | 76 ++++++++++--------- .../core/clientImpl/OfflineIterator.java | 4 +- 3 files changed, 50 insertions(+), 39 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java b/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java index a8617c2178f..bea2c1e5881 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java +++ b/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java @@ -230,10 +230,13 @@ public Iterator> iterator() { SortedKeyValueIterator skvi; try { - IteratorEnvironment iterEnv = new ClientIteratorEnvironment.Builder() + ClientIteratorEnvironment.Builder builder = new ClientIteratorEnvironment.Builder() .withClient(context.get()).withAuthorizations(getAuthorizations()) - .withScope(IteratorScope.scan).withTableId(tableId.get()) - .withSamplerConfiguration(getIteratorSamplerConfigurationInternal()).build(); + .withScope(IteratorScope.scan).withTableId(tableId.get()); + if (smi.samplerConfig != null) { + builder.withSamplerConfiguration(getIteratorSamplerConfigurationInternal()); + } + IteratorEnvironment iterEnv = builder.build(); IteratorBuilder ib = IteratorBuilder.builder(tm.values()).opts(serverSideIteratorOptions).env(iterEnv).build(); diff --git a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java index 253443c017b..a21a52bd36d 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java +++ b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java @@ -89,6 +89,45 @@ public ClientIteratorEnvironment.Builder withEnvironment(ClientServiceEnvironmen } + private static class RFileScannerEnvironmentImpl extends ClientServiceEnvironmentImpl { + + private final Opts opts; + + public RFileScannerEnvironmentImpl(Opts opts) { + super(null); + this.opts = opts; + } + + @Override + public String getTableName(TableId tableId) throws TableNotFoundException { + throw new IllegalStateException("TableId not known in RFileScanner"); + } + + @Override + public T instantiate(String className, Class base) + throws ReflectiveOperationException, IOException { + return RFileScanner.class.getClassLoader().loadClass(className).asSubclass(base) + .getDeclaredConstructor().newInstance(); + } + + @Override + public T instantiate(TableId tableId, String className, Class base) + throws ReflectiveOperationException, IOException { + return instantiate(className, base); + } + + @Override + public Configuration getConfiguration() { + return new ConfigurationImpl(new ConfigurationCopy(DefaultConfiguration.getInstance())); + } + + @Override + public Configuration getConfiguration(TableId tableId) { + return new ConfigurationImpl(new ConfigurationCopy(this.opts.tableConfig)); + } + + } + private static final byte[] EMPTY_BYTES = new byte[0]; private static final Range EMPTY_RANGE = new Range(); @@ -280,41 +319,10 @@ public Iterator> iterator() { EMPTY_BYTES, tableConf); } - ClientServiceEnvironmentImpl senv = new ClientServiceEnvironmentImpl(null) { - - @Override - public String getTableName(TableId tableId) throws TableNotFoundException { - throw new IllegalStateException("TableId not known in RFileScanner"); - } - - @Override - public T instantiate(String className, Class base) - throws ReflectiveOperationException, IOException { - return RFileScanner.class.getClassLoader().loadClass(className).asSubclass(base) - .getDeclaredConstructor().newInstance(); - } - - @Override - public T instantiate(TableId tableId, String className, Class base) - throws ReflectiveOperationException, IOException { - return instantiate(className, base); - } - - @Override - public Configuration getConfiguration() { - return new ConfigurationImpl(new ConfigurationCopy(DefaultConfiguration.getInstance())); - } - - @Override - public Configuration getConfiguration(TableId tableId) { - return new ConfigurationImpl(new ConfigurationCopy(opts.tableConfig)); - } - - }; - ClientIteratorEnvironment.Builder iterEnvBuilder = - new RFileScannerIteratorEnvironmentBuilder().withEnvironment(senv) - .withAuthorizations(opts.auths).withScope(IteratorScope.scan); + new RFileScannerIteratorEnvironmentBuilder() + .withEnvironment(new RFileScannerEnvironmentImpl(opts)).withAuthorizations(opts.auths) + .withScope(IteratorScope.scan); if (getSamplerConfiguration() != null) { iterEnvBuilder.withSamplerConfiguration(getSamplerConfiguration()); } diff --git a/core/src/main/java/org/apache/accumulo/core/clientImpl/OfflineIterator.java b/core/src/main/java/org/apache/accumulo/core/clientImpl/OfflineIterator.java index 89c4e35779a..02a103775a4 100644 --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/OfflineIterator.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/OfflineIterator.java @@ -248,8 +248,8 @@ private SortedKeyValueIterator createIterator(KeyExtent extent, ClientIteratorEnvironment.Builder iterEnvBuilder = new ClientIteratorEnvironment.Builder().withAuthorizations(authorizations) .withScope(IteratorScope.scan).withTableId(tableId).withClient(context); - if (scannerSamplerConfig != null) { - iterEnvBuilder.withSamplerConfiguration(scannerSamplerConfig); + if (samplerConfImpl != null) { + iterEnvBuilder.withSamplerConfiguration(samplerConfImpl.toSamplerConfiguration()); } IteratorEnvironment iterEnv = iterEnvBuilder.build(); From b9f12f4ce3ce8b48b861135306dd2a6886dd5c5e Mon Sep 17 00:00:00 2001 From: Dave Marion Date: Mon, 21 Apr 2025 20:07:40 +0000 Subject: [PATCH 06/11] Fixed RFileScannerEnvironmentImpl.getConfiguration(TableId) --- .../org/apache/accumulo/core/client/rfile/RFileScanner.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java index a21a52bd36d..bebf6d4254e 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java +++ b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java @@ -123,7 +123,9 @@ public Configuration getConfiguration() { @Override public Configuration getConfiguration(TableId tableId) { - return new ConfigurationImpl(new ConfigurationCopy(this.opts.tableConfig)); + ConfigurationCopy tableCC = new ConfigurationCopy(DefaultConfiguration.getInstance()); + opts.tableConfig.forEach(tableCC::set); + return new ConfigurationImpl(tableCC); } } From 3cec5f7c4fed6708ec47698b16bdfc7aa8e629ee Mon Sep 17 00:00:00 2001 From: Dave Marion Date: Tue, 22 Apr 2025 17:14:15 +0000 Subject: [PATCH 07/11] Added bool for sampling enabled, fixed up sampling configuration uses --- .../client/ClientSideIteratorScanner.java | 7 +-- .../core/client/rfile/RFileScanner.java | 1 + .../iterators/ClientIteratorEnvironment.java | 45 ++++++++++++------- .../accumulo/core/file/rfile/RFileTest.java | 7 ++- .../core/iterators/SortedMapIteratorTest.java | 2 +- 5 files changed, 40 insertions(+), 22 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java b/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java index bea2c1e5881..6550ccfd3f1 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java +++ b/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java @@ -232,9 +232,10 @@ public Iterator> iterator() { try { ClientIteratorEnvironment.Builder builder = new ClientIteratorEnvironment.Builder() .withClient(context.get()).withAuthorizations(getAuthorizations()) - .withScope(IteratorScope.scan).withTableId(tableId.get()); - if (smi.samplerConfig != null) { - builder.withSamplerConfiguration(getIteratorSamplerConfigurationInternal()); + .withScope(IteratorScope.scan).withTableId(tableId.get()) + .withSamplerConfiguration(getIteratorSamplerConfigurationInternal()); + if (getSamplerConfiguration() != null) { + builder.withSamplingEnabled(); } IteratorEnvironment iterEnv = builder.build(); IteratorBuilder ib = diff --git a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java index bebf6d4254e..9cd2978adf3 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java +++ b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java @@ -327,6 +327,7 @@ public Iterator> iterator() { .withScope(IteratorScope.scan); if (getSamplerConfiguration() != null) { iterEnvBuilder.withSamplerConfiguration(getSamplerConfiguration()); + iterEnvBuilder.withSamplingEnabled(); } IteratorEnvironment iterEnv = iterEnvBuilder.build(); try { diff --git a/core/src/main/java/org/apache/accumulo/core/iterators/ClientIteratorEnvironment.java b/core/src/main/java/org/apache/accumulo/core/iterators/ClientIteratorEnvironment.java index 70f68271b08..c6fdee1328c 100644 --- a/core/src/main/java/org/apache/accumulo/core/iterators/ClientIteratorEnvironment.java +++ b/core/src/main/java/org/apache/accumulo/core/iterators/ClientIteratorEnvironment.java @@ -27,6 +27,7 @@ import org.apache.accumulo.core.client.sample.SamplerConfiguration; import org.apache.accumulo.core.clientImpl.ClientContext; import org.apache.accumulo.core.clientImpl.ClientServiceEnvironmentImpl; +import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.data.Value; @@ -47,6 +48,7 @@ public static class Builder { private boolean isUserCompaction = false; private Optional tableId = Optional.empty(); private Optional samplerConfig = Optional.empty(); + private boolean samplingEnabled = false; protected Optional env = Optional.empty(); public Builder withScope(IteratorScope scope) { @@ -74,8 +76,13 @@ public Builder withTableId(TableId tableId) { return this; } + public Builder withSamplingEnabled() { + this.samplingEnabled = true; + return this; + } + public Builder withSamplerConfiguration(SamplerConfiguration sc) { - this.samplerConfig = Optional.of(sc); + this.samplerConfig = Optional.ofNullable(sc); return this; } @@ -90,9 +97,6 @@ public ClientIteratorEnvironment build() { } - private static final UnsupportedOperationException UOE = - new UnsupportedOperationException("Feature not supported"); - public static final IteratorEnvironment DEFAULT = new Builder().build(); private final Optional scope; @@ -101,6 +105,7 @@ public ClientIteratorEnvironment build() { private final boolean isUserCompaction; private final Optional tableId; private final Optional samplerConfig; + private final boolean samplingEnabled; private final Optional env; private ClientIteratorEnvironment(Builder builder) { @@ -111,18 +116,21 @@ private ClientIteratorEnvironment(Builder builder) { this.tableId = builder.tableId; this.samplerConfig = builder.samplerConfig; this.env = builder.env; + this.samplingEnabled = builder.samplingEnabled; } + /** + * Copy constructor used for enabling sample. Only called from {@link cloneWithSamplingEnabled}. + */ private ClientIteratorEnvironment(ClientIteratorEnvironment copy) { - this.scope = copy.scope.isPresent() ? Optional.of(copy.scope.orElseThrow()) : Optional.empty(); + this.scope = copy.scope; this.isFullMajorCompaction = copy.isFullMajorCompaction; - this.auths = copy.auths.isPresent() ? Optional.of(copy.auths.orElseThrow()) : Optional.empty(); + this.auths = copy.auths; this.isUserCompaction = copy.isUserCompaction; - this.tableId = - copy.tableId.isPresent() ? Optional.of(copy.tableId.orElseThrow()) : Optional.empty(); - this.samplerConfig = copy.samplerConfig.isPresent() - ? Optional.of(copy.samplerConfig.orElseThrow()) : Optional.empty(); - this.env = copy.env.isPresent() ? Optional.of(copy.env.orElseThrow()) : Optional.empty(); + this.tableId = copy.tableId; + this.samplerConfig = copy.samplerConfig; + this.env = copy.env; + this.samplingEnabled = true; } @Override @@ -150,20 +158,22 @@ public boolean isFullMajorCompaction() { @Override @Deprecated(since = "2.0.0") public void registerSideChannel(SortedKeyValueIterator iter) { - throw UOE; + throw new UnsupportedOperationException("Feature not supported"); } @Override public Authorizations getAuthorizations() { if (getIteratorScope() != IteratorScope.scan) { - throw new IllegalStateException("Iterator scope is not majc"); + throw new IllegalStateException("Iterator scope is not scan"); } return auths.orElseThrow(); } @Override public IteratorEnvironment cloneWithSamplingEnabled() { - if (!isSamplingEnabled()) { + String samplerClass = + getPluginEnv().getConfiguration(getTableId()).get(Property.TABLE_SAMPLER.getKey()); + if (samplerClass == null || samplerClass.isBlank()) { throw new SampleNotPresentException(); } return new ClientIteratorEnvironment(this); @@ -171,12 +181,15 @@ public IteratorEnvironment cloneWithSamplingEnabled() { @Override public boolean isSamplingEnabled() { - return this.samplerConfig.isPresent(); + return this.samplingEnabled; } @Override public SamplerConfiguration getSamplerConfiguration() { - return samplerConfig.orElse(null); + if (!isSamplingEnabled()) { + return null; + } + return samplerConfig.orElseThrow(); } @Override diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java index 59c522a59f8..9f34f62ef9e 100644 --- a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java +++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java @@ -2056,8 +2056,11 @@ public void testSample() throws IOException { assertEquals(expectedDataHash, hash(trf.reader)); - IteratorEnvironment ie = new ClientIteratorEnvironment.Builder() - .withSamplerConfiguration(sc.toSamplerConfiguration()).build(); + ClientIteratorEnvironment.Builder builder = new ClientIteratorEnvironment.Builder(); + if (sc != null) { + builder.withSamplerConfiguration(sc.toSamplerConfiguration()).withSamplingEnabled(); + } + IteratorEnvironment ie = builder.build(); for (int i = 0; i < 3; i++) { // test opening and closing deep copies a few times. diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/SortedMapIteratorTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/SortedMapIteratorTest.java index 9c0441187b0..754ee70fe97 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/SortedMapIteratorTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/SortedMapIteratorTest.java @@ -36,6 +36,6 @@ public void testSampleNotPresent() { assertThrows(SampleNotPresentException.class, () -> smi.deepCopy(new ClientIteratorEnvironment.Builder() .withSamplerConfiguration(new SamplerConfiguration(RowSampler.class.getName())) - .build())); + .withSamplingEnabled().build())); } } From bf294a411bc1524e71504934dd9526f2e0ea5b9b Mon Sep 17 00:00:00 2001 From: Dave Marion Date: Tue, 22 Apr 2025 17:51:05 +0000 Subject: [PATCH 08/11] Fixup table id and table name in RFileScanner --- .../accumulo/core/client/rfile/RFileScanner.java | 10 ++++++++-- .../org/apache/accumulo/core/file/rfile/RFileTest.java | 7 ++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java index 9cd2978adf3..2f7615e46aa 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java +++ b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java @@ -100,7 +100,9 @@ public RFileScannerEnvironmentImpl(Opts opts) { @Override public String getTableName(TableId tableId) throws TableNotFoundException { - throw new IllegalStateException("TableId not known in RFileScanner"); + Preconditions.checkArgument(tableId == TABLE_ID, "Expected " + TABLE_ID + " obtained" + + " from IteratorEnvironment.getTableId(), but got: " + tableId); + return TABLE_NAME; } @Override @@ -123,6 +125,8 @@ public Configuration getConfiguration() { @Override public Configuration getConfiguration(TableId tableId) { + Preconditions.checkArgument(tableId == TABLE_ID, "Expected " + TABLE_ID + " obtained" + + " from IteratorEnvironment.getTableId(), but got: " + tableId); ConfigurationCopy tableCC = new ConfigurationCopy(DefaultConfiguration.getInstance()); opts.tableConfig.forEach(tableCC::set); return new ConfigurationImpl(tableCC); @@ -132,6 +136,8 @@ public Configuration getConfiguration(TableId tableId) { private static final byte[] EMPTY_BYTES = new byte[0]; private static final Range EMPTY_RANGE = new Range(); + private static final String TABLE_NAME = "rfileScanner"; + private static final TableId TABLE_ID = TableId.of(TABLE_NAME); private Range range; private BlockCacheManager blockCacheManager = null; @@ -324,7 +330,7 @@ public Iterator> iterator() { ClientIteratorEnvironment.Builder iterEnvBuilder = new RFileScannerIteratorEnvironmentBuilder() .withEnvironment(new RFileScannerEnvironmentImpl(opts)).withAuthorizations(opts.auths) - .withScope(IteratorScope.scan); + .withScope(IteratorScope.scan).withTableId(TABLE_ID); if (getSamplerConfiguration() != null) { iterEnvBuilder.withSamplerConfiguration(getSamplerConfiguration()); iterEnvBuilder.withSamplingEnabled(); diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java index 9f34f62ef9e..a2967203ae9 100644 --- a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java +++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java @@ -2056,11 +2056,8 @@ public void testSample() throws IOException { assertEquals(expectedDataHash, hash(trf.reader)); - ClientIteratorEnvironment.Builder builder = new ClientIteratorEnvironment.Builder(); - if (sc != null) { - builder.withSamplerConfiguration(sc.toSamplerConfiguration()).withSamplingEnabled(); - } - IteratorEnvironment ie = builder.build(); + IteratorEnvironment ie = new ClientIteratorEnvironment.Builder() + .withSamplerConfiguration(sc.toSamplerConfiguration()).withSamplingEnabled().build(); for (int i = 0; i < 3; i++) { // test opening and closing deep copies a few times. From 2d39e4693cd738e433044f0e5697ef6269528369 Mon Sep 17 00:00:00 2001 From: Dave Marion Date: Wed, 23 Apr 2025 15:18:13 +0000 Subject: [PATCH 09/11] Moved ClientIteratorEnvironment out of API package, implement suggestions --- .../client/ClientSideIteratorScanner.java | 2 +- .../core/client/rfile/RFileScanner.java | 49 ++++++------------- .../core/clientImpl/OfflineIterator.java | 2 +- .../ClientIteratorEnvironment.java | 21 ++++---- .../accumulo/core/file/rfile/RFileTest.java | 2 +- .../FirstEntryInRowIteratorTest.java | 1 + .../core/iterators/SortedMapIteratorTest.java | 1 + .../iterators/user/ColumnSliceFilterTest.java | 2 +- .../core/iterators/user/CombinerTest.java | 2 +- .../core/iterators/user/FilterTest.java | 2 +- .../user/IndexedDocIteratorTest.java | 2 +- .../user/IntersectingIteratorTest.java | 2 +- .../iterators/user/LargeRowFilterTest.java | 2 +- .../core/iterators/user/RegExFilterTest.java | 2 +- .../user/RowDeletingIteratorTest.java | 2 +- .../user/RowEncodingIteratorTest.java | 2 +- .../core/iterators/user/RowFilterTest.java | 2 +- .../user/TransformingIteratorTest.java | 2 +- .../iteratorsImpl/IteratorConfigUtilTest.java | 1 - .../iteratortest/IteratorTestInput.java | 2 +- .../replication/StatusCombinerTest.java | 2 +- .../test/iterator/SummingCombinerTest.java | 2 +- 22 files changed, 43 insertions(+), 64 deletions(-) rename core/src/main/java/org/apache/accumulo/core/{iterators => iteratorsImpl}/ClientIteratorEnvironment.java (91%) diff --git a/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java b/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java index 6550ccfd3f1..8a646219511 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java +++ b/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java @@ -44,11 +44,11 @@ import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.dataImpl.thrift.IterInfo; -import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorAdapter; import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; import org.apache.accumulo.core.iteratorsImpl.IteratorBuilder; import org.apache.accumulo.core.iteratorsImpl.IteratorConfigUtil; import org.apache.accumulo.core.security.Authorizations; diff --git a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java index 2f7615e46aa..38172594afb 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java +++ b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java @@ -25,7 +25,6 @@ import java.util.Iterator; import java.util.List; import java.util.Map.Entry; -import java.util.Optional; import java.util.Set; import java.util.SortedSet; @@ -54,11 +53,11 @@ import org.apache.accumulo.core.file.blockfile.impl.CacheProvider; import org.apache.accumulo.core.file.rfile.RFile; import org.apache.accumulo.core.file.rfile.RFile.Reader; -import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorAdapter; import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; import org.apache.accumulo.core.iteratorsImpl.IteratorBuilder; import org.apache.accumulo.core.iteratorsImpl.IteratorConfigUtil; import org.apache.accumulo.core.iteratorsImpl.system.MultiIterator; @@ -79,23 +78,19 @@ class RFileScanner extends ScannerOptions implements Scanner { - private static class RFileScannerIteratorEnvironmentBuilder - extends ClientIteratorEnvironment.Builder { - - public ClientIteratorEnvironment.Builder withEnvironment(ClientServiceEnvironmentImpl env) { - this.env = Optional.of(env); - return this; - } - - } - private static class RFileScannerEnvironmentImpl extends ClientServiceEnvironmentImpl { - private final Opts opts; + private final Configuration conf; + private final Configuration tableConf; public RFileScannerEnvironmentImpl(Opts opts) { super(null); - this.opts = opts; + conf = new ConfigurationImpl(new ConfigurationCopy(DefaultConfiguration.getInstance())); + ConfigurationCopy tableCC = new ConfigurationCopy(DefaultConfiguration.getInstance()); + if (opts.tableConfig != null) { + opts.tableConfig.forEach(tableCC::set); + } + tableConf = new ConfigurationImpl(tableCC); } @Override @@ -105,31 +100,16 @@ public String getTableName(TableId tableId) throws TableNotFoundException { return TABLE_NAME; } - @Override - public T instantiate(String className, Class base) - throws ReflectiveOperationException, IOException { - return RFileScanner.class.getClassLoader().loadClass(className).asSubclass(base) - .getDeclaredConstructor().newInstance(); - } - - @Override - public T instantiate(TableId tableId, String className, Class base) - throws ReflectiveOperationException, IOException { - return instantiate(className, base); - } - @Override public Configuration getConfiguration() { - return new ConfigurationImpl(new ConfigurationCopy(DefaultConfiguration.getInstance())); + return conf; } @Override public Configuration getConfiguration(TableId tableId) { Preconditions.checkArgument(tableId == TABLE_ID, "Expected " + TABLE_ID + " obtained" + " from IteratorEnvironment.getTableId(), but got: " + tableId); - ConfigurationCopy tableCC = new ConfigurationCopy(DefaultConfiguration.getInstance()); - opts.tableConfig.forEach(tableCC::set); - return new ConfigurationImpl(tableCC); + return tableConf; } } @@ -327,10 +307,9 @@ public Iterator> iterator() { EMPTY_BYTES, tableConf); } - ClientIteratorEnvironment.Builder iterEnvBuilder = - new RFileScannerIteratorEnvironmentBuilder() - .withEnvironment(new RFileScannerEnvironmentImpl(opts)).withAuthorizations(opts.auths) - .withScope(IteratorScope.scan).withTableId(TABLE_ID); + ClientIteratorEnvironment.Builder iterEnvBuilder = new ClientIteratorEnvironment.Builder() + .withEnvironment(new RFileScannerEnvironmentImpl(opts)).withAuthorizations(opts.auths) + .withScope(IteratorScope.scan).withTableId(TABLE_ID); if (getSamplerConfiguration() != null) { iterEnvBuilder.withSamplerConfiguration(getSamplerConfiguration()); iterEnvBuilder.withSamplingEnabled(); diff --git a/core/src/main/java/org/apache/accumulo/core/clientImpl/OfflineIterator.java b/core/src/main/java/org/apache/accumulo/core/clientImpl/OfflineIterator.java index 02a103775a4..d64e0181487 100644 --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/OfflineIterator.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/OfflineIterator.java @@ -50,10 +50,10 @@ import org.apache.accumulo.core.dataImpl.KeyExtent; import org.apache.accumulo.core.file.FileOperations; import org.apache.accumulo.core.file.FileSKVIterator; -import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; import org.apache.accumulo.core.iteratorsImpl.IteratorConfigUtil; import org.apache.accumulo.core.iteratorsImpl.system.MultiIterator; import org.apache.accumulo.core.iteratorsImpl.system.SystemIteratorUtil; diff --git a/core/src/main/java/org/apache/accumulo/core/iterators/ClientIteratorEnvironment.java b/core/src/main/java/org/apache/accumulo/core/iteratorsImpl/ClientIteratorEnvironment.java similarity index 91% rename from core/src/main/java/org/apache/accumulo/core/iterators/ClientIteratorEnvironment.java rename to core/src/main/java/org/apache/accumulo/core/iteratorsImpl/ClientIteratorEnvironment.java index c6fdee1328c..c5de2da88c9 100644 --- a/core/src/main/java/org/apache/accumulo/core/iterators/ClientIteratorEnvironment.java +++ b/core/src/main/java/org/apache/accumulo/core/iteratorsImpl/ClientIteratorEnvironment.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.accumulo.core.iterators; +package org.apache.accumulo.core.iteratorsImpl; import java.io.IOException; import java.util.Optional; @@ -27,16 +27,14 @@ import org.apache.accumulo.core.client.sample.SamplerConfiguration; import org.apache.accumulo.core.clientImpl.ClientContext; import org.apache.accumulo.core.clientImpl.ClientServiceEnvironmentImpl; -import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; -import org.apache.accumulo.core.iteratorsImpl.system.MapFileIterator; +import org.apache.accumulo.core.iterators.SortedKeyValueIterator; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.spi.common.ServiceEnvironment; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; public class ClientIteratorEnvironment implements IteratorEnvironment { @@ -86,6 +84,11 @@ public Builder withSamplerConfiguration(SamplerConfiguration sc) { return this; } + public ClientIteratorEnvironment.Builder withEnvironment(ClientServiceEnvironmentImpl env) { + this.env = Optional.of(env); + return this; + } + public Builder withClient(AccumuloClient client) { this.env = Optional.of(new ClientServiceEnvironmentImpl((ClientContext) client)); return this; @@ -137,9 +140,7 @@ private ClientIteratorEnvironment(ClientIteratorEnvironment copy) { @Deprecated(since = "2.0.0") public SortedKeyValueIterator reserveMapFileReader(String mapFileName) throws IOException { - Configuration hadoopConf = new Configuration(); - FileSystem fs = FileSystem.get(hadoopConf); - return new MapFileIterator(fs, mapFileName, hadoopConf); + throw new UnsupportedOperationException("Feature not supported"); } @Override @@ -171,9 +172,7 @@ public Authorizations getAuthorizations() { @Override public IteratorEnvironment cloneWithSamplingEnabled() { - String samplerClass = - getPluginEnv().getConfiguration(getTableId()).get(Property.TABLE_SAMPLER.getKey()); - if (samplerClass == null || samplerClass.isBlank()) { + if (samplerConfig.isEmpty()) { throw new SampleNotPresentException(); } return new ClientIteratorEnvironment(this); diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java index a2967203ae9..0bb58e6bb55 100644 --- a/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java +++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java @@ -71,9 +71,9 @@ import org.apache.accumulo.core.file.blockfile.impl.CachableBlockFile.CachableBuilder; import org.apache.accumulo.core.file.rfile.RFile.Reader; import org.apache.accumulo.core.file.rfile.bcfile.BCFile; -import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; import org.apache.accumulo.core.iteratorsImpl.system.ColumnFamilySkippingIterator; import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/FirstEntryInRowIteratorTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/FirstEntryInRowIteratorTest.java index c76a77bf0d6..06e8311677a 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/FirstEntryInRowIteratorTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/FirstEntryInRowIteratorTest.java @@ -30,6 +30,7 @@ import org.apache.accumulo.core.data.PartialKey; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; import org.apache.accumulo.core.iteratorsImpl.system.CountingIterator; import org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator; import org.junit.jupiter.api.Test; diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/SortedMapIteratorTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/SortedMapIteratorTest.java index 754ee70fe97..3dab1bf667e 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/SortedMapIteratorTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/SortedMapIteratorTest.java @@ -25,6 +25,7 @@ import org.apache.accumulo.core.client.SampleNotPresentException; import org.apache.accumulo.core.client.sample.RowSampler; import org.apache.accumulo.core.client.sample.SamplerConfiguration; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; import org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator; import org.junit.jupiter.api.Test; diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/ColumnSliceFilterTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/ColumnSliceFilterTest.java index 18225c5b1b3..0a0716006e9 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/ColumnSliceFilterTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/ColumnSliceFilterTest.java @@ -34,8 +34,8 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorEnvironment; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; import org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator; import org.apache.hadoop.io.Text; import org.junit.jupiter.api.BeforeEach; diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java index d6e0f7ca279..04c64acf6dc 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java @@ -37,7 +37,6 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; import org.apache.accumulo.core.iterators.Combiner; import org.apache.accumulo.core.iterators.Combiner.ValueIterator; import org.apache.accumulo.core.iterators.CombinerTestUtil; @@ -50,6 +49,7 @@ import org.apache.accumulo.core.iterators.SortedKeyValueIterator; import org.apache.accumulo.core.iterators.TypedValueCombiner; import org.apache.accumulo.core.iterators.ValueFormatException; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; import org.apache.accumulo.core.iteratorsImpl.system.MultiIterator; import org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator; import org.apache.hadoop.io.Text; diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/FilterTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/FilterTest.java index a3574cbc8b5..c1dd3466769 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/FilterTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/FilterTest.java @@ -40,9 +40,9 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; import org.apache.accumulo.core.iterators.Filter; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; import org.apache.accumulo.core.iteratorsImpl.system.ColumnQualifierFilter; import org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator; import org.apache.accumulo.core.iteratorsImpl.system.VisibilityFilter; diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/IndexedDocIteratorTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/IndexedDocIteratorTest.java index 8a1d15ed2b1..b9ee9c6a3a9 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/IndexedDocIteratorTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/IndexedDocIteratorTest.java @@ -38,9 +38,9 @@ import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.file.rfile.RFileTest; import org.apache.accumulo.core.file.rfile.RFileTest.TestRFile; -import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; import org.apache.accumulo.core.iteratorsImpl.system.MultiIterator; import org.apache.hadoop.io.Text; import org.junit.jupiter.api.Test; diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/IntersectingIteratorTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/IntersectingIteratorTest.java index e2eedb454f1..7106db27706 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/IntersectingIteratorTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/IntersectingIteratorTest.java @@ -33,9 +33,9 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; import org.apache.accumulo.core.iteratorsImpl.system.MultiIterator; import org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator; import org.apache.hadoop.io.Text; diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/LargeRowFilterTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/LargeRowFilterTest.java index 2a4523e6561..fd6c7c6b9b5 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/LargeRowFilterTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/LargeRowFilterTest.java @@ -33,8 +33,8 @@ import org.apache.accumulo.core.data.PartialKey; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; import org.apache.accumulo.core.iteratorsImpl.system.ColumnFamilySkippingIterator; import org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator; import org.junit.jupiter.api.Test; diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/RegExFilterTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/RegExFilterTest.java index f2de0ebc9bb..8bcf1dac763 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/RegExFilterTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/RegExFilterTest.java @@ -33,7 +33,7 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; import org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator; import org.apache.hadoop.io.Text; import org.junit.jupiter.api.Test; diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/RowDeletingIteratorTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/RowDeletingIteratorTest.java index c5a211a8a3a..0a8ee00998a 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/RowDeletingIteratorTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/RowDeletingIteratorTest.java @@ -32,8 +32,8 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; import org.apache.accumulo.core.iteratorsImpl.system.ColumnFamilySkippingIterator; import org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator; import org.apache.hadoop.io.Text; diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/RowEncodingIteratorTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/RowEncodingIteratorTest.java index 4a4fc587372..63da92a45c3 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/RowEncodingIteratorTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/RowEncodingIteratorTest.java @@ -38,8 +38,8 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; import org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator; import org.apache.hadoop.io.Text; import org.junit.jupiter.api.Test; diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/RowFilterTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/RowFilterTest.java index 2bdc7e2db31..4c106559ecd 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/RowFilterTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/RowFilterTest.java @@ -39,9 +39,9 @@ import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; import org.apache.accumulo.core.iteratorsImpl.system.ColumnFamilySkippingIterator; import org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator; import org.apache.hadoop.io.Text; diff --git a/core/src/test/java/org/apache/accumulo/core/iterators/user/TransformingIteratorTest.java b/core/src/test/java/org/apache/accumulo/core/iterators/user/TransformingIteratorTest.java index f0c83154b98..a9c9c92c938 100644 --- a/core/src/test/java/org/apache/accumulo/core/iterators/user/TransformingIteratorTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/user/TransformingIteratorTest.java @@ -43,11 +43,11 @@ import org.apache.accumulo.core.data.PartialKey; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; import org.apache.accumulo.core.iterators.WrappingIterator; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; import org.apache.accumulo.core.iteratorsImpl.system.ColumnFamilySkippingIterator; import org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator; import org.apache.accumulo.core.iteratorsImpl.system.VisibilityFilter; diff --git a/core/src/test/java/org/apache/accumulo/core/iteratorsImpl/IteratorConfigUtilTest.java b/core/src/test/java/org/apache/accumulo/core/iteratorsImpl/IteratorConfigUtilTest.java index 4b360162da6..251a3441b36 100644 --- a/core/src/test/java/org/apache/accumulo/core/iteratorsImpl/IteratorConfigUtilTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iteratorsImpl/IteratorConfigUtilTest.java @@ -39,7 +39,6 @@ import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.dataImpl.thrift.IterInfo; -import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; diff --git a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/IteratorTestInput.java b/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/IteratorTestInput.java index a0363e0f30d..4794c4f713f 100644 --- a/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/IteratorTestInput.java +++ b/iterator-test-harness/src/main/java/org/apache/accumulo/iteratortest/IteratorTestInput.java @@ -31,9 +31,9 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; /** * The necessary user-input to invoke a test on a {@link SortedKeyValueIterator}. diff --git a/server/base/src/test/java/org/apache/accumulo/server/replication/StatusCombinerTest.java b/server/base/src/test/java/org/apache/accumulo/server/replication/StatusCombinerTest.java index 6671e47ef99..faa24bdf736 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/replication/StatusCombinerTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/replication/StatusCombinerTest.java @@ -32,10 +32,10 @@ import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.IteratorSetting.Column; import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; import org.apache.accumulo.core.iterators.Combiner; import org.apache.accumulo.core.iterators.DevNull; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; import org.apache.accumulo.core.replication.ReplicationSchema.StatusSection; import org.apache.accumulo.server.replication.proto.Replication.Status; import org.junit.jupiter.api.BeforeEach; diff --git a/test/src/test/java/org/apache/accumulo/test/iterator/SummingCombinerTest.java b/test/src/test/java/org/apache/accumulo/test/iterator/SummingCombinerTest.java index 2043f992209..0bbb6a85139 100644 --- a/test/src/test/java/org/apache/accumulo/test/iterator/SummingCombinerTest.java +++ b/test/src/test/java/org/apache/accumulo/test/iterator/SummingCombinerTest.java @@ -28,11 +28,11 @@ import org.apache.accumulo.core.data.PartialKey; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.ClientIteratorEnvironment; import org.apache.accumulo.core.iterators.Combiner; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; import org.apache.accumulo.core.iterators.LongCombiner; import org.apache.accumulo.core.iterators.user.SummingCombiner; +import org.apache.accumulo.core.iteratorsImpl.ClientIteratorEnvironment; import org.apache.accumulo.iteratortest.IteratorTestBase; import org.apache.accumulo.iteratortest.IteratorTestInput; import org.apache.accumulo.iteratortest.IteratorTestOutput; From 1c75b6c646bbd4bf5d7cdfff7d387445ff636445 Mon Sep 17 00:00:00 2001 From: Dave Marion Date: Wed, 23 Apr 2025 19:59:37 +0000 Subject: [PATCH 10/11] Throw exception when env is already set via the Builder --- .../core/iteratorsImpl/ClientIteratorEnvironment.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/src/main/java/org/apache/accumulo/core/iteratorsImpl/ClientIteratorEnvironment.java b/core/src/main/java/org/apache/accumulo/core/iteratorsImpl/ClientIteratorEnvironment.java index c5de2da88c9..95e93e41e37 100644 --- a/core/src/main/java/org/apache/accumulo/core/iteratorsImpl/ClientIteratorEnvironment.java +++ b/core/src/main/java/org/apache/accumulo/core/iteratorsImpl/ClientIteratorEnvironment.java @@ -85,11 +85,17 @@ public Builder withSamplerConfiguration(SamplerConfiguration sc) { } public ClientIteratorEnvironment.Builder withEnvironment(ClientServiceEnvironmentImpl env) { + if (this.env.isPresent()) { + throw new IllegalStateException("withEnvironment and withClient are mutually exclusive"); + } this.env = Optional.of(env); return this; } public Builder withClient(AccumuloClient client) { + if (this.env.isPresent()) { + throw new IllegalStateException("withEnvironment and withClient are mutually exclusive"); + } this.env = Optional.of(new ClientServiceEnvironmentImpl((ClientContext) client)); return this; } From aa7e54829ecf229d6a426f8f9c46145f37a2945a Mon Sep 17 00:00:00 2001 From: Dave Marion Date: Wed, 23 Apr 2025 21:00:24 +0000 Subject: [PATCH 11/11] Added validation to ClientIteratorEnvironment.Builder --- .../iteratorsImpl/ClientIteratorEnvironment.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/iteratorsImpl/ClientIteratorEnvironment.java b/core/src/main/java/org/apache/accumulo/core/iteratorsImpl/ClientIteratorEnvironment.java index 95e93e41e37..41a514f19aa 100644 --- a/core/src/main/java/org/apache/accumulo/core/iteratorsImpl/ClientIteratorEnvironment.java +++ b/core/src/main/java/org/apache/accumulo/core/iteratorsImpl/ClientIteratorEnvironment.java @@ -18,6 +18,8 @@ */ package org.apache.accumulo.core.iteratorsImpl; +import static com.google.common.base.Preconditions.checkState; + import java.io.IOException; import java.util.Optional; @@ -50,6 +52,7 @@ public static class Builder { protected Optional env = Optional.empty(); public Builder withScope(IteratorScope scope) { + checkState(this.scope.isEmpty(), "Scope has already been set"); this.scope = Optional.of(scope); return this; } @@ -60,6 +63,7 @@ public Builder isFullMajorCompaction() { } public Builder withAuthorizations(Authorizations auths) { + checkState(this.auths.isEmpty(), "Authorizations have already been set"); this.auths = Optional.of(auths); return this; } @@ -70,6 +74,7 @@ public Builder isUserCompaction() { } public Builder withTableId(TableId tableId) { + checkState(this.tableId.isEmpty(), "TableId has already been set"); this.tableId = Optional.of(tableId); return this; } @@ -80,22 +85,19 @@ public Builder withSamplingEnabled() { } public Builder withSamplerConfiguration(SamplerConfiguration sc) { + checkState(this.samplerConfig.isEmpty(), "SamplerConfiguration has already been set"); this.samplerConfig = Optional.ofNullable(sc); return this; } public ClientIteratorEnvironment.Builder withEnvironment(ClientServiceEnvironmentImpl env) { - if (this.env.isPresent()) { - throw new IllegalStateException("withEnvironment and withClient are mutually exclusive"); - } + checkState(this.env.isEmpty(), "ClientServiceEnvironmentImpl has already been set"); this.env = Optional.of(env); return this; } public Builder withClient(AccumuloClient client) { - if (this.env.isPresent()) { - throw new IllegalStateException("withEnvironment and withClient are mutually exclusive"); - } + checkState(this.env.isEmpty(), "ClientServiceEnvironmentImpl has already been set"); this.env = Optional.of(new ClientServiceEnvironmentImpl((ClientContext) client)); return this; }