From 0fca9bc046484a813309d8eea0ca4f774d6b4440 Mon Sep 17 00:00:00 2001 From: Adam Rauch Date: Fri, 14 Nov 2025 17:08:25 -0800 Subject: [PATCH 1/9] Filter signed snapshots based on notebook filter --- .../DatabaseMigrationConfiguration.java | 2 +- .../migration/DatabaseMigrationService.java | 1 + .../DefaultDatabaseMigrationConfiguration.java | 2 +- .../DefaultMigrationSchemaHandler.java | 10 ++++++---- .../api/migration/MigrationTableHandler.java | 18 ++++++++++++++++++ 5 files changed, 27 insertions(+), 6 deletions(-) create mode 100644 api/src/org/labkey/api/migration/MigrationTableHandler.java diff --git a/api/src/org/labkey/api/migration/DatabaseMigrationConfiguration.java b/api/src/org/labkey/api/migration/DatabaseMigrationConfiguration.java index 9d0a549cb12..49cd45b4ff5 100644 --- a/api/src/org/labkey/api/migration/DatabaseMigrationConfiguration.java +++ b/api/src/org/labkey/api/migration/DatabaseMigrationConfiguration.java @@ -19,7 +19,7 @@ default void beforeMigration(){} DbScope getTargetScope(); @NotNull Set getSkipSchemas(); Predicate getColumnNameFilter(); - @Nullable TableSelector getTableSelector(DbSchemaType schemaType, TableInfo sourceTable, TableInfo targetTable, Set selectColumnNames, MigrationSchemaHandler schemaHandler); + @Nullable TableSelector getTableSelector(DbSchemaType schemaType, TableInfo sourceTable, TableInfo targetTable, Set selectColumnNames, MigrationSchemaHandler schemaHandler, @Nullable MigrationTableHandler tableHandler); default void copyAttachments(DbSchema sourceSchema, DbSchema targetSchema, MigrationSchemaHandler schemaHandler){} default void afterMigration(){} } diff --git a/api/src/org/labkey/api/migration/DatabaseMigrationService.java b/api/src/org/labkey/api/migration/DatabaseMigrationService.java index ec2c0760a7c..b5ec00fac29 100644 --- a/api/src/org/labkey/api/migration/DatabaseMigrationService.java +++ b/api/src/org/labkey/api/migration/DatabaseMigrationService.java @@ -48,6 +48,7 @@ default void migrate(DatabaseMigrationConfiguration configuration) // By default, no-op implementations default void registerSchemaHandler(MigrationSchemaHandler schemaHandler) {} + default void registerTableHandler(MigrationTableHandler tableHandler) {} default void registerMigrationFilter(MigrationFilter filter) {} default @Nullable MigrationFilter getMigrationFilter(String propertyName) diff --git a/api/src/org/labkey/api/migration/DefaultDatabaseMigrationConfiguration.java b/api/src/org/labkey/api/migration/DefaultDatabaseMigrationConfiguration.java index 7dea199479d..8b5017a0edf 100644 --- a/api/src/org/labkey/api/migration/DefaultDatabaseMigrationConfiguration.java +++ b/api/src/org/labkey/api/migration/DefaultDatabaseMigrationConfiguration.java @@ -43,7 +43,7 @@ public Predicate getColumnNameFilter() } @Override - public TableSelector getTableSelector(DbSchemaType schemaType, TableInfo sourceTable, TableInfo targetTable, Set selectColumnNames, MigrationSchemaHandler schemaHandler) + public TableSelector getTableSelector(DbSchemaType schemaType, TableInfo sourceTable, TableInfo targetTable, Set selectColumnNames, MigrationSchemaHandler schemaHandler, @Nullable MigrationTableHandler tableHandler) { return null; } diff --git a/api/src/org/labkey/api/migration/DefaultMigrationSchemaHandler.java b/api/src/org/labkey/api/migration/DefaultMigrationSchemaHandler.java index 08f4fcd3c2c..b7b6853f376 100644 --- a/api/src/org/labkey/api/migration/DefaultMigrationSchemaHandler.java +++ b/api/src/org/labkey/api/migration/DefaultMigrationSchemaHandler.java @@ -27,6 +27,7 @@ import org.labkey.api.query.FieldKey; import org.labkey.api.query.SchemaKey; import org.labkey.api.query.TableSorter; +import org.labkey.api.util.ConfigurationException; import org.labkey.api.util.GUID; import org.labkey.api.util.StringUtilsLabKey; @@ -253,9 +254,10 @@ public void copyAttachments(DatabaseMigrationConfiguration configuration, DbSche sourceSchema.getSqlDialect().appendInClauseSql(selectParents, entityIds, getTempTableInClauseGenerator(sourceSchema.getScope())); copyAttachments(configuration, sourceSchema, new SQLClause(selectParents), type); } - - // TODO: fail if type.getSelectParentEntityIdsSql() returns null? - // TODO: throw if some registered AttachmentType is not seen + else + { + throw new ConfigurationException("AttachmentType \"" + type.getUniqueName() + "\" is not configured to find parent EntityIds!"); + } }); } @@ -293,7 +295,7 @@ public static void logUnseenAttachmentTypes() if (unseen.isEmpty()) DatabaseMigrationService.LOG.info("All AttachmentTypes have been seen"); else - DatabaseMigrationService.LOG.info("These AttachmentTypes have not been seen: {}", unseen.stream().map(type -> type.getClass().getSimpleName()).collect(Collectors.joining(", "))); + throw new ConfigurationException("These AttachmentTypes have not been seen: " + unseen.stream().map(type -> type.getClass().getSimpleName()).collect(Collectors.joining(", "))); } @Override diff --git a/api/src/org/labkey/api/migration/MigrationTableHandler.java b/api/src/org/labkey/api/migration/MigrationTableHandler.java new file mode 100644 index 00000000000..a16ad7a114a --- /dev/null +++ b/api/src/org/labkey/api/migration/MigrationTableHandler.java @@ -0,0 +1,18 @@ +package org.labkey.api.migration; + +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.TableInfo; +import org.labkey.api.util.GUID; + +import java.util.Set; + +/** + * Rarely needed, this interface lets a module filter the rows of another module's table. The specific use case: LabBook + * needs to filter the compliance.SignedSnapshots table of snapshots associated with Notebooks that are excluded by a + * NotebookFilter. + */ +public interface MigrationTableHandler +{ + TableInfo getTableInfo(); + void adjustFilter(SimpleFilter filter, Set containers); +} From c1de9bfa8e8010c98bf5d29af0fe5fa9d8e5019a Mon Sep 17 00:00:00 2001 From: Adam Rauch Date: Fri, 14 Nov 2025 17:29:31 -0800 Subject: [PATCH 2/9] Use a more verbose name for rarely needed method --- .../org/labkey/api/data/dialect/BasePostgreSqlDialect.java | 4 ++-- api/src/org/labkey/api/data/dialect/SqlDialect.java | 6 +++--- .../labkey/api/migration/DefaultMigrationSchemaHandler.java | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/src/org/labkey/api/data/dialect/BasePostgreSqlDialect.java b/api/src/org/labkey/api/data/dialect/BasePostgreSqlDialect.java index 5cba2821b96..ecdb80e1507 100644 --- a/api/src/org/labkey/api/data/dialect/BasePostgreSqlDialect.java +++ b/api/src/org/labkey/api/data/dialect/BasePostgreSqlDialect.java @@ -292,11 +292,11 @@ public String addReselect(SQLFragment sql, ColumnInfo column, @Nullable String p @Override public SQLFragment appendInClauseSql(SQLFragment sql, @NotNull Collection params) { - return appendInClauseSql(sql, params, _tempTableInClauseGenerator); + return appendInClauseSqlWithCustomInClauseGenerator(sql, params, _tempTableInClauseGenerator); } @Override - public SQLFragment appendInClauseSql(SQLFragment sql, @NotNull Collection params, InClauseGenerator tempTableGenerator) + public SQLFragment appendInClauseSqlWithCustomInClauseGenerator(SQLFragment sql, @NotNull Collection params, InClauseGenerator tempTableGenerator) { if (params.size() >= TEMPTABLE_GENERATOR_MINSIZE) { diff --git a/api/src/org/labkey/api/data/dialect/SqlDialect.java b/api/src/org/labkey/api/data/dialect/SqlDialect.java index 5c19524f10c..b11a782b34a 100644 --- a/api/src/org/labkey/api/data/dialect/SqlDialect.java +++ b/api/src/org/labkey/api/data/dialect/SqlDialect.java @@ -530,11 +530,11 @@ protected Set getJdbcKeywords(SqlExecutor executor) throws SQLException, // Most callers should use this method public SQLFragment appendInClauseSql(SQLFragment sql, @NotNull Collection params) { - return appendInClauseSql(sql, params, null); + return appendInClauseSqlWithCustomInClauseGenerator(sql, params, null); } - // Use in cases where the default temp schema won't do, e.g., you need to apply a large IN clause in an external data source - public SQLFragment appendInClauseSql(SQLFragment sql, @NotNull Collection params, InClauseGenerator tempTableGenerator) + // Use only in cases where the default temp-table generator won't do, e.g., you need to apply a large IN clause in an external data source + public SQLFragment appendInClauseSqlWithCustomInClauseGenerator(SQLFragment sql, @NotNull Collection params, InClauseGenerator tempTableGenerator) { return DEFAULT_GENERATOR.appendInClauseSql(sql, params); } diff --git a/api/src/org/labkey/api/migration/DefaultMigrationSchemaHandler.java b/api/src/org/labkey/api/migration/DefaultMigrationSchemaHandler.java index b7b6853f376..39bafe83579 100644 --- a/api/src/org/labkey/api/migration/DefaultMigrationSchemaHandler.java +++ b/api/src/org/labkey/api/migration/DefaultMigrationSchemaHandler.java @@ -251,7 +251,7 @@ public void copyAttachments(DatabaseMigrationConfiguration configuration, DbSche Collection entityIds = new SqlSelector(targetSchema, sql).getCollection(String.class); SQLFragment selectParents = new SQLFragment("Parent"); // This query against the source database is likely to contain a large IN clause, so use an alternative InClauseGenerator - sourceSchema.getSqlDialect().appendInClauseSql(selectParents, entityIds, getTempTableInClauseGenerator(sourceSchema.getScope())); + sourceSchema.getSqlDialect().appendInClauseSqlWithCustomInClauseGenerator(selectParents, entityIds, getTempTableInClauseGenerator(sourceSchema.getScope())); copyAttachments(configuration, sourceSchema, new SQLClause(selectParents), type); } else From 0c5ccdb60853ec0d4c53afdabbd44fd9c33c2742 Mon Sep 17 00:00:00 2001 From: Adam Rauch Date: Fri, 14 Nov 2025 21:40:10 -0800 Subject: [PATCH 3/9] Transfer core.Documents in the background --- .../DefaultMigrationSchemaHandler.java | 7 +++-- api/src/org/labkey/api/util/JobRunner.java | 26 ++++++++++--------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/api/src/org/labkey/api/migration/DefaultMigrationSchemaHandler.java b/api/src/org/labkey/api/migration/DefaultMigrationSchemaHandler.java index 39bafe83579..ce030c31c39 100644 --- a/api/src/org/labkey/api/migration/DefaultMigrationSchemaHandler.java +++ b/api/src/org/labkey/api/migration/DefaultMigrationSchemaHandler.java @@ -29,6 +29,7 @@ import org.labkey.api.query.TableSorter; import org.labkey.api.util.ConfigurationException; import org.labkey.api.util.GUID; +import org.labkey.api.util.JobRunner; import org.labkey.api.util.StringUtilsLabKey; import java.util.ArrayList; @@ -277,14 +278,16 @@ protected void copyAttachments(DatabaseMigrationConfiguration configuration, DbS String additionalMessage = " associated with " + Arrays.stream(type).map(t -> t.getClass().getSimpleName()).collect(Collectors.joining(", ")); TableInfo sourceDocumentsTable = sourceSchema.getScope().getSchema("core", DbSchemaType.Migration).getTable("Documents"); TableInfo targetDocumentsTable = CoreSchema.getInstance().getTableInfoDocuments(); - DatabaseMigrationService.get().copySourceTableToTargetTable(configuration, sourceDocumentsTable, targetDocumentsTable, DbSchemaType.Module, false, additionalMessage, new DefaultMigrationSchemaHandler(CoreSchema.getInstance().getSchema()) + + // Queue up the core.Documents transfers and let them run in the background + JobRunner.getDefault().execute(() -> DatabaseMigrationService.get().copySourceTableToTargetTable(configuration, sourceDocumentsTable, targetDocumentsTable, DbSchemaType.Module, false, additionalMessage, new DefaultMigrationSchemaHandler(CoreSchema.getInstance().getSchema()) { @Override public FilterClause getTableFilterClause(TableInfo sourceTable, Set containers) { return filterClause; } - }); + })); } public static void logUnseenAttachmentTypes() diff --git a/api/src/org/labkey/api/util/JobRunner.java b/api/src/org/labkey/api/util/JobRunner.java index a0d0b59ca00..9b1998dbc58 100644 --- a/api/src/org/labkey/api/util/JobRunner.java +++ b/api/src/org/labkey/api/util/JobRunner.java @@ -18,9 +18,11 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; import org.labkey.api.data.DbScope; import java.util.HashMap; +import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; import java.util.concurrent.Future; @@ -51,7 +53,7 @@ public class JobRunner implements Executor private static final JobRunner _defaultJobRunner = new JobRunner("Default", 1); private final ScheduledThreadPoolExecutor _executor; - private final HashMap _jobs = new HashMap<>(); + private final Map, Job> _jobs = new HashMap<>(); public JobRunner(String name, int max) @@ -77,11 +79,6 @@ public void shutdownPre() { _executor.shutdown(); } - - @Override - public void shutdownStarted() - { - } }); } @@ -111,11 +108,16 @@ public void shutdown() _executor.shutdown(); } + public boolean awaitTermination(long timeout, @NotNull TimeUnit unit) throws InterruptedException + { + return _executor.awaitTermination(timeout, unit); + } + /** * This will schedule the runnable to execute immediately, with no delay */ @Override - public void execute(Runnable command) + public void execute(@NotNull Runnable command) { execute(command, 0); } @@ -132,7 +134,7 @@ public void execute(Runnable command, long delay) { synchronized (_jobs) { - Future task = _executor.schedule(command, delay, TimeUnit.MILLISECONDS); + Future task = _executor.schedule(command, delay, TimeUnit.MILLISECONDS); if (command instanceof Job job) { job._task = task; @@ -141,7 +143,7 @@ public void execute(Runnable command, long delay) } } - public Future submit(Runnable run) + public Future submit(Runnable run) { if (run instanceof Job) { @@ -221,13 +223,13 @@ protected void afterExecute(Runnable r, Throwable t) } else { - if (r instanceof Future) + if (r instanceof Future f) { if (null == t) { try { - ((Future)r).get(); + f.get(); } catch (ExecutionException x) { @@ -277,7 +279,7 @@ static class JobThreadFactory implements ThreadFactory } @Override - public Thread newThread(Runnable r) + public Thread newThread(@NotNull Runnable r) { Thread t = new Thread(group, r, namePrefix + threadNumber.getAndIncrement(), 0); if (t.isDaemon()) From 58e024f5c6ed8b40b869c72736cf01a4da7f6909 Mon Sep 17 00:00:00 2001 From: Adam Rauch Date: Sun, 16 Nov 2025 14:30:47 -0800 Subject: [PATCH 4/9] Dedicated JobRunner for attachment copying. Correct JobRunner termination. --- .../DefaultMigrationSchemaHandler.java | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/api/src/org/labkey/api/migration/DefaultMigrationSchemaHandler.java b/api/src/org/labkey/api/migration/DefaultMigrationSchemaHandler.java index ce030c31c39..e54fe4953a0 100644 --- a/api/src/org/labkey/api/migration/DefaultMigrationSchemaHandler.java +++ b/api/src/org/labkey/api/migration/DefaultMigrationSchemaHandler.java @@ -1,5 +1,6 @@ package org.labkey.api.migration; +import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.labkey.api.attachments.AttachmentService; @@ -31,6 +32,7 @@ import org.labkey.api.util.GUID; import org.labkey.api.util.JobRunner; import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.logging.LogHelper; import java.util.ArrayList; import java.util.Arrays; @@ -40,10 +42,13 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; public class DefaultMigrationSchemaHandler implements MigrationSchemaHandler { + private static final Logger LOG = LogHelper.getLogger(DefaultMigrationSchemaHandler.class, "Migration shutdown status"); + private final DbSchema _schema; public DefaultMigrationSchemaHandler(DbSchema schema) @@ -79,7 +84,7 @@ public List getTablesToCopy() if (!allTables.isEmpty()) { - DatabaseMigrationService.LOG.info("These tables were removed by TableSorter: {}", allTables); + LOG.info("These tables were removed by TableSorter: {}", allTables); } return sortedTables.stream() @@ -270,9 +275,10 @@ protected InClauseGenerator getTempTableInClauseGenerator(DbScope sourceScope) } private static final Set SEEN = new HashSet<>(); + private static final JobRunner ATTACHMENT_JOB_RUNNER = new JobRunner("Attachment JobRunner", 1); // Copy all core.Documents rows that match the provided filter clause - protected void copyAttachments(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, FilterClause filterClause, AttachmentType... type) + protected final void copyAttachments(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, FilterClause filterClause, AttachmentType... type) { SEEN.addAll(Arrays.asList(type)); String additionalMessage = " associated with " + Arrays.stream(type).map(t -> t.getClass().getSimpleName()).collect(Collectors.joining(", ")); @@ -280,7 +286,7 @@ protected void copyAttachments(DatabaseMigrationConfiguration configuration, DbS TableInfo targetDocumentsTable = CoreSchema.getInstance().getTableInfoDocuments(); // Queue up the core.Documents transfers and let them run in the background - JobRunner.getDefault().execute(() -> DatabaseMigrationService.get().copySourceTableToTargetTable(configuration, sourceDocumentsTable, targetDocumentsTable, DbSchemaType.Module, false, additionalMessage, new DefaultMigrationSchemaHandler(CoreSchema.getInstance().getSchema()) + ATTACHMENT_JOB_RUNNER.execute(() -> DatabaseMigrationService.get().copySourceTableToTargetTable(configuration, sourceDocumentsTable, targetDocumentsTable, DbSchemaType.Module, false, additionalMessage, new DefaultMigrationSchemaHandler(CoreSchema.getInstance().getSchema()) { @Override public FilterClause getTableFilterClause(TableInfo sourceTable, Set containers) @@ -290,15 +296,25 @@ public FilterClause getTableFilterClause(TableInfo sourceTable, Set contai })); } - public static void logUnseenAttachmentTypes() + // Global (not schema- or configuration-specific) cleanup + public static void afterMigration() throws InterruptedException { + // Report any unseen attachment types Set unseen = new HashSet<>(AttachmentService.get().getAttachmentTypes()); unseen.removeAll(SEEN); if (unseen.isEmpty()) - DatabaseMigrationService.LOG.info("All AttachmentTypes have been seen"); + LOG.info("All AttachmentTypes have been seen"); else throw new ConfigurationException("These AttachmentTypes have not been seen: " + unseen.stream().map(type -> type.getClass().getSimpleName()).collect(Collectors.joining(", "))); + + // Shut down the attachment JobRunner + LOG.info("Waiting for core.Documents background transfer to complete"); + ATTACHMENT_JOB_RUNNER.shutdown(); + if (ATTACHMENT_JOB_RUNNER.awaitTermination(1, TimeUnit.HOURS)) + LOG.info("core.Documents background transfer is complete"); + else + LOG.error("core.Documents background transfer did not complete after one hour! Giving up."); } @Override From 3c62f50b755c5da0499bd44080421c7362f3a339 Mon Sep 17 00:00:00 2001 From: Adam Rauch Date: Mon, 17 Nov 2025 16:20:15 -0800 Subject: [PATCH 5/9] Exclude assay experiment runs in the assay-skip containers --- api/src/org/labkey/api/data/SimpleFilter.java | 13 ++- .../ExperimentMigrationSchemaHandler.java | 86 ++++++++++++++++--- .../src/org/labkey/search/SearchModule.java | 2 +- 3 files changed, 89 insertions(+), 12 deletions(-) diff --git a/api/src/org/labkey/api/data/SimpleFilter.java b/api/src/org/labkey/api/data/SimpleFilter.java index e114942285d..8dff53f20ff 100644 --- a/api/src/org/labkey/api/data/SimpleFilter.java +++ b/api/src/org/labkey/api/data/SimpleFilter.java @@ -664,11 +664,19 @@ public static DatabaseIdentifier getAliasForColumnFilter(SqlDialect dialect, Col public static class InClause extends MultiValuedFilterClause { + private InClauseGenerator _tempTableGenerator = null; + public InClause(FieldKey fieldKey, Collection params) { this(fieldKey, params, false, false); } + public InClause(FieldKey fieldKey, Collection params, InClauseGenerator tempTableGenerator) + { + this(fieldKey, params, false, false); + _tempTableGenerator = tempTableGenerator; + } + public InClause(FieldKey fieldKey, Collection params, boolean urlClause) { this(fieldKey, params, urlClause, false); @@ -837,7 +845,10 @@ public SQLFragment toSQLFragment(Map columnMap, in.appendIdentifier(alias); // Dialect may want to generate database-specific SQL, especially for very large IN clauses - dialect.appendInClauseSql(in, convertedParams); + if (null == _tempTableGenerator) + dialect.appendInClauseSql(in, convertedParams); + else + dialect.appendInClauseSqlWithCustomInClauseGenerator(in, convertedParams, _tempTableGenerator); if (isIncludeNull()) { diff --git a/experiment/src/org/labkey/experiment/ExperimentMigrationSchemaHandler.java b/experiment/src/org/labkey/experiment/ExperimentMigrationSchemaHandler.java index db5b16f572d..ff88f1924a6 100644 --- a/experiment/src/org/labkey/experiment/ExperimentMigrationSchemaHandler.java +++ b/experiment/src/org/labkey/experiment/ExperimentMigrationSchemaHandler.java @@ -4,20 +4,25 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.labkey.api.attachments.AttachmentType; +import org.labkey.api.collections.CsvSet; import org.labkey.api.data.CompareType; import org.labkey.api.data.CompareType.CompareClause; import org.labkey.api.data.DbSchema; import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.SimpleFilter.AndClause; import org.labkey.api.data.SimpleFilter.FilterClause; import org.labkey.api.data.SimpleFilter.InClause; +import org.labkey.api.data.SimpleFilter.NotClause; import org.labkey.api.data.SimpleFilter.OrClause; import org.labkey.api.data.SimpleFilter.SQLClause; import org.labkey.api.data.SqlExecutor; import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; import org.labkey.api.exp.OntologyManager; import org.labkey.api.exp.api.ExpProtocolAttachmentType; import org.labkey.api.exp.api.ExpRunAttachmentType; +import org.labkey.api.migration.AssaySkipContainers; import org.labkey.api.migration.DatabaseMigrationConfiguration; import org.labkey.api.migration.DefaultMigrationSchemaHandler; import org.labkey.api.query.FieldKey; @@ -26,6 +31,7 @@ import org.labkey.experiment.api.ExperimentServiceImpl; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Set; @@ -65,27 +71,87 @@ public void beforeSchema() @Override public List getTablesToCopy() { - // No need to populate the MaterialIndexed or DataIndexed tables -- new server should be completely re-indexed after migration + // No need to populate the MaterialIndexed or DataIndexed tables -- new server will be completely re-indexed after migration List tables = super.getTablesToCopy(); tables.remove(ExperimentServiceImpl.get().getTinfoMaterialIndexed()); tables.remove(ExperimentServiceImpl.get().getTinfoDataIndexed()); return tables; } + @Override + public FilterClause getTableFilterClause(TableInfo sourceTable, Set containers) + { + FilterClause containerClause = getContainerClause(sourceTable, containers); + return switch (sourceTable.getName()) + { + case "ExperimentRun" -> getExperimentRunExclusionFilter(sourceTable, containerClause, FieldKey.fromParts("RowId"), false); + case "ProtocolApplication" -> getExperimentRunExclusionFilter(sourceTable, containerClause, FieldKey.fromParts("RunId"), false); + case "Data", "Edge" -> getExperimentRunExclusionFilter(sourceTable, containerClause, FieldKey.fromParts("RunId"), true); + case "DataInput", "MaterialInput" -> getExperimentRunExclusionFilter(sourceTable, containerClause, FieldKey.fromParts("TargetApplicationId", "RunId"), false); + case "RunList" -> getExperimentRunExclusionFilter(sourceTable, containerClause, FieldKey.fromParts("ExperimentRunId"), false); + default -> containerClause; + }; + } + + // Combine the full container clause with the assay experiment run exclusion filter, if present + private FilterClause getExperimentRunExclusionFilter(TableInfo sourceTable, FilterClause containerClause, FieldKey runIdFieldKey, boolean nullable) + { + Collection experimentRunsToExclude = getExperimentRunExcludeRowIds(sourceTable.getSchema()); + if (experimentRunsToExclude.isEmpty()) + return containerClause; + + FilterClause excludedRowIdClause = new NotClause(new InClause(runIdFieldKey, experimentRunsToExclude, getTempTableInClauseGenerator(sourceTable.getSchema().getScope()))); + + return new AndClause( + containerClause, + nullable ? + new OrClause( + new CompareClause(runIdFieldKey, CompareType.ISBLANK, null), + excludedRowIdClause + ) : + excludedRowIdClause + ); + } + + private Collection _experimentRunExcludeRowIds = null; + + private @NotNull Collection getExperimentRunExcludeRowIds(DbSchema expSchema) + { + if (null == _experimentRunExcludeRowIds) + { + if (AssaySkipContainers.getContainers().isEmpty()) + { + _experimentRunExcludeRowIds = Collections.emptyList(); + } + else + { + // Select all the assay runs (same filter used by assay.AssayRuns) + SQLClause assayRun = new SQLClause(new SQLFragment( + "ProtocolLSID IN (SELECT LSID FROM exp.Protocol x WHERE (ApplicationType = 'ExperimentRun') AND " + + "((SELECT MAX(pd.PropertyId) from exp.Object o, exp.ObjectProperty op, exp.PropertyDescriptor pd WHERE " + + "pd.PropertyId = op.PropertyId and op.ObjectId = o.ObjectId and o.ObjectURI = LSID AND pd.PropertyURI LIKE '%AssayDomain-Run%') IS NOT NULL))" + )); + + // Select all assay runs in the assay-skip containers + SimpleFilter filter = new SimpleFilter( + new InClause(FieldKey.fromParts("Container"), AssaySkipContainers.getContainers()), + assayRun + ); + + // Select the assay experiment run RowIds that we don't want to copy. All tables with FKs to ExperimentRun + // (or FKs to other tables with FKs to ExperimentRun) must exclude these run IDs. + _experimentRunExcludeRowIds = new TableSelector(expSchema.getTable("ExperimentRun"), new CsvSet("RowId, ProtocolLSID"), filter, null).getCollection(Integer.class); + } + } + + return _experimentRunExcludeRowIds; + } + @Override public FilterClause getContainerClause(TableInfo sourceTable, Set containers) { -// Set assayFilteredContainers = assayFilteredContainers(containers); return switch (sourceTable.getName()) { -// case "ExperimentRun", "ProtocolApplication" -> super.getContainerClause(sourceTable, assayFilteredContainers); -// case "Data" -> new AndClause( -// new InClause(FieldKey.fromParts("Container"), containers), -// new OrClause( -// new CompareClause(FieldKey.fromParts("RunId"), CompareType.ISBLANK, null), -// new InClause(FieldKey.fromParts("RunId", "Container"), assayFilteredContainers) -// ) -// ); case "DataInput" -> new AndClause( new InClause(FieldKey.fromParts("DataId", "Container"), containers), new InClause(FieldKey.fromParts("TargetApplicationId", "RunId", "Container"), containers) diff --git a/search/src/org/labkey/search/SearchModule.java b/search/src/org/labkey/search/SearchModule.java index f4f9a3a7603..d4cd37bca6e 100644 --- a/search/src/org/labkey/search/SearchModule.java +++ b/search/src/org/labkey/search/SearchModule.java @@ -193,7 +193,7 @@ public void handle(Map properties @Override public List getTablesToCopy() { - return List.of(); // Leave empty -- target server needs to index all documents + return List.of(); // Leave empty -- target server will re-index all documents } }); } From a2d15cf4688bc5a746ae3e550b8da68dc85327ff Mon Sep 17 00:00:00 2001 From: Adam Rauch Date: Tue, 18 Nov 2025 10:34:15 -0800 Subject: [PATCH 6/9] Handle ReplacedByRunId, DataAncestors, Exclusions, and ExclusionMaps --- .../ExperimentMigrationSchemaHandler.java | 91 +++++++++++++------ .../labkey/experiment/ExperimentModule.java | 45 ++++++++- 2 files changed, 109 insertions(+), 27 deletions(-) diff --git a/experiment/src/org/labkey/experiment/ExperimentMigrationSchemaHandler.java b/experiment/src/org/labkey/experiment/ExperimentMigrationSchemaHandler.java index ff88f1924a6..665f425f430 100644 --- a/experiment/src/org/labkey/experiment/ExperimentMigrationSchemaHandler.java +++ b/experiment/src/org/labkey/experiment/ExperimentMigrationSchemaHandler.java @@ -8,6 +8,7 @@ import org.labkey.api.data.CompareType; import org.labkey.api.data.CompareType.CompareClause; import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; import org.labkey.api.data.SQLFragment; import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.SimpleFilter.AndClause; @@ -27,6 +28,7 @@ import org.labkey.api.migration.DefaultMigrationSchemaHandler; import org.labkey.api.query.FieldKey; import org.labkey.api.util.GUID; +import org.labkey.api.util.StringUtilsLabKey; import org.labkey.api.util.logging.LogHelper; import org.labkey.experiment.api.ExperimentServiceImpl; @@ -37,7 +39,7 @@ class ExperimentMigrationSchemaHandler extends DefaultMigrationSchemaHandler { - private static final Logger LOG = LogHelper.getLogger(ExperimentMigrationSchemaHandler.class, "Progress of exp deletes during database migration"); + private static final Logger LOG = LogHelper.getLogger(ExperimentMigrationSchemaHandler.class, "Progress of database migration"); public ExperimentMigrationSchemaHandler() { @@ -89,6 +91,7 @@ public FilterClause getTableFilterClause(TableInfo sourceTable, Set contai case "Data", "Edge" -> getExperimentRunExclusionFilter(sourceTable, containerClause, FieldKey.fromParts("RunId"), true); case "DataInput", "MaterialInput" -> getExperimentRunExclusionFilter(sourceTable, containerClause, FieldKey.fromParts("TargetApplicationId", "RunId"), false); case "RunList" -> getExperimentRunExclusionFilter(sourceTable, containerClause, FieldKey.fromParts("ExperimentRunId"), false); + case "DataAncestors" -> getExperimentRunExclusionFilter(sourceTable, containerClause, FieldKey.fromParts("RowId", "RunId"), true); default -> containerClause; }; } @@ -96,55 +99,91 @@ public FilterClause getTableFilterClause(TableInfo sourceTable, Set contai // Combine the full container clause with the assay experiment run exclusion filter, if present private FilterClause getExperimentRunExclusionFilter(TableInfo sourceTable, FilterClause containerClause, FieldKey runIdFieldKey, boolean nullable) { - Collection experimentRunsToExclude = getExperimentRunExcludeRowIds(sourceTable.getSchema()); - if (experimentRunsToExclude.isEmpty()) - return containerClause; + FilterClause excludedRowIdClause = getExcludedRowIdClause(sourceTable, runIdFieldKey); - FilterClause excludedRowIdClause = new NotClause(new InClause(runIdFieldKey, experimentRunsToExclude, getTempTableInClauseGenerator(sourceTable.getSchema().getScope()))); - - return new AndClause( - containerClause, - nullable ? - new OrClause( - new CompareClause(runIdFieldKey, CompareType.ISBLANK, null), + return excludedRowIdClause == null ? + containerClause : + new AndClause( + containerClause, + nullable ? + new OrClause( + new CompareClause(runIdFieldKey, CompareType.ISBLANK, null), + excludedRowIdClause + ) : excludedRowIdClause - ) : - excludedRowIdClause - ); + ); } - private Collection _experimentRunExcludeRowIds = null; + @Nullable FilterClause getExcludedRowIdClause(TableInfo sourceTable, FieldKey runIdFieldKey) + { + Collection experimentRunsToExclude = getExperimentRunExcludedRowIds(sourceTable.getSchema()); - private @NotNull Collection getExperimentRunExcludeRowIds(DbSchema expSchema) + return experimentRunsToExclude.isEmpty() ? + null : + new NotClause(new InClause(runIdFieldKey, experimentRunsToExclude, getTempTableInClauseGenerator(sourceTable.getSchema().getScope()))); + } + + private Collection _experimentRunExcludedRowIds = null; + + private @NotNull Collection getExperimentRunExcludedRowIds(DbSchema schema) { - if (null == _experimentRunExcludeRowIds) + if (null == _experimentRunExcludedRowIds) { if (AssaySkipContainers.getContainers().isEmpty()) { - _experimentRunExcludeRowIds = Collections.emptyList(); + _experimentRunExcludedRowIds = Collections.emptyList(); } else { + // We need the source exp schema; if it wasn't passed in, retrieve it from the scope. + DbSchema expSchema = "exp".equals(schema.getName()) ? schema : schema.getScope().getSchema("exp", DbSchemaType.Migration); + // Select all the assay runs (same filter used by assay.AssayRuns) - SQLClause assayRun = new SQLClause(new SQLFragment( + SQLFragment assayRunSql = new SQLFragment( "ProtocolLSID IN (SELECT LSID FROM exp.Protocol x WHERE (ApplicationType = 'ExperimentRun') AND " + "((SELECT MAX(pd.PropertyId) from exp.Object o, exp.ObjectProperty op, exp.PropertyDescriptor pd WHERE " + "pd.PropertyId = op.PropertyId and op.ObjectId = o.ObjectId and o.ObjectURI = LSID AND pd.PropertyURI LIKE '%AssayDomain-Run%') IS NOT NULL))" - )); + ); // Select all assay runs in the assay-skip containers - SimpleFilter filter = new SimpleFilter( + FilterClause assayRunClause = new AndClause( new InClause(FieldKey.fromParts("Container"), AssaySkipContainers.getContainers()), - assayRun + new SQLClause(assayRunSql) + ); + + // Select assay runs (regardless of their container) that were replaced by assay runs that are being + // excluded + FilterClause replaceByRunIdClause = new SQLClause( + new SQLFragment("ReplacedByRunId IN (SELECT RowId FROM exp.ExperimentRun WHERE ") + .append(assayRunClause.toSQLFragment(null, expSchema.getSqlDialect())) + .append(")") + ); + + // Select assay runs that were replaced by assay runs that are being excluded because they were replaced + // by an excluded assay run. Yes, we actually have to do this... + FilterClause replaceByReplacedRunIdClause = new SQLClause( + new SQLFragment("ReplacedByRunId IN (SELECT RowId FROM exp.ExperimentRun WHERE ") + .append(replaceByRunIdClause.toSQLFragment(null, expSchema.getSqlDialect())) + .append(")") + ); + + // Select all assay runs that need to be excluded + SimpleFilter filter = new SimpleFilter( + new OrClause( + assayRunClause, + replaceByRunIdClause, + replaceByReplacedRunIdClause + ) ); - // Select the assay experiment run RowIds that we don't want to copy. All tables with FKs to ExperimentRun - // (or FKs to other tables with FKs to ExperimentRun) must exclude these run IDs. - _experimentRunExcludeRowIds = new TableSelector(expSchema.getTable("ExperimentRun"), new CsvSet("RowId, ProtocolLSID"), filter, null).getCollection(Integer.class); + // Select the excluded assay experiment run RowIds. All tables with FKs to ExperimentRun (or FKs to + // other tables with FKs to ExperimentRun) must exclude these run IDs. + _experimentRunExcludedRowIds = new TableSelector(expSchema.getTable("ExperimentRun"), new CsvSet("RowId, ProtocolLSID, ReplacedByRunId"), filter, null).getCollection(Integer.class); + LOG.info(" {} being excluded due to the configured AssaySkipContainers parameter", StringUtilsLabKey.pluralize(_experimentRunExcludedRowIds.size(), "assay experiment run is", "assay experiment runs are")); } } - return _experimentRunExcludeRowIds; + return _experimentRunExcludedRowIds; } @Override diff --git a/experiment/src/org/labkey/experiment/ExperimentModule.java b/experiment/src/org/labkey/experiment/ExperimentModule.java index 67ab8161805..8001108dfd3 100644 --- a/experiment/src/org/labkey/experiment/ExperimentModule.java +++ b/experiment/src/org/labkey/experiment/ExperimentModule.java @@ -31,9 +31,14 @@ import org.labkey.api.data.ContainerManager; import org.labkey.api.data.CoreSchema; import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; import org.labkey.api.data.JdbcType; import org.labkey.api.data.NameGenerator; import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.SimpleFilter.FilterClause; +import org.labkey.api.data.SimpleFilter.InClause; +import org.labkey.api.data.SimpleFilter.NotClause; import org.labkey.api.data.SqlSelector; import org.labkey.api.data.TableInfo; import org.labkey.api.data.TableSelector; @@ -74,6 +79,7 @@ import org.labkey.api.files.TableUpdaterFileListener; import org.labkey.api.migration.DatabaseMigrationService; import org.labkey.api.migration.ExperimentDeleteService; +import org.labkey.api.migration.MigrationTableHandler; import org.labkey.api.module.ModuleContext; import org.labkey.api.module.ModuleLoader; import org.labkey.api.module.SpringModule; @@ -82,6 +88,7 @@ import org.labkey.api.ontology.Quantity; import org.labkey.api.ontology.Unit; import org.labkey.api.pipeline.PipelineService; +import org.labkey.api.query.FieldKey; import org.labkey.api.query.FilteredTable; import org.labkey.api.query.QueryService; import org.labkey.api.query.UserSchema; @@ -91,6 +98,7 @@ import org.labkey.api.settings.AppProps; import org.labkey.api.settings.OptionalFeatureService; import org.labkey.api.usageMetrics.UsageMetricsService; +import org.labkey.api.util.GUID; import org.labkey.api.util.JspTestCase; import org.labkey.api.util.PageFlowUtil; import org.labkey.api.util.StringUtilsLabKey; @@ -874,7 +882,42 @@ SELECT COUNT(DISTINCT DD.DomainURI) FROM }); } - DatabaseMigrationService.get().registerSchemaHandler(new ExperimentMigrationSchemaHandler()); + ExperimentMigrationSchemaHandler handler = new ExperimentMigrationSchemaHandler(); + DatabaseMigrationService.get().registerSchemaHandler(handler); + DatabaseMigrationService.get().registerTableHandler(new MigrationTableHandler() + { + @Override + public TableInfo getTableInfo() + { + return DbSchema.get("premium", DbSchemaType.Module).getTable("Exclusions"); + } + + @Override + public void adjustFilter(SimpleFilter filter, Set containers) + { + // Exclude assay experiment runs that weren't copied + FilterClause excludedClause = handler.getExcludedRowIdClause(getTableInfo(), FieldKey.fromParts("RunId")); + if (excludedClause != null) + filter.addClause(excludedClause); + } + }); + DatabaseMigrationService.get().registerTableHandler(new MigrationTableHandler() + { + @Override + public TableInfo getTableInfo() + { + return DbSchema.get("premium", DbSchemaType.Module).getTable("ExclusionMaps"); + } + + @Override + public void adjustFilter(SimpleFilter filter, Set containers) + { + // Exclude assay experiment runs that weren't copied + FilterClause excludedClause = handler.getExcludedRowIdClause(getTableInfo(), FieldKey.fromParts("ExclusionId", "RowId")); + if (excludedClause != null) + filter.addClause(excludedClause); + } + }); DatabaseMigrationService.get().registerSchemaHandler(new SampleTypeMigrationSchemaHandler()); DataClassMigrationSchemaHandler dcHandler = new DataClassMigrationSchemaHandler(); DatabaseMigrationService.get().registerSchemaHandler(dcHandler); From 0a55c0660ccd8561755c7b67dd897a7703527cb4 Mon Sep 17 00:00:00 2001 From: Adam Rauch Date: Tue, 18 Nov 2025 11:37:19 -0800 Subject: [PATCH 7/9] Consistent naming --- .../ExperimentMigrationSchemaHandler.java | 30 +++++++++---------- .../labkey/experiment/ExperimentModule.java | 4 +-- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/experiment/src/org/labkey/experiment/ExperimentMigrationSchemaHandler.java b/experiment/src/org/labkey/experiment/ExperimentMigrationSchemaHandler.java index 665f425f430..75ade1977e5 100644 --- a/experiment/src/org/labkey/experiment/ExperimentMigrationSchemaHandler.java +++ b/experiment/src/org/labkey/experiment/ExperimentMigrationSchemaHandler.java @@ -86,18 +86,18 @@ public FilterClause getTableFilterClause(TableInfo sourceTable, Set contai FilterClause containerClause = getContainerClause(sourceTable, containers); return switch (sourceTable.getName()) { - case "ExperimentRun" -> getExperimentRunExclusionFilter(sourceTable, containerClause, FieldKey.fromParts("RowId"), false); - case "ProtocolApplication" -> getExperimentRunExclusionFilter(sourceTable, containerClause, FieldKey.fromParts("RunId"), false); - case "Data", "Edge" -> getExperimentRunExclusionFilter(sourceTable, containerClause, FieldKey.fromParts("RunId"), true); - case "DataInput", "MaterialInput" -> getExperimentRunExclusionFilter(sourceTable, containerClause, FieldKey.fromParts("TargetApplicationId", "RunId"), false); - case "RunList" -> getExperimentRunExclusionFilter(sourceTable, containerClause, FieldKey.fromParts("ExperimentRunId"), false); - case "DataAncestors" -> getExperimentRunExclusionFilter(sourceTable, containerClause, FieldKey.fromParts("RowId", "RunId"), true); + case "ExperimentRun" -> getExcludedExperimentRunsFilter(sourceTable, containerClause, FieldKey.fromParts("RowId"), false); + case "ProtocolApplication" -> getExcludedExperimentRunsFilter(sourceTable, containerClause, FieldKey.fromParts("RunId"), false); + case "Data", "Edge" -> getExcludedExperimentRunsFilter(sourceTable, containerClause, FieldKey.fromParts("RunId"), true); + case "DataInput", "MaterialInput" -> getExcludedExperimentRunsFilter(sourceTable, containerClause, FieldKey.fromParts("TargetApplicationId", "RunId"), false); + case "RunList" -> getExcludedExperimentRunsFilter(sourceTable, containerClause, FieldKey.fromParts("ExperimentRunId"), false); + case "DataAncestors" -> getExcludedExperimentRunsFilter(sourceTable, containerClause, FieldKey.fromParts("RowId", "RunId"), true); default -> containerClause; }; } // Combine the full container clause with the assay experiment run exclusion filter, if present - private FilterClause getExperimentRunExclusionFilter(TableInfo sourceTable, FilterClause containerClause, FieldKey runIdFieldKey, boolean nullable) + private FilterClause getExcludedExperimentRunsFilter(TableInfo sourceTable, FilterClause containerClause, FieldKey runIdFieldKey, boolean nullable) { FilterClause excludedRowIdClause = getExcludedRowIdClause(sourceTable, runIdFieldKey); @@ -116,22 +116,22 @@ private FilterClause getExperimentRunExclusionFilter(TableInfo sourceTable, Filt @Nullable FilterClause getExcludedRowIdClause(TableInfo sourceTable, FieldKey runIdFieldKey) { - Collection experimentRunsToExclude = getExperimentRunExcludedRowIds(sourceTable.getSchema()); + Collection experimentRunsToExclude = getExcludedExperimentRunRowIds(sourceTable.getSchema()); return experimentRunsToExclude.isEmpty() ? null : new NotClause(new InClause(runIdFieldKey, experimentRunsToExclude, getTempTableInClauseGenerator(sourceTable.getSchema().getScope()))); } - private Collection _experimentRunExcludedRowIds = null; + private Collection _excludedExperimentRunRowIds = null; - private @NotNull Collection getExperimentRunExcludedRowIds(DbSchema schema) + private @NotNull Collection getExcludedExperimentRunRowIds(DbSchema schema) { - if (null == _experimentRunExcludedRowIds) + if (null == _excludedExperimentRunRowIds) { if (AssaySkipContainers.getContainers().isEmpty()) { - _experimentRunExcludedRowIds = Collections.emptyList(); + _excludedExperimentRunRowIds = Collections.emptyList(); } else { @@ -178,12 +178,12 @@ private FilterClause getExperimentRunExclusionFilter(TableInfo sourceTable, Filt // Select the excluded assay experiment run RowIds. All tables with FKs to ExperimentRun (or FKs to // other tables with FKs to ExperimentRun) must exclude these run IDs. - _experimentRunExcludedRowIds = new TableSelector(expSchema.getTable("ExperimentRun"), new CsvSet("RowId, ProtocolLSID, ReplacedByRunId"), filter, null).getCollection(Integer.class); - LOG.info(" {} being excluded due to the configured AssaySkipContainers parameter", StringUtilsLabKey.pluralize(_experimentRunExcludedRowIds.size(), "assay experiment run is", "assay experiment runs are")); + _excludedExperimentRunRowIds = new TableSelector(expSchema.getTable("ExperimentRun"), new CsvSet("RowId, ProtocolLSID, ReplacedByRunId"), filter, null).getCollection(Integer.class); + LOG.info(" {} being excluded due to the configured AssaySkipContainers parameter", StringUtilsLabKey.pluralize(_excludedExperimentRunRowIds.size(), "assay experiment run is", "assay experiment runs are")); } } - return _experimentRunExcludedRowIds; + return _excludedExperimentRunRowIds; } @Override diff --git a/experiment/src/org/labkey/experiment/ExperimentModule.java b/experiment/src/org/labkey/experiment/ExperimentModule.java index 8001108dfd3..9d95cfd5538 100644 --- a/experiment/src/org/labkey/experiment/ExperimentModule.java +++ b/experiment/src/org/labkey/experiment/ExperimentModule.java @@ -37,8 +37,6 @@ import org.labkey.api.data.SQLFragment; import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.SimpleFilter.FilterClause; -import org.labkey.api.data.SimpleFilter.InClause; -import org.labkey.api.data.SimpleFilter.NotClause; import org.labkey.api.data.SqlSelector; import org.labkey.api.data.TableInfo; import org.labkey.api.data.TableSelector; @@ -913,7 +911,7 @@ public TableInfo getTableInfo() public void adjustFilter(SimpleFilter filter, Set containers) { // Exclude assay experiment runs that weren't copied - FilterClause excludedClause = handler.getExcludedRowIdClause(getTableInfo(), FieldKey.fromParts("ExclusionId", "RowId")); + FilterClause excludedClause = handler.getExcludedRowIdClause(getTableInfo(), FieldKey.fromParts("ExclusionId", "RunId")); if (excludedClause != null) filter.addClause(excludedClause); } From 4fd97ab0708d91a26f38fe7236088c12fab565b4 Mon Sep 17 00:00:00 2001 From: Adam Rauch Date: Tue, 18 Nov 2025 16:16:26 -0800 Subject: [PATCH 8/9] Pass source table to table handlers. Fix assay run filters. --- .../api/migration/MigrationTableHandler.java | 2 +- .../labkey/experiment/ExperimentModule.java | 25 ++++++++++++++++--- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/api/src/org/labkey/api/migration/MigrationTableHandler.java b/api/src/org/labkey/api/migration/MigrationTableHandler.java index a16ad7a114a..37ae71f7206 100644 --- a/api/src/org/labkey/api/migration/MigrationTableHandler.java +++ b/api/src/org/labkey/api/migration/MigrationTableHandler.java @@ -14,5 +14,5 @@ public interface MigrationTableHandler { TableInfo getTableInfo(); - void adjustFilter(SimpleFilter filter, Set containers); + void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers); } diff --git a/experiment/src/org/labkey/experiment/ExperimentModule.java b/experiment/src/org/labkey/experiment/ExperimentModule.java index 9d95cfd5538..b420a972ec4 100644 --- a/experiment/src/org/labkey/experiment/ExperimentModule.java +++ b/experiment/src/org/labkey/experiment/ExperimentModule.java @@ -891,10 +891,10 @@ public TableInfo getTableInfo() } @Override - public void adjustFilter(SimpleFilter filter, Set containers) + public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) { // Exclude assay experiment runs that weren't copied - FilterClause excludedClause = handler.getExcludedRowIdClause(getTableInfo(), FieldKey.fromParts("RunId")); + FilterClause excludedClause = handler.getExcludedRowIdClause(sourceTable, FieldKey.fromParts("RunId")); if (excludedClause != null) filter.addClause(excludedClause); } @@ -908,10 +908,27 @@ public TableInfo getTableInfo() } @Override - public void adjustFilter(SimpleFilter filter, Set containers) + public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) { // Exclude assay experiment runs that weren't copied - FilterClause excludedClause = handler.getExcludedRowIdClause(getTableInfo(), FieldKey.fromParts("ExclusionId", "RunId")); + FilterClause excludedClause = handler.getExcludedRowIdClause(sourceTable, FieldKey.fromParts("ExclusionId", "RunId")); + if (excludedClause != null) + filter.addClause(excludedClause); + } + }); + DatabaseMigrationService.get().registerTableHandler(new MigrationTableHandler() + { + @Override + public TableInfo getTableInfo() + { + return DbSchema.get("assayrequest", DbSchemaType.Module).getTable("RequestRunsJunction"); + } + + @Override + public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) + { + // Exclude assay experiment runs that weren't copied + FilterClause excludedClause = handler.getExcludedRowIdClause(sourceTable, FieldKey.fromParts("RunId")); if (excludedClause != null) filter.addClause(excludedClause); } From d994934843bebd757bee128e3e93ceefe78ffade Mon Sep 17 00:00:00 2001 From: Adam Rauch Date: Tue, 18 Nov 2025 23:32:39 -0800 Subject: [PATCH 9/9] Tolerate module schemas when the module isn't present --- .../src/org/labkey/experiment/ExperimentModule.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/experiment/src/org/labkey/experiment/ExperimentModule.java b/experiment/src/org/labkey/experiment/ExperimentModule.java index b420a972ec4..592f80e9d7b 100644 --- a/experiment/src/org/labkey/experiment/ExperimentModule.java +++ b/experiment/src/org/labkey/experiment/ExperimentModule.java @@ -16,7 +16,6 @@ package org.labkey.experiment; import org.apache.commons.lang3.math.NumberUtils; -import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.labkey.api.admin.FolderSerializationRegistry; @@ -101,7 +100,6 @@ import org.labkey.api.util.PageFlowUtil; import org.labkey.api.util.StringUtilsLabKey; import org.labkey.api.util.SystemMaintenance; -import org.labkey.api.util.logging.LogHelper; import org.labkey.api.view.AlwaysAvailableWebPartFactory; import org.labkey.api.view.BaseWebPartFactory; import org.labkey.api.view.HttpView; @@ -185,7 +183,6 @@ public class ExperimentModule extends SpringModule { - private static final Logger LOG = LogHelper.getLogger(ExperimentModule.class, "Database migration status"); private static final String SAMPLE_TYPE_WEB_PART_NAME = "Sample Types"; private static final String PROTOCOL_WEB_PART_NAME = "Protocols"; @@ -887,7 +884,7 @@ SELECT COUNT(DISTINCT DD.DomainURI) FROM @Override public TableInfo getTableInfo() { - return DbSchema.get("premium", DbSchemaType.Module).getTable("Exclusions"); + return DbSchema.get("premium", DbSchemaType.Bare).getTable("Exclusions"); } @Override @@ -904,7 +901,7 @@ public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set c @Override public TableInfo getTableInfo() { - return DbSchema.get("premium", DbSchemaType.Module).getTable("ExclusionMaps"); + return DbSchema.get("premium", DbSchemaType.Bare).getTable("ExclusionMaps"); } @Override @@ -921,7 +918,7 @@ public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set c @Override public TableInfo getTableInfo() { - return DbSchema.get("assayrequest", DbSchemaType.Module).getTable("RequestRunsJunction"); + return DbSchema.get("assayrequest", DbSchemaType.Bare).getTable("RequestRunsJunction"); } @Override