diff --git a/api/src/org/labkey/api/data/SimpleFilter.java b/api/src/org/labkey/api/data/SimpleFilter.java index e114942285d..8dff53f20ff 100644 --- a/api/src/org/labkey/api/data/SimpleFilter.java +++ b/api/src/org/labkey/api/data/SimpleFilter.java @@ -664,11 +664,19 @@ public static DatabaseIdentifier getAliasForColumnFilter(SqlDialect dialect, Col public static class InClause extends MultiValuedFilterClause { + private InClauseGenerator _tempTableGenerator = null; + public InClause(FieldKey fieldKey, Collection params) { this(fieldKey, params, false, false); } + public InClause(FieldKey fieldKey, Collection params, InClauseGenerator tempTableGenerator) + { + this(fieldKey, params, false, false); + _tempTableGenerator = tempTableGenerator; + } + public InClause(FieldKey fieldKey, Collection params, boolean urlClause) { this(fieldKey, params, urlClause, false); @@ -837,7 +845,10 @@ public SQLFragment toSQLFragment(Map columnMap, in.appendIdentifier(alias); // Dialect may want to generate database-specific SQL, especially for very large IN clauses - dialect.appendInClauseSql(in, convertedParams); + if (null == _tempTableGenerator) + dialect.appendInClauseSql(in, convertedParams); + else + dialect.appendInClauseSqlWithCustomInClauseGenerator(in, convertedParams, _tempTableGenerator); if (isIncludeNull()) { diff --git a/api/src/org/labkey/api/data/dialect/BasePostgreSqlDialect.java b/api/src/org/labkey/api/data/dialect/BasePostgreSqlDialect.java index 5cba2821b96..ecdb80e1507 100644 --- a/api/src/org/labkey/api/data/dialect/BasePostgreSqlDialect.java +++ b/api/src/org/labkey/api/data/dialect/BasePostgreSqlDialect.java @@ -292,11 +292,11 @@ public String addReselect(SQLFragment sql, ColumnInfo column, @Nullable String p @Override public SQLFragment appendInClauseSql(SQLFragment sql, @NotNull Collection params) { - return appendInClauseSql(sql, params, _tempTableInClauseGenerator); + return appendInClauseSqlWithCustomInClauseGenerator(sql, params, _tempTableInClauseGenerator); } @Override - public SQLFragment appendInClauseSql(SQLFragment sql, @NotNull Collection params, InClauseGenerator tempTableGenerator) + public SQLFragment appendInClauseSqlWithCustomInClauseGenerator(SQLFragment sql, @NotNull Collection params, InClauseGenerator tempTableGenerator) { if (params.size() >= TEMPTABLE_GENERATOR_MINSIZE) { diff --git a/api/src/org/labkey/api/data/dialect/SqlDialect.java b/api/src/org/labkey/api/data/dialect/SqlDialect.java index 5c19524f10c..b11a782b34a 100644 --- a/api/src/org/labkey/api/data/dialect/SqlDialect.java +++ b/api/src/org/labkey/api/data/dialect/SqlDialect.java @@ -530,11 +530,11 @@ protected Set getJdbcKeywords(SqlExecutor executor) throws SQLException, // Most callers should use this method public SQLFragment appendInClauseSql(SQLFragment sql, @NotNull Collection params) { - return appendInClauseSql(sql, params, null); + return appendInClauseSqlWithCustomInClauseGenerator(sql, params, null); } - // Use in cases where the default temp schema won't do, e.g., you need to apply a large IN clause in an external data source - public SQLFragment appendInClauseSql(SQLFragment sql, @NotNull Collection params, InClauseGenerator tempTableGenerator) + // Use only in cases where the default temp-table generator won't do, e.g., you need to apply a large IN clause in an external data source + public SQLFragment appendInClauseSqlWithCustomInClauseGenerator(SQLFragment sql, @NotNull Collection params, InClauseGenerator tempTableGenerator) { return DEFAULT_GENERATOR.appendInClauseSql(sql, params); } diff --git a/api/src/org/labkey/api/migration/DatabaseMigrationConfiguration.java b/api/src/org/labkey/api/migration/DatabaseMigrationConfiguration.java index 9d0a549cb12..49cd45b4ff5 100644 --- a/api/src/org/labkey/api/migration/DatabaseMigrationConfiguration.java +++ b/api/src/org/labkey/api/migration/DatabaseMigrationConfiguration.java @@ -19,7 +19,7 @@ default void beforeMigration(){} DbScope getTargetScope(); @NotNull Set getSkipSchemas(); Predicate getColumnNameFilter(); - @Nullable TableSelector getTableSelector(DbSchemaType schemaType, TableInfo sourceTable, TableInfo targetTable, Set selectColumnNames, MigrationSchemaHandler schemaHandler); + @Nullable TableSelector getTableSelector(DbSchemaType schemaType, TableInfo sourceTable, TableInfo targetTable, Set selectColumnNames, MigrationSchemaHandler schemaHandler, @Nullable MigrationTableHandler tableHandler); default void copyAttachments(DbSchema sourceSchema, DbSchema targetSchema, MigrationSchemaHandler schemaHandler){} default void afterMigration(){} } diff --git a/api/src/org/labkey/api/migration/DatabaseMigrationService.java b/api/src/org/labkey/api/migration/DatabaseMigrationService.java index ec2c0760a7c..b5ec00fac29 100644 --- a/api/src/org/labkey/api/migration/DatabaseMigrationService.java +++ b/api/src/org/labkey/api/migration/DatabaseMigrationService.java @@ -48,6 +48,7 @@ default void migrate(DatabaseMigrationConfiguration configuration) // By default, no-op implementations default void registerSchemaHandler(MigrationSchemaHandler schemaHandler) {} + default void registerTableHandler(MigrationTableHandler tableHandler) {} default void registerMigrationFilter(MigrationFilter filter) {} default @Nullable MigrationFilter getMigrationFilter(String propertyName) diff --git a/api/src/org/labkey/api/migration/DefaultDatabaseMigrationConfiguration.java b/api/src/org/labkey/api/migration/DefaultDatabaseMigrationConfiguration.java index 7dea199479d..8b5017a0edf 100644 --- a/api/src/org/labkey/api/migration/DefaultDatabaseMigrationConfiguration.java +++ b/api/src/org/labkey/api/migration/DefaultDatabaseMigrationConfiguration.java @@ -43,7 +43,7 @@ public Predicate getColumnNameFilter() } @Override - public TableSelector getTableSelector(DbSchemaType schemaType, TableInfo sourceTable, TableInfo targetTable, Set selectColumnNames, MigrationSchemaHandler schemaHandler) + public TableSelector getTableSelector(DbSchemaType schemaType, TableInfo sourceTable, TableInfo targetTable, Set selectColumnNames, MigrationSchemaHandler schemaHandler, @Nullable MigrationTableHandler tableHandler) { return null; } diff --git a/api/src/org/labkey/api/migration/DefaultMigrationSchemaHandler.java b/api/src/org/labkey/api/migration/DefaultMigrationSchemaHandler.java index 08f4fcd3c2c..e54fe4953a0 100644 --- a/api/src/org/labkey/api/migration/DefaultMigrationSchemaHandler.java +++ b/api/src/org/labkey/api/migration/DefaultMigrationSchemaHandler.java @@ -1,5 +1,6 @@ package org.labkey.api.migration; +import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.labkey.api.attachments.AttachmentService; @@ -27,8 +28,11 @@ import org.labkey.api.query.FieldKey; import org.labkey.api.query.SchemaKey; import org.labkey.api.query.TableSorter; +import org.labkey.api.util.ConfigurationException; import org.labkey.api.util.GUID; +import org.labkey.api.util.JobRunner; import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.logging.LogHelper; import java.util.ArrayList; import java.util.Arrays; @@ -38,10 +42,13 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; public class DefaultMigrationSchemaHandler implements MigrationSchemaHandler { + private static final Logger LOG = LogHelper.getLogger(DefaultMigrationSchemaHandler.class, "Migration shutdown status"); + private final DbSchema _schema; public DefaultMigrationSchemaHandler(DbSchema schema) @@ -77,7 +84,7 @@ public List getTablesToCopy() if (!allTables.isEmpty()) { - DatabaseMigrationService.LOG.info("These tables were removed by TableSorter: {}", allTables); + LOG.info("These tables were removed by TableSorter: {}", allTables); } return sortedTables.stream() @@ -250,12 +257,13 @@ public void copyAttachments(DatabaseMigrationConfiguration configuration, DbSche Collection entityIds = new SqlSelector(targetSchema, sql).getCollection(String.class); SQLFragment selectParents = new SQLFragment("Parent"); // This query against the source database is likely to contain a large IN clause, so use an alternative InClauseGenerator - sourceSchema.getSqlDialect().appendInClauseSql(selectParents, entityIds, getTempTableInClauseGenerator(sourceSchema.getScope())); + sourceSchema.getSqlDialect().appendInClauseSqlWithCustomInClauseGenerator(selectParents, entityIds, getTempTableInClauseGenerator(sourceSchema.getScope())); copyAttachments(configuration, sourceSchema, new SQLClause(selectParents), type); } - - // TODO: fail if type.getSelectParentEntityIdsSql() returns null? - // TODO: throw if some registered AttachmentType is not seen + else + { + throw new ConfigurationException("AttachmentType \"" + type.getUniqueName() + "\" is not configured to find parent EntityIds!"); + } }); } @@ -267,33 +275,46 @@ protected InClauseGenerator getTempTableInClauseGenerator(DbScope sourceScope) } private static final Set SEEN = new HashSet<>(); + private static final JobRunner ATTACHMENT_JOB_RUNNER = new JobRunner("Attachment JobRunner", 1); // Copy all core.Documents rows that match the provided filter clause - protected void copyAttachments(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, FilterClause filterClause, AttachmentType... type) + protected final void copyAttachments(DatabaseMigrationConfiguration configuration, DbSchema sourceSchema, FilterClause filterClause, AttachmentType... type) { SEEN.addAll(Arrays.asList(type)); String additionalMessage = " associated with " + Arrays.stream(type).map(t -> t.getClass().getSimpleName()).collect(Collectors.joining(", ")); TableInfo sourceDocumentsTable = sourceSchema.getScope().getSchema("core", DbSchemaType.Migration).getTable("Documents"); TableInfo targetDocumentsTable = CoreSchema.getInstance().getTableInfoDocuments(); - DatabaseMigrationService.get().copySourceTableToTargetTable(configuration, sourceDocumentsTable, targetDocumentsTable, DbSchemaType.Module, false, additionalMessage, new DefaultMigrationSchemaHandler(CoreSchema.getInstance().getSchema()) + + // Queue up the core.Documents transfers and let them run in the background + ATTACHMENT_JOB_RUNNER.execute(() -> DatabaseMigrationService.get().copySourceTableToTargetTable(configuration, sourceDocumentsTable, targetDocumentsTable, DbSchemaType.Module, false, additionalMessage, new DefaultMigrationSchemaHandler(CoreSchema.getInstance().getSchema()) { @Override public FilterClause getTableFilterClause(TableInfo sourceTable, Set containers) { return filterClause; } - }); + })); } - public static void logUnseenAttachmentTypes() + // Global (not schema- or configuration-specific) cleanup + public static void afterMigration() throws InterruptedException { + // Report any unseen attachment types Set unseen = new HashSet<>(AttachmentService.get().getAttachmentTypes()); unseen.removeAll(SEEN); if (unseen.isEmpty()) - DatabaseMigrationService.LOG.info("All AttachmentTypes have been seen"); + LOG.info("All AttachmentTypes have been seen"); + else + throw new ConfigurationException("These AttachmentTypes have not been seen: " + unseen.stream().map(type -> type.getClass().getSimpleName()).collect(Collectors.joining(", "))); + + // Shut down the attachment JobRunner + LOG.info("Waiting for core.Documents background transfer to complete"); + ATTACHMENT_JOB_RUNNER.shutdown(); + if (ATTACHMENT_JOB_RUNNER.awaitTermination(1, TimeUnit.HOURS)) + LOG.info("core.Documents background transfer is complete"); else - DatabaseMigrationService.LOG.info("These AttachmentTypes have not been seen: {}", unseen.stream().map(type -> type.getClass().getSimpleName()).collect(Collectors.joining(", "))); + LOG.error("core.Documents background transfer did not complete after one hour! Giving up."); } @Override diff --git a/api/src/org/labkey/api/migration/MigrationTableHandler.java b/api/src/org/labkey/api/migration/MigrationTableHandler.java new file mode 100644 index 00000000000..37ae71f7206 --- /dev/null +++ b/api/src/org/labkey/api/migration/MigrationTableHandler.java @@ -0,0 +1,18 @@ +package org.labkey.api.migration; + +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.TableInfo; +import org.labkey.api.util.GUID; + +import java.util.Set; + +/** + * Rarely needed, this interface lets a module filter the rows of another module's table. The specific use case: LabBook + * needs to filter the compliance.SignedSnapshots table of snapshots associated with Notebooks that are excluded by a + * NotebookFilter. + */ +public interface MigrationTableHandler +{ + TableInfo getTableInfo(); + void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers); +} diff --git a/api/src/org/labkey/api/util/JobRunner.java b/api/src/org/labkey/api/util/JobRunner.java index a0d0b59ca00..9b1998dbc58 100644 --- a/api/src/org/labkey/api/util/JobRunner.java +++ b/api/src/org/labkey/api/util/JobRunner.java @@ -18,9 +18,11 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; import org.labkey.api.data.DbScope; import java.util.HashMap; +import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; import java.util.concurrent.Future; @@ -51,7 +53,7 @@ public class JobRunner implements Executor private static final JobRunner _defaultJobRunner = new JobRunner("Default", 1); private final ScheduledThreadPoolExecutor _executor; - private final HashMap _jobs = new HashMap<>(); + private final Map, Job> _jobs = new HashMap<>(); public JobRunner(String name, int max) @@ -77,11 +79,6 @@ public void shutdownPre() { _executor.shutdown(); } - - @Override - public void shutdownStarted() - { - } }); } @@ -111,11 +108,16 @@ public void shutdown() _executor.shutdown(); } + public boolean awaitTermination(long timeout, @NotNull TimeUnit unit) throws InterruptedException + { + return _executor.awaitTermination(timeout, unit); + } + /** * This will schedule the runnable to execute immediately, with no delay */ @Override - public void execute(Runnable command) + public void execute(@NotNull Runnable command) { execute(command, 0); } @@ -132,7 +134,7 @@ public void execute(Runnable command, long delay) { synchronized (_jobs) { - Future task = _executor.schedule(command, delay, TimeUnit.MILLISECONDS); + Future task = _executor.schedule(command, delay, TimeUnit.MILLISECONDS); if (command instanceof Job job) { job._task = task; @@ -141,7 +143,7 @@ public void execute(Runnable command, long delay) } } - public Future submit(Runnable run) + public Future submit(Runnable run) { if (run instanceof Job) { @@ -221,13 +223,13 @@ protected void afterExecute(Runnable r, Throwable t) } else { - if (r instanceof Future) + if (r instanceof Future f) { if (null == t) { try { - ((Future)r).get(); + f.get(); } catch (ExecutionException x) { @@ -277,7 +279,7 @@ static class JobThreadFactory implements ThreadFactory } @Override - public Thread newThread(Runnable r) + public Thread newThread(@NotNull Runnable r) { Thread t = new Thread(group, r, namePrefix + threadNumber.getAndIncrement(), 0); if (t.isDaemon()) diff --git a/experiment/src/org/labkey/experiment/ExperimentMigrationSchemaHandler.java b/experiment/src/org/labkey/experiment/ExperimentMigrationSchemaHandler.java index db5b16f572d..75ade1977e5 100644 --- a/experiment/src/org/labkey/experiment/ExperimentMigrationSchemaHandler.java +++ b/experiment/src/org/labkey/experiment/ExperimentMigrationSchemaHandler.java @@ -4,34 +4,42 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.labkey.api.attachments.AttachmentType; +import org.labkey.api.collections.CsvSet; import org.labkey.api.data.CompareType; import org.labkey.api.data.CompareType.CompareClause; import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.SimpleFilter.AndClause; import org.labkey.api.data.SimpleFilter.FilterClause; import org.labkey.api.data.SimpleFilter.InClause; +import org.labkey.api.data.SimpleFilter.NotClause; import org.labkey.api.data.SimpleFilter.OrClause; import org.labkey.api.data.SimpleFilter.SQLClause; import org.labkey.api.data.SqlExecutor; import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; import org.labkey.api.exp.OntologyManager; import org.labkey.api.exp.api.ExpProtocolAttachmentType; import org.labkey.api.exp.api.ExpRunAttachmentType; +import org.labkey.api.migration.AssaySkipContainers; import org.labkey.api.migration.DatabaseMigrationConfiguration; import org.labkey.api.migration.DefaultMigrationSchemaHandler; import org.labkey.api.query.FieldKey; import org.labkey.api.util.GUID; +import org.labkey.api.util.StringUtilsLabKey; import org.labkey.api.util.logging.LogHelper; import org.labkey.experiment.api.ExperimentServiceImpl; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Set; class ExperimentMigrationSchemaHandler extends DefaultMigrationSchemaHandler { - private static final Logger LOG = LogHelper.getLogger(ExperimentMigrationSchemaHandler.class, "Progress of exp deletes during database migration"); + private static final Logger LOG = LogHelper.getLogger(ExperimentMigrationSchemaHandler.class, "Progress of database migration"); public ExperimentMigrationSchemaHandler() { @@ -65,27 +73,124 @@ public void beforeSchema() @Override public List getTablesToCopy() { - // No need to populate the MaterialIndexed or DataIndexed tables -- new server should be completely re-indexed after migration + // No need to populate the MaterialIndexed or DataIndexed tables -- new server will be completely re-indexed after migration List tables = super.getTablesToCopy(); tables.remove(ExperimentServiceImpl.get().getTinfoMaterialIndexed()); tables.remove(ExperimentServiceImpl.get().getTinfoDataIndexed()); return tables; } + @Override + public FilterClause getTableFilterClause(TableInfo sourceTable, Set containers) + { + FilterClause containerClause = getContainerClause(sourceTable, containers); + return switch (sourceTable.getName()) + { + case "ExperimentRun" -> getExcludedExperimentRunsFilter(sourceTable, containerClause, FieldKey.fromParts("RowId"), false); + case "ProtocolApplication" -> getExcludedExperimentRunsFilter(sourceTable, containerClause, FieldKey.fromParts("RunId"), false); + case "Data", "Edge" -> getExcludedExperimentRunsFilter(sourceTable, containerClause, FieldKey.fromParts("RunId"), true); + case "DataInput", "MaterialInput" -> getExcludedExperimentRunsFilter(sourceTable, containerClause, FieldKey.fromParts("TargetApplicationId", "RunId"), false); + case "RunList" -> getExcludedExperimentRunsFilter(sourceTable, containerClause, FieldKey.fromParts("ExperimentRunId"), false); + case "DataAncestors" -> getExcludedExperimentRunsFilter(sourceTable, containerClause, FieldKey.fromParts("RowId", "RunId"), true); + default -> containerClause; + }; + } + + // Combine the full container clause with the assay experiment run exclusion filter, if present + private FilterClause getExcludedExperimentRunsFilter(TableInfo sourceTable, FilterClause containerClause, FieldKey runIdFieldKey, boolean nullable) + { + FilterClause excludedRowIdClause = getExcludedRowIdClause(sourceTable, runIdFieldKey); + + return excludedRowIdClause == null ? + containerClause : + new AndClause( + containerClause, + nullable ? + new OrClause( + new CompareClause(runIdFieldKey, CompareType.ISBLANK, null), + excludedRowIdClause + ) : + excludedRowIdClause + ); + } + + @Nullable FilterClause getExcludedRowIdClause(TableInfo sourceTable, FieldKey runIdFieldKey) + { + Collection experimentRunsToExclude = getExcludedExperimentRunRowIds(sourceTable.getSchema()); + + return experimentRunsToExclude.isEmpty() ? + null : + new NotClause(new InClause(runIdFieldKey, experimentRunsToExclude, getTempTableInClauseGenerator(sourceTable.getSchema().getScope()))); + } + + private Collection _excludedExperimentRunRowIds = null; + + private @NotNull Collection getExcludedExperimentRunRowIds(DbSchema schema) + { + if (null == _excludedExperimentRunRowIds) + { + if (AssaySkipContainers.getContainers().isEmpty()) + { + _excludedExperimentRunRowIds = Collections.emptyList(); + } + else + { + // We need the source exp schema; if it wasn't passed in, retrieve it from the scope. + DbSchema expSchema = "exp".equals(schema.getName()) ? schema : schema.getScope().getSchema("exp", DbSchemaType.Migration); + + // Select all the assay runs (same filter used by assay.AssayRuns) + SQLFragment assayRunSql = new SQLFragment( + "ProtocolLSID IN (SELECT LSID FROM exp.Protocol x WHERE (ApplicationType = 'ExperimentRun') AND " + + "((SELECT MAX(pd.PropertyId) from exp.Object o, exp.ObjectProperty op, exp.PropertyDescriptor pd WHERE " + + "pd.PropertyId = op.PropertyId and op.ObjectId = o.ObjectId and o.ObjectURI = LSID AND pd.PropertyURI LIKE '%AssayDomain-Run%') IS NOT NULL))" + ); + + // Select all assay runs in the assay-skip containers + FilterClause assayRunClause = new AndClause( + new InClause(FieldKey.fromParts("Container"), AssaySkipContainers.getContainers()), + new SQLClause(assayRunSql) + ); + + // Select assay runs (regardless of their container) that were replaced by assay runs that are being + // excluded + FilterClause replaceByRunIdClause = new SQLClause( + new SQLFragment("ReplacedByRunId IN (SELECT RowId FROM exp.ExperimentRun WHERE ") + .append(assayRunClause.toSQLFragment(null, expSchema.getSqlDialect())) + .append(")") + ); + + // Select assay runs that were replaced by assay runs that are being excluded because they were replaced + // by an excluded assay run. Yes, we actually have to do this... + FilterClause replaceByReplacedRunIdClause = new SQLClause( + new SQLFragment("ReplacedByRunId IN (SELECT RowId FROM exp.ExperimentRun WHERE ") + .append(replaceByRunIdClause.toSQLFragment(null, expSchema.getSqlDialect())) + .append(")") + ); + + // Select all assay runs that need to be excluded + SimpleFilter filter = new SimpleFilter( + new OrClause( + assayRunClause, + replaceByRunIdClause, + replaceByReplacedRunIdClause + ) + ); + + // Select the excluded assay experiment run RowIds. All tables with FKs to ExperimentRun (or FKs to + // other tables with FKs to ExperimentRun) must exclude these run IDs. + _excludedExperimentRunRowIds = new TableSelector(expSchema.getTable("ExperimentRun"), new CsvSet("RowId, ProtocolLSID, ReplacedByRunId"), filter, null).getCollection(Integer.class); + LOG.info(" {} being excluded due to the configured AssaySkipContainers parameter", StringUtilsLabKey.pluralize(_excludedExperimentRunRowIds.size(), "assay experiment run is", "assay experiment runs are")); + } + } + + return _excludedExperimentRunRowIds; + } + @Override public FilterClause getContainerClause(TableInfo sourceTable, Set containers) { -// Set assayFilteredContainers = assayFilteredContainers(containers); return switch (sourceTable.getName()) { -// case "ExperimentRun", "ProtocolApplication" -> super.getContainerClause(sourceTable, assayFilteredContainers); -// case "Data" -> new AndClause( -// new InClause(FieldKey.fromParts("Container"), containers), -// new OrClause( -// new CompareClause(FieldKey.fromParts("RunId"), CompareType.ISBLANK, null), -// new InClause(FieldKey.fromParts("RunId", "Container"), assayFilteredContainers) -// ) -// ); case "DataInput" -> new AndClause( new InClause(FieldKey.fromParts("DataId", "Container"), containers), new InClause(FieldKey.fromParts("TargetApplicationId", "RunId", "Container"), containers) diff --git a/experiment/src/org/labkey/experiment/ExperimentModule.java b/experiment/src/org/labkey/experiment/ExperimentModule.java index 67ab8161805..592f80e9d7b 100644 --- a/experiment/src/org/labkey/experiment/ExperimentModule.java +++ b/experiment/src/org/labkey/experiment/ExperimentModule.java @@ -16,7 +16,6 @@ package org.labkey.experiment; import org.apache.commons.lang3.math.NumberUtils; -import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.labkey.api.admin.FolderSerializationRegistry; @@ -31,9 +30,12 @@ import org.labkey.api.data.ContainerManager; import org.labkey.api.data.CoreSchema; import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; import org.labkey.api.data.JdbcType; import org.labkey.api.data.NameGenerator; import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.SimpleFilter.FilterClause; import org.labkey.api.data.SqlSelector; import org.labkey.api.data.TableInfo; import org.labkey.api.data.TableSelector; @@ -74,6 +76,7 @@ import org.labkey.api.files.TableUpdaterFileListener; import org.labkey.api.migration.DatabaseMigrationService; import org.labkey.api.migration.ExperimentDeleteService; +import org.labkey.api.migration.MigrationTableHandler; import org.labkey.api.module.ModuleContext; import org.labkey.api.module.ModuleLoader; import org.labkey.api.module.SpringModule; @@ -82,6 +85,7 @@ import org.labkey.api.ontology.Quantity; import org.labkey.api.ontology.Unit; import org.labkey.api.pipeline.PipelineService; +import org.labkey.api.query.FieldKey; import org.labkey.api.query.FilteredTable; import org.labkey.api.query.QueryService; import org.labkey.api.query.UserSchema; @@ -91,11 +95,11 @@ import org.labkey.api.settings.AppProps; import org.labkey.api.settings.OptionalFeatureService; import org.labkey.api.usageMetrics.UsageMetricsService; +import org.labkey.api.util.GUID; import org.labkey.api.util.JspTestCase; import org.labkey.api.util.PageFlowUtil; import org.labkey.api.util.StringUtilsLabKey; import org.labkey.api.util.SystemMaintenance; -import org.labkey.api.util.logging.LogHelper; import org.labkey.api.view.AlwaysAvailableWebPartFactory; import org.labkey.api.view.BaseWebPartFactory; import org.labkey.api.view.HttpView; @@ -179,7 +183,6 @@ public class ExperimentModule extends SpringModule { - private static final Logger LOG = LogHelper.getLogger(ExperimentModule.class, "Database migration status"); private static final String SAMPLE_TYPE_WEB_PART_NAME = "Sample Types"; private static final String PROTOCOL_WEB_PART_NAME = "Protocols"; @@ -874,7 +877,59 @@ SELECT COUNT(DISTINCT DD.DomainURI) FROM }); } - DatabaseMigrationService.get().registerSchemaHandler(new ExperimentMigrationSchemaHandler()); + ExperimentMigrationSchemaHandler handler = new ExperimentMigrationSchemaHandler(); + DatabaseMigrationService.get().registerSchemaHandler(handler); + DatabaseMigrationService.get().registerTableHandler(new MigrationTableHandler() + { + @Override + public TableInfo getTableInfo() + { + return DbSchema.get("premium", DbSchemaType.Bare).getTable("Exclusions"); + } + + @Override + public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) + { + // Exclude assay experiment runs that weren't copied + FilterClause excludedClause = handler.getExcludedRowIdClause(sourceTable, FieldKey.fromParts("RunId")); + if (excludedClause != null) + filter.addClause(excludedClause); + } + }); + DatabaseMigrationService.get().registerTableHandler(new MigrationTableHandler() + { + @Override + public TableInfo getTableInfo() + { + return DbSchema.get("premium", DbSchemaType.Bare).getTable("ExclusionMaps"); + } + + @Override + public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) + { + // Exclude assay experiment runs that weren't copied + FilterClause excludedClause = handler.getExcludedRowIdClause(sourceTable, FieldKey.fromParts("ExclusionId", "RunId")); + if (excludedClause != null) + filter.addClause(excludedClause); + } + }); + DatabaseMigrationService.get().registerTableHandler(new MigrationTableHandler() + { + @Override + public TableInfo getTableInfo() + { + return DbSchema.get("assayrequest", DbSchemaType.Bare).getTable("RequestRunsJunction"); + } + + @Override + public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) + { + // Exclude assay experiment runs that weren't copied + FilterClause excludedClause = handler.getExcludedRowIdClause(sourceTable, FieldKey.fromParts("RunId")); + if (excludedClause != null) + filter.addClause(excludedClause); + } + }); DatabaseMigrationService.get().registerSchemaHandler(new SampleTypeMigrationSchemaHandler()); DataClassMigrationSchemaHandler dcHandler = new DataClassMigrationSchemaHandler(); DatabaseMigrationService.get().registerSchemaHandler(dcHandler); diff --git a/search/src/org/labkey/search/SearchModule.java b/search/src/org/labkey/search/SearchModule.java index f4f9a3a7603..d4cd37bca6e 100644 --- a/search/src/org/labkey/search/SearchModule.java +++ b/search/src/org/labkey/search/SearchModule.java @@ -193,7 +193,7 @@ public void handle(Map properties @Override public List getTablesToCopy() { - return List.of(); // Leave empty -- target server needs to index all documents + return List.of(); // Leave empty -- target server will re-index all documents } }); }