diff --git a/src/org/labkey/serverapi/util/UsageReportingLevel.java b/src/org/labkey/serverapi/util/UsageReportingLevel.java new file mode 100644 index 0000000000..075012b8c6 --- /dev/null +++ b/src/org/labkey/serverapi/util/UsageReportingLevel.java @@ -0,0 +1,11 @@ +package org.labkey.serverapi.util; + +/** + * org.labkey.api.util.UsageReportingLevel + */ +public enum UsageReportingLevel +{ + NONE, + ON, + ON_WITHOUT_UPGRADE_MESSAGE +} diff --git a/src/org/labkey/test/pages/admin/ConfigureSystemMaintenancePage.java b/src/org/labkey/test/pages/admin/ConfigureSystemMaintenancePage.java new file mode 100644 index 0000000000..15961fa834 --- /dev/null +++ b/src/org/labkey/test/pages/admin/ConfigureSystemMaintenancePage.java @@ -0,0 +1,47 @@ +package org.labkey.test.pages.admin; + +import org.labkey.test.Locator; +import org.labkey.test.WebDriverWrapper; +import org.labkey.test.WebTestHelper; +import org.labkey.test.pages.LabKeyPage; +import org.labkey.test.pages.pipeline.PipelineStatusDetailsPage; +import org.openqa.selenium.WebDriver; + +public class ConfigureSystemMaintenancePage extends LabKeyPage +{ + public ConfigureSystemMaintenancePage(WebDriver driver) + { + super(driver); + } + + public static ConfigureSystemMaintenancePage beginAt(WebDriverWrapper webDriverWrapper) + { + webDriverWrapper.beginAt(WebTestHelper.buildURL("admin", "configureSystemMaintenance")); + return new ConfigureSystemMaintenancePage(webDriverWrapper.getDriver()); + } + + /** + * Run the specified maintenance task and switch to the window that opens + * @param description task description + */ + public PipelineStatusDetailsPage runMaintenanceTask(String description) + { + click(Locator.tagWithAttribute("input", "type", "checkbox") + .followingSibling("a").withText(description)); + getDriver().switchTo().window("systemMaintenance"); + + PipelineStatusDetailsPage pipelineStatusDetailsPage = new PipelineStatusDetailsPage(getDriver()); + pipelineStatusDetailsPage.waitForComplete(); + return pipelineStatusDetailsPage; + } + + @Override + protected ElementCache newElementCache() + { + return new ElementCache(); + } + + protected class ElementCache extends LabKeyPage.ElementCache + { + } +} diff --git a/src/org/labkey/test/pages/core/admin/ShowAdminPage.java b/src/org/labkey/test/pages/core/admin/ShowAdminPage.java index 16a591b25d..11af29747b 100644 --- a/src/org/labkey/test/pages/core/admin/ShowAdminPage.java +++ b/src/org/labkey/test/pages/core/admin/ShowAdminPage.java @@ -21,6 +21,7 @@ import org.labkey.test.components.DomainDesignerPage; import org.labkey.test.pages.ConfigureReportsAndScriptsPage; import org.labkey.test.pages.LabKeyPage; +import org.labkey.test.pages.admin.ConfigureSystemMaintenancePage; import org.labkey.test.pages.admin.ExternalSourcesPage; import org.labkey.test.pages.compliance.ComplianceSettingsAccountsPage; import org.labkey.test.pages.core.login.LoginConfigurePage; @@ -230,10 +231,11 @@ public void clickSiteWideTerms() clickAndWait(elementCache().siteWideTermsLink); } - public void clickSystemMaintenance() + public ConfigureSystemMaintenancePage clickSystemMaintenance() { goToSettingsSection(); clickAndWait(elementCache().systemMaintenanceLink); + return new ConfigureSystemMaintenancePage(getDriver()); } public void clickSystemProperties() diff --git a/src/org/labkey/test/tests/filecontent/FileContentUploadTest.java b/src/org/labkey/test/tests/filecontent/FileContentUploadTest.java index 1b24e83e25..e15f49c784 100644 --- a/src/org/labkey/test/tests/filecontent/FileContentUploadTest.java +++ b/src/org/labkey/test/tests/filecontent/FileContentUploadTest.java @@ -16,11 +16,13 @@ package org.labkey.test.tests.filecontent; +import org.assertj.core.api.Assertions; import org.hamcrest.CoreMatchers; import org.jetbrains.annotations.NotNull; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.labkey.remoteapi.CommandException; import org.labkey.test.BaseWebDriverTest; import org.labkey.test.Locator; import org.labkey.test.TestFileUtils; @@ -31,6 +33,7 @@ import org.labkey.test.components.domain.DomainFieldRow; import org.labkey.test.components.ext4.ComboBox; import org.labkey.test.components.ext4.Window; +import org.labkey.test.pages.admin.UsageStatisticsPage; import org.labkey.test.pages.files.WebDavPage; import org.labkey.test.params.FieldDefinition; import org.labkey.test.params.FieldDefinition.ColumnType; @@ -46,8 +49,9 @@ import org.labkey.test.util.PortalHelper; import org.labkey.test.util.SearchHelper; import org.labkey.test.util.Timer; +import org.labkey.test.util.core.admin.ServerUsageUtils; import org.labkey.test.util.core.webdav.WebDavUtils; -import org.openqa.selenium.WebElement; +import org.labkey.test.util.data.JSONUtils; import java.io.File; import java.io.IOException; @@ -241,7 +245,7 @@ public void testAbsoluteFilePath() throws Exception _fileBrowserHelper.goToConfigureButtonsTab(); _fileBrowserHelper.unhideGridColumn(FileBrowserHelper.ABSOLUTE_FILE_PATH_COLUMN_ID); click(Ext4Helper.Locators.ext4Button("submit")); - WebElement columnHeader = waitForElement(Locator.byClass("x4-column-header").withText("Absolute File Path").notHidden()); + waitForElement(Locator.byClass("x4-column-header").withText("Absolute File Path").notHidden()); String absolutePath = FileBrowserHelper.Locators.gridRowWithNodeId(filename) .append(Locator.byClass("x4-grid-cell").last()).findElement(getDriver()).getText(); @@ -283,6 +287,7 @@ public void testFolderNameCharacters() Set folders = new HashSet<>(_fileBrowserHelper.getFileList()); assertEquals("Didn't create expected folders", expectedFolders, folders); } + @Test public void testFileNameCharacters() throws IOException { @@ -343,6 +348,35 @@ public void testDrop() assertElementPresent(Locator.tagWithText("span", testFile.getName())); } + @Test + public void testCalculateFileRootSize() throws Exception + { + String calculateFileRootSizeTask = "Calculate file root sizes"; + goToAdminConsole().clickSystemMaintenance().runMaintenanceTask(calculateFileRootSizeTask); + Integer initialFileRootSize = getFileRootSize(); + + goToProjectHome(); + File testFile = TestFileUtils.getSampleData("fileTypes/tsv_sample.tsv"); + + log("Dropping the file object in drop zone"); + _fileBrowserHelper.uploadFile(testFile); + + goToAdminConsole().clickSystemMaintenance().runMaintenanceTask(calculateFileRootSizeTask); + Integer finalFileRootSize = getFileRootSize(); + if (!checker().wrapAssertion(() -> Assertions.assertThat(finalFileRootSize) + .as("Crawled file root size").isGreaterThan(initialFileRootSize))) + { + UsageStatisticsPage.beginAt(this).setJsonPathInput("modules.FileContent"); + checker().screenShotIfNewError("file_root_size"); + } + } + + private @NotNull Integer getFileRootSize() throws IOException, CommandException + { + return JSONUtils.getProperty("fileRootsTotalSize", + ServerUsageUtils.getModuleMetrics(createDefaultConnection(), "FileContent")); + } + @NotNull protected List folderSubstringsToVerify() { @@ -367,7 +401,7 @@ private void setupNotifications() table.checkCheckbox(table.getRowIndex("Email", TEST_USER)); shortWait().until(LabKeyExpectedConditions.elementIsEnabled(Locator.lkButton(MessagesLongTest.USERS_UPDATE_BUTTON))); table.clickHeaderMenu(MessagesLongTest.USERS_UPDATE_BUTTON, false, MessagesLongTest.FILES_MENU_ITEM); - final Window window = Window(getDriver()).withTitle("Update user settings for files").waitFor(); + final Window window = Window(getDriver()).withTitle("Update user settings for files").waitFor(); ComboBox.ComboBox(getDriver()).withLabel(MessagesLongTest.NEW_SETTING_LABEL).find(window).selectComboBoxItem("No Email"); window.clickButton(MessagesLongTest.POPUP_UPDATE_BUTTON, true); table.doAndWaitForUpdate(() -> Window(getDriver()).withTitle("Update selected users").waitFor(). diff --git a/src/org/labkey/test/tests/filecontent/FileRootMigrationTest.java b/src/org/labkey/test/tests/filecontent/FileRootMigrationTest.java index 526ee5383d..8137703ebc 100644 --- a/src/org/labkey/test/tests/filecontent/FileRootMigrationTest.java +++ b/src/org/labkey/test/tests/filecontent/FileRootMigrationTest.java @@ -77,7 +77,7 @@ private void doSetup() public void cleanFiles() throws IOException { FileUtils.deleteDirectory(targetFileRoot); - targetFileRoot.mkdirs(); + FileUtils.forceMkdir(targetFileRoot); FileRootsManagementPage.beginAt(this, getProjectName()) .useDefaultFileRoot() .clickSave(); @@ -101,7 +101,7 @@ public void testMigrateMove() final File folderFile2 = TestFileUtils.getSampleData("fileTypes/cmd_sample.cmd"); List sourceFiles = new ArrayList<>(); - String folderName = "folder \u2603";// + TRICKY_CHARACTERS_FOR_PROJECT_NAMES; + String folderName = "folder " + TRICKY_CHARACTERS_FOR_PROJECT_NAMES; log("Upload files to project"); goToProjectHome(); @@ -165,7 +165,7 @@ public void testMigrateCopy() File folderFile2 = TestFileUtils.getSampleData("fileTypes/cmd_sample.cmd"); List sourceFiles = new ArrayList<>(); - String folderName = "folder \u2603";// + TRICKY_CHARACTERS_FOR_PROJECT_NAMES; + String folderName = "folder " + TRICKY_CHARACTERS_FOR_PROJECT_NAMES; log("Upload files to project"); goToProjectHome(); @@ -238,7 +238,7 @@ public void testMigrateToNonExistentFolder() } @Test - public void testMigratingProjectWithNonInheritingSubfolder() + public void testMigratingProjectWithNonInheritingSubfolder() throws IOException { final File projFile1 = TestFileUtils.getSampleData("fileTypes/sample.txt"); final File projFile2 = TestFileUtils.getSampleData("fileTypes/rtf_sample.rtf"); @@ -247,9 +247,9 @@ public void testMigratingProjectWithNonInheritingSubfolder() List sourceFiles = new ArrayList<>(); File nonInheritingFileRoot = new File(TestFileUtils.getTestTempDir(), "custom"); TestFileUtils.deleteDir(nonInheritingFileRoot); - nonInheritingFileRoot.mkdirs(); + FileUtils.forceMkdir(nonInheritingFileRoot); - String folderName = "folder \u2603";// + TRICKY_CHARACTERS_FOR_PROJECT_NAMES; + String folderName = "folder " + TRICKY_CHARACTERS_FOR_PROJECT_NAMES; log("Upload files to project"); goToProjectHome(); diff --git a/src/org/labkey/test/tests/filecontent/FilesWebpartFileRootTest.java b/src/org/labkey/test/tests/filecontent/FilesWebpartFileRootTest.java index 06cb305823..b8c117f125 100644 --- a/src/org/labkey/test/tests/filecontent/FilesWebpartFileRootTest.java +++ b/src/org/labkey/test/tests/filecontent/FilesWebpartFileRootTest.java @@ -17,7 +17,6 @@ import org.jetbrains.annotations.Nullable; import org.junit.Assert; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -38,7 +37,10 @@ @BaseWebDriverTest.ClassTimeout(minutes = 5) public class FilesWebpartFileRootTest extends BaseWebDriverTest { - private static final String CHILD_CONTAINER = "ChildContainerNotForFileRootSelection"; + private static final String PROJECT_NAME = "FilesWebpartFileRoot Project" + TRICKY_CHARACTERS_FOR_PROJECT_NAMES; + private static final String WORK_FOLDER = "Work Folder" + TRICKY_CHARACTERS_FOR_PROJECT_NAMES; + private static final String WORK_CONTAINER_PATH = PROJECT_NAME + "/" + WORK_FOLDER; + private static final String GRANDCHILD_CONTAINER = "ChildContainerNotForFileRootSelection"; PortalHelper portalHelper = new PortalHelper(this); FileBrowserHelper fileBrowserHelper = new FileBrowserHelper(this); @@ -51,7 +53,7 @@ public List getAssociatedModules() @Override protected @Nullable String getProjectName() { - return "FilesWebpartFileRoot Project" + TRICKY_CHARACTERS_FOR_PROJECT_NAMES; + return PROJECT_NAME; } @BeforeClass @@ -64,16 +66,16 @@ public static void initTest() private void doInit() { _containerHelper.createProject(getProjectName(), null); - _containerHelper.createSubfolder(getProjectName(), CHILD_CONTAINER); + _containerHelper.createSubfolder(getProjectName(), WORK_FOLDER); + _containerHelper.createSubfolder(WORK_CONTAINER_PATH, GRANDCHILD_CONTAINER); - goToProjectHome(); + goToWorkFolder(); portalHelper.addWebPart("Files"); } - @Before - public void preTest() + private void goToWorkFolder() { - goToProjectHome(); + goToProjectHome(WORK_CONTAINER_PATH); } @Test @@ -82,6 +84,8 @@ public void testCustomFileRoot() String folderName = "Folder " + TRICKY_CHARACTERS_FOR_PROJECT_NAMES; File testFile = TestFileUtils.getSampleData("fileTypes/sample.txt"); + goToWorkFolder(); + _fileBrowserHelper.createFolder(folderName); _fileBrowserHelper.selectFileBrowserItem(folderName + "/"); _fileBrowserHelper.uploadFile(testFile); @@ -92,17 +96,17 @@ public void testCustomFileRoot() customizePage.setFileRoot("@files", folderName); Assert.assertEquals("File in custom file root", List.of(testFile.getName()), _fileBrowserHelper.getFileList()); - goToProjectHome(); + goToWorkFolder(); portalHelper.clickWebpartMenuItem("Files", true, "Customize"); - customizePage.verifyFileRootNodeNotPresent(CHILD_CONTAINER); //child container shouldn't show up as file root options + customizePage.verifyFileRootNodeNotPresent(GRANDCHILD_CONTAINER); //child container shouldn't show up as file root options customizePage.setFileRoot("@files"); Locator.XPathLocator importDataBtn = Locator.tagWithClass("a", "importDataBtn"); Assert.assertTrue("Import Data button should be present when file root is @files and no pipeline override exists", isElementPresent(importDataBtn)); log("Override pipeline root for project"); setPipelineRoot(TestFileUtils.getSampleData("AssayAPI").getParentFile().getAbsolutePath()); - goToProjectHome(); + goToWorkFolder(); Assert.assertTrue("Import Data button should not be present when file root is @files and pipeline override exists", !isElementPresent(importDataBtn)); log("Set webpart file root to @pipeline"); diff --git a/src/org/labkey/test/util/core/admin/ServerUsageUtils.java b/src/org/labkey/test/util/core/admin/ServerUsageUtils.java new file mode 100644 index 0000000000..285308710f --- /dev/null +++ b/src/org/labkey/test/util/core/admin/ServerUsageUtils.java @@ -0,0 +1,62 @@ +package org.labkey.test.util.core.admin; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.remoteapi.CommandException; +import org.labkey.remoteapi.CommandResponse; +import org.labkey.remoteapi.Connection; +import org.labkey.remoteapi.SimpleGetCommand; +import org.labkey.serverapi.util.UsageReportingLevel; +import org.labkey.test.WebTestHelper; +import org.labkey.test.util.data.JSONUtils; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.NoSuchElementException; + +public class ServerUsageUtils +{ + public static Map getUsageReportJson(Connection connection) throws IOException, CommandException + { + SimpleGetCommand command = new SimpleGetCommand("admin", "testMothershipReport"); + command.setParameters(getMothershipReportParams("CheckForUpdates", UsageReportingLevel.ON, false, null)); + CommandResponse response = command.execute(connection, "/"); + return response.getParsedData(); + } + + public static Map getUsageMetrics(Connection connection) throws IOException, CommandException + { + return JSONUtils.getProperty("jsonMetrics", getUsageReportJson(connection)); + } + + public static Map getModuleMetrics(Connection connection, String module) throws IOException, CommandException + { + Map modules = JSONUtils.getProperty("jsonMetrics.modules", getUsageReportJson(connection)); + if (modules.containsKey(module)) + return JSONUtils.getProperty(module, modules); + else + throw new NoSuchElementException("Server metrics for " + module + " module do not exist. Found: " + modules.keySet()); + } + + @NotNull + public static String getTestMothershipReportUrl(String type, UsageReportingLevel level, boolean submit, @Nullable String forwardedFor) + { + Map params = getMothershipReportParams(type, level, submit, forwardedFor); + return WebTestHelper.buildURL("admin", "testMothershipReport", params); + } + + @NotNull + private static Map getMothershipReportParams(String type, UsageReportingLevel level, boolean submit, @Nullable String forwardedFor) + { + Map params = new HashMap<>(); + params.put("type", type); + params.put("level", level.toString()); + params.put("submit", submit); + params.put("testMode", true); + if (null != forwardedFor) + params.put("forwardedFor", forwardedFor); + return params; + } + +} diff --git a/src/org/labkey/test/util/data/JSONUtils.java b/src/org/labkey/test/util/data/JSONUtils.java new file mode 100644 index 0000000000..f65d863690 --- /dev/null +++ b/src/org/labkey/test/util/data/JSONUtils.java @@ -0,0 +1,107 @@ +package org.labkey.test.util.data; + +import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class JSONUtils +{ + private JSONUtils() {} + + + /** + * Returns the value of a specific property in the parsed data + * given a path to that property. + *

+ * The path is a period-delimited list of property names. For + * example, to obtain the 'bar' property from the Map associated + * with the 'foo' property, the path would be 'foo.bar'. + * Property names may include an array index. For example, 'foos[2].bar' + * will return the 'bar' property for the third item of the 'foos' array + * @param path The property path. + * @param data JSON data in Map form + * @param the type of the property. + * @return The property value + */ + public static T getProperty(String path, Map data) + { + String[] pathParts = StringUtils.trimToEmpty(path).split("\\."); + if (StringUtils.isAnyBlank(pathParts)) + throw new IllegalArgumentException("Path cannot contain blank parts: " + path); + return getProperty(Arrays.asList(pathParts), 0, data); + } + + /** + * Called by {@link #getProperty(String, Map)} after splitting the path into + * a String[], and recursively by itself as it descends the property + * hierarchy. + * @param path The path split into a String[]. + * @param pathIndex The current index into the path array. + * @param parent The current parent map. + * @param The type of the property. + * @return The property value + */ + @SuppressWarnings("unchecked") + private static T getProperty(List path, int pathIndex, Map parent) + { + if (null == parent) + throw new NullPointerException("object is null"); + + String key = path.get(pathIndex); + Integer arrayIndex = null; + Pattern arrayPattern = Pattern.compile("(.+)\\[([0-9]+)]$"); + Matcher matcher = arrayPattern.matcher(key); + if (matcher.find()) + { + key = matcher.group(1); + arrayIndex = Integer.parseInt(matcher.group(2)); + } + + Object prop = parent.get(key); + if (arrayIndex != null) + { + if (prop instanceof List list) + { + if (list.size() > arrayIndex) + prop = list.get(arrayIndex); + else + throw new NoSuchElementException("Array index out of bounds [size = %s]: '%s'" + .formatted(list.size(), getSubPath(path, pathIndex))); + } + else + throw new NoSuchElementException("No array found at path: '%s'. Found '%s'" + .formatted(getSubPath(path, pathIndex), (prop == null ? "null" : prop.getClass().getSimpleName()))); + } + + // if this was the last path part, return the prop + if (pathIndex == (path.size() - 1)) + { + if (prop != null) + return (T) prop; + else + throw new NoSuchElementException("No item found at path: '%s'" + .formatted(getSubPath(path, pathIndex))); + } + else + { + // recurse if prop is non-null and instance of map + if (prop instanceof Map) + return getProperty(path, pathIndex + 1, (Map)prop); + else + throw new NoSuchElementException("No map found at path: '%s'. Found: '%s'" + .formatted(getSubPath(path, pathIndex), (prop == null ? "null" : prop.getClass().getSimpleName()))); + } + } + + private static @NotNull String getSubPath(List path, int pathIndex) + { + return String.join(".", path.subList(0, pathIndex + 1)); + } + +}