diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java b/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java index c3dd6474f5b..6bb4a57a920 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java @@ -138,6 +138,15 @@ private static class SubCommandOpts { public boolean help = false; } + @Parameters(commandNames = "find", commandDescription = "Compaction Temp Files Utility") + static class FindCompactionTmpFilesCommand { + @Parameter(names = {"-t", "--tables"}, description = "comma separated list of table names") + String tables; + + @Parameter(names = {"-d", "--delete"}, description = "if true, will delete tmp files") + boolean delete = false; + } + @Parameters(commandNames = "stop", commandDescription = "Stop the servers at the given addresses allowing them to complete current task but not start new task. Hostnames only are no longer supported; you must use . To Stop all services on a host, use 'accumulo admin serviceStatus' to list all hosts and then pass them to this command.") static class StopCommand extends SubCommandOpts { @@ -481,6 +490,9 @@ public void execute(final String[] args) { VolumesCommand volumesCommand = new VolumesCommand(); cl.addCommand(volumesCommand); + FindCompactionTmpFilesCommand filesCommand = new FindCompactionTmpFilesCommand(); + cl.addCommand("compactionTempFiles", filesCommand); + try { cl.parse(args); } catch (MissingCommandException e) { @@ -554,6 +566,8 @@ public void execute(final String[] args) { RestoreZookeeper.execute(conf, restoreZooOpts.file, restoreZooOpts.overwrite); } else if (cl.getParsedCommand().equals("fate")) { executeFateOpsCommand(context, fateOpsCommand); + } else if (cl.getParsedCommand().equals("compactionTempFiles")) { + FindCompactionTmpFiles.execute(context, filesCommand.tables, filesCommand.delete); } else if (cl.getParsedCommand().equals("serviceStatus")) { ServiceStatusCmd ssc = new ServiceStatusCmd(); ssc.execute(context, serviceStatusCommandOpts.json, serviceStatusCommandOpts.showHosts); diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/FindCompactionTmpFiles.java b/server/base/src/main/java/org/apache/accumulo/server/util/FindCompactionTmpFiles.java index be95a9a8006..6aa766833d0 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/FindCompactionTmpFiles.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/FindCompactionTmpFiles.java @@ -183,6 +183,36 @@ public static DeleteStats deleteTempFiles(ServerContext context, Set files return stats; } + public static void execute(ServerContext context, final String tablesToSearch, + final boolean delete) throws Exception { + LOG.info("Looking for compaction tmp files over tables: {}, deleting: {}", tablesToSearch, + delete); + + String[] tables = tablesToSearch.split(","); + for (String table : tables) { + + table = table.trim(); + String tableId = context.tableOperations().tableIdMap().get(table); + if (tableId == null || tableId.isEmpty()) { + LOG.warn("TableId for table: {} does not exist, maybe the table was deleted?", table); + continue; + } + + final Set matches = findTempFiles(context, tableId); + LOG.info("Found the following compaction tmp files for table {}:", table); + matches.forEach(p -> LOG.info("{}", p)); + + if (delete) { + LOG.info("Deleting compaction tmp files for table {}...", table); + DeleteStats stats = deleteTempFiles(context, matches); + LOG.info( + "Deletion of compaction tmp files for table {} complete. Success:{}, Failure:{}, Error:{}", + table, stats.success, stats.failure, stats.error); + } + + } + } + public static void main(String[] args) throws Exception { Opts opts = new Opts(); opts.parseArgs(FindCompactionTmpFiles.class.getName(), args); @@ -193,31 +223,7 @@ public static void main(String[] args) throws Exception { try (Scope scope = span.makeCurrent()) { ServerContext context = opts.getServerContext(); - String[] tables = opts.tables.split(","); - - final var stringStringMap = context.tableOperations().tableIdMap(); - for (String table : tables) { - - table = table.trim(); - String tableId = stringStringMap.get(table); - if (tableId == null || tableId.isEmpty()) { - LOG.warn("TableId for table: {} does not exist, maybe the table was deleted?", table); - continue; - } - - final Set matches = findTempFiles(context, tableId); - LOG.info("Found the following compaction tmp files for table {}:", table); - matches.forEach(p -> LOG.info("{}", p)); - - if (opts.delete) { - LOG.info("Deleting compaction tmp files for table {}...", table); - DeleteStats stats = deleteTempFiles(context, matches); - LOG.info( - "Deletion of compaction tmp files for table {} complete. Success:{}, Failure:{}, Error:{}", - table, stats.success, stats.failure, stats.error); - } - - } + execute(context, opts.tables, opts.delete); } finally { span.end(); diff --git a/server/base/src/test/java/org/apache/accumulo/server/util/AdminCommandsTest.java b/server/base/src/test/java/org/apache/accumulo/server/util/AdminCommandsTest.java index 831e7286bb1..f4288fe4d66 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/util/AdminCommandsTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/util/AdminCommandsTest.java @@ -77,4 +77,11 @@ public void testStopOpts() { Admin.StopCommand opts = new Admin.StopCommand(); assertFalse(opts.force); } + + @Test + public void testFindCompactionTmpFilesCommand() { + Admin.FindCompactionTmpFilesCommand filesCommand = new Admin.FindCompactionTmpFilesCommand(); + assertNull(filesCommand.tables); + assertFalse(filesCommand.delete); + } } diff --git a/test/src/main/java/org/apache/accumulo/test/functional/FindCompactionTmpFilesIT_SimpleSuite.java b/test/src/main/java/org/apache/accumulo/test/functional/FindCompactionTmpFilesIT_SimpleSuite.java index aa56f17ba28..a323b716235 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/FindCompactionTmpFilesIT_SimpleSuite.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/FindCompactionTmpFilesIT_SimpleSuite.java @@ -20,6 +20,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotSame; import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.HashSet; @@ -36,6 +37,7 @@ import org.apache.accumulo.harness.SharedMiniClusterBase; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.tablets.TabletNameGenerator; +import org.apache.accumulo.server.util.Admin; import org.apache.accumulo.server.util.FindCompactionTmpFiles; import org.apache.accumulo.server.util.FindCompactionTmpFiles.DeleteStats; import org.apache.hadoop.fs.FileSystem; @@ -216,4 +218,57 @@ public void testFindCompactionTmpFilesMainWithDelete() throws Exception { } } + @Test + public void testFindCompactionTmpFilesAdminCommand() throws Exception { + + try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) { + + String tableName = getUniqueNames(1)[0]; + c.tableOperations().create(tableName); + ReadWriteIT.ingest(c, 100, 1, 1, 0, tableName); + c.tableOperations().flush(tableName); + + String tableId = c.tableOperations().tableIdMap().get(tableName); + TableId tid = TableId.of(tableId); + + ServerContext ctx = getCluster().getServerContext(); + FileSystem fs = getCluster().getFileSystem(); + + Set tablesDirs = ctx.getTablesDirs(); + assertEquals(1, tablesDirs.size()); + + String tdir = tablesDirs.iterator().next() + "/" + tid.canonical() + "/default_tablet"; + Path defaultTabletPath = new Path(tdir); + assertTrue(fs.exists(defaultTabletPath)); + + Set generatedPaths = generateTmpFilePaths(ctx, tid, defaultTabletPath, 100); + for (Path p : generatedPaths) { + assertFalse(fs.exists(p)); + assertTrue(fs.createNewFile(p)); + assertTrue(fs.exists(p)); + } + + Set foundPaths = FindCompactionTmpFiles.findTempFiles(ctx, tid.canonical()); + assertEquals(100, foundPaths.size()); + assertEquals(foundPaths, generatedPaths); + + System.setProperty("accumulo.properties", + "file://" + getCluster().getAccumuloPropertiesPath()); + assertEquals(0, getCluster().exec(Admin.class, "compactionTempFiles", "-t", tableName) + .getProcess().waitFor()); + + foundPaths = FindCompactionTmpFiles.findTempFiles(ctx, tid.canonical()); + assertEquals(100, foundPaths.size()); + assertEquals(foundPaths, generatedPaths); + + assertEquals(0, getCluster().exec(Admin.class, "compactionTempFiles", "-t", tableName, "-d") + .getProcess().waitFor()); + + foundPaths = FindCompactionTmpFiles.findTempFiles(ctx, tid.canonical()); + assertEquals(0, foundPaths.size()); + assertNotSame(foundPaths, generatedPaths); + + } + } + }