From 91a5a76279a6f0a563c58d850cfde61809b8061d Mon Sep 17 00:00:00 2001 From: arbaazkhan1 Date: Tue, 16 Apr 2024 17:32:41 -0400 Subject: [PATCH 1/3] added FindCompactionTmpFiles option to Admin --- .../org/apache/accumulo/server/util/Admin.java | 18 ++++++++++++++++++ .../server/util/AdminCommandsTest.java | 7 +++++++ 2 files changed, 25 insertions(+) diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java b/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java index 09435c4d8fc..dfd348965dd 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java @@ -106,6 +106,15 @@ static class AdminOpts extends ServerUtilOpts { boolean force = false; } + @Parameters(commandDescription = "Compaction Temp Files Utility") + static class FindCompactionTmpFilesCommand { + @Parameter(names = "--tables", description = "comma separated list of table names") + String tables; + + @Parameter(names = "--delete", description = "if true, will delete tmp files") + boolean delete = false; + } + @Parameters(commandDescription = "stop the tablet server on the given hosts") static class StopCommand { @Parameter(description = " { ... }") @@ -321,6 +330,9 @@ public void execute(final String[] args) { VolumesCommand volumesCommand = new VolumesCommand(); cl.addCommand("volumes", volumesCommand); + FindCompactionTmpFilesCommand filesCommand = new FindCompactionTmpFilesCommand(); + cl.addCommand("compactionTempFiles", filesCommand); + cl.parse(args); if (opts.help || cl.getParsedCommand() == null) { @@ -385,6 +397,12 @@ public void execute(final String[] args) { tServerLocksOpts.delete); } else if (cl.getParsedCommand().equals("fate")) { executeFateOpsCommand(context, fateOpsCommand); + } else if (cl.getParsedCommand().equals("compactionTempFiles")) { + if (filesCommand.delete) { + FindCompactionTmpFiles.main(new String[] {filesCommand.tables, "--delete"}); + } else { + FindCompactionTmpFiles.main(new String[] {filesCommand.tables}); + } } else { everything = cl.getParsedCommand().equals("stopAll"); diff --git a/server/base/src/test/java/org/apache/accumulo/server/util/AdminCommandsTest.java b/server/base/src/test/java/org/apache/accumulo/server/util/AdminCommandsTest.java index 36388f96305..5a8688e802d 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/util/AdminCommandsTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/util/AdminCommandsTest.java @@ -84,4 +84,11 @@ public void testAdminOpts() { Admin.AdminOpts opts = new Admin.AdminOpts(); assertFalse(opts.force); } + + @Test + public void testFindCompactionTmpFilesCommand() { + Admin.FindCompactionTmpFilesCommand filesCommand = new Admin.FindCompactionTmpFilesCommand(); + assertNull(filesCommand.tables); + assertFalse(filesCommand.delete); + } } From 4a481ac427554a9d688694566abf547c33e5a38a Mon Sep 17 00:00:00 2001 From: arbaazkhan1 Date: Mon, 6 May 2024 14:08:48 -0400 Subject: [PATCH 2/3] fixed changes --- .../apache/accumulo/server/util/Admin.java | 6 +- .../server/util/FindCompactionTmpFiles.java | 55 +++++++++++-------- 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java b/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java index dfd348965dd..96e022396c6 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java @@ -398,11 +398,7 @@ public void execute(final String[] args) { } else if (cl.getParsedCommand().equals("fate")) { executeFateOpsCommand(context, fateOpsCommand); } else if (cl.getParsedCommand().equals("compactionTempFiles")) { - if (filesCommand.delete) { - FindCompactionTmpFiles.main(new String[] {filesCommand.tables, "--delete"}); - } else { - FindCompactionTmpFiles.main(new String[] {filesCommand.tables}); - } + FindCompactionTmpFiles.execute(context, filesCommand.tables, filesCommand.delete); } else { everything = cl.getParsedCommand().equals("stopAll"); diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/FindCompactionTmpFiles.java b/server/base/src/main/java/org/apache/accumulo/server/util/FindCompactionTmpFiles.java index eacfac6691e..35cdb331817 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/FindCompactionTmpFiles.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/FindCompactionTmpFiles.java @@ -176,6 +176,36 @@ public static DeleteStats deleteTempFiles(ServerContext context, Set files return stats; } + public static void execute(ServerContext context, final String tablesToSearch, + final boolean delete) throws Exception { + LOG.info("Looking for compaction tmp files over tables: {}, deleting: {}", tablesToSearch, + delete); + + String[] tables = tablesToSearch.split(","); + for (String table : tables) { + + table = table.trim(); + String tableId = context.tableOperations().tableIdMap().get(table); + if (tableId == null || tableId.isEmpty()) { + LOG.warn("TableId for table: {} does not exist, maybe the table was deleted?", table); + continue; + } + + final Set matches = findTempFiles(context, tableId); + LOG.info("Found the following compaction tmp files for table {}:", table); + matches.forEach(p -> LOG.info("{}", p)); + + if (delete) { + LOG.info("Deleting compaction tmp files for table {}...", table); + DeleteStats stats = deleteTempFiles(context, matches); + LOG.info( + "Deletion of compaction tmp files for table {} complete. Success:{}, Failure:{}, Error:{}", + table, stats.success, stats.failure, stats.error); + } + + } + } + public static void main(String[] args) throws Exception { Opts opts = new Opts(); opts.parseArgs(FindCompactionTmpFiles.class.getName(), args); @@ -186,30 +216,7 @@ public static void main(String[] args) throws Exception { try (Scope scope = span.makeCurrent()) { ServerContext context = opts.getServerContext(); - String[] tables = opts.tables.split(","); - - for (String table : tables) { - - table = table.trim(); - String tableId = context.tableOperations().tableIdMap().get(table); - if (tableId == null || tableId.isEmpty()) { - LOG.warn("TableId for table: {} does not exist, maybe the table was deleted?", table); - continue; - } - - final Set matches = findTempFiles(context, tableId); - LOG.info("Found the following compaction tmp files for table {}:", table); - matches.forEach(p -> LOG.info("{}", p)); - - if (opts.delete) { - LOG.info("Deleting compaction tmp files for table {}...", table); - DeleteStats stats = deleteTempFiles(context, matches); - LOG.info( - "Deletion of compaction tmp files for table {} complete. Success:{}, Failure:{}, Error:{}", - table, stats.success, stats.failure, stats.error); - } - - } + execute(context, opts.tables, opts.delete); } finally { span.end(); From cb763dbab13f54251485cb9dd12d58f5239b2bf6 Mon Sep 17 00:00:00 2001 From: arbaazkhan1 Date: Mon, 2 Jun 2025 15:00:04 -0400 Subject: [PATCH 3/3] test new admin command --- .../apache/accumulo/server/util/Admin.java | 4 +- .../FindCompactionTmpFilesIT_SimpleSuite.java | 55 +++++++++++++++++++ 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java b/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java index b0b646c68a3..de5065ea122 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/Admin.java @@ -141,10 +141,10 @@ static class GracefulShutdownCommand extends SubCommandOpts { @Parameters(commandDescription = "Compaction Temp Files Utility") static class FindCompactionTmpFilesCommand { - @Parameter(names = "--tables", description = "comma separated list of table names") + @Parameter(names = {"-t", "--tables"}, description = "comma separated list of table names") String tables; - @Parameter(names = "--delete", description = "if true, will delete tmp files") + @Parameter(names = {"-d", "--delete"}, description = "if true, will delete tmp files") boolean delete = false; } diff --git a/test/src/main/java/org/apache/accumulo/test/functional/FindCompactionTmpFilesIT_SimpleSuite.java b/test/src/main/java/org/apache/accumulo/test/functional/FindCompactionTmpFilesIT_SimpleSuite.java index aa56f17ba28..a323b716235 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/FindCompactionTmpFilesIT_SimpleSuite.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/FindCompactionTmpFilesIT_SimpleSuite.java @@ -20,6 +20,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotSame; import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.HashSet; @@ -36,6 +37,7 @@ import org.apache.accumulo.harness.SharedMiniClusterBase; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.tablets.TabletNameGenerator; +import org.apache.accumulo.server.util.Admin; import org.apache.accumulo.server.util.FindCompactionTmpFiles; import org.apache.accumulo.server.util.FindCompactionTmpFiles.DeleteStats; import org.apache.hadoop.fs.FileSystem; @@ -216,4 +218,57 @@ public void testFindCompactionTmpFilesMainWithDelete() throws Exception { } } + @Test + public void testFindCompactionTmpFilesAdminCommand() throws Exception { + + try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) { + + String tableName = getUniqueNames(1)[0]; + c.tableOperations().create(tableName); + ReadWriteIT.ingest(c, 100, 1, 1, 0, tableName); + c.tableOperations().flush(tableName); + + String tableId = c.tableOperations().tableIdMap().get(tableName); + TableId tid = TableId.of(tableId); + + ServerContext ctx = getCluster().getServerContext(); + FileSystem fs = getCluster().getFileSystem(); + + Set tablesDirs = ctx.getTablesDirs(); + assertEquals(1, tablesDirs.size()); + + String tdir = tablesDirs.iterator().next() + "/" + tid.canonical() + "/default_tablet"; + Path defaultTabletPath = new Path(tdir); + assertTrue(fs.exists(defaultTabletPath)); + + Set generatedPaths = generateTmpFilePaths(ctx, tid, defaultTabletPath, 100); + for (Path p : generatedPaths) { + assertFalse(fs.exists(p)); + assertTrue(fs.createNewFile(p)); + assertTrue(fs.exists(p)); + } + + Set foundPaths = FindCompactionTmpFiles.findTempFiles(ctx, tid.canonical()); + assertEquals(100, foundPaths.size()); + assertEquals(foundPaths, generatedPaths); + + System.setProperty("accumulo.properties", + "file://" + getCluster().getAccumuloPropertiesPath()); + assertEquals(0, getCluster().exec(Admin.class, "compactionTempFiles", "-t", tableName) + .getProcess().waitFor()); + + foundPaths = FindCompactionTmpFiles.findTempFiles(ctx, tid.canonical()); + assertEquals(100, foundPaths.size()); + assertEquals(foundPaths, generatedPaths); + + assertEquals(0, getCluster().exec(Admin.class, "compactionTempFiles", "-t", tableName, "-d") + .getProcess().waitFor()); + + foundPaths = FindCompactionTmpFiles.findTempFiles(ctx, tid.canonical()); + assertEquals(0, foundPaths.size()); + assertNotSame(foundPaths, generatedPaths); + + } + } + }