Skip to content

Run FindCompactionTmpFiles utility via accumulo Admin #4465

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ArbaazKhan1 wants to merge 10 commits into
base: main
Choose a base branch
from
+107 −25
Open

Run FindCompactionTmpFiles utility via accumulo Admin #4465

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
91a5a76
added FindCompactionTmpFiles option to Admin
ArbaazKhan1 Apr 16, 2024
4a481ac
fixed changes
ArbaazKhan1 May 6, 2024
7dd4dc0
Merge branch 'main' into accumulo-3986
ArbaazKhan1 Aug 27, 2024
3368def
Merge branch 'apache:main' into accumulo-3986
ArbaazKhan1 May 22, 2025
aa94be8
Merge branch 'apache:main' into accumulo-3986
ArbaazKhan1 May 30, 2025
cb763db
test new admin command
ArbaazKhan1 Jun 2, 2025
e316773
Merge branch 'main' into accumulo-3986
ArbaazKhan1 Jun 12, 2025
967386c
Merge branch 'main' into accumulo-3986
ArbaazKhan1 Aug 14, 2025
229861c
Merge branch 'main' into accumulo-3986
ArbaazKhan1 Jan 13, 2026
e1be88d
Merge branch 'apache:main' into accumulo-3986
ArbaazKhan1 Jan 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,15 @@ private static class SubCommandOpts {
public boolean help = false;
}

@Parameters(commandNames = "find", commandDescription = "Compaction Temp Files Utility")
static class FindCompactionTmpFilesCommand {
@Parameter(names = {"-t", "--tables"}, description = "comma separated list of table names")
String tables;

@Parameter(names = {"-d", "--delete"}, description = "if true, will delete tmp files")
boolean delete = false;
}

@Parameters(commandNames = "stop",
commandDescription = "Stop the servers at the given addresses allowing them to complete current task but not start new task. Hostnames only are no longer supported; you must use <host:port>. To Stop all services on a host, use 'accumulo admin serviceStatus' to list all hosts and then pass them to this command.")
static class StopCommand extends SubCommandOpts {
Expand Down Expand Up @@ -481,6 +490,9 @@ public void execute(final String[] args) {
VolumesCommand volumesCommand = new VolumesCommand();
cl.addCommand(volumesCommand);

FindCompactionTmpFilesCommand filesCommand = new FindCompactionTmpFilesCommand();
cl.addCommand("compactionTempFiles", filesCommand);

try {
cl.parse(args);
} catch (MissingCommandException e) {
Expand Down Expand Up @@ -554,6 +566,8 @@ public void execute(final String[] args) {
RestoreZookeeper.execute(conf, restoreZooOpts.file, restoreZooOpts.overwrite);
} else if (cl.getParsedCommand().equals("fate")) {
executeFateOpsCommand(context, fateOpsCommand);
} else if (cl.getParsedCommand().equals("compactionTempFiles")) {
FindCompactionTmpFiles.execute(context, filesCommand.tables, filesCommand.delete);
} else if (cl.getParsedCommand().equals("serviceStatus")) {
ServiceStatusCmd ssc = new ServiceStatusCmd();
ssc.execute(context, serviceStatusCommandOpts.json, serviceStatusCommandOpts.showHosts);
Expand Down
56 changes: 31 additions & 25 deletions server/base/src/main/java/org/apache/accumulo/server/util/FindCompactionTmpFiles.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,36 @@ public static DeleteStats deleteTempFiles(ServerContext context, Set<Path> files
return stats;
}

public static void execute(ServerContext context, final String tablesToSearch,
final boolean delete) throws Exception {
LOG.info("Looking for compaction tmp files over tables: {}, deleting: {}", tablesToSearch,
delete);

String[] tables = tablesToSearch.split(",");
for (String table : tables) {

table = table.trim();
String tableId = context.tableOperations().tableIdMap().get(table);
if (tableId == null || tableId.isEmpty()) {
LOG.warn("TableId for table: {} does not exist, maybe the table was deleted?", table);
continue;
}

final Set<Path> matches = findTempFiles(context, tableId);
LOG.info("Found the following compaction tmp files for table {}:", table);
matches.forEach(p -> LOG.info("{}", p));

if (delete) {
LOG.info("Deleting compaction tmp files for table {}...", table);
DeleteStats stats = deleteTempFiles(context, matches);
LOG.info(
"Deletion of compaction tmp files for table {} complete. Success:{}, Failure:{}, Error:{}",
table, stats.success, stats.failure, stats.error);
}

}
}

public static void main(String[] args) throws Exception {
Opts opts = new Opts();
opts.parseArgs(FindCompactionTmpFiles.class.getName(), args);
Expand All @@ -193,31 +223,7 @@ public static void main(String[] args) throws Exception {
try (Scope scope = span.makeCurrent()) {

ServerContext context = opts.getServerContext();
String[] tables = opts.tables.split(",");

final var stringStringMap = context.tableOperations().tableIdMap();
for (String table : tables) {

table = table.trim();
String tableId = stringStringMap.get(table);
if (tableId == null || tableId.isEmpty()) {
LOG.warn("TableId for table: {} does not exist, maybe the table was deleted?", table);
continue;
}

final Set<Path> matches = findTempFiles(context, tableId);
LOG.info("Found the following compaction tmp files for table {}:", table);
matches.forEach(p -> LOG.info("{}", p));

if (opts.delete) {
LOG.info("Deleting compaction tmp files for table {}...", table);
DeleteStats stats = deleteTempFiles(context, matches);
LOG.info(
"Deletion of compaction tmp files for table {} complete. Success:{}, Failure:{}, Error:{}",
table, stats.success, stats.failure, stats.error);
}

}
execute(context, opts.tables, opts.delete);

} finally {
span.end();
Expand Down
7 changes: 7 additions & 0 deletions server/base/src/test/java/org/apache/accumulo/server/util/AdminCommandsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,4 +77,11 @@ public void testStopOpts() {
Admin.StopCommand opts = new Admin.StopCommand();
assertFalse(opts.force);
}

@Test
public void testFindCompactionTmpFilesCommand() {
Admin.FindCompactionTmpFilesCommand filesCommand = new Admin.FindCompactionTmpFilesCommand();
assertNull(filesCommand.tables);
assertFalse(filesCommand.delete);
}
}
55 changes: 55 additions & 0 deletions ...c/main/java/org/apache/accumulo/test/functional/FindCompactionTmpFilesIT_SimpleSuite.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotSame;
import static org.junit.jupiter.api.Assertions.assertTrue;

import java.util.HashSet;
Expand All @@ -36,6 +37,7 @@
import org.apache.accumulo.harness.SharedMiniClusterBase;
import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.server.tablets.TabletNameGenerator;
import org.apache.accumulo.server.util.Admin;
import org.apache.accumulo.server.util.FindCompactionTmpFiles;
import org.apache.accumulo.server.util.FindCompactionTmpFiles.DeleteStats;
import org.apache.hadoop.fs.FileSystem;
Expand Down Expand Up @@ -216,4 +218,57 @@ public void testFindCompactionTmpFilesMainWithDelete() throws Exception {
}
}

@Test
public void testFindCompactionTmpFilesAdminCommand() throws Exception {

try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) {

String tableName = getUniqueNames(1)[0];
c.tableOperations().create(tableName);
ReadWriteIT.ingest(c, 100, 1, 1, 0, tableName);
c.tableOperations().flush(tableName);

String tableId = c.tableOperations().tableIdMap().get(tableName);
TableId tid = TableId.of(tableId);

ServerContext ctx = getCluster().getServerContext();
FileSystem fs = getCluster().getFileSystem();

Set<String> tablesDirs = ctx.getTablesDirs();
assertEquals(1, tablesDirs.size());

String tdir = tablesDirs.iterator().next() + "/" + tid.canonical() + "/default_tablet";
Path defaultTabletPath = new Path(tdir);
assertTrue(fs.exists(defaultTabletPath));

Set<Path> generatedPaths = generateTmpFilePaths(ctx, tid, defaultTabletPath, 100);
for (Path p : generatedPaths) {
assertFalse(fs.exists(p));
assertTrue(fs.createNewFile(p));
assertTrue(fs.exists(p));
}

Set<Path> foundPaths = FindCompactionTmpFiles.findTempFiles(ctx, tid.canonical());
assertEquals(100, foundPaths.size());
assertEquals(foundPaths, generatedPaths);

System.setProperty("accumulo.properties",
"file://" + getCluster().getAccumuloPropertiesPath());
assertEquals(0, getCluster().exec(Admin.class, "compactionTempFiles", "-t", tableName)
.getProcess().waitFor());

foundPaths = FindCompactionTmpFiles.findTempFiles(ctx, tid.canonical());
assertEquals(100, foundPaths.size());
assertEquals(foundPaths, generatedPaths);

assertEquals(0, getCluster().exec(Admin.class, "compactionTempFiles", "-t", tableName, "-d")
.getProcess().waitFor());

foundPaths = FindCompactionTmpFiles.findTempFiles(ctx, tid.canonical());
assertEquals(0, foundPaths.size());
assertNotSame(foundPaths, generatedPaths);

}
}

}