From 1f5067068e379eba4e5391838360cbcaea228cfc Mon Sep 17 00:00:00 2001 From: Florian Kinder Date: Tue, 3 Feb 2026 21:07:37 +0100 Subject: [PATCH 1/3] fix: normalize filename handling to strip both .json and .gz extensions Previously, uploaded files like `foo.json.gz` would only have `.gz` stripped, resulting in filenames like `foo.json` stored in the database and conversion output directories named `foo.json/`. Now both extensions are stripped consistently: - Database stores base name without extensions (e.g., `foo`) - Files on disk use explicit `.json.gz` extension - Conversion output directories use clean base name (e.g., `foo/`) This change requires migration for existing deployments. --- cmd/ocap-webserver/main.go | 5 ++++- cmd/ocap-webserver/main_test.go | 12 ++++++------ internal/conversion/worker.go | 2 +- internal/conversion/worker_test.go | 20 ++++++++++---------- internal/server/handler.go | 10 ++++++---- internal/server/handler_test.go | 20 ++++++++++---------- internal/server/integration_test.go | 2 +- internal/storage/json.go | 2 +- internal/storage/json_test.go | 6 +++--- 9 files changed, 42 insertions(+), 37 deletions(-) diff --git a/cmd/ocap-webserver/main.go b/cmd/ocap-webserver/main.go index f4b34e61..255a7e1d 100644 --- a/cmd/ocap-webserver/main.go +++ b/cmd/ocap-webserver/main.go @@ -123,11 +123,14 @@ func showConversionStatus(ctx context.Context, repo *server.RepoOperation) error } func convertSingleFile(ctx context.Context, repo *server.RepoOperation, inputFile, dataDir string, chunkSize uint32, format string) error { - // Determine filename - only strip .gz to match database filename format + // Determine filename - strip .gz and .json to get base name baseName := filepath.Base(inputFile) if ext := filepath.Ext(baseName); ext == ".gz" { baseName = baseName[:len(baseName)-len(ext)] } + if ext := filepath.Ext(baseName); ext == ".json" { + baseName = baseName[:len(baseName)-len(ext)] + } // Check if operation exists in database - if so, use worker for consistent behavior if op, err := repo.GetByFilename(ctx, baseName); err == nil && op != nil { diff --git a/cmd/ocap-webserver/main_test.go b/cmd/ocap-webserver/main_test.go index 02b4ba8d..035a1113 100644 --- a/cmd/ocap-webserver/main_test.go +++ b/cmd/ocap-webserver/main_test.go @@ -281,8 +281,8 @@ func TestConvertSingleFile(t *testing.T) { err = convertSingleFile(ctx, repo, inputPath, dataDir, 300, "protobuf") require.NoError(t, err) - // Verify output was created (keeps .json suffix to match database filename) - outputDir := filepath.Join(dataDir, "test_mission.json") + // Verify output was created + outputDir := filepath.Join(dataDir, "test_mission") _, err = os.Stat(filepath.Join(outputDir, "manifest.pb")) require.NoError(t, err) } @@ -359,7 +359,7 @@ func TestConvertSingleFile_WithDatabaseEntry(t *testing.T) { WorldName: "Stratis", MissionName: "Test Op", MissionDuration: 10, - Filename: "db_test.json", + Filename: "db_test", Date: "2024-01-01", StorageFormat: "json", ConversionStatus: "pending", @@ -375,12 +375,12 @@ func TestConvertSingleFile_WithDatabaseEntry(t *testing.T) { require.NoError(t, err) // Verify output was created - outputDir := filepath.Join(dataDir, "db_test.json") + outputDir := filepath.Join(dataDir, "db_test") _, err = os.Stat(filepath.Join(outputDir, "manifest.pb")) require.NoError(t, err) // Verify database was updated - result, err := repo.GetByFilename(ctx, "db_test.json") + result, err := repo.GetByFilename(ctx, "db_test") require.NoError(t, err) assert.Equal(t, "completed", result.ConversionStatus) assert.Equal(t, "protobuf", result.StorageFormat) @@ -449,7 +449,7 @@ func TestConvertAll_WithOperations(t *testing.T) { }` // Write gzipped JSON - jsonPath := filepath.Join(dataDir, "test_op.gz") + jsonPath := filepath.Join(dataDir, "test_op.json.gz") f, err := os.Create(jsonPath) require.NoError(t, err) gw := gzip.NewWriter(f) diff --git a/internal/conversion/worker.go b/internal/conversion/worker.go index 20a3e8b0..b2c3c555 100644 --- a/internal/conversion/worker.go +++ b/internal/conversion/worker.go @@ -174,7 +174,7 @@ func (w *Worker) convertOperation(ctx context.Context, op Operation) error { } // Determine paths - jsonPath := filepath.Join(w.dataDir, op.Filename+".gz") + jsonPath := filepath.Join(w.dataDir, op.Filename+".json.gz") outputPath := filepath.Join(w.dataDir, op.Filename) // Check if JSON file exists diff --git a/internal/conversion/worker_test.go b/internal/conversion/worker_test.go index 5be0d24c..871b3025 100644 --- a/internal/conversion/worker_test.go +++ b/internal/conversion/worker_test.go @@ -112,7 +112,7 @@ func TestWorker_ConvertOne(t *testing.T) { }` // Write gzipped JSON file - jsonPath := filepath.Join(dir, "test_mission.gz") + jsonPath := filepath.Join(dir, "test_mission.json.gz") f, err := os.Create(jsonPath) assert.NoError(t, err) gw := gzip.NewWriter(f) @@ -161,7 +161,7 @@ func TestWorker_ProcessOnce(t *testing.T) { // Write gzipped JSON files for _, name := range []string{"mission1", "mission2"} { - jsonPath := filepath.Join(dir, name+".gz") + jsonPath := filepath.Join(dir, name+".json.gz") f, _ := os.Create(jsonPath) gw := gzip.NewWriter(f) gw.Write([]byte(testData)) @@ -303,7 +303,7 @@ func TestTriggerConversion(t *testing.T) { }` // Write gzipped JSON file - jsonPath := filepath.Join(dir, "trigger_mission.gz") + jsonPath := filepath.Join(dir, "trigger_mission.json.gz") f, err := os.Create(jsonPath) assert.NoError(t, err) gw := gzip.NewWriter(f) @@ -397,7 +397,7 @@ func TestWorker_FlatBuffersFormat(t *testing.T) { }` // Write gzipped JSON file - jsonPath := filepath.Join(dir, "fb_test.gz") + jsonPath := filepath.Join(dir, "fb_test.json.gz") f, err := os.Create(jsonPath) assert.NoError(t, err) gw := gzip.NewWriter(f) @@ -442,7 +442,7 @@ func TestWorker_ContextCancellation(t *testing.T) { // Write multiple gzipped JSON files for i := 1; i <= 3; i++ { - jsonPath := filepath.Join(dir, fmt.Sprintf("cancel_%d.gz", i)) + jsonPath := filepath.Join(dir, fmt.Sprintf("cancel_%d.json.gz", i)) f, _ := os.Create(jsonPath) gw := gzip.NewWriter(f) gw.Write([]byte(testData)) @@ -613,7 +613,7 @@ func TestConvertOperation_UpdateConvertingStatusError(t *testing.T) { // Create test JSON file testData := `{"worldName": "test", "missionName": "Test", "endFrame": 5, "captureDelay": 1, "entities": [], "events": [], "times": []}` - jsonPath := filepath.Join(dir, "test.gz") + jsonPath := filepath.Join(dir, "test.json.gz") f, _ := os.Create(jsonPath) gw := gzip.NewWriter(f) gw.Write([]byte(testData)) @@ -640,7 +640,7 @@ func TestConvertOperation_UpdateStorageFormatError(t *testing.T) { // Create test JSON file testData := `{"worldName": "test", "missionName": "Test", "endFrame": 5, "captureDelay": 1, "entities": [], "events": [], "times": []}` - jsonPath := filepath.Join(dir, "test.gz") + jsonPath := filepath.Join(dir, "test.json.gz") f, _ := os.Create(jsonPath) gw := gzip.NewWriter(f) gw.Write([]byte(testData)) @@ -666,7 +666,7 @@ func TestConvertOperation_UpdateCompletedStatusError(t *testing.T) { // Create test JSON file testData := `{"worldName": "test", "missionName": "Test", "endFrame": 5, "captureDelay": 1, "entities": [], "events": [], "times": []}` - jsonPath := filepath.Join(dir, "test.gz") + jsonPath := filepath.Join(dir, "test.json.gz") f, _ := os.Create(jsonPath) gw := gzip.NewWriter(f) gw.Write([]byte(testData)) @@ -693,7 +693,7 @@ func TestConvertOperation_UpdateDurationError(t *testing.T) { // Create test JSON file testData := `{"worldName": "test", "missionName": "Test", "endFrame": 5, "captureDelay": 1, "entities": [], "events": [], "times": []}` - jsonPath := filepath.Join(dir, "test.gz") + jsonPath := filepath.Join(dir, "test.json.gz") f, _ := os.Create(jsonPath) gw := gzip.NewWriter(f) gw.Write([]byte(testData)) @@ -720,7 +720,7 @@ func TestConvertOperation_InvalidStorageFormat(t *testing.T) { // Create test JSON file testData := `{"worldName": "test", "missionName": "Test", "endFrame": 5, "captureDelay": 1, "entities": [], "events": [], "times": []}` - jsonPath := filepath.Join(dir, "test.gz") + jsonPath := filepath.Join(dir, "test.json.gz") f, _ := os.Create(jsonPath) gw := gzip.NewWriter(f) gw.Write([]byte(testData)) diff --git a/internal/server/handler.go b/internal/server/handler.go index eb9031a8..29637db7 100644 --- a/internal/server/handler.go +++ b/internal/server/handler.go @@ -362,7 +362,9 @@ func (h *Handler) StoreOperation(c echo.Context) error { return echo.ErrForbidden } - filename := strings.TrimSuffix(filepath.Base(c.FormValue("filename")), ".gz") + filename := filepath.Base(c.FormValue("filename")) + filename = strings.TrimSuffix(filename, ".gz") + filename = strings.TrimSuffix(filename, ".json") op := Operation{ WorldName: c.FormValue("worldName"), @@ -391,7 +393,7 @@ func (h *Handler) StoreOperation(c echo.Context) error { } defer file.Close() - writer, err := os.Create(filepath.Join(h.setting.Data, filename+".gz")) + writer, err := os.Create(filepath.Join(h.setting.Data, filename+".json.gz")) if err != nil { return err } @@ -414,7 +416,7 @@ func (h *Handler) GetCapture(c echo.Context) error { return err } - upath := filepath.Join(h.setting.Data, filepath.Base(name+".gz")) + upath := filepath.Join(h.setting.Data, filepath.Base(name+".json.gz")) c.Response().Header().Set("Content-Encoding", "gzip") c.Response().Header().Set("Content-Type", "application/json") @@ -428,7 +430,7 @@ func (h *Handler) GetCaptureFile(c echo.Context) error { return err } - filename := filepath.Base(name + ".gz") + filename := filepath.Base(name + ".json.gz") c.Response().Header().Set("Content-Disposition", "attachment;filename=\""+filename+"\"") diff --git a/internal/server/handler_test.go b/internal/server/handler_test.go index c673198c..2f5f24b9 100644 --- a/internal/server/handler_test.go +++ b/internal/server/handler_test.go @@ -232,7 +232,7 @@ func TestGetOperationManifest(t *testing.T) { "times": [], "Ede": [] }` - testDataPath := filepath.Join(dataDir, "test_mission.gz") + testDataPath := filepath.Join(dataDir, "test_mission.json.gz") err = writeGzipped(testDataPath, []byte(testData)) assert.NoError(t, err) @@ -731,7 +731,7 @@ func TestStoreOperation(t *testing.T) { writer.WriteField("tag", "coop") // Create file part - fileWriter, err := writer.CreateFormFile("file", "test_upload.gz") + fileWriter, err := writer.CreateFormFile("file", "test_upload.json.gz") require.NoError(t, err) // Write gzipped JSON data @@ -752,7 +752,7 @@ func TestStoreOperation(t *testing.T) { assert.Equal(t, http.StatusOK, rec.Code) // Verify file was created - _, err = os.Stat(filepath.Join(dataDir, "test_upload.gz")) + _, err = os.Stat(filepath.Join(dataDir, "test_upload.json.gz")) assert.NoError(t, err) }) @@ -798,7 +798,7 @@ func TestGetCapture(t *testing.T) { // Create test gzipped file testData := `{"test": "capture data"}` - testPath := filepath.Join(dataDir, "test_capture.gz") + testPath := filepath.Join(dataDir, "test_capture.json.gz") err = writeGzipped(testPath, []byte(testData)) require.NoError(t, err) @@ -841,7 +841,7 @@ func TestGetCaptureFile(t *testing.T) { require.NoError(t, err) // Create test gzipped file - testPath := filepath.Join(dataDir, "download_test.gz") + testPath := filepath.Join(dataDir, "download_test.json.gz") err = writeGzipped(testPath, []byte(`{"download": "test"}`)) require.NoError(t, err) @@ -859,7 +859,7 @@ func TestGetCaptureFile(t *testing.T) { err = hdlr.GetCaptureFile(c) assert.NoError(t, err) assert.Contains(t, rec.Header().Get("Content-Disposition"), "attachment") - assert.Contains(t, rec.Header().Get("Content-Disposition"), "download_test.gz") + assert.Contains(t, rec.Header().Get("Content-Disposition"), "download_test.json.gz") } func TestGetMapTitle(t *testing.T) { @@ -1107,7 +1107,7 @@ func TestStoreOperationWithConversionTrigger(t *testing.T) { writer.WriteField("filename", "trigger_test") writer.WriteField("tag", "coop") - fileWriter, _ := writer.CreateFormFile("file", "trigger_test.gz") + fileWriter, _ := writer.CreateFormFile("file", "trigger_test.json.gz") gw := gzip.NewWriter(fileWriter) gw.Write([]byte(`{"test": "trigger"}`)) gw.Close() @@ -1415,7 +1415,7 @@ func TestGetOperationManifest_JSONFormat(t *testing.T) { "entities": [], "events": [] }` - err = writeGzipped(filepath.Join(dataDir, "json_manifest_test.gz"), []byte(testJSON)) + err = writeGzipped(filepath.Join(dataDir, "json_manifest_test.json.gz"), []byte(testJSON)) require.NoError(t, err) // Register JSON engine @@ -1680,7 +1680,7 @@ func TestStoreOperation_InvalidMissionDuration(t *testing.T) { writer.WriteField("filename", "invalid_duration_test") writer.WriteField("tag", "coop") - fileWriter, _ := writer.CreateFormFile("file", "test.gz") + fileWriter, _ := writer.CreateFormFile("file", "test.json.gz") gw := gzip.NewWriter(fileWriter) gw.Write([]byte(`{"test": "data"}`)) gw.Close() @@ -1725,7 +1725,7 @@ func TestStoreOperation_WrongSecret(t *testing.T) { writer.WriteField("filename", "wrong_secret_test") writer.WriteField("tag", "coop") - fileWriter, _ := writer.CreateFormFile("file", "test.gz") + fileWriter, _ := writer.CreateFormFile("file", "test.json.gz") gw := gzip.NewWriter(fileWriter) gw.Write([]byte(`{"test": "data"}`)) gw.Close() diff --git a/internal/server/integration_test.go b/internal/server/integration_test.go index d99f79d9..c4fcae1b 100644 --- a/internal/server/integration_test.go +++ b/internal/server/integration_test.go @@ -97,7 +97,7 @@ func TestIntegration_ConversionAndPlayback(t *testing.T) { } // Write gzipped JSON file - jsonPath := filepath.Join(dataDir, "test_integration.gz") + jsonPath := filepath.Join(dataDir, "test_integration.json.gz") writeTestGzippedJSON(t, jsonPath, testRecording) // Store operation in database diff --git a/internal/storage/json.go b/internal/storage/json.go index 20b1be93..b5ef6d47 100644 --- a/internal/storage/json.go +++ b/internal/storage/json.go @@ -87,7 +87,7 @@ func (e *JSONEngine) Convert(ctx context.Context, jsonPath, outputPath string) e func (e *JSONEngine) loadJSON(filename string) (map[string]interface{}, error) { // Try gzipped first - path := filepath.Join(e.dataDir, filename+".gz") + path := filepath.Join(e.dataDir, filename+".json.gz") if _, err := os.Stat(path); err == nil { return e.loadGzipJSON(path) } diff --git a/internal/storage/json_test.go b/internal/storage/json_test.go index cc071ddb..cc6a874e 100644 --- a/internal/storage/json_test.go +++ b/internal/storage/json_test.go @@ -80,7 +80,7 @@ func TestJSONEngineGetManifestGzipped(t *testing.T) { ] }` - gzPath := filepath.Join(dir, "gztest.gz") + gzPath := filepath.Join(dir, "gztest.json.gz") f, err := os.Create(gzPath) require.NoError(t, err) @@ -191,7 +191,7 @@ func TestJSONEngineInvalidGzip(t *testing.T) { dir := t.TempDir() // Create a file with .gz extension but invalid gzip data - invalidGzPath := filepath.Join(dir, "invalid.gz") + invalidGzPath := filepath.Join(dir, "invalid.json.gz") err := os.WriteFile(invalidGzPath, []byte("not valid gzip data"), 0644) require.NoError(t, err) @@ -204,7 +204,7 @@ func TestJSONEngineInvalidJSONInGzip(t *testing.T) { dir := t.TempDir() // Create gzipped file with invalid JSON content - gzPath := filepath.Join(dir, "badjson.gz") + gzPath := filepath.Join(dir, "badjson.json.gz") f, err := os.Create(gzPath) require.NoError(t, err) From 32f6572473a2e0ec24c23ee5d1d2a1def5b1c1c6 Mon Sep 17 00:00:00 2001 From: Florian Kinder Date: Tue, 3 Feb 2026 21:25:24 +0100 Subject: [PATCH 2/3] feat: add migration script for filename extension handling Migrates existing deployments from old naming convention to new: - DB: strips .json suffix from filenames - Filesystem: renames output directories (foo.json/ -> foo/) --- scripts/migrate-filenames.sh | 62 ++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100755 scripts/migrate-filenames.sh diff --git a/scripts/migrate-filenames.sh b/scripts/migrate-filenames.sh new file mode 100755 index 00000000..66cfbe4a --- /dev/null +++ b/scripts/migrate-filenames.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Migration script: Strip .json suffix from filenames +# Run this once after upgrading from old naming convention +# +# Old: DB stores "foo.json", output dirs as "foo.json/" +# New: DB stores "foo", output dirs as "foo/" + +set -e + +DB="${OCAP_DB:-data.db}" +DATA_DIR="${OCAP_DATA:-data}" + +# Check sqlite3 is available +if ! command -v sqlite3 &> /dev/null; then + echo "Error: sqlite3 is required" + exit 1 +fi + +# Check database exists +if [ ! -f "$DB" ]; then + echo "Error: Database not found: $DB" + echo "Set OCAP_DB to your database path" + exit 1 +fi + +echo "Migrating database: $DB" +echo "Data directory: $DATA_DIR" +echo + +# Count affected rows +COUNT=$(sqlite3 "$DB" "SELECT COUNT(*) FROM operations WHERE filename LIKE '%.json'") +echo "Found $COUNT filenames with .json suffix" + +if [ "$COUNT" -eq 0 ]; then + echo "Nothing to migrate in database" +else + # Update database + sqlite3 "$DB" "UPDATE operations SET filename = SUBSTR(filename, 1, LENGTH(filename) - 5) WHERE filename LIKE '%.json'" + echo "Database updated" +fi + +# Rename output directories +if [ -d "$DATA_DIR" ]; then + RENAMED=0 + for dir in "$DATA_DIR"/*.json; do + [ -d "$dir" ] || continue + newdir="${dir%.json}" + if [ -d "$newdir" ]; then + echo "Warning: Both exist, skipping: $dir -> $newdir" + else + mv "$dir" "$newdir" + echo "Renamed: $dir -> $newdir" + ((RENAMED++)) + fi + done + echo "Renamed $RENAMED directories" +else + echo "Data directory not found: $DATA_DIR" +fi + +echo +echo "Migration complete" From aabf27c7044a0c8c166b7f6c7a90670c9542224e Mon Sep 17 00:00:00 2001 From: Florian Kinder Date: Tue, 3 Feb 2026 21:26:29 +0100 Subject: [PATCH 3/3] chore: remove migration script --- scripts/migrate-filenames.sh | 62 ------------------------------------ 1 file changed, 62 deletions(-) delete mode 100755 scripts/migrate-filenames.sh diff --git a/scripts/migrate-filenames.sh b/scripts/migrate-filenames.sh deleted file mode 100755 index 66cfbe4a..00000000 --- a/scripts/migrate-filenames.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -# Migration script: Strip .json suffix from filenames -# Run this once after upgrading from old naming convention -# -# Old: DB stores "foo.json", output dirs as "foo.json/" -# New: DB stores "foo", output dirs as "foo/" - -set -e - -DB="${OCAP_DB:-data.db}" -DATA_DIR="${OCAP_DATA:-data}" - -# Check sqlite3 is available -if ! command -v sqlite3 &> /dev/null; then - echo "Error: sqlite3 is required" - exit 1 -fi - -# Check database exists -if [ ! -f "$DB" ]; then - echo "Error: Database not found: $DB" - echo "Set OCAP_DB to your database path" - exit 1 -fi - -echo "Migrating database: $DB" -echo "Data directory: $DATA_DIR" -echo - -# Count affected rows -COUNT=$(sqlite3 "$DB" "SELECT COUNT(*) FROM operations WHERE filename LIKE '%.json'") -echo "Found $COUNT filenames with .json suffix" - -if [ "$COUNT" -eq 0 ]; then - echo "Nothing to migrate in database" -else - # Update database - sqlite3 "$DB" "UPDATE operations SET filename = SUBSTR(filename, 1, LENGTH(filename) - 5) WHERE filename LIKE '%.json'" - echo "Database updated" -fi - -# Rename output directories -if [ -d "$DATA_DIR" ]; then - RENAMED=0 - for dir in "$DATA_DIR"/*.json; do - [ -d "$dir" ] || continue - newdir="${dir%.json}" - if [ -d "$newdir" ]; then - echo "Warning: Both exist, skipping: $dir -> $newdir" - else - mv "$dir" "$newdir" - echo "Renamed: $dir -> $newdir" - ((RENAMED++)) - fi - done - echo "Renamed $RENAMED directories" -else - echo "Data directory not found: $DATA_DIR" -fi - -echo -echo "Migration complete"