Skip to content

fix: normalize filename handling with migration script #120

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
fank merged 3 commits into from
Feb 3, 2026
Merged

fix: normalize filename handling with migration script #120

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1f50670
fix: normalize filename handling to strip both .json and .gz extensions
fank Feb 3, 2026
32f6572
feat: add migration script for filename extension handling
fank Feb 3, 2026
aabf27c
chore: remove migration script
fank Feb 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion cmd/ocap-webserver/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,11 +123,14 @@ func showConversionStatus(ctx context.Context, repo *server.RepoOperation) error
}

func convertSingleFile(ctx context.Context, repo *server.RepoOperation, inputFile, dataDir string, chunkSize uint32, format string) error {
// Determine filename - only strip .gz to match database filename format
// Determine filename - strip .gz and .json to get base name
baseName := filepath.Base(inputFile)
if ext := filepath.Ext(baseName); ext == ".gz" {
baseName = baseName[:len(baseName)-len(ext)]
}
if ext := filepath.Ext(baseName); ext == ".json" {
baseName = baseName[:len(baseName)-len(ext)]
}

// Check if operation exists in database - if so, use worker for consistent behavior
if op, err := repo.GetByFilename(ctx, baseName); err == nil && op != nil {
Expand Down
12 changes: 6 additions & 6 deletions cmd/ocap-webserver/main_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -281,8 +281,8 @@ func TestConvertSingleFile(t *testing.T) {
err = convertSingleFile(ctx, repo, inputPath, dataDir, 300, "protobuf")
require.NoError(t, err)

// Verify output was created (keeps .json suffix to match database filename)
outputDir := filepath.Join(dataDir, "test_mission.json")
// Verify output was created
outputDir := filepath.Join(dataDir, "test_mission")
_, err = os.Stat(filepath.Join(outputDir, "manifest.pb"))
require.NoError(t, err)
}
Expand Down Expand Up @@ -359,7 +359,7 @@ func TestConvertSingleFile_WithDatabaseEntry(t *testing.T) {
WorldName: "Stratis",
MissionName: "Test Op",
MissionDuration: 10,
Filename: "db_test.json",
Filename: "db_test",
Date: "2024-01-01",
StorageFormat: "json",
ConversionStatus: "pending",
Expand All @@ -375,12 +375,12 @@ func TestConvertSingleFile_WithDatabaseEntry(t *testing.T) {
require.NoError(t, err)

// Verify output was created
outputDir := filepath.Join(dataDir, "db_test.json")
outputDir := filepath.Join(dataDir, "db_test")
_, err = os.Stat(filepath.Join(outputDir, "manifest.pb"))
require.NoError(t, err)

// Verify database was updated
result, err := repo.GetByFilename(ctx, "db_test.json")
result, err := repo.GetByFilename(ctx, "db_test")
require.NoError(t, err)
assert.Equal(t, "completed", result.ConversionStatus)
assert.Equal(t, "protobuf", result.StorageFormat)
Expand Down Expand Up @@ -449,7 +449,7 @@ func TestConvertAll_WithOperations(t *testing.T) {
}`

// Write gzipped JSON
jsonPath := filepath.Join(dataDir, "test_op.gz")
jsonPath := filepath.Join(dataDir, "test_op.json.gz")
f, err := os.Create(jsonPath)
require.NoError(t, err)
gw := gzip.NewWriter(f)
Expand Down
2 changes: 1 addition & 1 deletion internal/conversion/worker.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,7 @@ func (w *Worker) convertOperation(ctx context.Context, op Operation) error {
}

// Determine paths
jsonPath := filepath.Join(w.dataDir, op.Filename+".gz")
jsonPath := filepath.Join(w.dataDir, op.Filename+".json.gz")
outputPath := filepath.Join(w.dataDir, op.Filename)

// Check if JSON file exists
Expand Down
20 changes: 10 additions & 10 deletions internal/conversion/worker_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ func TestWorker_ConvertOne(t *testing.T) {
}`

// Write gzipped JSON file
jsonPath := filepath.Join(dir, "test_mission.gz")
jsonPath := filepath.Join(dir, "test_mission.json.gz")
f, err := os.Create(jsonPath)
assert.NoError(t, err)
gw := gzip.NewWriter(f)
Expand Down Expand Up @@ -161,7 +161,7 @@ func TestWorker_ProcessOnce(t *testing.T) {

// Write gzipped JSON files
for _, name := range []string{"mission1", "mission2"} {
jsonPath := filepath.Join(dir, name+".gz")
jsonPath := filepath.Join(dir, name+".json.gz")
f, _ := os.Create(jsonPath)
gw := gzip.NewWriter(f)
gw.Write([]byte(testData))
Expand Down Expand Up @@ -303,7 +303,7 @@ func TestTriggerConversion(t *testing.T) {
}`

// Write gzipped JSON file
jsonPath := filepath.Join(dir, "trigger_mission.gz")
jsonPath := filepath.Join(dir, "trigger_mission.json.gz")
f, err := os.Create(jsonPath)
assert.NoError(t, err)
gw := gzip.NewWriter(f)
Expand Down Expand Up @@ -397,7 +397,7 @@ func TestWorker_FlatBuffersFormat(t *testing.T) {
}`

// Write gzipped JSON file
jsonPath := filepath.Join(dir, "fb_test.gz")
jsonPath := filepath.Join(dir, "fb_test.json.gz")
f, err := os.Create(jsonPath)
assert.NoError(t, err)
gw := gzip.NewWriter(f)
Expand Down Expand Up @@ -442,7 +442,7 @@ func TestWorker_ContextCancellation(t *testing.T) {

// Write multiple gzipped JSON files
for i := 1; i <= 3; i++ {
jsonPath := filepath.Join(dir, fmt.Sprintf("cancel_%d.gz", i))
jsonPath := filepath.Join(dir, fmt.Sprintf("cancel_%d.json.gz", i))
f, _ := os.Create(jsonPath)
gw := gzip.NewWriter(f)
gw.Write([]byte(testData))
Expand Down Expand Up @@ -613,7 +613,7 @@ func TestConvertOperation_UpdateConvertingStatusError(t *testing.T) {

// Create test JSON file
testData := `{"worldName": "test", "missionName": "Test", "endFrame": 5, "captureDelay": 1, "entities": [], "events": [], "times": []}`
jsonPath := filepath.Join(dir, "test.gz")
jsonPath := filepath.Join(dir, "test.json.gz")
f, _ := os.Create(jsonPath)
gw := gzip.NewWriter(f)
gw.Write([]byte(testData))
Expand All @@ -640,7 +640,7 @@ func TestConvertOperation_UpdateStorageFormatError(t *testing.T) {

// Create test JSON file
testData := `{"worldName": "test", "missionName": "Test", "endFrame": 5, "captureDelay": 1, "entities": [], "events": [], "times": []}`
jsonPath := filepath.Join(dir, "test.gz")
jsonPath := filepath.Join(dir, "test.json.gz")
f, _ := os.Create(jsonPath)
gw := gzip.NewWriter(f)
gw.Write([]byte(testData))
Expand All @@ -666,7 +666,7 @@ func TestConvertOperation_UpdateCompletedStatusError(t *testing.T) {

// Create test JSON file
testData := `{"worldName": "test", "missionName": "Test", "endFrame": 5, "captureDelay": 1, "entities": [], "events": [], "times": []}`
jsonPath := filepath.Join(dir, "test.gz")
jsonPath := filepath.Join(dir, "test.json.gz")
f, _ := os.Create(jsonPath)
gw := gzip.NewWriter(f)
gw.Write([]byte(testData))
Expand All @@ -693,7 +693,7 @@ func TestConvertOperation_UpdateDurationError(t *testing.T) {

// Create test JSON file
testData := `{"worldName": "test", "missionName": "Test", "endFrame": 5, "captureDelay": 1, "entities": [], "events": [], "times": []}`
jsonPath := filepath.Join(dir, "test.gz")
jsonPath := filepath.Join(dir, "test.json.gz")
f, _ := os.Create(jsonPath)
gw := gzip.NewWriter(f)
gw.Write([]byte(testData))
Expand All @@ -720,7 +720,7 @@ func TestConvertOperation_InvalidStorageFormat(t *testing.T) {

// Create test JSON file
testData := `{"worldName": "test", "missionName": "Test", "endFrame": 5, "captureDelay": 1, "entities": [], "events": [], "times": []}`
jsonPath := filepath.Join(dir, "test.gz")
jsonPath := filepath.Join(dir, "test.json.gz")
f, _ := os.Create(jsonPath)
gw := gzip.NewWriter(f)
gw.Write([]byte(testData))
Expand Down
10 changes: 6 additions & 4 deletions internal/server/handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -362,7 +362,9 @@ func (h *Handler) StoreOperation(c echo.Context) error {
return echo.ErrForbidden
}

filename := strings.TrimSuffix(filepath.Base(c.FormValue("filename")), ".gz")
filename := filepath.Base(c.FormValue("filename"))
filename = strings.TrimSuffix(filename, ".gz")
filename = strings.TrimSuffix(filename, ".json")

op := Operation{
WorldName: c.FormValue("worldName"),
Expand Down Expand Up @@ -391,7 +393,7 @@ func (h *Handler) StoreOperation(c echo.Context) error {
}
defer file.Close()

writer, err := os.Create(filepath.Join(h.setting.Data, filename+".gz"))
writer, err := os.Create(filepath.Join(h.setting.Data, filename+".json.gz"))
if err != nil {
return err
}
Expand All @@ -414,7 +416,7 @@ func (h *Handler) GetCapture(c echo.Context) error {
return err
}

upath := filepath.Join(h.setting.Data, filepath.Base(name+".gz"))
upath := filepath.Join(h.setting.Data, filepath.Base(name+".json.gz"))

c.Response().Header().Set("Content-Encoding", "gzip")
c.Response().Header().Set("Content-Type", "application/json")
Expand All @@ -428,7 +430,7 @@ func (h *Handler) GetCaptureFile(c echo.Context) error {
return err
}

filename := filepath.Base(name + ".gz")
filename := filepath.Base(name + ".json.gz")

c.Response().Header().Set("Content-Disposition", "attachment;filename=\""+filename+"\"")

Expand Down
20 changes: 10 additions & 10 deletions internal/server/handler_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -232,7 +232,7 @@ func TestGetOperationManifest(t *testing.T) {
"times": [],
"Ede": []
}`
testDataPath := filepath.Join(dataDir, "test_mission.gz")
testDataPath := filepath.Join(dataDir, "test_mission.json.gz")
err = writeGzipped(testDataPath, []byte(testData))
assert.NoError(t, err)

Expand Down Expand Up @@ -731,7 +731,7 @@ func TestStoreOperation(t *testing.T) {
writer.WriteField("tag", "coop")

// Create file part
fileWriter, err := writer.CreateFormFile("file", "test_upload.gz")
fileWriter, err := writer.CreateFormFile("file", "test_upload.json.gz")
require.NoError(t, err)

// Write gzipped JSON data
Expand All @@ -752,7 +752,7 @@ func TestStoreOperation(t *testing.T) {
assert.Equal(t, http.StatusOK, rec.Code)

// Verify file was created
_, err = os.Stat(filepath.Join(dataDir, "test_upload.gz"))
_, err = os.Stat(filepath.Join(dataDir, "test_upload.json.gz"))
assert.NoError(t, err)
})

Expand Down Expand Up @@ -798,7 +798,7 @@ func TestGetCapture(t *testing.T) {

// Create test gzipped file
testData := `{"test": "capture data"}`
testPath := filepath.Join(dataDir, "test_capture.gz")
testPath := filepath.Join(dataDir, "test_capture.json.gz")
err = writeGzipped(testPath, []byte(testData))
require.NoError(t, err)

Expand Down Expand Up @@ -841,7 +841,7 @@ func TestGetCaptureFile(t *testing.T) {
require.NoError(t, err)

// Create test gzipped file
testPath := filepath.Join(dataDir, "download_test.gz")
testPath := filepath.Join(dataDir, "download_test.json.gz")
err = writeGzipped(testPath, []byte(`{"download": "test"}`))
require.NoError(t, err)

Expand All @@ -859,7 +859,7 @@ func TestGetCaptureFile(t *testing.T) {
err = hdlr.GetCaptureFile(c)
assert.NoError(t, err)
assert.Contains(t, rec.Header().Get("Content-Disposition"), "attachment")
assert.Contains(t, rec.Header().Get("Content-Disposition"), "download_test.gz")
assert.Contains(t, rec.Header().Get("Content-Disposition"), "download_test.json.gz")
}

func TestGetMapTitle(t *testing.T) {
Expand Down Expand Up @@ -1107,7 +1107,7 @@ func TestStoreOperationWithConversionTrigger(t *testing.T) {
writer.WriteField("filename", "trigger_test")
writer.WriteField("tag", "coop")

fileWriter, _ := writer.CreateFormFile("file", "trigger_test.gz")
fileWriter, _ := writer.CreateFormFile("file", "trigger_test.json.gz")
gw := gzip.NewWriter(fileWriter)
gw.Write([]byte(`{"test": "trigger"}`))
gw.Close()
Expand Down Expand Up @@ -1415,7 +1415,7 @@ func TestGetOperationManifest_JSONFormat(t *testing.T) {
"entities": [],
"events": []
}`
err = writeGzipped(filepath.Join(dataDir, "json_manifest_test.gz"), []byte(testJSON))
err = writeGzipped(filepath.Join(dataDir, "json_manifest_test.json.gz"), []byte(testJSON))
require.NoError(t, err)

// Register JSON engine
Expand Down Expand Up @@ -1680,7 +1680,7 @@ func TestStoreOperation_InvalidMissionDuration(t *testing.T) {
writer.WriteField("filename", "invalid_duration_test")
writer.WriteField("tag", "coop")

fileWriter, _ := writer.CreateFormFile("file", "test.gz")
fileWriter, _ := writer.CreateFormFile("file", "test.json.gz")
gw := gzip.NewWriter(fileWriter)
gw.Write([]byte(`{"test": "data"}`))
gw.Close()
Expand Down Expand Up @@ -1725,7 +1725,7 @@ func TestStoreOperation_WrongSecret(t *testing.T) {
writer.WriteField("filename", "wrong_secret_test")
writer.WriteField("tag", "coop")

fileWriter, _ := writer.CreateFormFile("file", "test.gz")
fileWriter, _ := writer.CreateFormFile("file", "test.json.gz")
gw := gzip.NewWriter(fileWriter)
gw.Write([]byte(`{"test": "data"}`))
gw.Close()
Expand Down
2 changes: 1 addition & 1 deletion internal/server/integration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ func TestIntegration_ConversionAndPlayback(t *testing.T) {
}

// Write gzipped JSON file
jsonPath := filepath.Join(dataDir, "test_integration.gz")
jsonPath := filepath.Join(dataDir, "test_integration.json.gz")
writeTestGzippedJSON(t, jsonPath, testRecording)

// Store operation in database
Expand Down
2 changes: 1 addition & 1 deletion internal/storage/json.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ func (e *JSONEngine) Convert(ctx context.Context, jsonPath, outputPath string) e

func (e *JSONEngine) loadJSON(filename string) (map[string]interface{}, error) {
// Try gzipped first
path := filepath.Join(e.dataDir, filename+".gz")
path := filepath.Join(e.dataDir, filename+".json.gz")
if _, err := os.Stat(path); err == nil {
return e.loadGzipJSON(path)
}
Expand Down
6 changes: 3 additions & 3 deletions internal/storage/json_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ func TestJSONEngineGetManifestGzipped(t *testing.T) {
]
}`

gzPath := filepath.Join(dir, "gztest.gz")
gzPath := filepath.Join(dir, "gztest.json.gz")
f, err := os.Create(gzPath)
require.NoError(t, err)

Expand Down Expand Up @@ -191,7 +191,7 @@ func TestJSONEngineInvalidGzip(t *testing.T) {
dir := t.TempDir()

// Create a file with .gz extension but invalid gzip data
invalidGzPath := filepath.Join(dir, "invalid.gz")
invalidGzPath := filepath.Join(dir, "invalid.json.gz")
err := os.WriteFile(invalidGzPath, []byte("not valid gzip data"), 0644)
require.NoError(t, err)

Expand All @@ -204,7 +204,7 @@ func TestJSONEngineInvalidJSONInGzip(t *testing.T) {
dir := t.TempDir()

// Create gzipped file with invalid JSON content
gzPath := filepath.Join(dir, "badjson.gz")
gzPath := filepath.Join(dir, "badjson.json.gz")
f, err := os.Create(gzPath)
require.NoError(t, err)

Expand Down
Loading