diff --git a/cmd/ocap-webserver/main.go b/cmd/ocap-webserver/main.go index f4b34e61..255a7e1d 100644 --- a/cmd/ocap-webserver/main.go +++ b/cmd/ocap-webserver/main.go @@ -123,11 +123,14 @@ func showConversionStatus(ctx context.Context, repo *server.RepoOperation) error } func convertSingleFile(ctx context.Context, repo *server.RepoOperation, inputFile, dataDir string, chunkSize uint32, format string) error { - // Determine filename - only strip .gz to match database filename format + // Determine filename - strip .gz and .json to get base name baseName := filepath.Base(inputFile) if ext := filepath.Ext(baseName); ext == ".gz" { baseName = baseName[:len(baseName)-len(ext)] } + if ext := filepath.Ext(baseName); ext == ".json" { + baseName = baseName[:len(baseName)-len(ext)] + } // Check if operation exists in database - if so, use worker for consistent behavior if op, err := repo.GetByFilename(ctx, baseName); err == nil && op != nil { diff --git a/cmd/ocap-webserver/main_test.go b/cmd/ocap-webserver/main_test.go index 02b4ba8d..035a1113 100644 --- a/cmd/ocap-webserver/main_test.go +++ b/cmd/ocap-webserver/main_test.go @@ -281,8 +281,8 @@ func TestConvertSingleFile(t *testing.T) { err = convertSingleFile(ctx, repo, inputPath, dataDir, 300, "protobuf") require.NoError(t, err) - // Verify output was created (keeps .json suffix to match database filename) - outputDir := filepath.Join(dataDir, "test_mission.json") + // Verify output was created + outputDir := filepath.Join(dataDir, "test_mission") _, err = os.Stat(filepath.Join(outputDir, "manifest.pb")) require.NoError(t, err) } @@ -359,7 +359,7 @@ func TestConvertSingleFile_WithDatabaseEntry(t *testing.T) { WorldName: "Stratis", MissionName: "Test Op", MissionDuration: 10, - Filename: "db_test.json", + Filename: "db_test", Date: "2024-01-01", StorageFormat: "json", ConversionStatus: "pending", @@ -375,12 +375,12 @@ func TestConvertSingleFile_WithDatabaseEntry(t *testing.T) { require.NoError(t, err) // Verify output was created - outputDir := filepath.Join(dataDir, "db_test.json") + outputDir := filepath.Join(dataDir, "db_test") _, err = os.Stat(filepath.Join(outputDir, "manifest.pb")) require.NoError(t, err) // Verify database was updated - result, err := repo.GetByFilename(ctx, "db_test.json") + result, err := repo.GetByFilename(ctx, "db_test") require.NoError(t, err) assert.Equal(t, "completed", result.ConversionStatus) assert.Equal(t, "protobuf", result.StorageFormat) @@ -449,7 +449,7 @@ func TestConvertAll_WithOperations(t *testing.T) { }` // Write gzipped JSON - jsonPath := filepath.Join(dataDir, "test_op.gz") + jsonPath := filepath.Join(dataDir, "test_op.json.gz") f, err := os.Create(jsonPath) require.NoError(t, err) gw := gzip.NewWriter(f) diff --git a/internal/conversion/worker.go b/internal/conversion/worker.go index 20a3e8b0..b2c3c555 100644 --- a/internal/conversion/worker.go +++ b/internal/conversion/worker.go @@ -174,7 +174,7 @@ func (w *Worker) convertOperation(ctx context.Context, op Operation) error { } // Determine paths - jsonPath := filepath.Join(w.dataDir, op.Filename+".gz") + jsonPath := filepath.Join(w.dataDir, op.Filename+".json.gz") outputPath := filepath.Join(w.dataDir, op.Filename) // Check if JSON file exists diff --git a/internal/conversion/worker_test.go b/internal/conversion/worker_test.go index 5be0d24c..871b3025 100644 --- a/internal/conversion/worker_test.go +++ b/internal/conversion/worker_test.go @@ -112,7 +112,7 @@ func TestWorker_ConvertOne(t *testing.T) { }` // Write gzipped JSON file - jsonPath := filepath.Join(dir, "test_mission.gz") + jsonPath := filepath.Join(dir, "test_mission.json.gz") f, err := os.Create(jsonPath) assert.NoError(t, err) gw := gzip.NewWriter(f) @@ -161,7 +161,7 @@ func TestWorker_ProcessOnce(t *testing.T) { // Write gzipped JSON files for _, name := range []string{"mission1", "mission2"} { - jsonPath := filepath.Join(dir, name+".gz") + jsonPath := filepath.Join(dir, name+".json.gz") f, _ := os.Create(jsonPath) gw := gzip.NewWriter(f) gw.Write([]byte(testData)) @@ -303,7 +303,7 @@ func TestTriggerConversion(t *testing.T) { }` // Write gzipped JSON file - jsonPath := filepath.Join(dir, "trigger_mission.gz") + jsonPath := filepath.Join(dir, "trigger_mission.json.gz") f, err := os.Create(jsonPath) assert.NoError(t, err) gw := gzip.NewWriter(f) @@ -397,7 +397,7 @@ func TestWorker_FlatBuffersFormat(t *testing.T) { }` // Write gzipped JSON file - jsonPath := filepath.Join(dir, "fb_test.gz") + jsonPath := filepath.Join(dir, "fb_test.json.gz") f, err := os.Create(jsonPath) assert.NoError(t, err) gw := gzip.NewWriter(f) @@ -442,7 +442,7 @@ func TestWorker_ContextCancellation(t *testing.T) { // Write multiple gzipped JSON files for i := 1; i <= 3; i++ { - jsonPath := filepath.Join(dir, fmt.Sprintf("cancel_%d.gz", i)) + jsonPath := filepath.Join(dir, fmt.Sprintf("cancel_%d.json.gz", i)) f, _ := os.Create(jsonPath) gw := gzip.NewWriter(f) gw.Write([]byte(testData)) @@ -613,7 +613,7 @@ func TestConvertOperation_UpdateConvertingStatusError(t *testing.T) { // Create test JSON file testData := `{"worldName": "test", "missionName": "Test", "endFrame": 5, "captureDelay": 1, "entities": [], "events": [], "times": []}` - jsonPath := filepath.Join(dir, "test.gz") + jsonPath := filepath.Join(dir, "test.json.gz") f, _ := os.Create(jsonPath) gw := gzip.NewWriter(f) gw.Write([]byte(testData)) @@ -640,7 +640,7 @@ func TestConvertOperation_UpdateStorageFormatError(t *testing.T) { // Create test JSON file testData := `{"worldName": "test", "missionName": "Test", "endFrame": 5, "captureDelay": 1, "entities": [], "events": [], "times": []}` - jsonPath := filepath.Join(dir, "test.gz") + jsonPath := filepath.Join(dir, "test.json.gz") f, _ := os.Create(jsonPath) gw := gzip.NewWriter(f) gw.Write([]byte(testData)) @@ -666,7 +666,7 @@ func TestConvertOperation_UpdateCompletedStatusError(t *testing.T) { // Create test JSON file testData := `{"worldName": "test", "missionName": "Test", "endFrame": 5, "captureDelay": 1, "entities": [], "events": [], "times": []}` - jsonPath := filepath.Join(dir, "test.gz") + jsonPath := filepath.Join(dir, "test.json.gz") f, _ := os.Create(jsonPath) gw := gzip.NewWriter(f) gw.Write([]byte(testData)) @@ -693,7 +693,7 @@ func TestConvertOperation_UpdateDurationError(t *testing.T) { // Create test JSON file testData := `{"worldName": "test", "missionName": "Test", "endFrame": 5, "captureDelay": 1, "entities": [], "events": [], "times": []}` - jsonPath := filepath.Join(dir, "test.gz") + jsonPath := filepath.Join(dir, "test.json.gz") f, _ := os.Create(jsonPath) gw := gzip.NewWriter(f) gw.Write([]byte(testData)) @@ -720,7 +720,7 @@ func TestConvertOperation_InvalidStorageFormat(t *testing.T) { // Create test JSON file testData := `{"worldName": "test", "missionName": "Test", "endFrame": 5, "captureDelay": 1, "entities": [], "events": [], "times": []}` - jsonPath := filepath.Join(dir, "test.gz") + jsonPath := filepath.Join(dir, "test.json.gz") f, _ := os.Create(jsonPath) gw := gzip.NewWriter(f) gw.Write([]byte(testData)) diff --git a/internal/server/handler.go b/internal/server/handler.go index eb9031a8..29637db7 100644 --- a/internal/server/handler.go +++ b/internal/server/handler.go @@ -362,7 +362,9 @@ func (h *Handler) StoreOperation(c echo.Context) error { return echo.ErrForbidden } - filename := strings.TrimSuffix(filepath.Base(c.FormValue("filename")), ".gz") + filename := filepath.Base(c.FormValue("filename")) + filename = strings.TrimSuffix(filename, ".gz") + filename = strings.TrimSuffix(filename, ".json") op := Operation{ WorldName: c.FormValue("worldName"), @@ -391,7 +393,7 @@ func (h *Handler) StoreOperation(c echo.Context) error { } defer file.Close() - writer, err := os.Create(filepath.Join(h.setting.Data, filename+".gz")) + writer, err := os.Create(filepath.Join(h.setting.Data, filename+".json.gz")) if err != nil { return err } @@ -414,7 +416,7 @@ func (h *Handler) GetCapture(c echo.Context) error { return err } - upath := filepath.Join(h.setting.Data, filepath.Base(name+".gz")) + upath := filepath.Join(h.setting.Data, filepath.Base(name+".json.gz")) c.Response().Header().Set("Content-Encoding", "gzip") c.Response().Header().Set("Content-Type", "application/json") @@ -428,7 +430,7 @@ func (h *Handler) GetCaptureFile(c echo.Context) error { return err } - filename := filepath.Base(name + ".gz") + filename := filepath.Base(name + ".json.gz") c.Response().Header().Set("Content-Disposition", "attachment;filename=\""+filename+"\"") diff --git a/internal/server/handler_test.go b/internal/server/handler_test.go index c673198c..2f5f24b9 100644 --- a/internal/server/handler_test.go +++ b/internal/server/handler_test.go @@ -232,7 +232,7 @@ func TestGetOperationManifest(t *testing.T) { "times": [], "Ede": [] }` - testDataPath := filepath.Join(dataDir, "test_mission.gz") + testDataPath := filepath.Join(dataDir, "test_mission.json.gz") err = writeGzipped(testDataPath, []byte(testData)) assert.NoError(t, err) @@ -731,7 +731,7 @@ func TestStoreOperation(t *testing.T) { writer.WriteField("tag", "coop") // Create file part - fileWriter, err := writer.CreateFormFile("file", "test_upload.gz") + fileWriter, err := writer.CreateFormFile("file", "test_upload.json.gz") require.NoError(t, err) // Write gzipped JSON data @@ -752,7 +752,7 @@ func TestStoreOperation(t *testing.T) { assert.Equal(t, http.StatusOK, rec.Code) // Verify file was created - _, err = os.Stat(filepath.Join(dataDir, "test_upload.gz")) + _, err = os.Stat(filepath.Join(dataDir, "test_upload.json.gz")) assert.NoError(t, err) }) @@ -798,7 +798,7 @@ func TestGetCapture(t *testing.T) { // Create test gzipped file testData := `{"test": "capture data"}` - testPath := filepath.Join(dataDir, "test_capture.gz") + testPath := filepath.Join(dataDir, "test_capture.json.gz") err = writeGzipped(testPath, []byte(testData)) require.NoError(t, err) @@ -841,7 +841,7 @@ func TestGetCaptureFile(t *testing.T) { require.NoError(t, err) // Create test gzipped file - testPath := filepath.Join(dataDir, "download_test.gz") + testPath := filepath.Join(dataDir, "download_test.json.gz") err = writeGzipped(testPath, []byte(`{"download": "test"}`)) require.NoError(t, err) @@ -859,7 +859,7 @@ func TestGetCaptureFile(t *testing.T) { err = hdlr.GetCaptureFile(c) assert.NoError(t, err) assert.Contains(t, rec.Header().Get("Content-Disposition"), "attachment") - assert.Contains(t, rec.Header().Get("Content-Disposition"), "download_test.gz") + assert.Contains(t, rec.Header().Get("Content-Disposition"), "download_test.json.gz") } func TestGetMapTitle(t *testing.T) { @@ -1107,7 +1107,7 @@ func TestStoreOperationWithConversionTrigger(t *testing.T) { writer.WriteField("filename", "trigger_test") writer.WriteField("tag", "coop") - fileWriter, _ := writer.CreateFormFile("file", "trigger_test.gz") + fileWriter, _ := writer.CreateFormFile("file", "trigger_test.json.gz") gw := gzip.NewWriter(fileWriter) gw.Write([]byte(`{"test": "trigger"}`)) gw.Close() @@ -1415,7 +1415,7 @@ func TestGetOperationManifest_JSONFormat(t *testing.T) { "entities": [], "events": [] }` - err = writeGzipped(filepath.Join(dataDir, "json_manifest_test.gz"), []byte(testJSON)) + err = writeGzipped(filepath.Join(dataDir, "json_manifest_test.json.gz"), []byte(testJSON)) require.NoError(t, err) // Register JSON engine @@ -1680,7 +1680,7 @@ func TestStoreOperation_InvalidMissionDuration(t *testing.T) { writer.WriteField("filename", "invalid_duration_test") writer.WriteField("tag", "coop") - fileWriter, _ := writer.CreateFormFile("file", "test.gz") + fileWriter, _ := writer.CreateFormFile("file", "test.json.gz") gw := gzip.NewWriter(fileWriter) gw.Write([]byte(`{"test": "data"}`)) gw.Close() @@ -1725,7 +1725,7 @@ func TestStoreOperation_WrongSecret(t *testing.T) { writer.WriteField("filename", "wrong_secret_test") writer.WriteField("tag", "coop") - fileWriter, _ := writer.CreateFormFile("file", "test.gz") + fileWriter, _ := writer.CreateFormFile("file", "test.json.gz") gw := gzip.NewWriter(fileWriter) gw.Write([]byte(`{"test": "data"}`)) gw.Close() diff --git a/internal/server/integration_test.go b/internal/server/integration_test.go index d99f79d9..c4fcae1b 100644 --- a/internal/server/integration_test.go +++ b/internal/server/integration_test.go @@ -97,7 +97,7 @@ func TestIntegration_ConversionAndPlayback(t *testing.T) { } // Write gzipped JSON file - jsonPath := filepath.Join(dataDir, "test_integration.gz") + jsonPath := filepath.Join(dataDir, "test_integration.json.gz") writeTestGzippedJSON(t, jsonPath, testRecording) // Store operation in database diff --git a/internal/storage/json.go b/internal/storage/json.go index 20b1be93..b5ef6d47 100644 --- a/internal/storage/json.go +++ b/internal/storage/json.go @@ -87,7 +87,7 @@ func (e *JSONEngine) Convert(ctx context.Context, jsonPath, outputPath string) e func (e *JSONEngine) loadJSON(filename string) (map[string]interface{}, error) { // Try gzipped first - path := filepath.Join(e.dataDir, filename+".gz") + path := filepath.Join(e.dataDir, filename+".json.gz") if _, err := os.Stat(path); err == nil { return e.loadGzipJSON(path) } diff --git a/internal/storage/json_test.go b/internal/storage/json_test.go index cc071ddb..cc6a874e 100644 --- a/internal/storage/json_test.go +++ b/internal/storage/json_test.go @@ -80,7 +80,7 @@ func TestJSONEngineGetManifestGzipped(t *testing.T) { ] }` - gzPath := filepath.Join(dir, "gztest.gz") + gzPath := filepath.Join(dir, "gztest.json.gz") f, err := os.Create(gzPath) require.NoError(t, err) @@ -191,7 +191,7 @@ func TestJSONEngineInvalidGzip(t *testing.T) { dir := t.TempDir() // Create a file with .gz extension but invalid gzip data - invalidGzPath := filepath.Join(dir, "invalid.gz") + invalidGzPath := filepath.Join(dir, "invalid.json.gz") err := os.WriteFile(invalidGzPath, []byte("not valid gzip data"), 0644) require.NoError(t, err) @@ -204,7 +204,7 @@ func TestJSONEngineInvalidJSONInGzip(t *testing.T) { dir := t.TempDir() // Create gzipped file with invalid JSON content - gzPath := filepath.Join(dir, "badjson.gz") + gzPath := filepath.Join(dir, "badjson.json.gz") f, err := os.Create(gzPath) require.NoError(t, err)