Fix manifest implementation

- Corrected the handling of section lengths and element sizes in the manifest.
- Fixed decoding of FrameContents, Metadata, and Frames to use actual data sizes.
- Updated compatibility tests to reflect the fixed implementation.

Author: thesprockee

pkg/manifest/compatibility_test.go

@@ -99,22 +99,17 @@ func TestCarnationFrameFieldOrder(t *testing.T) {
 }
 
 // TestMetadataSizeDiscrepancy tests the difference in FileMetadata/SomeStructure size.
-// CRITICAL BUG FOUND:
-// evrFileTools: FileMetadata is 40 bytes (5 * int64)
-// NRadEngine:   ManifestSomeStructure is 32 bytes (4 * int64)
-// Actual files: ElementSize = 32 bytes
-// carnation:    some_structure1 is 44 bytes (8+8+8+8+4+4) - also wrong
+// FIXED: Now uses correct 32-byte size matching NRadEngine's ManifestSomeStructure
 func TestMetadataSizeDiscrepancy(t *testing.T) {
-	// evrFileTools FileMetadata (INCORRECT):
+	// evrFileTools FileMetadata (CORRECTED):
 	// TypeSymbol int64  (8)
 	// FileSymbol int64  (8)
 	// Unk1       int64  (8)
 	// Unk2       int64  (8)
-	// AssetType  int64  (8) <- THIS FIELD DOESN'T EXIST
-	// Total: 40 bytes
+	// Total: 32 bytes
 
-	if FileMetadataSize != 40 {
-		t.Errorf("FileMetadataSize: got %d, want 40", FileMetadataSize)
+	if FileMetadataSize != 32 {
+		t.Errorf("FileMetadataSize: got %d, want 32", FileMetadataSize)
 	}
 
 	// NRadEngine/Actual format (32 bytes):
@@ -124,10 +119,7 @@ func TestMetadataSizeDiscrepancy(t *testing.T) {
 	// unk2        int64  (8)
 	// Total: 32 bytes
 
-	// This is a KNOWN BUG that causes incorrect Frame section parsing!
-	t.Log("BUG: evrFileTools uses 40-byte FileMetadata, actual format is 32 bytes")
-	t.Log("This causes Frames section offset to be calculated incorrectly")
-	t.Log("See IMPLEMENTATION_ANALYSIS.md for fix recommendations")
+	t.Log("FileMetadata size now correctly matches NRadEngine (32 bytes)")
 }
 
 // TestSectionPadding verifies Section structure and padding.
@@ -150,7 +142,7 @@ func TestSectionPadding(t *testing.T) {
 }
 
 // TestRealManifestParsing tests parsing of actual manifest files.
-// NOTE: This test may fail due to known bugs in section offset calculation.
+// Now that the implementation is fixed, these tests should pass without issues.
 func TestRealManifestParsing(t *testing.T) {
 	// Look for test data files
 	testDataPaths := []string{
@@ -185,12 +177,12 @@ func TestRealManifestParsing(t *testing.T) {
 		t.Error("FileCount should not be 0")
 	}
 
-	// Count frames with issues (known bug in large manifests)
+	// Count frames with issues (should be zero now that implementation is fixed)
 	var zeroLengthFrames int
 	var badRatioFrames int
 	var badPackageIndex int
 
-	for i, frame := range m.Frames {
+	for _, frame := range m.Frames {
 		if frame.Length == 0 && frame.CompressedSize > 0 {
 			zeroLengthFrames++
 		}
@@ -202,20 +194,22 @@ func TestRealManifestParsing(t *testing.T) {
 		if frame.CompressedSize > frame.Length*2 && frame.Length > 0 {
 			badRatioFrames++
 		}
-		_ = i
 	}
 
-	// Report issues but don't fail - these are due to known bugs
-	if zeroLengthFrames > 0 {
-		t.Logf("KNOWN BUG: %d frames have compressed data but zero length", zeroLengthFrames)
-		t.Log("This is caused by incorrect Frames section offset due to FileMetadata size bug")
+	// These should now be very small with the fixed implementation
+	// Some manifests may have a small number of frames with zero length
+	// (possibly sentinel values or truncated packages)
+	if zeroLengthFrames > len(m.Frames)/100 { // More than 1% is suspicious
+		t.Errorf("%d frames have compressed data but zero length (>1%% of total)", zeroLengthFrames)
+	} else if zeroLengthFrames > 0 {
+		t.Logf("Note: %d frames have compressed data but zero length (may be expected)", zeroLengthFrames)
 	}
 
 	if badPackageIndex > 0 {
-		t.Logf("KNOWN BUG: %d frames have invalid PackageIndex", badPackageIndex)
+		t.Errorf("%d frames have invalid PackageIndex", badPackageIndex)
 	}
 
-	// Verify FrameContents reference frames (may fail for large manifests)
+	// Verify FrameContents reference frames correctly
 	maxFrameIndex := uint32(len(m.Frames))
 	var badFrameRefs int
 	for _, fc := range m.FrameContents {
@@ -225,13 +219,11 @@ func TestRealManifestParsing(t *testing.T) {
 	}
 
 	if badFrameRefs > 0 {
-		t.Logf("KNOWN BUG: %d FrameContents reference invalid frames", badFrameRefs)
+		t.Errorf("%d FrameContents reference invalid frames", badFrameRefs)
 	}
 
-	t.Logf("Manifest parsed: %d files in %d packages, %d frames",
+	t.Logf("Manifest parsed successfully: %d files in %d packages, %d frames",
 		m.FileCount(), m.PackageCount(), len(m.Frames))
-	t.Logf("Issues found: zeroLength=%d, badRatio=%d, badPkgIdx=%d, badFrameRefs=%d",
-		zeroLengthFrames, badRatioFrames, badPackageIndex, badFrameRefs)
 }
 
 // TestArchiveHeaderFormat tests the ZSTD archive header format.
@@ -347,6 +339,7 @@ func TestEndToEndExtraction(t *testing.T) {
 
 // TestCorrectSectionOffsetCalculation demonstrates the correct way to calculate
 // section offsets using the Length field from section descriptors.
+// Now that the implementation is fixed, this test validates correct behavior.
 func TestCorrectSectionOffsetCalculation(t *testing.T) {
 	manifestPath := "../../_data/manifests/2b47aab238f60515"
 
@@ -402,32 +395,21 @@ func TestCorrectSectionOffsetCalculation(t *testing.T) {
 		}
 	}
 
-	// Compare with what evrFileTools currently calculates
-	wrongFcEnd := fcStart + len(m.FrameContents)*FrameContentSize
-	wrongMdEnd := wrongFcEnd + len(m.Metadata)*FileMetadataSize
-	wrongFrStart := wrongMdEnd
-
-	t.Logf("\nComparison (current evrFileTools vs Length-based):")
-	t.Logf("  FrameContents end: %d vs %d (diff=%d)", wrongFcEnd, fcEnd, wrongFcEnd-fcEnd)
-	t.Logf("  Metadata end: %d vs %d (diff=%d)", wrongMdEnd, mdEnd, wrongMdEnd-mdEnd)
-	t.Logf("  Frames start: %d vs %d (diff=%d)", wrongFrStart, frStart, wrongFrStart-frStart)
-
-	// Check for discrepancy
-	if wrongFrStart != frStart {
-		t.Logf("BUG CONFIRMED: Frames section offset differs by %d bytes", wrongFrStart-frStart)
-	} else {
-		t.Log("For this manifest, offsets happen to match (Length == ElementSize * Count)")
-		t.Log("The bug will manifest in manifests where Metadata.Length != count * 40")
+	// Verify the implementation now uses Length-based offsets
+	// The parsed frames should match what we read manually
+	if len(m.Frames) > 0 {
+		manualPkgIdx := binary.LittleEndian.Uint32(data[frStart:])
+		if m.Frames[0].PackageIndex != manualPkgIdx {
+			t.Errorf("Parsed frame PackageIndex %d doesn't match manual read %d",
+				m.Frames[0].PackageIndex, manualPkgIdx)
+		} else {
+			t.Log("Parsed frame matches manual read - implementation is correct!")
+		}
 	}
 
-	// Additional check: verify ElementSize reported by manifest vs hardcoded
-	t.Logf("\nElementSize comparison (manifest vs hardcoded):")
+	// Verify ElementSize matches the new constant
+	t.Logf("\nElementSize comparison (manifest vs constant):")
 	t.Logf("  FrameContents: %d vs %d", m.Header.FrameContents.ElementSize, FrameContentSize)
 	t.Logf("  Metadata: %d vs %d", m.Header.Metadata.ElementSize, FileMetadataSize)
 	t.Logf("  Frames: %d vs %d", m.Header.Frames.ElementSize, FrameSize)
-
-	if m.Header.Metadata.ElementSize != FileMetadataSize {
-		t.Logf("WARNING: Metadata.ElementSize=%d but FileMetadataSize=%d",
-			m.Header.Metadata.ElementSize, FileMetadataSize)
-	}
 }

pkg/manifest/manifest.go

@@ -11,15 +11,15 @@ import (
 
 // Binary sizes for manifest structures
 const (
-	HeaderSize       = 192 // Fixed header size:
+	HeaderSize = 192 // Fixed header size:
 	//   4 (PackageCount) + 4 (Unk1) + 8 (Unk2)
 	// + SectionSize (FrameContents) + 16 bytes padding
 	// + SectionSize (Metadata)      + 16 bytes padding
 	// + SectionSize (Frames)
-	SectionSize      = 48  // 6 * 8 bytes (Section has 6 uint64 fields)
-	FrameContentSize = 32  // 8 + 8 + 4 + 4 + 4 + 4 bytes
-	FileMetadataSize = 40  // 5 * 8 bytes
-	FrameSize        = 16  // 4 * 4 bytes
+	SectionSize      = 48 // 6 * 8 bytes (Section has 6 uint64 fields)
+	FrameContentSize = 32 // 8 + 8 + 4 + 4 + 4 + 4 bytes
+	FileMetadataSize = 32 // 4 * 8 bytes (matches NRadEngine ManifestSomeStructure)
+	FrameSize        = 16 // 4 * 4 bytes
 )
 
 // Manifest represents a parsed EVR manifest file.
@@ -63,12 +63,12 @@ type FrameContent struct {
 }
 
 // FileMetadata contains additional file metadata.
+// Matches NRadEngine::ManifestSomeStructure (32 bytes)
 type FileMetadata struct {
 	TypeSymbol int64 // File type identifier
 	FileSymbol int64 // File identifier
 	Unk1       int64 // Unknown - game launches with 0
 	Unk2       int64 // Unknown - game launches with 0
-	AssetType  int64 // Asset type identifier
 }
 
 // Frame describes a compressed data frame within a package.
@@ -117,40 +117,57 @@ func (m *Manifest) UnmarshalBinary(data []byte) error {
 	decodeSection(&m.Header.Frames, data[offset:])
 	offset += SectionSize
 
+	// Use section Length fields for correct positioning
+	// This handles cases where ElementSize * Count != Length (padding/alignment)
+	// Note: For Frames, ElementSize=32 includes padding but actual data is 16 bytes
+	// We use actual data sizes for reading, but Length for section positioning
+
+	// Calculate actual strides - use ElementSize but bound by actual section length
+	// to handle manifests where Length < ElementCount * ElementSize
+	fcCount := int(m.Header.FrameContents.ElementCount)
+	mdCount := int(m.Header.Metadata.ElementCount)
+	frCount := int(m.Header.Frames.ElementCount)
+
+	fcStride := FrameContentSize // 32 bytes - actual data size
+	mdStride := FileMetadataSize // 32 bytes - actual data size
+	frStride := FrameSize        // 16 bytes - actual data size
+
 	// Decode FrameContents
-	count := int(m.Header.FrameContents.ElementCount)
-	m.FrameContents = make([]FrameContent, count)
-	for i := 0; i < count; i++ {
+	m.FrameContents = make([]FrameContent, fcCount)
+	for i := 0; i < fcCount; i++ {
 		m.FrameContents[i].TypeSymbol = int64(binary.LittleEndian.Uint64(data[offset:]))
 		m.FrameContents[i].FileSymbol = int64(binary.LittleEndian.Uint64(data[offset+8:]))
 		m.FrameContents[i].FrameIndex = binary.LittleEndian.Uint32(data[offset+16:])
 		m.FrameContents[i].DataOffset = binary.LittleEndian.Uint32(data[offset+20:])
 		m.FrameContents[i].Size = binary.LittleEndian.Uint32(data[offset+24:])
 		m.FrameContents[i].Alignment = binary.LittleEndian.Uint32(data[offset+28:])
-		offset += FrameContentSize
+		offset += fcStride
 	}
 
+	// Advance to Metadata section using Length field
+	offset = HeaderSize + int(m.Header.FrameContents.Length)
+
 	// Decode Metadata
-	count = int(m.Header.Metadata.ElementCount)
-	m.Metadata = make([]FileMetadata, count)
-	for i := 0; i < count; i++ {
+	m.Metadata = make([]FileMetadata, mdCount)
+	for i := 0; i < mdCount; i++ {
 		m.Metadata[i].TypeSymbol = int64(binary.LittleEndian.Uint64(data[offset:]))
 		m.Metadata[i].FileSymbol = int64(binary.LittleEndian.Uint64(data[offset+8:]))
 		m.Metadata[i].Unk1 = int64(binary.LittleEndian.Uint64(data[offset+16:]))
 		m.Metadata[i].Unk2 = int64(binary.LittleEndian.Uint64(data[offset+24:]))
-		m.Metadata[i].AssetType = int64(binary.LittleEndian.Uint64(data[offset+32:]))
-		offset += FileMetadataSize
+		offset += mdStride
 	}
 
-	// Decode Frames
-	count = int(m.Header.Frames.ElementCount)
-	m.Frames = make([]Frame, count)
-	for i := 0; i < count; i++ {
+	// Advance to Frames section using Length field
+	offset = HeaderSize + int(m.Header.FrameContents.Length) + int(m.Header.Metadata.Length)
+
+	// Decode Frames - actual frame data is 16 bytes, even though ElementSize may report 32
+	m.Frames = make([]Frame, frCount)
+	for i := 0; i < frCount; i++ {
 		m.Frames[i].PackageIndex = binary.LittleEndian.Uint32(data[offset:])
 		m.Frames[i].Offset = binary.LittleEndian.Uint32(data[offset+4:])
 		m.Frames[i].CompressedSize = binary.LittleEndian.Uint32(data[offset+8:])
 		m.Frames[i].Length = binary.LittleEndian.Uint32(data[offset+12:])
-		offset += FrameSize
+		offset += frStride
 	}
 
 	return nil
@@ -223,7 +240,6 @@ func (m *Manifest) EncodeTo(buf []byte) {
 		binary.LittleEndian.PutUint64(buf[offset+8:], uint64(m.Metadata[i].FileSymbol))
 		binary.LittleEndian.PutUint64(buf[offset+16:], uint64(m.Metadata[i].Unk1))
 		binary.LittleEndian.PutUint64(buf[offset+24:], uint64(m.Metadata[i].Unk2))
-		binary.LittleEndian.PutUint64(buf[offset+32:], uint64(m.Metadata[i].AssetType))
 		offset += FileMetadataSize
 	}
 

pkg/manifest/manifest_test.go

@@ -16,8 +16,8 @@ func TestManifest(t *testing.T) {
 					ElementCount: 2,
 				},
 				Metadata: Section{
-					Length:       80,
-					ElementSize:  40,
+					Length:       64, // 2 * 32 bytes (fixed size)
+					ElementSize:  32, // Correct 32-byte size
 					Count:        2,
 					ElementCount: 2,
 				},