diff --git a/Project/GNU/CLI/Makefile.am b/Project/GNU/CLI/Makefile.am
index cb11835b..0f9bfe1b 100644
--- a/Project/GNU/CLI/Makefile.am
+++ b/Project/GNU/CLI/Makefile.am
@@ -27,6 +27,9 @@ rawcooked_SOURCES = \
../../../Source/Lib/RawFrame/RawFrame.cpp \
../../../Source/Lib/TIFF/TIFF.cpp \
../../../Source/Lib/WAV/WAV.cpp \
+ ../../../Source/Lib/ThirdParty/Reed-Solomon/Galois.cpp \
+ ../../../Source/Lib/ThirdParty/Reed-Solomon/Matrix.cpp \
+ ../../../Source/Lib/ThirdParty/Reed-Solomon/Reed-Solomon.cpp \
../../../Source/Lib/ThirdParty/flac/src/libFLAC/bitmath.c \
../../../Source/Lib/ThirdParty/flac/src/libFLAC/bitreader.c \
../../../Source/Lib/ThirdParty/flac/src/libFLAC/cpu.c \
@@ -59,7 +62,8 @@ AM_CPPFLAGS = -I../../../Source \
-I../../../Source/Lib/ThirdParty/flac/src/libFLAC/include \
-I../../../Source/Lib/ThirdParty/md5 \
-I../../../Source/Lib/ThirdParty/thread-pool/include\
- -I../../../Source/Lib/ThirdParty/zlib
+ -I../../../Source/Lib/ThirdParty/zlib \
+ -I../../../Source/Lib/ThirdParty
AM_CXXFLAGS = -std=c++0x
diff --git a/Project/MSVC2017/Lib/RAWcooked_Lib.vcxproj b/Project/MSVC2017/Lib/RAWcooked_Lib.vcxproj
index 9036455b..f44cccbc 100644
--- a/Project/MSVC2017/Lib/RAWcooked_Lib.vcxproj
+++ b/Project/MSVC2017/Lib/RAWcooked_Lib.vcxproj
@@ -42,6 +42,9 @@
+
+
+
@@ -105,6 +108,9 @@
+
+
+
@@ -214,7 +220,7 @@
Level3
Disabled
WIN32;_DEBUG;_CONSOLE;FLAC__NO_DLL;FLAC__HAS_OGG=0;FLAC__NO_ASM;%(PreprocessorDefinitions)
- ../../../Source;../../../Source/Lib/ThirdParty/flac/include;../../../Source/Lib/ThirdParty/flac/src/libFLAC/include;../../../Source/Lib/ThirdParty/md5;../../../Source/Lib/ThirdParty/thread-pool/include;../../../Source/Lib/ThirdParty/zlib
+ ../../../Source;../../../Source/Lib/ThirdParty/flac/include;../../../Source/Lib/ThirdParty/flac/src/libFLAC/include;../../../Source/Lib/ThirdParty/md5;../../../Source/Lib/ThirdParty/thread-pool/include;../../../Source/Lib/ThirdParty/zlib;../../../Source/Lib/ThirdParty
true
false
MultiThreadedDebugDLL
@@ -230,7 +236,7 @@
Level3
Disabled
_DEBUG;_CONSOLE;FLAC__NO_DLL;FLAC__HAS_OGG=0;FLAC__NO_ASM;%(PreprocessorDefinitions)
- ../../../Source;../../../Source/Lib/ThirdParty/flac/include;../../../Source/Lib/ThirdParty/flac/src/libFLAC/include;../../../Source/Lib/ThirdParty/md5;../../../Source/Lib/ThirdParty/thread-pool/include;../../../Source/Lib/ThirdParty/zlib
+ ../../../Source;../../../Source/Lib/ThirdParty/flac/include;../../../Source/Lib/ThirdParty/flac/src/libFLAC/include;../../../Source/Lib/ThirdParty/md5;../../../Source/Lib/ThirdParty/thread-pool/include;../../../Source/Lib/ThirdParty/zlib;../../../Source/Lib/ThirdParty
true
false
MultiThreadedDebugDLL
@@ -248,7 +254,7 @@
true
true
WIN32;NDEBUG;_CONSOLE;FLAC__NO_DLL;FLAC__HAS_OGG=0;FLAC__NO_ASM;%(PreprocessorDefinitions)
- ../../../Source;../../../Source/Lib/ThirdParty/flac/include;../../../Source/Lib/ThirdParty/flac/src/libFLAC/include;../../../Source/Lib/ThirdParty/md5;../../../Source/Lib/ThirdParty/thread-pool/include;../../../Source/Lib/ThirdParty/zlib
+ ../../../Source;../../../Source/Lib/ThirdParty/flac/include;../../../Source/Lib/ThirdParty/flac/src/libFLAC/include;../../../Source/Lib/ThirdParty/md5;../../../Source/Lib/ThirdParty/thread-pool/include;../../../Source/Lib/ThirdParty/zlib;../../../Source/Lib/ThirdParty
true
MultiThreadedDLL
@@ -267,7 +273,7 @@
true
true
NDEBUG;_CONSOLE;FLAC__NO_DLL;FLAC__HAS_OGG=0;FLAC__NO_ASM;%(PreprocessorDefinitions)
- ../../../Source;../../../Source/Lib/ThirdParty/flac/include;../../../Source/Lib/ThirdParty/flac/src/libFLAC/include;../../../Source/Lib/ThirdParty/md5;../../../Source/Lib/ThirdParty/thread-pool/include;../../../Source/Lib/ThirdParty/zlib
+ ../../../Source;../../../Source/Lib/ThirdParty/flac/include;../../../Source/Lib/ThirdParty/flac/src/libFLAC/include;../../../Source/Lib/ThirdParty/md5;../../../Source/Lib/ThirdParty/thread-pool/include;../../../Source/Lib/ThirdParty/zlib;../../../Source/Lib/ThirdParty
true
MultiThreadedDLL
AnySuitable
diff --git a/Project/MSVC2017/Lib/RAWcooked_Lib.vcxproj.filters b/Project/MSVC2017/Lib/RAWcooked_Lib.vcxproj.filters
index 208fe363..a40dafd4 100644
--- a/Project/MSVC2017/Lib/RAWcooked_Lib.vcxproj.filters
+++ b/Project/MSVC2017/Lib/RAWcooked_Lib.vcxproj.filters
@@ -103,6 +103,12 @@
{889f93d5-50da-4606-8480-a1ec82aa200f}
+
+ {bea62b93-d3d5-4cf1-a957-65810ce3ac52}
+
+
+ {e97f8d7f-a3f3-4d1b-87c3-c17fcf86decf}
+
@@ -303,6 +309,15 @@
Header Files\HashSum
+
+ ThirdParty\Reed-Solomon
+
+
+ ThirdParty\Reed-Solomon
+
+
+ ThirdParty\Reed-Solomon
+
@@ -440,6 +455,15 @@
Source Files\HashSum
+
+ ThirdParty\Reed-Solomon
+
+
+ ThirdParty\Reed-Solomon
+
+
+ ThirdParty\Reed-Solomon
+
diff --git a/Project/MSVC2019/Lib/RAWcooked_Lib.vcxproj b/Project/MSVC2019/Lib/RAWcooked_Lib.vcxproj
index 7012af7d..5c489a4e 100644
--- a/Project/MSVC2019/Lib/RAWcooked_Lib.vcxproj
+++ b/Project/MSVC2019/Lib/RAWcooked_Lib.vcxproj
@@ -42,6 +42,9 @@
+
+
+
@@ -105,6 +108,9 @@
+
+
+
@@ -214,7 +220,7 @@
Level3
Disabled
WIN32;_DEBUG;_CONSOLE;FLAC__NO_DLL;FLAC__HAS_OGG=0;FLAC__NO_ASM;%(PreprocessorDefinitions)
- ../../../Source;../../../Source/Lib/ThirdParty/flac/include;../../../Source/Lib/ThirdParty/flac/src/libFLAC/include;../../../Source/Lib/ThirdParty/md5;../../../Source/Lib/ThirdParty/thread-pool/include;../../../Source/Lib/ThirdParty/zlib
+ ../../../Source;../../../Source/Lib/ThirdParty/flac/include;../../../Source/Lib/ThirdParty/flac/src/libFLAC/include;../../../Source/Lib/ThirdParty/md5;../../../Source/Lib/ThirdParty/thread-pool/include;../../../Source/Lib/ThirdParty/zlib;../../../Source/Lib/ThirdParty
true
false
MultiThreadedDebugDLL
@@ -230,7 +236,7 @@
Level3
Disabled
_DEBUG;_CONSOLE;FLAC__NO_DLL;FLAC__HAS_OGG=0;FLAC__NO_ASM;%(PreprocessorDefinitions)
- ../../../Source;../../../Source/Lib/ThirdParty/flac/include;../../../Source/Lib/ThirdParty/flac/src/libFLAC/include;../../../Source/Lib/ThirdParty/md5;../../../Source/Lib/ThirdParty/thread-pool/include;../../../Source/Lib/ThirdParty/zlib
+ ../../../Source;../../../Source/Lib/ThirdParty/flac/include;../../../Source/Lib/ThirdParty/flac/src/libFLAC/include;../../../Source/Lib/ThirdParty/md5;../../../Source/Lib/ThirdParty/thread-pool/include;../../../Source/Lib/ThirdParty/zlib;../../../Source/Lib/ThirdParty
true
false
MultiThreadedDebugDLL
@@ -248,7 +254,7 @@
true
true
WIN32;NDEBUG;_CONSOLE;FLAC__NO_DLL;FLAC__HAS_OGG=0;FLAC__NO_ASM;%(PreprocessorDefinitions)
- ../../../Source;../../../Source/Lib/ThirdParty/flac/include;../../../Source/Lib/ThirdParty/flac/src/libFLAC/include;../../../Source/Lib/ThirdParty/md5;../../../Source/Lib/ThirdParty/thread-pool/include;../../../Source/Lib/ThirdParty/zlib
+ ../../../Source;../../../Source/Lib/ThirdParty/flac/include;../../../Source/Lib/ThirdParty/flac/src/libFLAC/include;../../../Source/Lib/ThirdParty/md5;../../../Source/Lib/ThirdParty/thread-pool/include;../../../Source/Lib/ThirdParty/zlib;../../../Source/Lib/ThirdParty
true
MultiThreadedDLL
@@ -267,7 +273,7 @@
true
true
NDEBUG;_CONSOLE;FLAC__NO_DLL;FLAC__HAS_OGG=0;FLAC__NO_ASM;%(PreprocessorDefinitions)
- ../../../Source;../../../Source/Lib/ThirdParty/flac/include;../../../Source/Lib/ThirdParty/flac/src/libFLAC/include;../../../Source/Lib/ThirdParty/md5;../../../Source/Lib/ThirdParty/thread-pool/include;../../../Source/Lib/ThirdParty/zlib
+ ../../../Source;../../../Source/Lib/ThirdParty/flac/include;../../../Source/Lib/ThirdParty/flac/src/libFLAC/include;../../../Source/Lib/ThirdParty/md5;../../../Source/Lib/ThirdParty/thread-pool/include;../../../Source/Lib/ThirdParty/zlib;../../../Source/Lib/ThirdParty
true
MultiThreadedDLL
AnySuitable
diff --git a/Project/MSVC2019/Lib/RAWcooked_Lib.vcxproj.filters b/Project/MSVC2019/Lib/RAWcooked_Lib.vcxproj.filters
index 208fe363..a40dafd4 100644
--- a/Project/MSVC2019/Lib/RAWcooked_Lib.vcxproj.filters
+++ b/Project/MSVC2019/Lib/RAWcooked_Lib.vcxproj.filters
@@ -103,6 +103,12 @@
{889f93d5-50da-4606-8480-a1ec82aa200f}
+
+ {bea62b93-d3d5-4cf1-a957-65810ce3ac52}
+
+
+ {e97f8d7f-a3f3-4d1b-87c3-c17fcf86decf}
+
@@ -303,6 +309,15 @@
Header Files\HashSum
+
+ ThirdParty\Reed-Solomon
+
+
+ ThirdParty\Reed-Solomon
+
+
+ ThirdParty\Reed-Solomon
+
@@ -440,6 +455,15 @@
Source Files\HashSum
+
+ ThirdParty\Reed-Solomon
+
+
+ ThirdParty\Reed-Solomon
+
+
+ ThirdParty\Reed-Solomon
+
diff --git a/Source/CLI/Global.cpp b/Source/CLI/Global.cpp
index 7672615f..05a6bc8e 100644
--- a/Source/CLI/Global.cpp
+++ b/Source/CLI/Global.cpp
@@ -127,6 +127,20 @@ int global::SetHash(bool Value)
return 0;
}
+//---------------------------------------------------------------------------
+int global::SetEcc(bool Value)
+{
+ Actions.set(Action_Ecc, Value);
+ return 0;
+}
+
+//---------------------------------------------------------------------------
+int global::SetFix(bool Value)
+{
+ Actions.set(Action_Fix, Value);
+ return 0;
+}
+
//---------------------------------------------------------------------------
int global::SetAll(bool Value)
{
@@ -142,6 +156,10 @@ int global::SetAll(bool Value)
return ReturnValue;
if (int ReturnValue = SetHash(Value))
return ReturnValue;
+ if (int ReturnValue = SetEcc(Value))
+ return ReturnValue;
+ if (int ReturnValue = SetFix(Value))
+ return ReturnValue;
return 0;
}
@@ -441,12 +459,24 @@ int global::ManageCommandLine(const char* argv[], int argc)
return Value;
License.Feature(Feature_GeneralOptions);
}
+ else if (strcmp(argv[i], "--ecc") == 0)
+ {
+ int Value = SetEcc(true);
+ if (Value)
+ return Value;
+ }
else if (strcmp(argv[i], "--encode") == 0)
{
int Value = SetEncode(true);
if (Value)
return Value;
}
+ else if (strcmp(argv[i], "--fix") == 0)
+ {
+ int Value = SetFix(true);
+ if (Value)
+ return Value;
+ }
else if (strcmp(argv[i], "--hash") == 0)
{
int Value = SetHash(true);
@@ -497,12 +527,24 @@ int global::ManageCommandLine(const char* argv[], int argc)
if (Value)
return Value;
}
+ else if (strcmp(argv[i], "--no-ecc") == 0)
+ {
+ int Value = SetEcc(false);
+ if (Value)
+ return Value;
+ }
else if (strcmp(argv[i], "--no-encode") == 0)
{
int Value = SetEncode(false);
if (Value)
return Value;
}
+ else if (strcmp(argv[i], "--no-fix") == 0)
+ {
+ int Value = SetFix(false);
+ if (Value)
+ return Value;
+ }
else if (strcmp(argv[i], "--no-hash") == 0)
{
int Value = SetHash(false);
diff --git a/Source/CLI/Global.h b/Source/CLI/Global.h
index 1dcabcfe..7ea3a182 100644
--- a/Source/CLI/Global.h
+++ b/Source/CLI/Global.h
@@ -86,6 +86,8 @@ class global
int SetConch(bool Value);
int SetEncode(bool Value);
int SetHash(bool Value);
+ int SetEcc(bool Value);
+ int SetFix(bool Value);
int SetAll(bool Value);
// Progress indicator
diff --git a/Source/CLI/Help.cpp b/Source/CLI/Help.cpp
index 86e9b107..6de1e3f9 100644
--- a/Source/CLI/Help.cpp
+++ b/Source/CLI/Help.cpp
@@ -143,6 +143,13 @@ ReturnValue Help(const char* Name)
cout << " Don't do conformance check (see above)." << endl;
cout << " Is default (it may change in the future)" << endl;
cout << endl;
+ cout << " --ecc" << endl;
+ cout << " Add Error Correction Codes to the resulting Matroska file." << endl;
+ cout << " --no-ecc" << endl;
+ cout << " Don't add Error Correction Codes to the resulting Matroska file" << endl;
+ cout << " (see above)." << endl;
+ cout << " Is default (it may change in the future)" << endl;
+ cout << endl;
cout << " --encode" << endl;
cout << " Encode audio-visual RAW data into a compressed stream." << endl;
cout << " Is default" << endl;
diff --git a/Source/CLI/Input.cpp b/Source/CLI/Input.cpp
index 0628f611..052c70ed 100644
--- a/Source/CLI/Input.cpp
+++ b/Source/CLI/Input.cpp
@@ -398,9 +398,9 @@ void input::CheckDurations(vector const& Durations, vector const
}
//---------------------------------------------------------------------------
-bool input::OpenInput(filemap& FileMap, const string& Name, errors* Errors)
+bool input::OpenInput(filemap& FileMap, const string& Name, bool Write, errors* Errors)
{
- if (FileMap.Open_ReadMode(Name))
+ if (FileMap.Open(Name, Write))
{
if (Errors)
{
diff --git a/Source/CLI/Input.h b/Source/CLI/Input.h
index 1b9a755e..8206ec18 100644
--- a/Source/CLI/Input.h
+++ b/Source/CLI/Input.h
@@ -30,7 +30,7 @@ class input
static void CheckDurations(vector const& Durations, vector const& Durations_FileName, errors* Errors = nullptr);
// I/O
- static bool OpenInput(filemap& FileMap, const string& Name, errors* Errors);
+ static bool OpenInput(filemap& FileMap, const string& Name, bool Write, errors* Errors);
};
#endif
diff --git a/Source/CLI/Main.cpp b/Source/CLI/Main.cpp
index 9e6630cd..1a3845de 100644
--- a/Source/CLI/Main.cpp
+++ b/Source/CLI/Main.cpp
@@ -67,6 +67,7 @@ user_mode Ask_Callback(user_mode* Mode, const string& FileName, const string& Ex
struct parse_info
{
string* Name;
+ string Name2; // TODO: simplify code related to file names
filemap FileMap;
vector RemovedFiles;
string FileName_Template;
@@ -144,7 +145,7 @@ bool parse_info::ParseFile_Input(input_base_uncompressed& SingleFile, input& Inp
for (size_t i = 1; i < SingleFile.InputInfo->FrameCount; i++)
{
Name = &RemovedFiles[i];
- if (input::OpenInput(FileMap, *Name, &Global.Errors))
+ if (input::OpenInput(FileMap, *Name, Global.Actions[Action_Fix], &Global.Errors))
return true;
RAWcooked.OutputFileName = Name->substr(Global.Path_Pos_Global);
FormatPath(RAWcooked.OutputFileName);
@@ -321,6 +322,13 @@ int ParseFile_Compressed(parse_info& ParseInfo)
{
ReturnValue = 1;
}
+ if (ParseInfo.IsDetected && Global.Actions[Action_Ecc])
+ {
+ ParseInfo.FileMap.Close();
+ const string* Name = ParseInfo.Name2.empty() ? ParseInfo.Name : &ParseInfo.Name2 ;
+ if (Name)
+ M.Erasure_Write(Name->c_str());
+ }
}
// End
@@ -345,7 +353,7 @@ int ParseFile(size_t Files_Pos)
ParseInfo.Name = &Input.Files[Files_Pos];
// Open file
- if (input::OpenInput(ParseInfo.FileMap, *ParseInfo.Name, &Global.Errors))
+ if (input::OpenInput(ParseInfo.FileMap, *ParseInfo.Name, Global.Actions[Action_Fix], &Global.Errors))
return 1;
// Compressed content
@@ -424,7 +432,7 @@ int main(int argc, const char* argv[])
case AlwaysYes: break;
default:
if ((!Value && Global.Actions[Action_Encode] && !Output.Streams.empty())
- || (Global.Check && !Global.Errors.HasErrors() && !Global.OutputFileName.empty() && !Output.Streams.empty()))
+ || ((Global.Check || Global.Actions[Action_Ecc]) && !Global.Errors.HasErrors() && !Global.OutputFileName.empty() && !Output.Streams.empty()))
{
cerr << "Do you want to continue despite warnings? [y/N] ";
string Result;
@@ -435,23 +443,25 @@ int main(int argc, const char* argv[])
}
}
- // FFmpeg
+ // FFmpeg
if (!Value && Global.Actions[Action_Encode])
- Value = Output.Process(Global);
+ Value = Output.Process (Global);
// RAWcooked file
if (!Global.DisplayCommand)
RAWcooked.Delete();
// Check result
- if (Global.Check && !Global.Errors.HasErrors() && !Global.OutputFileName.empty() && !Output.Streams.empty())
+ if ((Global.Check || Global.Actions[Action_Ecc]) && !Global.Errors.HasErrors() && !Global.OutputFileName.empty() && !Output.Streams.empty())
{
parse_info ParseInfo;
- Value = ParseInfo.FileMap.Open_ReadMode(Global.OutputFileName);
+ Value = ParseInfo.FileMap.Open(Global.OutputFileName, Global.Actions[Action_Fix]);
if (!Value)
{
// Configure for a 2nd pass
ParseInfo.Name = NULL;
+ ParseInfo.Name2 = Global.OutputFileName;
+ Global.Check = true;
Global.OutputFileName = Global.Inputs[0];
if (!Global.Actions[Action_Hash])
Global.OutputFileName_IsProvided = true;
diff --git a/Source/Lib/FileIO.cpp b/Source/Lib/FileIO.cpp
index c9aad2ec..8513829f 100644
--- a/Source/Lib/FileIO.cpp
+++ b/Source/Lib/FileIO.cpp
@@ -5,7 +5,9 @@
*/
//---------------------------------------------------------------------------
-#define _GNU_SOURCE // Needed for ftruncate on GNU compiler
+#if !defined(_GNU_SOURCE)
+ #define _GNU_SOURCE // Needed for ftruncate on GNU compiler
+#endif
#include "Lib/FileIO.h"
#include
#include
@@ -27,7 +29,7 @@
//---------------------------------------------------------------------------
//---------------------------------------------------------------------------
-int filemap::Open_ReadMode(const char* FileName)
+int filemap::Open(const char* FileName, bool Write)
{
Close();
@@ -35,7 +37,7 @@ int filemap::Open_ReadMode(const char* FileName)
#if defined(_WIN32) || defined(_WINDOWS)
HANDLE& File = (HANDLE&)Private;
- File = CreateFileA(FileName, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
+ File = CreateFileA(FileName, Write ? (GENERIC_READ | GENERIC_WRITE) : (GENERIC_READ), FILE_SHARE_READ | FILE_SHARE_WRITE, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
if (File != INVALID_HANDLE_VALUE)
{
DWORD FileSizeHigh;
@@ -46,7 +48,7 @@ int filemap::Open_ReadMode(const char* FileName)
if (Buffer_Size)
{
HANDLE& Mapping = (HANDLE&)Private2;
- Mapping = CreateFileMapping(File, 0, PAGE_READONLY, 0, 0, 0);
+ Mapping = CreateFileMapping(File, 0, Write ? PAGE_READWRITE : PAGE_READONLY , 0, 0, 0);
if (Mapping)
FileIsOpen = true;
else
@@ -68,7 +70,7 @@ int filemap::Open_ReadMode(const char* FileName)
FileIsOpen = false;
#else
int& P = (int&)Private;
- P = open(FileName, O_RDONLY, 0);
+ P = open(FileName, Write ? O_RDWR : O_RDONLY, 0);
if (P != -1)
{
struct stat Fstat;
@@ -89,7 +91,7 @@ int filemap::Open_ReadMode(const char* FileName)
#endif
if (FileIsOpen)
- FileIsOpen = Remap() ? false : true;
+ FileIsOpen = Remap(Write) ? false : true;
if (!FileIsOpen)
return 1;
@@ -98,7 +100,7 @@ int filemap::Open_ReadMode(const char* FileName)
}
//---------------------------------------------------------------------------
-int filemap::Remap()
+int filemap::Remap(bool Write)
{
if (Buffer)
{
@@ -115,7 +117,7 @@ int filemap::Remap()
#if defined(_WIN32) || defined(_WINDOWS)
HANDLE& Mapping = (HANDLE&)Private2;
- Buffer = (unsigned char*)MapViewOfFile(Mapping, FILE_MAP_READ, 0, 0, 0);
+ Buffer = (unsigned char*)MapViewOfFile(Mapping, Write ? (FILE_MAP_READ | FILE_MAP_WRITE) : (FILE_MAP_READ), 0, 0, 0);
if (!Buffer)
{
CloseHandle(Mapping);
@@ -128,7 +130,7 @@ int filemap::Remap()
}
#else
int& P = (int&)Private;
- Buffer = (unsigned char*)mmap(NULL, Buffer_Size, PROT_READ, MAP_FILE | MAP_PRIVATE, P, 0);
+ Buffer = (unsigned char*)mmap(NULL, Buffer_Size, Write ? (PROT_READ | PROT_WRITE) : (PROT_READ), MAP_SHARED, P, 0);
if (Buffer == MAP_FAILED)
{
Buffer = NULL;
diff --git a/Source/Lib/FileIO.h b/Source/Lib/FileIO.h
index 82649b56..ee9aa8bb 100644
--- a/Source/Lib/FileIO.h
+++ b/Source/Lib/FileIO.h
@@ -28,10 +28,10 @@ class filemap
size_t Buffer_Size = 0;
// Actions
- int Open_ReadMode(const char* FileName);
- int Open_ReadMode(const string& FileName) { return Open_ReadMode(FileName.c_str()); }
+ int Open(const char* FileName, bool Write = false);
+ int Open(const string& FileName, bool Write = false) { return Open(FileName.c_str(), Write); }
bool IsOpen() { return Private == (void*)-1 ? false : true; }
- int Remap();
+ int Remap(bool Write = false);
int Close();
private:
diff --git a/Source/Lib/HashSum/HashSum.cpp b/Source/Lib/HashSum/HashSum.cpp
index 365363d4..fd76ea05 100644
--- a/Source/Lib/HashSum/HashSum.cpp
+++ b/Source/Lib/HashSum/HashSum.cpp
@@ -60,8 +60,8 @@ const char** ErrorTexts[] =
{
undecodable::MessageText,
nullptr,
- invalid::MessageText,
nullptr,
+ invalid::MessageText,
};
static_assert(error::Type_Max == sizeof(ErrorTexts) / sizeof(const char**), IncoherencyMessage);
diff --git a/Source/Lib/Input_Base.cpp b/Source/Lib/Input_Base.cpp
index c0b4b525..4e3c95d5 100644
--- a/Source/Lib/Input_Base.cpp
+++ b/Source/Lib/Input_Base.cpp
@@ -232,6 +232,24 @@ uint64_t input_base::Get_EB()
return ToReturn;
}
+
+//---------------------------------------------------------------------------
+int64_t input_base::Get_BXs(size_t Size)
+{
+ TEST_BUFFEROVERFLOW(Size);
+
+ int64_t ToReturn = 0;
+ int64_t Mask = ((int64_t)-1) << (Size * 8);
+ while (Size)
+ {
+ ToReturn = (ToReturn << 8) | Buffer[Buffer_Offset];
+ Size--;
+ Buffer_Offset++;
+ }
+
+ return ToReturn | Mask;
+}
+
//---------------------------------------------------------------------------
void input_base::Error(error::type Type, error::generic::code Code)
{
diff --git a/Source/Lib/Input_Base.h b/Source/Lib/Input_Base.h
index 9ad47a29..43677a15 100644
--- a/Source/Lib/Input_Base.h
+++ b/Source/Lib/Input_Base.h
@@ -29,6 +29,8 @@ enum action : uint8_t
Action_Conch,
Action_CheckPadding,
Action_AcceptTruncated,
+ Action_Ecc,
+ Action_Fix,
Action_Max
};
@@ -101,6 +103,7 @@ class input_base
uint64_t Get_B8();
long double Get_BF10();
uint64_t Get_EB();
+ int64_t Get_BXs(size_t Size);
// Error message
void Undecodable(error::undecodable::code Code) { Error(error::Undecodable, (error::generic::code)Code); }
diff --git a/Source/Lib/Matroska/Matroska_Common.cpp b/Source/Lib/Matroska/Matroska_Common.cpp
index 711b2c22..bda0fe1a 100644
--- a/Source/Lib/Matroska/Matroska_Common.cpp
+++ b/Source/Lib/Matroska/Matroska_Common.cpp
@@ -14,6 +14,7 @@
#include "Lib/RawFrame/RawFrame.h"
#include "Lib/Config.h"
#include "Lib/FileIO.h"
+#include "Reed-Solomon/Reed-Solomon.h"
#include
#include
#include
@@ -93,25 +94,47 @@ static const char* MessageText[] =
{
"cannot open file for reading",
"files are not same",
+ "input file, it is corrupted, can not correct so many errors",
};
enum code : uint8_t
{
FileOpen,
FileComparison,
+ Erasure_TooManyErrors,
+ Max
+};
+
+static_assert(Max == sizeof(MessageText) / sizeof(const char*), IncoherencyMessage);
+
+} // undecodable
+
+namespace invalid
+{
+
+static const char* MessageText[] =
+{
+ "input file, it is corrupted, use --fix for fixing it",
+ "input file, it is corrupted (parity part), use --fix for rebuilding",
+};
+
+enum code : uint8_t
+{
+ Erasure_DataCorrupted,
+ Erasure_ParityCorrupted,
Max
};
namespace undecodable { static_assert(Max == sizeof(MessageText) / sizeof(const char*), IncoherencyMessage); }
-} // unparsable
+} // invalid
const char** ErrorTexts[] =
{
undecodable::MessageText,
nullptr,
nullptr,
- nullptr,
+ invalid::MessageText,
};
static_assert(error::Type_Max == sizeof(ErrorTexts) / sizeof(const char**), IncoherencyMessage);
@@ -161,7 +184,7 @@ void frame_writer::FrameCall(raw_frame* RawFrame, const string& OutputFileName)
if (!Mode[NoOutputCheck] && !File_Write.IsOpen())
{
// File already exists, we want to check it
- if (File_Read.Open_ReadMode(BaseDirectory + OutputFileName))
+ if (File_Read.Open(BaseDirectory + OutputFileName))
{
Offset = (size_t)-1;
SizeOnDisk = (size_t)-1;
@@ -655,6 +678,7 @@ matroska::call matroska::SubElements_##_VALUE(uint64_t Name) \
ELEMENT_BEGIN(_)
ELEMENT_CASE( 8538067, Segment)
+ELEMENT_CASE( 7273, Rawcooked_Segment)
ELEMENT_END()
ELEMENT_BEGIN(Segment)
@@ -737,6 +761,103 @@ ELEMENT_VOID( 30, Segment_Tracks_TrackEntry_Video_PixelWidth)
ELEMENT_VOID( 3A, Segment_Tracks_TrackEntry_Video_PixelHeight)
ELEMENT_END()
+ELEMENT_BEGIN(Rawcooked_Segment)
+ELEMENT_VOID( 70, Rawcooked_Segment_LibraryName)
+ELEMENT_VOID( 71, Rawcooked_Segment_LibraryVersion)
+ELEMENT_CASE( 726365, Rawcooked_Segment_Erasure)
+ELEMENT_END()
+
+ELEMENT_BEGIN(Rawcooked_Segment_Erasure)
+ELEMENT_VOID( 726368, Rawcooked_Segment_Erasure_ShardHashes)
+ELEMENT_VOID( 726369, Rawcooked_Segment_Erasure_ShardInfo)
+ELEMENT_VOID( 72636C, Rawcooked_Segment_Erasure_ParityShardsLocation)
+ELEMENT_VOID( 726370, Rawcooked_Segment_Erasure_ParityShards)
+ELEMENT_VOID( 726373, Rawcooked_Segment_Erasure_EbmlStartLocation)
+ELEMENT_END()
+
+//---------------------------------------------------------------------------
+// CRC_32_Table (Little Endian bitstream, )
+// The CRC in use is the IEEE-CRC-32 algorithm as used in the ISO 3309 standard and in section 8.1.1.6.2 of ITU-T recommendation V.42, with initial value of 0xFFFFFFFF. The CRC value MUST be computed on a little endian bitstream and MUST use little endian storage.
+// A CRC is computed like this:
+// Init: uint32_t CRC32 ^= 0;
+// for each data byte do
+// CRC32=(CRC32>>8) ^ Mk_CRC32_Table[(CRC32&0xFF)^*Buffer_Current++];
+// End: CRC32 ^= 0;
+static const uint32_t Matroska_CRC32_Table[256] =
+{
+ 0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA,
+ 0x076DC419, 0x706AF48F, 0xE963A535, 0x9E6495A3,
+ 0x0EDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988,
+ 0x09B64C2B, 0x7EB17CBD, 0xE7B82D07, 0x90BF1D91,
+ 0x1DB71064, 0x6AB020F2, 0xF3B97148, 0x84BE41DE,
+ 0x1ADAD47D, 0x6DDDE4EB, 0xF4D4B551, 0x83D385C7,
+ 0x136C9856, 0x646BA8C0, 0xFD62F97A, 0x8A65C9EC,
+ 0x14015C4F, 0x63066CD9, 0xFA0F3D63, 0x8D080DF5,
+ 0x3B6E20C8, 0x4C69105E, 0xD56041E4, 0xA2677172,
+ 0x3C03E4D1, 0x4B04D447, 0xD20D85FD, 0xA50AB56B,
+ 0x35B5A8FA, 0x42B2986C, 0xDBBBC9D6, 0xACBCF940,
+ 0x32D86CE3, 0x45DF5C75, 0xDCD60DCF, 0xABD13D59,
+ 0x26D930AC, 0x51DE003A, 0xC8D75180, 0xBFD06116,
+ 0x21B4F4B5, 0x56B3C423, 0xCFBA9599, 0xB8BDA50F,
+ 0x2802B89E, 0x5F058808, 0xC60CD9B2, 0xB10BE924,
+ 0x2F6F7C87, 0x58684C11, 0xC1611DAB, 0xB6662D3D,
+ 0x76DC4190, 0x01DB7106, 0x98D220BC, 0xEFD5102A,
+ 0x71B18589, 0x06B6B51F, 0x9FBFE4A5, 0xE8B8D433,
+ 0x7807C9A2, 0x0F00F934, 0x9609A88E, 0xE10E9818,
+ 0x7F6A0DBB, 0x086D3D2D, 0x91646C97, 0xE6635C01,
+ 0x6B6B51F4, 0x1C6C6162, 0x856530D8, 0xF262004E,
+ 0x6C0695ED, 0x1B01A57B, 0x8208F4C1, 0xF50FC457,
+ 0x65B0D9C6, 0x12B7E950, 0x8BBEB8EA, 0xFCB9887C,
+ 0x62DD1DDF, 0x15DA2D49, 0x8CD37CF3, 0xFBD44C65,
+ 0x4DB26158, 0x3AB551CE, 0xA3BC0074, 0xD4BB30E2,
+ 0x4ADFA541, 0x3DD895D7, 0xA4D1C46D, 0xD3D6F4FB,
+ 0x4369E96A, 0x346ED9FC, 0xAD678846, 0xDA60B8D0,
+ 0x44042D73, 0x33031DE5, 0xAA0A4C5F, 0xDD0D7CC9,
+ 0x5005713C, 0x270241AA, 0xBE0B1010, 0xC90C2086,
+ 0x5768B525, 0x206F85B3, 0xB966D409, 0xCE61E49F,
+ 0x5EDEF90E, 0x29D9C998, 0xB0D09822, 0xC7D7A8B4,
+ 0x59B33D17, 0x2EB40D81, 0xB7BD5C3B, 0xC0BA6CAD,
+ 0xEDB88320, 0x9ABFB3B6, 0x03B6E20C, 0x74B1D29A,
+ 0xEAD54739, 0x9DD277AF, 0x04DB2615, 0x73DC1683,
+ 0xE3630B12, 0x94643B84, 0x0D6D6A3E, 0x7A6A5AA8,
+ 0xE40ECF0B, 0x9309FF9D, 0x0A00AE27, 0x7D079EB1,
+ 0xF00F9344, 0x8708A3D2, 0x1E01F268, 0x6906C2FE,
+ 0xF762575D, 0x806567CB, 0x196C3671, 0x6E6B06E7,
+ 0xFED41B76, 0x89D32BE0, 0x10DA7A5A, 0x67DD4ACC,
+ 0xF9B9DF6F, 0x8EBEEFF9, 0x17B7BE43, 0x60B08ED5,
+ 0xD6D6A3E8, 0xA1D1937E, 0x38D8C2C4, 0x4FDFF252,
+ 0xD1BB67F1, 0xA6BC5767, 0x3FB506DD, 0x48B2364B,
+ 0xD80D2BDA, 0xAF0A1B4C, 0x36034AF6, 0x41047A60,
+ 0xDF60EFC3, 0xA867DF55, 0x316E8EEF, 0x4669BE79,
+ 0xCB61B38C, 0xBC66831A, 0x256FD2A0, 0x5268E236,
+ 0xCC0C7795, 0xBB0B4703, 0x220216B9, 0x5505262F,
+ 0xC5BA3BBE, 0xB2BD0B28, 0x2BB45A92, 0x5CB36A04,
+ 0xC2D7FFA7, 0xB5D0CF31, 0x2CD99E8B, 0x5BDEAE1D,
+ 0x9B64C2B0, 0xEC63F226, 0x756AA39C, 0x026D930A,
+ 0x9C0906A9, 0xEB0E363F, 0x72076785, 0x05005713,
+ 0x95BF4A82, 0xE2B87A14, 0x7BB12BAE, 0x0CB61B38,
+ 0x92D28E9B, 0xE5D5BE0D, 0x7CDCEFB7, 0x0BDBDF21,
+ 0x86D3D2D4, 0xF1D4E242, 0x68DDB3F8, 0x1FDA836E,
+ 0x81BE16CD, 0xF6B9265B, 0x6FB077E1, 0x18B74777,
+ 0x88085AE6, 0xFF0F6A70, 0x66063BCA, 0x11010B5C,
+ 0x8F659EFF, 0xF862AE69, 0x616BFFD3, 0x166CCF45,
+ 0xA00AE278, 0xD70DD2EE, 0x4E048354, 0x3903B3C2,
+ 0xA7672661, 0xD06016F7, 0x4969474D, 0x3E6E77DB,
+ 0xAED16A4A, 0xD9D65ADC, 0x40DF0B66, 0x37D83BF0,
+ 0xA9BCAE53, 0xDEBB9EC5, 0x47B2CF7F, 0x30B5FFE9,
+ 0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6,
+ 0xBAD03605, 0xCDD70693, 0x54DE5729, 0x23D967BF,
+ 0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94,
+ 0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02Ef8D,
+};
+
+//---------------------------------------------------------------------------
+void Matroska_CRC32_Compute(uint32_t& CRC32, const uint8_t* Buffer_Current, const uint8_t* Buffer_End)
+{
+ while (Buffer_Current < Buffer_End)
+ CRC32 = (CRC32 >> 8) ^ Matroska_CRC32_Table[(CRC32 & 0xFF) ^ *Buffer_Current++];
+}
+
//---------------------------------------------------------------------------
// Glue
void matroska_ProgressIndicator_Show(matroska* M)
@@ -818,6 +939,305 @@ matroska::call matroska::SubElements_Void(uint64_t Name)
Levels[Level].SubElements = &matroska::SubElements_Void; return &matroska::Void;
}
+//---------------------------------------------------------------------------
+matroska::erasure_encode::erasure_encode(erasure_info& Info_)
+{
+ Info = Info_;
+
+ size_t DataHashes_Count = (Info.erasureLength + Info.shardSize - 1) / (Info.shardSize);
+ size_t ParityHashes_Count = (DataHashes_Count + Info.dataShardCount - 1) / Info.dataShardCount;
+ ParityHashes_Count *= Info.parityShardCount;
+
+ RS = new ReedSolomon(Info.dataShardCount, Info.parityShardCount);
+ HashValues = new rawcooked::hash_value[DataHashes_Count + ParityHashes_Count];
+ ParityShards = new uint8_t[ParityHashes_Count * Info.shardSize];
+}
+
+//---------------------------------------------------------------------------
+size_t matroska::erasure_encode::Encode(uint8_t* Buffer, size_t Buffer_Max, size_t Offset)
+{
+ // Init
+ uint8_t* Shards[256];
+ size_t Package_Pos = Offset / Info.shardSize / Info.dataShardCount;
+ size_t Hash_Pos = Package_Pos * (Info.dataShardCount + Info.parityShardCount);
+ size_t Parity_Pos = Package_Pos * Info.parityShardCount;
+
+ // Data shards init
+ for (uint8_t i = 0; i < Info.dataShardCount; i++)
+ {
+ size_t Size = Buffer_Max - Offset;
+ if (Size > Info.shardSize)
+ Size = Info.shardSize;
+
+ if (Size == Info.shardSize)
+ {
+ Shards[i] = Buffer + Offset;
+ }
+ else
+ {
+ Shards[i] = new uint8_t[Info.shardSize];
+ memcpy(Shards[i], Buffer + Offset, Size);
+ memset(Shards[i] + Size, 0x00, Info.shardSize - Size);
+ }
+
+ if (Size)
+ {
+ HashValues[Hash_Pos++] = Compute_MD5(Buffer + Offset, Size);
+ Offset += Size;
+ }
+ }
+
+ // Parity shards init
+ auto ParityShards_Begin = ParityShards + Parity_Pos * Info.shardSize;
+ auto ParityShards_Temp = ParityShards_Begin;
+ for (uint8_t i = 0; i < Info.parityShardCount; i++)
+ {
+ Shards[Info.dataShardCount + i] = ParityShards_Temp;
+ ParityShards_Temp += Info.shardSize;
+ }
+ memset(ParityShards_Begin, 0x00, ParityShards_Temp - ParityShards_Begin);
+
+ // Process
+ RS->encode(Shards, Info.shardSize);
+
+ // Store hashes
+ for (uint8_t i = 0; i < Info.parityShardCount; i++)
+ {
+ HashValues[Hash_Pos++] = Compute_MD5(Shards[Info.dataShardCount + i], Info.shardSize);
+ }
+
+ return Offset;
+}
+
+//---------------------------------------------------------------------------
+bool matroska::erasure_check::Init()
+{
+ DataHashes_Count = (Info.erasureLength + Info.shardSize - 1) / (Info.shardSize);
+ ParityHashes_Count = (DataHashes_Count + Info.dataShardCount - 1) / Info.dataShardCount;
+ ParityHashes_Count *= Info.parityShardCount;
+ if (16 * (DataHashes_Count + ParityHashes_Count) != HashValues_Size)
+ {
+ return true;
+ }
+
+ return false;
+}
+
+//---------------------------------------------------------------------------
+size_t matroska::erasure_check::Check(uint8_t* Buffer, size_t Buffer_Max, size_t Offset)
+{
+ // Coherency
+ size_t isNOK = 0;
+ if (HashValues_Size != 16 * (DataHashes_Count + ParityHashes_Count))
+ {
+ isNOK = 1;
+ }
+ if (ParityShards_Size != Info.shardSize * ParityHashes_Count)
+ {
+ if (ParityShards_Size || !Info.shardSize || !ParityHashes_Count)
+ isNOK = 1;
+ else
+ ParityShards_Size = Info.shardSize * ParityHashes_Count;
+ }
+ if (isNOK)
+ return Offset; // TEMP TODO: Issues
+
+ // Init
+ size_t Offset_Base = Offset;
+ size_t Package_Pos = Offset / Info.shardSize / Info.dataShardCount;
+ size_t Hash_Pos = Package_Pos * (Info.dataShardCount + Info.parityShardCount);
+ size_t Parity_Pos = Package_Pos * Info.parityShardCount;
+
+ // Compute hash of data shards
+ std::vector shardPresent;
+ for (uint8_t i = 0; i < Info.dataShardCount; i++)
+ {
+ size_t Size = Buffer_Max - Offset;
+ if (Size > Info.shardSize)
+ Size = Info.shardSize;
+ if (!Size)
+ break;
+
+ shardPresent.push_back(memcmp(HashValues[Hash_Pos + i].Values, Compute_MD5(Buffer + Offset, Size).Values, 16) ? false : true);
+ if (!shardPresent.back())
+ isNOK++;
+
+ Offset += Size;
+ }
+
+ //if (!isNOK)
+ // return Offset; // All is fine
+
+ // Compute hash of parity shards
+ size_t dataShardTempCount = shardPresent.size();
+ shardPresent.resize(248, true);
+ for (uint8_t i = 0; i < Info.parityShardCount; i++)
+ {
+ shardPresent.push_back(memcmp(HashValues[Hash_Pos + dataShardTempCount + i].Values, Compute_MD5(ParityShards + (Parity_Pos + i) * Info.shardSize, Info.shardSize).Values, 16) ? false : true);
+ if (!shardPresent.back())
+ isNOK++;
+ }
+
+ if (isNOK)
+ {
+ if (isNOK > Info.parityShardCount)
+ {
+ if (Errors)
+ {
+ std::stringstream in, out;
+ in << std::hex << Offset_Base;
+ out << std::hex << Offset_Base + Info.dataShardCount * Info.shardSize - 1;
+ Errors->Error(IO_FileChecker, error::Undecodable, (error::generic::code)filechecker_issue::undecodable::Erasure_TooManyErrors, "at 0x" + in.str() + "-0x" + out.str());
+ }
+
+ return (uint64_t)-1;
+ }
+
+ uint8_t** Shards = new uint8_t* [Info.dataShardCount + Info.parityShardCount];
+ Offset = Offset_Base;
+
+ for (uint8_t i = 0; i < Info.dataShardCount; i++)
+ {
+ size_t Size = Buffer_Max - Offset;
+ if (Size > Info.shardSize)
+ Size = Info.shardSize;
+
+ if (Size == Info.shardSize)
+ {
+ Shards[i] = Buffer + Offset;
+ }
+ else
+ {
+ Shards[i] = new uint8_t[Info.shardSize];
+ memcpy(Shards[i], Buffer + Offset, Size);
+ memset(Shards[i] + Size, 0x00, Info.shardSize - Size);
+ }
+
+ Offset += Size;
+ }
+
+ for (uint8_t i = 0; i < Info.parityShardCount; i++)
+ {
+ Shards[Info.dataShardCount + i] = ParityShards + (Parity_Pos + i) * Info.shardSize ;
+ }
+
+ if (Write)
+ {
+ ReedSolomon RS(Info.dataShardCount, Info.parityShardCount);
+ if (!RS.repair(Shards, Info.shardSize, shardPresent))
+ {
+ if (Errors)
+ {
+ std::stringstream in, out;
+ in << std::hex << Offset_Base;
+ out << std::hex << Offset_Base + Info.dataShardCount * Info.shardSize - 1;
+ Errors->Error(IO_FileChecker, error::Undecodable, (error::generic::code)filechecker_issue::undecodable::Erasure_TooManyErrors, "at 0x" + in.str() + "-0x" + out.str());
+ }
+
+ return (uint64_t)-1;
+ }
+ }
+ for (size_t i = 0; i < shardPresent.size(); i++)
+ if (!shardPresent[i])
+ {
+ if (i < Info.dataShardCount) // Data shard
+ {
+ size_t Write_Offset = Offset_Base + i * Info.shardSize;
+ size_t Write_Size = Buffer_Max - Write_Offset;
+ if (Write_Size > Info.shardSize)
+ Write_Size = Info.shardSize;
+
+ if (Write)
+ {
+ memcpy(Buffer + Write_Offset, Shards[i], Write_Size);
+ }
+ else
+ {
+ if (Errors)
+ {
+ std::stringstream in, out;
+ in << std::hex << Write_Offset;
+ out << std::hex << Write_Offset + Write_Size - 1;
+ Errors->Error(IO_FileChecker, error::Invalid, (error::generic::code)filechecker_issue::invalid::Erasure_DataCorrupted, "at 0x" + in.str() + "-0x" + out.str());
+ }
+ }
+ }
+ else // Parity shard
+ {
+ size_t Offset_Temp = (Parity_Pos + i - Info.dataShardCount) * Info.shardSize;
+
+ if (Write)
+ {
+ memcpy(ParityShards + Offset_Temp, Shards[i], Info.shardSize);
+ }
+ else
+ {
+ if (Errors)
+ {
+ std::stringstream in, out;
+ in << std::hex << ParityShards - Buffer + Offset_Temp;
+ out << std::hex << ParityShards - Buffer + Offset_Temp + Info.shardSize - 1;
+ Errors->Error(IO_FileChecker, error::Invalid, (error::generic::code)filechecker_issue::invalid::Erasure_ParityCorrupted, "at 0x" + in.str() + "-0x" + out.str());
+ }
+ }
+ }
+ }
+
+ if (!Write)
+ return (uint64_t)-1;
+ }
+
+ return Offset;
+}
+
+//---------------------------------------------------------------------------
+rawcooked::hash_value matroska::erasure::Compute_MD5(uint8_t* Buffer, size_t Buffer_Size)
+{
+ MD5_CTX Hash;
+ MD5_Init(&Hash);
+ MD5_Update(&Hash, Buffer, (unsigned long)Buffer_Size);
+ rawcooked::hash_value HashValue;
+ MD5_Final(HashValue.Values, &Hash);
+ return HashValue;
+}
+
+//---------------------------------------------------------------------------
+void matroska::Erasure_Init()
+{
+ if (Buffer_Size_Matroska)
+ return; // Already donc
+ Buffer_Size_Matroska = Levels[Level].Offset_End;
+
+ // Erasure encode?
+ if (!Erasure_Encode && Actions[Action_Ecc])
+ {
+ erasure_info Info;
+ Info.dataShardCount = 248;
+ Info.parityShardCount = 8;
+ Info.shardSize = 1024 * 1024;
+ Info.erasureStart = 0;
+ Info.erasureLength = Buffer_Size_Matroska;
+
+ Erasure_Encode = new erasure_encode(Info);
+ }
+
+ // Erasure check?
+ if (Buffer_Size_Matroska != Buffer_Size)
+ {
+ const uint8_t* SearchBuffer = Buffer + Buffer_Size - 4;
+ const uint8_t* MatroskaEnd = Buffer + Buffer_Size_Matroska;
+ for (; SearchBuffer >= MatroskaEnd; SearchBuffer--)
+ {
+ if (SearchBuffer[0] != 0x1A || SearchBuffer[1] != 0x45 || SearchBuffer[2] != 0xDF || SearchBuffer[3] != 0xA3)
+ continue;
+ Buffer_Offset = SearchBuffer - Buffer;
+ Levels[Level].Offset_End = Buffer_Offset;
+ Matroska_ShouldBeParsed = true;
+ break;
+ }
+ }
+}
+
//---------------------------------------------------------------------------
void matroska::ParseBuffer()
{
@@ -845,15 +1265,47 @@ void matroska::ParseBuffer()
Level++;
size_t Buffer_Offset_LowerLimit = 0; // Used for indicating the system that we'll not need anymore memory below this value
+ size_t Buffer_Offset_HigherLimit = 0; // Used for indicating the system that we didn't use memory beyond this value
+ size_t Erasure_Encode_Offset = 0;
+ size_t Erasure_Check_Offset = 0;
while (Buffer_Offset < Buffer_Size)
{
+ // Erasure encode
+ if (Erasure_Encode && Buffer_Size_Matroska && Buffer_Offset + 16 >= Erasure_Encode_Offset && Buffer_Offset < Buffer_Size_Matroska)
+ {
+ Erasure_Encode_Offset = Erasure_Encode->Encode(Buffer, Buffer_Size_Matroska, Erasure_Encode_Offset);
+ if (Buffer_Offset_HigherLimit < Erasure_Encode_Offset)
+ Buffer_Offset_HigherLimit = Erasure_Encode_Offset;
+ }
+
+ // Erasure check
+ if (Erasure_Check && Buffer_Size_Matroska && Buffer_Offset >= Erasure_Check_Offset && Buffer_Offset < Buffer_Size_Matroska)
+ {
+ Erasure_Check_Offset = Erasure_Check->Check(Buffer, Buffer_Size_Matroska, Erasure_Check_Offset);
+ if (Erasure_Check_Offset == (uint64_t)-1)
+ break; // Error detected, we stop
+ if (Buffer_Offset_HigherLimit < Erasure_Check_Offset)
+ Buffer_Offset_HigherLimit = Erasure_Check_Offset;
+ }
+
+ // Parse header
uint64_t Name = Get_EB();
uint64_t Size = Get_EB();
if (Size <= Levels[Level - 1].Offset_End - Buffer_Offset)
Levels[Level].Offset_End = Buffer_Offset + Size;
else
Levels[Level].Offset_End = Levels[Level - 1].Offset_End;
+
+ // Erasure encode
+ if (Erasure_Encode && Buffer_Size_Matroska && Levels[Level].Offset_End >= Erasure_Encode_Offset && Levels[Level].Offset_End < Buffer_Size_Matroska)
+ {
+ Erasure_Encode_Offset = Erasure_Encode->Encode(Buffer, Buffer_Size_Matroska, Erasure_Encode_Offset);
+ if (Buffer_Offset_HigherLimit < Erasure_Encode_Offset)
+ Buffer_Offset_HigherLimit = Erasure_Encode_Offset;
+ }
+
+ // Parse data
call Call = (this->*Levels[Level - 1].SubElements)(Name);
IsList = false;
(this->*Call)();
@@ -873,10 +1325,42 @@ void matroska::ParseBuffer()
// Check if we can indicate the system that we'll not need anymore memory below this value, without indicating it too much
if (Buffer_Offset > Buffer_Offset_LowerLimit + 1024 * 1024) // TODO: when multi-threaded frame decoding is implemented, we need to check that all thread don't need anymore memory below this value
{
- FileMap->Remap();
+ FileMap->Remap(Actions[Action_Fix]);
Buffer = FileMap->Buffer;
Buffer_Offset_LowerLimit = Buffer_Offset;
}
+
+ if (Matroska_ShouldBeParsed && Buffer_Offset >= Buffer_Size)
+ {
+ Matroska_ShouldBeParsed = false;
+ Buffer_Offset = 0;
+ Buffer_Size = Buffer_Size_Matroska;
+ Levels[0].Offset_End = Buffer_Size;
+ Level = 1;
+
+ if (Erasure_Check)
+ {
+ if (Erasure_Check->Init())
+ {
+ // TODO
+ delete Erasure_Check;
+ Erasure_Check = nullptr;
+ }
+
+ if (Erasure_Encode)
+ {
+ delete Erasure_Encode;
+ Erasure_Encode = nullptr;
+ }
+ }
+ }
+ }
+
+ // Erasure encode
+ if (Erasure_Encode)
+ {
+ Erasure.Erasure(Erasure_Encode->HashValues, Erasure_Encode->ParityShards, Buffer_Size);
+ delete[] Erasure_Encode->HashValues;
}
// Progress indicator
@@ -904,6 +1388,8 @@ void matroska::Void()
void matroska::Segment()
{
SetDetected();
+ Erasure_Init();
+
IsList = true;
}
@@ -984,7 +1470,7 @@ void matroska::Segment_Attachments_AttachedFile_FileData_RawCookedAttachment_Fil
{
if (!RAWcooked_FileNameIsValid)
return; // File name should come first. TODO: support when file name comes after
- if (Levels[Level].Offset_End - Buffer_Offset != 17 || Buffer[Buffer_Offset] != 0x00)
+ if (Levels[Level].Offset_End - Buffer_Offset != 17 || Buffer[Buffer_Offset] != 0x80)
return; // MD5 support only
Buffer_Offset++;
@@ -1017,7 +1503,7 @@ void matroska::Segment_Attachments_AttachedFile_FileData_RawCookedBlock_FileHash
{
if (!RAWcooked_FileNameIsValid)
return; // File name should come first. TODO: support when file name comes after
- if (Levels[Level].Offset_End - Buffer_Offset != 17 || Buffer[Buffer_Offset] != 0x00)
+ if (Levels[Level].Offset_End - Buffer_Offset != 17 || Buffer[Buffer_Offset] != 0x80)
return; // MD5 support only
Buffer_Offset++;
@@ -1297,7 +1783,7 @@ void matroska::Segment_Attachments_AttachedFile_FileData_RawCookedTrack_FileHash
{
if (!RAWcooked_FileNameIsValid)
return; // File name should come first. TODO: support when file name comes after
- if (Levels[Level].Offset_End - Buffer_Offset != 17 || Buffer[Buffer_Offset] != 0x00)
+ if (Levels[Level].Offset_End - Buffer_Offset != 17 || Buffer[Buffer_Offset] != 0x80)
return; // MD5 support only
Buffer_Offset++;
@@ -1717,6 +2203,104 @@ void matroska::Segment_Tracks_TrackEntry_Video_PixelHeight()
TrackInfo_Current->Frame.SetHeight(Data);
}
+//---------------------------------------------------------------------------
+void matroska::Rawcooked_Segment()
+{
+ IsList = true;
+}
+
+//---------------------------------------------------------------------------
+void matroska::Rawcooked_Segment_LibraryName()
+{
+}
+
+//---------------------------------------------------------------------------
+void matroska::Rawcooked_Segment_LibraryVersion()
+{
+}
+
+//---------------------------------------------------------------------------
+void matroska::Rawcooked_Segment_Erasure()
+{
+ IsList = true;
+}
+
+//---------------------------------------------------------------------------
+void matroska::Rawcooked_Segment_Erasure_EbmlStartLocation()
+{
+ size_t Size = Levels[Level].Offset_End - Buffer_Offset;
+ int64_t Data = Get_BXs(Size);
+}
+
+//---------------------------------------------------------------------------
+void matroska::Rawcooked_Segment_Erasure_ShardHashes()
+{
+ uint64_t Kind = Get_EB();
+ size_t Size = (Levels[Level].Offset_End - Buffer_Offset);
+ if (Kind == 0 && !(Size % 16)) //MD5
+ {
+ if (!Erasure_Check)
+ Erasure_Check = new erasure_check(Actions[Action_Fix], Errors);
+ Erasure_Check->HashValues_Size = Size;
+ Erasure_Check->HashValues = new rawcooked::hash_value[Size / 16];
+ memcpy(Erasure_Check->HashValues, Buffer + Buffer_Offset, Size);
+ Buffer_Offset += Size;
+ }
+}
+
+//---------------------------------------------------------------------------
+void matroska::Rawcooked_Segment_Erasure_ShardInfo()
+{
+ uint64_t dataShardCount = Get_EB();
+ uint64_t parityShardCount = Get_EB();
+ uint64_t shardSize = Get_EB();
+ uint64_t erasureStart = Get_EB();
+ uint64_t erasureLength = Get_EB();
+
+ if (Buffer_Offset <= Levels[Level].Offset_End
+ && dataShardCount && dataShardCount <= 0xFF
+ && parityShardCount && parityShardCount <= 0xFF
+ && dataShardCount + parityShardCount <= 0x100
+ && erasureStart < Buffer_Size
+ && erasureLength < Buffer_Size
+ && erasureStart < Buffer_Size - erasureLength)
+ {
+ if (!Erasure_Check)
+ Erasure_Check = new erasure_check(Actions[Action_Fix], Errors);
+ Erasure_Check->Info.dataShardCount = (uint8_t)dataShardCount;
+ Erasure_Check->Info.parityShardCount = (uint8_t)parityShardCount;
+ Erasure_Check->Info.shardSize = shardSize;
+ Erasure_Check->Info.erasureStart = erasureStart;
+ Erasure_Check->Info.erasureLength = erasureLength;
+ }
+}
+
+//---------------------------------------------------------------------------
+void matroska::Rawcooked_Segment_Erasure_ParityShards()
+{
+ size_t Size = Levels[Level].Offset_End - Buffer_Offset;
+ if (!Erasure_Check)
+ Erasure_Check = new erasure_check(Actions[Action_Fix], Errors);
+ Erasure_Check->ParityShards = Buffer + Buffer_Offset;
+ Erasure_Check->ParityShards_Size = Size;
+ Buffer_Offset += Size;
+}
+
+//---------------------------------------------------------------------------
+void matroska::Rawcooked_Segment_Erasure_ParityShardsLocation()
+{
+ size_t Size = Levels[Level].Offset_End - Buffer_Offset;
+ int64_t Data = Get_BXs(Size);
+
+ uint64_t Remaining = Buffer_Size - Buffer_Offset;
+ if ((Data <= 0 || (Remaining < INT64_MAX && Data < (int64_t)Remaining)) && (Data >= 0 || (Buffer_Offset >= (uint64_t)-Data)))
+ {
+ if (!Erasure_Check)
+ Erasure_Check = new erasure_check(Actions[Action_Fix], Errors);
+ Erasure_Check->ParityShards = Buffer + (size_t) (Buffer_Offset + Data);
+ }
+}
+
//***************************************************************************
// Utils
//***************************************************************************
@@ -2050,3 +2634,10 @@ void matroska::ProcessFrame_FLAC()
break;
}
}
+
+//---------------------------------------------------------------------------
+void matroska::Erasure_Write(const char* FileName)
+{
+ Erasure.FileName = FileName;
+ Erasure.Erasure_AppendToFile();
+}
diff --git a/Source/Lib/Matroska/Matroska_Common.h b/Source/Lib/Matroska/Matroska_Common.h
index 3b17aabd..953391f9 100644
--- a/Source/Lib/Matroska/Matroska_Common.h
+++ b/Source/Lib/Matroska/Matroska_Common.h
@@ -11,6 +11,7 @@
//---------------------------------------------------------------------------
#include "Lib/FFV1/FFV1_Frame.h"
+#include "Lib/RAWcooked/RAWcooked.h"
#include "Lib/Input_Base.h"
#include "Lib/FileIO.h"
#include
@@ -30,6 +31,7 @@ class matroska_mapping;
class ThreadPool;
class flac_info;
class hashes;
+class ReedSolomon;
class frame_writer
{
@@ -91,6 +93,9 @@ class matroska : public input_base
hashes* Hashes_FromRAWcooked;
hashes* Hashes_FromAttachments;
+ // Erasure
+ void Erasure_Write(const char* FileName);
+
// Theading relating functions
void ProgressIndicator_Show();
@@ -170,6 +175,15 @@ class matroska : public input_base
MATROSKA_ELEMENT(Segment_Tracks_TrackEntry_Video_PixelWidth);
MATROSKA_ELEMENT(Segment_Tracks_TrackEntry_Video_PixelHeight);
MATROSKA_ELEMENT(Void);
+ MATROSKA_ELEMENT(Rawcooked_Segment);
+ MATROSKA_ELEMENT(Rawcooked_Segment_LibraryName);
+ MATROSKA_ELEMENT(Rawcooked_Segment_LibraryVersion);
+ MATROSKA_ELEMENT(Rawcooked_Segment_Erasure);
+ MATROSKA_ELEMENT(Rawcooked_Segment_Erasure_EbmlStartLocation);
+ MATROSKA_ELEMENT(Rawcooked_Segment_Erasure_ShardHashes);
+ MATROSKA_ELEMENT(Rawcooked_Segment_Erasure_ShardInfo);
+ MATROSKA_ELEMENT(Rawcooked_Segment_Erasure_ParityShards);
+ MATROSKA_ELEMENT(Rawcooked_Segment_Erasure_ParityShardsLocation);
enum format
{
@@ -255,6 +269,63 @@ class matroska : public input_base
int16_t Block_Timestamp;
friend class frame_writer;
+ // Erasure code
+ size_t Buffer_Size_Matroska = 0;
+ bool Matroska_ShouldBeParsed = false;
+ struct erasure_info
+ {
+ uint8_t dataShardCount = 0;
+ uint8_t parityShardCount = 0;
+ size_t shardSize = 0;
+ size_t erasureStart = 0;
+ size_t erasureLength = 0;
+ };
+ struct erasure
+ {
+ erasure_info Info;
+ rawcooked::hash_value Compute_MD5(uint8_t* Buffer, size_t Buffer_Size);
+ };
+ struct erasure_encode : public erasure
+ {
+ erasure_encode(erasure_info& Info);
+
+ ReedSolomon* RS = nullptr;
+ rawcooked::hash_value* HashValues = nullptr;
+ uint8_t* ParityShards = nullptr;
+
+ size_t Encode(uint8_t* Buffer, size_t Buffer_Max, size_t Offset);
+ };
+ struct erasure_check : public erasure
+ {
+ erasure_check(bool Write_Source = false, errors* Errors_Source = nullptr) :
+ Write(Write_Source),
+ Errors(Errors_Source)
+ {
+ }
+
+ bool Init();
+
+ //
+ rawcooked::hash_value* HashValues = nullptr;
+ size_t HashValues_Size = 0; // In bytes
+ uint8_t* ParityShards = nullptr;
+ size_t ParityShards_Size = 0; // In bytes
+
+ //
+ size_t DataHashes_Count = 0;
+ size_t ParityHashes_Count = 0;
+
+ size_t Check(uint8_t* Buffer, size_t Buffer_Max, size_t Offset);
+
+ private:
+ errors* Errors;
+ bool Write;
+ };
+ erasure_encode* Erasure_Encode = nullptr;
+ erasure_check* Erasure_Check = nullptr;
+ rawcooked Erasure;
+ void Erasure_Init();
+
//Utils
bool GetFormatAndFlavor(trackinfo* TrackInfo, input_base_uncompressed* PotentialParser, raw_frame::flavor Flavor);
void ParseDecodedFrame(trackinfo* TrackInfo);
diff --git a/Source/Lib/RAWcooked/RAWcooked.cpp b/Source/Lib/RAWcooked/RAWcooked.cpp
index ee698ef6..3bf14779 100644
--- a/Source/Lib/RAWcooked/RAWcooked.cpp
+++ b/Source/Lib/RAWcooked/RAWcooked.cpp
@@ -55,12 +55,30 @@ static_assert(error::Type_Max == sizeof(ErrorTexts) / sizeof(const char**), Inco
} // intermediatewrite_issue
+//---------------------------------------------------------------------------
+extern void Matroska_CRC32_Compute(uint32_t& CRC32, const uint8_t* Buffer_Current, const uint8_t* Buffer_End);
+static void CRC32_Fill(uint8_t* Buffer, size_t Buffer_Size)
+{
+ // Compute and fill CRC32
+ uint32_t CRC32Computed = 0xFFFFFFFF;
+ Matroska_CRC32_Compute(CRC32Computed, Buffer, Buffer + Buffer_Size);
+ CRC32Computed ^= 0xFFFFFFFF;
+ Buffer[-4] = (uint8_t)(CRC32Computed);
+ Buffer[-3] = (uint8_t)(CRC32Computed >> 8);
+ Buffer[-2] = (uint8_t)(CRC32Computed >> 16);
+ Buffer[-1] = (uint8_t)(CRC32Computed >> 24);
+}
+
//---------------------------------------------------------------------------
// Library name and version
const char* LibraryName = "RAWcooked";
const char* LibraryVersion = "18.10.1";
+// Global
+static const uint32_t CRC32 = 0x3F;
+static const uint32_t Void = 0x6C;
+
// EBML
static const uint32_t Name_EBML = 0x0A45DFA3;
static const uint32_t Name_EBML_Doctype = 0x0282;
@@ -93,14 +111,21 @@ static const uint32_t Name_RawCooked_FileHash = 0x20;
static const uint32_t Name_RawCooked_FileSize = 0x30;
static const uint32_t Name_RawCooked_MaskBaseFileSize = 0x31; // In BlockGroup only
static const uint32_t Name_RawCooked_MaskAdditionFileSize = 0x32; // In Track only
-// Global information
+// Segment information
static const uint32_t Name_RawCooked_LibraryName = 0x70;
static const uint32_t Name_RawCooked_LibraryVersion = 0x71;
static const uint32_t Name_RawCooked_PathSeparator = 0x72;
+// Erasure
+static const uint32_t Name_Rawcooked_Erasure = 0x726365; // "rce", RAWcooked global part, EBML class D
+static const uint32_t Name_Rawcooked_Erasure_ShardInfo = 0x726369; // "rci", RAWcooked global part, EBML class D
+static const uint32_t Name_Rawcooked_Erasure_ShardHashes = 0x726368; // "rch", RAWcooked global part, EBML class D
+static const uint32_t Name_Rawcooked_Erasure_ParityShards = 0x726370; // "rcp", RAWcooked global part, EBML class D
+static const uint32_t Name_Rawcooked_Erasure_EbmlStartLocation = 0x726373; // "rcs", RAWcooked global part, EBML class D
+static const uint32_t Name_Rawcooked_Erasure_ParityShardsLocation = 0x72636C; // "rcl", RAWcooked global part, EBML class D
// Parameters
static const char* DocType = "rawcooked";
-static const uint8_t DocTypeVersion = 1;
-static const uint8_t DocTypeReadVersion = 1;
+static const uint64_t DocTypeVersion = 1;
+static const uint64_t DocTypeReadVersion = 1;
//---------------------------------------------------------------------------
enum hashformat
@@ -112,18 +137,40 @@ enum hashformat
static size_t Size_EB(uint64_t Value)
{
size_t S_l = 1;
- while (Value >> (S_l * 7))
+ while (S_l < 7 && Value >> (S_l * 7))
S_l++;
if (Value == (uint64_t)-1 >> ((sizeof(Value) - S_l) * 8 + S_l))
S_l++;
return S_l;
}
+//---------------------------------------------------------------------------
+static void Write_EB(uint8_t* Buffer, size_t& Buffer_Offset, uint64_t Value)
+{
+ size_t S_l = Size_EB(Value);
+ uint64_t S2 = Value | (((uint64_t)1) << (S_l * 7));
+ while (S_l)
+ {
+ Buffer[Buffer_Offset] = (uint8_t)(S2 >> ((S_l - 1) * 8));
+ Buffer_Offset++;
+ S_l--;
+ }
+}
+
//---------------------------------------------------------------------------
static size_t Size_Number(uint64_t Value)
{
size_t S_l = 1;
- while (Value >> (S_l * 8))
+ while (S_l < 7 && Value >> (S_l * 8))
+ S_l++;
+ return S_l;
+}
+
+//---------------------------------------------------------------------------
+static size_t Size_Number(int64_t Value)
+{
+ size_t S_l = 1;
+ while (S_l < 7 && Value >> (S_l * 8) == ((int64_t)-1) >> (S_l * 8))
S_l++;
return S_l;
}
@@ -136,10 +183,11 @@ class ebml_writer
~ebml_writer();
// Types
- void EB(uint64_t Size);
- void Block(uint32_t Name, uint64_t Size, const uint8_t* Data);
+ void EB(uint64_t Value, size_t Length = (size_t)-1);
+ void Block(uint32_t Name, uint64_t Size, const uint8_t* Data, size_t BlockSize_Length = (size_t)-1);
void String(uint32_t Name, const char* Data);
- void Number(uint32_t Name, uint64_t Number);
+ void Number(uint32_t Name, uint64_t Number, size_t BlockSize_Length = (size_t)-1);
+ void Number(uint32_t Name, int64_t Number, size_t BlockSize_Length = (size_t)-1);
void DataWithEncodedPrefix(uint32_t Name, uint64_t Prefix, uint64_t Size, const uint8_t* Data);
void CompressableData(uint32_t Name, const uint8_t* Data, uint64_t Size, uint64_t CompressedSize);
@@ -153,37 +201,88 @@ class ebml_writer
size_t GetBufferSize();
private:
- std::vector BlockSizes;
+ struct block_info
+ {
+ size_t Buffer_PreviousOffset = 0;
+ size_t Size = 0;
+ std::vector Blocks;
+
+ size_t GetSize(size_t Level)
+ {
+ if (Level)
+ return Blocks.back().GetSize(Level - 1);
+ else
+ return Blocks.back().Size;
+ }
+
+ void Remove(size_t Level)
+ {
+ if (Level)
+ return Blocks.back().Remove(Level - 1);
+ else
+ Blocks.pop_back();
+ }
+
+ void Begin(size_t Offset, size_t Level)
+ {
+ if (Level)
+ return Blocks.back().Begin(Offset, Level - 1);
+ else
+ {
+ Blocks.push_back(block_info());
+ Blocks.back().Buffer_PreviousOffset = Offset;
+ }
+ }
+
+ void End(size_t Offset, size_t Level)
+ {
+ if (Level)
+ return Blocks.back().End(Offset, Level - 1);
+ else
+ {
+ Blocks.back().Size = Offset - Blocks.back().Buffer_PreviousOffset;
+ }
+ }
+
+ void Set2ndPass()
+ {
+ Buffer_PreviousOffset = 0;
+ reverse(Blocks.begin(), Blocks.end()); // We'll decrement size for keeping track about where we are
+ for (auto Block : Blocks)
+ Block.Set2ndPass();
+ }
+ };
+ block_info BlockInfo;
+ size_t Level = (size_t)-1;
uint8_t* Buffer = NULL;
- size_t Buffer_PreviousOffset = 0;
size_t Buffer_Offset = 0;
};
//---------------------------------------------------------------------------
-void ebml_writer::EB(uint64_t Size)
+void ebml_writer::EB(uint64_t Value, size_t S_l)
{
- size_t S_l = Size_EB(Size);
+ if (S_l > 7)
+ S_l = Size_EB(Value);
if (Buffer)
{
- uint64_t S2 = Size | (((uint64_t)1) << (S_l * 7));
+ uint64_t S2 = Value | (((uint64_t)1) << (S_l * 7));
while (S_l)
{
Buffer[Buffer_Offset] = (uint8_t)(S2 >> ((S_l - 1) * 8));
Buffer_Offset++;
S_l--;
}
-
}
else
Buffer_Offset += S_l;
}
//---------------------------------------------------------------------------
-void ebml_writer::Block(uint32_t Name, uint64_t Size, const uint8_t* Data)
+void ebml_writer::Block(uint32_t Name, uint64_t Size, const uint8_t* Data, size_t S_l)
{
EB(Name);
- EB(Size);
+ EB(Size, S_l);
if (Buffer)
{
@@ -199,10 +298,32 @@ void ebml_writer::String(uint32_t Name, const char* Data)
}
//---------------------------------------------------------------------------
-void ebml_writer::Number(uint32_t Name, uint64_t Number)
+void ebml_writer::Number(uint32_t Name, uint64_t Number, size_t S_l)
{
EB(Name);
- size_t S_l = Size_Number(Number);
+ if (S_l > 7)
+ S_l = Size_Number(Number);
+ EB(S_l);
+
+ if (Buffer)
+ {
+ while (S_l)
+ {
+ Buffer[Buffer_Offset] = (uint8_t)(Number >> ((S_l - 1) * 8));
+ Buffer_Offset++;
+ S_l--;
+ }
+ }
+ else
+ Buffer_Offset += S_l;
+}
+
+//---------------------------------------------------------------------------
+void ebml_writer::Number(uint32_t Name, int64_t Number, size_t S_l)
+{
+ EB(Name);
+ if (S_l > 7)
+ S_l = Size_Number(Number);
EB(S_l);
if (Buffer)
@@ -246,14 +367,17 @@ void ebml_writer::CompressableData(uint32_t Name, const uint8_t* Data, uint64_t
//---------------------------------------------------------------------------
void ebml_writer::Block_Begin(uint32_t Name)
{
+ Level++;
+
EB(Name);
if (Buffer)
{
- EB(BlockSizes.back());
- BlockSizes.pop_back();
+ EB(BlockInfo.GetSize(Level));
}
else
- Buffer_PreviousOffset = Buffer_Offset;
+ {
+ BlockInfo.Begin(Buffer_Offset, Level);
+ }
}
//---------------------------------------------------------------------------
@@ -261,9 +385,15 @@ void ebml_writer::Block_End()
{
if (!Buffer)
{
- BlockSizes.push_back(Buffer_Offset - Buffer_PreviousOffset);
- EB(Buffer_Offset - Buffer_PreviousOffset);
+ BlockInfo.End(Buffer_Offset, Level);
+ EB(BlockInfo.GetSize(Level));
}
+ else
+ {
+ BlockInfo.Remove(Level);
+ }
+
+ Level--;
}
//---------------------------------------------------------------------------
@@ -272,9 +402,8 @@ void ebml_writer::Set2ndPass()
if (Buffer)
return;
- reverse(BlockSizes.begin(), BlockSizes.end()); // We'll decrement size for keeping track about where we are
+ BlockInfo.Set2ndPass();
Buffer = new uint8_t[Buffer_Offset];
- Buffer_PreviousOffset = 0;
Buffer_Offset = 0;
}
@@ -310,6 +439,7 @@ rawcooked::rawcooked() :
rawcooked::~rawcooked()
{
Close();
+ delete EbmlWriter;
}
//---------------------------------------------------------------------------
@@ -470,8 +600,8 @@ void rawcooked::Parse()
{
Writer.Block_Begin(Name_EBML);
Writer.String(Name_EBML_Doctype, DocType);
- Writer.Number(Name_EBML_DoctypeVersion, 1);
- Writer.Number(Name_EBML_DoctypeReadVersion, 1);
+ Writer.Number(Name_EBML_DoctypeVersion, DocTypeVersion);
+ Writer.Number(Name_EBML_DoctypeReadVersion, DocTypeReadVersion);
Writer.Block_End();
}
@@ -568,11 +698,11 @@ void rawcooked::Delete()
}
//---------------------------------------------------------------------------
-void rawcooked::WriteToDisk(uint8_t* Buffer, size_t Buffer_Size)
+void rawcooked::WriteToDisk(uint8_t* Buffer, size_t Buffer_Size, bool Append)
{
if (!File_WasCreated)
{
- bool RejectIfExists = (Mode && *Mode == AlwaysYes) ? false : true;
+ bool RejectIfExists = (Append || (Mode && *Mode == AlwaysYes)) ? false : true;
if (file::return_value Result = File.Open_WriteMode(string(), FileName, RejectIfExists))
{
if (RejectIfExists && Result == file::Error_FileAlreadyExists && Mode && *Mode == Ask && Ask_Callback)
@@ -595,9 +725,141 @@ void rawcooked::WriteToDisk(uint8_t* Buffer, size_t Buffer_Size)
return;
}
}
+ if (Append)
+ {
+ if (File.Seek(0, file::End))
+ {
+ if (Errors)
+ Errors->Error(IO_IntermediateWriter, error::Undecodable, (error::generic::code)intermediatewrite_issue::undecodable::FileWrite, FileName);
+ return;
+ }
+ }
File_WasCreated = true;
}
if (File.Write(Buffer, Buffer_Size))
return;
}
+
+//---------------------------------------------------------------------------
+void rawcooked::Erasure(hash_value* HashValues, uint8_t* ParityShards, size_t Buffer_Size)
+{
+ // Create
+ size_t DataHashes_Count = (Buffer_Size + 1024 * 1024 - 1) / (1024 * 1024);
+ size_t ParityHashes_Count = (DataHashes_Count + 284 - 1) / 248;
+ ParityHashes_Count *= 8;
+
+ size_t Offset;
+ size_t ShardInfo_Size = Size_EB(248) + Size_EB(8) + Size_EB(1024 * 1024) + Size_EB(0) + Size_EB(Buffer_Size);
+ uint8_t* ShardInfo = new uint8_t[ShardInfo_Size];
+ Offset = 0;
+ Write_EB(ShardInfo, Offset, 248);
+ Write_EB(ShardInfo, Offset, 8);
+ Write_EB(ShardInfo, Offset, 1024 * 1024);
+ Write_EB(ShardInfo, Offset, 0);
+ Write_EB(ShardInfo, Offset, Buffer_Size);
+ size_t ShardHashes_Size = Size_EB(0) + 16 * (DataHashes_Count + ParityHashes_Count);
+ uint8_t* ShardHashes = new uint8_t[ShardHashes_Size];
+ Offset = 0;
+ Write_EB(ShardHashes, Offset, 0);
+ for (size_t i = 0; i < DataHashes_Count; i++)
+ {
+ memcpy(ShardHashes + Offset, HashValues[i].Values, 16);
+ Offset += 16;
+ }
+ for (size_t i = 0; i < ParityHashes_Count; i++)
+ {
+ memcpy(ShardHashes + Offset, HashValues[DataHashes_Count + i].Values, 16);
+ Offset += 16;
+ }
+
+ delete EbmlWriter;
+ EbmlWriter = new ebml_writer;
+ auto& Writer = *EbmlWriter;
+ size_t Padding_Size, Padding_Size2;
+ vector Locations;
+ for (uint8_t Pass = 0; Pass < 2; Pass++)
+ {
+ // EBML header
+ if (!Pass)
+ Locations.push_back(Writer.GetBufferSize());
+ Writer.Block_Begin(Name_EBML);
+ Writer.String(Name_EBML_Doctype, DocType);
+ Writer.Number(Name_EBML_DoctypeVersion, DocTypeVersion);
+ Writer.Number(Name_EBML_DoctypeReadVersion, DocTypeReadVersion);
+ Writer.Block_End();
+
+ // Segment - before padding
+ Writer.Block_Begin(Name_RawCookedSegment);
+ Writer.String(Name_RawCooked_LibraryName, LibraryName);
+ Writer.String(Name_RawCooked_LibraryVersion, LibraryVersion);
+ Writer.Block_Begin(Name_Rawcooked_Erasure);
+ Writer.Block(Name_Rawcooked_Erasure_ShardInfo, ShardInfo_Size, ShardInfo);
+ Writer.Block(Name_Rawcooked_Erasure_ShardHashes, ShardHashes_Size, ShardHashes);
+
+ // Segment - padding
+ if (!Pass)
+ Padding_Size = 1024 * 1024 - ((Buffer_Size + Writer.GetBufferSize() + 12 /*TEMP*/ + Size_EB(Name_Rawcooked_Erasure_ParityShards) + Size_EB(ParityHashes_Count * 1024 * 1024)) % (1024 * 1024));
+ Writer.Block(Void, Padding_Size, ParityShards, 3); //TEMP ParityShards
+
+ // Segment - after padding
+ Writer.Block(Name_Rawcooked_Erasure_ParityShards, ParityHashes_Count * 1024 * 1024, ParityShards);
+ Writer.Block_End();
+ Writer.Block_End();
+ if (!Pass)
+ Locations.push_back(Writer.GetBufferSize());
+
+ for (uint8_t i = 0; i < 8 + 1; i++)
+ {
+ // EBML header
+ Writer.Block_Begin(Name_EBML);
+ Writer.String(Name_EBML_Doctype, DocType);
+ Writer.Number(Name_EBML_DoctypeVersion, DocTypeVersion);
+ Writer.Number(Name_EBML_DoctypeReadVersion, DocTypeReadVersion);
+ Writer.Block_End();
+
+ // Segment - before padding
+ Writer.Block_Begin(Name_RawCookedSegment);
+ Writer.String(Name_RawCooked_LibraryName, LibraryName);
+ Writer.String(Name_RawCooked_LibraryVersion, LibraryVersion);
+ Writer.Block_Begin(Name_Rawcooked_Erasure);
+ const uint32_t Zero = 0;
+ Writer.Block(CRC32, 4, (const uint8_t*)&Zero);
+ size_t CRC32_Start = Writer.GetBufferSize();
+ Writer.Block(Name_Rawcooked_Erasure_ShardInfo, ShardInfo_Size, ShardInfo);
+ Writer.Block(Name_Rawcooked_Erasure_ShardHashes, ShardHashes_Size, ShardHashes);
+
+ // Segment - padding
+ size_t LastPartBlockSize = (Size_EB(Name_Rawcooked_Erasure_EbmlStartLocation) + Size_EB(4) + 4);
+ if (!i && !Pass)
+ Padding_Size2 = 1024 * 1024 - ((Writer.GetBufferSize() - Locations[1] + 10 /*TEMP*/ + LastPartBlockSize * (1 + 8 + 1 + 1)) % (1024 * 1024));
+ Writer.Block(Void, Padding_Size2, ParityShards, 3); //TEMP ParityShards
+
+ // Segment - after padding
+ Writer.Number(Name_Rawcooked_Erasure_EbmlStartLocation, Pass ? (Locations[0] - Locations[1 + i + 1] + LastPartBlockSize * (1 + 8 + 1)) : 0, 4);
+ for (uint8_t j = 0; j < 8 + 1; j++)
+ Writer.Number(Name_Rawcooked_Erasure_EbmlStartLocation, Pass ? (Locations[1 + j] - Locations[1 + i + 1] + LastPartBlockSize * (1 + 8 - j)) : 0, 4);
+ Writer.Number(Name_Rawcooked_Erasure_ParityShardsLocation, Pass ? (Locations[1] - ParityHashes_Count * 1024 * 1024 - Locations[1 + i + 1]) : 0, 4);
+ Writer.Block_End();
+ Writer.Block_End();
+ if (!Pass)
+ Locations.push_back(Writer.GetBufferSize());
+ else
+ CRC32_Fill(Writer.GetBuffer() + CRC32_Start, Writer.GetBufferSize() - CRC32_Start); // Compute and fill CRC32
+ }
+
+ // Init 2nd pass
+ Writer.Set2ndPass();
+ }
+}
+
+//---------------------------------------------------------------------------
+void rawcooked::Erasure_AppendToFile()
+{
+ if (!EbmlWriter)
+ return; // Nothing to do
+
+ // Write
+ auto& Writer = *EbmlWriter;
+ WriteToDisk(Writer.GetBuffer(), Writer.GetBufferSize(), true);
+}
diff --git a/Source/Lib/RAWcooked/RAWcooked.h b/Source/Lib/RAWcooked/RAWcooked.h
index af3acd97..d96e39f5 100644
--- a/Source/Lib/RAWcooked/RAWcooked.h
+++ b/Source/Lib/RAWcooked/RAWcooked.h
@@ -17,6 +17,7 @@
#include
#include
using namespace std;
+class ebml_writer;
//---------------------------------------------------------------------------
class rawcooked
@@ -48,6 +49,14 @@ class rawcooked
string OutputFileName;
uint64_t FileSize;
+ // Erasure
+ struct hash_value
+ {
+ unsigned char Values[16];
+ };
+ void Erasure(hash_value* Hashes_Values, uint8_t* ParityShards, size_t Buffer_Size);
+ void Erasure_AppendToFile();
+
// Errors
user_mode* Mode = nullptr;
ask_callback Ask_Callback = nullptr;
@@ -60,7 +69,7 @@ class rawcooked
file File;
size_t BlockCount;
bool File_WasCreated = false;
- void WriteToDisk(uint8_t* Buffer, size_t Buffer_Size);
+ void WriteToDisk(uint8_t* Buffer, size_t Buffer_Size, bool Append = false);
// First frame info
uint8_t* FirstFrame_Before;
@@ -69,6 +78,9 @@ class rawcooked
size_t FirstFrame_After_Size;
uint8_t* FirstFrame_FileName;
size_t FirstFrame_FileName_Size;
+
+ // Erasure
+ ebml_writer* EbmlWriter = nullptr;
};
//---------------------------------------------------------------------------
diff --git a/Source/Lib/ThirdParty/Reed-Solomon/Galois.cpp b/Source/Lib/ThirdParty/Reed-Solomon/Galois.cpp
new file mode 100644
index 00000000..a31d0caf
--- /dev/null
+++ b/Source/Lib/ThirdParty/Reed-Solomon/Galois.cpp
@@ -0,0 +1,195 @@
+/* Copyright (c) MediaArea.net SARL.
+ *
+ * Use of this source code is governed by a MIT-style license that can
+ * be found in the License.html file in the root of the source tree.
+ */
+
+/*
+ * Based on https://www.backblaze.com/blog/reed-solomon
+ * Based on https://github.com/Backblaze/JavaReedSolomon
+ * Based on https://github.com/DrPizza/reed-solomon
+ * Initial copyrights:
+ * copyright 2015 Peter Bright, Backblaze.
+ */
+
+//---------------------------------------------------------------------------
+#include "Reed-Solomon/Galois.h"
+//---------------------------------------------------------------------------
+
+Galois galois;
+
+/**
+ * The polynomial used to generate the logarithm table.
+ *
+ * There are a number of polynomials that work to generate
+ * a Galois field of 256 elements. The choice is arbitrary,
+ * and we just use the first one.
+ *
+ * The possibilities are: 29, 43, 45, 77, 95, 99, 101, 105,
+ * 113, 135, 141, 169, 195, 207, 231, and 245.
+ */
+
+/**
+ * Mapping from members of the Galois Field to their
+ * integer logarithms. The entry for 0 is meaningless
+ * because there is no log of 0.
+ */
+static uint8_t LogTable[256] = {
+ 0, 0, 1, 25, 2, 50, 26, 198,
+ 3, 223, 51, 238, 27, 104, 199, 75,
+ 4, 100, 224, 14, 52, 141, 239, 129,
+ 28, 193, 105, 248, 200, 8, 76, 113,
+ 5, 138, 101, 47, 225, 36, 15, 33,
+ 53, 147, 142, 218, 240, 18, 130, 69,
+ 29, 181, 194, 125, 106, 39, 249, 185,
+ 201, 154, 9, 120, 77, 228, 114, 166,
+ 6, 191, 139, 98, 102, 221, 48, 253,
+ 226, 152, 37, 179, 16, 145, 34, 136,
+ 54, 208, 148, 206, 143, 150, 219, 189,
+ 241, 210, 19, 92, 131, 56, 70, 64,
+ 30, 66, 182, 163, 195, 72, 126, 110,
+ 107, 58, 40, 84, 250, 133, 186, 61,
+ 202, 94, 155, 159, 10, 21, 121, 43,
+ 78, 212, 229, 172, 115, 243, 167, 87,
+ 7, 112, 192, 247, 140, 128, 99, 13,
+ 103, 74, 222, 237, 49, 197, 254, 24,
+ 227, 165, 153, 119, 38, 184, 180, 124,
+ 17, 68, 146, 217, 35, 32, 137, 46,
+ 55, 63, 209, 91, 149, 188, 207, 205,
+ 144, 135, 151, 178, 220, 252, 190, 97,
+ 242, 86, 211, 171, 20, 42, 93, 158,
+ 132, 60, 57, 83, 71, 109, 65, 162,
+ 31, 45, 67, 216, 183, 123, 164, 118,
+ 196, 23, 73, 236, 127, 12, 111, 246,
+ 108, 161, 59, 82, 41, 157, 85, 170,
+ 251, 96, 134, 177, 187, 204, 62, 90,
+ 203, 89, 95, 176, 156, 169, 160, 81,
+ 11, 245, 22, 235, 122, 117, 44, 215,
+ 79, 174, 213, 233, 230, 231, 173, 232,
+ 116, 214, 244, 234, 168, 80, 88, 175
+};
+
+/**
+ * Inverse of the logarithm table. Maps integer logarithms
+ * to members of the field. There is no entry for 255
+ * because the highest log is 254.
+ */
+static uint8_t ExpTable[255] = {
+ 1, 2, 4, 8, 16, 32, 64, 128, 29,
+ 58, 116, 232, 205, 135, 19, 38, 76,
+ 152, 45, 90, 180, 117, 234, 201, 143,
+ 3, 6, 12, 24, 48, 96, 192, 157,
+ 39, 78, 156, 37, 74, 148, 53, 106,
+ 212, 181, 119, 238, 193, 159, 35, 70,
+ 140, 5, 10, 20, 40, 80, 160, 93,
+ 186, 105, 210, 185, 111, 222, 161, 95,
+ 190, 97, 194, 153, 47, 94, 188, 101,
+ 202, 137, 15, 30, 60, 120, 240, 253,
+ 231, 211, 187, 107, 214, 177, 127, 254,
+ 225, 223, 163, 91, 182, 113, 226, 217,
+ 175, 67, 134, 17, 34, 68, 136, 13,
+ 26, 52, 104, 208, 189, 103, 206, 129,
+ 31, 62, 124, 248, 237, 199, 147, 59,
+ 118, 236, 197, 151, 51, 102, 204, 133,
+ 23, 46, 92, 184, 109, 218, 169, 79,
+ 158, 33, 66, 132, 21, 42, 84, 168,
+ 77, 154, 41, 82, 164, 85, 170, 73,
+ 146, 57, 114, 228, 213, 183, 115, 230,
+ 209, 191, 99, 198, 145, 63, 126, 252,
+ 229, 215, 179, 123, 246, 241, 255, 227,
+ 219, 171, 75, 150, 49, 98, 196, 149,
+ 55, 110, 220, 165, 87, 174, 65, 130,
+ 25, 50, 100, 200, 141, 7, 14, 28,
+ 56, 112, 224, 221, 167, 83, 166, 81,
+ 162, 89, 178, 121, 242, 249, 239, 195,
+ 155, 43, 86, 172, 69, 138, 9, 18,
+ 36, 72, 144, 61, 122, 244, 245, 247,
+ 243, 251, 235, 203, 139, 11, 22, 44,
+ 88, 176, 125, 250, 233, 207, 131, 27,
+ 54, 108, 216, 173, 71, 142,
+};
+
+Galois::Galois()
+{
+ uint8_t a = 0;
+ do
+ {
+ uint8_t b = 0;
+ do
+ {
+ MultiplicationTable[a][b] = multiply(a, b);
+ if (!(b & 0x0F))
+ {
+ uint8_t result = multiply(a, b);
+ MultiplicationTableSplit[a].High[b >> 4] = result;
+ MultiplicationTableSplit[a].High[(b >> 4) + 16] = result;
+ MultiplicationTableSplit[a].High[(b >> 4) + 32] = result;
+ MultiplicationTableSplit[a].High[(b >> 4) + 48] = result;
+ }
+ if (!(b & 0xF0))
+ {
+ uint8_t result = multiply(a, b);
+ MultiplicationTableSplit[a].Low[b] = result;
+ MultiplicationTableSplit[a].Low[b + 16] = result;
+ MultiplicationTableSplit[a].Low[b + 32] = result;
+ MultiplicationTableSplit[a].Low[b + 48] = result;
+ }
+ }
+ while (b++ != 255);
+ }
+ while (a++ != 255);
+}
+
+/**
+ * Multiplies to elements of the field.
+ */
+uint8_t Galois::multiply(uint8_t a, uint8_t b)
+{
+ if (!a || !b)
+ return 0;
+
+ uint16_t logA = LogTable[a];
+ uint16_t logB = LogTable[b];
+ uint16_t logResult = logA + logB;
+ if (255 <= logResult)
+ logResult -= 255;
+ return ExpTable[logResult];
+}
+
+/**
+ * Inverse of multiplication.
+ */
+uint8_t Galois::divide(uint8_t a, uint8_t b)
+{
+ if (!a)
+ return 0;
+ if (!b)
+ throw; // b should never be 0
+ int16_t logA = LogTable[a];
+ int16_t logB = LogTable[b];
+ int16_t logResult = logA - logB;
+ if (logResult < 0)
+ logResult += 255;
+ return ExpTable[logResult];
+}
+
+/**
+ * Computes a**n.
+ *
+ * The result will be the same as multiplying a times itself n times.
+ *
+ * @param a A member of the field.
+ * @param n A plain-old integer.
+ * @return The result of multiplying a by itself n times.
+ */
+uint8_t Galois::exp(uint8_t a, uint8_t n)
+{
+ if (!n)
+ return 1;
+ if (!a)
+ return 0;
+ uint16_t logResult = LogTable[a] * n;
+ while (255 <= logResult)
+ logResult -= 255;
+ return ExpTable[logResult];
+}
diff --git a/Source/Lib/ThirdParty/Reed-Solomon/Galois.h b/Source/Lib/ThirdParty/Reed-Solomon/Galois.h
new file mode 100644
index 00000000..17451acd
--- /dev/null
+++ b/Source/Lib/ThirdParty/Reed-Solomon/Galois.h
@@ -0,0 +1,58 @@
+/* Copyright (c) MediaArea.net SARL.
+ *
+ * Use of this source code is governed by a MIT-style license that can
+ * be found in the License.html file in the root of the source tree.
+ */
+
+ //---------------------------------------------------------------------------
+#ifndef GaloisH
+#define GaloisH
+//---------------------------------------------------------------------------
+
+/**
+ * 8-bit Galois Field
+ */
+
+ //---------------------------------------------------------------------------
+#include
+//---------------------------------------------------------------------------
+
+class Galois
+{
+public:
+ Galois();
+
+ /**
+ * Multiplies to elements of the field.
+ */
+ uint8_t multiply(uint8_t a, uint8_t b);
+
+ /**
+ * Inverse of multiplication.
+ */
+ uint8_t divide(uint8_t a, uint8_t b);
+
+ /**
+ * Computes a**n.
+ *
+ * The result will be the same as multiplying a times itself n times.
+ *
+ * @param a A member of the field.
+ * @param n A plain-old integer.
+ * @return The result of multiplying a by itself n times.
+ */
+ uint8_t exp(uint8_t a, uint8_t n);
+
+ // Members
+ uint8_t MultiplicationTable[256][256];
+ struct multiplication_table_split
+ {
+ uint8_t High[4 * 16]; // 4 high bits
+ uint8_t Low[4 * 16]; // 4 low bits
+ };
+ multiplication_table_split MultiplicationTableSplit[256];
+};
+
+extern Galois galois;
+
+#endif
diff --git a/Source/Lib/ThirdParty/Reed-Solomon/Matrix.cpp b/Source/Lib/ThirdParty/Reed-Solomon/Matrix.cpp
new file mode 100644
index 00000000..a8049d46
--- /dev/null
+++ b/Source/Lib/ThirdParty/Reed-Solomon/Matrix.cpp
@@ -0,0 +1,186 @@
+/* Copyright (c) MediaArea.net SARL.
+ *
+ * Use of this source code is governed by a BSD-style license that can
+ * be found in the License.html file in the root of the source tree.
+ */
+
+/*
+ * Based on https://www.backblaze.com/blog/reed-solomon
+ * Based on https://github.com/Backblaze/JavaReedSolomon
+ * Based on https://github.com/DrPizza/reed-solomon
+ * Initial copyrights:
+ * copyright 2015 Peter Bright, Backblaze.
+ */
+
+//---------------------------------------------------------------------------
+#include "Reed-Solomon/Galois.h"
+#include "Reed-Solomon/Matrix.h"
+#include
+#include
+#include
+using namespace std;
+//---------------------------------------------------------------------------
+
+Matrix::Matrix(uint8_t rows_minus1, uint8_t columns_minus1) :
+ rowCountMinus1{ rows_minus1 },
+ columnCountMinus1{ columns_minus1 },
+ data{ new uint8_t * [rowCountMinus1 + 1] },
+ dataPointer{ new uint8_t[(rowCountMinus1 + 1) * (columnCountMinus1 + 1) * 2] } // Twice the needed size in order to anticipate invert()
+{
+ uint8_t* dataTemp = dataPointer;
+ uint16_t columnCount = columnCountMinus1 + 1;
+ uint8_t r = 0;
+ do
+ {
+ data[r] = dataTemp;
+ dataTemp += columnCount;
+ }
+ while (r++ != rowCountMinus1);
+}
+
+// Move
+Matrix::Matrix(Matrix&& rhs) :
+ rowCountMinus1{ rhs.rowCountMinus1 },
+ columnCountMinus1{ rhs.columnCountMinus1 },
+ dataPointer{ rhs.dataPointer },
+ data{ rhs.data }
+{
+ rhs.dataPointer = nullptr;
+ rhs.data = nullptr;
+}
+
+Matrix::~Matrix()
+{
+ delete[] data;
+ delete[] dataPointer;
+}
+
+Matrix Matrix::submatrix(uint8_t rmin, uint8_t cmin, uint8_t rmax, uint8_t cmax) const
+{
+ Matrix result{ static_cast(rmax - rmin - 1), static_cast(cmax - cmin - 1) };
+ for (uint8_t r = rmin; r < rmax; ++r)
+ for (uint8_t c = cmin; c < cmax; ++c)
+ result.data[r - rmin][c - cmin] = data[r][c];
+ return result;
+}
+
+Matrix Matrix::times(const Matrix & rhs) const
+{
+ Matrix result{ rowCountMinus1, rhs.columnCountMinus1 };
+ uint8_t r = 0;
+ do
+ {
+ uint8_t c = 0;
+ do
+ {
+ uint8_t value = 0;
+ uint8_t i = 0;
+ do
+ {
+ value ^= galois.multiply(data[r][i], rhs.data[i][c]);
+ }
+ while (i++ != columnCountMinus1);
+ result.data[r][c] = value;
+ }
+ while (c++ != rhs.columnCountMinus1);
+ }
+ while (r++ != rowCountMinus1);
+ return result;
+}
+
+void Matrix::invert()
+{
+ // Augmenting this matrix with
+ // an identity matrix on the right.
+ const uint16_t size = (rowCountMinus1 + 1) * (columnCountMinus1 + 1);
+ uint8_t * dataTemp = dataPointer + size;
+ uint8_t * dataTemp2 = dataTemp + size;
+ uint16_t r = rowCountMinus1 + 1;
+ uint16_t columnCount = columnCountMinus1 + 1;
+ do
+ {
+ dataTemp2 -= columnCount;
+ memset(dataTemp2, 0, columnCount);
+ dataTemp2[--r] = 1;
+ dataTemp2 -= columnCount;
+ dataTemp -= columnCount;
+ memcpy(dataTemp2, dataTemp, columnCount);
+ data[r] = dataTemp2;
+ }
+ while (r);
+
+ // Do Gaussian elimination to transform the left half into
+ // an identity matrix.
+ gaussianElimination();
+
+ // The right half is now the inverse.
+ dataTemp2 += columnCount;
+ uint16_t columnCount2 = columnCount * 2;
+ while (r <= rowCountMinus1)
+ {
+ memcpy(dataTemp, dataTemp2, columnCount);
+ data[r++] = dataTemp;
+ dataTemp += columnCount;
+ dataTemp2 += columnCount2;
+ }
+}
+
+void Matrix::gaussianElimination()
+{
+ const uint16_t columnCount2 = (columnCountMinus1 + 1) * 2;
+
+ // Clear out the part below the main diagonal and scale the main
+ // diagonal to be 1.
+ uint8_t r = 0;
+ do
+ {
+ // If the element on the diagonal is 0, find a row below
+ // that has a non-zero and swap them.
+ if (!data[r][r])
+ {
+ uint8_t rowBelow = r + 1;
+ do
+ {
+ if (data[rowBelow][r])
+ {
+ //Swap
+ for (uint16_t c = 0; c < columnCount2; c++)
+ swap(data[r][c], data[rowBelow][c]);
+ }
+ }
+ while (rowBelow++ != rowCountMinus1);
+
+ // If we couldn't find one, the matrix is singular.
+ if (!data[r][r])
+ {
+ throw runtime_error("matrix is singular");
+ }
+ }
+
+ // Scale to 1.
+ if (data[r][r] != 1)
+ {
+ uint8_t scale = galois.divide(1, data[r][r]);
+ for (uint16_t c = 0; c < columnCount2; ++c)
+ data[r][c] = galois.multiply(data[r][c], scale);
+ }
+
+ // Make everything below the 1 be a 0 by subtracting
+ // a multiple of it. (Subtraction and addition are
+ // both exclusive or in the Galois field.)
+ uint8_t r2 = 0;
+ do
+ {
+ if (r2 == r)
+ continue;
+ uint8_t scale = data[r2][r];
+ if (scale)
+ {
+ for (uint16_t c = 0; c < columnCount2; c++)
+ data[r2][c] ^= galois.multiply(scale, data[r][c]);
+ }
+ }
+ while (r2++ != rowCountMinus1);
+ }
+ while (r++ != rowCountMinus1);
+}
diff --git a/Source/Lib/ThirdParty/Reed-Solomon/Matrix.h b/Source/Lib/ThirdParty/Reed-Solomon/Matrix.h
new file mode 100644
index 00000000..17503ec4
--- /dev/null
+++ b/Source/Lib/ThirdParty/Reed-Solomon/Matrix.h
@@ -0,0 +1,86 @@
+/* Copyright (c) MediaArea.net SARL.
+ *
+ * Use of this source code is governed by a MIT-style license that can
+ * be found in the License.html file in the root of the source tree.
+ */
+
+//---------------------------------------------------------------------------
+#ifndef MatrixH
+#define MatrixH
+//---------------------------------------------------------------------------
+
+ //---------------------------------------------------------------------------
+#include
+//---------------------------------------------------------------------------
+
+/**
+ * A matrix over the 8-bit Galois field.
+ *
+ * Note: input is not verified
+ */
+
+class Matrix
+{
+public:
+ /**
+ * Initialize a matrix of zeros.
+ *
+ * @param rows_minus1 The number of rows in the matrix minus 1.
+ * @param columns_minus1 The number of columns in the matrix minus 1.
+ */
+ Matrix(uint8_t rows_minus1, uint8_t columns_minus1);
+ Matrix(Matrix&& rhs);
+ ~Matrix();
+
+ /**
+ * Returns pointer on one row.
+ */
+ inline const uint8_t* getRow(uint8_t r) const { return data[r]; }
+
+ /**
+ * Returns the value at row r, column c.
+ */
+ inline uint8_t get(uint8_t r, uint8_t c) const { return data[r][c]; }
+
+ /**
+ * Sets the value at row r, column c.
+ */
+ inline void set(uint8_t r, uint8_t c, uint8_t value) { data[r][c] = value; }
+
+ /**
+ * Returns a part of this matrix.
+ */
+ Matrix submatrix(uint8_t rmin, uint8_t cmin, uint8_t rmax, uint8_t cmax) const;
+
+ /**
+ * Multiplies this matrix (the one on the left) by another
+ * matrix (the one on the right).
+ */
+ Matrix times(const Matrix& rhs) const;
+
+ /**
+ * Inverse this matrix.
+ */
+ void invert();
+
+private:
+ /**
+ * Does the work of matrix inversion.
+ *
+ * Assumes that this is an r by 2r matrix.
+ */
+ void gaussianElimination();
+
+ // Members
+ const uint8_t rowCountMinus1;
+ const uint8_t columnCountMinus1;
+ uint8_t** data;
+ uint8_t* dataPointer;
+
+ // Forbidden
+ Matrix(const Matrix& rhs) = delete;
+ bool operator==(const Matrix& rhs) const = delete;
+ bool operator!=(const Matrix& rhs) const = delete;
+};
+
+#endif
diff --git a/Source/Lib/ThirdParty/Reed-Solomon/Reed-Solomon.cpp b/Source/Lib/ThirdParty/Reed-Solomon/Reed-Solomon.cpp
new file mode 100644
index 00000000..3509dac1
--- /dev/null
+++ b/Source/Lib/ThirdParty/Reed-Solomon/Reed-Solomon.cpp
@@ -0,0 +1,502 @@
+/* Copyright (c) MediaArea.net SARL.
+ *
+ * Use of this source code is governed by a BSD-style license that can
+ * be found in the License.html file in the root of the source tree.
+ */
+
+/*
+ * Based on https://www.backblaze.com/blog/reed-solomon
+ * Based on https://github.com/Backblaze/JavaReedSolomon
+ * Based on https://github.com/DrPizza/reed-solomon
+ * Initial copyrights:
+ * copyright 2015 Peter Bright, Backblaze.
+ */
+
+//---------------------------------------------------------------------------
+#include "Reed-Solomon/Galois.h"
+#include "Reed-Solomon/Matrix.h"
+#include "Reed-Solomon/Reed-Solomon.h"
+#include
+#if defined(__x86_64) || defined(_M_AMD64)
+ #if defined(_MSC_VER)
+ #include
+ #else
+ #include
+ #endif
+ #include
+#endif
+using namespace std;
+
+//---------------------------------------------------------------------------
+
+/**
+ * Create a Vandermonde matrix, which is guaranteed to have the
+ * property that any subset of rows that forms a square matrix
+ * is invertible.
+ *
+ * @param rows Number of rows in the result.
+ * @param cols Number of columns in the result.
+ * @return A Matrix.
+ */
+static Matrix vandermonde(uint8_t rows_minus1, uint8_t cols)
+{
+ uint8_t cols_minus1 = cols - 1;
+ Matrix result{ rows_minus1, cols_minus1 };
+ uint8_t r = 0;
+ do
+ {
+ uint8_t c = 0;
+ do
+ {
+ result.set(r, c, galois.exp(r, c));
+ }
+ while (c++ != cols_minus1);
+ }
+ while (r++ != rows_minus1);
+ return result;
+}
+
+/**
+ * Create the matrix to use for encoding, given the number of
+ * data shards and the number of total shards.
+ *
+ * The top square of the matrix is guaranteed to be an identity
+ * matrix, which means that the data shards are unchanged after
+ * encoding.
+ */
+static Matrix buildMatrix(uint8_t dataShards, uint8_t totalShardsMinus1)
+{
+ if (!totalShardsMinus1)
+ return Matrix(0, 0); // Fake one, it will be trashed
+
+ // Start with a Vandermonde matrix. This matrix would work,
+ // in theory, but doesn't have the property that the data
+ // shards are unchanged after encoding.
+ Matrix v = vandermonde(totalShardsMinus1, dataShards);
+
+ // Multiple by the inverse of the top square of the matrix.
+ // This will make the top square be the identity matrix, but
+ // preserve the property that any square subset of rows is
+ // invertible.
+ Matrix top = v.submatrix(0, 0, dataShards, dataShards);
+ top.invert();
+ return v.times(top);
+}
+
+ReedSolomon::ReedSolomon(uint8_t dataShardCount_, uint8_t parityShardCount_) :
+ dataShardCount{ dataShardCount_ },
+ parityShardCount{ parityShardCount_ },
+ totalShardCountMinus1{ !dataShardCount || !parityShardCount || dataShardCount + parityShardCount > 256 ? static_cast(0) : static_cast(dataShardCount + parityShardCount - 1) },
+ matrixRows{ new const uint8_t * [parityShardCount] },
+ matrix{ buildMatrix(dataShardCount, totalShardCountMinus1) }
+{
+ if (!totalShardCountMinus1)
+ return;
+
+ // Fill parity_rows
+ for (uint8_t i = 0; i < parityShardCount; i++)
+ matrixRows[i] = matrix.getRow(dataShardCount + i);
+}
+
+ReedSolomon::~ReedSolomon()
+{
+ delete[] matrixRows;
+}
+
+static size_t singleInstructionMultipleData = 0;
+#if defined(__x86_64) || defined(_M_AMD64)
+ #if defined(_MSC_VER)
+ inline void cpuid(unsigned int cpuInfo[4], unsigned int id) { __cpuid((int*)cpuInfo, (int)id); }
+ #else
+ inline void cpuid(unsigned int cpuInfo[4], unsigned int id) { __get_cpuid(id, &cpuInfo[0], &cpuInfo[1], &cpuInfo[2], &cpuInfo[3]); }
+ #endif
+#endif
+bool ReedSolomon::setSingleInstructionMultipleData(uint8_t exponent)
+{
+ if (exponent == (uint8_t )-1)
+ {
+ /*
+ if (setSingleInstructionMultipleData(6))
+ return true;
+ if (setSingleInstructionMultipleData(5))
+ return true;
+ if (setSingleInstructionMultipleData(4))
+ return true;
+ */
+ if (setSingleInstructionMultipleData(0))
+ return true;
+ return false;
+ }
+
+ #if defined(__x86_64) || defined(_M_AMD64)
+ unsigned int cpuInfo[4];
+ #endif
+
+ switch (exponent)
+ {
+ case 0: break; // No SIMD, supported everywhere
+ case 4:
+ #if defined(__x86_64) || defined(_M_AMD64)
+ break; // SSE3 everywhere
+ #else
+ return false;
+ #endif
+ case 5:
+ #if defined(__x86_64) || defined(_M_AMD64)
+ cpuid(cpuInfo, 7);
+ if (cpuInfo[1] && (1 << 5))
+ break; // AVX2 supported
+ #else
+ return false;
+ #endif
+ case 6:
+ #if defined(__x86_64) || defined(_M_AMD64)
+ cpuid(cpuInfo, 7);
+ if (cpuInfo[1] && ((1 << 16) | (1 << 30)))
+ break; // AVX512F + AVX512BW supported
+ #else
+ return false;
+ #endif
+ default:
+ return false; // Supported nowhere
+ }
+
+ singleInstructionMultipleData = exponent;
+ return true;
+}
+
+static int processedBufferSize = 4096;
+bool ReedSolomon::setProcessedBufferSize(size_t bufferSize)
+{
+ processedBufferSize = static_cast(bufferSize);
+ return true;
+}
+
+static size_t processedOrder = 0;
+bool ReedSolomon::setProcessedOrder(uint8_t order)
+{
+ processedOrder = order;
+ return true;
+}
+
+void ReedSolomon::encode(uint8_t* __restrict__ const* __restrict__ const shards, size_t shardByteSize) const
+{
+ // Build the array of data/parity shards
+ auto dataShards = &shards[0];
+ auto parityShards = &shards[dataShardCount];
+
+ // Do the coding
+ process(matrixRows, dataShards, parityShards, parityShardCount, shardByteSize);
+}
+
+bool ReedSolomon::verify(uint8_t const* __restrict__ const* __restrict__ const shards, size_t shardByteSize) const
+{
+ // Build the array of data/parity shards
+ auto dataShards = &shards[0];
+ auto parityShards = &shards[dataShardCount];
+
+ // Create fake parity shards
+ uint8_t* newParityShardsPointer = new uint8_t[parityShardCount * shardByteSize];
+ memset(newParityShardsPointer, 0, parityShardCount * shardByteSize);
+ uint8_t** newParityShards = new uint8_t * [parityShardCount];
+ for (uint8_t i = 0; i < parityShardCount; i++)
+ newParityShards[i] = &newParityShardsPointer[i * shardByteSize];
+
+ // Do the coding as if there is no parity shards in the input
+ process(matrixRows, dataShards, newParityShards, parityShardCount, shardByteSize);
+
+ bool ok = true;
+ for (uint8_t r = 0; r < parityShardCount; r++)
+ if (memcmp(newParityShards[r], parityShards[r], shardByteSize))
+ ok = false;
+
+ delete[] newParityShards;
+ delete[] newParityShardsPointer;
+ return ok;
+
+}
+
+bool ReedSolomon::repair(uint8_t* __restrict__ const* __restrict__ const shards, size_t shardByteSize, std::vector shardPresent) const
+{
+ // Integrity check
+ if (shardPresent.size() != totalShardCountMinus1 + 1)
+ return false;
+
+ // Quick check: are all of the shards present? If so, there's nothing to do.
+ uint16_t number_present = 0;
+ {
+ uint8_t i = 0;
+ do
+ {
+ if (shardPresent[i])
+ number_present++;
+ }
+ while (i++ != totalShardCountMinus1);
+ }
+ if (number_present == totalShardCountMinus1 + 1)
+ {
+ // All of the shards data data. We don't need to do anything.
+ return true;
+ }
+ if (number_present < dataShardCount)
+ {
+ // Not enough valid shards. We don't need to do anything.
+ return false;
+ }
+
+ // Init invalid shard content to 0,
+ // needed for process()
+ {
+ uint8_t i = 0;
+ do
+ {
+ if (!shardPresent[i])
+ memset(shards[i], 0, shardByteSize);
+ }
+ while (i++ != totalShardCountMinus1);
+ }
+
+ // Pull out the rows of the matrix that correspond to the
+ // shards that we have and build a square matrix. This
+ // matrix could be used to generate the shards that we have
+ // from the original data.
+ //
+ // Also, pull out an array holding just the shards that
+ // correspond to the rows of the submatrix. These shards
+ // will be the input to the decoding process that re-creates
+ // the missing data shards.
+ Matrix dataDecodeMatrix{ static_cast(dataShardCount - 1), static_cast(dataShardCount - 1) };
+ const uint8_t ** validDataShards = new const uint8_t * [dataShardCount];
+ uint8_t validDataShardCount = 0;
+ {
+ uint8_t r = 0;
+ do
+ {
+ if (shardPresent[r])
+ {
+ for (uint8_t c = 0; c < dataShardCount; ++c)
+ dataDecodeMatrix.set(validDataShardCount, c, matrix.get(r, c));
+ validDataShards[validDataShardCount] = shards[r];
+ validDataShardCount++;
+ }
+ }
+ while (r++ <= totalShardCountMinus1 && validDataShardCount < dataShardCount);
+ }
+
+ // Invert the matrix, so we can go from the encoded shards
+ // back to the original data. Then pull out the row that
+ // generates the shard that we want to decode. Note that
+ // since this matrix maps back to the orginal data, it can
+ // be used to create a data shard, but not a parity shard.
+ dataDecodeMatrix.invert();
+
+ // Re-create any data shards that were missing.
+ //
+ // The input to the coding is all of the shards we actually
+ // have, and the output is the missing data shards. The computation
+ // is done using the special decode matrix we just built.
+ uint8_t** newParityShards = new uint8_t * [parityShardCount];
+ const uint8_t** validMatrixRows = new const uint8_t * [parityShardCount];
+ uint8_t newParityShardCount = 0;
+ for (uint8_t i = 0; i < dataShardCount; i++)
+ {
+ if (!shardPresent[i])
+ {
+ newParityShards[newParityShardCount] = shards[i];
+ validMatrixRows[newParityShardCount] = dataDecodeMatrix.getRow(i);
+ newParityShardCount++;
+ }
+ }
+ process(validMatrixRows, validDataShards, newParityShards, newParityShardCount, shardByteSize);
+
+ // Now that we have all of the data shards intact, we can
+ // compute any of the parity that is missing.
+ //
+ // The input to the coding is ALL of the data shards, including
+ // any that we just calculated. The output is whichever of the
+ // data shards were missing.
+ newParityShardCount = 0;
+ {
+ uint16_t i = dataShardCount;
+ do
+ {
+ if (!shardPresent[i])
+ {
+ newParityShards[newParityShardCount] = shards[i];
+ validMatrixRows[newParityShardCount] = matrixRows[i - dataShardCount];
+ newParityShardCount++;
+ }
+ }
+ while (i++ != totalShardCountMinus1);
+ }
+ const uint8_t* __restrict__ * __restrict__ dataShards = const_cast(&shards[0]);
+ process(validMatrixRows, dataShards, newParityShards, newParityShardCount, shardByteSize);
+
+ delete[] validDataShards;
+ delete[] newParityShards;
+ delete[] validMatrixRows;
+
+ return true;
+}
+
+/**
+ * Multiplies a subset of rows from a coding matrix by a full set of
+ * input shards to produce some output shards.
+ *
+ * @param validMatrixRows The rows from the matrix to use.
+ * @param validDataShards An array of byte arrays, each of which is one input shard.
+ * The inputs array may have extra buffers after the ones
+ * that are used. They will be ignored. The number of
+ * inputs used is determined by the length of the
+ * each matrix row.
+ * @param newParityShards Byte arrays where the computed shards are stored. The
+ * outputs array may also have extra, unused, elements
+ * at the end. The number of outputs computed, and the
+ * number of matrix rows used, is determined by
+ * outputCount.
+ * @param newParityShardCount The number of newParityShards to compute.
+ * @param shardByteSize The number of bytes to process.
+ */
+void ReedSolomon::process(uint8_t const* __restrict__ const* __restrict__ const validMatrixRows, uint8_t const* __restrict__ const* __restrict__ const validDataShards, uint8_t* __restrict__ const* __restrict__ const newParityShards, uint8_t newParityShardCount, size_t shardByteSize) const
+{
+ size_t loop1, loop2, loop3;
+ switch (processedOrder)
+ {
+ case 3 :
+ loop1 = shardByteSize / processedBufferSize;
+ shardByteSize = processedBufferSize;
+ loop2 = dataShardCount;
+ loop3 = newParityShardCount;
+ break;
+ case 2:
+ loop1 = shardByteSize / processedBufferSize;
+ shardByteSize = processedBufferSize;
+ loop2 = newParityShardCount;
+ loop3 = dataShardCount;
+ break;
+ case 1 :
+ loop1 = dataShardCount;
+ loop2 = newParityShardCount;
+ loop3 = 1;
+ break;
+ default :
+ loop1 = newParityShardCount;
+ loop2 = dataShardCount;
+ loop3 = 1;
+ break;
+ }
+
+ for (int l1 = 0; l1 < loop1; l1++)
+ for (int l2 = 0; l2 < loop2; l2++)
+ for (int l3 = 0; l3 < loop3; l3++)
+ {
+ int r, c, o;
+ switch (processedOrder)
+ {
+ case 3 :
+ r = l3;
+ c = l2;
+ o = l1 * processedBufferSize;
+ break;
+ case 2:
+ r = l2;
+ c = l3;
+ o = l1 * processedBufferSize;
+ break;
+ case 1 :
+ r = l2;
+ c = l1;
+ o = 0;
+ break;
+ default :
+ r = l1;
+ c = l2;
+ o = 0;
+ break;
+ }
+
+ uint8_t matrixValue = validMatrixRows[r][c];
+ const uint8_t* __restrict__ dataShard = validDataShards[c] + o;
+ uint8_t * __restrict__ const newParityShard = newParityShards[r] + o;
+
+ if (singleInstructionMultipleData == 0)
+ {
+ for (size_t i = 0; i < shardByteSize; i++)
+ newParityShard[i] ^= galois.MultiplicationTable[matrixValue][dataShard[i]];
+ continue;
+ }
+
+ /*
+ size_t end = shardByteSize & (~((size_t(1) << singleInstructionMultipleData) - 1));
+
+ if (singleInstructionMultipleData == 4)
+ {
+ const __m128i* __restrict__ input_ptr = reinterpret_cast(&dataShard[0]);
+ __m128i* __restrict__ output_ptr = reinterpret_cast<__m128i*>(&newParityShard[0]);
+ const __m128i* __restrict__ input_end = input_ptr + end / 16;
+
+ const __m128i low_table = _mm_loadu_si128(reinterpret_cast(galois.MultiplicationTableSplit[matrixValue].Low));
+ const __m128i high_table = _mm_loadu_si128(reinterpret_cast(galois.MultiplicationTableSplit[matrixValue].High));
+ const __m128i mask = _mm_set1_epi8(0x0F);
+ while (input_ptr < input_end)
+ {
+ __m128i input = _mm_loadu_si128(input_ptr++);
+ __m128i low_indices = _mm_and_si128(input, mask);
+ __m128i low_parts = _mm_shuffle_epi8(low_table, low_indices);
+ __m128i high_indices = _mm_and_si128(_mm_srli_epi32(input, 4), mask); // Equivalent to _mm_srli_epi8(input, 4);
+ __m128i high_parts = _mm_shuffle_epi8(high_table, high_indices);
+ __m128i intermediate = _mm_xor_si128(low_parts, high_parts);
+ _mm_storeu_si128(output_ptr, _mm_xor_si128(_mm_loadu_si128(output_ptr), intermediate)); // output = output XOR intermediate
+ output_ptr++;
+ }
+ }
+
+ if (singleInstructionMultipleData == 5)
+ {
+ const __m256i* __restrict__ input_ptr = reinterpret_cast(&dataShard[0]);
+ __m256i* __restrict__ output_ptr = reinterpret_cast<__m256i*>(&newParityShard[0]);
+ const __m256i* __restrict__ input_end = input_ptr + end / 32;
+
+ const __m256i low_table = _mm256_loadu_si256(reinterpret_cast(galois.MultiplicationTableSplit[matrixValue].Low));
+ const __m256i high_table = _mm256_loadu_si256(reinterpret_cast(galois.MultiplicationTableSplit[matrixValue].High));
+ const __m256i mask = _mm256_set1_epi8(0x0F);
+ while (input_ptr < input_end)
+ {
+ __m256i input = _mm256_loadu_si256(input_ptr++);
+ __m256i low_indices = _mm256_and_si256(input, mask);
+ __m256i low_parts = _mm256_shuffle_epi8(low_table, low_indices);
+ __m256i high_indices = _mm256_and_si256(_mm256_srli_epi32(input, 4), mask); // Equivalent to _mm256_srli_epi8(input, 4);
+ __m256i high_parts = _mm256_shuffle_epi8(high_table, high_indices);
+ __m256i intermediate = _mm256_xor_si256(low_parts, high_parts);
+ _mm256_storeu_si256(output_ptr, _mm256_xor_si256(_mm256_loadu_si256(output_ptr), intermediate)); // output = output XOR intermediate
+ output_ptr++;
+ }
+ }
+
+ if (singleInstructionMultipleData == 6)
+ {
+ const __m512i* __restrict__ input_ptr = reinterpret_cast(&dataShard[0]);
+ __m512i* __restrict__ output_ptr = reinterpret_cast<__m512i*>(&newParityShard[0]);
+ const __m512i* __restrict__ input_end = input_ptr + end / 64;
+
+ const __m512i low_table = _mm512_loadu_si512(reinterpret_cast(galois.MultiplicationTableSplit[matrixValue].Low));
+ const __m512i high_table = _mm512_loadu_si512(reinterpret_cast(galois.MultiplicationTableSplit[matrixValue].High));
+ const __m512i mask = _mm512_set1_epi8(0x0F);
+ while (input_ptr < input_end)
+ {
+ __m512i input = _mm512_loadu_si512(input_ptr++);
+ __m512i low_indices = _mm512_and_si512(input, mask);
+ __m512i low_parts = _mm512_shuffle_epi8(low_table, low_indices);
+ __m512i high_indices = _mm512_and_si512(_mm512_srli_epi32(input, 4), mask); // Equivalent to _mm512_srli_epi8(input, 4);
+ __m512i high_parts = _mm512_shuffle_epi8(high_table, high_indices);
+ __m512i intermediate = _mm512_xor_si512(low_parts, high_parts);
+ _mm512_storeu_si512(output_ptr, _mm512_xor_si512(_mm512_loadu_si512(output_ptr), intermediate)); // output = output XOR intermediate
+ output_ptr++;
+ }
+ }
+
+ for (; end < shardByteSize; end++)
+ newParityShard[end] ^= galois.MultiplicationTable[matrixValue][dataShard[end]];
+ */
+ }
+}
diff --git a/Source/Lib/ThirdParty/Reed-Solomon/Reed-Solomon.h b/Source/Lib/ThirdParty/Reed-Solomon/Reed-Solomon.h
new file mode 100644
index 00000000..3e3ec59d
--- /dev/null
+++ b/Source/Lib/ThirdParty/Reed-Solomon/Reed-Solomon.h
@@ -0,0 +1,129 @@
+/* Copyright (c) MediaArea.net SARL.
+ *
+ * Use of this source code is governed by a MIT-style license that can
+ * be found in the License.html file in the root of the source tree.
+ */
+
+ //---------------------------------------------------------------------------
+#ifndef ReedSolomonH
+#define ReedSolomonH
+//---------------------------------------------------------------------------
+
+//---------------------------------------------------------------------------
+#include "Reed-Solomon/Matrix.h"
+#include
+using size_t = decltype(sizeof 1);
+//---------------------------------------------------------------------------
+
+//---------------------------------------------------------------------------
+#if defined (_MSC_VER)
+#define __restrict__ __restrict
+#endif
+//---------------------------------------------------------------------------
+
+/**
+ * Reed-Solomon Coding over 8-bit values.
+ */
+
+class ReedSolomon
+{
+public:
+ /**
+ * Initializes a new encoder/decoder.
+ */
+ ReedSolomon(uint8_t dataShardCount_, uint8_t parityShardCount_);
+ ~ReedSolomon();
+
+ /**
+ * Set SIMD (Single Instruction Multiple Data)
+ * @param exponent Exponent of 2 of count of bytes in a single instruction
+ * e.g. 4 for SSE (128-bit)
+ * if set to -1, the highest exponent avalaible is selected
+ * @return false if exponent is not supported, else true
+ */
+ static bool setSingleInstructionMultipleData(uint8_t exponent = -1);
+
+ /**
+ * Set processed buffer size
+ * @return false if size is not supported, else true
+ */
+ static bool setProcessedBufferSize(size_t bufferSize);
+
+ /**
+ * Set encoding order
+ * @param order 0 =
+ *
+ * @return false if order is not supported, else true
+ */
+ static bool setProcessedOrder(uint8_t order);
+
+ /**
+ * Returns false if constructor failed to initialize.
+ */
+ bool isValid() const { return !totalShardCountMinus1; }
+
+ /**
+ * Returns the number of data shards.
+ */
+ uint8_t getDataShardCount() const { return dataShardCount; }
+
+ /**
+ * Returns the number of parity shards.
+ */
+ uint8_t getParityShardCount() const { return parityShardCount; }
+
+ /**
+ * Returns the total number of shards.
+ */
+ uint16_t getTotalShardCount() const { return totalShardCountMinus1 + 1; }
+
+ /**
+ * Encodes parity for a set of data shards.
+ *
+ * @param shards An array containing data shards followed by parity shards.
+ * Each shard is a byte array, and they must all be the same
+ * size.
+ * @param shardByteSize The number of bytes to encode in each shard.
+ *
+ */
+ void encode(uint8_t* __restrict__ const* __restrict__ const shards, size_t shardByteSize) const;
+
+ /**
+ * Returns true if the parity shards contain the right data.
+ *
+ * @param shards An array containing data shards followed by parity shards.
+ * Each shard is a byte array, and they must all be the same
+ * size.
+ * @param shardByteSize The number of bytes to check in each shard.
+ */
+ bool verify(uint8_t const* __restrict__ const* __restrict__ const shards, size_t shardByteSize) const;
+
+ /**
+ * Given a list of shards, some of which contain data, fills in the
+ * ones that don't have data.
+ *
+ * Quickly does nothing if all of the shards are present.
+ *
+ * If any shards are missing (based on the flags in shardsPresent),
+ * the data in those shards is recomputed and filled in.
+ */
+ bool repair(uint8_t* __restrict__ const* __restrict__ const shards, size_t shardByteSize, std::vector shardPresent) const;
+
+private:
+ void process(uint8_t const* __restrict__ const* __restrict__ const validMatrixRows, uint8_t const* __restrict__ const* __restrict__ const validDataShards, uint8_t* __restrict__ const* __restrict__ const newParityShards, uint8_t newParityShardCount, size_t shardByteSize) const;
+
+private:
+ // Members
+ const uint8_t dataShardCount;
+ const uint8_t parityShardCount;
+ const uint8_t totalShardCountMinus1;
+
+ /**
+ * Rows from the matrix for encoding parity, each one as its own
+ * byte array to allow for efficient access while encoding.
+ */
+ uint8_t const* __restrict* __restrict__ matrixRows;
+ Matrix matrix;
+};
+
+#endif
\ No newline at end of file