From f48c6b435cbfcb6c2453ffd546c2820718e18b73 Mon Sep 17 00:00:00 2001 From: RaphaelIT7 Date: Fri, 9 Feb 2024 00:02:02 +0100 Subject: [PATCH 1/2] Added Addon::Writer + A few other changes [+] Added -lowmemory option to write everything directly to the .gma file Reduces the needed memory to only the size of the biggest file in the entire addon. [+] Added -lowmemory and -nocrc to the help list. [#] Reallocating the file buffer only once instead of every time. [#] if necessary, write the .gma in chunks if the buffer gets too full. NOTE: If this happens we cannot create a CRC of the .gma because at this point it would probably fail to load or cause a crash. --- include/AddonWriter.h | 178 ++++++++++++++++++++++++++++++++++++++++++ src/create_gmad.cpp | 77 +++++++++++------- src/gmadd.cpp | 2 + 3 files changed, 228 insertions(+), 29 deletions(-) create mode 100644 include/AddonWriter.h diff --git a/include/AddonWriter.h b/include/AddonWriter.h new file mode 100644 index 0000000..39e947b --- /dev/null +++ b/include/AddonWriter.h @@ -0,0 +1,178 @@ + +#ifndef ADDONWRITER_H +#define ADDONWRITER_H + +#include +#include "Bootil/Bootil.h" +#include "AddonFormat.h" + +// +// +// +// +namespace Addon +{ + class Writer : public Bootil::AutoBuffer + { + public: + + Writer( Bootil::BString strOutput, bool nobuffer ) + { + m_usebuffer = !nobuffer; + m_writeblocks = false; + m_output = strOutput; + m_written = 0; + + remove( strOutput.c_str() ); + m_file.open( strOutput, std::ios::out | std::ios::binary | std::ios::app ); + if ( !m_file.is_open() ) + { + Bootil::Output::Error( "Failed to open %s!\n", strOutput ); + } + } + + uint64_t GetWritten() + { + if ( m_usebuffer ) + return AutoBuffer::GetWritten() + m_written; + + return m_written; + } + + bool EnsureCapacity( uint64_t iSize ) + { + if ( m_usebuffer ) + { + uint64_t max = INT_MAX / 1.5; // ~1.4GB will be ready for the buffer. If we try to use more, it will fail to allocate it for some reason. + if ( iSize > max ) + { + m_writeblocks = true; + iSize = max; + } + + return AutoBuffer::EnsureCapacity( iSize ); + } + + return true; + } + + void Reset() + { + m_iPos = 0; + m_iWritten = 0; + } + + void WriteBuffer( const Bootil::Buffer & bufferOut ) + { + if ( m_usebuffer ) + { + if ( m_writeblocks && ( this->GetSize() - AutoBuffer::GetWritten() ) <= bufferOut.GetWritten() ) + { + m_file.write( reinterpret_cast( this->GetBase() ), AutoBuffer::GetWritten() ); + m_written += AutoBuffer::GetWritten(); + Reset(); + + if ( bufferOut.GetWritten() >= this->GetSize() ) + { + m_file.write( reinterpret_cast( bufferOut.GetBase() ), bufferOut.GetWritten() ); + m_written += bufferOut.GetWritten(); + } + else + { + AutoBuffer::WriteBuffer( bufferOut ); + } + } + else + { + AutoBuffer::WriteBuffer( bufferOut ); + } + } + else + { + if ( AutoBuffer::GetWritten() > 0 ) + { + m_file.write( reinterpret_cast( this->GetBase() ), AutoBuffer::GetWritten() ); + m_written += AutoBuffer::GetWritten(); + Reset(); + } + + m_file.write( reinterpret_cast( bufferOut.GetBase() ), bufferOut.GetWritten() ); + m_written += bufferOut.GetWritten(); + } + } + + bool WriteFile() + { + if ( m_usebuffer ) + { + if ( m_writeblocks ) + { + m_written += AutoBuffer::GetWritten(); + m_file.write( reinterpret_cast( this->GetBase() ), AutoBuffer::GetWritten() ); + return true; + } + else + { + return Bootil::File::Write( m_output, *(Bootil::AutoBuffer*)this ); + } + } + + Close(); + + return true; + } + + uint32_t CRC() + { + if ( !m_usebuffer || m_writeblocks ) + return 0; + + return Bootil::Hasher::CRC32::Easy( this->GetBase(), this->GetWritten() ); + } + + void Close() + { + if ( m_usebuffer ) + { + AutoBuffer::Clear(); + } + else + { + m_file.close(); + } + } + + Bootil::BString FormatSize() // Bootil::String::Format::Memory has a 2GB limit. + { + uint64_t iBytes = GetWritten(); + float gb = iBytes / 1024.0f / 1024.0f / 1024.0f; + if ( gb >= 1.0 ) + { + return Bootil::String::Format::Print( "%.1f GB", gb ); + } + + float mb = iBytes / 1024.0f / 1024.0f; + if ( mb >= 1.0 ) + { + return Bootil::String::Format::Print( "%.1f MB", mb ); + } + + float kb = iBytes / 1024.0f; + if ( kb >= 1.0 ) + { + return Bootil::String::Format::Print( "%.1f KB", kb ); + } + + return Bootil::String::Format::Print( "%i B", iBytes ); + } + + private: + Bootil::BString m_output; + std::ofstream m_file; + bool m_usebuffer; + bool m_writeblocks; + uint64_t m_written; + }; +} + +#endif \ No newline at end of file diff --git a/src/create_gmad.cpp b/src/create_gmad.cpp index 4be185b..d666758 100644 --- a/src/create_gmad.cpp +++ b/src/create_gmad.cpp @@ -3,6 +3,7 @@ #include "AddonWhiteList.h" #include "AddonFormat.h" #include "Addon_Json.h" +#include "AddonWriter.h" #include using namespace Bootil; @@ -76,7 +77,7 @@ namespace CreateAddon // // Create an uncompressed GMAD file from a list of files // - bool Create( AutoBuffer& buffer, BString strFolder, String::List& files, BString strTitle, BString strDescription ) + bool Create( Addon::Writer& writer, BString strFolder, String::List& files, BString strTitle, BString strDescription ) { bool quiet = CommandLine::HasSwitch( "-quiet" ); @@ -84,23 +85,33 @@ namespace CreateAddon bool doCRCs = true; if ( CommandLine::HasSwitch( "-nocrc" ) ) doCRCs = false; + // Appends the file content directly to the .gma instead of writing everything in memory. + bool lowMemory = false; + if ( CommandLine::HasSwitch( "-lowmemory" ) ) lowMemory = true; + + if ( doCRCs && lowMemory ) + { + Output::Warning( "CRCs have been disabled due to -lowmemory being enabled!\n" ); + doCRCs = false; + } + // Header (5) - buffer.Write( Addon::Ident, 4 ); // Ident (4) - buffer.WriteType( ( char ) Addon::Version ); // Version (1) + writer.Write( Addon::Ident, 4 ); // Ident (4) + writer.WriteType( ( char ) Addon::Version ); // Version (1) // SteamID (8) [unused] - buffer.WriteType( ( uint64_t ) 0ULL ); + writer.WriteType( ( uint64_t ) 0ULL ); // TimeStamp (8) - buffer.WriteType( ( uint64_t ) Time::UnixTimestamp() ); + writer.WriteType( ( uint64_t ) Time::UnixTimestamp() ); // Required content (a list of strings) - buffer.WriteType( ( char ) 0 ); // signifies nothing + writer.WriteType( ( char ) 0 ); // signifies nothing // Addon Name (n) - buffer.WriteString( strTitle ); + writer.WriteString( strTitle ); // Addon Description (n) - buffer.WriteString( strDescription ); + writer.WriteString( strDescription ); // Addon Author (n) [unused] - buffer.WriteString( "Author Name" ); + writer.WriteString( "Author Name" ); // Addon Version (4) [unused] - buffer.WriteType( ( int32_t ) 1 ); + writer.WriteType( ( int32_t ) 1 ); Output::Msg( "Writing file list...\n" ); @@ -110,7 +121,7 @@ namespace CreateAddon iFileListSize = iFileListSize + 4 + 180 + 8 + 4; // File number (4) + File name (180 file name length limit) + File size (8) + File CRC (4) } - if ( !buffer.EnsureCapacity( buffer.GetWritten() + iFileListSize ) ) + if ( !writer.EnsureCapacity( writer.GetWritten() + iFileListSize ) ) { Output::Warning( "Failed to allocate buffer. Expect problems!\n" ); } @@ -118,6 +129,7 @@ namespace CreateAddon // File list uint32_t iFileNum = 0; uint64_t iTotalSize = 0; + uint64_t iBiggestFile = 0; BOOTIL_FOREACH( f, files, String::List ) { int64_t iSize = File::Size( strFolder + *f ); @@ -129,40 +141,47 @@ namespace CreateAddon iFileNum++; iTotalSize = iTotalSize + iSize; - buffer.WriteType( ( uint32_t ) iFileNum ); // File number (4) - buffer.WriteString( String::GetLower( *f ) ); // File name (all lower case!) (n) - buffer.WriteType( ( int64_t ) iSize ); // File size (8) + writer.WriteType( ( uint32_t ) iFileNum ); // File number (4) + writer.WriteString( String::GetLower( *f ) ); // File name (all lower case!) (n) + writer.WriteType( ( int64_t ) iSize ); // File size (8) + + if ( iSize > iBiggestFile ) + iBiggestFile = iSize; if ( doCRCs ) { uint32_t iCRC = File::CRC( strFolder + *f ); - buffer.WriteType( ( uint32_t ) iCRC ); // File CRC (4) + writer.WriteType( ( uint32_t ) iCRC ); // File CRC (4) } else { - buffer.WriteType( ( uint32_t ) 0 ); + writer.WriteType( ( uint32_t ) 0 ); } //Output::Msg( "\tFile index: %s [CRC:%u] [Size:%s]\n", f->c_str(), iCRC, String::Format::Memory( iSize ).c_str() ); } - if ( !buffer.EnsureCapacity( buffer.GetWritten() + iTotalSize + 8 ) ) + if ( !writer.EnsureCapacity( writer.GetWritten() + iTotalSize + 8 ) ) { Output::Warning( "Failed to allocate buffer. Expect problems!\n" ); } // Zero to signify end of files iFileNum = 0; - buffer.WriteType( ( uint32_t ) iFileNum ); + writer.WriteType( ( uint32_t ) iFileNum ); Output::Msg( "Writing files...\n" ); + AutoBuffer filebuffer; + filebuffer.EnsureCapacity(iBiggestFile + 32); + // The files BOOTIL_FOREACH( f, files, String::List ) { if ( !quiet ) Output::Msg( "\tWriting %s...\n", f->c_str() ); - AutoBuffer filebuffer; + filebuffer.SetPos(0); + filebuffer.SetWritten(0); bool res = File::Read( strFolder + *f, filebuffer ); //Output::Msg( "\tReading %s bool = %i %u\n", f->c_str(), res, filebuffer.GetWritten() ); @@ -172,9 +191,10 @@ namespace CreateAddon return false; } - uint64_t before = buffer.GetWritten(); - buffer.WriteBuffer( filebuffer ); - uint64_t diff = buffer.GetWritten() - before; + uint64_t before = writer.GetWritten(); + writer.WriteBuffer( filebuffer ); + uint64_t diff = writer.GetWritten() - before; + if ( diff < 1 ) { Output::Warning( "Failed to write file '%s' - written %llu bytes! (Can't grow buffer?)\n", ( *f ).c_str(), diff ); @@ -185,12 +205,11 @@ namespace CreateAddon // CRC what we've written (to verify that the download isn't shitted) (4) if ( doCRCs ) { - uint32_t AddonCRC = Hasher::CRC32::Easy( buffer.GetBase(), buffer.GetWritten() ); - buffer.WriteType( AddonCRC ); + writer.WriteType( writer.CRC() ); } else { - buffer.WriteType( 0 ); + writer.WriteType( 0 ); } return true; @@ -261,8 +280,8 @@ int CreateAddonFile( BString strFolder, BString strOutfile, bool warnInvalid ) // // Create an addon file in a buffer // - AutoBuffer buffer( 256 ); - if ( !CreateAddon::Create( buffer, strFolder, files, addoninfo.GetTitle(), addoninfo.BuildDescription() ) ) + Addon::Writer writer( strOutfile, CommandLine::HasSwitch( "-lowmemory" ) ); + if ( !CreateAddon::Create( writer, strFolder, files, addoninfo.GetTitle(), addoninfo.BuildDescription() ) ) { Output::Warning( "Failed to create the addon\n" ); return 1; @@ -273,7 +292,7 @@ int CreateAddonFile( BString strFolder, BString strOutfile, bool warnInvalid ) // // Save the buffer to the provided name // - if ( !File::Write( strOutfile, buffer ) ) + if ( !writer.WriteFile() ) { Output::Warning( "Couldn't save to file \"%s\"\n", strOutfile.c_str() ); return 1; @@ -282,7 +301,7 @@ int CreateAddonFile( BString strFolder, BString strOutfile, bool warnInvalid ) // // Success! // - Output::Msg( "Successfully saved to \"%s\" [%s]\n", strOutfile.c_str(), String::Format::Memory( buffer.GetWritten() ).c_str() ); + Output::Msg( "Successfully saved to \"%s\" [%s]\n", strOutfile.c_str(), writer.FormatSize().c_str() ); Output::Msg( "Files ignored: %i\n", addoninfo.m_IgnoredFiles ); return 0; } diff --git a/src/gmadd.cpp b/src/gmadd.cpp index 0c0b738..2d16618 100644 --- a/src/gmadd.cpp +++ b/src/gmadd.cpp @@ -80,6 +80,8 @@ int main( int argc, char* argv[] ) Output::Msg("\tAdd -warninvalid to automatically skip invalid files\n\n"); Output::Msg("\tAdd -quiet to not spam file paths to output\n\n"); + Output::Msg("\tAdd -nocrc to not write crcs to the output\n\n"); + Output::Msg("\tAdd -lowmemory to write directly to the output file\n\n"); #ifdef _WIN32 // Make sure they see how to use it From 7f35b5cc0f6f72ef42bd4f5db478a884b2fd0a28 Mon Sep 17 00:00:00 2001 From: RaphaelIT7 Date: Sat, 28 Dec 2024 18:43:46 +0100 Subject: [PATCH 2/2] Optimize a few things [#] Optimized CRC to use the file buffer instead of reading the file again [#] Highly optimized file whitelist [#] Optimized RemoveIgnoredFiles --- include/AddonWhiteList.h | 85 +++++++++++++++++++++++++++++----------- include/Addon_Json.h | 53 ++++++++++++++++--------- include/Addon_Util.h | 64 ++++++++++++++++++++++++++++++ src/create_gmad.cpp | 51 +++++++++++++----------- 4 files changed, 188 insertions(+), 65 deletions(-) create mode 100644 include/Addon_Util.h diff --git a/include/AddonWhiteList.h b/include/AddonWhiteList.h index 949c518..2148e0b 100644 --- a/include/AddonWhiteList.h +++ b/include/AddonWhiteList.h @@ -3,16 +3,16 @@ #define ADDONWHITELIST_H #include "Bootil/Bootil.h" +#include "Addon_Util.h" -// -// -// -// namespace Addon { namespace WhiteList { - static const char* Wildcard[] = + // Entries that start with ! invalidate all previous matches + // Caveat: Order of these now matters + // Ideally Bootil would support something like /**/ for "only one folder" or "no slashes" (or backwards to mimic gitignore) + static const Bootil::BString Allowed[] = { "lua/*.lua", "scenes/*.vcd", @@ -35,12 +35,15 @@ namespace Addon "materials/*.jpeg", "materials/colorcorrection/*.raw", "models/*.mdl", - "models/*.vtx", "models/*.phy", "models/*.ani", "models/*.vvd", + + "models/*.vtx", + "gamemodes/*/*.txt", "gamemodes/*/*.fgd", + "gamemodes/*/logo.png", "gamemodes/*/icon24.png", "gamemodes/*/gamemode/*.lua", @@ -51,10 +54,12 @@ namespace Addon "gamemodes/*/backgrounds/*.jpg", "gamemodes/*/backgrounds/*.jpeg", "gamemodes/*/content/models/*.mdl", - "gamemodes/*/content/models/*.vtx", "gamemodes/*/content/models/*.phy", "gamemodes/*/content/models/*.ani", "gamemodes/*/content/models/*.vvd", + + "gamemodes/*/content/models/*.vtx", + "gamemodes/*/content/materials/*.vmt", "gamemodes/*/content/materials/*.vtf", "gamemodes/*/content/materials/*.png", @@ -76,40 +81,74 @@ namespace Addon // static version of the data/ folder // (because you wouldn't be able to modify these) + // We only allow filetypes here that are not already allowed above "data_static/*.txt", "data_static/*.dat", "data_static/*.json", "data_static/*.xml", "data_static/*.csv", - "data_static/*.dem", - "data_static/*.vcd", - "data_static/*.vtf", - "data_static/*.vmt", - "data_static/*.png", - "data_static/*.jpg", - "data_static/*.jpeg", + "", + }; - "data_static/*.mp3", - "data_static/*.wav", - "data_static/*.ogg", + static const Bootil::BString Blocked[] = + { + "!models/*.sw.vtx", // These variations are unused by the game + "!models/*.360.vtx", + "!models/*.xbox.vtx", - NULL + "!gamemodes/*/*/*.txt", // Only in the root gamemode folder please! + "!gamemodes/*/*/*.fgd", + + "!gamemodes/*/content/models/*.sw.vtx", + "!gamemodes/*/content/models/*.360.vtx", + "!gamemodes/*/content/models/*.xbox.vtx", + + "", }; // - // Call on a filename including relative path to determine - // whether file is allowed to be in the addon. + // Call on a filename including relative path to determine whether file is allowed to be in the addon. + // This whitelist only serves to warn about bad files at upload stage - the game has its own whitelist. // inline bool Check( const Bootil::BString& strname ) { bool bValid = false; - for ( int i = 0;; i++ ) + for ( int i = 0;; ++i ) + { + if ( WhiteList::Allowed[i].empty() ) break; + + if ( CheckWildcard( Allowed[i], strname ) ) + { + bValid = true; + break; + } + } + + for ( int i = 0;; ++i ) + { + if ( WhiteList::Blocked[i].empty() ) break; + + if ( CheckWildcard( Blocked[i], strname ) ) + { + bValid = false; + break; + } + } + + if ( !bValid ) { - if ( bValid || WhiteList::Wildcard[i] == NULL ) break; + for ( int i = 0;; ++i ) + { + if ( WhiteList::Allowed[i].empty() ) break; - bValid = Bootil::String::Test::Wildcard( Wildcard[i], strname ); + if ( CheckWildcard( Allowed[i], strname ) ) + { + bValid = true; + break; + } + } } return bValid; diff --git a/include/Addon_Json.h b/include/Addon_Json.h index ab26398..2e4d242 100644 --- a/include/Addon_Json.h +++ b/include/Addon_Json.h @@ -2,6 +2,8 @@ #ifndef ADDON_JSON_H #define ADDON_JSON_H +#include "Addon_Util.h" + class CAddonJson { public: @@ -113,54 +115,67 @@ class CAddonJson void RemoveIgnoredFiles( Bootil::String::List& files, bool quiet ) { - Bootil::String::List old_files = files; - files.clear(); + // We don't use them directly since the compiler then would create a std::string for every iteration... + static const Bootil::BString addon_json = "addon.json"; + static const Bootil::BString thumbs_db = "*thumbs.db"; + static const Bootil::BString desktop_ini = "*desktop.ini"; + static const Bootil::BString git = ".git*"; + static const Bootil::BString ds_storage = "*/.DS_Store"; + + static Bootil::BString strLow; + strLow.reserve( 255 ); - BOOTIL_FOREACH( f, old_files, Bootil::String::List ) + BOOTIL_FOREACH_MANUAL( f, files, Bootil::String::List ) { + Bootil::Output::Msg("%s\n", (*f).c_str()); bool bSkipFile = false; // // Never include our json file! // - if ( *f == "addon.json" ) continue; + if ( *f == addon_json ) bSkipFile = true; // // Don't include Windows specifiic files // - Bootil::BString strLow = Bootil::String::GetLower( *f ); - if ( Bootil::String::Test::Wildcard( "*thumbs.db", strLow ) ) continue; - if ( Bootil::String::Test::Wildcard( "*desktop.ini", strLow ) ) continue; + strLow.assign( *f ); + Addon::GetLower( strLow ); + + if ( Addon::CheckWildcard( thumbs_db, strLow ) ) bSkipFile = true; + if ( Addon::CheckWildcard( ds_storage, strLow ) ) bSkipFile = true; // Git stuff - if ( Bootil::String::Test::Wildcard( ".git*", strLow ) ) continue; + if ( Addon::CheckWildcard( git, strLow ) ) bSkipFile = true; // // Don't include OS X metadata files // - if ( *f == ".DS_Store" ) continue; - if ( Bootil::String::Test::Wildcard( "*/.DS_Store", *f ) ) continue; + if ( *f == ".DS_Store" ) bSkipFile = true; + if ( Addon::CheckWildcard( ds_storage, *f ) ) bSkipFile = true; // // Check against our loaded ignores list // - BOOTIL_FOREACH( ignore, m_Ignores, Bootil::String::List ) + if ( !bSkipFile ) { - if ( Bootil::String::Test::Wildcard( *ignore, *f ) ) + BOOTIL_FOREACH( ignore, m_Ignores, Bootil::String::List ) { - bSkipFile = true; - break; + + if ( Addon::CheckWildcard( *ignore, *f ) ) + { + bSkipFile = true; + break; + } } } - if ( !bSkipFile ) - { - files.push_back( *f ); - } - else + if ( bSkipFile ) { m_IgnoredFiles++; if ( !quiet ) Bootil::Output::Msg( "\tIgnored %s\n", f->c_str() ); + f = files.erase( f ); + } else { + ++f; // Manually incrementing it since if we were to increment after a file was ignored/erased we would skip the next element } } } diff --git a/include/Addon_Util.h b/include/Addon_Util.h new file mode 100644 index 0000000..5c8e84b --- /dev/null +++ b/include/Addon_Util.h @@ -0,0 +1,64 @@ +#ifndef ADDONUTIL_H +#define ADDONUTIL_H + +#include "Bootil/Bootil.h" + +// These should probably be added to Bootil itself, but since bootil didn't have any changes for 5+ years I expect nothing. + +// Iterator but you manually increment the iterator +#define BOOTIL_FOREACH_MANUAL( varname, arrayname, arraytype ) for ( arraytype::iterator varname = arrayname.begin(); varname != arrayname.end(); ) + +namespace Addon +{ + // + // From this URL: http://www.codeproject.com/KB/string/wildcmp.aspx + // Written by Jack Handy - jakkhandy@hotmail.com + // + + inline bool globber( const char *wild, const char *string ) + { + const char *cp = 0, *mp = 0; + + while ((*string) && (*wild != '*')) { + if ((*wild != *string) && (*wild != '?')) { + return false; + } + wild++; + string++; + } + + while (*string) { + if (*wild == '*') { + if (!*++wild) { + return true; + } + mp = wild; + cp = string+1; + } else if ((*wild == *string) || (*wild == '?')) { + wild++; + string++; + } else { + wild = mp; + string = cp++; + } + } + + while (*wild == '*') { + wild++; + } + return !*wild; + } + + // This is faster than Bootil::String::Test::Wildcard since were passing a reference instead of creating a copy. + inline bool CheckWildcard( const Bootil::BString& wildCard, const Bootil::BString& hayStack ) + { + return globber( wildCard.c_str(), hayStack.c_str() ); + } + + // NOTE: This directly modifies the given string instead of making a copy. + inline void GetLower( Bootil::BString& fileName ) + { + std::transform( fileName.begin(), fileName.end(), fileName.begin(), ::tolower ); + } +} +#endif \ No newline at end of file diff --git a/src/create_gmad.cpp b/src/create_gmad.cpp index d666758..7cfba2d 100644 --- a/src/create_gmad.cpp +++ b/src/create_gmad.cpp @@ -25,37 +25,39 @@ namespace CreateAddon Output::Warning( "No files found, can't continue!\n" ); bOk = false; } - - String::List old_files = files; - files.clear(); + + BString lowerFileName; + lowerFileName.reserve( 255 ); // // Print each found file, check they're ok // - BOOTIL_FOREACH( file, old_files, String::List ) + BOOTIL_FOREACH_MANUAL( file, files, String::List ) { if ( !quiet ) Output::Msg( "\t%s\n", file->c_str() ); + lowerFileName.assign( *file ); + Addon::GetLower( lowerFileName ); + // // Check the file against the whitelist // Lowercase the name (addon filesystem is case insensitive) // - if ( Addon::WhiteList::Check( String::GetLower( *file ) ) ) - { - files.push_back( *file ); - } - else + if ( !Addon::WhiteList::Check( lowerFileName ) ) { if ( !quiet ) Output::Warning( "\t\t[Not allowed by whitelist]\n" ); if ( !warnInvalid ) bOk = false; errorFiles.emplace( *file, "Not allowed by whitelist" ); - } + file = files.erase( file ); + } else { + ++file; + } // // Warn that we're gonna lowercase the filename // - if ( String::GetLower( *file ) != *file ) + if ( lowerFileName != *file ) { if ( !quiet ) Output::Warning( "\t\t[Filename contains captial letters]\n" ); } @@ -130,6 +132,7 @@ namespace CreateAddon uint32_t iFileNum = 0; uint64_t iTotalSize = 0; uint64_t iBiggestFile = 0; + std::unordered_map pCRCPos; BOOTIL_FOREACH( f, files, String::List ) { int64_t iSize = File::Size( strFolder + *f ); @@ -148,15 +151,8 @@ namespace CreateAddon if ( iSize > iBiggestFile ) iBiggestFile = iSize; - if ( doCRCs ) - { - uint32_t iCRC = File::CRC( strFolder + *f ); - writer.WriteType( ( uint32_t ) iCRC ); // File CRC (4) - } - else - { - writer.WriteType( ( uint32_t ) 0 ); - } + pCRCPos[ *f ] = writer.GetPos(); // We save this position to write the CRC later + writer.WriteType( ( uint32_t ) 0 ); // File CRC (4) //Output::Msg( "\tFile index: %s [CRC:%u] [Size:%s]\n", f->c_str(), iCRC, String::Format::Memory( iSize ).c_str() ); } @@ -200,16 +196,24 @@ namespace CreateAddon Output::Warning( "Failed to write file '%s' - written %llu bytes! (Can't grow buffer?)\n", ( *f ).c_str(), diff ); return false; } + + if ( doCRCs ) // Writing it here allows us to directly create the CRC without having to open & read the file again. + { + unsigned int currentPos = writer.GetPos(); + writer.SetPos( pCRCPos[ *f ] ); + writer.WriteType( ( uint32_t ) Bootil::Hasher::CRC32::Easy( filebuffer.GetBase(), filebuffer.GetWritten() ) ); + writer.SetPos( currentPos ); + } } // CRC what we've written (to verify that the download isn't shitted) (4) if ( doCRCs ) { - writer.WriteType( writer.CRC() ); + writer.WriteType( (uint32_t) writer.CRC() ); } else { - writer.WriteType( 0 ); + writer.WriteType( (uint32_t) 0 ); } return true; @@ -220,6 +224,7 @@ int CreateAddonFile( BString strFolder, BString strOutfile, bool warnInvalid ) { bool bErrors = false; bool quiet = CommandLine::HasSwitch( "-quiet" ); + Time::Timer startTime = Time::Timer(); // // Make sure there's a slash on the end @@ -301,7 +306,7 @@ int CreateAddonFile( BString strFolder, BString strOutfile, bool warnInvalid ) // // Success! // - Output::Msg( "Successfully saved to \"%s\" [%s]\n", strOutfile.c_str(), writer.FormatSize().c_str() ); + Output::Msg( "Successfully saved to \"%s\" [%s] in %f seconds \n", strOutfile.c_str(), writer.FormatSize().c_str(), startTime.Seconds() ); Output::Msg( "Files ignored: %i\n", addoninfo.m_IgnoredFiles ); return 0; }