feat: mmap

.clang-format

@@ -0,0 +1,162 @@
+# Adapted from llama.cpp
+# fab5d30ff6729ff6ff615c41e8c0215d6bc30393 by Diego Devesa
+---
+Language:        Cpp
+AlignAfterOpenBracket: Align
+AlignArrayOfStructures: Left
+AlignConsecutiveAssignments: AcrossComments
+AlignConsecutiveBitFields: AcrossComments
+AlignConsecutiveDeclarations: AcrossComments
+AlignConsecutiveMacros: AcrossComments
+# AlignConsecutiveShortCaseStatements: AcrossComments
+AlignEscapedNewlines: Left # LeftWithLastLine
+AlignOperands:   Align
+AlignTrailingComments:
+  Kind: Always
+  OverEmptyLines: 1
+AllowAllArgumentsOnNextLine: true
+AllowAllParametersOfDeclarationOnNextLine: false
+# AllowBreakBeforeNoexceptSpecifier: OnlyWithParen
+AllowShortBlocksOnASingleLine: Never
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: Inline
+AllowShortIfStatementsOnASingleLine: Never
+AllowShortLambdasOnASingleLine: Inline
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakBeforeMultilineStrings: true
+BinPackArguments: true
+BinPackParameters: true # OnePerLine
+BitFieldColonSpacing: Both
+BreakBeforeBraces: Custom # Attach
+BraceWrapping:
+  AfterCaseLabel:  true
+  AfterClass:      false
+  AfterControlStatement: false
+  AfterEnum:       false
+  AfterFunction:   false
+  AfterNamespace:  false
+  AfterObjCDeclaration: false
+  AfterStruct:     false
+  AfterUnion:      false
+  AfterExternBlock: false
+  BeforeCatch:     false
+  BeforeElse:      false
+  BeforeLambdaBody: false
+  BeforeWhile: false
+  IndentBraces:    false
+  SplitEmptyFunction: false
+  SplitEmptyRecord: false
+  SplitEmptyNamespace: false
+# BreakAdjacentStringLiterals: true
+BreakAfterAttributes: Never
+BreakBeforeBinaryOperators: None
+BreakBeforeInlineASMColon: OnlyMultiline
+BreakBeforeTernaryOperators: false
+# BreakBinaryOperations: Never
+BreakConstructorInitializers: AfterColon
+# BreakFunctionDefinitionParameters: false
+BreakInheritanceList: AfterComma
+BreakStringLiterals: true
+# BreakTemplateDeclarations: Yes
+ColumnLimit:     120
+CommentPragmas:  '^ IWYU pragma:'
+CompactNamespaces: false
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: false
+DerivePointerAlignment: false
+DisableFormat:   false
+EmptyLineBeforeAccessModifier: Leave
+EmptyLineAfterAccessModifier: Never
+ExperimentalAutoDetectBinPacking: false
+FixNamespaceComments: true
+IncludeBlocks:   Regroup
+IncludeCategories:
+  - Regex:           '^<.*\.h>'
+    Priority:        1
+    SortPriority:    0
+  - Regex:           '^<.*'
+    Priority:        2
+    SortPriority:    0
+  - Regex:           '.*'
+    Priority:        3
+    SortPriority:    0
+IncludeIsMainRegex: '([-_](test|unittest))?$'
+IncludeIsMainSourceRegex: ''
+IndentAccessModifiers: false
+IndentCaseBlocks: true
+IndentCaseLabels: true
+IndentExternBlock: NoIndent
+IndentGotoLabels: false
+IndentPPDirectives: AfterHash
+IndentWidth:     4
+IndentWrappedFunctionNames: false
+InsertBraces:    true # NOTE: may lead to incorrect formatting
+InsertNewlineAtEOF: true
+JavaScriptQuotes: Leave
+JavaScriptWrapImports: true
+KeepEmptyLinesAtTheStartOfBlocks: false
+LambdaBodyIndentation: Signature
+LineEnding: LF
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCBinPackProtocolList: Auto
+ObjCBlockIndentWidth: 4
+ObjCSpaceAfterProperty: true
+ObjCSpaceBeforeProtocolList: true
+PPIndentWidth: -1
+PackConstructorInitializers: CurrentLine
+PenaltyBreakAssignment: 2
+PenaltyBreakBeforeFirstCallParameter: 1
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyBreakTemplateDeclaration: 10
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 200
+PointerAlignment: Middle
+QualifierAlignment: Left
+#QualifierOrder: ['static', 'inline', 'friend', 'constexpr', 'const', 'volatile', 'type', 'restrict']
+RawStringFormats:
+  - Language:        Cpp
+    Delimiters:
+      - cc
+      - CC
+      - cpp
+      - Cpp
+      - CPP
+      - 'c++'
+      - 'C++'
+    CanonicalDelimiter: ''
+ReferenceAlignment: Middle
+ReflowComments:  false # IndentOnly
+SeparateDefinitionBlocks: Always
+SortIncludes:    CaseInsensitive
+SortUsingDeclarations: LexicographicNumeric
+SpaceAfterCStyleCast: true
+SpaceAfterLogicalNot: false
+SpaceAfterTemplateKeyword: true
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeCpp11BracedList: false
+SpaceBeforeCtorInitializerColon: true
+SpaceBeforeInheritanceColon: true
+SpaceBeforeParens: ControlStatements
+SpaceBeforeRangeBasedForLoopColon: true
+SpaceInEmptyBlock: false
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 2
+SpacesInAngles:  Never
+SpacesInContainerLiterals: true
+SpacesInLineCommentPrefix:
+  Minimum: 1
+  Maximum: -1
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+SpaceBeforeSquareBrackets: false
+Standard:        c++20
+TabWidth:        4
+UseTab:          Never
+WhitespaceSensitiveMacros: ['STRINGIZE']
+...

CMakeLists.txt

@@ -42,13 +42,7 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/")
 
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
 
-if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
-    set(TTS_STANDALONE ON)
-
-    include(git-vars)
-else()
-    set(TTS_STANDALONE OFF)
-endif()
+include(git-vars)
 
 option(BUILD_SHARED_LIBS "build shared libraries" ${BUILD_SHARED_LIBS_DEFAULT})
 
@@ -82,10 +76,8 @@ set(GGML_FATAL_WARNINGS     ${TTS_FATAL_WARNINGS})
 
 
 # build lib
-if (NOT TARGET ggml)
-    add_subdirectory(ggml)
-    # ... otherwise assume ggml is added by a parent CMakeLists.txt
-endif()
+add_subdirectory(ggml)
+add_subdirectory(ggml-patches)
 add_subdirectory(src)
 
 # install tts

examples/cli/cli.cpp

@@ -1,10 +1,12 @@
-#include "tts.h"
+#include <thread>
+
+#include "../../src/models/loaders.h"
 #include "args.h"
 #include "common.h"
+#include "ggml.h"
 #include "playback.h"
 #include "vad.h"
 #include "write_file.h"
-#include <thread>
 
 class tts_timing_printer {
     const int64_t start_us{[] {
@@ -64,24 +66,24 @@ int main(int argc, const char ** argv) {
         exit(1);
     }
 
-    generation_configuration * config = new generation_configuration(
-        args.get_string_param("--voice"), 
-        *args.get_int_param("--topk"), 
-        *args.get_float_param("--temperature"), 
-        *args.get_float_param("--repetition-penalty"), 
+    const generation_configuration config{
+        args.get_string_param("--voice"),
+        *args.get_int_param("--topk"),
+        *args.get_float_param("--temperature"),
+        *args.get_float_param("--repetition-penalty"),
         !args.get_bool_param("--no-cross-attn"),
         args.get_string_param("--espeak-voice-id"),
         *args.get_int_param("--max-tokens"),
-        *args.get_float_param("--top-p"));
+        *args.get_float_param("--top-p")};
 
-    struct tts_runner * runner = runner_from_file(args.get_string_param("--model-path"), *args.get_int_param("--n-threads"), config, !args.get_bool_param("--use-metal"));
+    unique_ptr<tts_generation_runner> runner{runner_from_file(args.get_string_param("--model-path").c_str(), *args.get_int_param("--n-threads"), config, !args.get_bool_param("--use-metal"))};
 
-    if (conditional_prompt.size() > 0) {
-        update_conditional_prompt(runner, text_encoder_path, conditional_prompt, true);
+    if (!conditional_prompt.empty()) {
+        runner->update_conditional_prompt(text_encoder_path.c_str(), conditional_prompt.c_str());
     }
     tts_response data;
 
-    generate(runner, args.get_string_param("--prompt"), &data, config);
+    runner->generate(args.get_string_param("--prompt").c_str(), data, config);
     if (data.n_outputs == 0) {
         fprintf(stderr, "Got empty response for prompt, '%s'.\n", args.get_string_param("--prompt").c_str());
         exit(1);
@@ -92,5 +94,6 @@ int main(int argc, const char ** argv) {
     if (!play_tts_response(args, data, runner->sampling_rate)) {
         write_audio_file(data, args.get_string_param("--save-path"), runner->sampling_rate);
     }
+    static_cast<void>(!runner.release()); // TODO the destructor doesn't work yet
     return 0;
 }

examples/perf_battery/perf_battery.cpp

@@ -1,16 +1,12 @@
-#include "tts.h"
-#include "args.h"
-#include "common.h"
 #include <stdio.h>
+
 #include <chrono>
 #include <functional>
 #include <thread>
 
-
-std::vector<std::string> ARCH_LOOKUP = {
-	"parler-tts",
-	"kokoro",
-};
+#include "../../src/models/loaders.h"
+#include "args.h"
+#include "common.h"
 
 using perf_cb = std::function<void()>;
 
@@ -67,15 +63,14 @@ double mean(std::vector<double> series) {
 	return (double) sum / series.size();
 }
 
-std::string benchmark_printout(tts_arch arch, std::vector<double> generation_samples, std::vector<double> output_times) {
-	std::string arch_name = ARCH_LOOKUP[(int)arch];
+std::string benchmark_printout(const char * arch, std::vector<double> generation_samples, std::vector<double> output_times) {
 	double gen_mean = mean(generation_samples);
 	std::vector<double> gen_output;
 	for (int i = 0; i < (int) output_times.size(); i++) {
 		gen_output.push_back(generation_samples[i]/output_times[i]);
 	}
 	double gen_out_mean = mean(gen_output);
-	std::string printout = (std::string) "Mean Stats for arch " + arch_name + ":\n\n" + (std::string) "  Generation Time (ms):             " +  std::to_string(gen_mean) + (std::string) "\n";
+	std::string printout = (std::string) "Mean Stats for arch " + arch + ":\n\n" + (std::string) "  Generation Time (ms):             " +  std::to_string(gen_mean) + (std::string) "\n";
 	printout += (std::string) "  Generation Real Time Factor (ms): " + std::to_string(gen_out_mean) + (std::string) "\n";
 	return printout;
 }
@@ -102,22 +97,23 @@ int main(int argc, const char ** argv) {
     }
     args.validate();
 
-    generation_configuration * config = new generation_configuration(args.get_string_param("--voice"), *args.get_int_param("--topk"), *args.get_float_param("--temperature"), *args.get_float_param("--repetition-penalty"), !args.get_bool_param("--no-cross-attn"));
+    const generation_configuration config{args.get_string_param("--voice"), *args.get_int_param("--topk"), *args.get_float_param("--temperature"), *args.get_float_param("--repetition-penalty"), !args.get_bool_param("--no-cross-attn")};
 
-    struct tts_runner * runner = runner_from_file(args.get_string_param("--model-path"), *args.get_int_param("--n-threads"), config, !args.get_bool_param("--use-metal"));
+    unique_ptr<tts_generation_runner> runner{runner_from_file(args.get_string_param("--model-path").c_str(), *args.get_int_param("--n-threads"), config, !args.get_bool_param("--use-metal"))};
     std::vector<double> generation_samples;
     std::vector<double> output_times;
 
     for (std::string sentence : TEST_SENTENCES) {
     	tts_response response;
     	perf_cb cb = [&]{
-    		generate(runner, sentence, &response, config);
+    		runner->generate(sentence.c_str(), response, config);
     	};
     	double generation_ms = benchmark_ms(cb);
     	output_times.push_back((double)(response.n_outputs / 44.1));
     	generation_samples.push_back(generation_ms);
     }
 
-    fprintf(stdout, "%s", benchmark_printout(runner->arch, generation_samples, output_times).c_str());
+    fprintf(stdout, "%s", benchmark_printout(runner->loader.get().arch, generation_samples, output_times).c_str());
+    static_cast<void>(!runner.release()); // TODO the destructor doesn't work yet
 	return 0;
 }

examples/phonemize/phonemize.cpp

@@ -1,7 +1,8 @@
-#include "phonemizer.h"
-#include "args.h"
 #include <stdio.h>
 
+#include "../../src/models/kokoro/phonemizer.h"
+#include "args.h"
+
 int main(int argc, const char ** argv) {
     arg_list args;
     args.add_argument(string_arg("--phonemizer-path", "(OPTIONAL) The local path of the gguf phonemiser file for TTS.cpp phonemizer. This is required if not using espeak.", "-mp"));

examples/quantize/CMakeLists.txt

@@ -1,2 +1,6 @@
-add_executable(quantize quantize.cpp)
+add_executable(quantize
+        quantize.cpp
+        quantize_impl.cpp
+        quantize_impl.h
+)
 target_link_libraries(quantize PRIVATE ggml tts)

examples/quantize/quantize.cpp

@@ -1,11 +1,12 @@
-#include <stdio.h>
-#include <thread>
+#include <cstdio>
 #include <map>
+#include <thread>
 #include <vector>
 
+#include "../../src/models/loaders.h"
 #include "args.h"
 #include "ggml.h"
-#include "tts.h"
+#include "quantize_impl.h"
 
 const std::map<std::string, ggml_type> valid_quantization_types = {
     {"FP16", GGML_TYPE_F16},
@@ -42,12 +43,15 @@ int main(int argc, const char ** argv) {
                 qtype.c_str());
         exit(1);
     }
-    struct quantization_params * qp = new quantization_params((uint32_t) *args.get_int_param("--n-threads"), valid_quantization_types.at(qtype));
-    qp->quantize_output_heads = args.get_bool_param("--quantize-output-heads");
-    qp->quantize_text_embeddings = args.get_bool_param("--quantize-text-embedding");
-    qp->quantize_cross_attn_kv = args.get_bool_param("--quantize-cross-attn-kv");
-    qp->convert_dac_to_f16 = args.get_bool_param("--convert-dac-to-f16");
-    qp->convert_non_quantizable_to_f16 = args.get_bool_param("--convert-non-quantized-to-f16");
-  	quantize_gguf(args.get_string_param("--model-path"), args.get_string_param("--quantized-model-path"), qp);
+    quantization_params qp {
+        .n_threads{ static_cast<uint32_t>(*args.get_int_param("--n-threads")) },
+        .quantize_type{valid_quantization_types.at(qtype)},  // quantization type
+        .quantize_output_heads{ args.get_bool_param("--quantize-output-heads")},
+        .quantize_text_embeddings{args.get_bool_param("--quantize-text-embedding")},
+        .quantize_cross_attn_kv{ args.get_bool_param("--quantize-cross-attn-kv")},
+        .convert_dac_to_f16{ args.get_bool_param("--convert-dac-to-f16")},
+        .convert_non_quantizable_to_f16{ args.get_bool_param("--convert-non-quantized-to-f16")},
+    };
+    quantize_gguf(args.get_string_param("--model-path").c_str(), args.get_string_param("--quantized-model-path").c_str(), qp);
     return 0;
 }