From 5c95a48270bbf77ef36ab97c9cea95d70ffd1bcb Mon Sep 17 00:00:00 2001 From: Ziyang Xu Date: Fri, 7 Oct 2022 09:46:32 -0400 Subject: [PATCH 01/97] reasoning for loopaa --- liberty/lib/Repl/repl.cpp | 113 +++++++++++++++++++++++++++++++++++--- 1 file changed, 104 insertions(+), 9 deletions(-) diff --git a/liberty/lib/Repl/repl.cpp b/liberty/lib/Repl/repl.cpp index 29d8b1d5..7124863f 100644 --- a/liberty/lib/Repl/repl.cpp +++ b/liberty/lib/Repl/repl.cpp @@ -22,6 +22,7 @@ #include "noelle/core/PDG.hpp" #include "noelle/core/PDGPrinter.hpp" #include "noelle/core/SCCDAG.hpp" +#include "noelle/core/LoopDependenceInfo.hpp" #include "ReplParse.hpp" using namespace llvm; @@ -42,12 +43,12 @@ char OptRepl::ID = 0; static RegisterPass rp("opt-repl", "Opt Repl"); void OptRepl::getAnalysisUsage(AnalysisUsage &au) const { + au.addRequired(); au.addRequired(); au.addRequired(); au.addRequired(); au.addRequired< LoopProfLoad >(); au.addRequired< ProfilePerformanceEstimator >(); - au.addRequired(); au.setPreservesAll(); } @@ -132,6 +133,8 @@ char** completer(const char* text, int start, int end) { bool OptRepl::runOnModule(Module &M) { bool modified = false; + auto &noelle = getAnalysis(); + ModuleLoops &mloops = getAnalysis(); const Targets &targets = getAnalysis(); PDGBuilder &pdgbuilder = getAnalysis(); @@ -150,6 +153,20 @@ bool OptRepl::runOnModule(Module &M) { unique_ptr depIdMap; shared_ptr depIdLookupMap; + // have a vector of all the loop aas + LoopAA* loopAA = (LoopAA*)getSCAFLoopAA(); + vector loopAAs; + auto aa = loopAA; + while (aa) { + loopAAs.push_back(aa); + aa = aa->getNextAA(); + } + unsigned numLoopAAs = loopAAs.size(); + vector loopAAEnabled(numLoopAAs, true); + + outs() << "LoopAA (" << numLoopAAs << "): "; + loopAA->dump(); + // prepare hot loops from the targets { unsigned loopId = 0; @@ -193,7 +210,7 @@ bool OptRepl::runOnModule(Module &M) { }; // select one loop - auto selectFn = [&loopIdMap, &parser, &selectedLoop, &selectedPDG, &selectedSCCDAG, &pdgbuilder, &instIdMap, &instIdLookupMap]() { + auto selectFn = [&loopIdMap, &parser, &selectedLoop, &selectedPDG, &selectedSCCDAG, &pdgbuilder, &instIdMap, &instIdLookupMap, &noelle]() { int loopId = parser.getActionId(); if (loopId == -1) { outs() << "No number specified\n"; @@ -211,7 +228,12 @@ bool OptRepl::runOnModule(Module &M) { << "::" << loop->getHeader()->getName() << '\n'; selectedLoop = loop; - selectedPDG = pdgbuilder.getLoopPDG(loop); + //selectedPDG = pdgbuilder.getLoopPDG(loop); + LoopStructure loopStructure(loop); + auto ldi = noelle.getLoop(&loopStructure); + + selectedPDG = std::make_unique(*(ldi->getLoopDG())); + selectedSCCDAG = std::make_unique(selectedPDG.get()); instIdMap = createInstIdMap(selectedPDG.get()); @@ -438,13 +460,14 @@ bool OptRepl::runOnModule(Module &M) { }; // modref: create a modref query and (optionally explore the loopaa stack) - auto modrefFn = [this, &parser, &instIdMap, &selectedLoop]() { + auto modrefFn = [this, &parser, &instIdMap, &selectedLoop, &loopAA, + &loopAAs, &loopAAEnabled, &numLoopAAs]() { int fromId = parser.getFromId(); int toId = parser.getToId(); if (fromId == -1) { - outs() << "From InstId not set\n"; - return; + outs() << "From InstId not set\n"; + return; } else { if (instIdMap->find(fromId) == instIdMap->end()) { @@ -454,8 +477,8 @@ bool OptRepl::runOnModule(Module &M) { } if (toId == -1) { - outs() << "To InstId not set\n"; - return; + outs() << "To InstId not set\n"; + return; } else { if (instIdMap->find(toId) == instIdMap->end()) { @@ -472,11 +495,83 @@ bool OptRepl::runOnModule(Module &M) { return; } - LoopAA *aa = getAnalysis().getTopAA(); + LoopAA *aa = loopAA; Remedies remeds; if (parser.isVerbose()) { // TODO: try all combination of analysis and find a setting that the result is different + + // try all loopAA, from only the first one, to all of them, the last one is always NoLoopAA + liberty::LoopAA::ModRefResult lastRet[3] = {liberty::LoopAA::ModRef, liberty::LoopAA::ModRef, liberty::LoopAA::ModRef}; + for (auto i = 1; i < numLoopAAs - 1; i++) { + // set the first i loopAA to be enabled(loopAAEnabled[i] = true) + // and the rest to be disabled (loopAAEnabled[i] = false) + for (auto j = 0; j < i; j++) { + loopAAEnabled[j] = true; + } + for (auto j = i; j < numLoopAAs; j++) { + loopAAEnabled[j] = false; + } + + // configure the loop AAs + for (auto j = 0; j < i; j++) { + // if (j >= i) { + // loopAAs[j]->disable(); + // continue; + // } + + // set up the correct prev and next + LoopAA *prev, *next; + if (j == 0) { + prev = nullptr; + } else { + prev = loopAAs[j - 1]; + } + if (j == i - 1) { + // NoLoopAA + next = loopAAs[numLoopAAs - 1]; + } else { + next = loopAAs[j + 1]; + } + + loopAAs[j]->configure(prev, next); + } + loopAAs[numLoopAAs - 1]->configure(loopAAs[i - 1], nullptr); + + // aa->dump(); + auto ret = aa->modref(fromInst, liberty::LoopAA::TemporalRelation::Same, toInst, selectedLoop, remeds); + auto red = "\033[1;31m"; + auto green = "\033[1;32m"; + auto reset = "\033[0m"; + if (ret != lastRet[0]) { + outs() << "Modref (same) refine from " << red << lastRet[0] << reset + << " to " << red << ret << reset << " with " << green + << loopAAs[i - 1]->getLoopAAName() << reset << "\n"; + } + lastRet[0] = ret; + + ret = aa->modref(fromInst, liberty::LoopAA::TemporalRelation::Before, toInst, selectedLoop, remeds); + if (ret != lastRet[1]) { + outs() << "Modref (before) refine from " << red << lastRet[1] << reset + << " to " << red << ret << reset << " with " << green + << loopAAs[i - 1]->getLoopAAName() << reset << "\n"; + } + lastRet[1] = ret; + + ret = aa->modref(fromInst, liberty::LoopAA::TemporalRelation::After, toInst, selectedLoop, remeds); + if (ret != lastRet[2]) { + outs() << "Modref (after) refine from " << red << lastRet[2] << reset + << " to " << red << ret << reset << " with " << green + << loopAAs[i - 1]->getLoopAAName() << reset << "\n"; + } + lastRet[2] = ret; + + // outs() << *fromInst << "->" << *toInst << ": (Same)" << ret << " with " << remeds.size() << " remedies\n"; + // ret = aa->modref(fromInst, liberty::LoopAA::TemporalRelation::Before, toInst, selectedLoop, remeds); + // outs() << *fromInst << "->" << *toInst << ": (Before)" << ret << " with " << remeds.size() << " remedies\n"; + // ret = aa->modref(fromInst, liberty::LoopAA::TemporalRelation::After, toInst, selectedLoop, remeds); + // outs() << *fromInst << "->" << *toInst << ": (After)" << ret << " with " << remeds.size() << " remedies\n"; + } } else { auto ret = aa->modref(fromInst, liberty::LoopAA::TemporalRelation::Same, toInst, selectedLoop, remeds); From edbf7fc3615bf6243383d555935d22a72094b47c Mon Sep 17 00:00:00 2001 From: Ziyang Xu Date: Mon, 31 Oct 2022 13:43:38 -0400 Subject: [PATCH 02/97] match new noelle changes --- liberty/lib/Speculation/Selector.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/liberty/lib/Speculation/Selector.cpp b/liberty/lib/Speculation/Selector.cpp index b71616bf..8b3e18fc 100644 --- a/liberty/lib/Speculation/Selector.cpp +++ b/liberty/lib/Speculation/Selector.cpp @@ -534,7 +534,7 @@ void Selector::computeEdges(const Vertices &vertices, Edges &edges) BasicBlock *hA = A->getHeader(); Function *fA = hA->getParent(); Module *m = fA->getParent(); - auto fm = FunctionsManager(*m, pdgAnalysis); + auto fm = FunctionsManager(*m, pdgAnalysis, nullptr); /* * Call graph. */ From 759c1477ae0468964decf5a4b56096e0c8b700be Mon Sep 17 00:00:00 2001 From: Ziyang Xu Date: Mon, 31 Oct 2022 15:12:19 -0400 Subject: [PATCH 03/97] add inst select with namer id --- liberty/lib/Repl/repl.cpp | 41 +++++++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/liberty/lib/Repl/repl.cpp b/liberty/lib/Repl/repl.cpp index 7124863f..4ef57b8f 100644 --- a/liberty/lib/Repl/repl.cpp +++ b/liberty/lib/Repl/repl.cpp @@ -298,22 +298,39 @@ bool OptRepl::runOnModule(Module &M) { // show instructions with id auto instsFn = [&parser, &instIdMap]() { auto printDebug = parser.isVerbose(); - for (auto &[instId, node] : *instIdMap) { - auto *inst = dyn_cast(node->getT()); - // not an instruction - if (!inst) { - outs() << instId << "\t" << *node->getT() << "\n"; - continue; + int queryInstId = parser.getActionId(); + + // print the selected instruction + if (queryInstId != -1) { + for (auto &[instId, node] : *instIdMap) { + auto *inst = dyn_cast(node->getT()); + auto instNamerId = Namer::getInstrId(inst); + if (queryInstId == instNamerId) { + outs() << instId << " (" << queryInstId << ")\t" << *inst; + if (printDebug) { + liberty::printInstDebugInfo(inst); + } + outs() << "\n"; + } } + } else { // print all instructions + for (auto &[instId, node] : *instIdMap) { + auto *inst = dyn_cast(node->getT()); + // not an instruction + if (!inst) { + outs() << instId << "\t" << *node->getT() << "\n"; + continue; + } - auto instNamerId = Namer::getInstrId(inst); - outs() << instId << " (" << instNamerId << ")\t" << *node->getT(); + auto instNamerId = Namer::getInstrId(inst); + outs() << instId << " (" << instNamerId << ")\t" << *node->getT(); - if (printDebug) { - liberty::printInstDebugInfo(inst); - } + if (printDebug) { + liberty::printInstDebugInfo(inst); + } - outs()<< "\n"; + outs()<< "\n"; + } } }; From 9e2fed04441699f6af21ccddb4b36ac8533fcb7a Mon Sep 17 00:00:00 2001 From: Ziyang Xu Date: Mon, 31 Oct 2022 15:15:36 -0400 Subject: [PATCH 04/97] add inst select with namer id --- liberty/lib/Repl/repl.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/liberty/lib/Repl/repl.cpp b/liberty/lib/Repl/repl.cpp index 4ef57b8f..a0803048 100644 --- a/liberty/lib/Repl/repl.cpp +++ b/liberty/lib/Repl/repl.cpp @@ -302,6 +302,7 @@ bool OptRepl::runOnModule(Module &M) { // print the selected instruction if (queryInstId != -1) { + bool found = false; for (auto &[instId, node] : *instIdMap) { auto *inst = dyn_cast(node->getT()); auto instNamerId = Namer::getInstrId(inst); @@ -311,8 +312,14 @@ bool OptRepl::runOnModule(Module &M) { liberty::printInstDebugInfo(inst); } outs() << "\n"; + found = true; + break; } } + + if (!found) { + outs() << "Instruction with NamerId " << queryInstId << " not found\n"; + } } else { // print all instructions for (auto &[instId, node] : *instIdMap) { auto *inst = dyn_cast(node->getT()); From 48f00642f9cb6179d668e53b9ce9cd3c59c5fcd8 Mon Sep 17 00:00:00 2001 From: Ziyang Xu Date: Fri, 16 Sep 2022 21:32:51 -0400 Subject: [PATCH 05/97] add new context tracking for SLAMP --- liberty/lib/SLAMP/SLAMPlib/hooks/context.h | 102 +++++++++++++++++ .../lib/SLAMP/SLAMPlib/hooks/slamp_hooks.cpp | 104 ++++++++++++++---- 2 files changed, 187 insertions(+), 19 deletions(-) create mode 100644 liberty/lib/SLAMP/SLAMPlib/hooks/context.h diff --git a/liberty/lib/SLAMP/SLAMPlib/hooks/context.h b/liberty/lib/SLAMP/SLAMPlib/hooks/context.h new file mode 100644 index 00000000..8622d8b5 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPlib/hooks/context.h @@ -0,0 +1,102 @@ +#ifndef SLAMPLIB_HOOKS_SLAMP_CONTEXT_H +#define SLAMPLIB_HOOKS_SLAMP_CONTEXT_H + +/* The context management + * ====================== + * The definition of a context is very specific to the client. For SpecPriv, + * the dynamic context is defined as the function and loop stack. + * + * For loop-aware memory dependnece profiling, the context is the loop and the + * invocation and iteration count. + * + * The context information usually needs to be stored in the shadow memory. + * + * The context management is responsible for keeping track the current context + * and creating a fixed length representation of any context. + * + */ + +#include +#include +#include "llvm/IR/Instruction.h" +#include "scaf/Utilities/Metadata.h" + +namespace SLAMPLib { + +/// The context is stack of the ContextId +template struct Context { + ContextId id; + Context *parent; + + Context(ContextId id, Context *parent) : id(std::move(id)), parent(parent) {} + + /// Add a new context to the stack + Context *chain(ContextId id) { return new Context(id, this); } + + size_t hash() { + size_t h = id.hash(); + if (parent) + h ^= parent->hash(); + } + + /// Remove the top context from the stack + Context *pop() { + auto parent = this->parent; + // FIXME: is this ok to use? + delete this; + return parent; + } + using FlattenContext = std::vector; + + /// make the context into a vector of only the context id + FlattenContext flatten() { + std::vector result; + Context *cur = this; + while (cur) { + result.push_back(cur->id); + cur = cur->parent; + } + return result; + }; +}; + +template +struct ContextManager { + Context *activeContext; + + ContextManager() : activeContext(nullptr) {} + std::unordered_map contextMap; + + void addContext(size_t hash, Context context) { + contextMap[hash] = context.flatten(); + } +}; + +namespace SpecPrivLib { +enum SpecPrivContextType { + TopContext = 0, + FunctionContext, + LoopContext, +}; + +struct ContextId { + SpecPrivContextType type; + int32_t metaId; + ContextId(SpecPrivContextType type, int32_t id) : type(type), metaId(id) {} + ContextId(std::pair p) + : type(p.first), metaId(p.second) {} + // id left shift 2 bits and or with type + size_t hash() { + return (static_cast(metaId) << 2) | static_cast(type); + } + + bool operator==(const ContextId &other) const { + return type == other.type && metaId == other.metaId; + } +}; +using Context = Context; +}; // namespace SpecPrivLib + + +} // namespace SLAMPLib +#endif // SLAMPLIB_HOOKS_SLAMP_CONTEXT_H diff --git a/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_hooks.cpp b/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_hooks.cpp index 285bb430..33f20c52 100644 --- a/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_hooks.cpp +++ b/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_hooks.cpp @@ -20,6 +20,7 @@ #include "slamp_debug.h" #include "slamp_timer.h" +#include "context.h" #include @@ -30,6 +31,8 @@ #define UNW_LOCAL_ONLY #include +using namespace SLAMPLib; + #define TURN_OFF_CUSTOM_MALLOC do {\ __malloc_hook = old_malloc_hook; \ __free_hook = old_free_hook; \ @@ -115,6 +118,7 @@ uint64_t __slamp_load_count = 0; uint64_t __slamp_store_count = 0; uint64_t __slamp_malloc_count = 0; uint64_t __slamp_free_count = 0; +static SpecPrivLib::Context *currentContext = nullptr; // Type of the access callback function // instr, bare_instr, address, value, size @@ -138,7 +142,8 @@ void SLAMP_callback_stack_alloca(uint64_t array_size, uint64_t type_size, uint32 void* shadow = (void*)GET_SHADOW(addr, TIMESTAMP_SIZE_IN_POWER_OF_TWO); if (shadow && POINTS_TO_MODULE) { TS *s = (TS *)shadow; - TS ts = CREATE_TS(instr, __slamp_iteration, __slamp_invocation); + auto hash = currentContext->hash(); + TS ts = CREATE_TS(instr, hash, __slamp_invocation); for (auto i = 0; i < size; i++) s[i] = ts; } @@ -378,7 +383,7 @@ void slamp_access_callback_linear_address(bool isLoad, uint32_t instr, uint32_t } } -static uint32_t context = 0; +static uint32_t StaticInstIdOfLOI = 0; static uint32_t ext_context = 0; slamp::MemoryMap* smmap = nullptr; @@ -422,8 +427,12 @@ void SLAMP_points_to_module_use(uint32_t instr, uint64_t addr) { } if (cond && tss[i] != 0) { + // mask off the iteration count + TS ts = tss[i]; + ts = ts & 0xfffffffffffffff0; + // ts = ts & 0xfffff0000000000f; //create set of objects for each load/store - (*pointsToMap)[instr].insert(tss[i]); + (*pointsToMap)[instr].insert(ts); } } } @@ -440,6 +449,9 @@ void SLAMP_points_to_module_use(uint32_t instr, uint64_t addr, unsigned size) { TS ts; ts = s[i]; + // mask off the iteration count + ts = ts & 0xfffffffffffffff0; + // ts = ts & 0xfffff0000000000f; if (m.count(ts) == 0) { (*pointsToMap)[instr].insert(ts); m.insert(ts); @@ -704,6 +716,8 @@ static void* SLAMP_memalign_hook(size_t alignment, size_t size, const void *call void SLAMP_init(uint32_t fn_id, uint32_t loop_id) { + // update the current context + currentContext = new SpecPrivLib::Context(std::make_pair(SpecPrivLib::TopContext, 0), nullptr); // auto heapStart = sbrk(0); // fprintf(stderr, "heap start: %lx\n", (unsigned long)heapStart); @@ -964,6 +978,49 @@ void SLAMP_main_entry(uint32_t argc, char** argv, char** env) TURN_ON_CUSTOM_MALLOC; } +/// Keep track of the context of the function +void SLAMP_enter_fcn(uint32_t fcnId) { + auto contextId = SpecPrivLib::ContextId(SpecPrivLib::FunctionContext, fcnId); + + // update the current context + assert(currentContext && "currentContext is null"); + currentContext = currentContext->chain(contextId); +} + +/// Keep track of the context of the function +void SLAMP_exit_fcn(uint32_t fcnId) { + auto contextId = SpecPrivLib::ContextId(SpecPrivLib::FunctionContext, fcnId); + + assert(currentContext && "currentContext is null"); + // update the current context + if (currentContext->id == contextId) { + currentContext = currentContext->pop(); + } else { + // The context is not matching, could be due to the longjmp/setjmp etc or exception handling in C++ + std::cerr << "Context mismatch! Current context: "; + auto tmp = currentContext; + while (tmp->parent) { + std::cerr << "(" << tmp->id.type << "," << tmp->id.metaId << ")->"; + tmp = tmp->parent; + } + std::cerr << "(" << tmp->id.type << "," << tmp->id.metaId << ")->"; + std::cerr << "Exiting context: (" << contextId.type << "," << contextId.metaId << ")" << std::endl; + + // Let's try to find the correct context + bool foundInStack = false; + while (currentContext) { + if (currentContext->id == contextId) { + foundInStack = true; + currentContext = currentContext->pop(); + break; + } + currentContext = currentContext->pop(); + } + + assert(foundInStack && "Could not find the exiting context in the stack"); + } +} + /// update the invocation count void SLAMP_loop_invocation() { // fprintf(stderr, "SLAMP_loop_invocation, depth: %u\n", invokedepth); @@ -1028,8 +1085,8 @@ void SLAMP_push(const uint32_t instr) { << std::flush; #endif - assert(context == 0); - context = instr; + assert(StaticInstIdOfLOI == 0); + StaticInstIdOfLOI = instr; } /// unset the context of the call inside a loop @@ -1044,7 +1101,7 @@ void SLAMP_pop() { << std::flush; #endif - context = 0; + StaticInstIdOfLOI = 0; } // FIXME: a temporary patch for out of handling program original heap @@ -1144,7 +1201,7 @@ void SLAMP_dependence_module_load_log(const uint32_t instr, const uint32_t bare_ template void SLAMP_load(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline) { if (invokedepth > 1) - instr = context; + instr = StaticInstIdOfLOI; if (SLAMP_isBadAlloc(addr)) return; @@ -1208,7 +1265,7 @@ void SLAMP_loadn(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, if (SLAMP_isBadAlloc(addr)) return; if (invokedepth > 1) - instr = context; + instr = StaticInstIdOfLOI; if (TRACE_MODULE) { __slamp_load_count++; @@ -1250,8 +1307,8 @@ void SLAMP_load_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t val << addr << std::dec << "\n" << std::flush; #endif - if (context) - SLAMP_load(context, addr, bare_instr, value); + if (StaticInstIdOfLOI) + SLAMP_load(StaticInstIdOfLOI, addr, bare_instr, value); } @@ -1284,8 +1341,8 @@ void SLAMP_loadn_ext(const uint64_t addr, const uint32_t bare_instr, size_t n) { << std::flush; #endif - if (context) - SLAMP_loadn(context, addr, bare_instr, n); + if (StaticInstIdOfLOI) + SLAMP_loadn(StaticInstIdOfLOI, addr, bare_instr, n); } template @@ -1331,7 +1388,7 @@ void SLAMP_store(uint32_t instr, uint32_t bare_instr, const uint64_t addr) ATTRI // TODO: do we care about recursive calls? if (invokedepth > 1) - instr = context; + instr = StaticInstIdOfLOI; if (TRACE_MODULE) { __slamp_store_count++; @@ -1397,7 +1454,7 @@ void SLAMP_storen(uint32_t instr, const uint64_t addr, size_t n) { if (SLAMP_isBadAlloc(addr)) return; if (invokedepth > 1) - instr = context; + instr = StaticInstIdOfLOI; if (TRACE_MODULE) { __slamp_store_count++; @@ -1441,8 +1498,8 @@ void SLAMP_store_ext(const uint64_t addr, const uint32_t bare_inst) ATTRIBUTE(al << std::flush; #endif - if (context) - SLAMP_store(context, bare_inst, addr); + if (StaticInstIdOfLOI) + SLAMP_store(StaticInstIdOfLOI, bare_inst, addr); } @@ -1471,8 +1528,8 @@ void SLAMP_storen_ext(const uint64_t addr, const uint32_t bare_inst, size_t n) { << std::flush; #endif - if (context) - SLAMP_storen(context, addr, n); + if (StaticInstIdOfLOI) + SLAMP_storen(StaticInstIdOfLOI, addr, n); } /* @@ -1512,7 +1569,11 @@ void* SLAMP_malloc(size_t size, uint32_t instr, size_t alignment) // cast as timestamp TS *s = (TS *)shadow; // log all data into sigle TS - TS ts = CREATE_TS(instr, __slamp_iteration, __slamp_invocation); + // FIXME: static instruction and the dynamic context? + // context: static instr + function + loop + auto hash = currentContext->hash(); + TS ts = CREATE_TS(instr, hash, __slamp_invocation); + //8 bytes per byte TODO: can we reduce this? for (auto i = 0; i < size; i++) s[i] = ts; @@ -1552,6 +1613,11 @@ void SLAMP_free(void* ptr) unsigned purge_cnt; bool purge = slamp::bound_free(ptr, starting_page, purge_cnt); + // need to check if the object is short-lived + if (POINTS_TO_MODULE) { + + } + if (DEPENDENCE_MODULE || POINTS_TO_MODULE) { if (purge) smmap->deallocate_pages(starting_page, purge_cnt); From e958f50682da677d50c93a18ae390638dd34782a Mon Sep 17 00:00:00 2001 From: Ziyang Xu Date: Mon, 19 Sep 2022 19:08:23 -0400 Subject: [PATCH 06/97] add context tracking and object lifetime tracking --- liberty/include/liberty/SLAMP/SLAMP.h | 1 + liberty/lib/SLAMP/SLAMP.cpp | 64 +++++++++ liberty/lib/SLAMP/SLAMPlib/hooks/context.h | 97 +++++++++++++- .../lib/SLAMP/SLAMPlib/hooks/slamp_hooks.cpp | 122 ++++++++++++------ .../lib/SLAMP/SLAMPlib/hooks/slamp_hooks.h | 2 + 5 files changed, 241 insertions(+), 45 deletions(-) diff --git a/liberty/include/liberty/SLAMP/SLAMP.h b/liberty/include/liberty/SLAMP/SLAMP.h index 5a9c66bd..215e1e67 100644 --- a/liberty/include/liberty/SLAMP/SLAMP.h +++ b/liberty/include/liberty/SLAMP/SLAMP.h @@ -59,6 +59,7 @@ class SLAMP: public ModulePass void instrumentNonStandards(Module& m, Function* ctor); void allocErrnoLocation(Module& m, Function* ctor); + void instrumentFunctionStartStop(Module&m); void instrumentLoopStartStop(Module&m, Loop* l); void instrumentInstructions(Module& m, Loop* l); diff --git a/liberty/lib/SLAMP/SLAMP.cpp b/liberty/lib/SLAMP/SLAMP.cpp index 5189ee50..acc5310c 100644 --- a/liberty/lib/SLAMP/SLAMP.cpp +++ b/liberty/lib/SLAMP/SLAMP.cpp @@ -457,6 +457,7 @@ bool SLAMP::runOnModule(Module &m) { instrumentMainFunction(m); + instrumentFunctionStartStop(m); instrumentLoopStartStop(m, this->target_loop); instrumentInstructions(m, this->target_loop); @@ -891,6 +892,8 @@ void SLAMP::reportEndOfAllocaLifetime(AllocaInst *inst, Instruction *end, bool e else { //TODO:search for terminator block auto *F = inst->getFunction(); + + // find all return instructions //IRBuilder<> Builder(); } return; @@ -1055,6 +1058,67 @@ void SLAMP::instrumentMainFunction(Module &m) { } } + +/// Instrumnent each function entry and exit with SLAMP function entry and exit calls +void SLAMP::instrumentFunctionStartStop(Module &m) { + // for each function body + for (auto &fi : m) { + Function *func = &fi; + if (func->isDeclaration()) + continue; + + // ignore all SLAMP calls + if (func->getName().startswith("SLAMP_")) + continue; + + + // find function ID + auto fcnID = Namer::getFuncId(func); + if (fcnID == -1) { + errs() << "Cannot find function ID for " << func->getName() << "\n"; + continue; + } + + // set parameters + vector args; + args.push_back(ConstantInt::get(I32, fcnID)); + + // find the function entry + auto *f_function_entry = cast( + m.getOrInsertFunction("SLAMP_enter_fcn", Void, I32).getCallee()); + auto *f_function_exit = cast( + m.getOrInsertFunction("SLAMP_exit_fcn", Void, I32).getCallee()); + + // insert SLAMP_enter_fcn at the beginning of the function + BasicBlock *entry = &(func->getEntryBlock()); + InstInsertPt pt = InstInsertPt::Before(entry->getFirstNonPHI()); + pt << updateDebugInfo(CallInst::Create(f_function_entry, args), pt.getPosition(), m); + + // find all exits of the function + vector exits; + for (auto &bi : *func) { + BasicBlock *bb = &bi; + if (isa(bb->getTerminator())) + exits.push_back(bb->getTerminator()); + // else if (isa(bb->getTerminator())) + // exits.push_back(bb->getTerminator()); + // // FIXME: should be at the beginning of the block + // else if (isa(bb->getTerminator())) + // exits.push_back(bb->getTerminator()); + //// FIXME: invoke the exception end + // else if (isa(bb->getTerminator())) + // exits.push_back(bb->getTerminator()); + } + + // insert SLAMP_exit_fcn at the end of the function + for (auto &exit : exits) { + InstInsertPt pt = InstInsertPt::Before(exit); + pt << updateDebugInfo(CallInst::Create(f_function_exit, args), pt.getPosition(), m); + } + + } +} + /// Pass in the loop and instrument invocation/iteration/exit hooks void SLAMP::instrumentLoopStartStop(Module &m, Loop *loop) { // TODO: check setjmp/longjmp diff --git a/liberty/lib/SLAMP/SLAMPlib/hooks/context.h b/liberty/lib/SLAMP/SLAMPlib/hooks/context.h index 8622d8b5..817045c4 100644 --- a/liberty/lib/SLAMP/SLAMPlib/hooks/context.h +++ b/liberty/lib/SLAMP/SLAMPlib/hooks/context.h @@ -16,6 +16,7 @@ * */ +#include #include #include #include "llvm/IR/Instruction.h" @@ -23,6 +24,8 @@ namespace SLAMPLib { +using ContextHash = size_t; + /// The context is stack of the ContextId template struct Context { ContextId id; @@ -33,10 +36,15 @@ template struct Context { /// Add a new context to the stack Context *chain(ContextId id) { return new Context(id, this); } - size_t hash() { - size_t h = id.hash(); + static Context* getTopContext() { + return new Context(ContextId::getTopContextId(), nullptr); + } + + ContextHash hash() { + ContextHash h = id.hash(); if (parent) h ^= parent->hash(); + return h; } /// Remove the top context from the stack @@ -47,6 +55,7 @@ template struct Context { return parent; } using FlattenContext = std::vector; + using ContextIdType = ContextId; /// make the context into a vector of only the context id FlattenContext flatten() { @@ -58,16 +67,85 @@ template struct Context { } return result; }; + + /// print context + void print(std::ostream &os) { + os << "Context:\n"; + Context *cur = this; + while (cur) { + cur->id.print(os); + cur = cur->parent; + } + } }; +/// Note that we do no need to keep track of +/// all the objects that are alive template struct ContextManager { Context *activeContext; - ContextManager() : activeContext(nullptr) {} - std::unordered_map contextMap; + ContextManager() { + activeContext = Context::getTopContext(); + } + + std::unordered_map contextMap; - void addContext(size_t hash, Context context) { + ContextHash activeHash() { + if (activeContext) + return activeContext->hash(); + else + return 0; + } + + void updateContext(typename Context::ContextIdType contextId) { + assert(activeContext && "active context is null"); + /// maybe this is not a great idea, expose the semantics of the context to the manager + activeContext = activeContext->chain(contextId); + addContext(activeContext->hash(), *activeContext); + } + + void popContext(typename Context::ContextIdType contextId) { + assert(activeContext && "active context is null"); + + if (activeContext->id == contextId) { + activeContext = activeContext->pop(); + } else { + // The context is not matching, could be due to the longjmp/setjmp etc or + // exception handling in C++ + std::cerr << "Context mismatch! Current context: "; + auto tmp = activeContext; + while (tmp->parent) { + std::cerr << "(" << tmp->id.type << "," << tmp->id.metaId << ")->"; + tmp = tmp->parent; + } + std::cerr << "(" << tmp->id.type << "," << tmp->id.metaId << ")->"; + std::cerr << "Exiting context: (" << contextId.type << "," + << contextId.metaId << ")" << std::endl; + + // Let's try to find the correct context + bool foundInStack = false; + while (activeContext->parent) { + activeContext = activeContext->pop(); + if (activeContext->id == contextId) { + foundInStack = true; + break; + } + } + + assert(foundInStack && "Could not find the exiting context in the stack"); + } + } + + void addContext(ContextHash hash, Context &context) { + + // if exist one but not the same, then we have a problem + if (contextMap.count(hash) && contextMap[hash] != context.flatten()) { + // FIXME: need to turn this back on + // assert(false && "Context hash collision"); + } + + // insert the context contextMap[hash] = context.flatten(); } }; @@ -85,14 +163,19 @@ struct ContextId { ContextId(SpecPrivContextType type, int32_t id) : type(type), metaId(id) {} ContextId(std::pair p) : type(p.first), metaId(p.second) {} + static ContextId getTopContextId() { return {TopContext, 0}; } // id left shift 2 bits and or with type - size_t hash() { - return (static_cast(metaId) << 2) | static_cast(type); + ContextHash hash() { + return (static_cast(metaId) << 2) | static_cast(type); } bool operator==(const ContextId &other) const { return type == other.type && metaId == other.metaId; } + + void print(std::ostream &os) { + os << "ContextId: " << type << " " << metaId << "\n"; + } }; using Context = Context; }; // namespace SpecPrivLib diff --git a/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_hooks.cpp b/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_hooks.cpp index 33f20c52..511c0cf3 100644 --- a/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_hooks.cpp +++ b/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_hooks.cpp @@ -118,7 +118,8 @@ uint64_t __slamp_load_count = 0; uint64_t __slamp_store_count = 0; uint64_t __slamp_malloc_count = 0; uint64_t __slamp_free_count = 0; -static SpecPrivLib::Context *currentContext = nullptr; +static ContextManager contextManager; +std::unordered_set *shortLivedObjects, *longLivedObjects; // Type of the access callback function // instr, bare_instr, address, value, size @@ -142,7 +143,7 @@ void SLAMP_callback_stack_alloca(uint64_t array_size, uint64_t type_size, uint32 void* shadow = (void*)GET_SHADOW(addr, TIMESTAMP_SIZE_IN_POWER_OF_TWO); if (shadow && POINTS_TO_MODULE) { TS *s = (TS *)shadow; - auto hash = currentContext->hash(); + auto hash = contextManager.activeHash(); TS ts = CREATE_TS(instr, hash, __slamp_invocation); for (auto i = 0; i < size; i++) s[i] = ts; @@ -152,7 +153,33 @@ void SLAMP_callback_stack_alloca(uint64_t array_size, uint64_t type_size, uint32 } -void SLAMP_callback_stack_free(void) {} +void SLAMP_callback_stack_free(void) { + // do not need to free memory or shadow memory, it gets reused implicitly + + // // Need to record short-lived objects + // // need to check if the object is short-lived + // if (POINTS_TO_MODULE) { + // // if we are still in the loop and the iteration is the same, mark it as local + // // otherwise mark it as not local + // TS* s = (TS*)GET_SHADOW(ptr, TIMESTAMP_SIZE_IN_POWER_OF_TWO); + // auto instr = GET_INSTR(s[0]); + // auto iteration = GET_ITER(s[0]); + // auto invocation = GET_INVOC(s[0]); + + // // if invokedepth is 0, it means we are not in a loop + // if (iteration == (0xffffffffff & __slamp_iteration) + // && invocation == (0xf & __slamp_invocation) + // && invokedepth > 0) { + // // is short-lived, put in the set + // shortLivedObjects.insert(instr); + // } + // else { + // // is not short-lived + // longLivedObjects.insert(instr); + // } + + // } +} // Callback function pointers std::list *access_callbacks; @@ -429,7 +456,8 @@ void SLAMP_points_to_module_use(uint32_t instr, uint64_t addr) { if (cond && tss[i] != 0) { // mask off the iteration count TS ts = tss[i]; - ts = ts & 0xfffffffffffffff0; + ts = ts & 0xfffff00000000000; + // ts = ts & 0xfffffffffffffff0; // ts = ts & 0xfffff0000000000f; //create set of objects for each load/store (*pointsToMap)[instr].insert(ts); @@ -450,7 +478,8 @@ void SLAMP_points_to_module_use(uint32_t instr, uint64_t addr, unsigned size) { ts = s[i]; // mask off the iteration count - ts = ts & 0xfffffffffffffff0; + ts = ts & 0xfffff00000000000; + // ts = ts & 0xfffffffffffffff0; // ts = ts & 0xfffff0000000000f; if (m.count(ts) == 0) { (*pointsToMap)[instr].insert(ts); @@ -716,8 +745,11 @@ static void* SLAMP_memalign_hook(size_t alignment, size_t size, const void *call void SLAMP_init(uint32_t fn_id, uint32_t loop_id) { - // update the current context - currentContext = new SpecPrivLib::Context(std::make_pair(SpecPrivLib::TopContext, 0), nullptr); + + if (POINTS_TO_MODULE) { + shortLivedObjects = new std::unordered_set(); + longLivedObjects = new std::unordered_set(); + } // auto heapStart = sbrk(0); // fprintf(stderr, "heap start: %lx\n", (unsigned long)heapStart); @@ -890,6 +922,18 @@ void SLAMP_fini(const char* filename) } ofs << "\n"; } + ofs << "Short-lived object:\n"; + ofs << shortLivedObjects->size() << " " << longLivedObjects->size() << "\n"; + for (auto &obj: *shortLivedObjects) { + // if not long lived + ofs << obj << "\n"; + } + + for (auto &obj: *longLivedObjects) { + // if not long lived + ofs << obj << "\n"; + } + ofs.close(); } } @@ -983,42 +1027,15 @@ void SLAMP_enter_fcn(uint32_t fcnId) { auto contextId = SpecPrivLib::ContextId(SpecPrivLib::FunctionContext, fcnId); // update the current context - assert(currentContext && "currentContext is null"); - currentContext = currentContext->chain(contextId); + contextManager.updateContext(contextId); } /// Keep track of the context of the function void SLAMP_exit_fcn(uint32_t fcnId) { auto contextId = SpecPrivLib::ContextId(SpecPrivLib::FunctionContext, fcnId); - assert(currentContext && "currentContext is null"); // update the current context - if (currentContext->id == contextId) { - currentContext = currentContext->pop(); - } else { - // The context is not matching, could be due to the longjmp/setjmp etc or exception handling in C++ - std::cerr << "Context mismatch! Current context: "; - auto tmp = currentContext; - while (tmp->parent) { - std::cerr << "(" << tmp->id.type << "," << tmp->id.metaId << ")->"; - tmp = tmp->parent; - } - std::cerr << "(" << tmp->id.type << "," << tmp->id.metaId << ")->"; - std::cerr << "Exiting context: (" << contextId.type << "," << contextId.metaId << ")" << std::endl; - - // Let's try to find the correct context - bool foundInStack = false; - while (currentContext) { - if (currentContext->id == contextId) { - foundInStack = true; - currentContext = currentContext->pop(); - break; - } - currentContext = currentContext->pop(); - } - - assert(foundInStack && "Could not find the exiting context in the stack"); - } + contextManager.popContext(contextId); } /// update the invocation count @@ -1571,8 +1588,13 @@ void* SLAMP_malloc(size_t size, uint32_t instr, size_t alignment) // log all data into sigle TS // FIXME: static instruction and the dynamic context? // context: static instr + function + loop - auto hash = currentContext->hash(); - TS ts = CREATE_TS(instr, hash, __slamp_invocation); + auto hash = contextManager.activeHash(); + + // currentContext->print(std::cerr); + // std::cerr << "hash: " << hash << "\n"; + + // TS ts = CREATE_TS(instr, hash, __slamp_invocation); + TS ts = CREATE_TS(instr, __slamp_iteration, __slamp_invocation); //8 bytes per byte TODO: can we reduce this? for (auto i = 0; i < size; i++) @@ -1607,6 +1629,12 @@ void* SLAMP_malloc(size_t size, uint32_t instr, size_t alignment) void SLAMP_free(void* ptr) { + if (ptr == nullptr) + return; + + if (SLAMP_isBadAlloc((uint64_t)ptr)) + return; + TURN_OFF_CUSTOM_MALLOC; uint64_t starting_page; @@ -1615,6 +1643,24 @@ void SLAMP_free(void* ptr) // need to check if the object is short-lived if (POINTS_TO_MODULE) { + // if we are still in the loop and the iteration is the same, mark it as local + // otherwise mark it as not local + TS* s = (TS*)GET_SHADOW(ptr, TIMESTAMP_SIZE_IN_POWER_OF_TWO); + auto instr = GET_INSTR(s[0]); + auto iteration = GET_ITER(s[0]); + auto invocation = GET_INVOC(s[0]); + + // if invokedepth is 0, it means we are not in a loop + if (iteration == (0xffffffffff & __slamp_iteration) + && invocation == (0xf & __slamp_invocation) + && invokedepth > 0) { + // is short-lived, put in the set + shortLivedObjects->insert(instr); + } + else { + // is not short-lived + longLivedObjects->insert(instr); + } } diff --git a/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_hooks.h b/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_hooks.h index fb803f93..a5456abd 100644 --- a/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_hooks.h +++ b/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_hooks.h @@ -61,6 +61,8 @@ void SLAMP_allocated(uint64_t addr); void SLAMP_init_global_vars(const char *name, uint64_t addr, size_t size); void SLAMP_main_entry(uint32_t argc, char** argv, char** env); +void SLAMP_enter_fcn(uint32_t id); +void SLAMP_exit_fcn(uint32_t id); void SLAMP_loop_invocation(); void SLAMP_loop_iteration(); void SLAMP_loop_exit(); From ba6e419aeb0f50c7538d2f39c3fc03dfb36e3ad1 Mon Sep 17 00:00:00 2001 From: Ziyang Xu Date: Thu, 22 Sep 2022 19:04:54 -0400 Subject: [PATCH 07/97] add underlying object tracking at the pointer creation place --- liberty/include/liberty/SLAMP/SLAMP.h | 1 + liberty/lib/SLAMP/SLAMP.cpp | 242 ++++++++++++------ .../lib/SLAMP/SLAMPlib/hooks/slamp_hooks.cpp | 190 ++++++++++---- .../lib/SLAMP/SLAMPlib/hooks/slamp_hooks.h | 2 + .../SLAMP/SLAMPlib/hooks/slamp_timestamp.h | 7 +- 5 files changed, 325 insertions(+), 117 deletions(-) diff --git a/liberty/include/liberty/SLAMP/SLAMP.h b/liberty/include/liberty/SLAMP/SLAMP.h index 215e1e67..a4074ac0 100644 --- a/liberty/include/liberty/SLAMP/SLAMP.h +++ b/liberty/include/liberty/SLAMP/SLAMP.h @@ -49,6 +49,7 @@ class SLAMP: public ModulePass void instrumentDestructor(Module& m); void instrumentGlobalVars(Module& m, Function* ctor); void instrumentAllocas(Module& m); + void instrumentBasePointer(Module &m, Loop *l); // functions used in instrumentAllocas void findLifetimeMarkers(Value* i, set& already, std::vector& starts, std::vector& ends); diff --git a/liberty/lib/SLAMP/SLAMP.cpp b/liberty/lib/SLAMP/SLAMP.cpp index acc5310c..4e347c0c 100644 --- a/liberty/lib/SLAMP/SLAMP.cpp +++ b/liberty/lib/SLAMP/SLAMP.cpp @@ -14,6 +14,8 @@ #include "liberty/SLAMP/SLAMP.h" #include "liberty/SLAMP/externs.h" +#include "scaf/Utilities/CallSiteFactory.h" +#include "liberty/PointsToProfiler/Indeterminate.h" #include "llvm/IR/CFG.h" #include "llvm/ADT/Statistic.h" @@ -287,21 +289,21 @@ bool SLAMP::runOnModule(Module &m) { // debug LLVM_DEBUG(if (Namer::getInstrId(&I) == 21182) { - Remedies remedies; - LoopAA::ModRefResult modrefIIFW = aa->modref( - target_inst, LoopAA::Same, &I, target_loop, remedies); - auto modrefIIBW = aa->modref(&I, LoopAA::Same, target_inst, - target_loop, remedies); - errs() << "Target inst: " << *target_inst << "\n"; - errs() << "I: " << I << "\n"; - // convert retLCFW to int and print - errs() << "retLCFW: " << (int)(retLCFW) << "\n"; - errs() << "retLCBW: " << (int)(retLCBW) << "\n"; - errs() << "retIIFW: " << (int)(retIIFW) << "\n"; - errs() << "retIIBW: " << (int)(retIIBW) << "\n"; - errs() << "modrefIIFW: " << (modrefIIFW) << "\n"; - errs() << "modrefIIBW: " << (modrefIIBW) << "\n"; - }); + Remedies remedies; + LoopAA::ModRefResult modrefIIFW = aa->modref( + target_inst, LoopAA::Same, &I, target_loop, remedies); + auto modrefIIBW = aa->modref(&I, LoopAA::Same, target_inst, + target_loop, remedies); + errs() << "Target inst: " << *target_inst << "\n"; + errs() << "I: " << I << "\n"; + // convert retLCFW to int and print + errs() << "retLCFW: " << (int)(retLCFW) << "\n"; + errs() << "retLCBW: " << (int)(retLCBW) << "\n"; + errs() << "retIIFW: " << (int)(retIIFW) << "\n"; + errs() << "retIIBW: " << (int)(retIIBW) << "\n"; + errs() << "modrefIIFW: " << (modrefIIFW) << "\n"; + errs() << "modrefIIBW: " << (modrefIIBW) << "\n"; + }); // RAW disproved for all deps if ((retLCFW & 0b001) && (retLCBW & 0b001) && (retIIFW & 0b001) && (retIIBW & 0b001)) { @@ -453,6 +455,8 @@ bool SLAMP::runOnModule(Module &m) { if (UsePointsToModule){ instrumentAllocas(m); + // instrument all base pointer creation + instrumentBasePointer(m, this->target_loop); } instrumentMainFunction(m); @@ -518,7 +522,7 @@ bool SLAMP::mayCallSetjmpLongjmp(Loop *loop) { getCallableFunctions(loop, callables); return (find_if(callables.begin(), callables.end(), is_setjmp_or_longjmp) != - callables.end()); + callables.end()); } void SLAMP::getCallableFunctions(Loop *loop, set &callables) { @@ -559,7 +563,7 @@ void SLAMP::getCallableFunctions(CallInst *ci, set &callables) { void SLAMP::getCallableFunctions(Function *f, set &callables) { for (inst_iterator ii = inst_begin(f); ii != inst_end(f); ii++) { - // FIXME: not only callinst are callable + // FIXME: not only callinst are callable auto *ci = dyn_cast(&*ii); if (!ci) continue; @@ -588,7 +592,7 @@ void SLAMP::getFunctionsWithSign(CallInst *ci, set matched) { Function::arg_iterator fai; CallSite::arg_iterator cai; for (fai = func->arg_begin(), cai = cs.arg_begin(); - fai != func->arg_end(); fai++, cai++) { + fai != func->arg_end(); fai++, cai++) { Value *af = &*fai; Value *ac = *cai; if (af->getType() != ac->getType()) { @@ -610,18 +614,18 @@ void SLAMP::replaceExternalFunctionCalls(Module &m) { auto *push = cast( m.getOrInsertFunction("SLAMP_ext_push", Void, I32).getCallee()); auto *pop = - cast(m.getOrInsertFunction("SLAMP_ext_pop", Void).getCallee()); + cast(m.getOrInsertFunction("SLAMP_ext_pop", Void).getCallee()); set externs; for (unsigned i = 0, e = sizeof(externs_str) / sizeof(externs_str[0]); i < e; - i++) + i++) externs.insert(externs_str[i]); // initialize a set of external functions not to be implemented set ignores; for (unsigned i = 0, - e = sizeof(ignore_externs_str) / sizeof(ignore_externs_str[0]); - i < e; i++) + e = sizeof(ignore_externs_str) / sizeof(ignore_externs_str[0]); + i < e; i++) ignores.insert(ignore_externs_str[i]); vector funcs; @@ -641,7 +645,7 @@ void SLAMP::replaceExternalFunctionCalls(Module &m) { if (func->isIntrinsic()) { // just confirm that all uses is an intrinsic instruction for (Value::user_iterator ui = func->user_begin(); ui != func->user_end(); - ui++) + ui++) assert(isa(*ui)); continue; } @@ -651,75 +655,77 @@ void SLAMP::replaceExternalFunctionCalls(Module &m) { bool hasUnrecognizedFunction = false; for (auto func : funcs) { - string name = func->getName(); + string name = func->getName(); - if (externs.find(name) == externs.end()) { - // check if the function argument is `readnone`, then it's pure - if (func->hasFnAttribute(llvm::Attribute::AttrKind::ReadNone)) { + // start with SLAMP_, ignore it + if (name.find("SLAMP_") == 0) { + continue; + } + + // find all usage of the function + // add a slamp_push and slamp_pop around it + for (auto user : func->users()) { + // get instruction + auto *inst = dyn_cast(user); + if (inst == nullptr) continue; - } - // start with SLAMP_, ignore it - if (name.find("SLAMP_") == 0) { + + // FIXME: duplicated code as instrumentLoopInst + auto id = Namer:: getInstrId(inst); + if (id == -1) { continue; } + vector args; + args.push_back(ConstantInt::get(I32, id)); + InstInsertPt pt = InstInsertPt::Before(inst); + pt << updateDebugInfo(CallInst::Create(push, args), pt.getPosition(), m); - errs() << "WARNING: Wrapper for external function " << name - << " not implemented.\n"; - hasUnrecognizedFunction = true; - - // find all usage of the function - // add a slamp_push and slamp_pop around it - for (auto user : func->users()) { - // get instruction - auto *inst = dyn_cast(user); - if (inst == nullptr) - continue; - - // FIXME: duplicated code as instrumentLoopInst - auto id = Namer:: getInstrId(inst); - if (id == -1) { - continue; - } - vector args; - args.push_back(ConstantInt::get(I32, id)); - InstInsertPt pt = InstInsertPt::Before(inst); - pt << updateDebugInfo(CallInst::Create(push, args), pt.getPosition(), m); + if (isa(inst)) { + pt = InstInsertPt::After(inst); + pt << updateDebugInfo(CallInst::Create(pop), pt.getPosition(), m); + } else if (auto *invokeI = dyn_cast(inst)) { + // for invoke, need to find the two paths and add pop + auto insertPop = [&pop, &m](BasicBlock* entry){ + InstInsertPt pt; + if (isa(entry->getFirstNonPHI())) + pt = InstInsertPt::After(entry->getFirstNonPHI()); + else + pt = InstInsertPt::Before(entry->getFirstNonPHI()); - if (isa(inst)) { - pt = InstInsertPt::After(inst); pt << updateDebugInfo(CallInst::Create(pop), pt.getPosition(), m); - } else if (auto *invokeI = dyn_cast(inst)) { - // for invoke, need to find the two paths and add pop - auto insertPop = [&pop, &m](BasicBlock* entry){ - InstInsertPt pt; - if (isa(entry->getFirstNonPHI())) - pt = InstInsertPt::After(entry->getFirstNonPHI()); - else - pt = InstInsertPt::Before(entry->getFirstNonPHI()); - - pt << updateDebugInfo(CallInst::Create(pop), pt.getPosition(), m); - }; - - insertPop(invokeI->getNormalDest()); - // FIXME: will generate mulitiple `slamp_pop` after the landing pad - // Fine for now because `slamp_pop` only set the context to 0 - insertPop(invokeI->getUnwindDest()); + }; - } else { - assert(false && "Call but not CallInst nor InvokeInst"); - } + insertPop(invokeI->getNormalDest()); + // FIXME: will generate mulitiple `slamp_pop` after the landing pad + // Fine for now because `slamp_pop` only set the context to 0 + insertPop(invokeI->getUnwindDest()); + + } else { + assert(false && "Call but not CallInst nor InvokeInst"); + } + } + if (externs.find(name) == externs.end()) { + // check if the function argument is `readnone`, then it's pure + if (func->hasFnAttribute(llvm::Attribute::AttrKind::ReadNone)) { + continue; } + errs() << "WARNING: Wrapper for external function " << name + << " not implemented.\n"; + hasUnrecognizedFunction = true; + } else { string wrapper_name = "SLAMP_" + name; /* Function* wrapper = cast( m.getOrInsertFunction(wrapper_name, * func->getFunctionType() ) ); */ FunctionCallee wrapper = - m.getOrInsertFunction(wrapper_name, func->getFunctionType()); + m.getOrInsertFunction(wrapper_name, func->getFunctionType()); // replace 'func' to 'wrapper' in uses func->replaceAllUsesWith(wrapper.getCallee()); } + + } if (hasUnrecognizedFunction) { @@ -899,6 +905,96 @@ void SLAMP::reportEndOfAllocaLifetime(AllocaInst *inst, Instruction *end, bool e return; } +/// For each pointer use in the targeted loop +/// 1. find the base pointer +/// 2. Go to the creation time of the pointer +/// 3. Insert a call to SLAMP_report_base_pointer(instruction, address) +void SLAMP::instrumentBasePointer(Module &m, Loop* l) { + + const DataLayout &DL = m.getDataLayout(); + + auto *find_underlying_arg = cast( + m.getOrInsertFunction("SLAMP_report_base_pointer_arg", Void, I32, I32, I8Ptr) + .getCallee()); + auto *find_underlying_inst = cast( + m.getOrInsertFunction("SLAMP_report_base_pointer_inst", Void, I32, I8Ptr) + .getCallee()); + + // collect all pointer use by load, store, function argument in the targeted loop + std::set indeterminate_pointers, indeterminate_objects, already; + for(auto *bb: l->getBlocks()) + { + Indeterminate::findIndeterminateObjects(*bb, indeterminate_pointers, indeterminate_objects); + } + + for (auto &object : indeterminate_objects) { + if (const auto *const_arg = dyn_cast(object)) { + if (already.count(const_arg)) + continue; + already.insert(const_arg); + + LLVM_DEBUG( + errs() + << "Instrumenting indeterminate base object in function argument " + << *const_arg << "\n"); + + auto *arg = const_cast(const_arg); + Function *fcn = arg->getParent(); + + Instruction *cast = new BitCastInst(arg, I8Ptr); + auto fcnId = Namer::getFuncId(fcn); + auto argId = arg->getArgNo(); + + auto pt = InstInsertPt::Beginning(fcn); + Value *args[] = {ConstantInt::get(I32, fcnId), + ConstantInt::get(I32, argId), cast}; + pt << cast + << updateDebugInfo(CallInst::Create(find_underlying_arg, args), + pt.getPosition(), m); + } + else if (const auto *const_inst = dyn_cast(object)) { + if (already.count(const_inst)) + continue; + already.insert(const_inst); + + if (isa(const_inst)) + continue; + + LLVM_DEBUG(errs() << "Instrumenting indeterminate base object: " + << *const_inst << '\n'); + auto *inst = const_cast(const_inst); + + Instruction *cast = new BitCastInst(inst, I8Ptr); + + InstInsertPt where; + if (auto *invoke = dyn_cast(inst)) { + auto entry = invoke->getNormalDest(); + if (isa(entry->getFirstNonPHI())) + where = InstInsertPt::After(entry->getFirstNonPHI()); + else + where = InstInsertPt::Before(entry->getFirstNonPHI()); + } else if (auto *phi = dyn_cast(inst)) { + // Don't accidentally insert instrumentation before + // later PHIs or landing pad instructions. + where = InstInsertPt::Beginning(phi->getParent()); + } else + where = InstInsertPt::After(inst); + + auto instId = Namer::getInstrId(inst); + Value *args[] = { + ConstantInt::get(I32, instId), + cast + }; + where << cast + << updateDebugInfo(CallInst::Create(find_underlying_inst, args), where.getPosition(), m); + } + else { + errs() << "What is: " << *object << '\n'; + assert(false && "Unknown object type?!?!"); + } + } +} + // For each alloca, find the lifetime starts and ends // and insert calls to `SLAMP_callback_stack_alloca` and // `SLAMP_callback_stack_free` diff --git a/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_hooks.cpp b/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_hooks.cpp index 511c0cf3..bce4b193 100644 --- a/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_hooks.cpp +++ b/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_hooks.cpp @@ -51,6 +51,9 @@ using namespace SLAMPLib; #define HEAP_BOUND_HIGHER (0xFFFFFFFFFFFFL-BOUND_LOWER+1) // 8MB max stack size for a typical host machine `ulimit -s` #define SIZE_8M 0x800000 + +#define FORMAT_INST_ARG(fcnId, argId) (fcnId << 5 | (0x1f & (argId << 4) | 0x1)) +#define FORMAT_INST_INST(instId) (instId << 1 | 0x0) // #define LOCALWRITE(addr) if (true) // debugging tools @@ -111,6 +114,10 @@ extern size_t LOCALWRITE_PATTERN; #define LOCALWRITE(addr) ((!LOCALWRITE_MODULE || ((size_t)addr & LOCALWRITE_MASK) == LOCALWRITE_PATTERN)) +static void *(*old_malloc_hook)(size_t, const void *); +static void (*old_free_hook)(void *, const void *); +static void *(*old_memalign_hook)(size_t, size_t, const void *); + uint64_t __slamp_iteration = 0; uint64_t __slamp_invocation = 0; @@ -118,7 +125,7 @@ uint64_t __slamp_load_count = 0; uint64_t __slamp_store_count = 0; uint64_t __slamp_malloc_count = 0; uint64_t __slamp_free_count = 0; -static ContextManager contextManager; +ContextManager *contextManager; std::unordered_set *shortLivedObjects, *longLivedObjects; // Type of the access callback function @@ -143,7 +150,7 @@ void SLAMP_callback_stack_alloca(uint64_t array_size, uint64_t type_size, uint32 void* shadow = (void*)GET_SHADOW(addr, TIMESTAMP_SIZE_IN_POWER_OF_TWO); if (shadow && POINTS_TO_MODULE) { TS *s = (TS *)shadow; - auto hash = contextManager.activeHash(); + auto hash = contextManager->activeHash(); TS ts = CREATE_TS(instr, hash, __slamp_invocation); for (auto i = 0; i < size; i++) s[i] = ts; @@ -340,7 +347,7 @@ void slamp_access_callback_constant_value(bool isLoad, uint32_t instr, uint32_t } } } else { - auto cp = new Constant(1, 1, size, addr, value); + auto cp = new Constant(true, true, size, addr, value); constmap_value->insert(std::make_pair(key, cp)); } } @@ -441,6 +448,12 @@ std::unordered_map> *pointsToM template void SLAMP_points_to_module_use(uint32_t instr, uint64_t addr) { + TURN_OFF_CUSTOM_MALLOC; + + if (addr == 0) { + (*pointsToMap)[instr].insert(0); + return; + } TS* s = (TS*)GET_SHADOW(addr, TIMESTAMP_SIZE_IN_POWER_OF_TWO); TS tss[8]; // HACK: avoid using malloc for (auto i = 0; i < size; i++) { @@ -456,17 +469,22 @@ void SLAMP_points_to_module_use(uint32_t instr, uint64_t addr) { if (cond && tss[i] != 0) { // mask off the iteration count TS ts = tss[i]; - ts = ts & 0xfffff00000000000; + ts = ts & 0xfffffffffffffe00; // ts = ts & 0xfffffffffffffff0; // ts = ts & 0xfffff0000000000f; //create set of objects for each load/store (*pointsToMap)[instr].insert(ts); } } + TURN_ON_CUSTOM_MALLOC; } void SLAMP_points_to_module_use(uint32_t instr, uint64_t addr, unsigned size) { + if (addr == 0) { + (*pointsToMap)[instr].insert(0); + return; + } std::unordered_set m; TS *s = (TS *)GET_SHADOW(addr, TIMESTAMP_SIZE_IN_POWER_OF_TWO); @@ -478,7 +496,8 @@ void SLAMP_points_to_module_use(uint32_t instr, uint64_t addr, unsigned size) { ts = s[i]; // mask off the iteration count - ts = ts & 0xfffff00000000000; + // ts = ts & 0xfffff00000000000; + ts = ts & 0xfffffffffffffe00; // ts = ts & 0xfffffffffffffff0; // ts = ts & 0xfffff0000000000f; if (m.count(ts) == 0) { @@ -720,10 +739,6 @@ static void __slamp_sigsegv_handler(int sig, siginfo_t *siginfo, void *dummy) } -static void *(*old_malloc_hook)(size_t, const void *); -static void (*old_free_hook)(void *, const void *); -static void *(*old_memalign_hook)(size_t, size_t, const void *); - static void* SLAMP_malloc_hook(size_t size, const void * /*caller*/) { auto ptr = SLAMP_malloc(size, ext_context, 16); @@ -747,6 +762,7 @@ void SLAMP_init(uint32_t fn_id, uint32_t loop_id) { if (POINTS_TO_MODULE) { + contextManager = new ContextManager(); shortLivedObjects = new std::unordered_set(); longLivedObjects = new std::unordered_set(); } @@ -925,12 +941,12 @@ void SLAMP_fini(const char* filename) ofs << "Short-lived object:\n"; ofs << shortLivedObjects->size() << " " << longLivedObjects->size() << "\n"; for (auto &obj: *shortLivedObjects) { - // if not long lived + // if short-lived ofs << obj << "\n"; } for (auto &obj: *longLivedObjects) { - // if not long lived + // if long-lived ofs << obj << "\n"; } @@ -938,6 +954,63 @@ void SLAMP_fini(const char* filename) } } + // Dump compatible one as SpecPriv output + std::ofstream specprivfs("specpriv-profile.out"); + specprivfs << "BEGIN SPEC PRIV PROFILE\n"; + specprivfs << "COMPLETE ALLOCATION INFO ; \n"; + + // local objects + if (POINTS_TO_MODULE) { + for (auto &obj: *shortLivedObjects) { + // LOCAL OBJECT AU HEAP main if.else.i call.i4.i FROM CONTEXT { LOOP main for.cond15 1 WITHIN FUNCTION main WITHIN TOP } IS LOCAL TO CONTEXT { LOOP main for.cond15 1 WITHIN FUNCTION main WITHIN TOP } COUNT 300 ; + specprivfs << "LOCAL OBJECT " << obj << " ; \n"; + } + + } + + // predict int and predict ptr (only NULL) + // Note that the int prediction is in place, however SpecPriv rematerializes the load to the preheader of the loop + + // PRED INT main enqueue.exit23.i $0 AT CONTEXT { LOOP main for.cond15 1 WITHIN FUNCTION main WITHIN TOP } AS PREDICTABLE 301 SAMPLES OVER 1 VALUES { ( INT 0 COUNT 301 ) } ; + // PRED PTR main if.then29.i $17 AT CONTEXT { LOOP main for.cond15 1 WITHIN FUNCTION main WITHIN TOP } AS PREDICTABLE 301 SAMPLES OVER 1 VALUES { ( OFFSET 0 BASE AU NULL COUNT 301 ) } ; + + if (CONSTANT_VALUE_MODULE) { + for (auto &[key, cp] : *constmap_value) { + if (cp->valid) { + // instr and value + auto instr = key.first; + auto value = cp->value; + // later, we need to parse the instruction to see if it's a pointer or a regular integer + specprivfs << "PRED VAL " << instr << " " << value << " ; \n"; + } + } + } + // predict OBJ + // PRED OBJ main if.else.i $0 AT CONTEXT { LOOP main for.cond15 1 WITHIN FUNCTION main WITHIN TOP } AS PREDICTABLE 300 SAMPLES OVER 1 VALUES { ( OFFSET 0 BASE AU HEAP allocate_matrices for.end call7 FROM CONTEXT { FUNCTION allocate_matrices WITHIN FUNCTION main WITHIN TOP } COUNT 300 ) } ; + if (POINTS_TO_MODULE) { + for (auto &it : *pointsToMap) { + specprivfs << "PRED OBJ " << it.first << " "; // instruction ID + for (auto &it2 : it.second) { // the set of allocation units + auto hash = GET_HASH(it2); + auto iter = GET_ITER(it2) & 0xf; + + std::vector context = contextManager->contextMap[hash]; + + specprivfs << "instr - "<< GET_INSTR(it2) << " Context: " ; + for (auto &ctx : context) { + ctx.print(specprivfs); + } + + specprivfs << " iter - " << iter << " invoc - " << GET_INVOC(it2) << "\n"; + } + specprivfs << "\n"; + } + } + + + specprivfs << " END SPEC PRIV PROFILE\n"; + + if (DEPENDENCE_MODULE) { slamp::fini_logger(filename); // delete smmap; @@ -1024,18 +1097,23 @@ void SLAMP_main_entry(uint32_t argc, char** argv, char** env) /// Keep track of the context of the function void SLAMP_enter_fcn(uint32_t fcnId) { + + TURN_OFF_CUSTOM_MALLOC; auto contextId = SpecPrivLib::ContextId(SpecPrivLib::FunctionContext, fcnId); // update the current context - contextManager.updateContext(contextId); + contextManager->updateContext(contextId); + TURN_ON_CUSTOM_MALLOC; } /// Keep track of the context of the function void SLAMP_exit_fcn(uint32_t fcnId) { + TURN_OFF_CUSTOM_MALLOC; auto contextId = SpecPrivLib::ContextId(SpecPrivLib::FunctionContext, fcnId); // update the current context - contextManager.popContext(contextId); + contextManager->popContext(contextId); + TURN_ON_CUSTOM_MALLOC; } /// update the invocation count @@ -1079,6 +1157,33 @@ void SLAMP_loop_exit() { invokedepth--; } +// FIXME: a temporary patch for out of handling program original heap +bool SLAMP_isBadAlloc(uint64_t addr) ATTRIBUTE(always_inline) { + const uint64_t lower = 0x100000000L; + const uint64_t higher = 0x010000000000L; + const uint64_t heapStart = smmap->heapStart; + + if (addr < lower && addr > heapStart) { + return true; + } + + return false; +} + +void SLAMP_report_base_pointer_arg(uint32_t fcnId, uint32_t argId, void *ptr){ + if (SLAMP_isBadAlloc((uint64_t)ptr)) { + return; + } + SLAMP_points_to_module_use<1>(FORMAT_INST_ARG(fcnId, argId), (uint64_t)ptr); +} + +void SLAMP_report_base_pointer_inst(uint32_t instId, void *ptr){ + if (SLAMP_isBadAlloc((uint64_t)ptr)) { + return; + } + SLAMP_points_to_module_use<1>(FORMAT_INST_INST(instId), (uint64_t)ptr); +} + /// set the context of the call inside a loop void SLAMP_ext_push(const uint32_t instr) ATTRIBUTE(always_inline) { assert(ext_context == 0); @@ -1121,19 +1226,6 @@ void SLAMP_pop() { StaticInstIdOfLOI = 0; } -// FIXME: a temporary patch for out of handling program original heap -bool SLAMP_isBadAlloc(uint64_t addr) ATTRIBUTE(always_inline) { - const uint64_t lower = 0x100000000L; - const uint64_t higher = 0x010000000000L; - const uint64_t heapStart = smmap->heapStart; - - if (addr < lower && addr > heapStart) { - return true; - } - - return false; -} - template void SLAMP_dependence_module_load_log(const uint32_t instr, const uint32_t bare_instr, const uint64_t value, const uint64_t addr) ATTRIBUTE(noinline) { uint64_t START; @@ -1246,9 +1338,9 @@ void SLAMP_load(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, SLAMP_dependence_module_load_log(instr, bare_instr, value, addr); } - if (POINTS_TO_MODULE) { - SLAMP_points_to_module_use(instr, addr); - } + // if (POINTS_TO_MODULE) { + // SLAMP_points_to_module_use(instr, addr); + // } #ifndef ITO_ENABLE TURN_ON_CUSTOM_MALLOC; #endif @@ -1306,10 +1398,10 @@ void SLAMP_loadn(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, SLAMP_dependence_module_load_log(instr, bare_instr, 0, addr, n); } - if (POINTS_TO_MODULE) { - // only need to check once - SLAMP_points_to_module_use(instr, addr, n); - } + // if (POINTS_TO_MODULE) { + // // only need to check once + // SLAMP_points_to_module_use(instr, addr, n); + // } TURN_ON_CUSTOM_MALLOC; } @@ -1442,9 +1534,9 @@ void SLAMP_store(uint32_t instr, uint32_t bare_instr, const uint64_t addr) ATTRI SLAMP_dependence_module_store_log(instr, addr); } - if (POINTS_TO_MODULE) { - SLAMP_points_to_module_use(instr, addr); - } + // if (POINTS_TO_MODULE) { + // SLAMP_points_to_module_use(instr, addr); + // } #ifndef ITO_ENABLE TURN_ON_CUSTOM_MALLOC; #endif @@ -1497,10 +1589,10 @@ void SLAMP_storen(uint32_t instr, const uint64_t addr, size_t n) { SLAMP_dependence_module_store_log(instr, addr, n); } - if (POINTS_TO_MODULE) { - // only need to check once - SLAMP_points_to_module_use(instr, addr, n); - } + // if (POINTS_TO_MODULE) { + // // only need to check once + // SLAMP_points_to_module_use(instr, addr, n); + // } TURN_ON_CUSTOM_MALLOC; } } @@ -1588,13 +1680,17 @@ void* SLAMP_malloc(size_t size, uint32_t instr, size_t alignment) // log all data into sigle TS // FIXME: static instruction and the dynamic context? // context: static instr + function + loop - auto hash = contextManager.activeHash(); + auto hash = contextManager->activeHash(); // currentContext->print(std::cerr); // std::cerr << "hash: " << hash << "\n"; // TS ts = CREATE_TS(instr, hash, __slamp_invocation); - TS ts = CREATE_TS(instr, __slamp_iteration, __slamp_invocation); + TS ts = CREATE_TS_HASH(instr, hash, __slamp_iteration, __slamp_iteration); + // TS ts = CREATE_TS(instr, __slamp_iteration, __slamp_invocation); + if (instr == 0) { + std::cerr << "Ext context: " << ext_context << "\n"; + } //8 bytes per byte TODO: can we reduce this? for (auto i = 0; i < size; i++) @@ -1649,13 +1745,20 @@ void SLAMP_free(void* ptr) auto instr = GET_INSTR(s[0]); auto iteration = GET_ITER(s[0]); auto invocation = GET_INVOC(s[0]); + + if (instr == 0) { + std::cerr << "TS: " << s[0] << " ptr: " << ptr << "\n"; + } // if invokedepth is 0, it means we are not in a loop - if (iteration == (0xffffffffff & __slamp_iteration) + if (iteration == (0xf & __slamp_iteration) && invocation == (0xf & __slamp_invocation) && invokedepth > 0) { // is short-lived, put in the set shortLivedObjects->insert(instr); + for (auto &obj: *shortLivedObjects) { + std::cerr << "short lived object: " << obj << "\n"; + } } else { // is not short-lived @@ -1670,4 +1773,5 @@ void SLAMP_free(void* ptr) } TURN_ON_CUSTOM_MALLOC; + } diff --git a/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_hooks.h b/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_hooks.h index a5456abd..e719dd15 100644 --- a/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_hooks.h +++ b/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_hooks.h @@ -67,6 +67,8 @@ void SLAMP_loop_invocation(); void SLAMP_loop_iteration(); void SLAMP_loop_exit(); +void SLAMP_report_base_pointer_arg(uint32_t, uint32_t, void *ptr); +void SLAMP_report_base_pointer_inst(uint32_t, void *ptr); void SLAMP_callback_stack_alloca(uint64_t, uint64_t, uint32_t, uint64_t); void SLAMP_callback_stack_free(void); diff --git a/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_timestamp.h b/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_timestamp.h index 419d3f9b..aada0546 100644 --- a/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_timestamp.h +++ b/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_timestamp.h @@ -9,7 +9,12 @@ typedef uint64_t TS; // first 20 bits for instr and following 44 bits for iter #define ITERATION_SIZE 40 #define INVOCATION_SIZE 4 // 44-40 #define CREATE_TS(instr, iter, invoc) ( ((TS)instr << 44) | (((TS)iter & (TS)0xffffffffff) << INVOCATION_SIZE) | ((TS)invoc & (TS)0xf)) -#define GET_INSTR(ts) ( (ts >> 44) & 0xfffff ) +#define CREATE_TS_HASH(instr, hash, iter, invoc) \ + (((TS)instr << 44) | \ + (((TS)hash & (TS)0xfffffffff) << (INVOCATION_SIZE + 1)) | \ + (((TS)iter & (TS)0xf) << INVOCATION_SIZE) | ((TS)invoc & (TS)0xf)) +#define GET_INSTR(ts) ((ts >> 44) & 0xfffff) +#define GET_HASH(ts) ( (ts >> 5) & 0xfffffffff) #define GET_ITER(ts) ( (ts >> INVOCATION_SIZE) & 0xffffffffff) #define GET_INVOC(ts) ( ts & 0xf) From 21cac1a32f91d4793be4337e2446775865865f90 Mon Sep 17 00:00:00 2001 From: Ziyang Xu Date: Wed, 28 Sep 2022 21:11:32 -0400 Subject: [PATCH 08/97] update SLAMP to support more context printing; add a parsing script to generate SpecPriv input from PROMPT output --- liberty/include/liberty/SLAMP/SLAMP.h | 2 +- liberty/lib/SLAMP/SLAMP.cpp | 145 +++++++++++++- liberty/lib/SLAMP/SLAMPlib/hooks/context.h | 47 ++++- .../lib/SLAMP/SLAMPlib/hooks/slamp_hooks.cpp | 90 +++++++-- .../lib/SLAMP/SLAMPlib/hooks/slamp_hooks.h | 3 + .../SLAMP/SLAMPlib/hooks/slamp_timestamp.h | 4 +- tests/scripts/convertPromptToSpecPriv.py | 183 ++++++++++++++++++ tests/scripts/slamp-driver | 2 +- 8 files changed, 447 insertions(+), 29 deletions(-) create mode 100644 tests/scripts/convertPromptToSpecPriv.py diff --git a/liberty/include/liberty/SLAMP/SLAMP.h b/liberty/include/liberty/SLAMP/SLAMP.h index a4074ac0..942ada53 100644 --- a/liberty/include/liberty/SLAMP/SLAMP.h +++ b/liberty/include/liberty/SLAMP/SLAMP.h @@ -59,7 +59,7 @@ class SLAMP: public ModulePass void instrumentNonStandards(Module& m, Function* ctor); void allocErrnoLocation(Module& m, Function* ctor); - + void instrumentLoopStartStopForAll(Module &m); void instrumentFunctionStartStop(Module&m); void instrumentLoopStartStop(Module&m, Loop* l); void instrumentInstructions(Module& m, Loop* l); diff --git a/liberty/lib/SLAMP/SLAMP.cpp b/liberty/lib/SLAMP/SLAMP.cpp index 4e347c0c..dc9d7d50 100644 --- a/liberty/lib/SLAMP/SLAMP.cpp +++ b/liberty/lib/SLAMP/SLAMP.cpp @@ -4,6 +4,7 @@ // #include "llvm/IR/LLVMContext.h" +#include "llvm/Support/raw_ostream.h" #define DEBUG_TYPE "SLAMP" #define USE_PDG @@ -144,6 +145,7 @@ SLAMP::~SLAMP() = default; void SLAMP::getAnalysisUsage(AnalysisUsage &au) const { // au.addRequired(); // use static ID (requires the bitcode to be exact the same) au.addRequired(); + au.addRequired(); #ifdef USE_PDG au.addRequired(); au.addRequired(); @@ -193,7 +195,25 @@ Instruction* updateDebugInfo(Instruction *inserted, Instruction *location, Modul return inserted; } +unordered_map instOffsetMap; + +void generateInstOffset(Module &m) { + // generate instruction offset + for (auto &f : m) { + if (f.isDeclaration()) continue; + for (auto &bb : f) { + unsigned offsetWithinBlock = 0; + for (auto &inst : bb) { + instOffsetMap[&inst] = offsetWithinBlock++; + } + } + } +} + bool SLAMP::runOnModule(Module &m) { + // generate instruction offset + generateInstOffset(m); + LLVMContext &ctxt = m.getContext(); // frequently used types @@ -459,9 +479,10 @@ bool SLAMP::runOnModule(Module &m) { instrumentBasePointer(m, this->target_loop); } + instrumentFunctionStartStop(m); + instrumentLoopStartStopForAll(m); instrumentMainFunction(m); - instrumentFunctionStartStop(m); instrumentLoopStartStop(m, this->target_loop); instrumentInstructions(m, this->target_loop); @@ -607,6 +628,31 @@ void SLAMP::getFunctionsWithSign(CallInst *ci, set matched) { } } +std::string getInstructionName(Instruction *inst) { + auto fcn = inst->getParent()->getParent(); + auto bb = inst->getParent(); + + std::stringstream sout; + sout << fcn->getName().str() << ' ' << bb->getName().str() << ' '; + + if( inst->hasName() ) + sout << inst->getName().str(); + else { + // find the offset within the block + sout << '$' << instOffsetMap[inst]; + } + + return sout.str(); +} + +std::string getArgName(Argument *arg) { + Function *fcn = arg->getParent(); + + std::ostringstream name; + name << "argument " << fcn->getName().str() << " %" << arg->getArgNo(); + return name.str(); +} + // Replace external functions with SLAMP prefixed ones (SLAMP_xxx) // The list of SLAMP functions are given in `externs.h` void SLAMP::replaceExternalFunctionCalls(Module &m) { @@ -680,6 +726,9 @@ void SLAMP::replaceExternalFunctionCalls(Module &m) { InstInsertPt pt = InstInsertPt::Before(inst); pt << updateDebugInfo(CallInst::Create(push, args), pt.getPosition(), m); + errs() << "Malloc ID " << id << " : " + << getInstructionName(inst) << "\n"; + if (isa(inst)) { pt = InstInsertPt::After(inst); pt << updateDebugInfo(CallInst::Create(pop), pt.getPosition(), m); @@ -951,6 +1000,10 @@ void SLAMP::instrumentBasePointer(Module &m, Loop* l) { pt << cast << updateDebugInfo(CallInst::Create(find_underlying_arg, args), pt.getPosition(), m); + + // errs() << "UO Arg (" << fcnId << "," << argId << ") : " + errs() << "UO Arg " << (fcnId << 5 | ((0x1f & (argId << 4)) | 0x1)) << " : " + << getArgName(arg) << "\n"; } else if (const auto *const_inst = dyn_cast(object)) { if (already.count(const_inst)) @@ -987,6 +1040,8 @@ void SLAMP::instrumentBasePointer(Module &m, Loop* l) { }; where << cast << updateDebugInfo(CallInst::Create(find_underlying_inst, args), where.getPosition(), m); + + errs() << "UO Inst " << (instId << 1 | 0x0) << " : " << getInstructionName(inst) << "\n"; } else { errs() << "What is: " << *object << '\n'; @@ -1154,6 +1209,92 @@ void SLAMP::instrumentMainFunction(Module &m) { } } +/// Pass in the loop and instrument enter/exit hooks +void SLAMP::instrumentLoopStartStopForAll(Module &m) { + + // for all functions + for (auto &f : m) { + if (f.isDeclaration()) + continue; + // get all loops + LoopInfo &li = getAnalysis(f).getLoopInfo(); + for (auto &loop : li.getLoopsInPreorder()) { + + // TODO: check setjmp/longjmp + BasicBlock *header = loop->getHeader(); + unsigned loopId = Namer::getBlkId(header); + if (loopId == -1) { + assert(false && "Loop header has no id"); + } + BasicBlock *latch = loop->getLoopLatch(); + vector args; + args.push_back(ConstantInt::get(I32, loopId)); + + errs() << "Loop ID " << loopId << " : " << f.getName() << " " << header->getName() << " " << li.getLoopDepth(header) << "\n"; + + // check if loop-simplify pass executed + assert(loop->getNumBackEdges() == 1 && + "Should be only 1 back edge, loop-simplify?"); + assert(latch && "Loop latch needs to exist, loop-simplify?"); + + // add instrumentation on loop header: + // if new invocation, call SLAMP_loop_invocation, else, call + // SLAMP_loop_iteration + auto *f_loop_invoke = cast( + m.getOrInsertFunction("SLAMP_enter_loop", Void, I32).getCallee()); + auto *f_loop_iter= cast( + m.getOrInsertFunction("SLAMP_loop_iter_ctx", Void, I32).getCallee()); + auto *f_loop_exit = cast( + m.getOrInsertFunction("SLAMP_exit_loop", Void, I32).getCallee()); + + PHINode *funcphi = PHINode::Create(f_loop_invoke->getType(), 2, "funcphi_loop_context"); + InstInsertPt pt; + + if (isa(header->getFirstNonPHI())) + pt = InstInsertPt::After(header->getFirstNonPHI()); + else + pt = InstInsertPt::Before(header->getFirstNonPHI()); + + pt << funcphi; + + // choose which function to execute (iter or invoke) + for (auto pred : predecessors(header)) { + if (pred == latch) + funcphi->addIncoming(f_loop_iter, pred); + else + funcphi->addIncoming(f_loop_invoke, pred); + } + + updateDebugInfo(CallInst::Create(funcphi, args, "", header->getFirstNonPHI()), header->getFirstNonPHI(), m); + + // Add `SLAMP_loop_exit` to all loop exits + SmallVector exits; + loop->getExitBlocks(exits); + + // one instrumentation per block + set s; + + for (unsigned i = 0; i < exits.size(); i++) { + if (s.count(exits[i])) + continue; + + CallInst *ci = CallInst::Create(f_loop_exit, args); + + InstInsertPt pt2; + if (isa(exits[i]->getFirstNonPHI())) + pt2 = InstInsertPt::After(exits[i]->getFirstNonPHI()); + else + pt2 = InstInsertPt::Before(exits[i]->getFirstNonPHI()); + + pt2 << updateDebugInfo(ci, pt2.getPosition(), m); + + s.insert(exits[i]); + } + } + } + +} + /// Instrumnent each function entry and exit with SLAMP function entry and exit calls void SLAMP::instrumentFunctionStartStop(Module &m) { @@ -1175,6 +1316,8 @@ void SLAMP::instrumentFunctionStartStop(Module &m) { continue; } + errs() << "Function ID " << fcnID << " : " << func->getName() << "\n"; + // set parameters vector args; args.push_back(ConstantInt::get(I32, fcnID)); diff --git a/liberty/lib/SLAMP/SLAMPlib/hooks/context.h b/liberty/lib/SLAMP/SLAMPlib/hooks/context.h index 817045c4..b51622a7 100644 --- a/liberty/lib/SLAMP/SLAMPlib/hooks/context.h +++ b/liberty/lib/SLAMP/SLAMPlib/hooks/context.h @@ -19,6 +19,7 @@ #include #include #include +#include #include "llvm/IR/Instruction.h" #include "scaf/Utilities/Metadata.h" @@ -91,7 +92,7 @@ struct ContextManager { std::unordered_map contextMap; - ContextHash activeHash() { + ContextHash activeId() { if (activeContext) return activeContext->hash(); else @@ -102,7 +103,7 @@ struct ContextManager { assert(activeContext && "active context is null"); /// maybe this is not a great idea, expose the semantics of the context to the manager activeContext = activeContext->chain(contextId); - addContext(activeContext->hash(), *activeContext); + // addContext(activeContext->hash(), *activeContext); } void popContext(typename Context::ContextIdType contextId) { @@ -157,6 +158,7 @@ enum SpecPrivContextType { LoopContext, }; + struct ContextId { SpecPrivContextType type; int32_t metaId; @@ -173,13 +175,50 @@ struct ContextId { return type == other.type && metaId == other.metaId; } + bool operator<(const ContextId &other) const { + return type < other.type || (type == other.type && metaId < other.metaId); + } + void print(std::ostream &os) { os << "ContextId: " << type << " " << metaId << "\n"; } }; -using Context = Context; -}; // namespace SpecPrivLib +using SpecPrivContext = Context; + +struct SpecPrivContextManager : public ContextManager { + std::map, ContextHash> contextIdHashMap; + size_t contextIdHashCounter = 1; + + ContextHash encodeContext(SpecPrivContext context) { + std::vector flattenContext = context.flatten(); + if (contextIdHashMap.count(flattenContext)) { + return contextIdHashMap[flattenContext]; + } else { + contextIdHashMap[flattenContext] = contextIdHashCounter; + contextMap[contextIdHashCounter] = flattenContext; + + return contextIdHashCounter++; + } + } + + ContextHash encodeActiveContext() { + // std::cerr << "encodeActiveContext: "; + // activeContext->print(std::cerr); + return encodeContext(*activeContext); + } + + std::vector decodeContext(ContextHash hash) { + if (contextMap.count(hash)) { + return contextMap[hash]; + } else { + assert(false && "Could not find the context"); + return {}; + } + } + +}; +}; // namespace SpecPrivLib } // namespace SLAMPLib #endif // SLAMPLIB_HOOKS_SLAMP_CONTEXT_H diff --git a/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_hooks.cpp b/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_hooks.cpp index bce4b193..b9ca1223 100644 --- a/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_hooks.cpp +++ b/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_hooks.cpp @@ -125,7 +125,7 @@ uint64_t __slamp_load_count = 0; uint64_t __slamp_store_count = 0; uint64_t __slamp_malloc_count = 0; uint64_t __slamp_free_count = 0; -ContextManager *contextManager; +SpecPrivLib::SpecPrivContextManager *contextManager; std::unordered_set *shortLivedObjects, *longLivedObjects; // Type of the access callback function @@ -143,6 +143,7 @@ void slamp_global_callback(const char* name, uint64_t addr, uint64_t size) {} // TODO: activate shadow memory (SLAMP_malloc) void SLAMP_callback_stack_alloca(uint64_t array_size, uint64_t type_size, uint32_t instr, uint64_t addr) { + TURN_OFF_CUSTOM_MALLOC; uint64_t size = array_size*type_size; if (DEPENDENCE_MODULE || POINTS_TO_MODULE) { @@ -150,13 +151,15 @@ void SLAMP_callback_stack_alloca(uint64_t array_size, uint64_t type_size, uint32 void* shadow = (void*)GET_SHADOW(addr, TIMESTAMP_SIZE_IN_POWER_OF_TWO); if (shadow && POINTS_TO_MODULE) { TS *s = (TS *)shadow; - auto hash = contextManager->activeHash(); + auto hash = contextManager->encodeActiveContext(); TS ts = CREATE_TS(instr, hash, __slamp_invocation); for (auto i = 0; i < size; i++) s[i] = ts; } } - return; + + TURN_ON_CUSTOM_MALLOC; + return; } @@ -469,7 +472,7 @@ void SLAMP_points_to_module_use(uint32_t instr, uint64_t addr) { if (cond && tss[i] != 0) { // mask off the iteration count TS ts = tss[i]; - ts = ts & 0xfffffffffffffe00; + ts = ts & 0xffffffffffffff00; // ts = ts & 0xfffffffffffffff0; // ts = ts & 0xfffff0000000000f; //create set of objects for each load/store @@ -762,7 +765,7 @@ void SLAMP_init(uint32_t fn_id, uint32_t loop_id) { if (POINTS_TO_MODULE) { - contextManager = new ContextManager(); + contextManager = new SpecPrivLib::SpecPrivContextManager(); shortLivedObjects = new std::unordered_set(); longLivedObjects = new std::unordered_set(); } @@ -959,11 +962,23 @@ void SLAMP_fini(const char* filename) specprivfs << "BEGIN SPEC PRIV PROFILE\n"; specprivfs << "COMPLETE ALLOCATION INFO ; \n"; + auto printContext = [&specprivfs](const std::vector &ctx) { + for (auto &c : ctx) { + specprivfs << "(" << c.type << "," << c.metaId << ")"; + } + }; + // local objects if (POINTS_TO_MODULE) { for (auto &obj: *shortLivedObjects) { // LOCAL OBJECT AU HEAP main if.else.i call.i4.i FROM CONTEXT { LOOP main for.cond15 1 WITHIN FUNCTION main WITHIN TOP } IS LOCAL TO CONTEXT { LOOP main for.cond15 1 WITHIN FUNCTION main WITHIN TOP } COUNT 300 ; - specprivfs << "LOCAL OBJECT " << obj << " ; \n"; + auto instr = GET_INSTR(obj); + auto hash = GET_HASH(obj); + auto context = contextManager->decodeContext(hash); + specprivfs << "LOCAL OBJECT " << instr << " at context "; + printContext(context); + specprivfs << ";\n"; + } } @@ -989,21 +1004,23 @@ void SLAMP_fini(const char* filename) // PRED OBJ main if.else.i $0 AT CONTEXT { LOOP main for.cond15 1 WITHIN FUNCTION main WITHIN TOP } AS PREDICTABLE 300 SAMPLES OVER 1 VALUES { ( OFFSET 0 BASE AU HEAP allocate_matrices for.end call7 FROM CONTEXT { FUNCTION allocate_matrices WITHIN FUNCTION main WITHIN TOP } COUNT 300 ) } ; if (POINTS_TO_MODULE) { for (auto &it : *pointsToMap) { - specprivfs << "PRED OBJ " << it.first << " "; // instruction ID + specprivfs << "PRED OBJ " << it.first << ": "; // instruction ID for (auto &it2 : it.second) { // the set of allocation units auto hash = GET_HASH(it2); - auto iter = GET_ITER(it2) & 0xf; - std::vector context = contextManager->contextMap[hash]; + std::vector context = contextManager->decodeContext(hash); - specprivfs << "instr - "<< GET_INSTR(it2) << " Context: " ; - for (auto &ctx : context) { - ctx.print(specprivfs); - } + specprivfs << "AU "; - specprivfs << " iter - " << iter << " invoc - " << GET_INVOC(it2) << "\n"; + if (it2 == 0) { + specprivfs << " NULL"; + } else { + specprivfs <updateContext(contextId); + TURN_ON_CUSTOM_MALLOC; +} + +void SLAMP_exit_loop(uint32_t bbId) { + TURN_OFF_CUSTOM_MALLOC; + auto contextId = SpecPrivLib::ContextId(SpecPrivLib::LoopContext, bbId); + + // std::cerr << "exiting function " << fcnId << std::endl; + // update the current context + contextManager->popContext(contextId); + TURN_ON_CUSTOM_MALLOC; +} + /// Keep track of the context of the function void SLAMP_exit_fcn(uint32_t fcnId) { TURN_OFF_CUSTOM_MALLOC; auto contextId = SpecPrivLib::ContextId(SpecPrivLib::FunctionContext, fcnId); + // std::cerr << "exiting function " << fcnId << std::endl; // update the current context contextManager->popContext(contextId); TURN_ON_CUSTOM_MALLOC; } +void SLAMP_loop_iter_ctx(uint32_t id) {} + /// update the invocation count void SLAMP_loop_invocation() { // fprintf(stderr, "SLAMP_loop_invocation, depth: %u\n", invokedepth); @@ -1680,13 +1724,16 @@ void* SLAMP_malloc(size_t size, uint32_t instr, size_t alignment) // log all data into sigle TS // FIXME: static instruction and the dynamic context? // context: static instr + function + loop - auto hash = contextManager->activeHash(); + auto hash = contextManager->encodeActiveContext(); + // print active context + // contextManager->activeContext->print(std::cerr); // currentContext->print(std::cerr); - // std::cerr << "hash: " << hash << "\n"; + // std::cerr << "malloc hash: " << hash << "\n"; // TS ts = CREATE_TS(instr, hash, __slamp_invocation); TS ts = CREATE_TS_HASH(instr, hash, __slamp_iteration, __slamp_iteration); + // std::cerr << "TS: " << std::hex << ts << std::dec << "\n"; // TS ts = CREATE_TS(instr, __slamp_iteration, __slamp_invocation); if (instr == 0) { std::cerr << "Ext context: " << ext_context << "\n"; @@ -1743,8 +1790,11 @@ void SLAMP_free(void* ptr) // otherwise mark it as not local TS* s = (TS*)GET_SHADOW(ptr, TIMESTAMP_SIZE_IN_POWER_OF_TWO); auto instr = GET_INSTR(s[0]); - auto iteration = GET_ITER(s[0]); + auto hash = GET_HASH(s[0]); + auto iteration = GET_ITER(s[0]) & 0xF; auto invocation = GET_INVOC(s[0]); + + auto instrAndHash = s[0] & 0xFFFFFFFFFFFFFF00; if (instr == 0) { std::cerr << "TS: " << s[0] << " ptr: " << ptr << "\n"; @@ -1755,14 +1805,14 @@ void SLAMP_free(void* ptr) && invocation == (0xf & __slamp_invocation) && invokedepth > 0) { // is short-lived, put in the set - shortLivedObjects->insert(instr); + shortLivedObjects->insert(instrAndHash); for (auto &obj: *shortLivedObjects) { std::cerr << "short lived object: " << obj << "\n"; } } else { // is not short-lived - longLivedObjects->insert(instr); + longLivedObjects->insert(instrAndHash); } } diff --git a/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_hooks.h b/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_hooks.h index e719dd15..df546117 100644 --- a/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_hooks.h +++ b/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_hooks.h @@ -63,6 +63,9 @@ void SLAMP_main_entry(uint32_t argc, char** argv, char** env); void SLAMP_enter_fcn(uint32_t id); void SLAMP_exit_fcn(uint32_t id); +void SLAMP_enter_loop(uint32_t id); +void SLAMP_exit_loop(uint32_t id); +void SLAMP_loop_iter_ctx(uint32_t id); void SLAMP_loop_invocation(); void SLAMP_loop_iteration(); void SLAMP_loop_exit(); diff --git a/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_timestamp.h b/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_timestamp.h index aada0546..fc9c1428 100644 --- a/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_timestamp.h +++ b/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_timestamp.h @@ -11,10 +11,10 @@ typedef uint64_t TS; // first 20 bits for instr and following 44 bits for iter #define CREATE_TS(instr, iter, invoc) ( ((TS)instr << 44) | (((TS)iter & (TS)0xffffffffff) << INVOCATION_SIZE) | ((TS)invoc & (TS)0xf)) #define CREATE_TS_HASH(instr, hash, iter, invoc) \ (((TS)instr << 44) | \ - (((TS)hash & (TS)0xfffffffff) << (INVOCATION_SIZE + 1)) | \ + (((TS)hash & (TS)0xfffffffff) << (INVOCATION_SIZE + 4)) | \ (((TS)iter & (TS)0xf) << INVOCATION_SIZE) | ((TS)invoc & (TS)0xf)) #define GET_INSTR(ts) ((ts >> 44) & 0xfffff) -#define GET_HASH(ts) ( (ts >> 5) & 0xfffffffff) +#define GET_HASH(ts) ( (ts >> 8) & 0xfffffffff) #define GET_ITER(ts) ( (ts >> INVOCATION_SIZE) & 0xffffffffff) #define GET_INVOC(ts) ( ts & 0xf) diff --git a/tests/scripts/convertPromptToSpecPriv.py b/tests/scripts/convertPromptToSpecPriv.py new file mode 100644 index 00000000..72649185 --- /dev/null +++ b/tests/scripts/convertPromptToSpecPriv.py @@ -0,0 +1,183 @@ +import os +import argparse + + +# Parse the rabbit6 file +def parse_rabbit6(fname): + lines = [] + with open(fname, 'r') as f: + lines = f.readlines() + + # Find all "Malloc ID #id : #name" into a dict + malloc_id = {} + for line in lines: + if "Malloc ID" in line: + line = line.replace("Malloc ID", "").replace(":", "").strip() + mid, name = line.split(" ", 1) + name = name.strip() + # assert mid is an integer + assert mid.isdigit() + malloc_id[mid] = name + + # Find all "UO Inst #uoid : #name" into a dict + # Or all "UO Arg #uoid : #name" + uo_id = {} + for line in lines: + if "UO Inst" in line or "UO Arg" in line: + line = line.replace("UO Inst", "").replace("UO Arg", "").replace(":", "").strip() + print(line) + # split by the first space + mid, name = line.split(" ", 1) + name = name.strip() + # assert mid is an integer + assert mid.isdigit() + uo_id[mid] = name + + + # Find all "Function ID #fid: #name" into a dict + func_id = {} + for line in lines: + if "Function ID" in line: + line = line.replace("Function ID", "").replace(":", "").strip() + mid, name = line.split(" ", 1) + name = name.strip() + # assert mid is an integer + assert mid.isdigit() + func_id[mid] = name + + # Find all "Loop ID #lid: #name" into a dict + loop_id = {} + for line in lines: + if "Loop ID" in line: + line = line.replace("Loop ID", "").replace(":", "").strip() + mid, name = line.split(" ", 1) + name = name.strip() + # assert mid is an integer + assert mid.isdigit() + loop_id[mid] = name + + return malloc_id, uo_id, func_id, loop_id + +def parse_specpriv(fname, id_maps): + + with open(fname, 'r') as f: + lines = f.readlines() + + final_lines = [] + + # For context, if the first number is 2, it's a loop context, get the name from loop_id map + # if the first number is 1, it's a function context, get the name from func_id map + # if the first number is 0, it's a Top context, the name is "TOP" + def generateContextStr(contexts): + # Get context from contexts and parse + contexts = contexts.replace("]", "").strip().split(")(") + + context_str = "" + # type, id + for context in contexts: + context = context.replace("(", "").replace(")", "").split(",") + ctype, cid = context + # assert ctype is an integer + assert ctype.isdigit() + # assert cid is an integer + assert cid.isdigit() + + + if ctype == "2": + # loop context + ctype_name = "LOOP" + context_name = loop_ids[cid] + elif ctype == "1": + # function context + ctype_name = "FUNCTION" + context_name = func_ids[cid] + elif ctype == "0": + # top context + ctype_name = "TOP" + context_name = "" + else: + assert False + + context_str += ctype_name + " " + context_name + if ctype != "0": + context_str += " WITHIN " + + return context_str + + + malloc_ids = id_maps['malloc'] + uo_ids = id_maps['uo'] + func_ids = id_maps['func'] + loop_ids = id_maps['loop'] + + # Parse LOCAL OBJECT 279 at context (2,39)(2,35)(2,20)(1,10)(0,0); + # Get malloc ID + for line in lines: + if "LOCAL OBJECT" in line: + line = line.replace("LOCAL OBJECT", "").replace("at context", "").replace(";", "").strip() + print(line) + mid, contexts = line.split(" ", 1) + + # assert oid is an integer + assert mid.isdigit() + + # Get the name of the malloc inst + malloc_name = malloc_ids[mid] + context_str = generateContextStr(contexts) + + final_lines.append("LOCAL OBJECT AU HEAP " + malloc_name + " FROM CONTEXT { " + context_str + "} IS LOCAL TO CONTEXT { } COUNT 1") + + # Parse PRED OBJ 282: AU 24 FROM CONTEXT (1,0)(1,10)(0,0); + # The first number is UO ID, the second number is malloc ID + for line in lines: + if "PRED OBJ" in line: + line = line.replace("PRED OBJ", "").replace("AU ", "").replace("FROM CONTEXT", "").replace(":", "").replace(";", "").strip() + if "NULL" in line: + uoid, _ = line.split(" ", 1) + # assert uoid is an integer + assert uoid.isdigit() + # Get the name of the malloc inst + uo_name = uo_ids[uoid] + + final_lines.append("PRED OBJ " + uo_name + " OVER 1 VALUES { ( OFFSET 0 BASE AU NULL COUNT 1 ) }") + else: + uoid, malloc_id, contexts = line.split(" ", 2) + + # assert uoid is an integer + assert uoid.isdigit() + assert malloc_id.isdigit() + + # Get the name of the malloc inst + uo_name = uo_ids[uoid] + malloc_name = malloc_ids[malloc_id] + context_str = generateContextStr(contexts) + + final_lines.append("PRED OBJ " + uo_name + " OVER 1 VALUES { ( OFFSET 0 BASE AU HEAP " + malloc_id + " FROM CONTEXT { " + context_str + "} COUNT 1 }") + + return final_lines + + +# Parse the rabbit6 file and the specpriv-profile.out file +def parse_args(): + parser = argparse.ArgumentParser(description='Parse the rabbit6 file and the specpriv-profile.out file') + parser.add_argument('-r', '--rabbit6', help='Rabbit6 file', default="rabbit6", type=str) + parser.add_argument('-p', '--profile', help='specpriv-profile.out file', default="specpriv-profile.out", type=str) + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + rabbit6 = args.rabbit6 + profile = args.profile + + # Parse the rabbit6 file + malloc_id, uo_id, func_id, loop_id = parse_rabbit6(rabbit6) + + id_maps = {"malloc": malloc_id, "uo": uo_id, "func": func_id, "loop": loop_id} + + # Parse the specpriv-profile.out file + final_lines = parse_specpriv(profile, id_maps) + # print all lines + for line in final_lines: + print(line + ";") + diff --git a/tests/scripts/slamp-driver b/tests/scripts/slamp-driver index 3ddb8fc8..7fe157e9 100755 --- a/tests/scripts/slamp-driver +++ b/tests/scripts/slamp-driver @@ -22,7 +22,7 @@ function drive { -load $LIBERTY_LIBS_DIR/libStrategy.so -load $SCAF_LIBS_DIR/libLoopProf.so -load $SCAF_LIBS_DIR/libLAMPLoad.so - -load $SCAF_LIBS_DIR/libPointsToProfiler.so + -load $LIBERTY_LIBS_DIR/libPointsToProfiler.so -load $NOELLE_LIBS_DIR/AllocAA.so -load $NOELLE_LIBS_DIR/TalkDown.so -load $NOELLE_LIBS_DIR/CallGraph.so From 62beb3e7d066534c6f1f1b7932cb30590cc0403e Mon Sep 17 00:00:00 2001 From: Ziyang Xu Date: Thu, 29 Sep 2022 22:21:28 -0400 Subject: [PATCH 09/97] the points-to and local are working for dijkstra --- liberty/lib/SLAMP/SLAMP.cpp | 17 ++- liberty/lib/SLAMP/SLAMPlib/hooks/context.h | 11 +- .../lib/SLAMP/SLAMPlib/hooks/slamp_hooks.cpp | 126 ++++++++++++------ tests/scripts/convertPromptToSpecPriv.py | 100 ++++++++++---- .../slamp-test/specpriv_counter1/src/Makefile | 5 + tests/slamp-test/specpriv_counter1/src/test.c | 20 +++ 6 files changed, 209 insertions(+), 70 deletions(-) create mode 100644 tests/slamp-test/specpriv_counter1/src/Makefile create mode 100644 tests/slamp-test/specpriv_counter1/src/test.c diff --git a/liberty/lib/SLAMP/SLAMP.cpp b/liberty/lib/SLAMP/SLAMP.cpp index dc9d7d50..997c4737 100644 --- a/liberty/lib/SLAMP/SLAMP.cpp +++ b/liberty/lib/SLAMP/SLAMP.cpp @@ -101,7 +101,7 @@ static cl::opt UseLinearValueModule("slamp-linear-value-module", cl::init( static cl::opt UseConstantAddressModule("slamp-constant-address-module", cl::init(false), cl::NotHidden, cl::desc("Use address module")); // linear address module -static cl::opt UseLinearAddressModule("slamp-linear-address-module", cl::init(false), cl::NotHidden, cl::desc("Use linear address module")); + static cl::opt UseLinearAddressModule("slamp-linear-address-module", cl::init(false), cl::NotHidden, cl::desc("Use linear address module")); // trace module static cl::opt UseTraceModule("slamp-trace-module", cl::init(false), cl::NotHidden, cl::desc("Use trace module")); @@ -480,10 +480,10 @@ bool SLAMP::runOnModule(Module &m) { } instrumentFunctionStartStop(m); - instrumentLoopStartStopForAll(m); instrumentMainFunction(m); instrumentLoopStartStop(m, this->target_loop); + instrumentLoopStartStopForAll(m); instrumentInstructions(m, this->target_loop); @@ -971,11 +971,18 @@ void SLAMP::instrumentBasePointer(Module &m, Loop* l) { // collect all pointer use by load, store, function argument in the targeted loop std::set indeterminate_pointers, indeterminate_objects, already; - for(auto *bb: l->getBlocks()) - { - Indeterminate::findIndeterminateObjects(*bb, indeterminate_pointers, indeterminate_objects); + + for (auto &F : m) { + for (auto &BB : F) { + Indeterminate::findIndeterminateObjects(BB, indeterminate_pointers, indeterminate_objects); + } } + // for(auto *bb: l->getBlocks()) + // { + // Indeterminate::findIndeterminateObjects(*bb, indeterminate_pointers, indeterminate_objects); + // } + for (auto &object : indeterminate_objects) { if (const auto *const_arg = dyn_cast(object)) { if (already.count(const_arg)) diff --git a/liberty/lib/SLAMP/SLAMPlib/hooks/context.h b/liberty/lib/SLAMP/SLAMPlib/hooks/context.h index b51622a7..bb5ec242 100644 --- a/liberty/lib/SLAMP/SLAMPlib/hooks/context.h +++ b/liberty/lib/SLAMP/SLAMPlib/hooks/context.h @@ -189,6 +189,9 @@ struct SpecPrivContextManager : public ContextManager { std::map, ContextHash> contextIdHashMap; size_t contextIdHashCounter = 1; + + SpecPrivContext *cachedActiveContext; + ContextHash cachedContextHash; ContextHash encodeContext(SpecPrivContext context) { std::vector flattenContext = context.flatten(); @@ -205,7 +208,13 @@ struct SpecPrivContextManager : public ContextManager { ContextHash encodeActiveContext() { // std::cerr << "encodeActiveContext: "; // activeContext->print(std::cerr); - return encodeContext(*activeContext); + if (activeContext == cachedActiveContext) { + return cachedContextHash; + } else { + cachedActiveContext = activeContext; + cachedContextHash = encodeContext(*activeContext); + return cachedContextHash; + } } std::vector decodeContext(ContextHash hash) { diff --git a/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_hooks.cpp b/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_hooks.cpp index b9ca1223..700cb0b3 100644 --- a/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_hooks.cpp +++ b/liberty/lib/SLAMP/SLAMPlib/hooks/slamp_hooks.cpp @@ -127,6 +127,7 @@ uint64_t __slamp_malloc_count = 0; uint64_t __slamp_free_count = 0; SpecPrivLib::SpecPrivContextManager *contextManager; std::unordered_set *shortLivedObjects, *longLivedObjects; +std::unordered_set *targetLoopContexts; // Type of the access callback function // instr, bare_instr, address, value, size @@ -447,15 +448,18 @@ void updateInstruction(uint32_t instr, uint64_t addr) { using SlampAllocationUnit = TS; // map from load/store instruction to the allocation unit -std::unordered_map> *pointsToMap; +std::unordered_map> *pointsToMap; +// std::unordered_map> *pointsToMap; template void SLAMP_points_to_module_use(uint32_t instr, uint64_t addr) { TURN_OFF_CUSTOM_MALLOC; + auto contextHash = contextManager->encodeActiveContext(); + uint64_t instrAndHash = ((uint64_t)instr << 32) | contextHash; if (addr == 0) { - (*pointsToMap)[instr].insert(0); - return; + (*pointsToMap)[instrAndHash].insert(0); + return; } TS* s = (TS*)GET_SHADOW(addr, TIMESTAMP_SIZE_IN_POWER_OF_TWO); TS tss[8]; // HACK: avoid using malloc @@ -476,16 +480,19 @@ void SLAMP_points_to_module_use(uint32_t instr, uint64_t addr) { // ts = ts & 0xfffffffffffffff0; // ts = ts & 0xfffff0000000000f; //create set of objects for each load/store - (*pointsToMap)[instr].insert(ts); + (*pointsToMap)[instrAndHash].insert(ts); } } TURN_ON_CUSTOM_MALLOC; } void SLAMP_points_to_module_use(uint32_t instr, uint64_t addr, unsigned size) { + TURN_OFF_CUSTOM_MALLOC; + auto contextHash = contextManager->encodeActiveContext(); + uint64_t instrAndHash = ((uint64_t)instr << 32) | contextHash; if (addr == 0) { - (*pointsToMap)[instr].insert(0); + (*pointsToMap)[instrAndHash].insert(0); return; } std::unordered_set m; @@ -504,10 +511,11 @@ void SLAMP_points_to_module_use(uint32_t instr, uint64_t addr, unsigned size) { // ts = ts & 0xfffffffffffffff0; // ts = ts & 0xfffff0000000000f; if (m.count(ts) == 0) { - (*pointsToMap)[instr].insert(ts); + (*pointsToMap)[instrAndHash].insert(ts); m.insert(ts); } } + TURN_ON_CUSTOM_MALLOC; } // -1->inconclusive @@ -768,6 +776,7 @@ void SLAMP_init(uint32_t fn_id, uint32_t loop_id) contextManager = new SpecPrivLib::SpecPrivContextManager(); shortLivedObjects = new std::unordered_set(); longLivedObjects = new std::unordered_set(); + targetLoopContexts = new std::unordered_set(); } // auto heapStart = sbrk(0); // fprintf(stderr, "heap start: %lx\n", (unsigned long)heapStart); @@ -821,7 +830,7 @@ void SLAMP_init(uint32_t fn_id, uint32_t loop_id) } // initialize pointsToMap - pointsToMap = new std::unordered_map>(); + pointsToMap = new std::unordered_map>(); // #ifndef ITO_ENABLE // // FIXME: LOCALWRITE stuff should also be converted to constant if possible @@ -882,6 +891,14 @@ void SLAMP_init(uint32_t fn_id, uint32_t loop_id) smmap->allocate((void*)&stdout, sizeof(stdout)); smmap->allocate((void*)&stderr, sizeof(stderr)); smmap->allocate((void*)&sys_nerr, sizeof(sys_nerr)); + if (POINTS_TO_MODULE) { + auto size = sizeof(*stdout); + auto shadow = (TS*)smmap->allocate((void*)stdout, size); + for (auto i = 0; i < size; i+= size) { + TS ts = ~(TS)0; + shadow[i] = ts; + } + } { const unsigned short int* ctype_ptr = (*__ctype_b_loc()) - 128; @@ -968,8 +985,16 @@ void SLAMP_fini(const char* filename) } }; - // local objects if (POINTS_TO_MODULE) { + // print all loop contexts + specprivfs << "LOOP CONTEXTS: " << targetLoopContexts->size() << "\n"; + for (auto contextHash : *targetLoopContexts) { + auto context = contextManager->decodeContext(contextHash); + printContext(context); + specprivfs << "\n"; + } + + // local objects for (auto &obj: *shortLivedObjects) { // LOCAL OBJECT AU HEAP main if.else.i call.i4.i FROM CONTEXT { LOOP main for.cond15 1 WITHIN FUNCTION main WITHIN TOP } IS LOCAL TO CONTEXT { LOOP main for.cond15 1 WITHIN FUNCTION main WITHIN TOP } COUNT 300 ; auto instr = GET_INSTR(obj); @@ -1004,17 +1029,24 @@ void SLAMP_fini(const char* filename) // PRED OBJ main if.else.i $0 AT CONTEXT { LOOP main for.cond15 1 WITHIN FUNCTION main WITHIN TOP } AS PREDICTABLE 300 SAMPLES OVER 1 VALUES { ( OFFSET 0 BASE AU HEAP allocate_matrices for.end call7 FROM CONTEXT { FUNCTION allocate_matrices WITHIN FUNCTION main WITHIN TOP } COUNT 300 ) } ; if (POINTS_TO_MODULE) { for (auto &it : *pointsToMap) { - specprivfs << "PRED OBJ " << it.first << ": "; // instruction ID + auto instr = it.first >> 32; + auto instrHash = it.first & 0xFFFFFFFF; + std::vector instrContext = contextManager->decodeContext(instrHash); + specprivfs << "PRED OBJ " << instr << " at "; + printContext(instrContext); + specprivfs << ": " << it.second.size() << "\n"; // instruction ID for (auto &it2 : it.second) { // the set of allocation units auto hash = GET_HASH(it2); - std::vector context = contextManager->decodeContext(hash); - specprivfs << "AU "; - - if (it2 == 0) { + if (it2 == 0xffffffffffffff00) { + specprivfs << " UNMANAGED"; + } + else if (it2 == 0) { specprivfs << " NULL"; } else { + std::vector context = contextManager->decodeContext(hash); + specprivfs <updateContext(contextId); - TURN_ON_CUSTOM_MALLOC; + // update the current context + contextManager->updateContext(contextId); + TURN_ON_CUSTOM_MALLOC; + } } void SLAMP_enter_loop(uint32_t bbId) { + if (POINTS_TO_MODULE) { + TURN_OFF_CUSTOM_MALLOC; - TURN_OFF_CUSTOM_MALLOC; + // std::cerr << "entering function " << fcnId << std::endl; + auto contextId = SpecPrivLib::ContextId(SpecPrivLib::LoopContext, bbId); - // std::cerr << "entering function " << fcnId << std::endl; - auto contextId = SpecPrivLib::ContextId(SpecPrivLib::LoopContext, bbId); - - // update the current context - contextManager->updateContext(contextId); - TURN_ON_CUSTOM_MALLOC; + // update the current context + contextManager->updateContext(contextId); + TURN_ON_CUSTOM_MALLOC; + } } void SLAMP_exit_loop(uint32_t bbId) { - TURN_OFF_CUSTOM_MALLOC; - auto contextId = SpecPrivLib::ContextId(SpecPrivLib::LoopContext, bbId); + if (POINTS_TO_MODULE) { + TURN_OFF_CUSTOM_MALLOC; + auto contextId = SpecPrivLib::ContextId(SpecPrivLib::LoopContext, bbId); - // std::cerr << "exiting function " << fcnId << std::endl; - // update the current context - contextManager->popContext(contextId); - TURN_ON_CUSTOM_MALLOC; + // std::cerr << "exiting function " << fcnId << std::endl; + // update the current context + contextManager->popContext(contextId); + TURN_ON_CUSTOM_MALLOC; + } } /// Keep track of the context of the function void SLAMP_exit_fcn(uint32_t fcnId) { - TURN_OFF_CUSTOM_MALLOC; - auto contextId = SpecPrivLib::ContextId(SpecPrivLib::FunctionContext, fcnId); + if (POINTS_TO_MODULE) { + TURN_OFF_CUSTOM_MALLOC; + auto contextId = SpecPrivLib::ContextId(SpecPrivLib::FunctionContext, fcnId); - // std::cerr << "exiting function " << fcnId << std::endl; - // update the current context - contextManager->popContext(contextId); - TURN_ON_CUSTOM_MALLOC; + // std::cerr << "exiting function " << fcnId << std::endl; + // update the current context + contextManager->popContext(contextId); + TURN_ON_CUSTOM_MALLOC; + } } void SLAMP_loop_iter_ctx(uint32_t id) {} /// update the invocation count void SLAMP_loop_invocation() { + TURN_OFF_CUSTOM_MALLOC; // fprintf(stderr, "SLAMP_loop_invocation, depth: %u\n", invokedepth); invokedepth++; + if (POINTS_TO_MODULE) + targetLoopContexts->insert(contextManager->encodeActiveContext()); + if (invokedepth > 1) return; @@ -1179,6 +1221,8 @@ void SLAMP_loop_invocation() { if (__slamp_begin_trace) std::cout << "[invoke] " << (__slamp_invocation) << "\n" << std::flush; #endif + + TURN_ON_CUSTOM_MALLOC; } void SLAMP_loop_iteration() @@ -1215,6 +1259,8 @@ bool SLAMP_isBadAlloc(uint64_t addr) ATTRIBUTE(always_inline) { } void SLAMP_report_base_pointer_arg(uint32_t fcnId, uint32_t argId, void *ptr){ + if (invokedepth == 0) + return; if (SLAMP_isBadAlloc((uint64_t)ptr)) { return; } @@ -1222,6 +1268,8 @@ void SLAMP_report_base_pointer_arg(uint32_t fcnId, uint32_t argId, void *ptr){ } void SLAMP_report_base_pointer_inst(uint32_t instId, void *ptr){ + if (invokedepth == 0) + return; if (SLAMP_isBadAlloc((uint64_t)ptr)) { return; } diff --git a/tests/scripts/convertPromptToSpecPriv.py b/tests/scripts/convertPromptToSpecPriv.py index 72649185..91ef8208 100644 --- a/tests/scripts/convertPromptToSpecPriv.py +++ b/tests/scripts/convertPromptToSpecPriv.py @@ -109,6 +109,30 @@ def generateContextStr(contexts): uo_ids = id_maps['uo'] func_ids = id_maps['func'] loop_ids = id_maps['loop'] + loop_context_name = "" + + # Load loop contexts + # LOOP CONTEXTS: 1 + # (1,10)(0,0) + curLine = 0 + + while curLine < len(lines): + line = lines[curLine] + if "LOOP CONTEXTS" in line: + # Get the number of loop contexts + num_loop_contexts = int(line.replace("LOOP CONTEXTS: ", "").strip()) + # assert num_loop_contexts is an integer + if num_loop_contexts == 0 or num_loop_contexts > 1: + print("ERROR: num_loop_contexts is not 1") + exit(1) + + curLine += 1 + line = lines[curLine] + context_str = generateContextStr(line) + loop_context_name = context_str + + curLine += 1 + # Parse LOCAL OBJECT 279 at context (2,39)(2,35)(2,20)(1,10)(0,0); # Get malloc ID @@ -125,34 +149,52 @@ def generateContextStr(contexts): malloc_name = malloc_ids[mid] context_str = generateContextStr(contexts) - final_lines.append("LOCAL OBJECT AU HEAP " + malloc_name + " FROM CONTEXT { " + context_str + "} IS LOCAL TO CONTEXT { } COUNT 1") + final_lines.append("LOCAL OBJECT AU HEAP " + malloc_name + " FROM CONTEXT { " + context_str + "} IS LOCAL TO CONTEXT { " + loop_context_name + " } COUNT 1") - # Parse PRED OBJ 282: AU 24 FROM CONTEXT (1,0)(1,10)(0,0); + # Parse PRED OBJ 282: length of AUs + # AU 24 FROM CONTEXT (1,0)(1,10)(0,0); # The first number is UO ID, the second number is malloc ID - for line in lines: + curLine = 0 + while curLine < len(lines): + line = lines[curLine] + curLine += 1 if "PRED OBJ" in line: - line = line.replace("PRED OBJ", "").replace("AU ", "").replace("FROM CONTEXT", "").replace(":", "").replace(";", "").strip() - if "NULL" in line: - uoid, _ = line.split(" ", 1) - # assert uoid is an integer - assert uoid.isdigit() - # Get the name of the malloc inst - uo_name = uo_ids[uoid] - - final_lines.append("PRED OBJ " + uo_name + " OVER 1 VALUES { ( OFFSET 0 BASE AU NULL COUNT 1 ) }") - else: - uoid, malloc_id, contexts = line.split(" ", 2) + line = line.replace("PRED OBJ", "").replace("at ", "").replace(":", "").replace(";", "").strip() + uoid, contexts, lenAUs = line.split(" ", 2) + lenAUs = int(lenAUs) - # assert uoid is an integer - assert uoid.isdigit() - assert malloc_id.isdigit() + context_str = generateContextStr(contexts) + uo_name = uo_ids[uoid] + uo_str = "PRED OBJ " + uo_name + " AT CONTEXT { " + context_str + " } AS PREDICTABLE " + str(lenAUs) + " SAMPLES OVER " + str(lenAUs) + " VALUES { " - # Get the name of the malloc inst - uo_name = uo_ids[uoid] - malloc_name = malloc_ids[malloc_id] - context_str = generateContextStr(contexts) + # Parse the AU + for i in range(lenAUs): + line = lines[curLine] + line = line.replace("AU ", "").replace("FROM CONTEXT", "").replace(":", "").replace(";", "").strip() - final_lines.append("PRED OBJ " + uo_name + " OVER 1 VALUES { ( OFFSET 0 BASE AU HEAP " + malloc_id + " FROM CONTEXT { " + context_str + "} COUNT 1 }") + + if "NULL" in line: + # Get the name of the malloc inst + au_str = "( OFFSET 0 BASE AU NULL COUNT 1 )" + elif "UNMANAGED" in line: + au_str = "( OFFSET 0 BASE AU HEAP UNMANAGED fopen FROM CONTEXT { TOP } COUNT 1 )" + else: + malloc_id, contexts = line.split(" ", 1) + assert malloc_id.isdigit() + + # Get the name of the malloc inst + malloc_name = malloc_ids[malloc_id] + context_str = generateContextStr(contexts) + au_str = "( OFFSET 0 BASE AU HEAP " + malloc_name + " FROM CONTEXT { " + context_str + "} COUNT 1 )" + + uo_str += au_str + if i != lenAUs - 1: + uo_str += " , " + + curLine += 1 + + uo_str += " }" + final_lines.append(uo_str) return final_lines @@ -162,6 +204,9 @@ def parse_args(): parser = argparse.ArgumentParser(description='Parse the rabbit6 file and the specpriv-profile.out file') parser.add_argument('-r', '--rabbit6', help='Rabbit6 file', default="rabbit6", type=str) parser.add_argument('-p', '--profile', help='specpriv-profile.out file', default="specpriv-profile.out", type=str) + parser.add_argument('-o', '--output', help='Output file', default="specpriv-profile-converted.out", type=str) + return parser.parse_args() + return parser.parse_args() @@ -177,7 +222,12 @@ def parse_args(): # Parse the specpriv-profile.out file final_lines = parse_specpriv(profile, id_maps) - # print all lines - for line in final_lines: - print(line + ";") + # print all lines to out file + with open(args.output, 'w') as f: + f.write("BEGIN SPEC PRIV PROFILE\n") + f.write("COMPLETE ALLOCATION INFO ;\n") + + for line in final_lines: + f.write(line + " ;\n") + f.write("END SPEC PRIV PROFILE\n"); diff --git a/tests/slamp-test/specpriv_counter1/src/Makefile b/tests/slamp-test/specpriv_counter1/src/Makefile new file mode 100644 index 00000000..2b0747f7 --- /dev/null +++ b/tests/slamp-test/specpriv_counter1/src/Makefile @@ -0,0 +1,5 @@ +PROFILESETUP= +PROFILEARGS=10000 100 20 + +#NOINLINE=1 +include ../../../Makefile.generic diff --git a/tests/slamp-test/specpriv_counter1/src/test.c b/tests/slamp-test/specpriv_counter1/src/test.c new file mode 100644 index 00000000..77d70a47 --- /dev/null +++ b/tests/slamp-test/specpriv_counter1/src/test.c @@ -0,0 +1,20 @@ +#include +#include + +#define N 1000000 +int main() { + // have a loop that malloc in one iteration and free in the next + int *p; + for (int i = 0; i < N; i++) { + if (i == 0) { + p = (int *) malloc(sizeof(int)); + *p = 0; + } + else if (i == N - 1) { + free(p); + } else { + *p = *p + 1; + } + } + +} From 5272eac6670dc01e2730a8d7f5f3c1898b024d3f Mon Sep 17 00:00:00 2001 From: Ziyang Xu Date: Tue, 4 Oct 2022 18:41:29 -0400 Subject: [PATCH 10/97] add different queue implementation for SLAMP --- liberty/lib/SLAMP/CMakeLists.txt | 3 + liberty/lib/SLAMP/SLAMP.cpp | 2 +- liberty/lib/SLAMP/SLAMPatomicq/BoostSend.cpp | 381 ++++++++++++++++ liberty/lib/SLAMP/SLAMPatomicq/CMakeLists.txt | 34 ++ liberty/lib/SLAMP/SLAMPatomicq/slamp_hooks.h | 371 ++++++++++++++++ liberty/lib/SLAMP/SLAMPboost/BoostSend.cpp | 413 ++++++++++++++++++ liberty/lib/SLAMP/SLAMPboost/CMakeLists.txt | 34 ++ liberty/lib/SLAMP/SLAMPboost/slamp_hooks.h | 371 ++++++++++++++++ liberty/lib/SLAMP/SLAMPnng/CMakeLists.txt | 32 ++ liberty/lib/SLAMP/SLAMPnng/NngSend.cpp | 393 +++++++++++++++++ liberty/lib/SLAMP/SLAMPnng/slamp_hooks.h | 371 ++++++++++++++++ 11 files changed, 2404 insertions(+), 1 deletion(-) create mode 100644 liberty/lib/SLAMP/SLAMPatomicq/BoostSend.cpp create mode 100644 liberty/lib/SLAMP/SLAMPatomicq/CMakeLists.txt create mode 100644 liberty/lib/SLAMP/SLAMPatomicq/slamp_hooks.h create mode 100644 liberty/lib/SLAMP/SLAMPboost/BoostSend.cpp create mode 100644 liberty/lib/SLAMP/SLAMPboost/CMakeLists.txt create mode 100644 liberty/lib/SLAMP/SLAMPboost/slamp_hooks.h create mode 100644 liberty/lib/SLAMP/SLAMPnng/CMakeLists.txt create mode 100644 liberty/lib/SLAMP/SLAMPnng/NngSend.cpp create mode 100644 liberty/lib/SLAMP/SLAMPnng/slamp_hooks.h diff --git a/liberty/lib/SLAMP/CMakeLists.txt b/liberty/lib/SLAMP/CMakeLists.txt index 4bcfe78d..3c6bfeb2 100644 --- a/liberty/lib/SLAMP/CMakeLists.txt +++ b/liberty/lib/SLAMP/CMakeLists.txt @@ -19,3 +19,6 @@ include_directories(./) add_llvm_library(${PassName} SHARED ${SRCS}) # This is to generate libxxx.so add_subdirectory(SLAMPlib/hooks) +add_subdirectory(SLAMPnng) +add_subdirectory(SLAMPboost) +# add_subdirectory(SLAMPatomicq) diff --git a/liberty/lib/SLAMP/SLAMP.cpp b/liberty/lib/SLAMP/SLAMP.cpp index 997c4737..39959975 100644 --- a/liberty/lib/SLAMP/SLAMP.cpp +++ b/liberty/lib/SLAMP/SLAMP.cpp @@ -432,7 +432,7 @@ bool SLAMP::runOnModule(Module &m) { errs() << "Elided Hash: " << elidedHash(elidedLoopInstsId) << "\n"; // replace external function calls to wrapper function calls - replaceExternalFunctionCalls(m); + // replaceExternalFunctionCalls(m); auto setGlobalModule = [&m](string name, bool value) { diff --git a/liberty/lib/SLAMP/SLAMPatomicq/BoostSend.cpp b/liberty/lib/SLAMP/SLAMPatomicq/BoostSend.cpp new file mode 100644 index 00000000..4e8f48c9 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPatomicq/BoostSend.cpp @@ -0,0 +1,381 @@ +#include "slamp_hooks.h" +#include // ring buffer + +#include +#include +#include +#include + +#include "atomic_queue/atomic_queue.h" + +namespace bip = boost::interprocess; +namespace shm +{ + typedef bip::allocator char_alloc; + typedef bip::basic_string, char_alloc > shared_string; + + typedef boost::lockfree::spsc_queue< + shared_string, + boost::lockfree::capacity<65536> + > ring_buffer; + +} + +#include + +using Element = char; // Queue element type. +Element constexpr NIL = static_cast(-1); // Atomic elements require a special value that cannot be pushed/popped. +using Queue = atomic_queue::AtomicQueueB; // Use heap-allocated buffer. + +// create segment and corresponding allocator +bip::managed_shared_memory *segment; +shm::char_alloc *char_alloc; + +// Ringbuffer fully constructed in shared memory. The element strings are +// also allocated from the same shared memory segment. This vector can be +// safely accessed from other processes. +shm::ring_buffer *queue; +Queue *a_queue; +unsigned long counter4 = 0; +unsigned long counter8 = 0; + +void SLAMP_init(uint32_t fn_id, uint32_t loop_id) { + segment = new bip::managed_shared_memory(bip::open_or_create, "MySharedMemory", 65536UL*1600); + char_alloc = new shm::char_alloc(segment->get_segment_manager()); + // queue = segment->find_or_construct("queue")(); + a_queue = new atomic_queue::AtomicQueueB(65536); + + + // send a msg with "fn_id, loop_id" + char msg[100]; + sprintf(msg, "%d,%d", fn_id, loop_id); + a_queue->push(msg); + queue->push(shm::shared_string(msg, *char_alloc)); +} + +void SLAMP_fini(const char* filename){ + // send a msg with "fini" + // queue->push(shm::shared_string("fini", *char_alloc)); + std::cout << counter4 << " " << counter8 << std::endl; +} + +void SLAMP_allocated(uint64_t addr){} +void SLAMP_init_global_vars(const char *name, uint64_t addr, size_t size){} +void SLAMP_main_entry(uint32_t argc, char** argv, char** env){} + +void SLAMP_enter_fcn(uint32_t id){} +void SLAMP_exit_fcn(uint32_t id){} +void SLAMP_enter_loop(uint32_t id){} +void SLAMP_exit_loop(uint32_t id){} +void SLAMP_loop_iter_ctx(uint32_t id){} +void SLAMP_loop_invocation(){} +void SLAMP_loop_iteration(){} +void SLAMP_loop_exit(){} + +void SLAMP_report_base_pointer_arg(uint32_t, uint32_t, void *ptr){} +void SLAMP_report_base_pointer_inst(uint32_t, void *ptr){} +void SLAMP_callback_stack_alloca(uint64_t, uint64_t, uint32_t, uint64_t){} +void SLAMP_callback_stack_free(){} + +void SLAMP_ext_push(const uint32_t instr){} +void SLAMP_ext_pop(){} + +void SLAMP_push(const uint32_t instr){} +void SLAMP_pop(){} + +void SLAMP_load1(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value){ +} +void SLAMP_load2(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value){ +} +void SLAMP_load4(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + counter4++; + // char msg[] = "load4"; + // queue->push(shm::shared_string(msg, *char_alloc)); +} + +void SLAMP_load8(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + counter8++; + char msg[1] = {8}; + // // sprintf(msg, "load8,%d,%lu,%d,%lu", instr, addr, bare_instr, value); + queue->push(shm::shared_string(msg, *char_alloc)); +} + +void SLAMP_loadn(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, size_t n){} + +void SLAMP_load1_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value){} +void SLAMP_load2_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value){} +void SLAMP_load4_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value){} +void SLAMP_load8_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value){} +void SLAMP_loadn_ext(const uint64_t addr, const uint32_t bare_instr, size_t n){} + +void SLAMP_store1(uint32_t instr, const uint64_t addr){} +void SLAMP_store2(uint32_t instr, const uint64_t addr){} +void SLAMP_store4(uint32_t instr, const uint64_t addr){} +void SLAMP_store8(uint32_t instr, const uint64_t addr){} +void SLAMP_storen(uint32_t instr, const uint64_t addr, size_t n){} + +void SLAMP_store1_ext(const uint64_t addr, const uint32_t bare_inst){} +void SLAMP_store2_ext(const uint64_t addr, const uint32_t bare_inst){} +void SLAMP_store4_ext(const uint64_t addr, const uint32_t bare_inst){} +void SLAMP_store8_ext(const uint64_t addr, const uint32_t bare_inst){} +void SLAMP_storen_ext(const uint64_t addr, const uint32_t bare_inst, size_t n){} + +/* wrappers */ +static void* SLAMP_malloc_hook(size_t size, const void *caller){} +static void SLAMP_free_hook(void *ptr, const void *caller){} +static void* SLAMP_memalign_hook(size_t alignment, size_t size, const void *caller){} +void* SLAMP_malloc(size_t size, uint32_t instr, size_t alignment){} + +void* SLAMP_calloc(size_t nelem, size_t elsize){} +void* SLAMP_realloc(void* ptr, size_t size){} +void* SLAMP__Znam(size_t size){} +void* SLAMP__Znwm(size_t size){} + + + +char* SLAMP_strdup(const char *s1){} +char* SLAMP___strdup(const char *s1){} +void SLAMP_free(void* ptr){} +void SLAMP_cfree(void* ptr){} +void SLAMP__ZdlPv(void* ptr){} +void SLAMP__ZdaPv(void* ptr){} +int SLAMP_brk(void *end_data_segment){} +void* SLAMP_sbrk(intptr_t increment){} + +/* llvm memory intrinsics */ +void SLAMP_llvm_memcpy_p0i8_p0i8_i32(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint32_t sizeBytes){} +void SLAMP_llvm_memcpy_p0i8_p0i8_i64(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint64_t sizeBytes){} +void SLAMP_llvm_memmove_p0i8_p0i8_i32(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint32_t sizeBytes){} +void SLAMP_llvm_memmove_p0i8_p0i8_i64(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint64_t sizeBytes){} +void SLAMP_llvm_memset_p0i8_i32(const uint8_t* dstAddr, const uint32_t len){} +void SLAMP_llvm_memset_p0i8_i64(const uint8_t* dstAddr, const uint64_t len){} + +// void SLAMP_llvm_lifetime_start_p0i8(uint64_t size, uint8_t* ptr){} +// void SLAMP_llvm_lifetime_end_p0i8(uint64_t size, uint8_t* ptr){} + +/* String functions */ +size_t SLAMP_strlen(const char *str){} +char* SLAMP_strchr(char *s, int c){} +char* SLAMP_strrchr(char *s, int c){} +int SLAMP_strcmp(const char *s1, const char *s2){} +int SLAMP_strncmp(const char *s1, const char *s2, size_t n){} +char* SLAMP_strcpy(char *dest, const char *src){} +char* SLAMP_strncpy(char *dest, const char *src, size_t n){} +char* SLAMP_strcat(char *s1, const char *s2){} +char* SLAMP_strncat(char *s1, const char *s2, size_t n){} +char* SLAMP_strstr(char *s1, char *s2){} +size_t SLAMP_strspn(const char *s1, const char *s2){} +size_t SLAMP_strcspn(const char *s1, const char *s2){} +char* SLAMP_strtok(char *s, const char *delim){} +double SLAMP_strtod(const char *nptr, char **endptr){} +long int SLAMP_strtol(const char *nptr, char **endptr, int base){} +char* SLAMP_strpbrk(char *s1, char *s2){} + +/* Mem* and b* functions */ +void *SLAMP_memset (void *dest, int c, size_t n){} +void *SLAMP_memcpy (void *dest, const void *src, size_t n){} +void *SLAMP___builtin_memcpy (void *dest, const void *src, size_t n){} +void *SLAMP_memmove (void *dest, const void *src, size_t n){} +int SLAMP_memcmp(const void *s1, const void *s2, size_t n){} +void* SLAMP_memchr(void* ptr, int value, size_t num){} +void* SLAMP___rawmemchr(void* ptr, int value){} + +void SLAMP_bzero(void *s, size_t n){} +void SLAMP_bcopy(const void *s1, void *s2, size_t n){} + +/* IO */ +ssize_t SLAMP_read(int fd, void *buf, size_t count){} +int SLAMP_open(const char *pathname, int flags, mode_t mode){} +int SLAMP_close(int fd){} +ssize_t SLAMP_write(int fd, const void *buf, size_t count){} +off_t SLAMP_lseek(int fildes, off_t offset, int whence){} + +FILE * SLAMP_fopen(const char *path, const char *mode){} +FILE * SLAMP_fopen64(const char *path, const char *mode){} +FILE * SLAMP_freopen(const char *path, const char *mode, FILE* stream){} +int SLAMP_fflush(FILE *stream){} +int SLAMP_fclose(FILE *stream){} +int SLAMP_ferror(FILE *stream){} +int SLAMP_feof(FILE *stream){} +long SLAMP_ftell(FILE *stream){} +size_t SLAMP_fread(void * ptr, size_t size, size_t nitems, FILE *stream){} +size_t SLAMP_fwrite(const void *ptr, size_t size, size_t nitems, FILE *stream){} +int SLAMP_fseek(FILE *stream, long offset, int whence){} +void SLAMP_rewind(FILE *stream){} + +int SLAMP_fgetc(FILE *stream){} +int SLAMP_fputc(int c, FILE *stream){} +char * SLAMP_fgets(char *s, int n, FILE *stream){} +int SLAMP_fputs(const char *s, FILE *stream){} + +int SLAMP_ungetc(int c, FILE *stream){} +int SLAMP_putchar(int c){} +int SLAMP_getchar(void){} + +int SLAMP_fileno(FILE *stream){} +char * SLAMP_gets(char *s){} +int SLAMP_puts(const char *s){} + +int SLAMP_select(int n, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, struct timeval *timeout){} +int SLAMP_remove(const char *path){} + +void SLAMP_setbuf(FILE * stream, char * buf){} +void SLAMP_setvbuf(FILE * stream, char * buf, int mode, size_t size){} +char * SLAMP_tmpnam(char *s){} +FILE* SLAMP_tmpfile(void){} +char * SLAMP_ttyname(int fildes){} + +FILE * SLAMP_fdopen(int fildes, const char *mode){} +void SLAMP_clearerr(FILE *stream){} + +int SLAMP_truncate(const char *path, off_t length){} +int SLAMP_ftruncate(int fildes, off_t length){} + +int SLAMP_dup(int oldfd){} +int SLAMP_dup2(int oldfd, int newfd){} +int SLAMP_pipe(int filedes[2]){} + +int SLAMP_chmod(const char *path, mode_t mode){} +int SLAMP_fchmod(int fildes, mode_t mode){} +int SLAMP_fchown(int fd, uid_t owner, gid_t group){} +int SLAMP_access(const char *pathname, int mode){} +long SLAMP_pathconf(char *path, int name){} +int SLAMP_mkdir(const char *pathname, mode_t mode){} +int SLAMP_rmdir(const char *pathname){} +mode_t SLAMP_umask(mode_t mask){} +int SLAMP_fcntl(int fd, int cmd, struct flock *lock){} + +DIR* SLAMP_opendir(const char* name){} +struct dirent* SLAMP_readdir(DIR *dirp){} +struct dirent64* SLAMP_readdir64(DIR *dirp){} +int SLAMP_closedir(DIR* dirp){} + +/* Printf */ +int SLAMP_printf(const char *format, ...){} +int SLAMP_fprintf(FILE *stream, const char *format, ...){} +int SLAMP_sprintf(char *str, const char *format, ...){} +int SLAMP_snprintf(char *str, size_t size, const char *format, ...){} + +int SLAMP_vprintf(const char *format, va_list ap){} +int SLAMP_vfprintf(FILE *stream, const char *format, va_list ap){} +int SLAMP_vsprintf(char *str, const char *format, va_list ap){} +int SLAMP_vsnprintf(char *str, size_t size, const char *format, va_list ap){} + +/* Scanf */ +int SLAMP_fscanf(FILE *stream, const char *format, ... ){} +int SLAMP_scanf(const char *format, ... ){} +int SLAMP_sscanf(const char *s, const char *format, ... ){} +int SLAMP___isoc99_sscanf(const char *s, const char *format, ... ){} + +int SLAMP_vfscanf(FILE *stream, const char *format, va_list ap){} +int SLAMP_vscanf(const char *format, va_list ap){} +int SLAMP_vsscanf(const char *s, const char *format, va_list ap){} + +/* Time */ +time_t SLAMP_time(time_t *t){} +struct tm *SLAMP_localtime(const time_t *timer){} +struct lconv* SLAMP_localeconv(){} +struct tm *SLAMP_gmtime(const time_t *timer){} +int SLAMP_gettimeofday(struct timeval *tv, struct timezone *tz){} + +/* Math */ +double SLAMP_ldexp(double x, int exp){} +float SLAMP_ldexpf(float x, int exp){} +long double SLAMP_ldexpl(long double x, int exp){} +double SLAMP_log10(double x){} +float SLAMP_log10f(float x){} +long double SLAMP_log10l(long double x){} +double SLAMP_log(double x){} +float SLAMP_logf(float x){} +long double SLAMP_logl(long double x){} + +double SLAMP_exp(double x){} +float SLAMP_expf(float x){} +long double SLAMP_expl(long double x){} + +double SLAMP_cos(double x){} +float SLAMP_cosf(float x){} +long double SLAMP_cosl(long double x){} +double SLAMP_sin(double x){} +double SLAMP_tan(double x){} +float SLAMP_sinf(float x){} +long double SLAMP_sinl(long double x){} + +double SLAMP_atan(double x){} +float SLAMP_atanf(float x){} +long double SLAMP_atanl(long double x){} + +double SLAMP_floor(double x){} +float SLAMP_floorf(float x){} +long double SLAMP_floorl(long double x){} +double SLAMP_ceil(double x){} +float SLAMP_ceilf(float x){} +long double SLAMP_ceill(long double x){} + +double SLAMP_atan2(double y, double x){} +float SLAMP_atan2f(float y, float x){} +long double SLAMP_atan2l(long double y, long double x){} + +double SLAMP_sqrt(double x){} +float SLAMP_sqrtf(float x){} +long double SLAMP_sqrtl(long double x){} + +double SLAMP_pow(double x, double y){} +float SLAMP_powf(float x, float y){} +long double SLAMP_powl(long double x, long double y){} + +double SLAMP_fabs(double x){} +float SLAMP_fabsf(float x){} +long double SLAMP_fabsl(long double x){} + +double SLAMP_modf(double x, double *iptr){} +float SLAMP_modff(float x, float *iptr){} +long double SLAMP_modfl(long double x, long double *iptr){} + +double SLAMP_fmod(double x, double y){} + +double SLAMP_frexp(double num, int *exp){} +float SLAMP_frexpf(float num, int *exp){} +long double SLAMP_frexpl(long double num, int *exp){} + +int SLAMP_isnan(){} + +/* MISC */ +char *SLAMP_getenv(const char *name){} +int SLAMP_putenv(char* string){} +char *SLAMP_getcwd(char *buf, size_t size){} +char* SLAMP_strerror(int errnum){} +void SLAMP_exit(int status){} +void SLAMP__exit(int status){} +int SLAMP_link(const char *oldpath, const char *newpath){} +int SLAMP_unlink(const char *pathname){} +int SLAMP_isatty(int desc){} +int SLAMP_setuid(uid_t uid){} +uid_t SLAMP_getuid(void){} +uid_t SLAMP_geteuid(void){} +int SLAMP_setgid(gid_t gid){} +gid_t SLAMP_getgid(void){} +gid_t SLAMP_getegid(void){} +pid_t SLAMP_getpid(void){} +int SLAMP_chdir(const char *path){} +int SLAMP_execl(const char *path, const char *arg0, ... /*, (char *)0 */){} +int SLAMP_execv(const char *path, char *const argv[]){} +int SLAMP_execvp(const char *file, char *const argv[]){} +int SLAMP_kill(pid_t pid, int sig){} +pid_t SLAMP_fork(void){} +sighandler_t SLAMP___sysv_signal(int signum, sighandler_t handler){} +pid_t SLAMP_waitpid(pid_t pid, int* status, int options){} +void SLAMP_qsort(void* base, size_t nmemb, size_t size, int(*compar)(const void *, const void *)){} +int SLAMP_ioctl(int d, int request, ...){} +unsigned int SLAMP_sleep(unsigned int seconds){} +char* SLAMP_gcvt(double number, size_t ndigit, char* buf){} +char* SLAMP_nl_langinfo(nl_item item){} + +/* Compiler/Glibc Internals */ +void SLAMP___assert_fail(const char * assertion, const char * file, unsigned int line, const char * function){} +const unsigned short int **SLAMP___ctype_b_loc(void){} +int SLAMP__IO_getc(_IO_FILE * __fp){} +int SLAMP__IO_putc(int __c, _IO_FILE *__fp){} + +int SLAMP___fxstat (int __ver, int __fildes, struct stat *__stat_buf){} +int SLAMP___xstat (int __ver, __const char *__filename, struct stat *__stat_buf){} diff --git a/liberty/lib/SLAMP/SLAMPatomicq/CMakeLists.txt b/liberty/lib/SLAMP/SLAMPatomicq/CMakeLists.txt new file mode 100644 index 00000000..69ff2d42 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPatomicq/CMakeLists.txt @@ -0,0 +1,34 @@ +file(GLOB SRCS + "*.cpp" +) +set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} "/u/ziyangx/test/boost/boost_1_80_0/install/lib/cmake") + +# add CMAKE_PREFIX_PATH +set(CMAKE_C_COMPILER "clang") +set(CMAKE_CXX_COMPILER "clang++") + +# Compilation flags +# set_source_files_properties(${SRCS} PROPERTIES COMPILE_FLAGS "-Wl,-save-temps -std=c++17 -Wno-inline -O3 -fexceptions")# -emit-llvm") +set_source_files_properties(${SRCS} PROPERTIES COMPILE_FLAGS "-flto -std=c++17 -Wno-inline -O3 -fexceptions")# -emit-llvm") +set(PassName "slamp_hooks_atomic_queue") + +set(THREADS_PREFER_PTHREAD_FLAG ON) +find_package(Boost 1.80.0 REQUIRED COMPONENTS system) +include_directories(${Boost_INCLUDE_DIRS}) + + +# list(APPEND CMAKE_MODULE_PATH "${LLVM_CMAKE_DIR}") +#include(HandleLLVMOptions) +# include(AddLLVM) + + +include_directories(./ + /u/ziyangx/test/atomic_queue/atomic_queue/include) + +add_library(${PassName} STATIC ${SRCS}) +target_link_libraries(${PassName} ${Boost_LIBRARIES}) +# target_link_libraries(${PassName} nng::nng) +# add_llvm_library(${PassName}_shared SHARED ${SRCS}) +# set_target_properties(${PassName}_shared PROPERTIES OUTPUT_NAME ${PassName}) +# set_property(TARGET ${PassName} PROPERTY INTERPROCEDURAL_OPTIMIZATION TRUE) +#add_llvm_library(${PassName} SHARED ${SRCS}) # This is to generate libxxx.so diff --git a/liberty/lib/SLAMP/SLAMPatomicq/slamp_hooks.h b/liberty/lib/SLAMP/SLAMPatomicq/slamp_hooks.h new file mode 100644 index 00000000..df546117 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPatomicq/slamp_hooks.h @@ -0,0 +1,371 @@ +#ifndef SLAMPLIB_HOOKS_SLAMP_HOOKS_H +#define SLAMPLIB_HOOKS_SLAMP_HOOKS_H + +// FIXME: inline tweak actually make things worse in sequential and better with +// 16x, so turn it on at all time before understanding why +#define ATTRIBUTE(x) __attribute__((x)) +// #ifdef ITO_ENABLE +// // #define ATTRIBUTE(x) +// #define ATTRIBUTE(x) __attribute__((x)) +// #else +// #define ATTRIBUTE(x) +// #endif + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +void SLAMP_dbggv(int id); +void SLAMP_dbggvstr(char* str); + +// SLAMP measure functions +void SLAMP_measure_init(); +void SLAMP_measure_fini(); +void SLAMP_measure_load(uint32_t id, uint64_t size); +void SLAMP_measure_store(uint32_t id, uint64_t size); +static void* SLAMP_measure_malloc_hook(size_t size, const void *caller); +static void SLAMP_measure_free_hook(void *ptr, const void *caller); + +void SLAMP_init(uint32_t fn_id, uint32_t loop_id); +void SLAMP_fini(const char* filename); + +void SLAMP_allocated(uint64_t addr); +void SLAMP_init_global_vars(const char *name, uint64_t addr, size_t size); +void SLAMP_main_entry(uint32_t argc, char** argv, char** env); + +void SLAMP_enter_fcn(uint32_t id); +void SLAMP_exit_fcn(uint32_t id); +void SLAMP_enter_loop(uint32_t id); +void SLAMP_exit_loop(uint32_t id); +void SLAMP_loop_iter_ctx(uint32_t id); +void SLAMP_loop_invocation(); +void SLAMP_loop_iteration(); +void SLAMP_loop_exit(); + +void SLAMP_report_base_pointer_arg(uint32_t, uint32_t, void *ptr); +void SLAMP_report_base_pointer_inst(uint32_t, void *ptr); +void SLAMP_callback_stack_alloca(uint64_t, uint64_t, uint32_t, uint64_t); +void SLAMP_callback_stack_free(void); + +void SLAMP_ext_push(const uint32_t instr); +void SLAMP_ext_pop(); + +void SLAMP_push(const uint32_t instr); +void SLAMP_pop(); + +void SLAMP_load1(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_load2(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_load4(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_load8(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_loadn(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, size_t n) ATTRIBUTE(always_inline);; + +void SLAMP_load1_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_load2_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_load4_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_load8_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_loadn_ext(const uint64_t addr, const uint32_t bare_instr, size_t n) ATTRIBUTE(always_inline);; + +void SLAMP_store1(uint32_t instr, const uint64_t addr) ATTRIBUTE(always_inline); +void SLAMP_store2(uint32_t instr, const uint64_t addr) ATTRIBUTE(always_inline); +void SLAMP_store4(uint32_t instr, const uint64_t addr) ATTRIBUTE(always_inline); +void SLAMP_store8(uint32_t instr, const uint64_t addr) ATTRIBUTE(always_inline); +void SLAMP_storen(uint32_t instr, const uint64_t addr, size_t n) ATTRIBUTE(always_inline);; + +void SLAMP_store1_ext(const uint64_t addr, const uint32_t bare_inst) ATTRIBUTE(always_inline); +void SLAMP_store2_ext(const uint64_t addr, const uint32_t bare_inst) ATTRIBUTE(always_inline); +void SLAMP_store4_ext(const uint64_t addr, const uint32_t bare_inst) ATTRIBUTE(always_inline); +void SLAMP_store8_ext(const uint64_t addr, const uint32_t bare_inst) ATTRIBUTE(always_inline); +void SLAMP_storen_ext(const uint64_t addr, const uint32_t bare_inst, size_t n) ATTRIBUTE(always_inline);; + +/* wrappers */ +static void* SLAMP_malloc_hook(size_t size, const void *caller); +static void SLAMP_free_hook(void *ptr, const void *caller); +static void* SLAMP_memalign_hook(size_t alignment, size_t size, const void *caller); +void* SLAMP_malloc(size_t size, uint32_t instr=0, size_t alignment=16); + +void* SLAMP_calloc(size_t nelem, size_t elsize); +void* SLAMP_realloc(void* ptr, size_t size); +void* SLAMP__Znam(size_t size); +void* SLAMP__Znwm(size_t size); + +char* SLAMP_strdup(const char *s1); +char* SLAMP___strdup(const char *s1); +void SLAMP_free(void* ptr); +void SLAMP_cfree(void* ptr); +void SLAMP__ZdlPv(void* ptr); +void SLAMP__ZdaPv(void* ptr); +int SLAMP_brk(void *end_data_segment); +void* SLAMP_sbrk(intptr_t increment); + +/* llvm memory intrinsics */ +void SLAMP_llvm_memcpy_p0i8_p0i8_i32(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint32_t sizeBytes); +void SLAMP_llvm_memcpy_p0i8_p0i8_i64(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint64_t sizeBytes); +void SLAMP_llvm_memmove_p0i8_p0i8_i32(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint32_t sizeBytes); +void SLAMP_llvm_memmove_p0i8_p0i8_i64(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint64_t sizeBytes); +void SLAMP_llvm_memset_p0i8_i32(const uint8_t* dstAddr, const uint32_t len); +void SLAMP_llvm_memset_p0i8_i64(const uint8_t* dstAddr, const uint64_t len); + +// void SLAMP_llvm_lifetime_start_p0i8(uint64_t size, uint8_t* ptr); +// void SLAMP_llvm_lifetime_end_p0i8(uint64_t size, uint8_t* ptr); + +/* String functions */ +size_t SLAMP_strlen(const char *str); +char* SLAMP_strchr(char *s, int c); +char* SLAMP_strrchr(char *s, int c); +int SLAMP_strcmp(const char *s1, const char *s2); +int SLAMP_strncmp(const char *s1, const char *s2, size_t n); +char* SLAMP_strcpy(char *dest, const char *src); +char* SLAMP_strncpy(char *dest, const char *src, size_t n); +char* SLAMP_strcat(char *s1, const char *s2); +char* SLAMP_strncat(char *s1, const char *s2, size_t n); +char* SLAMP_strstr(char *s1, char *s2); +size_t SLAMP_strspn(const char *s1, const char *s2); +size_t SLAMP_strcspn(const char *s1, const char *s2); +char* SLAMP_strtok(char *s, const char *delim); +double SLAMP_strtod(const char *nptr, char **endptr); +long int SLAMP_strtol(const char *nptr, char **endptr, int base); +char* SLAMP_strpbrk(char *s1, char *s2); + +/* Mem* and b* functions */ +void *SLAMP_memset (void *dest, int c, size_t n); +void *SLAMP_memcpy (void *dest, const void *src, size_t n); +void *SLAMP___builtin_memcpy (void *dest, const void *src, size_t n); +void *SLAMP_memmove (void *dest, const void *src, size_t n); +int SLAMP_memcmp(const void *s1, const void *s2, size_t n); +void* SLAMP_memchr(void* ptr, int value, size_t num); +void* SLAMP___rawmemchr(void* ptr, int value); + +void SLAMP_bzero(void *s, size_t n); +void SLAMP_bcopy(const void *s1, void *s2, size_t n); + +/* IO */ +ssize_t SLAMP_read(int fd, void *buf, size_t count); +int SLAMP_open(const char *pathname, int flags, mode_t mode); +int SLAMP_close(int fd); +ssize_t SLAMP_write(int fd, const void *buf, size_t count); +off_t SLAMP_lseek(int fildes, off_t offset, int whence); + +FILE * SLAMP_fopen(const char *path, const char *mode); +FILE * SLAMP_fopen64(const char *path, const char *mode); +FILE * SLAMP_freopen(const char *path, const char *mode, FILE* stream); +int SLAMP_fflush(FILE *stream); +int SLAMP_fclose(FILE *stream); +int SLAMP_ferror(FILE *stream); +int SLAMP_feof(FILE *stream); +long SLAMP_ftell(FILE *stream); +size_t SLAMP_fread(void * ptr, size_t size, size_t nitems, FILE *stream); +size_t SLAMP_fwrite(const void *ptr, size_t size, size_t nitems, FILE *stream); +int SLAMP_fseek(FILE *stream, long offset, int whence); +void SLAMP_rewind(FILE *stream); + +int SLAMP_fgetc(FILE *stream); +int SLAMP_fputc(int c, FILE *stream); +char * SLAMP_fgets(char *s, int n, FILE *stream); +int SLAMP_fputs(const char *s, FILE *stream); + +int SLAMP_ungetc(int c, FILE *stream); +int SLAMP_putchar(int c); +int SLAMP_getchar(void); + +int SLAMP_fileno(FILE *stream); +char * SLAMP_gets(char *s); +int SLAMP_puts(const char *s); + +int SLAMP_select(int n, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, struct timeval *timeout); +int SLAMP_remove(const char *path); + +void SLAMP_setbuf(FILE * stream, char * buf); +void SLAMP_setvbuf(FILE * stream, char * buf, int mode, size_t size); +char * SLAMP_tmpnam(char *s); +FILE* SLAMP_tmpfile(void); +char * SLAMP_ttyname(int fildes); + +FILE * SLAMP_fdopen(int fildes, const char *mode); +void SLAMP_clearerr(FILE *stream); + +int SLAMP_truncate(const char *path, off_t length); +int SLAMP_ftruncate(int fildes, off_t length); + +int SLAMP_dup(int oldfd); +int SLAMP_dup2(int oldfd, int newfd); +int SLAMP_pipe(int filedes[2]); + +int SLAMP_chmod(const char *path, mode_t mode); +int SLAMP_fchmod(int fildes, mode_t mode); +int SLAMP_fchown(int fd, uid_t owner, gid_t group); +int SLAMP_access(const char *pathname, int mode); +long SLAMP_pathconf(char *path, int name); +int SLAMP_mkdir(const char *pathname, mode_t mode); +int SLAMP_rmdir(const char *pathname); +mode_t SLAMP_umask(mode_t mask); +int SLAMP_fcntl(int fd, int cmd, struct flock *lock); + +DIR* SLAMP_opendir(const char* name); +struct dirent* SLAMP_readdir(DIR *dirp); +struct dirent64* SLAMP_readdir64(DIR *dirp); +int SLAMP_closedir(DIR* dirp); + +/* Printf */ +int SLAMP_printf(const char *format, ...); +int SLAMP_fprintf(FILE *stream, const char *format, ...); +int SLAMP_sprintf(char *str, const char *format, ...); +int SLAMP_snprintf(char *str, size_t size, const char *format, ...); + +int SLAMP_vprintf(const char *format, va_list ap); +int SLAMP_vfprintf(FILE *stream, const char *format, va_list ap); +int SLAMP_vsprintf(char *str, const char *format, va_list ap); +int SLAMP_vsnprintf(char *str, size_t size, const char *format, va_list ap); + +/* Scanf */ +int SLAMP_fscanf(FILE *stream, const char *format, ... ); +int SLAMP_scanf(const char *format, ... ); +int SLAMP_sscanf(const char *s, const char *format, ... ); +int SLAMP___isoc99_sscanf(const char *s, const char *format, ... ); + +int SLAMP_vfscanf(FILE *stream, const char *format, va_list ap); +int SLAMP_vscanf(const char *format, va_list ap); +int SLAMP_vsscanf(const char *s, const char *format, va_list ap); + +/* Time */ +time_t SLAMP_time(time_t *t); +struct tm *SLAMP_localtime(const time_t *timer); +struct lconv* SLAMP_localeconv(); +struct tm *SLAMP_gmtime(const time_t *timer); +int SLAMP_gettimeofday(struct timeval *tv, struct timezone *tz); + +/* Math */ +double SLAMP_ldexp(double x, int exp); +float SLAMP_ldexpf(float x, int exp); +long double SLAMP_ldexpl(long double x, int exp); +double SLAMP_log10(double x); +float SLAMP_log10f(float x); +long double SLAMP_log10l(long double x); +double SLAMP_log(double x); +float SLAMP_logf(float x); +long double SLAMP_logl(long double x); + +double SLAMP_exp(double x); +float SLAMP_expf(float x); +long double SLAMP_expl(long double x); + +double SLAMP_cos(double x); +float SLAMP_cosf(float x); +long double SLAMP_cosl(long double x); +double SLAMP_sin(double x); +double SLAMP_tan(double x); +float SLAMP_sinf(float x); +long double SLAMP_sinl(long double x); + +double SLAMP_atan(double x); +float SLAMP_atanf(float x); +long double SLAMP_atanl(long double x); + +double SLAMP_floor(double x); +float SLAMP_floorf(float x); +long double SLAMP_floorl(long double x); +double SLAMP_ceil(double x); +float SLAMP_ceilf(float x); +long double SLAMP_ceill(long double x); + +double SLAMP_atan2(double y, double x); +float SLAMP_atan2f(float y, float x); +long double SLAMP_atan2l(long double y, long double x); + +double SLAMP_sqrt(double x); +float SLAMP_sqrtf(float x); +long double SLAMP_sqrtl(long double x); + +double SLAMP_pow(double x, double y); +float SLAMP_powf(float x, float y); +long double SLAMP_powl(long double x, long double y); + +double SLAMP_fabs(double x); +float SLAMP_fabsf(float x); +long double SLAMP_fabsl(long double x); + +double SLAMP_modf(double x, double *iptr); +float SLAMP_modff(float x, float *iptr); +long double SLAMP_modfl(long double x, long double *iptr); + +double SLAMP_fmod(double x, double y); + +double SLAMP_frexp(double num, int *exp); +float SLAMP_frexpf(float num, int *exp); +long double SLAMP_frexpl(long double num, int *exp); + +int SLAMP_isnan(); + +/* MISC */ +char *SLAMP_getenv(const char *name); +int SLAMP_putenv(char* string); +char *SLAMP_getcwd(char *buf, size_t size); +char* SLAMP_strerror(int errnum); +void SLAMP_exit(int status); +void SLAMP__exit(int status); +int SLAMP_link(const char *oldpath, const char *newpath); +int SLAMP_unlink(const char *pathname); +int SLAMP_isatty(int desc); +int SLAMP_setuid(uid_t uid); +uid_t SLAMP_getuid(void); +uid_t SLAMP_geteuid(void); +int SLAMP_setgid(gid_t gid); +gid_t SLAMP_getgid(void); +gid_t SLAMP_getegid(void); +pid_t SLAMP_getpid(void); +int SLAMP_chdir(const char *path); +int SLAMP_execl(const char *path, const char *arg0, ... /*, (char *)0 */); +int SLAMP_execv(const char *path, char *const argv[]); +int SLAMP_execvp(const char *file, char *const argv[]); +int SLAMP_kill(pid_t pid, int sig); +pid_t SLAMP_fork(void); +sighandler_t SLAMP___sysv_signal(int signum, sighandler_t handler); +pid_t SLAMP_waitpid(pid_t pid, int* status, int options); +void SLAMP_qsort(void* base, size_t nmemb, size_t size, int(*compar)(const void *, const void *)); +int SLAMP_ioctl(int d, int request, ...); +unsigned int SLAMP_sleep(unsigned int seconds); +char* SLAMP_gcvt(double number, size_t ndigit, char* buf); +char* SLAMP_nl_langinfo(nl_item item); + +/* Compiler/Glibc Internals */ +void SLAMP___assert_fail(const char * assertion, const char * file, unsigned int line, const char * function); +const unsigned short int **SLAMP___ctype_b_loc(void); +int SLAMP__IO_getc(_IO_FILE * __fp); +int SLAMP__IO_putc(int __c, _IO_FILE *__fp); +int* SLAMP___errno_location (void); + +int SLAMP___fxstat (int __ver, int __fildes, struct stat *__stat_buf); +int SLAMP___xstat (int __ver, __const char *__filename, struct stat *__stat_buf); + +#ifdef __cplusplus +} +#endif + +#endif /* SLAMP_HOOKS_H */ diff --git a/liberty/lib/SLAMP/SLAMPboost/BoostSend.cpp b/liberty/lib/SLAMP/SLAMPboost/BoostSend.cpp new file mode 100644 index 00000000..451e224a --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPboost/BoostSend.cpp @@ -0,0 +1,413 @@ +#include "slamp_hooks.h" +#include // ring buffer + +#include +#include +#include +#include + +#define LOCAL_BUFFER_SIZE 256 + +namespace bip = boost::interprocess; +namespace shm +{ + typedef bip::allocator char_alloc; + typedef bip::basic_string, char_alloc > shared_string; + + typedef boost::lockfree::spsc_queue< + shared_string, + boost::lockfree::capacity<65536> + > ring_buffer; +} + +#include + +// create segment and corresponding allocator +bip::managed_shared_memory *segment; +shm::char_alloc *char_alloc; + +// Ringbuffer fully constructed in shared memory. The element strings are +// also allocated from the same shared memory segment. This vector can be +// safely accessed from other processes. +shm::ring_buffer *queue; +unsigned long counter4 = 0; +unsigned long counter8 = 0; +unsigned long counter4_ext = 0; +unsigned long counter8_ext = 0; +char local_buffer[LOCAL_BUFFER_SIZE]; +unsigned buffer_counter = 0; + +void SLAMP_init(uint32_t fn_id, uint32_t loop_id) { + segment = new bip::managed_shared_memory(bip::open_or_create, "MySharedMemory", 65536UL*1600); + char_alloc = new shm::char_alloc(segment->get_segment_manager()); + queue = segment->find_or_construct("queue")(); + + + // send a msg with "fn_id, loop_id" + char msg[100]; + sprintf(msg, "%d,%d", fn_id, loop_id); + queue->push(shm::shared_string(msg, *char_alloc)); +} + +void SLAMP_fini(const char* filename){ + // send a msg with "fini" + // queue->push(shm::shared_string("fini", *char_alloc)); + std::cout << counter4 << " " << counter8 << " " << counter4_ext << " " << counter8_ext << std::endl; +} + +void SLAMP_allocated(uint64_t addr){} +void SLAMP_init_global_vars(const char *name, uint64_t addr, size_t size){} +void SLAMP_main_entry(uint32_t argc, char** argv, char** env){} + +void SLAMP_enter_fcn(uint32_t id){} +void SLAMP_exit_fcn(uint32_t id){} +void SLAMP_enter_loop(uint32_t id){} +void SLAMP_exit_loop(uint32_t id){} +void SLAMP_loop_iter_ctx(uint32_t id){} +void SLAMP_loop_invocation(){} +void SLAMP_loop_iteration(){} +void SLAMP_loop_exit(){} + +void SLAMP_report_base_pointer_arg(uint32_t, uint32_t, void *ptr){} +void SLAMP_report_base_pointer_inst(uint32_t, void *ptr){} +void SLAMP_callback_stack_alloca(uint64_t, uint64_t, uint32_t, uint64_t){} +void SLAMP_callback_stack_free(){} + +void SLAMP_ext_push(const uint32_t instr){} +void SLAMP_ext_pop(){} + +void SLAMP_push(const uint32_t instr){} +void SLAMP_pop(){} + +void SLAMP_load1(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value){ +} +void SLAMP_load2(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value){ +} +void SLAMP_load4(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + counter4++; + + local_buffer[buffer_counter++] = 4; + local_buffer[buffer_counter++] = instr & 0xFF; + + // // sprintf(msg, "load8,%d,%lu,%d,%lu", instr, addr, bare_instr, value); + if (buffer_counter == LOCAL_BUFFER_SIZE) { + queue->push(shm::shared_string(local_buffer, LOCAL_BUFFER_SIZE, *char_alloc)); + buffer_counter = 0; + } +} + +void SLAMP_load8(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + counter8++; + + local_buffer[buffer_counter++] = 8; + local_buffer[buffer_counter++] = instr & 0xFF; + + // // sprintf(msg, "load8,%d,%lu,%d,%lu", instr, addr, bare_instr, value); + if (buffer_counter == LOCAL_BUFFER_SIZE) { + queue->push(shm::shared_string(local_buffer, LOCAL_BUFFER_SIZE, *char_alloc)); + buffer_counter = 0; + } +} + +void SLAMP_loadn(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, size_t n){} + +void SLAMP_load1_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value){} +void SLAMP_load2_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value){} +void SLAMP_load4_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + counter4_ext++; + + local_buffer[buffer_counter++] = 14; + local_buffer[buffer_counter++] = addr & 0xFF; + + // // sprintf(msg, "load8,%d,%lu,%d,%lu", instr, addr, bare_instr, value); + if (buffer_counter == LOCAL_BUFFER_SIZE) { + queue->push(shm::shared_string(local_buffer, LOCAL_BUFFER_SIZE, *char_alloc)); + buffer_counter = 0; + } + +} +void SLAMP_load8_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + counter8_ext++; + + local_buffer[buffer_counter++] = 18; + local_buffer[buffer_counter++] = addr & 0xFF; + + // // sprintf(msg, "load8,%d,%lu,%d,%lu", instr, addr, bare_instr, value); + if (buffer_counter == LOCAL_BUFFER_SIZE) { + queue->push(shm::shared_string(local_buffer, LOCAL_BUFFER_SIZE, *char_alloc)); + buffer_counter = 0; + } +} +void SLAMP_loadn_ext(const uint64_t addr, const uint32_t bare_instr, size_t n){} + +void SLAMP_store1(uint32_t instr, const uint64_t addr){} +void SLAMP_store2(uint32_t instr, const uint64_t addr){} +void SLAMP_store4(uint32_t instr, const uint64_t addr){} +void SLAMP_store8(uint32_t instr, const uint64_t addr){} +void SLAMP_storen(uint32_t instr, const uint64_t addr, size_t n){} + +void SLAMP_store1_ext(const uint64_t addr, const uint32_t bare_inst){} +void SLAMP_store2_ext(const uint64_t addr, const uint32_t bare_inst){} +void SLAMP_store4_ext(const uint64_t addr, const uint32_t bare_inst){} +void SLAMP_store8_ext(const uint64_t addr, const uint32_t bare_inst){} +void SLAMP_storen_ext(const uint64_t addr, const uint32_t bare_inst, size_t n){} + +/* wrappers */ +static void* SLAMP_malloc_hook(size_t size, const void *caller){} +static void SLAMP_free_hook(void *ptr, const void *caller){} +static void* SLAMP_memalign_hook(size_t alignment, size_t size, const void *caller){} +void* SLAMP_malloc(size_t size, uint32_t instr, size_t alignment){} + +void* SLAMP_calloc(size_t nelem, size_t elsize){} +void* SLAMP_realloc(void* ptr, size_t size){} +void* SLAMP__Znam(size_t size){} +void* SLAMP__Znwm(size_t size){} + + + +char* SLAMP_strdup(const char *s1){} +char* SLAMP___strdup(const char *s1){} +void SLAMP_free(void* ptr){} +void SLAMP_cfree(void* ptr){} +void SLAMP__ZdlPv(void* ptr){} +void SLAMP__ZdaPv(void* ptr){} +int SLAMP_brk(void *end_data_segment){} +void* SLAMP_sbrk(intptr_t increment){} + +/* llvm memory intrinsics */ +void SLAMP_llvm_memcpy_p0i8_p0i8_i32(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint32_t sizeBytes){} +void SLAMP_llvm_memcpy_p0i8_p0i8_i64(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint64_t sizeBytes){} +void SLAMP_llvm_memmove_p0i8_p0i8_i32(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint32_t sizeBytes){} +void SLAMP_llvm_memmove_p0i8_p0i8_i64(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint64_t sizeBytes){} +void SLAMP_llvm_memset_p0i8_i32(const uint8_t* dstAddr, const uint32_t len){} +void SLAMP_llvm_memset_p0i8_i64(const uint8_t* dstAddr, const uint64_t len){} + +// void SLAMP_llvm_lifetime_start_p0i8(uint64_t size, uint8_t* ptr){} +// void SLAMP_llvm_lifetime_end_p0i8(uint64_t size, uint8_t* ptr){} + +/* String functions */ +size_t SLAMP_strlen(const char *str){} +char* SLAMP_strchr(char *s, int c){} +char* SLAMP_strrchr(char *s, int c){} +int SLAMP_strcmp(const char *s1, const char *s2){} +int SLAMP_strncmp(const char *s1, const char *s2, size_t n){} +char* SLAMP_strcpy(char *dest, const char *src){} +char* SLAMP_strncpy(char *dest, const char *src, size_t n){} +char* SLAMP_strcat(char *s1, const char *s2){} +char* SLAMP_strncat(char *s1, const char *s2, size_t n){} +char* SLAMP_strstr(char *s1, char *s2){} +size_t SLAMP_strspn(const char *s1, const char *s2){} +size_t SLAMP_strcspn(const char *s1, const char *s2){} +char* SLAMP_strtok(char *s, const char *delim){} +double SLAMP_strtod(const char *nptr, char **endptr){} +long int SLAMP_strtol(const char *nptr, char **endptr, int base){} +char* SLAMP_strpbrk(char *s1, char *s2){} + +/* Mem* and b* functions */ +void *SLAMP_memset (void *dest, int c, size_t n){} +void *SLAMP_memcpy (void *dest, const void *src, size_t n){} +void *SLAMP___builtin_memcpy (void *dest, const void *src, size_t n){} +void *SLAMP_memmove (void *dest, const void *src, size_t n){} +int SLAMP_memcmp(const void *s1, const void *s2, size_t n){} +void* SLAMP_memchr(void* ptr, int value, size_t num){} +void* SLAMP___rawmemchr(void* ptr, int value){} + +void SLAMP_bzero(void *s, size_t n){} +void SLAMP_bcopy(const void *s1, void *s2, size_t n){} + +/* IO */ +ssize_t SLAMP_read(int fd, void *buf, size_t count){} +int SLAMP_open(const char *pathname, int flags, mode_t mode){} +int SLAMP_close(int fd){} +ssize_t SLAMP_write(int fd, const void *buf, size_t count){} +off_t SLAMP_lseek(int fildes, off_t offset, int whence){} + +FILE * SLAMP_fopen(const char *path, const char *mode){} +FILE * SLAMP_fopen64(const char *path, const char *mode){} +FILE * SLAMP_freopen(const char *path, const char *mode, FILE* stream){} +int SLAMP_fflush(FILE *stream){} +int SLAMP_fclose(FILE *stream){} +int SLAMP_ferror(FILE *stream){} +int SLAMP_feof(FILE *stream){} +long SLAMP_ftell(FILE *stream){} +size_t SLAMP_fread(void * ptr, size_t size, size_t nitems, FILE *stream){} +size_t SLAMP_fwrite(const void *ptr, size_t size, size_t nitems, FILE *stream){} +int SLAMP_fseek(FILE *stream, long offset, int whence){} +void SLAMP_rewind(FILE *stream){} + +int SLAMP_fgetc(FILE *stream){} +int SLAMP_fputc(int c, FILE *stream){} +char * SLAMP_fgets(char *s, int n, FILE *stream){} +int SLAMP_fputs(const char *s, FILE *stream){} + +int SLAMP_ungetc(int c, FILE *stream){} +int SLAMP_putchar(int c){} +int SLAMP_getchar(void){} + +int SLAMP_fileno(FILE *stream){} +char * SLAMP_gets(char *s){} +int SLAMP_puts(const char *s){} + +int SLAMP_select(int n, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, struct timeval *timeout){} +int SLAMP_remove(const char *path){} + +void SLAMP_setbuf(FILE * stream, char * buf){} +void SLAMP_setvbuf(FILE * stream, char * buf, int mode, size_t size){} +char * SLAMP_tmpnam(char *s){} +FILE* SLAMP_tmpfile(void){} +char * SLAMP_ttyname(int fildes){} + +FILE * SLAMP_fdopen(int fildes, const char *mode){} +void SLAMP_clearerr(FILE *stream){} + +int SLAMP_truncate(const char *path, off_t length){} +int SLAMP_ftruncate(int fildes, off_t length){} + +int SLAMP_dup(int oldfd){} +int SLAMP_dup2(int oldfd, int newfd){} +int SLAMP_pipe(int filedes[2]){} + +int SLAMP_chmod(const char *path, mode_t mode){} +int SLAMP_fchmod(int fildes, mode_t mode){} +int SLAMP_fchown(int fd, uid_t owner, gid_t group){} +int SLAMP_access(const char *pathname, int mode){} +long SLAMP_pathconf(char *path, int name){} +int SLAMP_mkdir(const char *pathname, mode_t mode){} +int SLAMP_rmdir(const char *pathname){} +mode_t SLAMP_umask(mode_t mask){} +int SLAMP_fcntl(int fd, int cmd, struct flock *lock){} + +DIR* SLAMP_opendir(const char* name){} +struct dirent* SLAMP_readdir(DIR *dirp){} +struct dirent64* SLAMP_readdir64(DIR *dirp){} +int SLAMP_closedir(DIR* dirp){} + +/* Printf */ +int SLAMP_printf(const char *format, ...){} +int SLAMP_fprintf(FILE *stream, const char *format, ...){} +int SLAMP_sprintf(char *str, const char *format, ...){} +int SLAMP_snprintf(char *str, size_t size, const char *format, ...){} + +int SLAMP_vprintf(const char *format, va_list ap){} +int SLAMP_vfprintf(FILE *stream, const char *format, va_list ap){} +int SLAMP_vsprintf(char *str, const char *format, va_list ap){} +int SLAMP_vsnprintf(char *str, size_t size, const char *format, va_list ap){} + +/* Scanf */ +int SLAMP_fscanf(FILE *stream, const char *format, ... ){} +int SLAMP_scanf(const char *format, ... ){} +int SLAMP_sscanf(const char *s, const char *format, ... ){} +int SLAMP___isoc99_sscanf(const char *s, const char *format, ... ){} + +int SLAMP_vfscanf(FILE *stream, const char *format, va_list ap){} +int SLAMP_vscanf(const char *format, va_list ap){} +int SLAMP_vsscanf(const char *s, const char *format, va_list ap){} + +/* Time */ +time_t SLAMP_time(time_t *t){} +struct tm *SLAMP_localtime(const time_t *timer){} +struct lconv* SLAMP_localeconv(){} +struct tm *SLAMP_gmtime(const time_t *timer){} +int SLAMP_gettimeofday(struct timeval *tv, struct timezone *tz){} + +/* Math */ +double SLAMP_ldexp(double x, int exp){} +float SLAMP_ldexpf(float x, int exp){} +long double SLAMP_ldexpl(long double x, int exp){} +double SLAMP_log10(double x){} +float SLAMP_log10f(float x){} +long double SLAMP_log10l(long double x){} +double SLAMP_log(double x){} +float SLAMP_logf(float x){} +long double SLAMP_logl(long double x){} + +double SLAMP_exp(double x){} +float SLAMP_expf(float x){} +long double SLAMP_expl(long double x){} + +double SLAMP_cos(double x){} +float SLAMP_cosf(float x){} +long double SLAMP_cosl(long double x){} +double SLAMP_sin(double x){} +double SLAMP_tan(double x){} +float SLAMP_sinf(float x){} +long double SLAMP_sinl(long double x){} + +double SLAMP_atan(double x){} +float SLAMP_atanf(float x){} +long double SLAMP_atanl(long double x){} + +double SLAMP_floor(double x){} +float SLAMP_floorf(float x){} +long double SLAMP_floorl(long double x){} +double SLAMP_ceil(double x){} +float SLAMP_ceilf(float x){} +long double SLAMP_ceill(long double x){} + +double SLAMP_atan2(double y, double x){} +float SLAMP_atan2f(float y, float x){} +long double SLAMP_atan2l(long double y, long double x){} + +double SLAMP_sqrt(double x){} +float SLAMP_sqrtf(float x){} +long double SLAMP_sqrtl(long double x){} + +double SLAMP_pow(double x, double y){} +float SLAMP_powf(float x, float y){} +long double SLAMP_powl(long double x, long double y){} + +double SLAMP_fabs(double x){} +float SLAMP_fabsf(float x){} +long double SLAMP_fabsl(long double x){} + +double SLAMP_modf(double x, double *iptr){} +float SLAMP_modff(float x, float *iptr){} +long double SLAMP_modfl(long double x, long double *iptr){} + +double SLAMP_fmod(double x, double y){} + +double SLAMP_frexp(double num, int *exp){} +float SLAMP_frexpf(float num, int *exp){} +long double SLAMP_frexpl(long double num, int *exp){} + +int SLAMP_isnan(){} + +/* MISC */ +char *SLAMP_getenv(const char *name){} +int SLAMP_putenv(char* string){} +char *SLAMP_getcwd(char *buf, size_t size){} +char* SLAMP_strerror(int errnum){} +void SLAMP_exit(int status){} +void SLAMP__exit(int status){} +int SLAMP_link(const char *oldpath, const char *newpath){} +int SLAMP_unlink(const char *pathname){} +int SLAMP_isatty(int desc){} +int SLAMP_setuid(uid_t uid){} +uid_t SLAMP_getuid(void){} +uid_t SLAMP_geteuid(void){} +int SLAMP_setgid(gid_t gid){} +gid_t SLAMP_getgid(void){} +gid_t SLAMP_getegid(void){} +pid_t SLAMP_getpid(void){} +int SLAMP_chdir(const char *path){} +int SLAMP_execl(const char *path, const char *arg0, ... /*, (char *)0 */){} +int SLAMP_execv(const char *path, char *const argv[]){} +int SLAMP_execvp(const char *file, char *const argv[]){} +int SLAMP_kill(pid_t pid, int sig){} +pid_t SLAMP_fork(void){} +sighandler_t SLAMP___sysv_signal(int signum, sighandler_t handler){} +pid_t SLAMP_waitpid(pid_t pid, int* status, int options){} +void SLAMP_qsort(void* base, size_t nmemb, size_t size, int(*compar)(const void *, const void *)){} +int SLAMP_ioctl(int d, int request, ...){} +unsigned int SLAMP_sleep(unsigned int seconds){} +char* SLAMP_gcvt(double number, size_t ndigit, char* buf){} +char* SLAMP_nl_langinfo(nl_item item){} + +/* Compiler/Glibc Internals */ +void SLAMP___assert_fail(const char * assertion, const char * file, unsigned int line, const char * function){} +const unsigned short int **SLAMP___ctype_b_loc(void){} +int SLAMP__IO_getc(_IO_FILE * __fp){} +int SLAMP__IO_putc(int __c, _IO_FILE *__fp){} + +int SLAMP___fxstat (int __ver, int __fildes, struct stat *__stat_buf){} +int SLAMP___xstat (int __ver, __const char *__filename, struct stat *__stat_buf){} diff --git a/liberty/lib/SLAMP/SLAMPboost/CMakeLists.txt b/liberty/lib/SLAMP/SLAMPboost/CMakeLists.txt new file mode 100644 index 00000000..604d7ffd --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPboost/CMakeLists.txt @@ -0,0 +1,34 @@ +file(GLOB SRCS + "*.cpp" +) +set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} "/u/ziyangx/test/boost/boost_1_80_0/install/lib/cmake") + +# add CMAKE_PREFIX_PATH +set(CMAKE_C_COMPILER "clang") +set(CMAKE_CXX_COMPILER "clang++") + +# Compilation flags +# set_source_files_properties(${SRCS} PROPERTIES COMPILE_FLAGS "-Wl,-save-temps -std=c++17 -Wno-inline -O3 -fexceptions")# -emit-llvm") +set_source_files_properties(${SRCS} PROPERTIES COMPILE_FLAGS "-flto -std=c++17 -Wno-inline -O3 -fexceptions")# -emit-llvm") +set(PassName "slamp_hooks_boost") + +set(THREADS_PREFER_PTHREAD_FLAG ON) +find_package(Boost 1.80.0 REQUIRED COMPONENTS system) +include_directories(${Boost_INCLUDE_DIRS}) + + +# list(APPEND CMAKE_MODULE_PATH "${LLVM_CMAKE_DIR}") +#include(HandleLLVMOptions) +# include(AddLLVM) + + +include_directories(./ + /u/ziyangx/test/boost/boost_1_80_0/install/include) + +add_library(${PassName} STATIC ${SRCS}) +target_link_libraries(${PassName} ${Boost_LIBRARIES}) +# target_link_libraries(${PassName} nng::nng) +# add_llvm_library(${PassName}_shared SHARED ${SRCS}) +# set_target_properties(${PassName}_shared PROPERTIES OUTPUT_NAME ${PassName}) +# set_property(TARGET ${PassName} PROPERTY INTERPROCEDURAL_OPTIMIZATION TRUE) +#add_llvm_library(${PassName} SHARED ${SRCS}) # This is to generate libxxx.so diff --git a/liberty/lib/SLAMP/SLAMPboost/slamp_hooks.h b/liberty/lib/SLAMP/SLAMPboost/slamp_hooks.h new file mode 100644 index 00000000..df546117 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPboost/slamp_hooks.h @@ -0,0 +1,371 @@ +#ifndef SLAMPLIB_HOOKS_SLAMP_HOOKS_H +#define SLAMPLIB_HOOKS_SLAMP_HOOKS_H + +// FIXME: inline tweak actually make things worse in sequential and better with +// 16x, so turn it on at all time before understanding why +#define ATTRIBUTE(x) __attribute__((x)) +// #ifdef ITO_ENABLE +// // #define ATTRIBUTE(x) +// #define ATTRIBUTE(x) __attribute__((x)) +// #else +// #define ATTRIBUTE(x) +// #endif + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +void SLAMP_dbggv(int id); +void SLAMP_dbggvstr(char* str); + +// SLAMP measure functions +void SLAMP_measure_init(); +void SLAMP_measure_fini(); +void SLAMP_measure_load(uint32_t id, uint64_t size); +void SLAMP_measure_store(uint32_t id, uint64_t size); +static void* SLAMP_measure_malloc_hook(size_t size, const void *caller); +static void SLAMP_measure_free_hook(void *ptr, const void *caller); + +void SLAMP_init(uint32_t fn_id, uint32_t loop_id); +void SLAMP_fini(const char* filename); + +void SLAMP_allocated(uint64_t addr); +void SLAMP_init_global_vars(const char *name, uint64_t addr, size_t size); +void SLAMP_main_entry(uint32_t argc, char** argv, char** env); + +void SLAMP_enter_fcn(uint32_t id); +void SLAMP_exit_fcn(uint32_t id); +void SLAMP_enter_loop(uint32_t id); +void SLAMP_exit_loop(uint32_t id); +void SLAMP_loop_iter_ctx(uint32_t id); +void SLAMP_loop_invocation(); +void SLAMP_loop_iteration(); +void SLAMP_loop_exit(); + +void SLAMP_report_base_pointer_arg(uint32_t, uint32_t, void *ptr); +void SLAMP_report_base_pointer_inst(uint32_t, void *ptr); +void SLAMP_callback_stack_alloca(uint64_t, uint64_t, uint32_t, uint64_t); +void SLAMP_callback_stack_free(void); + +void SLAMP_ext_push(const uint32_t instr); +void SLAMP_ext_pop(); + +void SLAMP_push(const uint32_t instr); +void SLAMP_pop(); + +void SLAMP_load1(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_load2(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_load4(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_load8(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_loadn(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, size_t n) ATTRIBUTE(always_inline);; + +void SLAMP_load1_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_load2_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_load4_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_load8_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_loadn_ext(const uint64_t addr, const uint32_t bare_instr, size_t n) ATTRIBUTE(always_inline);; + +void SLAMP_store1(uint32_t instr, const uint64_t addr) ATTRIBUTE(always_inline); +void SLAMP_store2(uint32_t instr, const uint64_t addr) ATTRIBUTE(always_inline); +void SLAMP_store4(uint32_t instr, const uint64_t addr) ATTRIBUTE(always_inline); +void SLAMP_store8(uint32_t instr, const uint64_t addr) ATTRIBUTE(always_inline); +void SLAMP_storen(uint32_t instr, const uint64_t addr, size_t n) ATTRIBUTE(always_inline);; + +void SLAMP_store1_ext(const uint64_t addr, const uint32_t bare_inst) ATTRIBUTE(always_inline); +void SLAMP_store2_ext(const uint64_t addr, const uint32_t bare_inst) ATTRIBUTE(always_inline); +void SLAMP_store4_ext(const uint64_t addr, const uint32_t bare_inst) ATTRIBUTE(always_inline); +void SLAMP_store8_ext(const uint64_t addr, const uint32_t bare_inst) ATTRIBUTE(always_inline); +void SLAMP_storen_ext(const uint64_t addr, const uint32_t bare_inst, size_t n) ATTRIBUTE(always_inline);; + +/* wrappers */ +static void* SLAMP_malloc_hook(size_t size, const void *caller); +static void SLAMP_free_hook(void *ptr, const void *caller); +static void* SLAMP_memalign_hook(size_t alignment, size_t size, const void *caller); +void* SLAMP_malloc(size_t size, uint32_t instr=0, size_t alignment=16); + +void* SLAMP_calloc(size_t nelem, size_t elsize); +void* SLAMP_realloc(void* ptr, size_t size); +void* SLAMP__Znam(size_t size); +void* SLAMP__Znwm(size_t size); + +char* SLAMP_strdup(const char *s1); +char* SLAMP___strdup(const char *s1); +void SLAMP_free(void* ptr); +void SLAMP_cfree(void* ptr); +void SLAMP__ZdlPv(void* ptr); +void SLAMP__ZdaPv(void* ptr); +int SLAMP_brk(void *end_data_segment); +void* SLAMP_sbrk(intptr_t increment); + +/* llvm memory intrinsics */ +void SLAMP_llvm_memcpy_p0i8_p0i8_i32(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint32_t sizeBytes); +void SLAMP_llvm_memcpy_p0i8_p0i8_i64(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint64_t sizeBytes); +void SLAMP_llvm_memmove_p0i8_p0i8_i32(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint32_t sizeBytes); +void SLAMP_llvm_memmove_p0i8_p0i8_i64(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint64_t sizeBytes); +void SLAMP_llvm_memset_p0i8_i32(const uint8_t* dstAddr, const uint32_t len); +void SLAMP_llvm_memset_p0i8_i64(const uint8_t* dstAddr, const uint64_t len); + +// void SLAMP_llvm_lifetime_start_p0i8(uint64_t size, uint8_t* ptr); +// void SLAMP_llvm_lifetime_end_p0i8(uint64_t size, uint8_t* ptr); + +/* String functions */ +size_t SLAMP_strlen(const char *str); +char* SLAMP_strchr(char *s, int c); +char* SLAMP_strrchr(char *s, int c); +int SLAMP_strcmp(const char *s1, const char *s2); +int SLAMP_strncmp(const char *s1, const char *s2, size_t n); +char* SLAMP_strcpy(char *dest, const char *src); +char* SLAMP_strncpy(char *dest, const char *src, size_t n); +char* SLAMP_strcat(char *s1, const char *s2); +char* SLAMP_strncat(char *s1, const char *s2, size_t n); +char* SLAMP_strstr(char *s1, char *s2); +size_t SLAMP_strspn(const char *s1, const char *s2); +size_t SLAMP_strcspn(const char *s1, const char *s2); +char* SLAMP_strtok(char *s, const char *delim); +double SLAMP_strtod(const char *nptr, char **endptr); +long int SLAMP_strtol(const char *nptr, char **endptr, int base); +char* SLAMP_strpbrk(char *s1, char *s2); + +/* Mem* and b* functions */ +void *SLAMP_memset (void *dest, int c, size_t n); +void *SLAMP_memcpy (void *dest, const void *src, size_t n); +void *SLAMP___builtin_memcpy (void *dest, const void *src, size_t n); +void *SLAMP_memmove (void *dest, const void *src, size_t n); +int SLAMP_memcmp(const void *s1, const void *s2, size_t n); +void* SLAMP_memchr(void* ptr, int value, size_t num); +void* SLAMP___rawmemchr(void* ptr, int value); + +void SLAMP_bzero(void *s, size_t n); +void SLAMP_bcopy(const void *s1, void *s2, size_t n); + +/* IO */ +ssize_t SLAMP_read(int fd, void *buf, size_t count); +int SLAMP_open(const char *pathname, int flags, mode_t mode); +int SLAMP_close(int fd); +ssize_t SLAMP_write(int fd, const void *buf, size_t count); +off_t SLAMP_lseek(int fildes, off_t offset, int whence); + +FILE * SLAMP_fopen(const char *path, const char *mode); +FILE * SLAMP_fopen64(const char *path, const char *mode); +FILE * SLAMP_freopen(const char *path, const char *mode, FILE* stream); +int SLAMP_fflush(FILE *stream); +int SLAMP_fclose(FILE *stream); +int SLAMP_ferror(FILE *stream); +int SLAMP_feof(FILE *stream); +long SLAMP_ftell(FILE *stream); +size_t SLAMP_fread(void * ptr, size_t size, size_t nitems, FILE *stream); +size_t SLAMP_fwrite(const void *ptr, size_t size, size_t nitems, FILE *stream); +int SLAMP_fseek(FILE *stream, long offset, int whence); +void SLAMP_rewind(FILE *stream); + +int SLAMP_fgetc(FILE *stream); +int SLAMP_fputc(int c, FILE *stream); +char * SLAMP_fgets(char *s, int n, FILE *stream); +int SLAMP_fputs(const char *s, FILE *stream); + +int SLAMP_ungetc(int c, FILE *stream); +int SLAMP_putchar(int c); +int SLAMP_getchar(void); + +int SLAMP_fileno(FILE *stream); +char * SLAMP_gets(char *s); +int SLAMP_puts(const char *s); + +int SLAMP_select(int n, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, struct timeval *timeout); +int SLAMP_remove(const char *path); + +void SLAMP_setbuf(FILE * stream, char * buf); +void SLAMP_setvbuf(FILE * stream, char * buf, int mode, size_t size); +char * SLAMP_tmpnam(char *s); +FILE* SLAMP_tmpfile(void); +char * SLAMP_ttyname(int fildes); + +FILE * SLAMP_fdopen(int fildes, const char *mode); +void SLAMP_clearerr(FILE *stream); + +int SLAMP_truncate(const char *path, off_t length); +int SLAMP_ftruncate(int fildes, off_t length); + +int SLAMP_dup(int oldfd); +int SLAMP_dup2(int oldfd, int newfd); +int SLAMP_pipe(int filedes[2]); + +int SLAMP_chmod(const char *path, mode_t mode); +int SLAMP_fchmod(int fildes, mode_t mode); +int SLAMP_fchown(int fd, uid_t owner, gid_t group); +int SLAMP_access(const char *pathname, int mode); +long SLAMP_pathconf(char *path, int name); +int SLAMP_mkdir(const char *pathname, mode_t mode); +int SLAMP_rmdir(const char *pathname); +mode_t SLAMP_umask(mode_t mask); +int SLAMP_fcntl(int fd, int cmd, struct flock *lock); + +DIR* SLAMP_opendir(const char* name); +struct dirent* SLAMP_readdir(DIR *dirp); +struct dirent64* SLAMP_readdir64(DIR *dirp); +int SLAMP_closedir(DIR* dirp); + +/* Printf */ +int SLAMP_printf(const char *format, ...); +int SLAMP_fprintf(FILE *stream, const char *format, ...); +int SLAMP_sprintf(char *str, const char *format, ...); +int SLAMP_snprintf(char *str, size_t size, const char *format, ...); + +int SLAMP_vprintf(const char *format, va_list ap); +int SLAMP_vfprintf(FILE *stream, const char *format, va_list ap); +int SLAMP_vsprintf(char *str, const char *format, va_list ap); +int SLAMP_vsnprintf(char *str, size_t size, const char *format, va_list ap); + +/* Scanf */ +int SLAMP_fscanf(FILE *stream, const char *format, ... ); +int SLAMP_scanf(const char *format, ... ); +int SLAMP_sscanf(const char *s, const char *format, ... ); +int SLAMP___isoc99_sscanf(const char *s, const char *format, ... ); + +int SLAMP_vfscanf(FILE *stream, const char *format, va_list ap); +int SLAMP_vscanf(const char *format, va_list ap); +int SLAMP_vsscanf(const char *s, const char *format, va_list ap); + +/* Time */ +time_t SLAMP_time(time_t *t); +struct tm *SLAMP_localtime(const time_t *timer); +struct lconv* SLAMP_localeconv(); +struct tm *SLAMP_gmtime(const time_t *timer); +int SLAMP_gettimeofday(struct timeval *tv, struct timezone *tz); + +/* Math */ +double SLAMP_ldexp(double x, int exp); +float SLAMP_ldexpf(float x, int exp); +long double SLAMP_ldexpl(long double x, int exp); +double SLAMP_log10(double x); +float SLAMP_log10f(float x); +long double SLAMP_log10l(long double x); +double SLAMP_log(double x); +float SLAMP_logf(float x); +long double SLAMP_logl(long double x); + +double SLAMP_exp(double x); +float SLAMP_expf(float x); +long double SLAMP_expl(long double x); + +double SLAMP_cos(double x); +float SLAMP_cosf(float x); +long double SLAMP_cosl(long double x); +double SLAMP_sin(double x); +double SLAMP_tan(double x); +float SLAMP_sinf(float x); +long double SLAMP_sinl(long double x); + +double SLAMP_atan(double x); +float SLAMP_atanf(float x); +long double SLAMP_atanl(long double x); + +double SLAMP_floor(double x); +float SLAMP_floorf(float x); +long double SLAMP_floorl(long double x); +double SLAMP_ceil(double x); +float SLAMP_ceilf(float x); +long double SLAMP_ceill(long double x); + +double SLAMP_atan2(double y, double x); +float SLAMP_atan2f(float y, float x); +long double SLAMP_atan2l(long double y, long double x); + +double SLAMP_sqrt(double x); +float SLAMP_sqrtf(float x); +long double SLAMP_sqrtl(long double x); + +double SLAMP_pow(double x, double y); +float SLAMP_powf(float x, float y); +long double SLAMP_powl(long double x, long double y); + +double SLAMP_fabs(double x); +float SLAMP_fabsf(float x); +long double SLAMP_fabsl(long double x); + +double SLAMP_modf(double x, double *iptr); +float SLAMP_modff(float x, float *iptr); +long double SLAMP_modfl(long double x, long double *iptr); + +double SLAMP_fmod(double x, double y); + +double SLAMP_frexp(double num, int *exp); +float SLAMP_frexpf(float num, int *exp); +long double SLAMP_frexpl(long double num, int *exp); + +int SLAMP_isnan(); + +/* MISC */ +char *SLAMP_getenv(const char *name); +int SLAMP_putenv(char* string); +char *SLAMP_getcwd(char *buf, size_t size); +char* SLAMP_strerror(int errnum); +void SLAMP_exit(int status); +void SLAMP__exit(int status); +int SLAMP_link(const char *oldpath, const char *newpath); +int SLAMP_unlink(const char *pathname); +int SLAMP_isatty(int desc); +int SLAMP_setuid(uid_t uid); +uid_t SLAMP_getuid(void); +uid_t SLAMP_geteuid(void); +int SLAMP_setgid(gid_t gid); +gid_t SLAMP_getgid(void); +gid_t SLAMP_getegid(void); +pid_t SLAMP_getpid(void); +int SLAMP_chdir(const char *path); +int SLAMP_execl(const char *path, const char *arg0, ... /*, (char *)0 */); +int SLAMP_execv(const char *path, char *const argv[]); +int SLAMP_execvp(const char *file, char *const argv[]); +int SLAMP_kill(pid_t pid, int sig); +pid_t SLAMP_fork(void); +sighandler_t SLAMP___sysv_signal(int signum, sighandler_t handler); +pid_t SLAMP_waitpid(pid_t pid, int* status, int options); +void SLAMP_qsort(void* base, size_t nmemb, size_t size, int(*compar)(const void *, const void *)); +int SLAMP_ioctl(int d, int request, ...); +unsigned int SLAMP_sleep(unsigned int seconds); +char* SLAMP_gcvt(double number, size_t ndigit, char* buf); +char* SLAMP_nl_langinfo(nl_item item); + +/* Compiler/Glibc Internals */ +void SLAMP___assert_fail(const char * assertion, const char * file, unsigned int line, const char * function); +const unsigned short int **SLAMP___ctype_b_loc(void); +int SLAMP__IO_getc(_IO_FILE * __fp); +int SLAMP__IO_putc(int __c, _IO_FILE *__fp); +int* SLAMP___errno_location (void); + +int SLAMP___fxstat (int __ver, int __fildes, struct stat *__stat_buf); +int SLAMP___xstat (int __ver, __const char *__filename, struct stat *__stat_buf); + +#ifdef __cplusplus +} +#endif + +#endif /* SLAMP_HOOKS_H */ diff --git a/liberty/lib/SLAMP/SLAMPnng/CMakeLists.txt b/liberty/lib/SLAMP/SLAMPnng/CMakeLists.txt new file mode 100644 index 00000000..cb9f8144 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPnng/CMakeLists.txt @@ -0,0 +1,32 @@ +file(GLOB SRCS + "*.cpp" +) +set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} "/u/ziyangx/test/nng/nng-1.5.2/install/lib/cmake") + +# add CMAKE_PREFIX_PATH +set(CMAKE_C_COMPILER "clang") +set(CMAKE_CXX_COMPILER "clang++") + +# Compilation flags +# set_source_files_properties(${SRCS} PROPERTIES COMPILE_FLAGS "-Wl,-save-temps -std=c++17 -Wno-inline -O3 -fexceptions")# -emit-llvm") +set_source_files_properties(${SRCS} PROPERTIES COMPILE_FLAGS "-std=c++17 -Wno-inline -O3 -fexceptions")# -emit-llvm") +set(PassName "slamp_hooks_nng") + +set(THREADS_PREFER_PTHREAD_FLAG ON) +find_package(Threads REQUIRED) +find_package(nng REQUIRED CONFIG) + +# list(APPEND CMAKE_MODULE_PATH "${LLVM_CMAKE_DIR}") +#include(HandleLLVMOptions) +# include(AddLLVM) + + +include_directories(./ + /u/ziyangx/test/nng/nng-1.5.2/install/include) + +add_library(${PassName} SHARED ${SRCS}) +# target_link_libraries(${PassName} nng::nng) +# add_llvm_library(${PassName}_shared SHARED ${SRCS}) +# set_target_properties(${PassName}_shared PROPERTIES OUTPUT_NAME ${PassName}) +# set_property(TARGET ${PassName} PROPERTY INTERPROCEDURAL_OPTIMIZATION TRUE) +#add_llvm_library(${PassName} SHARED ${SRCS}) # This is to generate libxxx.so diff --git a/liberty/lib/SLAMP/SLAMPnng/NngSend.cpp b/liberty/lib/SLAMP/SLAMPnng/NngSend.cpp new file mode 100644 index 00000000..a571415d --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPnng/NngSend.cpp @@ -0,0 +1,393 @@ +#include +#include +#include +#include + +#include "slamp_hooks.h" +#include +#include +#include + +nng_socket sock; +void SLAMP_init(uint32_t fn_id, uint32_t loop_id) { + int rv; + int bytes; + char *url = "ipc:///tmp/SLAMP.ipc"; + if ((rv = nng_push0_open(&sock)) != 0) { + fprintf(stderr, "nng_push0_open: %s", nng_strerror(rv)); + exit(1); + } + if ((rv = nng_dial(sock, url, nullptr, 0)) != 0) { + fprintf(stderr, "nng_dial: %s", nng_strerror(rv)); + exit(1); + } + // set NN_SNDBUF to 100MB + int sndbuf = 8192; + if ((rv = nng_setopt_int(sock, NNG_OPT_SENDBUF, sndbuf)) != 0) { + fprintf(stderr, "nng_setopt_int: %s", nng_strerror(rv)); + exit(1); + } + // send a msg with "fn_id, loop_id" + char msg[100]; + sprintf(msg, "%d,%d", fn_id, loop_id); + if ((rv = nng_send(sock, msg, strlen(msg), 0)) != 0) { + fprintf(stderr, "nng_send: %s", nng_strerror(rv)); + exit(1); + } +} + +void SLAMP_fini(const char* filename){ + nng_close(sock); +} + +void SLAMP_allocated(uint64_t addr){} +void SLAMP_init_global_vars(const char *name, uint64_t addr, size_t size){} +void SLAMP_main_entry(uint32_t argc, char** argv, char** env){} + +void SLAMP_enter_fcn(uint32_t id){} +void SLAMP_exit_fcn(uint32_t id){} +void SLAMP_enter_loop(uint32_t id){} +void SLAMP_exit_loop(uint32_t id){} +void SLAMP_loop_iter_ctx(uint32_t id){} +void SLAMP_loop_invocation(){} +void SLAMP_loop_iteration(){} +void SLAMP_loop_exit(){} + +void SLAMP_report_base_pointer_arg(uint32_t, uint32_t, void *ptr){} +void SLAMP_report_base_pointer_inst(uint32_t, void *ptr){} +void SLAMP_callback_stack_alloca(uint64_t, uint64_t, uint32_t, uint64_t){} +void SLAMP_callback_stack_free(){} + +void SLAMP_ext_push(const uint32_t instr){} +void SLAMP_ext_pop(){} + +void SLAMP_push(const uint32_t instr){} +void SLAMP_pop(){} + +void SLAMP_load1(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + int rv; + char buf[1024]; + // msg = instr + addr + value + sprintf(buf, "%d %lu %lu", instr, addr, value); + + if ((rv = nng_send(sock, buf, strlen(buf), 0)) != 0) { + fprintf(stderr, "nng_send: %s", nng_strerror(rv)); + exit(1); + } +} +void SLAMP_load2(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + int rv; + char buf[1024]; + // msg = instr + addr + value + sprintf(buf, "%d %lu %lu", instr, addr, value); + + if ((rv = nng_send(sock, buf, strlen(buf), 0)) != 0) { + fprintf(stderr, "nng_send: %s", nng_strerror(rv)); + exit(1); + } + +} +void SLAMP_load4(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + int rv; + char buf[1024]; + // msg = instr + addr + value + sprintf(buf, "%d %lu %lu", instr, addr, value); + + if ((rv = nng_send(sock, buf, strlen(buf), 0)) != 0) { + fprintf(stderr, "nng_send: %s", nng_strerror(rv)); + exit(1); + } +} + +void SLAMP_load8(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + int rv; + char buf[1024]; + // msg = instr + addr + value + sprintf(buf, "%d %lu %lu", instr, addr, value); + + if ((rv = nng_send(sock, buf, strlen(buf), 0)) != 0) { + fprintf(stderr, "nng_send: %s", nng_strerror(rv)); + exit(1); + } +} + +void SLAMP_loadn(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, size_t n){} + +void SLAMP_load1_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value){} +void SLAMP_load2_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value){} +void SLAMP_load4_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value){} +void SLAMP_load8_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value){} +void SLAMP_loadn_ext(const uint64_t addr, const uint32_t bare_instr, size_t n){} + +void SLAMP_store1(uint32_t instr, const uint64_t addr){} +void SLAMP_store2(uint32_t instr, const uint64_t addr){} +void SLAMP_store4(uint32_t instr, const uint64_t addr){} +void SLAMP_store8(uint32_t instr, const uint64_t addr){} +void SLAMP_storen(uint32_t instr, const uint64_t addr, size_t n){} + +void SLAMP_store1_ext(const uint64_t addr, const uint32_t bare_inst){} +void SLAMP_store2_ext(const uint64_t addr, const uint32_t bare_inst){} +void SLAMP_store4_ext(const uint64_t addr, const uint32_t bare_inst){} +void SLAMP_store8_ext(const uint64_t addr, const uint32_t bare_inst){} +void SLAMP_storen_ext(const uint64_t addr, const uint32_t bare_inst, size_t n){} + +/* wrappers */ +static void* SLAMP_malloc_hook(size_t size, const void *caller){} +static void SLAMP_free_hook(void *ptr, const void *caller){} +static void* SLAMP_memalign_hook(size_t alignment, size_t size, const void *caller){} +void* SLAMP_malloc(size_t size, uint32_t instr, size_t alignment){} + +void* SLAMP_calloc(size_t nelem, size_t elsize){} +void* SLAMP_realloc(void* ptr, size_t size){} +void* SLAMP__Znam(size_t size){} +void* SLAMP__Znwm(size_t size){} + + + +char* SLAMP_strdup(const char *s1){} +char* SLAMP___strdup(const char *s1){} +void SLAMP_free(void* ptr){} +void SLAMP_cfree(void* ptr){} +void SLAMP__ZdlPv(void* ptr){} +void SLAMP__ZdaPv(void* ptr){} +int SLAMP_brk(void *end_data_segment){} +void* SLAMP_sbrk(intptr_t increment){} + +/* llvm memory intrinsics */ +void SLAMP_llvm_memcpy_p0i8_p0i8_i32(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint32_t sizeBytes){} +void SLAMP_llvm_memcpy_p0i8_p0i8_i64(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint64_t sizeBytes){} +void SLAMP_llvm_memmove_p0i8_p0i8_i32(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint32_t sizeBytes){} +void SLAMP_llvm_memmove_p0i8_p0i8_i64(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint64_t sizeBytes){} +void SLAMP_llvm_memset_p0i8_i32(const uint8_t* dstAddr, const uint32_t len){} +void SLAMP_llvm_memset_p0i8_i64(const uint8_t* dstAddr, const uint64_t len){} + +// void SLAMP_llvm_lifetime_start_p0i8(uint64_t size, uint8_t* ptr){} +// void SLAMP_llvm_lifetime_end_p0i8(uint64_t size, uint8_t* ptr){} + +/* String functions */ +size_t SLAMP_strlen(const char *str){} +char* SLAMP_strchr(char *s, int c){} +char* SLAMP_strrchr(char *s, int c){} +int SLAMP_strcmp(const char *s1, const char *s2){} +int SLAMP_strncmp(const char *s1, const char *s2, size_t n){} +char* SLAMP_strcpy(char *dest, const char *src){} +char* SLAMP_strncpy(char *dest, const char *src, size_t n){} +char* SLAMP_strcat(char *s1, const char *s2){} +char* SLAMP_strncat(char *s1, const char *s2, size_t n){} +char* SLAMP_strstr(char *s1, char *s2){} +size_t SLAMP_strspn(const char *s1, const char *s2){} +size_t SLAMP_strcspn(const char *s1, const char *s2){} +char* SLAMP_strtok(char *s, const char *delim){} +double SLAMP_strtod(const char *nptr, char **endptr){} +long int SLAMP_strtol(const char *nptr, char **endptr, int base){} +char* SLAMP_strpbrk(char *s1, char *s2){} + +/* Mem* and b* functions */ +void *SLAMP_memset (void *dest, int c, size_t n){} +void *SLAMP_memcpy (void *dest, const void *src, size_t n){} +void *SLAMP___builtin_memcpy (void *dest, const void *src, size_t n){} +void *SLAMP_memmove (void *dest, const void *src, size_t n){} +int SLAMP_memcmp(const void *s1, const void *s2, size_t n){} +void* SLAMP_memchr(void* ptr, int value, size_t num){} +void* SLAMP___rawmemchr(void* ptr, int value){} + +void SLAMP_bzero(void *s, size_t n){} +void SLAMP_bcopy(const void *s1, void *s2, size_t n){} + +/* IO */ +ssize_t SLAMP_read(int fd, void *buf, size_t count){} +int SLAMP_open(const char *pathname, int flags, mode_t mode){} +int SLAMP_close(int fd){} +ssize_t SLAMP_write(int fd, const void *buf, size_t count){} +off_t SLAMP_lseek(int fildes, off_t offset, int whence){} + +FILE * SLAMP_fopen(const char *path, const char *mode){} +FILE * SLAMP_fopen64(const char *path, const char *mode){} +FILE * SLAMP_freopen(const char *path, const char *mode, FILE* stream){} +int SLAMP_fflush(FILE *stream){} +int SLAMP_fclose(FILE *stream){} +int SLAMP_ferror(FILE *stream){} +int SLAMP_feof(FILE *stream){} +long SLAMP_ftell(FILE *stream){} +size_t SLAMP_fread(void * ptr, size_t size, size_t nitems, FILE *stream){} +size_t SLAMP_fwrite(const void *ptr, size_t size, size_t nitems, FILE *stream){} +int SLAMP_fseek(FILE *stream, long offset, int whence){} +void SLAMP_rewind(FILE *stream){} + +int SLAMP_fgetc(FILE *stream){} +int SLAMP_fputc(int c, FILE *stream){} +char * SLAMP_fgets(char *s, int n, FILE *stream){} +int SLAMP_fputs(const char *s, FILE *stream){} + +int SLAMP_ungetc(int c, FILE *stream){} +int SLAMP_putchar(int c){} +int SLAMP_getchar(void){} + +int SLAMP_fileno(FILE *stream){} +char * SLAMP_gets(char *s){} +int SLAMP_puts(const char *s){} + +int SLAMP_select(int n, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, struct timeval *timeout){} +int SLAMP_remove(const char *path){} + +void SLAMP_setbuf(FILE * stream, char * buf){} +void SLAMP_setvbuf(FILE * stream, char * buf, int mode, size_t size){} +char * SLAMP_tmpnam(char *s){} +FILE* SLAMP_tmpfile(void){} +char * SLAMP_ttyname(int fildes){} + +FILE * SLAMP_fdopen(int fildes, const char *mode){} +void SLAMP_clearerr(FILE *stream){} + +int SLAMP_truncate(const char *path, off_t length){} +int SLAMP_ftruncate(int fildes, off_t length){} + +int SLAMP_dup(int oldfd){} +int SLAMP_dup2(int oldfd, int newfd){} +int SLAMP_pipe(int filedes[2]){} + +int SLAMP_chmod(const char *path, mode_t mode){} +int SLAMP_fchmod(int fildes, mode_t mode){} +int SLAMP_fchown(int fd, uid_t owner, gid_t group){} +int SLAMP_access(const char *pathname, int mode){} +long SLAMP_pathconf(char *path, int name){} +int SLAMP_mkdir(const char *pathname, mode_t mode){} +int SLAMP_rmdir(const char *pathname){} +mode_t SLAMP_umask(mode_t mask){} +int SLAMP_fcntl(int fd, int cmd, struct flock *lock){} + +DIR* SLAMP_opendir(const char* name){} +struct dirent* SLAMP_readdir(DIR *dirp){} +struct dirent64* SLAMP_readdir64(DIR *dirp){} +int SLAMP_closedir(DIR* dirp){} + +/* Printf */ +int SLAMP_printf(const char *format, ...){} +int SLAMP_fprintf(FILE *stream, const char *format, ...){} +int SLAMP_sprintf(char *str, const char *format, ...){} +int SLAMP_snprintf(char *str, size_t size, const char *format, ...){} + +int SLAMP_vprintf(const char *format, va_list ap){} +int SLAMP_vfprintf(FILE *stream, const char *format, va_list ap){} +int SLAMP_vsprintf(char *str, const char *format, va_list ap){} +int SLAMP_vsnprintf(char *str, size_t size, const char *format, va_list ap){} + +/* Scanf */ +int SLAMP_fscanf(FILE *stream, const char *format, ... ){} +int SLAMP_scanf(const char *format, ... ){} +int SLAMP_sscanf(const char *s, const char *format, ... ){} +int SLAMP___isoc99_sscanf(const char *s, const char *format, ... ){} + +int SLAMP_vfscanf(FILE *stream, const char *format, va_list ap){} +int SLAMP_vscanf(const char *format, va_list ap){} +int SLAMP_vsscanf(const char *s, const char *format, va_list ap){} + +/* Time */ +time_t SLAMP_time(time_t *t){} +struct tm *SLAMP_localtime(const time_t *timer){} +struct lconv* SLAMP_localeconv(){} +struct tm *SLAMP_gmtime(const time_t *timer){} +int SLAMP_gettimeofday(struct timeval *tv, struct timezone *tz){} + +/* Math */ +double SLAMP_ldexp(double x, int exp){} +float SLAMP_ldexpf(float x, int exp){} +long double SLAMP_ldexpl(long double x, int exp){} +double SLAMP_log10(double x){} +float SLAMP_log10f(float x){} +long double SLAMP_log10l(long double x){} +double SLAMP_log(double x){} +float SLAMP_logf(float x){} +long double SLAMP_logl(long double x){} + +double SLAMP_exp(double x){} +float SLAMP_expf(float x){} +long double SLAMP_expl(long double x){} + +double SLAMP_cos(double x){} +float SLAMP_cosf(float x){} +long double SLAMP_cosl(long double x){} +double SLAMP_sin(double x){} +double SLAMP_tan(double x){} +float SLAMP_sinf(float x){} +long double SLAMP_sinl(long double x){} + +double SLAMP_atan(double x){} +float SLAMP_atanf(float x){} +long double SLAMP_atanl(long double x){} + +double SLAMP_floor(double x){} +float SLAMP_floorf(float x){} +long double SLAMP_floorl(long double x){} +double SLAMP_ceil(double x){} +float SLAMP_ceilf(float x){} +long double SLAMP_ceill(long double x){} + +double SLAMP_atan2(double y, double x){} +float SLAMP_atan2f(float y, float x){} +long double SLAMP_atan2l(long double y, long double x){} + +double SLAMP_sqrt(double x){} +float SLAMP_sqrtf(float x){} +long double SLAMP_sqrtl(long double x){} + +double SLAMP_pow(double x, double y){} +float SLAMP_powf(float x, float y){} +long double SLAMP_powl(long double x, long double y){} + +double SLAMP_fabs(double x){} +float SLAMP_fabsf(float x){} +long double SLAMP_fabsl(long double x){} + +double SLAMP_modf(double x, double *iptr){} +float SLAMP_modff(float x, float *iptr){} +long double SLAMP_modfl(long double x, long double *iptr){} + +double SLAMP_fmod(double x, double y){} + +double SLAMP_frexp(double num, int *exp){} +float SLAMP_frexpf(float num, int *exp){} +long double SLAMP_frexpl(long double num, int *exp){} + +int SLAMP_isnan(){} + +/* MISC */ +char *SLAMP_getenv(const char *name){} +int SLAMP_putenv(char* string){} +char *SLAMP_getcwd(char *buf, size_t size){} +char* SLAMP_strerror(int errnum){} +void SLAMP_exit(int status){} +void SLAMP__exit(int status){} +int SLAMP_link(const char *oldpath, const char *newpath){} +int SLAMP_unlink(const char *pathname){} +int SLAMP_isatty(int desc){} +int SLAMP_setuid(uid_t uid){} +uid_t SLAMP_getuid(void){} +uid_t SLAMP_geteuid(void){} +int SLAMP_setgid(gid_t gid){} +gid_t SLAMP_getgid(void){} +gid_t SLAMP_getegid(void){} +pid_t SLAMP_getpid(void){} +int SLAMP_chdir(const char *path){} +int SLAMP_execl(const char *path, const char *arg0, ... /*, (char *)0 */){} +int SLAMP_execv(const char *path, char *const argv[]){} +int SLAMP_execvp(const char *file, char *const argv[]){} +int SLAMP_kill(pid_t pid, int sig){} +pid_t SLAMP_fork(void){} +sighandler_t SLAMP___sysv_signal(int signum, sighandler_t handler){} +pid_t SLAMP_waitpid(pid_t pid, int* status, int options){} +void SLAMP_qsort(void* base, size_t nmemb, size_t size, int(*compar)(const void *, const void *)){} +int SLAMP_ioctl(int d, int request, ...){} +unsigned int SLAMP_sleep(unsigned int seconds){} +char* SLAMP_gcvt(double number, size_t ndigit, char* buf){} +char* SLAMP_nl_langinfo(nl_item item){} + +/* Compiler/Glibc Internals */ +void SLAMP___assert_fail(const char * assertion, const char * file, unsigned int line, const char * function){} +const unsigned short int **SLAMP___ctype_b_loc(void){} +int SLAMP__IO_getc(_IO_FILE * __fp){} +int SLAMP__IO_putc(int __c, _IO_FILE *__fp){} +int* SLAMP___errno_location (void){} + +int SLAMP___fxstat (int __ver, int __fildes, struct stat *__stat_buf){} +int SLAMP___xstat (int __ver, __const char *__filename, struct stat *__stat_buf){} diff --git a/liberty/lib/SLAMP/SLAMPnng/slamp_hooks.h b/liberty/lib/SLAMP/SLAMPnng/slamp_hooks.h new file mode 100644 index 00000000..df546117 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPnng/slamp_hooks.h @@ -0,0 +1,371 @@ +#ifndef SLAMPLIB_HOOKS_SLAMP_HOOKS_H +#define SLAMPLIB_HOOKS_SLAMP_HOOKS_H + +// FIXME: inline tweak actually make things worse in sequential and better with +// 16x, so turn it on at all time before understanding why +#define ATTRIBUTE(x) __attribute__((x)) +// #ifdef ITO_ENABLE +// // #define ATTRIBUTE(x) +// #define ATTRIBUTE(x) __attribute__((x)) +// #else +// #define ATTRIBUTE(x) +// #endif + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +void SLAMP_dbggv(int id); +void SLAMP_dbggvstr(char* str); + +// SLAMP measure functions +void SLAMP_measure_init(); +void SLAMP_measure_fini(); +void SLAMP_measure_load(uint32_t id, uint64_t size); +void SLAMP_measure_store(uint32_t id, uint64_t size); +static void* SLAMP_measure_malloc_hook(size_t size, const void *caller); +static void SLAMP_measure_free_hook(void *ptr, const void *caller); + +void SLAMP_init(uint32_t fn_id, uint32_t loop_id); +void SLAMP_fini(const char* filename); + +void SLAMP_allocated(uint64_t addr); +void SLAMP_init_global_vars(const char *name, uint64_t addr, size_t size); +void SLAMP_main_entry(uint32_t argc, char** argv, char** env); + +void SLAMP_enter_fcn(uint32_t id); +void SLAMP_exit_fcn(uint32_t id); +void SLAMP_enter_loop(uint32_t id); +void SLAMP_exit_loop(uint32_t id); +void SLAMP_loop_iter_ctx(uint32_t id); +void SLAMP_loop_invocation(); +void SLAMP_loop_iteration(); +void SLAMP_loop_exit(); + +void SLAMP_report_base_pointer_arg(uint32_t, uint32_t, void *ptr); +void SLAMP_report_base_pointer_inst(uint32_t, void *ptr); +void SLAMP_callback_stack_alloca(uint64_t, uint64_t, uint32_t, uint64_t); +void SLAMP_callback_stack_free(void); + +void SLAMP_ext_push(const uint32_t instr); +void SLAMP_ext_pop(); + +void SLAMP_push(const uint32_t instr); +void SLAMP_pop(); + +void SLAMP_load1(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_load2(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_load4(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_load8(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_loadn(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, size_t n) ATTRIBUTE(always_inline);; + +void SLAMP_load1_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_load2_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_load4_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_load8_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_loadn_ext(const uint64_t addr, const uint32_t bare_instr, size_t n) ATTRIBUTE(always_inline);; + +void SLAMP_store1(uint32_t instr, const uint64_t addr) ATTRIBUTE(always_inline); +void SLAMP_store2(uint32_t instr, const uint64_t addr) ATTRIBUTE(always_inline); +void SLAMP_store4(uint32_t instr, const uint64_t addr) ATTRIBUTE(always_inline); +void SLAMP_store8(uint32_t instr, const uint64_t addr) ATTRIBUTE(always_inline); +void SLAMP_storen(uint32_t instr, const uint64_t addr, size_t n) ATTRIBUTE(always_inline);; + +void SLAMP_store1_ext(const uint64_t addr, const uint32_t bare_inst) ATTRIBUTE(always_inline); +void SLAMP_store2_ext(const uint64_t addr, const uint32_t bare_inst) ATTRIBUTE(always_inline); +void SLAMP_store4_ext(const uint64_t addr, const uint32_t bare_inst) ATTRIBUTE(always_inline); +void SLAMP_store8_ext(const uint64_t addr, const uint32_t bare_inst) ATTRIBUTE(always_inline); +void SLAMP_storen_ext(const uint64_t addr, const uint32_t bare_inst, size_t n) ATTRIBUTE(always_inline);; + +/* wrappers */ +static void* SLAMP_malloc_hook(size_t size, const void *caller); +static void SLAMP_free_hook(void *ptr, const void *caller); +static void* SLAMP_memalign_hook(size_t alignment, size_t size, const void *caller); +void* SLAMP_malloc(size_t size, uint32_t instr=0, size_t alignment=16); + +void* SLAMP_calloc(size_t nelem, size_t elsize); +void* SLAMP_realloc(void* ptr, size_t size); +void* SLAMP__Znam(size_t size); +void* SLAMP__Znwm(size_t size); + +char* SLAMP_strdup(const char *s1); +char* SLAMP___strdup(const char *s1); +void SLAMP_free(void* ptr); +void SLAMP_cfree(void* ptr); +void SLAMP__ZdlPv(void* ptr); +void SLAMP__ZdaPv(void* ptr); +int SLAMP_brk(void *end_data_segment); +void* SLAMP_sbrk(intptr_t increment); + +/* llvm memory intrinsics */ +void SLAMP_llvm_memcpy_p0i8_p0i8_i32(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint32_t sizeBytes); +void SLAMP_llvm_memcpy_p0i8_p0i8_i64(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint64_t sizeBytes); +void SLAMP_llvm_memmove_p0i8_p0i8_i32(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint32_t sizeBytes); +void SLAMP_llvm_memmove_p0i8_p0i8_i64(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint64_t sizeBytes); +void SLAMP_llvm_memset_p0i8_i32(const uint8_t* dstAddr, const uint32_t len); +void SLAMP_llvm_memset_p0i8_i64(const uint8_t* dstAddr, const uint64_t len); + +// void SLAMP_llvm_lifetime_start_p0i8(uint64_t size, uint8_t* ptr); +// void SLAMP_llvm_lifetime_end_p0i8(uint64_t size, uint8_t* ptr); + +/* String functions */ +size_t SLAMP_strlen(const char *str); +char* SLAMP_strchr(char *s, int c); +char* SLAMP_strrchr(char *s, int c); +int SLAMP_strcmp(const char *s1, const char *s2); +int SLAMP_strncmp(const char *s1, const char *s2, size_t n); +char* SLAMP_strcpy(char *dest, const char *src); +char* SLAMP_strncpy(char *dest, const char *src, size_t n); +char* SLAMP_strcat(char *s1, const char *s2); +char* SLAMP_strncat(char *s1, const char *s2, size_t n); +char* SLAMP_strstr(char *s1, char *s2); +size_t SLAMP_strspn(const char *s1, const char *s2); +size_t SLAMP_strcspn(const char *s1, const char *s2); +char* SLAMP_strtok(char *s, const char *delim); +double SLAMP_strtod(const char *nptr, char **endptr); +long int SLAMP_strtol(const char *nptr, char **endptr, int base); +char* SLAMP_strpbrk(char *s1, char *s2); + +/* Mem* and b* functions */ +void *SLAMP_memset (void *dest, int c, size_t n); +void *SLAMP_memcpy (void *dest, const void *src, size_t n); +void *SLAMP___builtin_memcpy (void *dest, const void *src, size_t n); +void *SLAMP_memmove (void *dest, const void *src, size_t n); +int SLAMP_memcmp(const void *s1, const void *s2, size_t n); +void* SLAMP_memchr(void* ptr, int value, size_t num); +void* SLAMP___rawmemchr(void* ptr, int value); + +void SLAMP_bzero(void *s, size_t n); +void SLAMP_bcopy(const void *s1, void *s2, size_t n); + +/* IO */ +ssize_t SLAMP_read(int fd, void *buf, size_t count); +int SLAMP_open(const char *pathname, int flags, mode_t mode); +int SLAMP_close(int fd); +ssize_t SLAMP_write(int fd, const void *buf, size_t count); +off_t SLAMP_lseek(int fildes, off_t offset, int whence); + +FILE * SLAMP_fopen(const char *path, const char *mode); +FILE * SLAMP_fopen64(const char *path, const char *mode); +FILE * SLAMP_freopen(const char *path, const char *mode, FILE* stream); +int SLAMP_fflush(FILE *stream); +int SLAMP_fclose(FILE *stream); +int SLAMP_ferror(FILE *stream); +int SLAMP_feof(FILE *stream); +long SLAMP_ftell(FILE *stream); +size_t SLAMP_fread(void * ptr, size_t size, size_t nitems, FILE *stream); +size_t SLAMP_fwrite(const void *ptr, size_t size, size_t nitems, FILE *stream); +int SLAMP_fseek(FILE *stream, long offset, int whence); +void SLAMP_rewind(FILE *stream); + +int SLAMP_fgetc(FILE *stream); +int SLAMP_fputc(int c, FILE *stream); +char * SLAMP_fgets(char *s, int n, FILE *stream); +int SLAMP_fputs(const char *s, FILE *stream); + +int SLAMP_ungetc(int c, FILE *stream); +int SLAMP_putchar(int c); +int SLAMP_getchar(void); + +int SLAMP_fileno(FILE *stream); +char * SLAMP_gets(char *s); +int SLAMP_puts(const char *s); + +int SLAMP_select(int n, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, struct timeval *timeout); +int SLAMP_remove(const char *path); + +void SLAMP_setbuf(FILE * stream, char * buf); +void SLAMP_setvbuf(FILE * stream, char * buf, int mode, size_t size); +char * SLAMP_tmpnam(char *s); +FILE* SLAMP_tmpfile(void); +char * SLAMP_ttyname(int fildes); + +FILE * SLAMP_fdopen(int fildes, const char *mode); +void SLAMP_clearerr(FILE *stream); + +int SLAMP_truncate(const char *path, off_t length); +int SLAMP_ftruncate(int fildes, off_t length); + +int SLAMP_dup(int oldfd); +int SLAMP_dup2(int oldfd, int newfd); +int SLAMP_pipe(int filedes[2]); + +int SLAMP_chmod(const char *path, mode_t mode); +int SLAMP_fchmod(int fildes, mode_t mode); +int SLAMP_fchown(int fd, uid_t owner, gid_t group); +int SLAMP_access(const char *pathname, int mode); +long SLAMP_pathconf(char *path, int name); +int SLAMP_mkdir(const char *pathname, mode_t mode); +int SLAMP_rmdir(const char *pathname); +mode_t SLAMP_umask(mode_t mask); +int SLAMP_fcntl(int fd, int cmd, struct flock *lock); + +DIR* SLAMP_opendir(const char* name); +struct dirent* SLAMP_readdir(DIR *dirp); +struct dirent64* SLAMP_readdir64(DIR *dirp); +int SLAMP_closedir(DIR* dirp); + +/* Printf */ +int SLAMP_printf(const char *format, ...); +int SLAMP_fprintf(FILE *stream, const char *format, ...); +int SLAMP_sprintf(char *str, const char *format, ...); +int SLAMP_snprintf(char *str, size_t size, const char *format, ...); + +int SLAMP_vprintf(const char *format, va_list ap); +int SLAMP_vfprintf(FILE *stream, const char *format, va_list ap); +int SLAMP_vsprintf(char *str, const char *format, va_list ap); +int SLAMP_vsnprintf(char *str, size_t size, const char *format, va_list ap); + +/* Scanf */ +int SLAMP_fscanf(FILE *stream, const char *format, ... ); +int SLAMP_scanf(const char *format, ... ); +int SLAMP_sscanf(const char *s, const char *format, ... ); +int SLAMP___isoc99_sscanf(const char *s, const char *format, ... ); + +int SLAMP_vfscanf(FILE *stream, const char *format, va_list ap); +int SLAMP_vscanf(const char *format, va_list ap); +int SLAMP_vsscanf(const char *s, const char *format, va_list ap); + +/* Time */ +time_t SLAMP_time(time_t *t); +struct tm *SLAMP_localtime(const time_t *timer); +struct lconv* SLAMP_localeconv(); +struct tm *SLAMP_gmtime(const time_t *timer); +int SLAMP_gettimeofday(struct timeval *tv, struct timezone *tz); + +/* Math */ +double SLAMP_ldexp(double x, int exp); +float SLAMP_ldexpf(float x, int exp); +long double SLAMP_ldexpl(long double x, int exp); +double SLAMP_log10(double x); +float SLAMP_log10f(float x); +long double SLAMP_log10l(long double x); +double SLAMP_log(double x); +float SLAMP_logf(float x); +long double SLAMP_logl(long double x); + +double SLAMP_exp(double x); +float SLAMP_expf(float x); +long double SLAMP_expl(long double x); + +double SLAMP_cos(double x); +float SLAMP_cosf(float x); +long double SLAMP_cosl(long double x); +double SLAMP_sin(double x); +double SLAMP_tan(double x); +float SLAMP_sinf(float x); +long double SLAMP_sinl(long double x); + +double SLAMP_atan(double x); +float SLAMP_atanf(float x); +long double SLAMP_atanl(long double x); + +double SLAMP_floor(double x); +float SLAMP_floorf(float x); +long double SLAMP_floorl(long double x); +double SLAMP_ceil(double x); +float SLAMP_ceilf(float x); +long double SLAMP_ceill(long double x); + +double SLAMP_atan2(double y, double x); +float SLAMP_atan2f(float y, float x); +long double SLAMP_atan2l(long double y, long double x); + +double SLAMP_sqrt(double x); +float SLAMP_sqrtf(float x); +long double SLAMP_sqrtl(long double x); + +double SLAMP_pow(double x, double y); +float SLAMP_powf(float x, float y); +long double SLAMP_powl(long double x, long double y); + +double SLAMP_fabs(double x); +float SLAMP_fabsf(float x); +long double SLAMP_fabsl(long double x); + +double SLAMP_modf(double x, double *iptr); +float SLAMP_modff(float x, float *iptr); +long double SLAMP_modfl(long double x, long double *iptr); + +double SLAMP_fmod(double x, double y); + +double SLAMP_frexp(double num, int *exp); +float SLAMP_frexpf(float num, int *exp); +long double SLAMP_frexpl(long double num, int *exp); + +int SLAMP_isnan(); + +/* MISC */ +char *SLAMP_getenv(const char *name); +int SLAMP_putenv(char* string); +char *SLAMP_getcwd(char *buf, size_t size); +char* SLAMP_strerror(int errnum); +void SLAMP_exit(int status); +void SLAMP__exit(int status); +int SLAMP_link(const char *oldpath, const char *newpath); +int SLAMP_unlink(const char *pathname); +int SLAMP_isatty(int desc); +int SLAMP_setuid(uid_t uid); +uid_t SLAMP_getuid(void); +uid_t SLAMP_geteuid(void); +int SLAMP_setgid(gid_t gid); +gid_t SLAMP_getgid(void); +gid_t SLAMP_getegid(void); +pid_t SLAMP_getpid(void); +int SLAMP_chdir(const char *path); +int SLAMP_execl(const char *path, const char *arg0, ... /*, (char *)0 */); +int SLAMP_execv(const char *path, char *const argv[]); +int SLAMP_execvp(const char *file, char *const argv[]); +int SLAMP_kill(pid_t pid, int sig); +pid_t SLAMP_fork(void); +sighandler_t SLAMP___sysv_signal(int signum, sighandler_t handler); +pid_t SLAMP_waitpid(pid_t pid, int* status, int options); +void SLAMP_qsort(void* base, size_t nmemb, size_t size, int(*compar)(const void *, const void *)); +int SLAMP_ioctl(int d, int request, ...); +unsigned int SLAMP_sleep(unsigned int seconds); +char* SLAMP_gcvt(double number, size_t ndigit, char* buf); +char* SLAMP_nl_langinfo(nl_item item); + +/* Compiler/Glibc Internals */ +void SLAMP___assert_fail(const char * assertion, const char * file, unsigned int line, const char * function); +const unsigned short int **SLAMP___ctype_b_loc(void); +int SLAMP__IO_getc(_IO_FILE * __fp); +int SLAMP__IO_putc(int __c, _IO_FILE *__fp); +int* SLAMP___errno_location (void); + +int SLAMP___fxstat (int __ver, int __fildes, struct stat *__stat_buf); +int SLAMP___xstat (int __ver, __const char *__filename, struct stat *__stat_buf); + +#ifdef __cplusplus +} +#endif + +#endif /* SLAMP_HOOKS_H */ From 33075566a69073183b8866fe58b75e758c5a9e3f Mon Sep 17 00:00:00 2001 From: Ziyang Xu Date: Wed, 5 Oct 2022 20:00:36 -0400 Subject: [PATCH 11/97] update different queue implementation --- liberty/lib/SLAMP/CMakeLists.txt | 2 +- .../{BoostSend.cpp => AtomicQueueSend.cpp} | 43 +++++------ liberty/lib/SLAMP/SLAMPboost/BoostSend.cpp | 74 ++++++++++--------- 3 files changed, 61 insertions(+), 58 deletions(-) rename liberty/lib/SLAMP/SLAMPatomicq/{BoostSend.cpp => AtomicQueueSend.cpp} (91%) diff --git a/liberty/lib/SLAMP/CMakeLists.txt b/liberty/lib/SLAMP/CMakeLists.txt index 3c6bfeb2..91167f99 100644 --- a/liberty/lib/SLAMP/CMakeLists.txt +++ b/liberty/lib/SLAMP/CMakeLists.txt @@ -21,4 +21,4 @@ add_llvm_library(${PassName} SHARED ${SRCS}) # This is to generate libxxx.so add_subdirectory(SLAMPlib/hooks) add_subdirectory(SLAMPnng) add_subdirectory(SLAMPboost) -# add_subdirectory(SLAMPatomicq) +add_subdirectory(SLAMPatomicq) diff --git a/liberty/lib/SLAMP/SLAMPatomicq/BoostSend.cpp b/liberty/lib/SLAMP/SLAMPatomicq/AtomicQueueSend.cpp similarity index 91% rename from liberty/lib/SLAMP/SLAMPatomicq/BoostSend.cpp rename to liberty/lib/SLAMP/SLAMPatomicq/AtomicQueueSend.cpp index 4e8f48c9..2b684d73 100644 --- a/liberty/lib/SLAMP/SLAMPatomicq/BoostSend.cpp +++ b/liberty/lib/SLAMP/SLAMPatomicq/AtomicQueueSend.cpp @@ -11,46 +11,40 @@ namespace bip = boost::interprocess; namespace shm { - typedef bip::allocator char_alloc; - typedef bip::basic_string, char_alloc > shared_string; - typedef boost::lockfree::spsc_queue< - shared_string, - boost::lockfree::capacity<65536> - > ring_buffer; + using Element = char; // Queue element type. + using char_alloc = bip::allocator; + Element constexpr NIL = static_cast(-1); // Atomic elements require a special value that cannot be pushed/popped. + // using Queue = atomic_queue::AtomicQueueB, NIL>; // Use heap-allocated buffer. + // using Queue = atomic_queue::AtomicQueueB; // Use heap-allocated buffer. + using Queue = atomic_queue::AtomicQueueB, NIL, false, false, true>; } #include -using Element = char; // Queue element type. -Element constexpr NIL = static_cast(-1); // Atomic elements require a special value that cannot be pushed/popped. -using Queue = atomic_queue::AtomicQueueB; // Use heap-allocated buffer. // create segment and corresponding allocator bip::managed_shared_memory *segment; shm::char_alloc *char_alloc; -// Ringbuffer fully constructed in shared memory. The element strings are -// also allocated from the same shared memory segment. This vector can be -// safely accessed from other processes. -shm::ring_buffer *queue; -Queue *a_queue; +shm::Queue *queue; unsigned long counter4 = 0; unsigned long counter8 = 0; void SLAMP_init(uint32_t fn_id, uint32_t loop_id) { segment = new bip::managed_shared_memory(bip::open_or_create, "MySharedMemory", 65536UL*1600); - char_alloc = new shm::char_alloc(segment->get_segment_manager()); - // queue = segment->find_or_construct("queue")(); - a_queue = new atomic_queue::AtomicQueueB(65536); + // auto a_queue = new atomic_queue::AtomicQueueB(65536); + auto q = static_cast(segment->find_or_construct("atomic_queue")[65536]()); + queue = new shm::Queue(65536, q); + queue->push((char)fn_id); // send a msg with "fn_id, loop_id" - char msg[100]; - sprintf(msg, "%d,%d", fn_id, loop_id); - a_queue->push(msg); - queue->push(shm::shared_string(msg, *char_alloc)); + // char msg[100]; + // sprintf(msg, "%d,%d", fn_id, loop_id); + // a_queue->push(msg); + // queue->push(shm::shared_string(msg, *char_alloc)); } void SLAMP_fini(const char* filename){ @@ -95,9 +89,10 @@ void SLAMP_load4(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, void SLAMP_load8(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value){ counter8++; - char msg[1] = {8}; - // // sprintf(msg, "load8,%d,%lu,%d,%lu", instr, addr, bare_instr, value); - queue->push(shm::shared_string(msg, *char_alloc)); + queue->push((char)instr); + // char msg[1] = {8}; + // // // sprintf(msg, "load8,%d,%lu,%d,%lu", instr, addr, bare_instr, value); + // queue->push(shm::shared_string(msg, *char_alloc)); } void SLAMP_loadn(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, size_t n){} diff --git a/liberty/lib/SLAMP/SLAMPboost/BoostSend.cpp b/liberty/lib/SLAMP/SLAMPboost/BoostSend.cpp index 451e224a..08e73b6a 100644 --- a/liberty/lib/SLAMP/SLAMPboost/BoostSend.cpp +++ b/liberty/lib/SLAMP/SLAMPboost/BoostSend.cpp @@ -14,10 +14,8 @@ namespace shm typedef bip::allocator char_alloc; typedef bip::basic_string, char_alloc > shared_string; - typedef boost::lockfree::spsc_queue< - shared_string, - boost::lockfree::capacity<65536> - > ring_buffer; + // using ring_buffer = boost::lockfree::spsc_queue>; + using ring_buffer = boost::lockfree::spsc_queue>; } #include @@ -46,7 +44,8 @@ void SLAMP_init(uint32_t fn_id, uint32_t loop_id) { // send a msg with "fn_id, loop_id" char msg[100]; sprintf(msg, "%d,%d", fn_id, loop_id); - queue->push(shm::shared_string(msg, *char_alloc)); + // queue->push(shm::shared_string(msg, *char_alloc)); + queue->push(fn_id); } void SLAMP_fini(const char* filename){ @@ -86,27 +85,32 @@ void SLAMP_load2(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, void SLAMP_load4(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value){ counter4++; - local_buffer[buffer_counter++] = 4; - local_buffer[buffer_counter++] = instr & 0xFF; + queue->push(4); + queue->push(instr); - // // sprintf(msg, "load8,%d,%lu,%d,%lu", instr, addr, bare_instr, value); - if (buffer_counter == LOCAL_BUFFER_SIZE) { - queue->push(shm::shared_string(local_buffer, LOCAL_BUFFER_SIZE, *char_alloc)); - buffer_counter = 0; - } + // local_buffer[buffer_counter++] = 4; + // local_buffer[buffer_counter++] = instr & 0xFF; + + // // // sprintf(msg, "load8,%d,%lu,%d,%lu", instr, addr, bare_instr, value); + // if (buffer_counter == LOCAL_BUFFER_SIZE) { + // queue->push(shm::shared_string(local_buffer, LOCAL_BUFFER_SIZE, *char_alloc)); + // buffer_counter = 0; + // } } void SLAMP_load8(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value){ counter8++; - local_buffer[buffer_counter++] = 8; - local_buffer[buffer_counter++] = instr & 0xFF; + queue->push(8); + queue->push(instr); + // local_buffer[buffer_counter++] = 8; + // local_buffer[buffer_counter++] = instr & 0xFF; - // // sprintf(msg, "load8,%d,%lu,%d,%lu", instr, addr, bare_instr, value); - if (buffer_counter == LOCAL_BUFFER_SIZE) { - queue->push(shm::shared_string(local_buffer, LOCAL_BUFFER_SIZE, *char_alloc)); - buffer_counter = 0; - } + // // // sprintf(msg, "load8,%d,%lu,%d,%lu", instr, addr, bare_instr, value); + // if (buffer_counter == LOCAL_BUFFER_SIZE) { + // queue->push(shm::shared_string(local_buffer, LOCAL_BUFFER_SIZE, *char_alloc)); + // buffer_counter = 0; + // } } void SLAMP_loadn(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, size_t n){} @@ -116,27 +120,31 @@ void SLAMP_load2_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t va void SLAMP_load4_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value){ counter4_ext++; - local_buffer[buffer_counter++] = 14; - local_buffer[buffer_counter++] = addr & 0xFF; + queue->push(14); + queue->push(addr & 0xFFFFFFFF); + // local_buffer[buffer_counter++] = 14; + // local_buffer[buffer_counter++] = addr & 0xFF; - // // sprintf(msg, "load8,%d,%lu,%d,%lu", instr, addr, bare_instr, value); - if (buffer_counter == LOCAL_BUFFER_SIZE) { - queue->push(shm::shared_string(local_buffer, LOCAL_BUFFER_SIZE, *char_alloc)); - buffer_counter = 0; - } + // // // sprintf(msg, "load8,%d,%lu,%d,%lu", instr, addr, bare_instr, value); + // if (buffer_counter == LOCAL_BUFFER_SIZE) { + // queue->push(shm::shared_string(local_buffer, LOCAL_BUFFER_SIZE, *char_alloc)); + // buffer_counter = 0; + // } } void SLAMP_load8_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value){ counter8_ext++; - local_buffer[buffer_counter++] = 18; - local_buffer[buffer_counter++] = addr & 0xFF; + queue->push(18); + queue->push(addr & 0xFFFFFFFF); + // local_buffer[buffer_counter++] = 18; + // local_buffer[buffer_counter++] = addr & 0xFF; - // // sprintf(msg, "load8,%d,%lu,%d,%lu", instr, addr, bare_instr, value); - if (buffer_counter == LOCAL_BUFFER_SIZE) { - queue->push(shm::shared_string(local_buffer, LOCAL_BUFFER_SIZE, *char_alloc)); - buffer_counter = 0; - } + // // // sprintf(msg, "load8,%d,%lu,%d,%lu", instr, addr, bare_instr, value); + // if (buffer_counter == LOCAL_BUFFER_SIZE) { + // queue->push(shm::shared_string(local_buffer, LOCAL_BUFFER_SIZE, *char_alloc)); + // buffer_counter = 0; + // } } void SLAMP_loadn_ext(const uint64_t addr, const uint32_t bare_instr, size_t n){} From d2d8e1df4a47421cf7301ed179c12ce1ed4d04e5 Mon Sep 17 00:00:00 2001 From: Ziyang Xu Date: Wed, 5 Oct 2022 20:00:59 -0400 Subject: [PATCH 12/97] add consumer code for each impl --- .../SLAMPatomicq/consumer/CMakeLists.txt | 18 ++++ .../SLAMP/SLAMPatomicq/consumer/consumer.cpp | 40 +++++++++ .../SLAMP/SLAMPboost/consumer/CMakeLists.txt | 17 ++++ .../SLAMP/SLAMPboost/consumer/consumer.cpp | 40 +++++++++ .../SLAMP/SLAMPnng/consumer/CMakeLists.txt | 28 +++++++ liberty/lib/SLAMP/SLAMPnng/consumer/raw.c | 82 +++++++++++++++++++ 6 files changed, 225 insertions(+) create mode 100644 liberty/lib/SLAMP/SLAMPatomicq/consumer/CMakeLists.txt create mode 100644 liberty/lib/SLAMP/SLAMPatomicq/consumer/consumer.cpp create mode 100644 liberty/lib/SLAMP/SLAMPboost/consumer/CMakeLists.txt create mode 100644 liberty/lib/SLAMP/SLAMPboost/consumer/consumer.cpp create mode 100644 liberty/lib/SLAMP/SLAMPnng/consumer/CMakeLists.txt create mode 100644 liberty/lib/SLAMP/SLAMPnng/consumer/raw.c diff --git a/liberty/lib/SLAMP/SLAMPatomicq/consumer/CMakeLists.txt b/liberty/lib/SLAMP/SLAMPatomicq/consumer/CMakeLists.txt new file mode 100644 index 00000000..65b55888 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPatomicq/consumer/CMakeLists.txt @@ -0,0 +1,18 @@ +cmake_minimum_required (VERSION 3.6.2 FATAL_ERROR) + +set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} "/u/ziyangx/test/boost/boost_1_80_0/install/lib/cmake") + +set(THREADS_PREFER_PTHREAD_FLAG ON) +find_package(Threads REQUIRED) + + +find_package(Boost 1.80.0 REQUIRED COMPONENTS system) +include_directories(${Boost_INCLUDE_DIRS} + /u/ziyangx/test/atomic_queue/atomic_queue/include) + +add_executable (consumer consumer.cpp) +target_link_libraries(consumer LINK_PUBLIC + ${Boost_LIBRARIES} + rt + Threads::Threads +) diff --git a/liberty/lib/SLAMP/SLAMPatomicq/consumer/consumer.cpp b/liberty/lib/SLAMP/SLAMPatomicq/consumer/consumer.cpp new file mode 100644 index 00000000..5cfca3fc --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPatomicq/consumer/consumer.cpp @@ -0,0 +1,40 @@ +#include // ring buffer + +#include +#include +#include + +#include "atomic_queue/atomic_queue.h" + +namespace bip = boost::interprocess; +namespace shm +{ + using Element = char; + Element constexpr NIL = static_cast(-1); + using Queue = atomic_queue::AtomicQueueB, NIL, false, false, true>; +} + +#include + +int main() +{ + // create segment and corresponding allocator + bip::managed_shared_memory segment(bip::open_or_create, "MySharedMemory", 104857600); + auto q = static_cast(segment.find_or_construct("atomic_queue")[65536]()); + std::cout << q << std::endl; + + shm::Queue *queue = new shm::Queue(65536, q); + + auto counter = 0; + // shm::shared_string v(char_alloc); + while (true) + { + if (shm::Element v = queue->pop()) { + counter++; + + if (counter % 1000000 == 0) { + std::cout << "counter: " << counter << std::endl; + } + } + } +} diff --git a/liberty/lib/SLAMP/SLAMPboost/consumer/CMakeLists.txt b/liberty/lib/SLAMP/SLAMPboost/consumer/CMakeLists.txt new file mode 100644 index 00000000..cdf39bea --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPboost/consumer/CMakeLists.txt @@ -0,0 +1,17 @@ +cmake_minimum_required (VERSION 3.6.2 FATAL_ERROR) + +set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} "/u/ziyangx/test/boost/boost_1_80_0/install/lib/cmake") + +set(THREADS_PREFER_PTHREAD_FLAG ON) +find_package(Threads REQUIRED) + + +find_package(Boost 1.80.0 REQUIRED COMPONENTS system) +include_directories(${Boost_INCLUDE_DIRS}) + +add_executable (consumer consumer.cpp) +target_link_libraries(consumer LINK_PUBLIC + ${Boost_LIBRARIES} + rt + Threads::Threads +) diff --git a/liberty/lib/SLAMP/SLAMPboost/consumer/consumer.cpp b/liberty/lib/SLAMP/SLAMPboost/consumer/consumer.cpp new file mode 100644 index 00000000..6df09897 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPboost/consumer/consumer.cpp @@ -0,0 +1,40 @@ +#include // ring buffer + +#include +#include +#include + +namespace bip = boost::interprocess; +namespace shm +{ + typedef bip::allocator char_alloc; + typedef bip::basic_string, char_alloc > shared_string; + + // shared_string, + using ring_buffer = boost::lockfree::spsc_queue>; +} + +#include + +int main() +{ + // create segment and corresponding allocator + bip::managed_shared_memory segment(bip::open_or_create, "MySharedMemory", 104857600); + shm::char_alloc char_alloc(segment.get_segment_manager()); + + shm::ring_buffer *queue = segment.find_or_construct("queue")(); + + auto counter = 0; + // shm::shared_string v(char_alloc); + unsigned int v; + while (true) + { + if (queue->pop(v)) { + counter++; + + if (counter % 1000000 == 0) { + std::cout << "counter: " << counter << std::endl; + } + } + } +} diff --git a/liberty/lib/SLAMP/SLAMPnng/consumer/CMakeLists.txt b/liberty/lib/SLAMP/SLAMPnng/consumer/CMakeLists.txt new file mode 100644 index 00000000..89331c08 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPnng/consumer/CMakeLists.txt @@ -0,0 +1,28 @@ +# +# Copyright 2018 Capitar IT Group BV +# Copyright 2018 Staysail Systems, Inc. +# +# This software is supplied under the terms of the MIT License, a +# copy of which should be located in the distribution where this +# file was obtained (LICENSE.txt). A copy of the license may also be +# found online at https://opensource.org/licenses/MIT. + +cmake_minimum_required (VERSION 2.8.7) + +# add CMAKE_PREFIX_PATH +set(CMAKE_C_COMPILER "clang") +set(CMAKE_CXX_COMPILER "clang++") + +set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} "/u/ziyangx/test/nng/nng-1.5.2/install/lib/cmake") +project(raw) + + +set(PARALLEL 128 CACHE STRING "Parallelism (min 4, max 1000)") + +set(THREADS_PREFER_PTHREAD_FLAG ON) +find_package(Threads REQUIRED) +find_package(nng CONFIG REQUIRED) + +add_executable(raw raw.c) +target_link_libraries(raw nng::nng) +target_compile_definitions(raw PRIVATE -DPARALLEL=${PARALLEL}) diff --git a/liberty/lib/SLAMP/SLAMPnng/consumer/raw.c b/liberty/lib/SLAMP/SLAMPnng/consumer/raw.c new file mode 100644 index 00000000..ba231cbb --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPnng/consumer/raw.c @@ -0,0 +1,82 @@ +#include +#include +#include +#include + +#include +#include +#include + +#define NODE0 "node0" +#define NODE1 "node1" + +void +fatal(const char *func, int rv) +{ + fprintf(stderr, "%s: %s\n", func, nng_strerror(rv)); + exit(1); +} + +int +node0(const char *url) +{ + nng_socket sock; + int rv; + + if ((rv = nng_pull0_open(&sock)) != 0) { + fatal("nng_pull0_open", rv); + } + if ((rv = nng_listen(sock, url, NULL, 0)) != 0) { + fatal("nng_listen", rv); + } + unsigned long counter = 0; + for (;;) { + char *buf = NULL; + size_t sz; + if ((rv = nng_recv(sock, &buf, &sz, NNG_FLAG_ALLOC)) != 0) { + fatal("nng_recv", rv); + } + nng_free(buf, sz); + counter++; + if (counter % 1000000 == 0) { + printf("node0: %lu\n", counter); + } + } +} + +int +node1(const char *url, char *msg) +{ + int sz_msg = strlen(msg) + 1; // '\0' too + nng_socket sock; + int rv; + int bytes; + + if ((rv = nng_push0_open(&sock)) != 0) { + fatal("nng_push0_open", rv); + } + if ((rv = nng_dial(sock, url, NULL, 0)) != 0) { + fatal("nng_dial", rv); + } + printf("NODE1: SENDING \"%s\"\n", msg); + if ((rv = nng_send(sock, msg, strlen(msg)+1, 0)) != 0) { + fatal("nng_send", rv); + } + sleep(1); // wait for messages to flush before shutting down + nng_close(sock); + return (0); +} + +int +main(int argc, char **argv) +{ + if ((argc > 1) && (strcmp(NODE0, argv[1]) == 0)) + return (node0(argv[2])); + + if ((argc > 2) && (strcmp(NODE1, argv[1]) == 0)) + return (node1(argv[2], argv[3])); + + fprintf(stderr, "Usage: pipeline %s|%s ...'\n", + NODE0, NODE1); + return (1); +} From 7f904927b37a415cd0a157d5271e2644ac208521 Mon Sep 17 00:00:00 2001 From: Ziyang Xu Date: Wed, 5 Oct 2022 20:01:11 -0400 Subject: [PATCH 13/97] add queue benchmark result on white --- liberty/lib/SLAMP/queue_benchmark.txt | 1100 +++++++++++++++++++++++++ 1 file changed, 1100 insertions(+) create mode 100644 liberty/lib/SLAMP/queue_benchmark.txt diff --git a/liberty/lib/SLAMP/queue_benchmark.txt b/liberty/lib/SLAMP/queue_benchmark.txt new file mode 100644 index 00000000..a1fbac6c --- /dev/null +++ b/liberty/lib/SLAMP/queue_benchmark.txt @@ -0,0 +1,1100 @@ +---- Running throughput benchmarks (higher is better) ---- + boost::lockfree::spsc_queue, 1,s: 107,997,734 msg/sec + boost::lockfree::spsc_queue, 1,i: 109,358,919 msg/sec + boost::lockfree::queue, 1,s: 4,048,308 msg/sec + boost::lockfree::queue, 1,i: 4,090,655 msg/sec + boost::lockfree::queue, 2,s: 3,582,047 msg/sec + boost::lockfree::queue, 2,i: 3,566,650 msg/sec + boost::lockfree::queue, 3,s: 3,451,386 msg/sec + boost::lockfree::queue, 3,i: 3,471,721 msg/sec + boost::lockfree::queue, 4,s: 3,265,429 msg/sec + boost::lockfree::queue, 4,i: 3,352,379 msg/sec + boost::lockfree::queue, 5,s: 3,168,874 msg/sec + boost::lockfree::queue, 5,i: 3,073,243 msg/sec + boost::lockfree::queue, 6,s: 2,981,684 msg/sec + boost::lockfree::queue, 6,i: 2,954,838 msg/sec + boost::lockfree::queue, 7,s: 2,848,137 msg/sec + boost::lockfree::queue, 7,i: 2,884,899 msg/sec + boost::lockfree::queue, 8,s: 1,315,964 msg/sec + boost::lockfree::queue, 8,i: 2,170,027 msg/sec + boost::lockfree::queue, 9,s: 1,212,512 msg/sec + boost::lockfree::queue, 9,i: 1,748,269 msg/sec + boost::lockfree::queue,10,s: 1,172,987 msg/sec + boost::lockfree::queue,10,i: 1,258,711 msg/sec + boost::lockfree::queue,11,s: 1,180,074 msg/sec + boost::lockfree::queue,11,i: 1,163,508 msg/sec + boost::lockfree::queue,12,s: 1,221,812 msg/sec + boost::lockfree::queue,12,i: 1,059,277 msg/sec + boost::lockfree::queue,13,s: 1,251,914 msg/sec + boost::lockfree::queue,13,i: 1,039,787 msg/sec + boost::lockfree::queue,14,s: 1,279,827 msg/sec + boost::lockfree::queue,14,i: 999,276 msg/sec + boost::lockfree::queue,15,s: 1,297,184 msg/sec + boost::lockfree::queue,15,i: 1,088,072 msg/sec + boost::lockfree::queue,16,s: 1,264,412 msg/sec + boost::lockfree::queue,16,i: 1,103,424 msg/sec + boost::lockfree::queue,17,s: 1,217,296 msg/sec + boost::lockfree::queue,17,i: 1,123,018 msg/sec + boost::lockfree::queue,18,s: 1,166,488 msg/sec + boost::lockfree::queue,18,i: 1,134,237 msg/sec + boost::lockfree::queue,19,s: 1,099,566 msg/sec + boost::lockfree::queue,19,i: 1,142,602 msg/sec + boost::lockfree::queue,20,s: 1,028,095 msg/sec + boost::lockfree::queue,20,i: 1,179,536 msg/sec + boost::lockfree::queue,21,s: 996,655 msg/sec + boost::lockfree::queue,21,i: 1,158,180 msg/sec + boost::lockfree::queue,22,s: 989,882 msg/sec + boost::lockfree::queue,22,i: 1,159,260 msg/sec + boost::lockfree::queue,23,s: 972,800 msg/sec + boost::lockfree::queue,23,i: 1,187,182 msg/sec + boost::lockfree::queue,24,s: 997,166 msg/sec + boost::lockfree::queue,24,i: 1,194,829 msg/sec + boost::lockfree::queue,25,s: 996,421 msg/sec + boost::lockfree::queue,25,i: 1,272,264 msg/sec + boost::lockfree::queue,26,s: 942,650 msg/sec + boost::lockfree::queue,26,i: 1,176,117 msg/sec + boost::lockfree::queue,27,s: 945,707 msg/sec + boost::lockfree::queue,27,i: 1,175,990 msg/sec + boost::lockfree::queue,28,s: 903,794 msg/sec + boost::lockfree::queue,28,i: 1,101,815 msg/sec + TicketSpinlock, 1,s: 3,347,824 msg/sec + TicketSpinlock, 1,i: 3,347,547 msg/sec + TicketSpinlock, 2,s: 2,967,686 msg/sec + TicketSpinlock, 2,i: 2,961,449 msg/sec + TicketSpinlock, 3,s: 2,520,313 msg/sec + TicketSpinlock, 3,i: 2,550,617 msg/sec + TicketSpinlock, 4,s: 2,210,679 msg/sec + TicketSpinlock, 4,i: 2,243,531 msg/sec + TicketSpinlock, 5,s: 2,049,747 msg/sec + TicketSpinlock, 5,i: 2,095,328 msg/sec + TicketSpinlock, 6,s: 1,875,237 msg/sec + TicketSpinlock, 6,i: 1,944,668 msg/sec + TicketSpinlock, 7,s: 1,753,580 msg/sec + TicketSpinlock, 7,i: 1,817,202 msg/sec + TicketSpinlock, 8,s: 1,277,627 msg/sec + TicketSpinlock, 8,i: 1,294,381 msg/sec + TicketSpinlock, 9,s: 1,222,606 msg/sec + TicketSpinlock, 9,i: 1,239,650 msg/sec + TicketSpinlock,10,s: 1,203,840 msg/sec + TicketSpinlock,10,i: 1,197,460 msg/sec + TicketSpinlock,11,s: 1,169,283 msg/sec + TicketSpinlock,11,i: 1,156,647 msg/sec + TicketSpinlock,12,s: 1,165,389 msg/sec + TicketSpinlock,12,i: 1,133,687 msg/sec + TicketSpinlock,13,s: 1,132,670 msg/sec + TicketSpinlock,13,i: 1,128,463 msg/sec + TicketSpinlock,14,s: 1,147,855 msg/sec + TicketSpinlock,14,i: 1,117,921 msg/sec + TicketSpinlock,15,s: 1,148,721 msg/sec + TicketSpinlock,15,i: 1,127,920 msg/sec + TicketSpinlock,16,s: 1,152,958 msg/sec + TicketSpinlock,16,i: 1,139,066 msg/sec + TicketSpinlock,17,s: 1,161,394 msg/sec + TicketSpinlock,17,i: 1,183,274 msg/sec + TicketSpinlock,18,s: 1,174,699 msg/sec + TicketSpinlock,18,i: 1,214,113 msg/sec + TicketSpinlock,19,s: 1,175,616 msg/sec + TicketSpinlock,19,i: 1,215,555 msg/sec + TicketSpinlock,20,s: 1,158,764 msg/sec + TicketSpinlock,20,i: 1,216,226 msg/sec + TicketSpinlock,21,s: 1,159,335 msg/sec + TicketSpinlock,21,i: 1,249,847 msg/sec + TicketSpinlock,22,s: 1,143,440 msg/sec + TicketSpinlock,22,i: 1,211,246 msg/sec + TicketSpinlock,23,s: 1,151,665 msg/sec + TicketSpinlock,23,i: 1,198,510 msg/sec + TicketSpinlock,24,s: 1,149,453 msg/sec + TicketSpinlock,24,i: 1,213,075 msg/sec + TicketSpinlock,25,s: 1,138,716 msg/sec + TicketSpinlock,25,i: 1,208,927 msg/sec + TicketSpinlock,26,s: 1,171,849 msg/sec + TicketSpinlock,26,i: 1,191,843 msg/sec + TicketSpinlock,27,s: 1,147,152 msg/sec + TicketSpinlock,27,i: 1,208,291 msg/sec + TicketSpinlock,28,s: 1,157,333 msg/sec + TicketSpinlock,28,i: 1,193,849 msg/sec + moodycamel::ReaderWriterQueue, 1,s: 225,144,209 msg/sec + moodycamel::ReaderWriterQueue, 1,i: 102,397,802 msg/sec + moodycamel::ConcurrentQueue, 1,s: 13,368,618 msg/sec + moodycamel::ConcurrentQueue, 1,i: 9,999,354 msg/sec + moodycamel::ConcurrentQueue, 2,s: 10,246,428 msg/sec + moodycamel::ConcurrentQueue, 2,i: 10,144,610 msg/sec + moodycamel::ConcurrentQueue, 3,s: 9,394,547 msg/sec + moodycamel::ConcurrentQueue, 3,i: 9,307,908 msg/sec + moodycamel::ConcurrentQueue, 4,s: 9,536,518 msg/sec + moodycamel::ConcurrentQueue, 4,i: 9,464,837 msg/sec + moodycamel::ConcurrentQueue, 5,s: 23,129,485 msg/sec + moodycamel::ConcurrentQueue, 5,i: 22,768,199 msg/sec + moodycamel::ConcurrentQueue, 6,s: 9,474,407 msg/sec + moodycamel::ConcurrentQueue, 6,i: 9,484,694 msg/sec + moodycamel::ConcurrentQueue, 7,s: 9,404,419 msg/sec + moodycamel::ConcurrentQueue, 7,i: 18,979,667 msg/sec + moodycamel::ConcurrentQueue, 8,s: 5,287,332 msg/sec + moodycamel::ConcurrentQueue, 8,i: 6,403,588 msg/sec + moodycamel::ConcurrentQueue, 9,s: 4,810,655 msg/sec + moodycamel::ConcurrentQueue, 9,i: 5,410,923 msg/sec + moodycamel::ConcurrentQueue,10,s: 11,319,209 msg/sec + moodycamel::ConcurrentQueue,10,i: 5,000,012 msg/sec + moodycamel::ConcurrentQueue,11,s: 9,739,263 msg/sec + moodycamel::ConcurrentQueue,11,i: 13,600,492 msg/sec + moodycamel::ConcurrentQueue,12,s: 12,174,409 msg/sec + moodycamel::ConcurrentQueue,12,i: 4,762,923 msg/sec + moodycamel::ConcurrentQueue,13,s: 13,138,679 msg/sec + moodycamel::ConcurrentQueue,13,i: 10,438,944 msg/sec + moodycamel::ConcurrentQueue,14,s: 9,505,382 msg/sec + moodycamel::ConcurrentQueue,14,i: 14,061,583 msg/sec + moodycamel::ConcurrentQueue,15,s: 22,268,559 msg/sec + moodycamel::ConcurrentQueue,15,i: 27,396,194 msg/sec + moodycamel::ConcurrentQueue,16,s: 39,867,824 msg/sec + moodycamel::ConcurrentQueue,16,i: 9,987,406 msg/sec + moodycamel::ConcurrentQueue,17,s: 4,815,220 msg/sec + moodycamel::ConcurrentQueue,17,i: 5,323,513 msg/sec + moodycamel::ConcurrentQueue,18,s: 31,809,138 msg/sec + moodycamel::ConcurrentQueue,18,i: 37,599,636 msg/sec + moodycamel::ConcurrentQueue,19,s: 4,641,828 msg/sec + moodycamel::ConcurrentQueue,19,i: 25,964,670 msg/sec + moodycamel::ConcurrentQueue,20,s: 4,699,633 msg/sec + moodycamel::ConcurrentQueue,20,i: 11,341,493 msg/sec + moodycamel::ConcurrentQueue,21,s: 9,199,439 msg/sec + moodycamel::ConcurrentQueue,21,i: 25,546,446 msg/sec + moodycamel::ConcurrentQueue,22,s: 40,199,001 msg/sec + moodycamel::ConcurrentQueue,22,i: 14,043,025 msg/sec + moodycamel::ConcurrentQueue,23,s: 16,944,418 msg/sec + moodycamel::ConcurrentQueue,23,i: 43,708,787 msg/sec + moodycamel::ConcurrentQueue,24,s: 4,560,747 msg/sec + moodycamel::ConcurrentQueue,24,i: 6,968,251 msg/sec + moodycamel::ConcurrentQueue,25,s: 12,175,776 msg/sec + moodycamel::ConcurrentQueue,25,i: 7,134,059 msg/sec + moodycamel::ConcurrentQueue,26,s: 33,074,197 msg/sec + moodycamel::ConcurrentQueue,26,i: 16,678,528 msg/sec + moodycamel::ConcurrentQueue,27,s: 48,724,662 msg/sec + moodycamel::ConcurrentQueue,27,i: 7,558,508 msg/sec + moodycamel::ConcurrentQueue,28,s: 4,540,667 msg/sec + moodycamel::ConcurrentQueue,28,i: 21,423,380 msg/sec + pthread_spinlock, 1,s: 1,388,219 msg/sec + pthread_spinlock, 1,i: 1,590,798 msg/sec + pthread_spinlock, 2,s: 4,061,346 msg/sec + pthread_spinlock, 2,i: 4,111,464 msg/sec + pthread_spinlock, 3,s: 3,707,084 msg/sec + pthread_spinlock, 3,i: 3,657,961 msg/sec + pthread_spinlock, 4,s: 3,356,248 msg/sec + pthread_spinlock, 4,i: 3,411,467 msg/sec + pthread_spinlock, 5,s: 2,922,106 msg/sec + pthread_spinlock, 5,i: 3,012,059 msg/sec + pthread_spinlock, 6,s: 2,814,872 msg/sec + pthread_spinlock, 6,i: 2,807,978 msg/sec + pthread_spinlock, 7,s: 2,277,902 msg/sec + pthread_spinlock, 7,i: 2,424,606 msg/sec +pthread_spinlock: producers: 8: consumer 1 received too few messages: 0.03% of expected. + pthread_spinlock, 8,s: 1,713,429 msg/sec +pthread_spinlock: producers: 8: consumer 2 received too few messages: 0.00% of expected. + pthread_spinlock, 8,i: 2,099,709 msg/sec +pthread_spinlock: producers: 9: consumer 2 received too few messages: 0.08% of expected. +pthread_spinlock: producers: 9: consumer 0 received too few messages: 0.04% of expected. + pthread_spinlock, 9,s: 1,702,071 msg/sec +pthread_spinlock: producers: 9: consumer 1 received too few messages: 0.01% of expected. + pthread_spinlock, 9,i: 2,129,815 msg/sec +pthread_spinlock: producers: 10: consumer 1 received too few messages: 0.06% of expected. +pthread_spinlock: producers: 10: consumer 3 received too few messages: 0.03% of expected. +pthread_spinlock: producers: 10: consumer 0 received too few messages: 0.04% of expected. + pthread_spinlock,10,s: 1,897,704 msg/sec +pthread_spinlock: producers: 10: consumer 5 received too few messages: 0.01% of expected. +pthread_spinlock: producers: 10: consumer 6 received too few messages: 0.03% of expected. + pthread_spinlock,10,i: 2,136,486 msg/sec +pthread_spinlock: producers: 11: consumer 0 received too few messages: 0.08% of expected. + pthread_spinlock,11,s: 1,917,518 msg/sec +pthread_spinlock: producers: 11: consumer 4 received too few messages: 0.02% of expected. + pthread_spinlock,11,i: 2,039,850 msg/sec + pthread_spinlock,12,s: 1,878,326 msg/sec +pthread_spinlock: producers: 12: consumer 1 received too few messages: 0.02% of expected. + pthread_spinlock,12,i: 2,017,794 msg/sec +pthread_spinlock: producers: 13: consumer 0 received too few messages: 0.10% of expected. +pthread_spinlock: producers: 13: consumer 0 received too few messages: 0.03% of expected. + pthread_spinlock,13,s: 1,617,323 msg/sec +pthread_spinlock: producers: 13: consumer 5 received too few messages: 0.02% of expected. + pthread_spinlock,13,i: 2,007,359 msg/sec + pthread_spinlock,14,s: 1,730,125 msg/sec +pthread_spinlock: producers: 14: consumer 3 received too few messages: 0.05% of expected. +pthread_spinlock: producers: 14: consumer 4 received too few messages: 0.10% of expected. + pthread_spinlock,14,i: 1,818,811 msg/sec +pthread_spinlock: producers: 15: consumer 13 received too few messages: 0.06% of expected. +pthread_spinlock: producers: 15: consumer 14 received too few messages: 0.08% of expected. + pthread_spinlock,15,s: 1,788,437 msg/sec + pthread_spinlock,15,i: 1,896,966 msg/sec +pthread_spinlock: producers: 16: consumer 12 received too few messages: 0.09% of expected. + pthread_spinlock,16,s: 1,778,092 msg/sec + pthread_spinlock,16,i: 1,937,498 msg/sec +pthread_spinlock: producers: 17: consumer 11 received too few messages: 0.09% of expected. +pthread_spinlock: producers: 17: consumer 15 received too few messages: 0.09% of expected. + pthread_spinlock,17,s: 1,805,306 msg/sec + pthread_spinlock,17,i: 1,967,797 msg/sec +pthread_spinlock: producers: 18: consumer 16 received too few messages: 0.06% of expected. +pthread_spinlock: producers: 18: consumer 17 received too few messages: 0.09% of expected. +pthread_spinlock: producers: 18: consumer 15 received too few messages: 0.07% of expected. +pthread_spinlock: producers: 18: consumer 14 received too few messages: 0.06% of expected. + pthread_spinlock,18,s: 1,897,631 msg/sec + pthread_spinlock,18,i: 1,971,578 msg/sec +pthread_spinlock: producers: 19: consumer 16 received too few messages: 0.10% of expected. +pthread_spinlock: producers: 19: consumer 10 received too few messages: 0.10% of expected. +pthread_spinlock: producers: 19: consumer 11 received too few messages: 0.09% of expected. +pthread_spinlock: producers: 19: consumer 16 received too few messages: 0.10% of expected. +pthread_spinlock: producers: 19: consumer 13 received too few messages: 0.08% of expected. + pthread_spinlock,19,s: 1,954,765 msg/sec +pthread_spinlock: producers: 19: consumer 6 received too few messages: 0.08% of expected. + pthread_spinlock,19,i: 2,002,153 msg/sec +pthread_spinlock: producers: 20: consumer 12 received too few messages: 0.02% of expected. + pthread_spinlock,20,s: 1,925,179 msg/sec +pthread_spinlock: producers: 20: consumer 2 received too few messages: 0.03% of expected. + pthread_spinlock,20,i: 1,988,383 msg/sec + pthread_spinlock,21,s: 2,008,688 msg/sec +pthread_spinlock: producers: 21: consumer 7 received too few messages: 0.09% of expected. + pthread_spinlock,21,i: 1,980,480 msg/sec +pthread_spinlock: producers: 22: consumer 7 received too few messages: 0.04% of expected. + pthread_spinlock,22,s: 1,969,265 msg/sec +pthread_spinlock: producers: 22: consumer 19 received too few messages: 0.03% of expected. +pthread_spinlock: producers: 22: consumer 4 received too few messages: 0.03% of expected. + pthread_spinlock,22,i: 1,946,497 msg/sec + pthread_spinlock,23,s: 1,968,276 msg/sec + pthread_spinlock,23,i: 1,960,031 msg/sec +pthread_spinlock: producers: 24: consumer 12 received too few messages: 0.06% of expected. + pthread_spinlock,24,s: 1,865,849 msg/sec +pthread_spinlock: producers: 24: consumer 18 received too few messages: 0.04% of expected. +pthread_spinlock: producers: 24: consumer 4 received too few messages: 0.03% of expected. +pthread_spinlock: producers: 24: consumer 5 received too few messages: 0.04% of expected. + pthread_spinlock,24,i: 1,933,067 msg/sec +pthread_spinlock: producers: 25: consumer 4 received too few messages: 0.08% of expected. +pthread_spinlock: producers: 25: consumer 7 received too few messages: 0.09% of expected. + pthread_spinlock,25,s: 1,796,967 msg/sec +pthread_spinlock: producers: 25: consumer 14 received too few messages: 0.02% of expected. + pthread_spinlock,25,i: 1,906,613 msg/sec + pthread_spinlock,26,s: 1,715,120 msg/sec +pthread_spinlock: producers: 26: consumer 3 received too few messages: 0.09% of expected. + pthread_spinlock,26,i: 1,744,655 msg/sec + pthread_spinlock,27,s: 1,396,728 msg/sec + pthread_spinlock,27,i: 1,460,071 msg/sec + pthread_spinlock,28,s: 1,397,343 msg/sec + pthread_spinlock,28,i: 1,453,740 msg/sec + std::mutex, 1,s: 4,831,124 msg/sec + std::mutex, 1,i: 4,686,462 msg/sec + std::mutex, 2,s: 5,043,814 msg/sec + std::mutex, 2,i: 5,020,252 msg/sec + std::mutex, 3,s: 3,908,694 msg/sec + std::mutex, 3,i: 3,895,097 msg/sec + std::mutex, 4,s: 3,554,815 msg/sec + std::mutex, 4,i: 3,528,514 msg/sec + std::mutex, 5,s: 3,843,170 msg/sec + std::mutex, 5,i: 3,771,042 msg/sec + std::mutex, 6,s: 3,767,452 msg/sec + std::mutex, 6,i: 3,667,091 msg/sec + std::mutex, 7,s: 3,610,762 msg/sec + std::mutex, 7,i: 3,477,284 msg/sec + std::mutex, 8,s: 2,197,057 msg/sec + std::mutex, 8,i: 2,246,655 msg/sec + std::mutex, 9,s: 1,893,438 msg/sec + std::mutex, 9,i: 1,950,847 msg/sec + std::mutex,10,s: 1,787,108 msg/sec + std::mutex,10,i: 1,865,565 msg/sec + std::mutex,11,s: 1,741,380 msg/sec + std::mutex,11,i: 1,807,413 msg/sec + std::mutex,12,s: 1,749,876 msg/sec + std::mutex,12,i: 1,804,944 msg/sec + std::mutex,13,s: 1,759,121 msg/sec + std::mutex,13,i: 1,769,137 msg/sec + std::mutex,14,s: 1,779,333 msg/sec + std::mutex,14,i: 1,776,040 msg/sec + std::mutex,15,s: 1,734,586 msg/sec + std::mutex,15,i: 1,718,733 msg/sec + std::mutex,16,s: 1,709,260 msg/sec + std::mutex,16,i: 1,714,574 msg/sec + std::mutex,17,s: 1,694,088 msg/sec + std::mutex,17,i: 1,702,783 msg/sec + std::mutex,18,s: 1,700,218 msg/sec + std::mutex,18,i: 1,703,590 msg/sec + std::mutex,19,s: 1,689,770 msg/sec + std::mutex,19,i: 1,685,577 msg/sec + std::mutex,20,s: 1,715,029 msg/sec + std::mutex,20,i: 1,668,976 msg/sec + std::mutex,21,s: 1,727,414 msg/sec + std::mutex,21,i: 1,684,466 msg/sec + std::mutex,22,s: 1,706,687 msg/sec + std::mutex,22,i: 1,663,748 msg/sec + std::mutex,23,s: 1,692,779 msg/sec + std::mutex,23,i: 1,651,100 msg/sec + std::mutex,24,s: 1,676,927 msg/sec + std::mutex,24,i: 1,647,381 msg/sec + std::mutex,25,s: 1,664,716 msg/sec + std::mutex,25,i: 1,626,701 msg/sec + std::mutex,26,s: 1,663,565 msg/sec + std::mutex,26,i: 1,618,815 msg/sec + std::mutex,27,s: 1,670,749 msg/sec + std::mutex,27,i: 1,620,646 msg/sec + std::mutex,28,s: 1,658,071 msg/sec + std::mutex,28,i: 1,604,433 msg/sec + tbb::spin_mutex, 1,s: 34,633,531 msg/sec + tbb::spin_mutex, 1,i: 8,803,958 msg/sec + tbb::spin_mutex, 2,s: 30,094,896 msg/sec + tbb::spin_mutex, 2,i: 29,262,071 msg/sec + tbb::spin_mutex, 3,s: 13,875,600 msg/sec + tbb::spin_mutex, 3,i: 14,174,790 msg/sec + tbb::spin_mutex, 4,s: 5,751,810 msg/sec + tbb::spin_mutex, 4,i: 6,723,603 msg/sec + tbb::spin_mutex, 5,s: 2,317,003 msg/sec + tbb::spin_mutex, 5,i: 4,720,911 msg/sec +tbb::spin_mutex: producers: 6: consumer 3 received too few messages: 0.09% of expected. + tbb::spin_mutex, 6,s: 2,993,706 msg/sec +tbb::spin_mutex: producers: 6: consumer 4 received too few messages: 0.02% of expected. + tbb::spin_mutex, 6,i: 3,336,064 msg/sec + tbb::spin_mutex, 7,s: 2,436,489 msg/sec + tbb::spin_mutex, 7,i: 1,506,125 msg/sec + tbb::spin_mutex, 8,s: 789,866 msg/sec +tbb::spin_mutex: producers: 8: consumer 4 received too few messages: 0.07% of expected. +tbb::spin_mutex: producers: 8: consumer 4 received too few messages: 0.09% of expected. + tbb::spin_mutex, 8,i: 829,718 msg/sec + tbb::spin_mutex, 9,s: 161,273 msg/sec + tbb::spin_mutex, 9,i: 412,053 msg/sec + tbb::spin_mutex,10,s: 99,406 msg/sec + tbb::spin_mutex,10,i: 201,820 msg/sec + tbb::spin_mutex,11,s: 95,459 msg/sec + tbb::spin_mutex,11,i: 145,884 msg/sec + tbb::spin_mutex,12,s: 95,252 msg/sec + tbb::spin_mutex,12,i: 111,953 msg/sec + tbb::spin_mutex,13,s: 95,234 msg/sec + tbb::spin_mutex,13,i: 94,810 msg/sec + tbb::spin_mutex,14,s: 87,071 msg/sec + tbb::spin_mutex,14,i: 90,292 msg/sec +^[[6~tbb::spin_mutex: producers: 15: consumer 2 received too few messages: 0.10% of expected. +tbb::spin_mutex: producers: 15: consumer 8 received too few messages: 0.09% of expected. + tbb::spin_mutex,15,s: 85,940 msg/sec + tbb::spin_mutex,15,i: 87,511 msg/sec + tbb::spin_mutex,16,s: 85,171 msg/sec + tbb::spin_mutex,16,i: 96,928 msg/sec +tbb::spin_mutex: producers: 17: consumer 15 received too few messages: 0.06% of expected. +tbb::spin_mutex: producers: 17: consumer 14 received too few messages: 0.09% of expected. + tbb::spin_mutex,17,s: 84,877 msg/sec + tbb::spin_mutex,17,i: 102,681 msg/sec + tbb::spin_mutex,18,s: 81,075 msg/sec + tbb::spin_mutex,18,i: 105,120 msg/sec +tbb::spin_mutex: producers: 19: consumer 13 received too few messages: 0.09% of expected. + tbb::spin_mutex,19,s: 108,166 msg/sec + tbb::spin_mutex,19,i: 122,460 msg/sec + tbb::spin_mutex,20,s: 103,006 msg/sec +tbb::spin_mutex: producers: 20: consumer 0 received too few messages: 0.04% of expected. + tbb::spin_mutex,20,i: 127,191 msg/sec +tbb::spin_mutex: producers: 21: consumer 8 received too few messages: 0.09% of expected. + tbb::spin_mutex,21,s: 104,653 msg/sec + tbb::spin_mutex,21,i: 132,129 msg/sec + tbb::spin_mutex,22,s: 92,899 msg/sec +tbb::spin_mutex: producers: 22: consumer 4 received too few messages: 0.03% of expected. +tbb::spin_mutex: producers: 22: consumer 18 received too few messages: 0.06% of expected. + tbb::spin_mutex,22,i: 117,358 msg/sec +tbb::spin_mutex: producers: 23: consumer 14 received too few messages: 0.03% of expected. + tbb::spin_mutex,23,s: 92,578 msg/sec + tbb::spin_mutex,23,i: 91,750 msg/sec +tbb::spin_mutex: producers: 24: consumer 13 received too few messages: 0.06% of expected. + tbb::spin_mutex,24,s: 89,071 msg/sec +tbb::spin_mutex: producers: 24: consumer 4 received too few messages: 0.10% of expected. + tbb::spin_mutex,24,i: 76,326 msg/sec + tbb::spin_mutex,25,s: 74,687 msg/sec +tbb::spin_mutex: producers: 25: consumer 6 received too few messages: 0.07% of expected. + tbb::spin_mutex,25,i: 64,215 msg/sec + tbb::spin_mutex,26,s: 65,585 msg/sec + tbb::spin_mutex,26,i: 60,131 msg/sec + tbb::spin_mutex,27,s: 63,543 msg/sec + tbb::spin_mutex,27,i: 58,732 msg/sec + tbb::spin_mutex,28,s: 62,861 msg/sec + tbb::spin_mutex,28,i: 57,504 msg/sec + tbb::concurrent_bounded_queue, 1,s: 8,247,335 msg/sec + tbb::concurrent_bounded_queue, 1,i: 8,127,504 msg/sec + tbb::concurrent_bounded_queue, 2,s: 7,910,361 msg/sec + tbb::concurrent_bounded_queue, 2,i: 8,042,523 msg/sec + tbb::concurrent_bounded_queue, 3,s: 6,576,622 msg/sec + tbb::concurrent_bounded_queue, 3,i: 6,935,144 msg/sec + tbb::concurrent_bounded_queue, 4,s: 5,100,359 msg/sec + tbb::concurrent_bounded_queue, 4,i: 5,416,538 msg/sec + tbb::concurrent_bounded_queue, 5,s: 4,366,804 msg/sec + tbb::concurrent_bounded_queue, 5,i: 4,692,855 msg/sec + tbb::concurrent_bounded_queue, 6,s: 3,753,167 msg/sec + tbb::concurrent_bounded_queue, 6,i: 3,901,458 msg/sec + tbb::concurrent_bounded_queue, 7,s: 3,419,297 msg/sec + tbb::concurrent_bounded_queue, 7,i: 3,432,102 msg/sec + tbb::concurrent_bounded_queue, 8,s: 1,214,040 msg/sec +tbb::concurrent_bounded_queue: producers: 8: consumer 4 received too few messages: 0.04% of expected. + tbb::concurrent_bounded_queue, 8,i: 2,916,297 msg/sec + tbb::concurrent_bounded_queue, 9,s: 873,003 msg/sec +tbb::concurrent_bounded_queue: producers: 9: consumer 6 received too few messages: 0.07% of expected. + tbb::concurrent_bounded_queue, 9,i: 2,403,944 msg/sec + tbb::concurrent_bounded_queue,10,s: 756,434 msg/sec + tbb::concurrent_bounded_queue,10,i: 1,674,241 msg/sec + tbb::concurrent_bounded_queue,11,s: 678,623 msg/sec + tbb::concurrent_bounded_queue,11,i: 1,455,146 msg/sec + tbb::concurrent_bounded_queue,12,s: 613,767 msg/sec + tbb::concurrent_bounded_queue,12,i: 1,348,600 msg/sec + tbb::concurrent_bounded_queue,13,s: 615,369 msg/sec + tbb::concurrent_bounded_queue,13,i: 1,152,422 msg/sec + tbb::concurrent_bounded_queue,14,s: 577,159 msg/sec + tbb::concurrent_bounded_queue,14,i: 1,001,476 msg/sec + tbb::concurrent_bounded_queue,15,s: 655,450 msg/sec + tbb::concurrent_bounded_queue,15,i: 1,042,384 msg/sec + tbb::concurrent_bounded_queue,16,s: 732,790 msg/sec + tbb::concurrent_bounded_queue,16,i: 1,099,887 msg/sec + tbb::concurrent_bounded_queue,17,s: 805,787 msg/sec + tbb::concurrent_bounded_queue,17,i: 1,086,630 msg/sec + tbb::concurrent_bounded_queue,18,s: 857,035 msg/sec + tbb::concurrent_bounded_queue,18,i: 1,176,985 msg/sec + tbb::concurrent_bounded_queue,19,s: 885,873 msg/sec + tbb::concurrent_bounded_queue,19,i: 1,146,171 msg/sec + tbb::concurrent_bounded_queue,20,s: 892,738 msg/sec + tbb::concurrent_bounded_queue,20,i: 1,227,082 msg/sec + tbb::concurrent_bounded_queue,21,s: 1,139,013 msg/sec + tbb::concurrent_bounded_queue,21,i: 1,180,454 msg/sec + tbb::concurrent_bounded_queue,22,s: 1,174,013 msg/sec + tbb::concurrent_bounded_queue,22,i: 1,077,110 msg/sec + tbb::concurrent_bounded_queue,23,s: 1,069,171 msg/sec + tbb::concurrent_bounded_queue,23,i: 1,063,105 msg/sec + tbb::concurrent_bounded_queue,24,s: 1,034,391 msg/sec + tbb::concurrent_bounded_queue,24,i: 1,134,641 msg/sec + tbb::concurrent_bounded_queue,25,s: 919,629 msg/sec + tbb::concurrent_bounded_queue,25,i: 1,034,531 msg/sec + tbb::concurrent_bounded_queue,26,s: 828,854 msg/sec + tbb::concurrent_bounded_queue,26,i: 1,050,141 msg/sec + tbb::concurrent_bounded_queue,27,s: 802,094 msg/sec + tbb::concurrent_bounded_queue,27,i: 1,087,126 msg/sec + tbb::concurrent_bounded_queue,28,s: 757,896 msg/sec + tbb::concurrent_bounded_queue,28,i: 1,001,942 msg/sec + xenium::michael_scott_queue, 1,s: 6,087,911 msg/sec + xenium::michael_scott_queue, 1,i: 5,973,660 msg/sec + xenium::michael_scott_queue, 2,s: 4,839,244 msg/sec + xenium::michael_scott_queue, 2,i: 4,943,268 msg/sec + xenium::michael_scott_queue, 3,s: 3,900,562 msg/sec + xenium::michael_scott_queue, 3,i: 3,844,148 msg/sec + xenium::michael_scott_queue, 4,s: 3,351,319 msg/sec + xenium::michael_scott_queue, 4,i: 3,283,584 msg/sec + xenium::michael_scott_queue, 5,s: 2,995,125 msg/sec + xenium::michael_scott_queue, 5,i: 2,966,782 msg/sec + xenium::michael_scott_queue, 6,s: 2,654,470 msg/sec + xenium::michael_scott_queue, 6,i: 2,631,512 msg/sec + xenium::michael_scott_queue, 7,s: 2,416,360 msg/sec + xenium::michael_scott_queue, 7,i: 2,412,721 msg/sec + xenium::michael_scott_queue, 8,s: 1,338,731 msg/sec +xenium::michael_scott_queue: producers: 8: consumer 1 received too few messages: 0.03% of expected. + xenium::michael_scott_queue, 8,i: 2,093,123 msg/sec + xenium::michael_scott_queue, 9,s: 957,648 msg/sec +xenium::michael_scott_queue: producers: 9: consumer 6 received too few messages: 0.06% of expected. +xenium::michael_scott_queue: producers: 9: consumer 0 received too few messages: 0.07% of expected. + xenium::michael_scott_queue, 9,i: 1,475,539 msg/sec + xenium::michael_scott_queue,10,s: 874,576 msg/sec + xenium::michael_scott_queue,10,i: 1,097,559 msg/sec + xenium::michael_scott_queue,11,s: 851,316 msg/sec + xenium::michael_scott_queue,11,i: 858,944 msg/sec + xenium::michael_scott_queue,12,s: 859,542 msg/sec + xenium::michael_scott_queue,12,i: 781,581 msg/sec + xenium::michael_scott_queue,13,s: 867,708 msg/sec + xenium::michael_scott_queue,13,i: 756,735 msg/sec + xenium::michael_scott_queue,14,s: 882,615 msg/sec + xenium::michael_scott_queue,14,i: 754,595 msg/sec + xenium::michael_scott_queue,15,s: 914,757 msg/sec + xenium::michael_scott_queue,15,i: 790,262 msg/sec + xenium::michael_scott_queue,16,s: 919,740 msg/sec + xenium::michael_scott_queue,16,i: 775,181 msg/sec + xenium::michael_scott_queue,17,s: 900,991 msg/sec + xenium::michael_scott_queue,17,i: 763,415 msg/sec + xenium::michael_scott_queue,18,s: 855,615 msg/sec + xenium::michael_scott_queue,18,i: 783,522 msg/sec + xenium::michael_scott_queue,19,s: 792,582 msg/sec + xenium::michael_scott_queue,19,i: 807,000 msg/sec + xenium::michael_scott_queue,20,s: 715,598 msg/sec + xenium::michael_scott_queue,20,i: 797,604 msg/sec + xenium::michael_scott_queue,21,s: 752,375 msg/sec + xenium::michael_scott_queue,21,i: 784,862 msg/sec + xenium::michael_scott_queue,22,s: 711,311 msg/sec + xenium::michael_scott_queue,22,i: 808,771 msg/sec + xenium::michael_scott_queue,23,s: 681,410 msg/sec + xenium::michael_scott_queue,23,i: 809,788 msg/sec + xenium::michael_scott_queue,24,s: 624,802 msg/sec + xenium::michael_scott_queue,24,i: 822,990 msg/sec + xenium::michael_scott_queue,25,s: 635,117 msg/sec + xenium::michael_scott_queue,25,i: 819,132 msg/sec + xenium::michael_scott_queue,26,s: 592,770 msg/sec + xenium::michael_scott_queue,26,i: 819,099 msg/sec +xenium::michael_scott_queue: producers: 27: consumer 0 received too few messages: 0.07% of expected. + xenium::michael_scott_queue,27,s: 550,901 msg/sec + xenium::michael_scott_queue,27,i: 820,302 msg/sec + xenium::michael_scott_queue,28,s: 560,111 msg/sec + xenium::michael_scott_queue,28,i: 811,424 msg/sec + xenium::ramalhete_queue, 1,s: 9,567,120 msg/sec + xenium::ramalhete_queue, 1,i: 9,619,168 msg/sec + xenium::ramalhete_queue, 2,s: 15,444,315 msg/sec + xenium::ramalhete_queue, 2,i: 15,050,621 msg/sec + xenium::ramalhete_queue, 3,s: 17,279,594 msg/sec + xenium::ramalhete_queue, 3,i: 17,121,031 msg/sec + xenium::ramalhete_queue, 4,s: 18,001,312 msg/sec + xenium::ramalhete_queue, 4,i: 17,566,641 msg/sec + xenium::ramalhete_queue, 5,s: 18,108,353 msg/sec + xenium::ramalhete_queue, 5,i: 17,789,638 msg/sec + xenium::ramalhete_queue, 6,s: 17,764,211 msg/sec + xenium::ramalhete_queue, 6,i: 17,466,043 msg/sec + xenium::ramalhete_queue, 7,s: 17,690,124 msg/sec + xenium::ramalhete_queue, 7,i: 17,417,677 msg/sec + xenium::ramalhete_queue, 8,s: 8,081,813 msg/sec + xenium::ramalhete_queue, 8,i: 12,095,927 msg/sec + xenium::ramalhete_queue, 9,s: 7,574,517 msg/sec + xenium::ramalhete_queue, 9,i: 11,374,828 msg/sec + xenium::ramalhete_queue,10,s: 7,254,124 msg/sec + xenium::ramalhete_queue,10,i: 8,472,110 msg/sec + xenium::ramalhete_queue,11,s: 4,033,612 msg/sec + xenium::ramalhete_queue,11,i: 8,149,429 msg/sec + xenium::ramalhete_queue,12,s: 2,885,247 msg/sec + xenium::ramalhete_queue,12,i: 7,637,845 msg/sec + xenium::ramalhete_queue,13,s: 9,003,628 msg/sec + xenium::ramalhete_queue,13,i: 7,624,103 msg/sec + xenium::ramalhete_queue,14,s: 16,436,702 msg/sec + xenium::ramalhete_queue,14,i: 7,578,451 msg/sec + xenium::ramalhete_queue,15,s: 8,977,087 msg/sec + xenium::ramalhete_queue,15,i: 8,052,175 msg/sec + xenium::ramalhete_queue,16,s: 8,142,653 msg/sec + xenium::ramalhete_queue,16,i: 8,543,981 msg/sec + xenium::ramalhete_queue,17,s: 7,631,201 msg/sec + xenium::ramalhete_queue,17,i: 8,731,522 msg/sec + xenium::ramalhete_queue,18,s: 7,264,692 msg/sec + xenium::ramalhete_queue,18,i: 9,256,970 msg/sec + xenium::ramalhete_queue,19,s: 7,086,574 msg/sec + xenium::ramalhete_queue,19,i: 9,702,664 msg/sec + xenium::ramalhete_queue,20,s: 7,437,731 msg/sec + xenium::ramalhete_queue,20,i: 9,872,264 msg/sec + xenium::ramalhete_queue,21,s: 7,643,080 msg/sec + xenium::ramalhete_queue,21,i: 10,232,115 msg/sec + xenium::ramalhete_queue,22,s: 6,765,281 msg/sec + xenium::ramalhete_queue,22,i: 10,917,282 msg/sec + xenium::ramalhete_queue,23,s: 6,062,503 msg/sec + xenium::ramalhete_queue,23,i: 11,357,220 msg/sec + xenium::ramalhete_queue,24,s: 5,784,848 msg/sec + xenium::ramalhete_queue,24,i: 11,837,857 msg/sec + xenium::ramalhete_queue,25,s: 5,575,124 msg/sec + xenium::ramalhete_queue,25,i: 12,200,564 msg/sec + xenium::ramalhete_queue,26,s: 5,449,886 msg/sec + xenium::ramalhete_queue,26,i: 12,756,375 msg/sec + xenium::ramalhete_queue,27,s: 5,428,683 msg/sec + xenium::ramalhete_queue,27,i: 13,079,069 msg/sec + xenium::ramalhete_queue,28,s: 5,300,291 msg/sec + xenium::ramalhete_queue,28,i: 13,643,916 msg/sec + xenium::vyukov_bounded_queue, 1,s: 77,693,727 msg/sec + xenium::vyukov_bounded_queue, 1,i: 57,935,240 msg/sec + xenium::vyukov_bounded_queue, 2,s: 6,351,933 msg/sec + xenium::vyukov_bounded_queue, 2,i: 6,393,902 msg/sec + xenium::vyukov_bounded_queue, 3,s: 4,721,665 msg/sec + xenium::vyukov_bounded_queue, 3,i: 4,665,999 msg/sec + xenium::vyukov_bounded_queue, 4,s: 4,404,695 msg/sec + xenium::vyukov_bounded_queue, 4,i: 4,396,911 msg/sec + xenium::vyukov_bounded_queue, 5,s: 4,768,806 msg/sec + xenium::vyukov_bounded_queue, 5,i: 4,873,888 msg/sec + xenium::vyukov_bounded_queue, 6,s: 4,939,120 msg/sec + xenium::vyukov_bounded_queue, 6,i: 4,461,586 msg/sec + xenium::vyukov_bounded_queue, 7,s: 4,306,018 msg/sec + xenium::vyukov_bounded_queue, 7,i: 3,818,955 msg/sec + xenium::vyukov_bounded_queue, 8,s: 1,908,903 msg/sec +xenium::vyukov_bounded_queue: producers: 8: consumer 4 received too few messages: 0.06% of expected. +xenium::vyukov_bounded_queue: producers: 8: consumer 2 received too few messages: 0.07% of expected. + xenium::vyukov_bounded_queue, 8,i: 3,475,570 msg/sec + xenium::vyukov_bounded_queue, 9,s: 1,547,119 msg/sec + xenium::vyukov_bounded_queue, 9,i: 2,063,884 msg/sec + xenium::vyukov_bounded_queue,10,s: 1,363,881 msg/sec + xenium::vyukov_bounded_queue,10,i: 2,274,182 msg/sec + xenium::vyukov_bounded_queue,11,s: 1,288,127 msg/sec + xenium::vyukov_bounded_queue,11,i: 1,666,304 msg/sec + xenium::vyukov_bounded_queue,12,s: 1,798,928 msg/sec + xenium::vyukov_bounded_queue,12,i: 1,429,573 msg/sec + xenium::vyukov_bounded_queue,13,s: 1,319,297 msg/sec + xenium::vyukov_bounded_queue,13,i: 1,238,154 msg/sec + xenium::vyukov_bounded_queue,14,s: 1,098,229 msg/sec + xenium::vyukov_bounded_queue,14,i: 1,308,885 msg/sec + xenium::vyukov_bounded_queue,15,s: 1,126,230 msg/sec + xenium::vyukov_bounded_queue,15,i: 1,284,529 msg/sec + xenium::vyukov_bounded_queue,16,s: 1,216,249 msg/sec + xenium::vyukov_bounded_queue,16,i: 1,306,070 msg/sec + xenium::vyukov_bounded_queue,17,s: 1,352,608 msg/sec + xenium::vyukov_bounded_queue,17,i: 1,296,785 msg/sec + xenium::vyukov_bounded_queue,18,s: 1,318,919 msg/sec + xenium::vyukov_bounded_queue,18,i: 1,323,451 msg/sec + xenium::vyukov_bounded_queue,19,s: 1,221,440 msg/sec + xenium::vyukov_bounded_queue,19,i: 1,338,820 msg/sec + xenium::vyukov_bounded_queue,20,s: 1,210,423 msg/sec + xenium::vyukov_bounded_queue,20,i: 1,354,683 msg/sec + xenium::vyukov_bounded_queue,21,s: 1,129,205 msg/sec + xenium::vyukov_bounded_queue,21,i: 1,390,226 msg/sec + xenium::vyukov_bounded_queue,22,s: 1,164,455 msg/sec + xenium::vyukov_bounded_queue,22,i: 1,386,801 msg/sec + xenium::vyukov_bounded_queue,23,s: 1,146,081 msg/sec + xenium::vyukov_bounded_queue,23,i: 1,385,629 msg/sec + xenium::vyukov_bounded_queue,24,s: 1,040,731 msg/sec + xenium::vyukov_bounded_queue,24,i: 1,394,698 msg/sec +xenium::vyukov_bounded_queue: producers: 25: consumer 21 received too few messages: 0.10% of expected. + xenium::vyukov_bounded_queue,25,s: 952,623 msg/sec + xenium::vyukov_bounded_queue,25,i: 1,439,827 msg/sec +xenium::vyukov_bounded_queue: producers: 26: consumer 17 received too few messages: 0.09% of expected. + xenium::vyukov_bounded_queue,26,s: 803,291 msg/sec + xenium::vyukov_bounded_queue,26,i: 1,432,610 msg/sec +xenium::vyukov_bounded_queue: producers: 27: consumer 17 received too few messages: 0.08% of expected. + xenium::vyukov_bounded_queue,27,s: 743,491 msg/sec + xenium::vyukov_bounded_queue,27,i: 1,457,092 msg/sec + xenium::vyukov_bounded_queue,28,s: 710,389 msg/sec + xenium::vyukov_bounded_queue,28,i: 1,452,180 msg/sec + AtomicQueue, 1,s: 79,772,865 msg/sec + AtomicQueue, 1,i: 77,217,926 msg/sec + AtomicQueue, 2,s: 5,228,301 msg/sec + AtomicQueue, 2,i: 5,322,278 msg/sec + AtomicQueue, 3,s: 4,041,654 msg/sec + AtomicQueue, 3,i: 4,152,137 msg/sec + AtomicQueue, 4,s: 3,290,899 msg/sec + AtomicQueue, 4,i: 3,371,436 msg/sec + AtomicQueue, 5,s: 3,088,270 msg/sec + AtomicQueue, 5,i: 3,032,945 msg/sec + AtomicQueue, 6,s: 2,632,802 msg/sec + AtomicQueue, 6,i: 2,665,903 msg/sec + AtomicQueue, 7,s: 2,323,482 msg/sec + AtomicQueue, 7,i: 2,361,146 msg/sec + AtomicQueue, 8,s: 908,494 msg/sec + AtomicQueue, 8,i: 2,207,479 msg/sec + AtomicQueue, 9,s: 752,080 msg/sec + AtomicQueue, 9,i: 1,641,588 msg/sec + AtomicQueue,10,s: 640,224 msg/sec + AtomicQueue,10,i: 1,205,194 msg/sec + AtomicQueue,11,s: 574,183 msg/sec + AtomicQueue,11,i: 889,738 msg/sec + AtomicQueue,12,s: 550,395 msg/sec + AtomicQueue,12,i: 592,877 msg/sec + AtomicQueue,13,s: 538,031 msg/sec + AtomicQueue,13,i: 517,048 msg/sec + AtomicQueue,14,s: 471,733 msg/sec + AtomicQueue,14,i: 482,650 msg/sec + AtomicQueue,15,s: 508,808 msg/sec + AtomicQueue,15,i: 498,364 msg/sec + AtomicQueue,16,s: 533,182 msg/sec + AtomicQueue,16,i: 506,302 msg/sec + AtomicQueue,17,s: 558,629 msg/sec + AtomicQueue,17,i: 492,533 msg/sec + AtomicQueue,18,s: 567,271 msg/sec + AtomicQueue,18,i: 500,401 msg/sec + AtomicQueue,19,s: 559,687 msg/sec + AtomicQueue,19,i: 524,712 msg/sec + AtomicQueue,20,s: 544,831 msg/sec + AtomicQueue,20,i: 512,759 msg/sec + AtomicQueue,21,s: 521,285 msg/sec + AtomicQueue,21,i: 511,789 msg/sec + AtomicQueue,22,s: 506,268 msg/sec + AtomicQueue,22,i: 532,692 msg/sec + AtomicQueue,23,s: 518,073 msg/sec + AtomicQueue,23,i: 537,241 msg/sec + AtomicQueue,24,s: 462,993 msg/sec + AtomicQueue,24,i: 507,631 msg/sec + AtomicQueue,25,s: 438,405 msg/sec + AtomicQueue,25,i: 523,575 msg/sec + AtomicQueue,26,s: 421,278 msg/sec + AtomicQueue,26,i: 501,521 msg/sec + AtomicQueue,27,s: 437,904 msg/sec + AtomicQueue,27,i: 482,943 msg/sec + AtomicQueue,28,s: 399,149 msg/sec + AtomicQueue,28,i: 477,047 msg/sec + AtomicQueueB, 1,s: 47,438,573 msg/sec + AtomicQueueB, 1,i: 36,715,771 msg/sec + AtomicQueueB, 2,s: 5,365,020 msg/sec + AtomicQueueB, 2,i: 5,376,771 msg/sec + AtomicQueueB, 3,s: 3,960,556 msg/sec + AtomicQueueB, 3,i: 4,112,612 msg/sec + AtomicQueueB, 4,s: 3,246,590 msg/sec + AtomicQueueB, 4,i: 3,314,863 msg/sec + AtomicQueueB, 5,s: 2,969,057 msg/sec + AtomicQueueB, 5,i: 3,017,942 msg/sec + AtomicQueueB, 6,s: 2,602,342 msg/sec + AtomicQueueB, 6,i: 2,652,390 msg/sec + AtomicQueueB, 7,s: 2,308,771 msg/sec + AtomicQueueB, 7,i: 2,348,205 msg/sec + AtomicQueueB, 8,s: 918,766 msg/sec + AtomicQueueB, 8,i: 2,201,598 msg/sec + AtomicQueueB, 9,s: 758,205 msg/sec +AtomicQueueB: producers: 9: consumer 6 received too few messages: 0.06% of expected. +AtomicQueueB: producers: 9: consumer 0 received too few messages: 0.06% of expected. + AtomicQueueB, 9,i: 1,633,309 msg/sec + AtomicQueueB,10,s: 649,014 msg/sec + AtomicQueueB,10,i: 1,195,059 msg/sec + AtomicQueueB,11,s: 589,662 msg/sec + AtomicQueueB,11,i: 879,485 msg/sec + AtomicQueueB,12,s: 575,181 msg/sec + AtomicQueueB,12,i: 621,402 msg/sec + AtomicQueueB,13,s: 542,750 msg/sec + AtomicQueueB,13,i: 502,794 msg/sec + AtomicQueueB,14,s: 472,781 msg/sec + AtomicQueueB,14,i: 468,631 msg/sec + AtomicQueueB,15,s: 505,662 msg/sec + AtomicQueueB,15,i: 499,555 msg/sec + AtomicQueueB,16,s: 529,470 msg/sec + AtomicQueueB,16,i: 492,589 msg/sec + AtomicQueueB,17,s: 550,989 msg/sec + AtomicQueueB,17,i: 484,494 msg/sec + AtomicQueueB,18,s: 548,113 msg/sec + AtomicQueueB,18,i: 504,459 msg/sec + AtomicQueueB,19,s: 536,477 msg/sec + AtomicQueueB,19,i: 498,815 msg/sec + AtomicQueueB,20,s: 546,760 msg/sec + AtomicQueueB,20,i: 501,568 msg/sec + AtomicQueueB,21,s: 499,480 msg/sec + AtomicQueueB,21,i: 524,874 msg/sec + AtomicQueueB,22,s: 505,888 msg/sec + AtomicQueueB,22,i: 516,875 msg/sec + AtomicQueueB,23,s: 513,906 msg/sec + AtomicQueueB,23,i: 514,174 msg/sec + AtomicQueueB,24,s: 483,252 msg/sec + AtomicQueueB,24,i: 525,254 msg/sec + AtomicQueueB,25,s: 451,654 msg/sec + AtomicQueueB,25,i: 506,582 msg/sec + AtomicQueueB,26,s: 420,500 msg/sec + AtomicQueueB,26,i: 505,109 msg/sec + AtomicQueueB,27,s: 426,964 msg/sec + AtomicQueueB,27,i: 482,598 msg/sec + AtomicQueueB,28,s: 414,367 msg/sec + AtomicQueueB,28,i: 478,045 msg/sec + OptimistAtomicQueue, 1,s: 490,054,624 msg/sec + OptimistAtomicQueue, 1,i: 484,095,327 msg/sec + OptimistAtomicQueue, 2,s: 13,720,515 msg/sec + OptimistAtomicQueue, 2,i: 13,769,344 msg/sec + OptimistAtomicQueue, 3,s: 14,601,848 msg/sec + OptimistAtomicQueue, 3,i: 14,715,831 msg/sec + OptimistAtomicQueue, 4,s: 15,384,078 msg/sec + OptimistAtomicQueue, 4,i: 15,464,966 msg/sec + OptimistAtomicQueue, 5,s: 16,770,529 msg/sec + OptimistAtomicQueue, 5,i: 17,059,718 msg/sec + OptimistAtomicQueue, 6,s: 17,237,155 msg/sec + OptimistAtomicQueue, 6,i: 17,537,609 msg/sec + OptimistAtomicQueue, 7,s: 17,569,838 msg/sec + OptimistAtomicQueue, 7,i: 17,760,428 msg/sec + OptimistAtomicQueue, 8,s: 7,534,351 msg/sec + OptimistAtomicQueue, 8,i: 11,499,230 msg/sec + OptimistAtomicQueue, 9,s: 6,675,629 msg/sec + OptimistAtomicQueue, 9,i: 7,806,498 msg/sec + OptimistAtomicQueue,10,s: 6,241,334 msg/sec + OptimistAtomicQueue,10,i: 6,834,659 msg/sec + OptimistAtomicQueue,11,s: 5,914,049 msg/sec + OptimistAtomicQueue,11,i: 6,181,994 msg/sec + OptimistAtomicQueue,12,s: 5,199,936 msg/sec + OptimistAtomicQueue,12,i: 5,909,534 msg/sec + OptimistAtomicQueue,13,s: 4,589,785 msg/sec + OptimistAtomicQueue,13,i: 6,147,249 msg/sec + OptimistAtomicQueue,14,s: 3,878,266 msg/sec + OptimistAtomicQueue,14,i: 6,104,766 msg/sec + OptimistAtomicQueue,15,s: 4,784,026 msg/sec + OptimistAtomicQueue,15,i: 6,449,264 msg/sec + OptimistAtomicQueue,16,s: 4,920,347 msg/sec + OptimistAtomicQueue,16,i: 6,435,966 msg/sec + OptimistAtomicQueue,17,s: 5,111,870 msg/sec + OptimistAtomicQueue,17,i: 6,615,520 msg/sec + OptimistAtomicQueue,18,s: 5,432,958 msg/sec + OptimistAtomicQueue,18,i: 6,731,241 msg/sec + OptimistAtomicQueue,19,s: 5,491,359 msg/sec + OptimistAtomicQueue,19,i: 6,858,386 msg/sec + OptimistAtomicQueue,20,s: 6,286,582 msg/sec + OptimistAtomicQueue,20,i: 6,960,726 msg/sec + OptimistAtomicQueue,21,s: 6,682,416 msg/sec + OptimistAtomicQueue,21,i: 7,092,639 msg/sec + OptimistAtomicQueue,22,s: 6,418,524 msg/sec + OptimistAtomicQueue,22,i: 7,401,114 msg/sec + OptimistAtomicQueue,23,s: 6,376,475 msg/sec + OptimistAtomicQueue,23,i: 7,683,225 msg/sec + OptimistAtomicQueue,24,s: 6,360,283 msg/sec + OptimistAtomicQueue,24,i: 8,006,488 msg/sec + OptimistAtomicQueue,25,s: 6,197,136 msg/sec + OptimistAtomicQueue,25,i: 8,313,302 msg/sec + OptimistAtomicQueue,26,s: 6,244,154 msg/sec + OptimistAtomicQueue,26,i: 8,619,479 msg/sec +OptimistAtomicQueue: producers: 27: consumer 11 received too few messages: 0.09% of expected. +OptimistAtomicQueue: producers: 27: consumer 24 received too few messages: 0.08% of expected. +OptimistAtomicQueue: producers: 27: consumer 7 received too few messages: 0.10% of expected. + OptimistAtomicQueue,27,s: 6,866,018 msg/sec + OptimistAtomicQueue,27,i: 8,939,372 msg/sec + OptimistAtomicQueue,28,s: 5,565,018 msg/sec + OptimistAtomicQueue,28,i: 9,232,001 msg/sec + OptimistAtomicQueueB, 1,s: 239,359,767 msg/sec + OptimistAtomicQueueB, 1,i: 80,413,426 msg/sec + OptimistAtomicQueueB, 2,s: 13,821,801 msg/sec + OptimistAtomicQueueB, 2,i: 13,835,022 msg/sec + OptimistAtomicQueueB, 3,s: 14,541,806 msg/sec + OptimistAtomicQueueB, 3,i: 14,648,662 msg/sec + OptimistAtomicQueueB, 4,s: 15,259,340 msg/sec + OptimistAtomicQueueB, 4,i: 15,385,569 msg/sec + OptimistAtomicQueueB, 5,s: 16,641,349 msg/sec + OptimistAtomicQueueB, 5,i: 16,917,226 msg/sec + OptimistAtomicQueueB, 6,s: 17,112,059 msg/sec + OptimistAtomicQueueB, 6,i: 17,415,295 msg/sec + OptimistAtomicQueueB, 7,s: 17,479,018 msg/sec + OptimistAtomicQueueB, 7,i: 17,632,618 msg/sec + OptimistAtomicQueueB, 8,s: 7,390,170 msg/sec + OptimistAtomicQueueB, 8,i: 11,741,384 msg/sec + OptimistAtomicQueueB, 9,s: 6,572,124 msg/sec + OptimistAtomicQueueB, 9,i: 7,500,283 msg/sec + OptimistAtomicQueueB,10,s: 6,313,129 msg/sec + OptimistAtomicQueueB,10,i: 6,104,042 msg/sec + OptimistAtomicQueueB,11,s: 5,859,540 msg/sec + OptimistAtomicQueueB,11,i: 6,225,458 msg/sec + OptimistAtomicQueueB,12,s: 5,288,305 msg/sec + OptimistAtomicQueueB,12,i: 5,923,237 msg/sec + OptimistAtomicQueueB,13,s: 4,584,836 msg/sec + OptimistAtomicQueueB,13,i: 5,996,519 msg/sec + OptimistAtomicQueueB,14,s: 3,847,023 msg/sec + OptimistAtomicQueueB,14,i: 6,529,716 msg/sec + OptimistAtomicQueueB,15,s: 4,707,856 msg/sec + OptimistAtomicQueueB,15,i: 6,206,118 msg/sec + OptimistAtomicQueueB,16,s: 4,879,939 msg/sec + OptimistAtomicQueueB,16,i: 6,583,111 msg/sec + OptimistAtomicQueueB,17,s: 5,119,063 msg/sec + OptimistAtomicQueueB,17,i: 6,535,731 msg/sec + OptimistAtomicQueueB,18,s: 5,446,519 msg/sec + OptimistAtomicQueueB,18,i: 6,701,125 msg/sec + OptimistAtomicQueueB,19,s: 5,549,130 msg/sec + OptimistAtomicQueueB,19,i: 6,805,469 msg/sec + OptimistAtomicQueueB,20,s: 6,164,500 msg/sec + OptimistAtomicQueueB,20,i: 6,975,990 msg/sec + OptimistAtomicQueueB,21,s: 6,417,871 msg/sec + OptimistAtomicQueueB,21,i: 7,071,665 msg/sec + OptimistAtomicQueueB,22,s: 6,212,319 msg/sec + OptimistAtomicQueueB,22,i: 7,399,107 msg/sec + OptimistAtomicQueueB,23,s: 6,119,000 msg/sec + OptimistAtomicQueueB,23,i: 7,690,375 msg/sec + OptimistAtomicQueueB,24,s: 5,897,361 msg/sec + OptimistAtomicQueueB,24,i: 7,996,467 msg/sec + OptimistAtomicQueueB,25,s: 5,787,206 msg/sec + OptimistAtomicQueueB,25,i: 8,293,432 msg/sec + OptimistAtomicQueueB,26,s: 6,110,304 msg/sec + OptimistAtomicQueueB,26,i: 8,622,487 msg/sec + OptimistAtomicQueueB,27,s: 6,550,818 msg/sec + OptimistAtomicQueueB,27,i: 8,938,106 msg/sec + OptimistAtomicQueueB,28,s: 5,473,162 msg/sec + OptimistAtomicQueueB,28,i: 9,241,486 msg/sec + AtomicQueue2, 1,s: 23,820,635 msg/sec + AtomicQueue2, 1,i: 22,969,183 msg/sec + AtomicQueue2, 2,s: 5,026,705 msg/sec + AtomicQueue2, 2,i: 5,077,014 msg/sec + AtomicQueue2, 3,s: 3,930,905 msg/sec + AtomicQueue2, 3,i: 4,044,969 msg/sec + AtomicQueue2, 4,s: 3,235,081 msg/sec + AtomicQueue2, 4,i: 3,300,821 msg/sec + AtomicQueue2, 5,s: 2,972,579 msg/sec + AtomicQueue2, 5,i: 2,996,009 msg/sec + AtomicQueue2, 6,s: 2,606,889 msg/sec + AtomicQueue2, 6,i: 2,653,808 msg/sec + AtomicQueue2, 7,s: 2,310,335 msg/sec + AtomicQueue2, 7,i: 2,350,312 msg/sec + AtomicQueue2, 8,s: 883,952 msg/sec + AtomicQueue2, 8,i: 2,164,039 msg/sec + AtomicQueue2, 9,s: 729,613 msg/sec + AtomicQueue2, 9,i: 1,632,855 msg/sec + AtomicQueue2,10,s: 627,970 msg/sec + AtomicQueue2,10,i: 1,213,308 msg/sec + AtomicQueue2,11,s: 571,190 msg/sec + AtomicQueue2,11,i: 773,100 msg/sec + AtomicQueue2,12,s: 552,257 msg/sec + AtomicQueue2,12,i: 674,055 msg/sec + AtomicQueue2,13,s: 539,411 msg/sec + AtomicQueue2,13,i: 518,344 msg/sec + AtomicQueue2,14,s: 467,152 msg/sec + AtomicQueue2,14,i: 478,024 msg/sec + AtomicQueue2,15,s: 504,016 msg/sec + AtomicQueue2,15,i: 490,031 msg/sec + AtomicQueue2,16,s: 531,799 msg/sec + AtomicQueue2,16,i: 486,517 msg/sec + AtomicQueue2,17,s: 543,079 msg/sec + AtomicQueue2,17,i: 496,588 msg/sec + AtomicQueue2,18,s: 547,615 msg/sec + AtomicQueue2,18,i: 491,143 msg/sec + AtomicQueue2,19,s: 542,636 msg/sec + AtomicQueue2,19,i: 541,999 msg/sec + AtomicQueue2,20,s: 521,610 msg/sec + AtomicQueue2,20,i: 547,004 msg/sec + AtomicQueue2,21,s: 499,529 msg/sec + AtomicQueue2,21,i: 519,443 msg/sec + AtomicQueue2,22,s: 509,717 msg/sec + AtomicQueue2,22,i: 523,333 msg/sec + AtomicQueue2,23,s: 470,999 msg/sec + AtomicQueue2,23,i: 529,548 msg/sec + AtomicQueue2,24,s: 480,228 msg/sec + AtomicQueue2,24,i: 522,050 msg/sec + AtomicQueue2,25,s: 429,897 msg/sec + AtomicQueue2,25,i: 501,764 msg/sec + AtomicQueue2,26,s: 422,101 msg/sec + AtomicQueue2,26,i: 497,520 msg/sec + AtomicQueue2,27,s: 422,020 msg/sec + AtomicQueue2,27,i: 483,837 msg/sec + AtomicQueue2,28,s: 429,190 msg/sec + AtomicQueue2,28,i: 476,448 msg/sec + AtomicQueueB2, 1,s: 17,215,273 msg/sec + AtomicQueueB2, 1,i: 17,210,621 msg/sec + AtomicQueueB2, 2,s: 5,094,130 msg/sec + AtomicQueueB2, 2,i: 5,175,396 msg/sec + AtomicQueueB2, 3,s: 3,922,155 msg/sec + AtomicQueueB2, 3,i: 4,052,567 msg/sec + AtomicQueueB2, 4,s: 3,215,549 msg/sec + AtomicQueueB2, 4,i: 3,290,586 msg/sec + AtomicQueueB2, 5,s: 2,978,908 msg/sec + AtomicQueueB2, 5,i: 3,006,456 msg/sec + AtomicQueueB2, 6,s: 2,632,036 msg/sec + AtomicQueueB2, 6,i: 2,672,123 msg/sec + AtomicQueueB2, 7,s: 2,315,208 msg/sec + AtomicQueueB2, 7,i: 2,361,934 msg/sec + AtomicQueueB2, 8,s: 890,429 msg/sec + AtomicQueueB2, 8,i: 2,133,729 msg/sec + AtomicQueueB2, 9,s: 724,498 msg/sec + AtomicQueueB2, 9,i: 1,589,044 msg/sec + AtomicQueueB2,10,s: 632,408 msg/sec + AtomicQueueB2,10,i: 1,194,042 msg/sec + AtomicQueueB2,11,s: 572,357 msg/sec + AtomicQueueB2,11,i: 895,109 msg/sec + AtomicQueueB2,12,s: 549,668 msg/sec + AtomicQueueB2,12,i: 670,994 msg/sec + AtomicQueueB2,13,s: 536,501 msg/sec + AtomicQueueB2,13,i: 502,503 msg/sec + AtomicQueueB2,14,s: 466,850 msg/sec + AtomicQueueB2,14,i: 460,185 msg/sec + AtomicQueueB2,15,s: 503,173 msg/sec + AtomicQueueB2,15,i: 493,614 msg/sec + AtomicQueueB2,16,s: 531,524 msg/sec + AtomicQueueB2,16,i: 482,348 msg/sec + AtomicQueueB2,17,s: 538,869 msg/sec + AtomicQueueB2,17,i: 490,159 msg/sec + AtomicQueueB2,18,s: 546,701 msg/sec + AtomicQueueB2,18,i: 508,125 msg/sec + AtomicQueueB2,19,s: 553,294 msg/sec + AtomicQueueB2,19,i: 523,688 msg/sec + AtomicQueueB2,20,s: 515,627 msg/sec + AtomicQueueB2,20,i: 529,239 msg/sec + AtomicQueueB2,21,s: 513,581 msg/sec + AtomicQueueB2,21,i: 517,527 msg/sec + AtomicQueueB2,22,s: 518,339 msg/sec + AtomicQueueB2,22,i: 532,753 msg/sec + AtomicQueueB2,23,s: 494,051 msg/sec + AtomicQueueB2,23,i: 534,986 msg/sec + AtomicQueueB2,24,s: 491,424 msg/sec + AtomicQueueB2,24,i: 510,188 msg/sec + AtomicQueueB2,25,s: 440,980 msg/sec + AtomicQueueB2,25,i: 508,473 msg/sec + AtomicQueueB2,26,s: 407,891 msg/sec + AtomicQueueB2,26,i: 486,349 msg/sec + AtomicQueueB2,27,s: 436,592 msg/sec + AtomicQueueB2,27,i: 476,802 msg/sec + AtomicQueueB2,28,s: 409,361 msg/sec + AtomicQueueB2,28,i: 478,694 msg/sec + OptimistAtomicQueue2, 1,s: 446,196,432 msg/sec + OptimistAtomicQueue2, 1,i: 40,201,612 msg/sec + OptimistAtomicQueue2, 2,s: 10,094,174 msg/sec + OptimistAtomicQueue2, 2,i: 10,110,329 msg/sec + OptimistAtomicQueue2, 3,s: 12,085,454 msg/sec + OptimistAtomicQueue2, 3,i: 12,080,441 msg/sec + OptimistAtomicQueue2, 4,s: 13,508,019 msg/sec + OptimistAtomicQueue2, 4,i: 13,528,834 msg/sec + OptimistAtomicQueue2, 5,s: 15,309,589 msg/sec + OptimistAtomicQueue2, 5,i: 15,468,559 msg/sec + OptimistAtomicQueue2, 6,s: 16,226,800 msg/sec + OptimistAtomicQueue2, 6,i: 16,453,961 msg/sec + OptimistAtomicQueue2, 7,s: 16,728,925 msg/sec + OptimistAtomicQueue2, 7,i: 16,943,866 msg/sec + OptimistAtomicQueue2, 8,s: 6,319,662 msg/sec + OptimistAtomicQueue2, 8,i: 8,171,789 msg/sec + OptimistAtomicQueue2, 9,s: 5,136,259 msg/sec + OptimistAtomicQueue2, 9,i: 6,530,491 msg/sec + OptimistAtomicQueue2,10,s: 4,791,581 msg/sec + OptimistAtomicQueue2,10,i: 5,880,087 msg/sec + OptimistAtomicQueue2,11,s: 4,724,867 msg/sec + OptimistAtomicQueue2,11,i: 5,433,285 msg/sec + OptimistAtomicQueue2,12,s: 4,771,642 msg/sec + OptimistAtomicQueue2,12,i: 4,869,808 msg/sec + OptimistAtomicQueue2,13,s: 4,341,967 msg/sec + OptimistAtomicQueue2,13,i: 4,918,200 msg/sec + OptimistAtomicQueue2,14,s: 3,987,235 msg/sec + OptimistAtomicQueue2,14,i: 4,722,288 msg/sec + OptimistAtomicQueue2,15,s: 4,579,344 msg/sec + OptimistAtomicQueue2,15,i: 5,054,095 msg/sec + OptimistAtomicQueue2,16,s: 4,963,793 msg/sec + OptimistAtomicQueue2,16,i: 5,373,581 msg/sec + OptimistAtomicQueue2,17,s: 5,331,598 msg/sec + OptimistAtomicQueue2,17,i: 5,692,601 msg/sec + OptimistAtomicQueue2,18,s: 5,481,421 msg/sec + OptimistAtomicQueue2,18,i: 6,043,113 msg/sec + OptimistAtomicQueue2,19,s: 5,534,724 msg/sec + OptimistAtomicQueue2,19,i: 6,332,701 msg/sec + OptimistAtomicQueue2,20,s: 6,281,056 msg/sec + OptimistAtomicQueue2,20,i: 6,646,543 msg/sec + OptimistAtomicQueue2,21,s: 6,372,744 msg/sec + OptimistAtomicQueue2,21,i: 6,939,570 msg/sec + OptimistAtomicQueue2,22,s: 5,645,002 msg/sec + OptimistAtomicQueue2,22,i: 7,274,754 msg/sec + OptimistAtomicQueue2,23,s: 5,913,588 msg/sec + OptimistAtomicQueue2,23,i: 7,560,869 msg/sec + OptimistAtomicQueue2,24,s: 5,483,244 msg/sec + OptimistAtomicQueue2,24,i: 7,923,552 msg/sec + OptimistAtomicQueue2,25,s: 5,593,249 msg/sec + OptimistAtomicQueue2,25,i: 8,236,931 msg/sec + OptimistAtomicQueue2,26,s: 5,698,893 msg/sec + OptimistAtomicQueue2,26,i: 8,565,898 msg/sec + OptimistAtomicQueue2,27,s: 5,748,027 msg/sec + OptimistAtomicQueue2,27,i: 8,878,784 msg/sec + OptimistAtomicQueue2,28,s: 6,950,219 msg/sec + OptimistAtomicQueue2,28,i: 9,228,086 msg/sec + OptimistAtomicQueueB2, 1,s: 18,483,689 msg/sec + OptimistAtomicQueueB2, 1,i: 18,514,940 msg/sec + OptimistAtomicQueueB2, 2,s: 10,169,071 msg/sec + OptimistAtomicQueueB2, 2,i: 10,103,438 msg/sec + OptimistAtomicQueueB2, 3,s: 11,951,116 msg/sec + OptimistAtomicQueueB2, 3,i: 12,040,370 msg/sec + OptimistAtomicQueueB2, 4,s: 13,262,983 msg/sec + OptimistAtomicQueueB2, 4,i: 13,296,339 msg/sec + OptimistAtomicQueueB2, 5,s: 15,060,880 msg/sec + OptimistAtomicQueueB2, 5,i: 15,209,565 msg/sec + OptimistAtomicQueueB2, 6,s: 15,932,102 msg/sec + OptimistAtomicQueueB2, 6,i: 16,172,766 msg/sec + OptimistAtomicQueueB2, 7,s: 16,324,971 msg/sec + OptimistAtomicQueueB2, 7,i: 16,556,597 msg/sec + OptimistAtomicQueueB2, 8,s: 6,301,104 msg/sec + OptimistAtomicQueueB2, 8,i: 8,119,023 msg/sec + OptimistAtomicQueueB2, 9,s: 5,202,266 msg/sec + OptimistAtomicQueueB2, 9,i: 7,389,147 msg/sec + OptimistAtomicQueueB2,10,s: 5,060,597 msg/sec + OptimistAtomicQueueB2,10,i: 5,805,180 msg/sec + OptimistAtomicQueueB2,11,s: 4,784,186 msg/sec + OptimistAtomicQueueB2,11,i: 4,900,037 msg/sec + OptimistAtomicQueueB2,12,s: 4,716,909 msg/sec + OptimistAtomicQueueB2,12,i: 5,085,442 msg/sec + OptimistAtomicQueueB2,13,s: 4,354,083 msg/sec + OptimistAtomicQueueB2,13,i: 4,731,783 msg/sec + OptimistAtomicQueueB2,14,s: 3,819,186 msg/sec + OptimistAtomicQueueB2,14,i: 4,651,880 msg/sec + OptimistAtomicQueueB2,15,s: 4,514,573 msg/sec + OptimistAtomicQueueB2,15,i: 4,988,294 msg/sec + OptimistAtomicQueueB2,16,s: 5,117,055 msg/sec + OptimistAtomicQueueB2,16,i: 5,318,373 msg/sec + OptimistAtomicQueueB2,17,s: 5,187,389 msg/sec + OptimistAtomicQueueB2,17,i: 5,669,221 msg/sec + OptimistAtomicQueueB2,18,s: 5,484,858 msg/sec + OptimistAtomicQueueB2,18,i: 5,917,342 msg/sec + OptimistAtomicQueueB2,19,s: 5,750,173 msg/sec + OptimistAtomicQueueB2,19,i: 6,403,314 msg/sec + OptimistAtomicQueueB2,20,s: 6,173,457 msg/sec + OptimistAtomicQueueB2,20,i: 6,561,965 msg/sec + OptimistAtomicQueueB2,21,s: 6,254,794 msg/sec + OptimistAtomicQueueB2,21,i: 6,873,429 msg/sec + OptimistAtomicQueueB2,22,s: 5,580,800 msg/sec + OptimistAtomicQueueB2,22,i: 7,177,269 msg/sec + OptimistAtomicQueueB2,23,s: 5,601,989 msg/sec + OptimistAtomicQueueB2,23,i: 7,487,255 msg/sec + OptimistAtomicQueueB2,24,s: 5,444,041 msg/sec + OptimistAtomicQueueB2,24,i: 7,871,700 msg/sec + OptimistAtomicQueueB2,25,s: 5,433,472 msg/sec + OptimistAtomicQueueB2,25,i: 8,172,443 msg/sec + OptimistAtomicQueueB2,26,s: 5,609,907 msg/sec + OptimistAtomicQueueB2,26,i: 8,518,634 msg/sec + OptimistAtomicQueueB2,27,s: 5,629,413 msg/sec + OptimistAtomicQueueB2,27,i: 8,854,258 msg/sec + OptimistAtomicQueueB2,28,s: 6,130,585 msg/sec + OptimistAtomicQueueB2,28,i: 9,193,558 msg/sec + +---- Running ping-pong benchmarks (lower is better) ---- + boost::lockfree::spsc_queue: 0.000000321 sec/round-trip + boost::lockfree::queue: 0.000000571 sec/round-trip + TicketSpinlock: 0.000000668 sec/round-trip + moodycamel::ReaderWriterQueue: 0.000000310 sec/round-trip + From 6973bdc1b808ebe1bf26d1a542c7d30d84e488e4 Mon Sep 17 00:00:00 2001 From: Ziyang Xu Date: Thu, 6 Oct 2022 17:51:22 -0400 Subject: [PATCH 14/97] add slamp stats runtime --- liberty/lib/SLAMP/CMakeLists.txt | 1 + liberty/lib/SLAMP/SLAMPstats/CMakeLists.txt | 24 + liberty/lib/SLAMP/SLAMPstats/Stats.cpp | 538 + liberty/lib/SLAMP/SLAMPstats/json.hpp | 22091 ++++++++++++++++++ liberty/lib/SLAMP/SLAMPstats/slamp_hooks.h | 371 + 5 files changed, 23025 insertions(+) create mode 100644 liberty/lib/SLAMP/SLAMPstats/CMakeLists.txt create mode 100644 liberty/lib/SLAMP/SLAMPstats/Stats.cpp create mode 100644 liberty/lib/SLAMP/SLAMPstats/json.hpp create mode 100644 liberty/lib/SLAMP/SLAMPstats/slamp_hooks.h diff --git a/liberty/lib/SLAMP/CMakeLists.txt b/liberty/lib/SLAMP/CMakeLists.txt index 91167f99..c96bcfd7 100644 --- a/liberty/lib/SLAMP/CMakeLists.txt +++ b/liberty/lib/SLAMP/CMakeLists.txt @@ -22,3 +22,4 @@ add_subdirectory(SLAMPlib/hooks) add_subdirectory(SLAMPnng) add_subdirectory(SLAMPboost) add_subdirectory(SLAMPatomicq) +add_subdirectory(SLAMPstats) diff --git a/liberty/lib/SLAMP/SLAMPstats/CMakeLists.txt b/liberty/lib/SLAMP/SLAMPstats/CMakeLists.txt new file mode 100644 index 00000000..53db9b61 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPstats/CMakeLists.txt @@ -0,0 +1,24 @@ +file(GLOB SRCS + "*.cpp" +) +# add CMAKE_PREFIX_PATH +set(CMAKE_C_COMPILER "clang") +set(CMAKE_CXX_COMPILER "clang++") + +# Compilation flags +# set_source_files_properties(${SRCS} PROPERTIES COMPILE_FLAGS "-Wl,-save-temps -std=c++17 -Wno-inline -O3 -fexceptions")# -emit-llvm") +set_source_files_properties(${SRCS} PROPERTIES COMPILE_FLAGS "-flto -std=c++17 -Wno-inline -O3 -fexceptions")# -emit-llvm") +set(PassName "slamp_hooks_stats") + + +# list(APPEND CMAKE_MODULE_PATH "${LLVM_CMAKE_DIR}") +#include(HandleLLVMOptions) +# include(AddLLVM) + + +add_llvm_library(${PassName} STATIC ${SRCS}) +# target_link_libraries(${PassName} nng::nng) +# add_llvm_library(${PassName}_shared SHARED ${SRCS}) +# set_target_properties(${PassName}_shared PROPERTIES OUTPUT_NAME ${PassName}) +# set_property(TARGET ${PassName} PROPERTY INTERPROCEDURAL_OPTIMIZATION TRUE) +#add_llvm_library(${PassName} SHARED ${SRCS}) # This is to generate libxxx.so diff --git a/liberty/lib/SLAMP/SLAMPstats/Stats.cpp b/liberty/lib/SLAMP/SLAMPstats/Stats.cpp new file mode 100644 index 00000000..1b884afe --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPstats/Stats.cpp @@ -0,0 +1,538 @@ +#include +#include + +#include "slamp_hooks.h" +#include "malloc.h" +#include "json.hpp" + +// context counters +unsigned long counter_fcn_enter = 0; +unsigned long counter_fcn_exit = 0; +unsigned long counter_loop_enter = 0; +unsigned long counter_loop_exit = 0; +unsigned long counter_loop_iter = 0; +unsigned long counter_ext_fcn_enter = 0; +unsigned long counter_ext_fcn_exit = 0; + +// load/store counters +unsigned long counter_load_1 = 0; +unsigned long counter_load_2 = 0; +unsigned long counter_load_4 = 0; +unsigned long counter_load_8 = 0; +unsigned long counter_load_n = 0; +unsigned long counter_store_1 = 0; +unsigned long counter_store_2 = 0; +unsigned long counter_store_4 = 0; +unsigned long counter_store_8 = 0; +unsigned long counter_store_n = 0; + +// memory access counters +unsigned long counter_global = 0; +unsigned long counter_alloca = 0; +unsigned long counter_alloca_bytes = 0; +unsigned long counter_malloc = 0; +unsigned long counter_malloc_bytes = 0; +unsigned long counter_memalign = 0; +unsigned long counter_free = 0; +unsigned long counter_report_base = 0; + +#define TURN_OFF_CUSTOM_MALLOC do {\ + __malloc_hook = old_malloc_hook; \ + __free_hook = old_free_hook; \ + __memalign_hook = old_memalign_hook; \ +} while (false); + +#define TURN_ON_CUSTOM_MALLOC do { \ + __malloc_hook = SLAMP_malloc_hook; \ + __free_hook = SLAMP_free_hook; \ + __memalign_hook = SLAMP_memalign_hook; \ +} while (false); + +static void *(*old_malloc_hook)(size_t, const void *); +static void (*old_free_hook)(void *, const void *); +static void *(*old_memalign_hook)(size_t, size_t, const void *); + +void SLAMP_init(uint32_t fn_id, uint32_t loop_id) { + // replace hooks + old_malloc_hook = __malloc_hook; + old_free_hook = __free_hook; + old_memalign_hook = __memalign_hook; + + TURN_ON_CUSTOM_MALLOC; +} + +void SLAMP_fini(const char* filename){ + TURN_OFF_CUSTOM_MALLOC; + // print stats to file + std::ofstream stats_file; + stats_file.open(filename); + + using json = nlohmann::json; + json outfile; + + outfile["context"]["fcn_enter"] = counter_fcn_enter; + outfile["context"]["fcn_exit"] = counter_fcn_exit; + outfile["context"]["loop_enter"] = counter_loop_enter; + outfile["context"]["loop_exit"] = counter_loop_exit; + outfile["context"]["loop_iter"] = counter_loop_iter; + outfile["context"]["ext_fcn_enter"] = counter_ext_fcn_enter; + outfile["context"]["ext_fcn_exit"] = counter_ext_fcn_exit; + + outfile["load_store"]["load_1"] = counter_load_1; + outfile["load_store"]["load_2"] = counter_load_2; + outfile["load_store"]["load_4"] = counter_load_4; + outfile["load_store"]["load_8"] = counter_load_8; + outfile["load_store"]["load_n"] = counter_load_n; + outfile["load_store"]["store_1"] = counter_store_1; + outfile["load_store"]["store_2"] = counter_store_2; + outfile["load_store"]["store_4"] = counter_store_4; + outfile["load_store"]["store_8"] = counter_store_8; + outfile["load_store"]["store_n"] = counter_store_n; + + outfile["memory_access"]["global"] = counter_global; + outfile["memory_access"]["alloca"] = counter_alloca; + outfile["memory_access"]["alloca_bytes"] = counter_alloca_bytes; + outfile["memory_access"]["malloc"] = counter_malloc; + outfile["memory_access"]["malloc_bytes"] = counter_malloc_bytes; + outfile["memory_access"]["memalign"] = counter_memalign; + outfile["memory_access"]["free"] = counter_free; + outfile["memory_access"]["report_base"] = counter_report_base; + + stats_file << outfile.dump(2); + stats_file.close(); + + // stats_file.open(filename); + // stats_file << "Context Counters: " << "\n" + // << "Fcn Enter: " << counter_fcn_enter << "\n" + // << "Fcn Exit: " << counter_fcn_exit << "\n" + // << "Loop Enter: " << counter_loop_enter << "\n" + // << "Loop Exit: " << counter_loop_exit << "\n" + // << "Loop Iter: " << counter_loop_iter << "\n" + // << "Ext Fcn Enter: " << counter_ext_fcn_enter << "\n" + // << "Ext Fcn Exit: " << counter_ext_fcn_exit << "\n\n"; + + // stats_file << "Load/Store Counters: " << "\n" + // << "Load 1: " << counter_load_1 << "\n" + // << "Load 2: " << counter_load_2 << "\n" + // << "Load 4: " << counter_load_4 << "\n" + // << "Load 8: " << counter_load_8 << "\n" + // << "Load N: " << counter_load_n << "\n" + // << "Store 1: " << counter_store_1 << "\n" + // << "Store 2: " << counter_store_2 << "\n" + // << "Store 4: " << counter_store_4 << "\n" + // << "Store 8: " << counter_store_8 << "\n" + // << "Store N: " << counter_store_n << "\n\n"; + + // stats_file << "Memory Access Counters: " << "\n" + // << "Global: " << counter_global << "\n" + // << "Alloca: " << counter_alloca << "\n" + // << "Malloc: " << counter_malloc << "\n" + // << "Memalign: " << counter_memalign << "\n" + // << "Free: " << counter_free << "\n" + // << "Report Base: " << counter_report_base << "\n"; +} + +void SLAMP_init_global_vars(const char *name, uint64_t addr, size_t size){ + counter_global++; +} +void SLAMP_allocated(uint64_t addr){} + +void SLAMP_main_entry(uint32_t argc, char** argv, char** env){} + +void SLAMP_enter_fcn(uint32_t id){ + counter_fcn_enter++; +} + +void SLAMP_exit_fcn(uint32_t id){ + counter_fcn_exit++; +} + +void SLAMP_enter_loop(uint32_t id){ + counter_loop_enter++; +} + +void SLAMP_exit_loop(uint32_t id){ + counter_loop_exit++; +} +void SLAMP_loop_iter_ctx(uint32_t id){ + counter_loop_iter++; +} + +void SLAMP_loop_invocation(){} +void SLAMP_loop_iteration(){} +void SLAMP_loop_exit(){} + +void SLAMP_report_base_pointer_arg(uint32_t, uint32_t, void *ptr){ + counter_report_base++; +} +void SLAMP_report_base_pointer_inst(uint32_t, void *ptr){ + counter_report_base++; +} + +void SLAMP_callback_stack_alloca(uint64_t array_sz, uint64_t type_sz, uint32_t instr, uint64_t addr){ + counter_alloca++; + counter_alloca_bytes += array_sz * type_sz; +} + +void SLAMP_callback_stack_free(){} + +void SLAMP_ext_push(const uint32_t instr){ + counter_ext_fcn_enter++; +} +void SLAMP_ext_pop(){ + counter_ext_fcn_exit++; +} + +void SLAMP_push(const uint32_t instr){} +void SLAMP_pop(){} + +void SLAMP_load1(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + counter_load_1++; +} + +void SLAMP_load2(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + counter_load_2++; +} + +void SLAMP_load4(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + counter_load_4++; +} + +void SLAMP_load8(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + counter_load_8++; +} + +void SLAMP_loadn(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, size_t n){ + counter_load_n++; +} + +void SLAMP_load1_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + counter_load_1++; +} + +void SLAMP_load2_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + counter_load_2++; +} +void SLAMP_load4_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + counter_load_4++; +} + +void SLAMP_load8_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + counter_load_8++; +} + +void SLAMP_loadn_ext(const uint64_t addr, const uint32_t bare_instr, size_t n){ + counter_load_n++; +} + +void SLAMP_store1(uint32_t instr, const uint64_t addr){ + counter_store_1++; +} +void SLAMP_store2(uint32_t instr, const uint64_t addr){ + counter_store_2++; +} +void SLAMP_store4(uint32_t instr, const uint64_t addr){ + counter_store_4++; +} +void SLAMP_store8(uint32_t instr, const uint64_t addr){ + counter_store_8++; +} +void SLAMP_storen(uint32_t instr, const uint64_t addr, size_t n){ + counter_store_n++; +} + +void SLAMP_store1_ext(const uint64_t addr, const uint32_t bare_inst){ + counter_store_1++; +} +void SLAMP_store2_ext(const uint64_t addr, const uint32_t bare_inst){ + counter_store_2++; +} +void SLAMP_store4_ext(const uint64_t addr, const uint32_t bare_inst){ + counter_store_4++; +} +void SLAMP_store8_ext(const uint64_t addr, const uint32_t bare_inst){ + counter_store_8++; +} +void SLAMP_storen_ext(const uint64_t addr, const uint32_t bare_inst, size_t n){ + counter_store_n++; +} + +/* wrappers */ +static void* SLAMP_malloc_hook(size_t size, const void *caller){ + void *result; + TURN_OFF_CUSTOM_MALLOC; + result = malloc(size); + TURN_ON_CUSTOM_MALLOC; + counter_malloc++; + counter_malloc_bytes += size; + return result; +} +static void SLAMP_free_hook(void *ptr, const void *caller){ + TURN_OFF_CUSTOM_MALLOC; + free(ptr); + TURN_ON_CUSTOM_MALLOC; + counter_free++; +} +static void* SLAMP_memalign_hook(size_t alignment, size_t size, const void *caller){ + void *result; + TURN_OFF_CUSTOM_MALLOC; + result = memalign(alignment, size); + TURN_ON_CUSTOM_MALLOC; + counter_memalign++; + return result; +} + +void* SLAMP_malloc(size_t size, uint32_t instr, size_t alignment){} +void* SLAMP_calloc(size_t nelem, size_t elsize){} +void* SLAMP_realloc(void* ptr, size_t size){} +void* SLAMP__Znam(size_t size){} +void* SLAMP__Znwm(size_t size){} + + + +char* SLAMP_strdup(const char *s1){} +char* SLAMP___strdup(const char *s1){} +void SLAMP_free(void* ptr){} +void SLAMP_cfree(void* ptr){} +void SLAMP__ZdlPv(void* ptr){} +void SLAMP__ZdaPv(void* ptr){} +int SLAMP_brk(void *end_data_segment){} +void* SLAMP_sbrk(intptr_t increment){} + +/* llvm memory intrinsics */ +void SLAMP_llvm_memcpy_p0i8_p0i8_i32(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint32_t sizeBytes){} +void SLAMP_llvm_memcpy_p0i8_p0i8_i64(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint64_t sizeBytes){} +void SLAMP_llvm_memmove_p0i8_p0i8_i32(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint32_t sizeBytes){} +void SLAMP_llvm_memmove_p0i8_p0i8_i64(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint64_t sizeBytes){} +void SLAMP_llvm_memset_p0i8_i32(const uint8_t* dstAddr, const uint32_t len){} +void SLAMP_llvm_memset_p0i8_i64(const uint8_t* dstAddr, const uint64_t len){} + +// void SLAMP_llvm_lifetime_start_p0i8(uint64_t size, uint8_t* ptr){} +// void SLAMP_llvm_lifetime_end_p0i8(uint64_t size, uint8_t* ptr){} + +/* String functions */ +size_t SLAMP_strlen(const char *str){} +char* SLAMP_strchr(char *s, int c){} +char* SLAMP_strrchr(char *s, int c){} +int SLAMP_strcmp(const char *s1, const char *s2){} +int SLAMP_strncmp(const char *s1, const char *s2, size_t n){} +char* SLAMP_strcpy(char *dest, const char *src){} +char* SLAMP_strncpy(char *dest, const char *src, size_t n){} +char* SLAMP_strcat(char *s1, const char *s2){} +char* SLAMP_strncat(char *s1, const char *s2, size_t n){} +char* SLAMP_strstr(char *s1, char *s2){} +size_t SLAMP_strspn(const char *s1, const char *s2){} +size_t SLAMP_strcspn(const char *s1, const char *s2){} +char* SLAMP_strtok(char *s, const char *delim){} +double SLAMP_strtod(const char *nptr, char **endptr){} +long int SLAMP_strtol(const char *nptr, char **endptr, int base){} +char* SLAMP_strpbrk(char *s1, char *s2){} + +/* Mem* and b* functions */ +void *SLAMP_memset (void *dest, int c, size_t n){} +void *SLAMP_memcpy (void *dest, const void *src, size_t n){} +void *SLAMP___builtin_memcpy (void *dest, const void *src, size_t n){} +void *SLAMP_memmove (void *dest, const void *src, size_t n){} +int SLAMP_memcmp(const void *s1, const void *s2, size_t n){} +void* SLAMP_memchr(void* ptr, int value, size_t num){} +void* SLAMP___rawmemchr(void* ptr, int value){} + +void SLAMP_bzero(void *s, size_t n){} +void SLAMP_bcopy(const void *s1, void *s2, size_t n){} + +/* IO */ +ssize_t SLAMP_read(int fd, void *buf, size_t count){} +int SLAMP_open(const char *pathname, int flags, mode_t mode){} +int SLAMP_close(int fd){} +ssize_t SLAMP_write(int fd, const void *buf, size_t count){} +off_t SLAMP_lseek(int fildes, off_t offset, int whence){} + +FILE * SLAMP_fopen(const char *path, const char *mode){} +FILE * SLAMP_fopen64(const char *path, const char *mode){} +FILE * SLAMP_freopen(const char *path, const char *mode, FILE* stream){} +int SLAMP_fflush(FILE *stream){} +int SLAMP_fclose(FILE *stream){} +int SLAMP_ferror(FILE *stream){} +int SLAMP_feof(FILE *stream){} +long SLAMP_ftell(FILE *stream){} +size_t SLAMP_fread(void * ptr, size_t size, size_t nitems, FILE *stream){} +size_t SLAMP_fwrite(const void *ptr, size_t size, size_t nitems, FILE *stream){} +int SLAMP_fseek(FILE *stream, long offset, int whence){} +void SLAMP_rewind(FILE *stream){} + +int SLAMP_fgetc(FILE *stream){} +int SLAMP_fputc(int c, FILE *stream){} +char * SLAMP_fgets(char *s, int n, FILE *stream){} +int SLAMP_fputs(const char *s, FILE *stream){} + +int SLAMP_ungetc(int c, FILE *stream){} +int SLAMP_putchar(int c){} +int SLAMP_getchar(void){} + +int SLAMP_fileno(FILE *stream){} +char * SLAMP_gets(char *s){} +int SLAMP_puts(const char *s){} + +int SLAMP_select(int n, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, struct timeval *timeout){} +int SLAMP_remove(const char *path){} + +void SLAMP_setbuf(FILE * stream, char * buf){} +void SLAMP_setvbuf(FILE * stream, char * buf, int mode, size_t size){} +char * SLAMP_tmpnam(char *s){} +FILE* SLAMP_tmpfile(void){} +char * SLAMP_ttyname(int fildes){} + +FILE * SLAMP_fdopen(int fildes, const char *mode){} +void SLAMP_clearerr(FILE *stream){} + +int SLAMP_truncate(const char *path, off_t length){} +int SLAMP_ftruncate(int fildes, off_t length){} + +int SLAMP_dup(int oldfd){} +int SLAMP_dup2(int oldfd, int newfd){} +int SLAMP_pipe(int filedes[2]){} + +int SLAMP_chmod(const char *path, mode_t mode){} +int SLAMP_fchmod(int fildes, mode_t mode){} +int SLAMP_fchown(int fd, uid_t owner, gid_t group){} +int SLAMP_access(const char *pathname, int mode){} +long SLAMP_pathconf(char *path, int name){} +int SLAMP_mkdir(const char *pathname, mode_t mode){} +int SLAMP_rmdir(const char *pathname){} +mode_t SLAMP_umask(mode_t mask){} +int SLAMP_fcntl(int fd, int cmd, struct flock *lock){} + +DIR* SLAMP_opendir(const char* name){} +struct dirent* SLAMP_readdir(DIR *dirp){} +struct dirent64* SLAMP_readdir64(DIR *dirp){} +int SLAMP_closedir(DIR* dirp){} + +/* Printf */ +int SLAMP_printf(const char *format, ...){} +int SLAMP_fprintf(FILE *stream, const char *format, ...){} +int SLAMP_sprintf(char *str, const char *format, ...){} +int SLAMP_snprintf(char *str, size_t size, const char *format, ...){} + +int SLAMP_vprintf(const char *format, va_list ap){} +int SLAMP_vfprintf(FILE *stream, const char *format, va_list ap){} +int SLAMP_vsprintf(char *str, const char *format, va_list ap){} +int SLAMP_vsnprintf(char *str, size_t size, const char *format, va_list ap){} + +/* Scanf */ +int SLAMP_fscanf(FILE *stream, const char *format, ... ){} +int SLAMP_scanf(const char *format, ... ){} +int SLAMP_sscanf(const char *s, const char *format, ... ){} +int SLAMP___isoc99_sscanf(const char *s, const char *format, ... ){} + +int SLAMP_vfscanf(FILE *stream, const char *format, va_list ap){} +int SLAMP_vscanf(const char *format, va_list ap){} +int SLAMP_vsscanf(const char *s, const char *format, va_list ap){} + +/* Time */ +time_t SLAMP_time(time_t *t){} +struct tm *SLAMP_localtime(const time_t *timer){} +struct lconv* SLAMP_localeconv(){} +struct tm *SLAMP_gmtime(const time_t *timer){} +int SLAMP_gettimeofday(struct timeval *tv, struct timezone *tz){} + +/* Math */ +double SLAMP_ldexp(double x, int exp){} +float SLAMP_ldexpf(float x, int exp){} +long double SLAMP_ldexpl(long double x, int exp){} +double SLAMP_log10(double x){} +float SLAMP_log10f(float x){} +long double SLAMP_log10l(long double x){} +double SLAMP_log(double x){} +float SLAMP_logf(float x){} +long double SLAMP_logl(long double x){} + +double SLAMP_exp(double x){} +float SLAMP_expf(float x){} +long double SLAMP_expl(long double x){} + +double SLAMP_cos(double x){} +float SLAMP_cosf(float x){} +long double SLAMP_cosl(long double x){} +double SLAMP_sin(double x){} +double SLAMP_tan(double x){} +float SLAMP_sinf(float x){} +long double SLAMP_sinl(long double x){} + +double SLAMP_atan(double x){} +float SLAMP_atanf(float x){} +long double SLAMP_atanl(long double x){} + +double SLAMP_floor(double x){} +float SLAMP_floorf(float x){} +long double SLAMP_floorl(long double x){} +double SLAMP_ceil(double x){} +float SLAMP_ceilf(float x){} +long double SLAMP_ceill(long double x){} + +double SLAMP_atan2(double y, double x){} +float SLAMP_atan2f(float y, float x){} +long double SLAMP_atan2l(long double y, long double x){} + +double SLAMP_sqrt(double x){} +float SLAMP_sqrtf(float x){} +long double SLAMP_sqrtl(long double x){} + +double SLAMP_pow(double x, double y){} +float SLAMP_powf(float x, float y){} +long double SLAMP_powl(long double x, long double y){} + +double SLAMP_fabs(double x){} +float SLAMP_fabsf(float x){} +long double SLAMP_fabsl(long double x){} + +double SLAMP_modf(double x, double *iptr){} +float SLAMP_modff(float x, float *iptr){} +long double SLAMP_modfl(long double x, long double *iptr){} + +double SLAMP_fmod(double x, double y){} + +double SLAMP_frexp(double num, int *exp){} +float SLAMP_frexpf(float num, int *exp){} +long double SLAMP_frexpl(long double num, int *exp){} + +int SLAMP_isnan(){} + +/* MISC */ +char *SLAMP_getenv(const char *name){} +int SLAMP_putenv(char* string){} +char *SLAMP_getcwd(char *buf, size_t size){} +char* SLAMP_strerror(int errnum){} +void SLAMP_exit(int status){} +void SLAMP__exit(int status){} +int SLAMP_link(const char *oldpath, const char *newpath){} +int SLAMP_unlink(const char *pathname){} +int SLAMP_isatty(int desc){} +int SLAMP_setuid(uid_t uid){} +uid_t SLAMP_getuid(void){} +uid_t SLAMP_geteuid(void){} +int SLAMP_setgid(gid_t gid){} +gid_t SLAMP_getgid(void){} +gid_t SLAMP_getegid(void){} +pid_t SLAMP_getpid(void){} +int SLAMP_chdir(const char *path){} +int SLAMP_execl(const char *path, const char *arg0, ... /*, (char *)0 */){} +int SLAMP_execv(const char *path, char *const argv[]){} +int SLAMP_execvp(const char *file, char *const argv[]){} +int SLAMP_kill(pid_t pid, int sig){} +pid_t SLAMP_fork(void){} +sighandler_t SLAMP___sysv_signal(int signum, sighandler_t handler){} +pid_t SLAMP_waitpid(pid_t pid, int* status, int options){} +void SLAMP_qsort(void* base, size_t nmemb, size_t size, int(*compar)(const void *, const void *)){} +int SLAMP_ioctl(int d, int request, ...){} +unsigned int SLAMP_sleep(unsigned int seconds){} +char* SLAMP_gcvt(double number, size_t ndigit, char* buf){} +char* SLAMP_nl_langinfo(nl_item item){} + +/* Compiler/Glibc Internals */ +void SLAMP___assert_fail(const char * assertion, const char * file, unsigned int line, const char * function){} +const unsigned short int **SLAMP___ctype_b_loc(void){} +int SLAMP__IO_getc(_IO_FILE * __fp){} +int SLAMP__IO_putc(int __c, _IO_FILE *__fp){} + +int SLAMP___fxstat (int __ver, int __fildes, struct stat *__stat_buf){} +int SLAMP___xstat (int __ver, __const char *__filename, struct stat *__stat_buf){} diff --git a/liberty/lib/SLAMP/SLAMPstats/json.hpp b/liberty/lib/SLAMP/SLAMPstats/json.hpp new file mode 100644 index 00000000..cb27e058 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPstats/json.hpp @@ -0,0 +1,22091 @@ +/* + __ _____ _____ _____ + __| | __| | | | JSON for Modern C++ +| | |__ | | | | | | version 3.10.5 +|_____|_____|_____|_|___| https://github.com/nlohmann/json + +Licensed under the MIT License . +SPDX-License-Identifier: MIT +Copyright (c) 2013-2022 Niels Lohmann . + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +/****************************************************************************\ + * Note on documentation: The source files contain links to the online * + * documentation of the public API at https://json.nlohmann.me. This URL * + * contains the most recent documentation and should also be applicable to * + * previous versions; documentation for deprecated functions is not * + * removed, but marked deprecated. See "Generate documentation" section in * + * file doc/README.md. * +\****************************************************************************/ + +#ifndef INCLUDE_NLOHMANN_JSON_HPP_ +#define INCLUDE_NLOHMANN_JSON_HPP_ + +#define NLOHMANN_JSON_VERSION_MAJOR 3 +#define NLOHMANN_JSON_VERSION_MINOR 10 +#define NLOHMANN_JSON_VERSION_PATCH 5 + +#include // all_of, find, for_each +#include // nullptr_t, ptrdiff_t, size_t +#include // hash, less +#include // initializer_list +#ifndef JSON_NO_IO + #include // istream, ostream +#endif // JSON_NO_IO +#include // random_access_iterator_tag +#include // unique_ptr +#include // accumulate +#include // string, stoi, to_string +#include // declval, forward, move, pair, swap +#include // vector + +// #include + + +#include +#include + +// #include + + +#include // transform +#include // array +#include // forward_list +#include // inserter, front_inserter, end +#include // map +#include // string +#include // tuple, make_tuple +#include // is_arithmetic, is_same, is_enum, underlying_type, is_convertible +#include // unordered_map +#include // pair, declval +#include // valarray + +// #include + + +#include // exception +#include // runtime_error +#include // to_string +#include // vector + +// #include + + +#include // array +#include // size_t +#include // uint8_t +#include // string + +namespace nlohmann +{ +namespace detail +{ +/////////////////////////// +// JSON type enumeration // +/////////////////////////// + +/*! +@brief the JSON type enumeration + +This enumeration collects the different JSON types. It is internally used to +distinguish the stored values, and the functions @ref basic_json::is_null(), +@ref basic_json::is_object(), @ref basic_json::is_array(), +@ref basic_json::is_string(), @ref basic_json::is_boolean(), +@ref basic_json::is_number() (with @ref basic_json::is_number_integer(), +@ref basic_json::is_number_unsigned(), and @ref basic_json::is_number_float()), +@ref basic_json::is_discarded(), @ref basic_json::is_primitive(), and +@ref basic_json::is_structured() rely on it. + +@note There are three enumeration entries (number_integer, number_unsigned, and +number_float), because the library distinguishes these three types for numbers: +@ref basic_json::number_unsigned_t is used for unsigned integers, +@ref basic_json::number_integer_t is used for signed integers, and +@ref basic_json::number_float_t is used for floating-point numbers or to +approximate integers which do not fit in the limits of their respective type. + +@sa see @ref basic_json::basic_json(const value_t value_type) -- create a JSON +value with the default value for a given type + +@since version 1.0.0 +*/ +enum class value_t : std::uint8_t +{ + null, ///< null value + object, ///< object (unordered set of name/value pairs) + array, ///< array (ordered collection of values) + string, ///< string value + boolean, ///< boolean value + number_integer, ///< number value (signed integer) + number_unsigned, ///< number value (unsigned integer) + number_float, ///< number value (floating-point) + binary, ///< binary array (ordered collection of bytes) + discarded ///< discarded by the parser callback function +}; + +/*! +@brief comparison operator for JSON types + +Returns an ordering that is similar to Python: +- order: null < boolean < number < object < array < string < binary +- furthermore, each type is not smaller than itself +- discarded values are not comparable +- binary is represented as a b"" string in python and directly comparable to a + string; however, making a binary array directly comparable with a string would + be surprising behavior in a JSON file. + +@since version 1.0.0 +*/ +inline bool operator<(const value_t lhs, const value_t rhs) noexcept +{ + static constexpr std::array order = {{ + 0 /* null */, 3 /* object */, 4 /* array */, 5 /* string */, + 1 /* boolean */, 2 /* integer */, 2 /* unsigned */, 2 /* float */, + 6 /* binary */ + } + }; + + const auto l_index = static_cast(lhs); + const auto r_index = static_cast(rhs); + return l_index < order.size() && r_index < order.size() && order[l_index] < order[r_index]; +} +} // namespace detail +} // namespace nlohmann + +// #include + + +#include +// #include + + +#include // declval, pair +// #include + + +/* Hedley - https://nemequ.github.io/hedley + * Created by Evan Nemerson + * + * To the extent possible under law, the author(s) have dedicated all + * copyright and related and neighboring rights to this software to + * the public domain worldwide. This software is distributed without + * any warranty. + * + * For details, see . + * SPDX-License-Identifier: CC0-1.0 + */ + +#if !defined(JSON_HEDLEY_VERSION) || (JSON_HEDLEY_VERSION < 15) +#if defined(JSON_HEDLEY_VERSION) + #undef JSON_HEDLEY_VERSION +#endif +#define JSON_HEDLEY_VERSION 15 + +#if defined(JSON_HEDLEY_STRINGIFY_EX) + #undef JSON_HEDLEY_STRINGIFY_EX +#endif +#define JSON_HEDLEY_STRINGIFY_EX(x) #x + +#if defined(JSON_HEDLEY_STRINGIFY) + #undef JSON_HEDLEY_STRINGIFY +#endif +#define JSON_HEDLEY_STRINGIFY(x) JSON_HEDLEY_STRINGIFY_EX(x) + +#if defined(JSON_HEDLEY_CONCAT_EX) + #undef JSON_HEDLEY_CONCAT_EX +#endif +#define JSON_HEDLEY_CONCAT_EX(a,b) a##b + +#if defined(JSON_HEDLEY_CONCAT) + #undef JSON_HEDLEY_CONCAT +#endif +#define JSON_HEDLEY_CONCAT(a,b) JSON_HEDLEY_CONCAT_EX(a,b) + +#if defined(JSON_HEDLEY_CONCAT3_EX) + #undef JSON_HEDLEY_CONCAT3_EX +#endif +#define JSON_HEDLEY_CONCAT3_EX(a,b,c) a##b##c + +#if defined(JSON_HEDLEY_CONCAT3) + #undef JSON_HEDLEY_CONCAT3 +#endif +#define JSON_HEDLEY_CONCAT3(a,b,c) JSON_HEDLEY_CONCAT3_EX(a,b,c) + +#if defined(JSON_HEDLEY_VERSION_ENCODE) + #undef JSON_HEDLEY_VERSION_ENCODE +#endif +#define JSON_HEDLEY_VERSION_ENCODE(major,minor,revision) (((major) * 1000000) + ((minor) * 1000) + (revision)) + +#if defined(JSON_HEDLEY_VERSION_DECODE_MAJOR) + #undef JSON_HEDLEY_VERSION_DECODE_MAJOR +#endif +#define JSON_HEDLEY_VERSION_DECODE_MAJOR(version) ((version) / 1000000) + +#if defined(JSON_HEDLEY_VERSION_DECODE_MINOR) + #undef JSON_HEDLEY_VERSION_DECODE_MINOR +#endif +#define JSON_HEDLEY_VERSION_DECODE_MINOR(version) (((version) % 1000000) / 1000) + +#if defined(JSON_HEDLEY_VERSION_DECODE_REVISION) + #undef JSON_HEDLEY_VERSION_DECODE_REVISION +#endif +#define JSON_HEDLEY_VERSION_DECODE_REVISION(version) ((version) % 1000) + +#if defined(JSON_HEDLEY_GNUC_VERSION) + #undef JSON_HEDLEY_GNUC_VERSION +#endif +#if defined(__GNUC__) && defined(__GNUC_PATCHLEVEL__) + #define JSON_HEDLEY_GNUC_VERSION JSON_HEDLEY_VERSION_ENCODE(__GNUC__, __GNUC_MINOR__, __GNUC_PATCHLEVEL__) +#elif defined(__GNUC__) + #define JSON_HEDLEY_GNUC_VERSION JSON_HEDLEY_VERSION_ENCODE(__GNUC__, __GNUC_MINOR__, 0) +#endif + +#if defined(JSON_HEDLEY_GNUC_VERSION_CHECK) + #undef JSON_HEDLEY_GNUC_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_GNUC_VERSION) + #define JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_GNUC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_MSVC_VERSION) + #undef JSON_HEDLEY_MSVC_VERSION +#endif +#if defined(_MSC_FULL_VER) && (_MSC_FULL_VER >= 140000000) && !defined(__ICL) + #define JSON_HEDLEY_MSVC_VERSION JSON_HEDLEY_VERSION_ENCODE(_MSC_FULL_VER / 10000000, (_MSC_FULL_VER % 10000000) / 100000, (_MSC_FULL_VER % 100000) / 100) +#elif defined(_MSC_FULL_VER) && !defined(__ICL) + #define JSON_HEDLEY_MSVC_VERSION JSON_HEDLEY_VERSION_ENCODE(_MSC_FULL_VER / 1000000, (_MSC_FULL_VER % 1000000) / 10000, (_MSC_FULL_VER % 10000) / 10) +#elif defined(_MSC_VER) && !defined(__ICL) + #define JSON_HEDLEY_MSVC_VERSION JSON_HEDLEY_VERSION_ENCODE(_MSC_VER / 100, _MSC_VER % 100, 0) +#endif + +#if defined(JSON_HEDLEY_MSVC_VERSION_CHECK) + #undef JSON_HEDLEY_MSVC_VERSION_CHECK +#endif +#if !defined(JSON_HEDLEY_MSVC_VERSION) + #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (0) +#elif defined(_MSC_VER) && (_MSC_VER >= 1400) + #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (_MSC_FULL_VER >= ((major * 10000000) + (minor * 100000) + (patch))) +#elif defined(_MSC_VER) && (_MSC_VER >= 1200) + #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (_MSC_FULL_VER >= ((major * 1000000) + (minor * 10000) + (patch))) +#else + #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (_MSC_VER >= ((major * 100) + (minor))) +#endif + +#if defined(JSON_HEDLEY_INTEL_VERSION) + #undef JSON_HEDLEY_INTEL_VERSION +#endif +#if defined(__INTEL_COMPILER) && defined(__INTEL_COMPILER_UPDATE) && !defined(__ICL) + #define JSON_HEDLEY_INTEL_VERSION JSON_HEDLEY_VERSION_ENCODE(__INTEL_COMPILER / 100, __INTEL_COMPILER % 100, __INTEL_COMPILER_UPDATE) +#elif defined(__INTEL_COMPILER) && !defined(__ICL) + #define JSON_HEDLEY_INTEL_VERSION JSON_HEDLEY_VERSION_ENCODE(__INTEL_COMPILER / 100, __INTEL_COMPILER % 100, 0) +#endif + +#if defined(JSON_HEDLEY_INTEL_VERSION_CHECK) + #undef JSON_HEDLEY_INTEL_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_INTEL_VERSION) + #define JSON_HEDLEY_INTEL_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_INTEL_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_INTEL_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_INTEL_CL_VERSION) + #undef JSON_HEDLEY_INTEL_CL_VERSION +#endif +#if defined(__INTEL_COMPILER) && defined(__INTEL_COMPILER_UPDATE) && defined(__ICL) + #define JSON_HEDLEY_INTEL_CL_VERSION JSON_HEDLEY_VERSION_ENCODE(__INTEL_COMPILER, __INTEL_COMPILER_UPDATE, 0) +#endif + +#if defined(JSON_HEDLEY_INTEL_CL_VERSION_CHECK) + #undef JSON_HEDLEY_INTEL_CL_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_INTEL_CL_VERSION) + #define JSON_HEDLEY_INTEL_CL_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_INTEL_CL_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_INTEL_CL_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_PGI_VERSION) + #undef JSON_HEDLEY_PGI_VERSION +#endif +#if defined(__PGI) && defined(__PGIC__) && defined(__PGIC_MINOR__) && defined(__PGIC_PATCHLEVEL__) + #define JSON_HEDLEY_PGI_VERSION JSON_HEDLEY_VERSION_ENCODE(__PGIC__, __PGIC_MINOR__, __PGIC_PATCHLEVEL__) +#endif + +#if defined(JSON_HEDLEY_PGI_VERSION_CHECK) + #undef JSON_HEDLEY_PGI_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_PGI_VERSION) + #define JSON_HEDLEY_PGI_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_PGI_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_PGI_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_SUNPRO_VERSION) + #undef JSON_HEDLEY_SUNPRO_VERSION +#endif +#if defined(__SUNPRO_C) && (__SUNPRO_C > 0x1000) + #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((((__SUNPRO_C >> 16) & 0xf) * 10) + ((__SUNPRO_C >> 12) & 0xf), (((__SUNPRO_C >> 8) & 0xf) * 10) + ((__SUNPRO_C >> 4) & 0xf), (__SUNPRO_C & 0xf) * 10) +#elif defined(__SUNPRO_C) + #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((__SUNPRO_C >> 8) & 0xf, (__SUNPRO_C >> 4) & 0xf, (__SUNPRO_C) & 0xf) +#elif defined(__SUNPRO_CC) && (__SUNPRO_CC > 0x1000) + #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((((__SUNPRO_CC >> 16) & 0xf) * 10) + ((__SUNPRO_CC >> 12) & 0xf), (((__SUNPRO_CC >> 8) & 0xf) * 10) + ((__SUNPRO_CC >> 4) & 0xf), (__SUNPRO_CC & 0xf) * 10) +#elif defined(__SUNPRO_CC) + #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((__SUNPRO_CC >> 8) & 0xf, (__SUNPRO_CC >> 4) & 0xf, (__SUNPRO_CC) & 0xf) +#endif + +#if defined(JSON_HEDLEY_SUNPRO_VERSION_CHECK) + #undef JSON_HEDLEY_SUNPRO_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_SUNPRO_VERSION) + #define JSON_HEDLEY_SUNPRO_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_SUNPRO_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_SUNPRO_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_EMSCRIPTEN_VERSION) + #undef JSON_HEDLEY_EMSCRIPTEN_VERSION +#endif +#if defined(__EMSCRIPTEN__) + #define JSON_HEDLEY_EMSCRIPTEN_VERSION JSON_HEDLEY_VERSION_ENCODE(__EMSCRIPTEN_major__, __EMSCRIPTEN_minor__, __EMSCRIPTEN_tiny__) +#endif + +#if defined(JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK) + #undef JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_EMSCRIPTEN_VERSION) + #define JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_EMSCRIPTEN_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_ARM_VERSION) + #undef JSON_HEDLEY_ARM_VERSION +#endif +#if defined(__CC_ARM) && defined(__ARMCOMPILER_VERSION) + #define JSON_HEDLEY_ARM_VERSION JSON_HEDLEY_VERSION_ENCODE(__ARMCOMPILER_VERSION / 1000000, (__ARMCOMPILER_VERSION % 1000000) / 10000, (__ARMCOMPILER_VERSION % 10000) / 100) +#elif defined(__CC_ARM) && defined(__ARMCC_VERSION) + #define JSON_HEDLEY_ARM_VERSION JSON_HEDLEY_VERSION_ENCODE(__ARMCC_VERSION / 1000000, (__ARMCC_VERSION % 1000000) / 10000, (__ARMCC_VERSION % 10000) / 100) +#endif + +#if defined(JSON_HEDLEY_ARM_VERSION_CHECK) + #undef JSON_HEDLEY_ARM_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_ARM_VERSION) + #define JSON_HEDLEY_ARM_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_ARM_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_ARM_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_IBM_VERSION) + #undef JSON_HEDLEY_IBM_VERSION +#endif +#if defined(__ibmxl__) + #define JSON_HEDLEY_IBM_VERSION JSON_HEDLEY_VERSION_ENCODE(__ibmxl_version__, __ibmxl_release__, __ibmxl_modification__) +#elif defined(__xlC__) && defined(__xlC_ver__) + #define JSON_HEDLEY_IBM_VERSION JSON_HEDLEY_VERSION_ENCODE(__xlC__ >> 8, __xlC__ & 0xff, (__xlC_ver__ >> 8) & 0xff) +#elif defined(__xlC__) + #define JSON_HEDLEY_IBM_VERSION JSON_HEDLEY_VERSION_ENCODE(__xlC__ >> 8, __xlC__ & 0xff, 0) +#endif + +#if defined(JSON_HEDLEY_IBM_VERSION_CHECK) + #undef JSON_HEDLEY_IBM_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_IBM_VERSION) + #define JSON_HEDLEY_IBM_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_IBM_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_IBM_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_VERSION) + #undef JSON_HEDLEY_TI_VERSION +#endif +#if \ + defined(__TI_COMPILER_VERSION__) && \ + ( \ + defined(__TMS470__) || defined(__TI_ARM__) || \ + defined(__MSP430__) || \ + defined(__TMS320C2000__) \ + ) +#if (__TI_COMPILER_VERSION__ >= 16000000) + #define JSON_HEDLEY_TI_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif +#endif + +#if defined(JSON_HEDLEY_TI_VERSION_CHECK) + #undef JSON_HEDLEY_TI_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_VERSION) + #define JSON_HEDLEY_TI_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_CL2000_VERSION) + #undef JSON_HEDLEY_TI_CL2000_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && defined(__TMS320C2000__) + #define JSON_HEDLEY_TI_CL2000_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_CL2000_VERSION_CHECK) + #undef JSON_HEDLEY_TI_CL2000_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_CL2000_VERSION) + #define JSON_HEDLEY_TI_CL2000_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CL2000_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_CL2000_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_CL430_VERSION) + #undef JSON_HEDLEY_TI_CL430_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && defined(__MSP430__) + #define JSON_HEDLEY_TI_CL430_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_CL430_VERSION_CHECK) + #undef JSON_HEDLEY_TI_CL430_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_CL430_VERSION) + #define JSON_HEDLEY_TI_CL430_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CL430_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_CL430_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_ARMCL_VERSION) + #undef JSON_HEDLEY_TI_ARMCL_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && (defined(__TMS470__) || defined(__TI_ARM__)) + #define JSON_HEDLEY_TI_ARMCL_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_ARMCL_VERSION_CHECK) + #undef JSON_HEDLEY_TI_ARMCL_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_ARMCL_VERSION) + #define JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_ARMCL_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_CL6X_VERSION) + #undef JSON_HEDLEY_TI_CL6X_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && defined(__TMS320C6X__) + #define JSON_HEDLEY_TI_CL6X_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_CL6X_VERSION_CHECK) + #undef JSON_HEDLEY_TI_CL6X_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_CL6X_VERSION) + #define JSON_HEDLEY_TI_CL6X_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CL6X_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_CL6X_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_CL7X_VERSION) + #undef JSON_HEDLEY_TI_CL7X_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && defined(__C7000__) + #define JSON_HEDLEY_TI_CL7X_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_CL7X_VERSION_CHECK) + #undef JSON_HEDLEY_TI_CL7X_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_CL7X_VERSION) + #define JSON_HEDLEY_TI_CL7X_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CL7X_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_CL7X_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_CLPRU_VERSION) + #undef JSON_HEDLEY_TI_CLPRU_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && defined(__PRU__) + #define JSON_HEDLEY_TI_CLPRU_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_CLPRU_VERSION_CHECK) + #undef JSON_HEDLEY_TI_CLPRU_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_CLPRU_VERSION) + #define JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CLPRU_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_CRAY_VERSION) + #undef JSON_HEDLEY_CRAY_VERSION +#endif +#if defined(_CRAYC) + #if defined(_RELEASE_PATCHLEVEL) + #define JSON_HEDLEY_CRAY_VERSION JSON_HEDLEY_VERSION_ENCODE(_RELEASE_MAJOR, _RELEASE_MINOR, _RELEASE_PATCHLEVEL) + #else + #define JSON_HEDLEY_CRAY_VERSION JSON_HEDLEY_VERSION_ENCODE(_RELEASE_MAJOR, _RELEASE_MINOR, 0) + #endif +#endif + +#if defined(JSON_HEDLEY_CRAY_VERSION_CHECK) + #undef JSON_HEDLEY_CRAY_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_CRAY_VERSION) + #define JSON_HEDLEY_CRAY_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_CRAY_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_CRAY_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_IAR_VERSION) + #undef JSON_HEDLEY_IAR_VERSION +#endif +#if defined(__IAR_SYSTEMS_ICC__) + #if __VER__ > 1000 + #define JSON_HEDLEY_IAR_VERSION JSON_HEDLEY_VERSION_ENCODE((__VER__ / 1000000), ((__VER__ / 1000) % 1000), (__VER__ % 1000)) + #else + #define JSON_HEDLEY_IAR_VERSION JSON_HEDLEY_VERSION_ENCODE(__VER__ / 100, __VER__ % 100, 0) + #endif +#endif + +#if defined(JSON_HEDLEY_IAR_VERSION_CHECK) + #undef JSON_HEDLEY_IAR_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_IAR_VERSION) + #define JSON_HEDLEY_IAR_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_IAR_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_IAR_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TINYC_VERSION) + #undef JSON_HEDLEY_TINYC_VERSION +#endif +#if defined(__TINYC__) + #define JSON_HEDLEY_TINYC_VERSION JSON_HEDLEY_VERSION_ENCODE(__TINYC__ / 1000, (__TINYC__ / 100) % 10, __TINYC__ % 100) +#endif + +#if defined(JSON_HEDLEY_TINYC_VERSION_CHECK) + #undef JSON_HEDLEY_TINYC_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TINYC_VERSION) + #define JSON_HEDLEY_TINYC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TINYC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TINYC_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_DMC_VERSION) + #undef JSON_HEDLEY_DMC_VERSION +#endif +#if defined(__DMC__) + #define JSON_HEDLEY_DMC_VERSION JSON_HEDLEY_VERSION_ENCODE(__DMC__ >> 8, (__DMC__ >> 4) & 0xf, __DMC__ & 0xf) +#endif + +#if defined(JSON_HEDLEY_DMC_VERSION_CHECK) + #undef JSON_HEDLEY_DMC_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_DMC_VERSION) + #define JSON_HEDLEY_DMC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_DMC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_DMC_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_COMPCERT_VERSION) + #undef JSON_HEDLEY_COMPCERT_VERSION +#endif +#if defined(__COMPCERT_VERSION__) + #define JSON_HEDLEY_COMPCERT_VERSION JSON_HEDLEY_VERSION_ENCODE(__COMPCERT_VERSION__ / 10000, (__COMPCERT_VERSION__ / 100) % 100, __COMPCERT_VERSION__ % 100) +#endif + +#if defined(JSON_HEDLEY_COMPCERT_VERSION_CHECK) + #undef JSON_HEDLEY_COMPCERT_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_COMPCERT_VERSION) + #define JSON_HEDLEY_COMPCERT_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_COMPCERT_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_COMPCERT_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_PELLES_VERSION) + #undef JSON_HEDLEY_PELLES_VERSION +#endif +#if defined(__POCC__) + #define JSON_HEDLEY_PELLES_VERSION JSON_HEDLEY_VERSION_ENCODE(__POCC__ / 100, __POCC__ % 100, 0) +#endif + +#if defined(JSON_HEDLEY_PELLES_VERSION_CHECK) + #undef JSON_HEDLEY_PELLES_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_PELLES_VERSION) + #define JSON_HEDLEY_PELLES_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_PELLES_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_PELLES_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_MCST_LCC_VERSION) + #undef JSON_HEDLEY_MCST_LCC_VERSION +#endif +#if defined(__LCC__) && defined(__LCC_MINOR__) + #define JSON_HEDLEY_MCST_LCC_VERSION JSON_HEDLEY_VERSION_ENCODE(__LCC__ / 100, __LCC__ % 100, __LCC_MINOR__) +#endif + +#if defined(JSON_HEDLEY_MCST_LCC_VERSION_CHECK) + #undef JSON_HEDLEY_MCST_LCC_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_MCST_LCC_VERSION) + #define JSON_HEDLEY_MCST_LCC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_MCST_LCC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_MCST_LCC_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_GCC_VERSION) + #undef JSON_HEDLEY_GCC_VERSION +#endif +#if \ + defined(JSON_HEDLEY_GNUC_VERSION) && \ + !defined(__clang__) && \ + !defined(JSON_HEDLEY_INTEL_VERSION) && \ + !defined(JSON_HEDLEY_PGI_VERSION) && \ + !defined(JSON_HEDLEY_ARM_VERSION) && \ + !defined(JSON_HEDLEY_CRAY_VERSION) && \ + !defined(JSON_HEDLEY_TI_VERSION) && \ + !defined(JSON_HEDLEY_TI_ARMCL_VERSION) && \ + !defined(JSON_HEDLEY_TI_CL430_VERSION) && \ + !defined(JSON_HEDLEY_TI_CL2000_VERSION) && \ + !defined(JSON_HEDLEY_TI_CL6X_VERSION) && \ + !defined(JSON_HEDLEY_TI_CL7X_VERSION) && \ + !defined(JSON_HEDLEY_TI_CLPRU_VERSION) && \ + !defined(__COMPCERT__) && \ + !defined(JSON_HEDLEY_MCST_LCC_VERSION) + #define JSON_HEDLEY_GCC_VERSION JSON_HEDLEY_GNUC_VERSION +#endif + +#if defined(JSON_HEDLEY_GCC_VERSION_CHECK) + #undef JSON_HEDLEY_GCC_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_GCC_VERSION) + #define JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_GCC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_HAS_ATTRIBUTE) + #undef JSON_HEDLEY_HAS_ATTRIBUTE +#endif +#if \ + defined(__has_attribute) && \ + ( \ + (!defined(JSON_HEDLEY_IAR_VERSION) || JSON_HEDLEY_IAR_VERSION_CHECK(8,5,9)) \ + ) +# define JSON_HEDLEY_HAS_ATTRIBUTE(attribute) __has_attribute(attribute) +#else +# define JSON_HEDLEY_HAS_ATTRIBUTE(attribute) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_ATTRIBUTE) + #undef JSON_HEDLEY_GNUC_HAS_ATTRIBUTE +#endif +#if defined(__has_attribute) + #define JSON_HEDLEY_GNUC_HAS_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_HAS_ATTRIBUTE(attribute) +#else + #define JSON_HEDLEY_GNUC_HAS_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_ATTRIBUTE) + #undef JSON_HEDLEY_GCC_HAS_ATTRIBUTE +#endif +#if defined(__has_attribute) + #define JSON_HEDLEY_GCC_HAS_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_HAS_ATTRIBUTE(attribute) +#else + #define JSON_HEDLEY_GCC_HAS_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_CPP_ATTRIBUTE) + #undef JSON_HEDLEY_HAS_CPP_ATTRIBUTE +#endif +#if \ + defined(__has_cpp_attribute) && \ + defined(__cplusplus) && \ + (!defined(JSON_HEDLEY_SUNPRO_VERSION) || JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,15,0)) + #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE(attribute) __has_cpp_attribute(attribute) +#else + #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE(attribute) (0) +#endif + +#if defined(JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS) + #undef JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS +#endif +#if !defined(__cplusplus) || !defined(__has_cpp_attribute) + #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS(ns,attribute) (0) +#elif \ + !defined(JSON_HEDLEY_PGI_VERSION) && \ + !defined(JSON_HEDLEY_IAR_VERSION) && \ + (!defined(JSON_HEDLEY_SUNPRO_VERSION) || JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,15,0)) && \ + (!defined(JSON_HEDLEY_MSVC_VERSION) || JSON_HEDLEY_MSVC_VERSION_CHECK(19,20,0)) + #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS(ns,attribute) JSON_HEDLEY_HAS_CPP_ATTRIBUTE(ns::attribute) +#else + #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS(ns,attribute) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE) + #undef JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE +#endif +#if defined(__has_cpp_attribute) && defined(__cplusplus) + #define JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) __has_cpp_attribute(attribute) +#else + #define JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE) + #undef JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE +#endif +#if defined(__has_cpp_attribute) && defined(__cplusplus) + #define JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) __has_cpp_attribute(attribute) +#else + #define JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_BUILTIN) + #undef JSON_HEDLEY_HAS_BUILTIN +#endif +#if defined(__has_builtin) + #define JSON_HEDLEY_HAS_BUILTIN(builtin) __has_builtin(builtin) +#else + #define JSON_HEDLEY_HAS_BUILTIN(builtin) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_BUILTIN) + #undef JSON_HEDLEY_GNUC_HAS_BUILTIN +#endif +#if defined(__has_builtin) + #define JSON_HEDLEY_GNUC_HAS_BUILTIN(builtin,major,minor,patch) __has_builtin(builtin) +#else + #define JSON_HEDLEY_GNUC_HAS_BUILTIN(builtin,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_BUILTIN) + #undef JSON_HEDLEY_GCC_HAS_BUILTIN +#endif +#if defined(__has_builtin) + #define JSON_HEDLEY_GCC_HAS_BUILTIN(builtin,major,minor,patch) __has_builtin(builtin) +#else + #define JSON_HEDLEY_GCC_HAS_BUILTIN(builtin,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_FEATURE) + #undef JSON_HEDLEY_HAS_FEATURE +#endif +#if defined(__has_feature) + #define JSON_HEDLEY_HAS_FEATURE(feature) __has_feature(feature) +#else + #define JSON_HEDLEY_HAS_FEATURE(feature) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_FEATURE) + #undef JSON_HEDLEY_GNUC_HAS_FEATURE +#endif +#if defined(__has_feature) + #define JSON_HEDLEY_GNUC_HAS_FEATURE(feature,major,minor,patch) __has_feature(feature) +#else + #define JSON_HEDLEY_GNUC_HAS_FEATURE(feature,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_FEATURE) + #undef JSON_HEDLEY_GCC_HAS_FEATURE +#endif +#if defined(__has_feature) + #define JSON_HEDLEY_GCC_HAS_FEATURE(feature,major,minor,patch) __has_feature(feature) +#else + #define JSON_HEDLEY_GCC_HAS_FEATURE(feature,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_EXTENSION) + #undef JSON_HEDLEY_HAS_EXTENSION +#endif +#if defined(__has_extension) + #define JSON_HEDLEY_HAS_EXTENSION(extension) __has_extension(extension) +#else + #define JSON_HEDLEY_HAS_EXTENSION(extension) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_EXTENSION) + #undef JSON_HEDLEY_GNUC_HAS_EXTENSION +#endif +#if defined(__has_extension) + #define JSON_HEDLEY_GNUC_HAS_EXTENSION(extension,major,minor,patch) __has_extension(extension) +#else + #define JSON_HEDLEY_GNUC_HAS_EXTENSION(extension,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_EXTENSION) + #undef JSON_HEDLEY_GCC_HAS_EXTENSION +#endif +#if defined(__has_extension) + #define JSON_HEDLEY_GCC_HAS_EXTENSION(extension,major,minor,patch) __has_extension(extension) +#else + #define JSON_HEDLEY_GCC_HAS_EXTENSION(extension,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE) + #undef JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE +#endif +#if defined(__has_declspec_attribute) + #define JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE(attribute) __has_declspec_attribute(attribute) +#else + #define JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE(attribute) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE) + #undef JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE +#endif +#if defined(__has_declspec_attribute) + #define JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) __has_declspec_attribute(attribute) +#else + #define JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE) + #undef JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE +#endif +#if defined(__has_declspec_attribute) + #define JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) __has_declspec_attribute(attribute) +#else + #define JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_WARNING) + #undef JSON_HEDLEY_HAS_WARNING +#endif +#if defined(__has_warning) + #define JSON_HEDLEY_HAS_WARNING(warning) __has_warning(warning) +#else + #define JSON_HEDLEY_HAS_WARNING(warning) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_WARNING) + #undef JSON_HEDLEY_GNUC_HAS_WARNING +#endif +#if defined(__has_warning) + #define JSON_HEDLEY_GNUC_HAS_WARNING(warning,major,minor,patch) __has_warning(warning) +#else + #define JSON_HEDLEY_GNUC_HAS_WARNING(warning,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_WARNING) + #undef JSON_HEDLEY_GCC_HAS_WARNING +#endif +#if defined(__has_warning) + #define JSON_HEDLEY_GCC_HAS_WARNING(warning,major,minor,patch) __has_warning(warning) +#else + #define JSON_HEDLEY_GCC_HAS_WARNING(warning,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if \ + (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) || \ + defined(__clang__) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(18,4,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,7,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(2,0,1) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,1,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,0,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(5,0,0) || \ + JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,17) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(8,0,0) || \ + (JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) && defined(__C99_PRAGMA_OPERATOR)) + #define JSON_HEDLEY_PRAGMA(value) _Pragma(#value) +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) + #define JSON_HEDLEY_PRAGMA(value) __pragma(value) +#else + #define JSON_HEDLEY_PRAGMA(value) +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_PUSH) + #undef JSON_HEDLEY_DIAGNOSTIC_PUSH +#endif +#if defined(JSON_HEDLEY_DIAGNOSTIC_POP) + #undef JSON_HEDLEY_DIAGNOSTIC_POP +#endif +#if defined(__clang__) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("clang diagnostic push") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("clang diagnostic pop") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("warning(push)") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("warning(pop)") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(4,6,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("GCC diagnostic push") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("GCC diagnostic pop") +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH __pragma(warning(push)) + #define JSON_HEDLEY_DIAGNOSTIC_POP __pragma(warning(pop)) +#elif JSON_HEDLEY_ARM_VERSION_CHECK(5,6,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("push") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("pop") +#elif \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,4,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,1,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("diag_push") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("diag_pop") +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(2,90,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("warning(push)") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("warning(pop)") +#else + #define JSON_HEDLEY_DIAGNOSTIC_PUSH + #define JSON_HEDLEY_DIAGNOSTIC_POP +#endif + +/* JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_ is for + HEDLEY INTERNAL USE ONLY. API subject to change without notice. */ +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_ +#endif +#if defined(__cplusplus) +# if JSON_HEDLEY_HAS_WARNING("-Wc++98-compat") +# if JSON_HEDLEY_HAS_WARNING("-Wc++17-extensions") +# if JSON_HEDLEY_HAS_WARNING("-Wc++1z-extensions") +# define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(xpr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wc++98-compat\"") \ + _Pragma("clang diagnostic ignored \"-Wc++17-extensions\"") \ + _Pragma("clang diagnostic ignored \"-Wc++1z-extensions\"") \ + xpr \ + JSON_HEDLEY_DIAGNOSTIC_POP +# else +# define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(xpr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wc++98-compat\"") \ + _Pragma("clang diagnostic ignored \"-Wc++17-extensions\"") \ + xpr \ + JSON_HEDLEY_DIAGNOSTIC_POP +# endif +# else +# define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(xpr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wc++98-compat\"") \ + xpr \ + JSON_HEDLEY_DIAGNOSTIC_POP +# endif +# endif +#endif +#if !defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(x) x +#endif + +#if defined(JSON_HEDLEY_CONST_CAST) + #undef JSON_HEDLEY_CONST_CAST +#endif +#if defined(__cplusplus) +# define JSON_HEDLEY_CONST_CAST(T, expr) (const_cast(expr)) +#elif \ + JSON_HEDLEY_HAS_WARNING("-Wcast-qual") || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,6,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) +# define JSON_HEDLEY_CONST_CAST(T, expr) (__extension__ ({ \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL \ + ((T) (expr)); \ + JSON_HEDLEY_DIAGNOSTIC_POP \ + })) +#else +# define JSON_HEDLEY_CONST_CAST(T, expr) ((T) (expr)) +#endif + +#if defined(JSON_HEDLEY_REINTERPRET_CAST) + #undef JSON_HEDLEY_REINTERPRET_CAST +#endif +#if defined(__cplusplus) + #define JSON_HEDLEY_REINTERPRET_CAST(T, expr) (reinterpret_cast(expr)) +#else + #define JSON_HEDLEY_REINTERPRET_CAST(T, expr) ((T) (expr)) +#endif + +#if defined(JSON_HEDLEY_STATIC_CAST) + #undef JSON_HEDLEY_STATIC_CAST +#endif +#if defined(__cplusplus) + #define JSON_HEDLEY_STATIC_CAST(T, expr) (static_cast(expr)) +#else + #define JSON_HEDLEY_STATIC_CAST(T, expr) ((T) (expr)) +#endif + +#if defined(JSON_HEDLEY_CPP_CAST) + #undef JSON_HEDLEY_CPP_CAST +#endif +#if defined(__cplusplus) +# if JSON_HEDLEY_HAS_WARNING("-Wold-style-cast") +# define JSON_HEDLEY_CPP_CAST(T, expr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wold-style-cast\"") \ + ((T) (expr)) \ + JSON_HEDLEY_DIAGNOSTIC_POP +# elif JSON_HEDLEY_IAR_VERSION_CHECK(8,3,0) +# define JSON_HEDLEY_CPP_CAST(T, expr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("diag_suppress=Pe137") \ + JSON_HEDLEY_DIAGNOSTIC_POP +# else +# define JSON_HEDLEY_CPP_CAST(T, expr) ((T) (expr)) +# endif +#else +# define JSON_HEDLEY_CPP_CAST(T, expr) (expr) +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wdeprecated-declarations") + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("clang diagnostic ignored \"-Wdeprecated-declarations\"") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("warning(disable:1478 1786)") +#elif JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED __pragma(warning(disable:1478 1786)) +#elif JSON_HEDLEY_PGI_VERSION_CHECK(20,7,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1215,1216,1444,1445") +#elif JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1215,1444") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(4,3,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"") +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED __pragma(warning(disable:4996)) +#elif JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1215,1444") +#elif \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1291,1718") +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,13,0) && !defined(__cplusplus) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("error_messages(off,E_DEPRECATED_ATT,E_DEPRECATED_ATT_MESS)") +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,13,0) && defined(__cplusplus) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("error_messages(off,symdeprecated,symdeprecated2)") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress=Pe1444,Pe1215") +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(2,90,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("warn(disable:2241)") +#else + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wunknown-pragmas") + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("clang diagnostic ignored \"-Wunknown-pragmas\"") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("warning(disable:161)") +#elif JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS __pragma(warning(disable:161)) +#elif JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 1675") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(4,3,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("GCC diagnostic ignored \"-Wunknown-pragmas\"") +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS __pragma(warning(disable:4068)) +#elif \ + JSON_HEDLEY_TI_VERSION_CHECK(16,9,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,0,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,3,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 163") +#elif JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 163") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress=Pe161") +#elif JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 161") +#else + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wunknown-attributes") + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("clang diagnostic ignored \"-Wunknown-attributes\"") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(4,6,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(17,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("warning(disable:1292)") +#elif JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES __pragma(warning(disable:1292)) +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(19,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES __pragma(warning(disable:5030)) +#elif JSON_HEDLEY_PGI_VERSION_CHECK(20,7,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress 1097,1098") +#elif JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress 1097") +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,14,0) && defined(__cplusplus) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("error_messages(off,attrskipunsup)") +#elif \ + JSON_HEDLEY_TI_VERSION_CHECK(18,1,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,3,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress 1173") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress=Pe1097") +#elif JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress 1097") +#else + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wcast-qual") + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL _Pragma("clang diagnostic ignored \"-Wcast-qual\"") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL _Pragma("warning(disable:2203 2331)") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(3,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL _Pragma("GCC diagnostic ignored \"-Wcast-qual\"") +#else + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wunused-function") + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION _Pragma("clang diagnostic ignored \"-Wunused-function\"") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(3,4,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION _Pragma("GCC diagnostic ignored \"-Wunused-function\"") +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(1,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION __pragma(warning(disable:4505)) +#elif JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION _Pragma("diag_suppress 3142") +#else + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION +#endif + +#if defined(JSON_HEDLEY_DEPRECATED) + #undef JSON_HEDLEY_DEPRECATED +#endif +#if defined(JSON_HEDLEY_DEPRECATED_FOR) + #undef JSON_HEDLEY_DEPRECATED_FOR +#endif +#if \ + JSON_HEDLEY_MSVC_VERSION_CHECK(14,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DEPRECATED(since) __declspec(deprecated("Since " # since)) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __declspec(deprecated("Since " #since "; use " #replacement)) +#elif \ + (JSON_HEDLEY_HAS_EXTENSION(attribute_deprecated_with_message) && !defined(JSON_HEDLEY_IAR_VERSION)) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,5,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(5,6,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,13,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(18,1,0) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(18,1,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,3,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,3,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_DEPRECATED(since) __attribute__((__deprecated__("Since " #since))) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __attribute__((__deprecated__("Since " #since "; use " #replacement))) +#elif defined(__cplusplus) && (__cplusplus >= 201402L) + #define JSON_HEDLEY_DEPRECATED(since) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[deprecated("Since " #since)]]) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[deprecated("Since " #since "; use " #replacement)]]) +#elif \ + JSON_HEDLEY_HAS_ATTRIBUTE(deprecated) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,10,0) + #define JSON_HEDLEY_DEPRECATED(since) __attribute__((__deprecated__)) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __attribute__((__deprecated__)) +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \ + JSON_HEDLEY_PELLES_VERSION_CHECK(6,50,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DEPRECATED(since) __declspec(deprecated) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __declspec(deprecated) +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_DEPRECATED(since) _Pragma("deprecated") + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) _Pragma("deprecated") +#else + #define JSON_HEDLEY_DEPRECATED(since) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) +#endif + +#if defined(JSON_HEDLEY_UNAVAILABLE) + #undef JSON_HEDLEY_UNAVAILABLE +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(warning) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,3,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_UNAVAILABLE(available_since) __attribute__((__warning__("Not available until " #available_since))) +#else + #define JSON_HEDLEY_UNAVAILABLE(available_since) +#endif + +#if defined(JSON_HEDLEY_WARN_UNUSED_RESULT) + #undef JSON_HEDLEY_WARN_UNUSED_RESULT +#endif +#if defined(JSON_HEDLEY_WARN_UNUSED_RESULT_MSG) + #undef JSON_HEDLEY_WARN_UNUSED_RESULT_MSG +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(warn_unused_result) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,4,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,15,0) && defined(__cplusplus)) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_WARN_UNUSED_RESULT __attribute__((__warn_unused_result__)) + #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) __attribute__((__warn_unused_result__)) +#elif (JSON_HEDLEY_HAS_CPP_ATTRIBUTE(nodiscard) >= 201907L) + #define JSON_HEDLEY_WARN_UNUSED_RESULT JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard]]) + #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard(msg)]]) +#elif JSON_HEDLEY_HAS_CPP_ATTRIBUTE(nodiscard) + #define JSON_HEDLEY_WARN_UNUSED_RESULT JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard]]) + #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard]]) +#elif defined(_Check_return_) /* SAL */ + #define JSON_HEDLEY_WARN_UNUSED_RESULT _Check_return_ + #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) _Check_return_ +#else + #define JSON_HEDLEY_WARN_UNUSED_RESULT + #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) +#endif + +#if defined(JSON_HEDLEY_SENTINEL) + #undef JSON_HEDLEY_SENTINEL +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(sentinel) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(5,4,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_SENTINEL(position) __attribute__((__sentinel__(position))) +#else + #define JSON_HEDLEY_SENTINEL(position) +#endif + +#if defined(JSON_HEDLEY_NO_RETURN) + #undef JSON_HEDLEY_NO_RETURN +#endif +#if JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_NO_RETURN __noreturn +#elif \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_NO_RETURN __attribute__((__noreturn__)) +#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L + #define JSON_HEDLEY_NO_RETURN _Noreturn +#elif defined(__cplusplus) && (__cplusplus >= 201103L) + #define JSON_HEDLEY_NO_RETURN JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[noreturn]]) +#elif \ + JSON_HEDLEY_HAS_ATTRIBUTE(noreturn) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,2,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,10,0) + #define JSON_HEDLEY_NO_RETURN __attribute__((__noreturn__)) +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) + #define JSON_HEDLEY_NO_RETURN _Pragma("does_not_return") +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_NO_RETURN __declspec(noreturn) +#elif JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,0,0) && defined(__cplusplus) + #define JSON_HEDLEY_NO_RETURN _Pragma("FUNC_NEVER_RETURNS;") +#elif JSON_HEDLEY_COMPCERT_VERSION_CHECK(3,2,0) + #define JSON_HEDLEY_NO_RETURN __attribute((noreturn)) +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(9,0,0) + #define JSON_HEDLEY_NO_RETURN __declspec(noreturn) +#else + #define JSON_HEDLEY_NO_RETURN +#endif + +#if defined(JSON_HEDLEY_NO_ESCAPE) + #undef JSON_HEDLEY_NO_ESCAPE +#endif +#if JSON_HEDLEY_HAS_ATTRIBUTE(noescape) + #define JSON_HEDLEY_NO_ESCAPE __attribute__((__noescape__)) +#else + #define JSON_HEDLEY_NO_ESCAPE +#endif + +#if defined(JSON_HEDLEY_UNREACHABLE) + #undef JSON_HEDLEY_UNREACHABLE +#endif +#if defined(JSON_HEDLEY_UNREACHABLE_RETURN) + #undef JSON_HEDLEY_UNREACHABLE_RETURN +#endif +#if defined(JSON_HEDLEY_ASSUME) + #undef JSON_HEDLEY_ASSUME +#endif +#if \ + JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_ASSUME(expr) __assume(expr) +#elif JSON_HEDLEY_HAS_BUILTIN(__builtin_assume) + #define JSON_HEDLEY_ASSUME(expr) __builtin_assume(expr) +#elif \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,2,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(4,0,0) + #if defined(__cplusplus) + #define JSON_HEDLEY_ASSUME(expr) std::_nassert(expr) + #else + #define JSON_HEDLEY_ASSUME(expr) _nassert(expr) + #endif +#endif +#if \ + (JSON_HEDLEY_HAS_BUILTIN(__builtin_unreachable) && (!defined(JSON_HEDLEY_ARM_VERSION))) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,5,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(18,10,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(13,1,5) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(10,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_UNREACHABLE() __builtin_unreachable() +#elif defined(JSON_HEDLEY_ASSUME) + #define JSON_HEDLEY_UNREACHABLE() JSON_HEDLEY_ASSUME(0) +#endif +#if !defined(JSON_HEDLEY_ASSUME) + #if defined(JSON_HEDLEY_UNREACHABLE) + #define JSON_HEDLEY_ASSUME(expr) JSON_HEDLEY_STATIC_CAST(void, ((expr) ? 1 : (JSON_HEDLEY_UNREACHABLE(), 1))) + #else + #define JSON_HEDLEY_ASSUME(expr) JSON_HEDLEY_STATIC_CAST(void, expr) + #endif +#endif +#if defined(JSON_HEDLEY_UNREACHABLE) + #if \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,2,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(4,0,0) + #define JSON_HEDLEY_UNREACHABLE_RETURN(value) return (JSON_HEDLEY_STATIC_CAST(void, JSON_HEDLEY_ASSUME(0)), (value)) + #else + #define JSON_HEDLEY_UNREACHABLE_RETURN(value) JSON_HEDLEY_UNREACHABLE() + #endif +#else + #define JSON_HEDLEY_UNREACHABLE_RETURN(value) return (value) +#endif +#if !defined(JSON_HEDLEY_UNREACHABLE) + #define JSON_HEDLEY_UNREACHABLE() JSON_HEDLEY_ASSUME(0) +#endif + +JSON_HEDLEY_DIAGNOSTIC_PUSH +#if JSON_HEDLEY_HAS_WARNING("-Wpedantic") + #pragma clang diagnostic ignored "-Wpedantic" +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wc++98-compat-pedantic") && defined(__cplusplus) + #pragma clang diagnostic ignored "-Wc++98-compat-pedantic" +#endif +#if JSON_HEDLEY_GCC_HAS_WARNING("-Wvariadic-macros",4,0,0) + #if defined(__clang__) + #pragma clang diagnostic ignored "-Wvariadic-macros" + #elif defined(JSON_HEDLEY_GCC_VERSION) + #pragma GCC diagnostic ignored "-Wvariadic-macros" + #endif +#endif +#if defined(JSON_HEDLEY_NON_NULL) + #undef JSON_HEDLEY_NON_NULL +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(nonnull) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,3,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) + #define JSON_HEDLEY_NON_NULL(...) __attribute__((__nonnull__(__VA_ARGS__))) +#else + #define JSON_HEDLEY_NON_NULL(...) +#endif +JSON_HEDLEY_DIAGNOSTIC_POP + +#if defined(JSON_HEDLEY_PRINTF_FORMAT) + #undef JSON_HEDLEY_PRINTF_FORMAT +#endif +#if defined(__MINGW32__) && JSON_HEDLEY_GCC_HAS_ATTRIBUTE(format,4,4,0) && !defined(__USE_MINGW_ANSI_STDIO) + #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __attribute__((__format__(ms_printf, string_idx, first_to_check))) +#elif defined(__MINGW32__) && JSON_HEDLEY_GCC_HAS_ATTRIBUTE(format,4,4,0) && defined(__USE_MINGW_ANSI_STDIO) + #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __attribute__((__format__(gnu_printf, string_idx, first_to_check))) +#elif \ + JSON_HEDLEY_HAS_ATTRIBUTE(format) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(5,6,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __attribute__((__format__(__printf__, string_idx, first_to_check))) +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(6,0,0) + #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __declspec(vaformat(printf,string_idx,first_to_check)) +#else + #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) +#endif + +#if defined(JSON_HEDLEY_CONSTEXPR) + #undef JSON_HEDLEY_CONSTEXPR +#endif +#if defined(__cplusplus) + #if __cplusplus >= 201103L + #define JSON_HEDLEY_CONSTEXPR JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(constexpr) + #endif +#endif +#if !defined(JSON_HEDLEY_CONSTEXPR) + #define JSON_HEDLEY_CONSTEXPR +#endif + +#if defined(JSON_HEDLEY_PREDICT) + #undef JSON_HEDLEY_PREDICT +#endif +#if defined(JSON_HEDLEY_LIKELY) + #undef JSON_HEDLEY_LIKELY +#endif +#if defined(JSON_HEDLEY_UNLIKELY) + #undef JSON_HEDLEY_UNLIKELY +#endif +#if defined(JSON_HEDLEY_UNPREDICTABLE) + #undef JSON_HEDLEY_UNPREDICTABLE +#endif +#if JSON_HEDLEY_HAS_BUILTIN(__builtin_unpredictable) + #define JSON_HEDLEY_UNPREDICTABLE(expr) __builtin_unpredictable((expr)) +#endif +#if \ + (JSON_HEDLEY_HAS_BUILTIN(__builtin_expect_with_probability) && !defined(JSON_HEDLEY_PGI_VERSION)) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(9,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) +# define JSON_HEDLEY_PREDICT(expr, value, probability) __builtin_expect_with_probability( (expr), (value), (probability)) +# define JSON_HEDLEY_PREDICT_TRUE(expr, probability) __builtin_expect_with_probability(!!(expr), 1 , (probability)) +# define JSON_HEDLEY_PREDICT_FALSE(expr, probability) __builtin_expect_with_probability(!!(expr), 0 , (probability)) +# define JSON_HEDLEY_LIKELY(expr) __builtin_expect (!!(expr), 1 ) +# define JSON_HEDLEY_UNLIKELY(expr) __builtin_expect (!!(expr), 0 ) +#elif \ + (JSON_HEDLEY_HAS_BUILTIN(__builtin_expect) && !defined(JSON_HEDLEY_INTEL_CL_VERSION)) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,15,0) && defined(__cplusplus)) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,7,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,1,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,1,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,27) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(8,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) +# define JSON_HEDLEY_PREDICT(expr, expected, probability) \ + (((probability) >= 0.9) ? __builtin_expect((expr), (expected)) : (JSON_HEDLEY_STATIC_CAST(void, expected), (expr))) +# define JSON_HEDLEY_PREDICT_TRUE(expr, probability) \ + (__extension__ ({ \ + double hedley_probability_ = (probability); \ + ((hedley_probability_ >= 0.9) ? __builtin_expect(!!(expr), 1) : ((hedley_probability_ <= 0.1) ? __builtin_expect(!!(expr), 0) : !!(expr))); \ + })) +# define JSON_HEDLEY_PREDICT_FALSE(expr, probability) \ + (__extension__ ({ \ + double hedley_probability_ = (probability); \ + ((hedley_probability_ >= 0.9) ? __builtin_expect(!!(expr), 0) : ((hedley_probability_ <= 0.1) ? __builtin_expect(!!(expr), 1) : !!(expr))); \ + })) +# define JSON_HEDLEY_LIKELY(expr) __builtin_expect(!!(expr), 1) +# define JSON_HEDLEY_UNLIKELY(expr) __builtin_expect(!!(expr), 0) +#else +# define JSON_HEDLEY_PREDICT(expr, expected, probability) (JSON_HEDLEY_STATIC_CAST(void, expected), (expr)) +# define JSON_HEDLEY_PREDICT_TRUE(expr, probability) (!!(expr)) +# define JSON_HEDLEY_PREDICT_FALSE(expr, probability) (!!(expr)) +# define JSON_HEDLEY_LIKELY(expr) (!!(expr)) +# define JSON_HEDLEY_UNLIKELY(expr) (!!(expr)) +#endif +#if !defined(JSON_HEDLEY_UNPREDICTABLE) + #define JSON_HEDLEY_UNPREDICTABLE(expr) JSON_HEDLEY_PREDICT(expr, 1, 0.5) +#endif + +#if defined(JSON_HEDLEY_MALLOC) + #undef JSON_HEDLEY_MALLOC +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(malloc) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(12,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_MALLOC __attribute__((__malloc__)) +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) + #define JSON_HEDLEY_MALLOC _Pragma("returns_new_memory") +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(14,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_MALLOC __declspec(restrict) +#else + #define JSON_HEDLEY_MALLOC +#endif + +#if defined(JSON_HEDLEY_PURE) + #undef JSON_HEDLEY_PURE +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(pure) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(2,96,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) +# define JSON_HEDLEY_PURE __attribute__((__pure__)) +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) +# define JSON_HEDLEY_PURE _Pragma("does_not_write_global_data") +#elif defined(__cplusplus) && \ + ( \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(2,0,1) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(4,0,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) \ + ) +# define JSON_HEDLEY_PURE _Pragma("FUNC_IS_PURE;") +#else +# define JSON_HEDLEY_PURE +#endif + +#if defined(JSON_HEDLEY_CONST) + #undef JSON_HEDLEY_CONST +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(const) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(2,5,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_CONST __attribute__((__const__)) +#elif \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) + #define JSON_HEDLEY_CONST _Pragma("no_side_effect") +#else + #define JSON_HEDLEY_CONST JSON_HEDLEY_PURE +#endif + +#if defined(JSON_HEDLEY_RESTRICT) + #undef JSON_HEDLEY_RESTRICT +#endif +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && !defined(__cplusplus) + #define JSON_HEDLEY_RESTRICT restrict +#elif \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_MSVC_VERSION_CHECK(14,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,2,4) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,1,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,14,0) && defined(__cplusplus)) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) || \ + defined(__clang__) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_RESTRICT __restrict +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,3,0) && !defined(__cplusplus) + #define JSON_HEDLEY_RESTRICT _Restrict +#else + #define JSON_HEDLEY_RESTRICT +#endif + +#if defined(JSON_HEDLEY_INLINE) + #undef JSON_HEDLEY_INLINE +#endif +#if \ + (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) || \ + (defined(__cplusplus) && (__cplusplus >= 199711L)) + #define JSON_HEDLEY_INLINE inline +#elif \ + defined(JSON_HEDLEY_GCC_VERSION) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(6,2,0) + #define JSON_HEDLEY_INLINE __inline__ +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(12,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,1,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,2,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,0,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_INLINE __inline +#else + #define JSON_HEDLEY_INLINE +#endif + +#if defined(JSON_HEDLEY_ALWAYS_INLINE) + #undef JSON_HEDLEY_ALWAYS_INLINE +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(always_inline) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,10,0) +# define JSON_HEDLEY_ALWAYS_INLINE __attribute__((__always_inline__)) JSON_HEDLEY_INLINE +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(12,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) +# define JSON_HEDLEY_ALWAYS_INLINE __forceinline +#elif defined(__cplusplus) && \ + ( \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,1,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) \ + ) +# define JSON_HEDLEY_ALWAYS_INLINE _Pragma("FUNC_ALWAYS_INLINE;") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) +# define JSON_HEDLEY_ALWAYS_INLINE _Pragma("inline=forced") +#else +# define JSON_HEDLEY_ALWAYS_INLINE JSON_HEDLEY_INLINE +#endif + +#if defined(JSON_HEDLEY_NEVER_INLINE) + #undef JSON_HEDLEY_NEVER_INLINE +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(noinline) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,10,0) + #define JSON_HEDLEY_NEVER_INLINE __attribute__((__noinline__)) +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_NEVER_INLINE __declspec(noinline) +#elif JSON_HEDLEY_PGI_VERSION_CHECK(10,2,0) + #define JSON_HEDLEY_NEVER_INLINE _Pragma("noinline") +#elif JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,0,0) && defined(__cplusplus) + #define JSON_HEDLEY_NEVER_INLINE _Pragma("FUNC_CANNOT_INLINE;") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_NEVER_INLINE _Pragma("inline=never") +#elif JSON_HEDLEY_COMPCERT_VERSION_CHECK(3,2,0) + #define JSON_HEDLEY_NEVER_INLINE __attribute((noinline)) +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(9,0,0) + #define JSON_HEDLEY_NEVER_INLINE __declspec(noinline) +#else + #define JSON_HEDLEY_NEVER_INLINE +#endif + +#if defined(JSON_HEDLEY_PRIVATE) + #undef JSON_HEDLEY_PRIVATE +#endif +#if defined(JSON_HEDLEY_PUBLIC) + #undef JSON_HEDLEY_PUBLIC +#endif +#if defined(JSON_HEDLEY_IMPORT) + #undef JSON_HEDLEY_IMPORT +#endif +#if defined(_WIN32) || defined(__CYGWIN__) +# define JSON_HEDLEY_PRIVATE +# define JSON_HEDLEY_PUBLIC __declspec(dllexport) +# define JSON_HEDLEY_IMPORT __declspec(dllimport) +#else +# if \ + JSON_HEDLEY_HAS_ATTRIBUTE(visibility) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,3,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(13,1,0) || \ + ( \ + defined(__TI_EABI__) && \ + ( \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) \ + ) \ + ) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) +# define JSON_HEDLEY_PRIVATE __attribute__((__visibility__("hidden"))) +# define JSON_HEDLEY_PUBLIC __attribute__((__visibility__("default"))) +# else +# define JSON_HEDLEY_PRIVATE +# define JSON_HEDLEY_PUBLIC +# endif +# define JSON_HEDLEY_IMPORT extern +#endif + +#if defined(JSON_HEDLEY_NO_THROW) + #undef JSON_HEDLEY_NO_THROW +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(nothrow) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,3,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_NO_THROW __attribute__((__nothrow__)) +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(13,1,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) + #define JSON_HEDLEY_NO_THROW __declspec(nothrow) +#else + #define JSON_HEDLEY_NO_THROW +#endif + +#if defined(JSON_HEDLEY_FALL_THROUGH) + #undef JSON_HEDLEY_FALL_THROUGH +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(fallthrough) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(7,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_FALL_THROUGH __attribute__((__fallthrough__)) +#elif JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS(clang,fallthrough) + #define JSON_HEDLEY_FALL_THROUGH JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[clang::fallthrough]]) +#elif JSON_HEDLEY_HAS_CPP_ATTRIBUTE(fallthrough) + #define JSON_HEDLEY_FALL_THROUGH JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[fallthrough]]) +#elif defined(__fallthrough) /* SAL */ + #define JSON_HEDLEY_FALL_THROUGH __fallthrough +#else + #define JSON_HEDLEY_FALL_THROUGH +#endif + +#if defined(JSON_HEDLEY_RETURNS_NON_NULL) + #undef JSON_HEDLEY_RETURNS_NON_NULL +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(returns_nonnull) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,9,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_RETURNS_NON_NULL __attribute__((__returns_nonnull__)) +#elif defined(_Ret_notnull_) /* SAL */ + #define JSON_HEDLEY_RETURNS_NON_NULL _Ret_notnull_ +#else + #define JSON_HEDLEY_RETURNS_NON_NULL +#endif + +#if defined(JSON_HEDLEY_ARRAY_PARAM) + #undef JSON_HEDLEY_ARRAY_PARAM +#endif +#if \ + defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && \ + !defined(__STDC_NO_VLA__) && \ + !defined(__cplusplus) && \ + !defined(JSON_HEDLEY_PGI_VERSION) && \ + !defined(JSON_HEDLEY_TINYC_VERSION) + #define JSON_HEDLEY_ARRAY_PARAM(name) (name) +#else + #define JSON_HEDLEY_ARRAY_PARAM(name) +#endif + +#if defined(JSON_HEDLEY_IS_CONSTANT) + #undef JSON_HEDLEY_IS_CONSTANT +#endif +#if defined(JSON_HEDLEY_REQUIRE_CONSTEXPR) + #undef JSON_HEDLEY_REQUIRE_CONSTEXPR +#endif +/* JSON_HEDLEY_IS_CONSTEXPR_ is for + HEDLEY INTERNAL USE ONLY. API subject to change without notice. */ +#if defined(JSON_HEDLEY_IS_CONSTEXPR_) + #undef JSON_HEDLEY_IS_CONSTEXPR_ +#endif +#if \ + JSON_HEDLEY_HAS_BUILTIN(__builtin_constant_p) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,4,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,19) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(13,1,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,1,0) || \ + (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) && !defined(__cplusplus)) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(8,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_IS_CONSTANT(expr) __builtin_constant_p(expr) +#endif +#if !defined(__cplusplus) +# if \ + JSON_HEDLEY_HAS_BUILTIN(__builtin_types_compatible_p) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,4,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(13,1,0) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(8,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(5,4,0) || \ + JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,24) +#if defined(__INTPTR_TYPE__) + #define JSON_HEDLEY_IS_CONSTEXPR_(expr) __builtin_types_compatible_p(__typeof__((1 ? (void*) ((__INTPTR_TYPE__) ((expr) * 0)) : (int*) 0)), int*) +#else + #include + #define JSON_HEDLEY_IS_CONSTEXPR_(expr) __builtin_types_compatible_p(__typeof__((1 ? (void*) ((intptr_t) ((expr) * 0)) : (int*) 0)), int*) +#endif +# elif \ + ( \ + defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) && \ + !defined(JSON_HEDLEY_SUNPRO_VERSION) && \ + !defined(JSON_HEDLEY_PGI_VERSION) && \ + !defined(JSON_HEDLEY_IAR_VERSION)) || \ + (JSON_HEDLEY_HAS_EXTENSION(c_generic_selections) && !defined(JSON_HEDLEY_IAR_VERSION)) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,9,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(17,0,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(12,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(5,3,0) +#if defined(__INTPTR_TYPE__) + #define JSON_HEDLEY_IS_CONSTEXPR_(expr) _Generic((1 ? (void*) ((__INTPTR_TYPE__) ((expr) * 0)) : (int*) 0), int*: 1, void*: 0) +#else + #include + #define JSON_HEDLEY_IS_CONSTEXPR_(expr) _Generic((1 ? (void*) ((intptr_t) * 0) : (int*) 0), int*: 1, void*: 0) +#endif +# elif \ + defined(JSON_HEDLEY_GCC_VERSION) || \ + defined(JSON_HEDLEY_INTEL_VERSION) || \ + defined(JSON_HEDLEY_TINYC_VERSION) || \ + defined(JSON_HEDLEY_TI_ARMCL_VERSION) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(18,12,0) || \ + defined(JSON_HEDLEY_TI_CL2000_VERSION) || \ + defined(JSON_HEDLEY_TI_CL6X_VERSION) || \ + defined(JSON_HEDLEY_TI_CL7X_VERSION) || \ + defined(JSON_HEDLEY_TI_CLPRU_VERSION) || \ + defined(__clang__) +# define JSON_HEDLEY_IS_CONSTEXPR_(expr) ( \ + sizeof(void) != \ + sizeof(*( \ + 1 ? \ + ((void*) ((expr) * 0L) ) : \ +((struct { char v[sizeof(void) * 2]; } *) 1) \ + ) \ + ) \ + ) +# endif +#endif +#if defined(JSON_HEDLEY_IS_CONSTEXPR_) + #if !defined(JSON_HEDLEY_IS_CONSTANT) + #define JSON_HEDLEY_IS_CONSTANT(expr) JSON_HEDLEY_IS_CONSTEXPR_(expr) + #endif + #define JSON_HEDLEY_REQUIRE_CONSTEXPR(expr) (JSON_HEDLEY_IS_CONSTEXPR_(expr) ? (expr) : (-1)) +#else + #if !defined(JSON_HEDLEY_IS_CONSTANT) + #define JSON_HEDLEY_IS_CONSTANT(expr) (0) + #endif + #define JSON_HEDLEY_REQUIRE_CONSTEXPR(expr) (expr) +#endif + +#if defined(JSON_HEDLEY_BEGIN_C_DECLS) + #undef JSON_HEDLEY_BEGIN_C_DECLS +#endif +#if defined(JSON_HEDLEY_END_C_DECLS) + #undef JSON_HEDLEY_END_C_DECLS +#endif +#if defined(JSON_HEDLEY_C_DECL) + #undef JSON_HEDLEY_C_DECL +#endif +#if defined(__cplusplus) + #define JSON_HEDLEY_BEGIN_C_DECLS extern "C" { + #define JSON_HEDLEY_END_C_DECLS } + #define JSON_HEDLEY_C_DECL extern "C" +#else + #define JSON_HEDLEY_BEGIN_C_DECLS + #define JSON_HEDLEY_END_C_DECLS + #define JSON_HEDLEY_C_DECL +#endif + +#if defined(JSON_HEDLEY_STATIC_ASSERT) + #undef JSON_HEDLEY_STATIC_ASSERT +#endif +#if \ + !defined(__cplusplus) && ( \ + (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) || \ + (JSON_HEDLEY_HAS_FEATURE(c_static_assert) && !defined(JSON_HEDLEY_INTEL_CL_VERSION)) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(6,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + defined(_Static_assert) \ + ) +# define JSON_HEDLEY_STATIC_ASSERT(expr, message) _Static_assert(expr, message) +#elif \ + (defined(__cplusplus) && (__cplusplus >= 201103L)) || \ + JSON_HEDLEY_MSVC_VERSION_CHECK(16,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) +# define JSON_HEDLEY_STATIC_ASSERT(expr, message) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(static_assert(expr, message)) +#else +# define JSON_HEDLEY_STATIC_ASSERT(expr, message) +#endif + +#if defined(JSON_HEDLEY_NULL) + #undef JSON_HEDLEY_NULL +#endif +#if defined(__cplusplus) + #if __cplusplus >= 201103L + #define JSON_HEDLEY_NULL JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(nullptr) + #elif defined(NULL) + #define JSON_HEDLEY_NULL NULL + #else + #define JSON_HEDLEY_NULL JSON_HEDLEY_STATIC_CAST(void*, 0) + #endif +#elif defined(NULL) + #define JSON_HEDLEY_NULL NULL +#else + #define JSON_HEDLEY_NULL ((void*) 0) +#endif + +#if defined(JSON_HEDLEY_MESSAGE) + #undef JSON_HEDLEY_MESSAGE +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wunknown-pragmas") +# define JSON_HEDLEY_MESSAGE(msg) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS \ + JSON_HEDLEY_PRAGMA(message msg) \ + JSON_HEDLEY_DIAGNOSTIC_POP +#elif \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,4,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) +# define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(message msg) +#elif JSON_HEDLEY_CRAY_VERSION_CHECK(5,0,0) +# define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(_CRI message msg) +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) +# define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(message(msg)) +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(2,0,0) +# define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(message(msg)) +#else +# define JSON_HEDLEY_MESSAGE(msg) +#endif + +#if defined(JSON_HEDLEY_WARNING) + #undef JSON_HEDLEY_WARNING +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wunknown-pragmas") +# define JSON_HEDLEY_WARNING(msg) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS \ + JSON_HEDLEY_PRAGMA(clang warning msg) \ + JSON_HEDLEY_DIAGNOSTIC_POP +#elif \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,8,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(18,4,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) +# define JSON_HEDLEY_WARNING(msg) JSON_HEDLEY_PRAGMA(GCC warning msg) +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) +# define JSON_HEDLEY_WARNING(msg) JSON_HEDLEY_PRAGMA(message(msg)) +#else +# define JSON_HEDLEY_WARNING(msg) JSON_HEDLEY_MESSAGE(msg) +#endif + +#if defined(JSON_HEDLEY_REQUIRE) + #undef JSON_HEDLEY_REQUIRE +#endif +#if defined(JSON_HEDLEY_REQUIRE_MSG) + #undef JSON_HEDLEY_REQUIRE_MSG +#endif +#if JSON_HEDLEY_HAS_ATTRIBUTE(diagnose_if) +# if JSON_HEDLEY_HAS_WARNING("-Wgcc-compat") +# define JSON_HEDLEY_REQUIRE(expr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wgcc-compat\"") \ + __attribute__((diagnose_if(!(expr), #expr, "error"))) \ + JSON_HEDLEY_DIAGNOSTIC_POP +# define JSON_HEDLEY_REQUIRE_MSG(expr,msg) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wgcc-compat\"") \ + __attribute__((diagnose_if(!(expr), msg, "error"))) \ + JSON_HEDLEY_DIAGNOSTIC_POP +# else +# define JSON_HEDLEY_REQUIRE(expr) __attribute__((diagnose_if(!(expr), #expr, "error"))) +# define JSON_HEDLEY_REQUIRE_MSG(expr,msg) __attribute__((diagnose_if(!(expr), msg, "error"))) +# endif +#else +# define JSON_HEDLEY_REQUIRE(expr) +# define JSON_HEDLEY_REQUIRE_MSG(expr,msg) +#endif + +#if defined(JSON_HEDLEY_FLAGS) + #undef JSON_HEDLEY_FLAGS +#endif +#if JSON_HEDLEY_HAS_ATTRIBUTE(flag_enum) && (!defined(__cplusplus) || JSON_HEDLEY_HAS_WARNING("-Wbitfield-enum-conversion")) + #define JSON_HEDLEY_FLAGS __attribute__((__flag_enum__)) +#else + #define JSON_HEDLEY_FLAGS +#endif + +#if defined(JSON_HEDLEY_FLAGS_CAST) + #undef JSON_HEDLEY_FLAGS_CAST +#endif +#if JSON_HEDLEY_INTEL_VERSION_CHECK(19,0,0) +# define JSON_HEDLEY_FLAGS_CAST(T, expr) (__extension__ ({ \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("warning(disable:188)") \ + ((T) (expr)); \ + JSON_HEDLEY_DIAGNOSTIC_POP \ + })) +#else +# define JSON_HEDLEY_FLAGS_CAST(T, expr) JSON_HEDLEY_STATIC_CAST(T, expr) +#endif + +#if defined(JSON_HEDLEY_EMPTY_BASES) + #undef JSON_HEDLEY_EMPTY_BASES +#endif +#if \ + (JSON_HEDLEY_MSVC_VERSION_CHECK(19,0,23918) && !JSON_HEDLEY_MSVC_VERSION_CHECK(20,0,0)) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_EMPTY_BASES __declspec(empty_bases) +#else + #define JSON_HEDLEY_EMPTY_BASES +#endif + +/* Remaining macros are deprecated. */ + +#if defined(JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK) + #undef JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK +#endif +#if defined(__clang__) + #define JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK(major,minor,patch) (0) +#else + #define JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK(major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_CLANG_HAS_ATTRIBUTE) + #undef JSON_HEDLEY_CLANG_HAS_ATTRIBUTE +#endif +#define JSON_HEDLEY_CLANG_HAS_ATTRIBUTE(attribute) JSON_HEDLEY_HAS_ATTRIBUTE(attribute) + +#if defined(JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE) + #undef JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE +#endif +#define JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE(attribute) JSON_HEDLEY_HAS_CPP_ATTRIBUTE(attribute) + +#if defined(JSON_HEDLEY_CLANG_HAS_BUILTIN) + #undef JSON_HEDLEY_CLANG_HAS_BUILTIN +#endif +#define JSON_HEDLEY_CLANG_HAS_BUILTIN(builtin) JSON_HEDLEY_HAS_BUILTIN(builtin) + +#if defined(JSON_HEDLEY_CLANG_HAS_FEATURE) + #undef JSON_HEDLEY_CLANG_HAS_FEATURE +#endif +#define JSON_HEDLEY_CLANG_HAS_FEATURE(feature) JSON_HEDLEY_HAS_FEATURE(feature) + +#if defined(JSON_HEDLEY_CLANG_HAS_EXTENSION) + #undef JSON_HEDLEY_CLANG_HAS_EXTENSION +#endif +#define JSON_HEDLEY_CLANG_HAS_EXTENSION(extension) JSON_HEDLEY_HAS_EXTENSION(extension) + +#if defined(JSON_HEDLEY_CLANG_HAS_DECLSPEC_DECLSPEC_ATTRIBUTE) + #undef JSON_HEDLEY_CLANG_HAS_DECLSPEC_DECLSPEC_ATTRIBUTE +#endif +#define JSON_HEDLEY_CLANG_HAS_DECLSPEC_ATTRIBUTE(attribute) JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE(attribute) + +#if defined(JSON_HEDLEY_CLANG_HAS_WARNING) + #undef JSON_HEDLEY_CLANG_HAS_WARNING +#endif +#define JSON_HEDLEY_CLANG_HAS_WARNING(warning) JSON_HEDLEY_HAS_WARNING(warning) + +#endif /* !defined(JSON_HEDLEY_VERSION) || (JSON_HEDLEY_VERSION < X) */ + +// #include + + +#include + +// #include + + +namespace nlohmann +{ +namespace detail +{ +template struct make_void +{ + using type = void; +}; +template using void_t = typename make_void::type; +} // namespace detail +} // namespace nlohmann + + +// https://en.cppreference.com/w/cpp/experimental/is_detected +namespace nlohmann +{ +namespace detail +{ +struct nonesuch +{ + nonesuch() = delete; + ~nonesuch() = delete; + nonesuch(nonesuch const&) = delete; + nonesuch(nonesuch const&&) = delete; + void operator=(nonesuch const&) = delete; + void operator=(nonesuch&&) = delete; +}; + +template class Op, + class... Args> +struct detector +{ + using value_t = std::false_type; + using type = Default; +}; + +template class Op, class... Args> +struct detector>, Op, Args...> +{ + using value_t = std::true_type; + using type = Op; +}; + +template class Op, class... Args> +using is_detected = typename detector::value_t; + +template class Op, class... Args> +struct is_detected_lazy : is_detected { }; + +template class Op, class... Args> +using detected_t = typename detector::type; + +template class Op, class... Args> +using detected_or = detector; + +template class Op, class... Args> +using detected_or_t = typename detected_or::type; + +template class Op, class... Args> +using is_detected_exact = std::is_same>; + +template class Op, class... Args> +using is_detected_convertible = + std::is_convertible, To>; +} // namespace detail +} // namespace nlohmann + + +// This file contains all internal macro definitions +// You MUST include macro_unscope.hpp at the end of json.hpp to undef all of them + +// exclude unsupported compilers +#if !defined(JSON_SKIP_UNSUPPORTED_COMPILER_CHECK) + #if defined(__clang__) + #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400 + #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" + #endif + #elif defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER)) + #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40800 + #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" + #endif + #endif +#endif + +// C++ language standard detection +// if the user manually specified the used c++ version this is skipped +#if !defined(JSON_HAS_CPP_20) && !defined(JSON_HAS_CPP_17) && !defined(JSON_HAS_CPP_14) && !defined(JSON_HAS_CPP_11) + #if (defined(__cplusplus) && __cplusplus >= 202002L) || (defined(_MSVC_LANG) && _MSVC_LANG >= 202002L) + #define JSON_HAS_CPP_20 + #define JSON_HAS_CPP_17 + #define JSON_HAS_CPP_14 + #elif (defined(__cplusplus) && __cplusplus >= 201703L) || (defined(_HAS_CXX17) && _HAS_CXX17 == 1) // fix for issue #464 + #define JSON_HAS_CPP_17 + #define JSON_HAS_CPP_14 + #elif (defined(__cplusplus) && __cplusplus >= 201402L) || (defined(_HAS_CXX14) && _HAS_CXX14 == 1) + #define JSON_HAS_CPP_14 + #endif + // the cpp 11 flag is always specified because it is the minimal required version + #define JSON_HAS_CPP_11 +#endif + +#if !defined(JSON_HAS_FILESYSTEM) && !defined(JSON_HAS_EXPERIMENTAL_FILESYSTEM) + #ifdef JSON_HAS_CPP_17 + #if defined(__cpp_lib_filesystem) + #define JSON_HAS_FILESYSTEM 1 + #elif defined(__cpp_lib_experimental_filesystem) + #define JSON_HAS_EXPERIMENTAL_FILESYSTEM 1 + #elif !defined(__has_include) + #define JSON_HAS_EXPERIMENTAL_FILESYSTEM 1 + #elif __has_include() + #define JSON_HAS_FILESYSTEM 1 + #elif __has_include() + #define JSON_HAS_EXPERIMENTAL_FILESYSTEM 1 + #endif + + // std::filesystem does not work on MinGW GCC 8: https://sourceforge.net/p/mingw-w64/bugs/737/ + #if defined(__MINGW32__) && defined(__GNUC__) && __GNUC__ == 8 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + + // no filesystem support before GCC 8: https://en.cppreference.com/w/cpp/compiler_support + #if defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 8 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + + // no filesystem support before Clang 7: https://en.cppreference.com/w/cpp/compiler_support + #if defined(__clang_major__) && __clang_major__ < 7 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + + // no filesystem support before MSVC 19.14: https://en.cppreference.com/w/cpp/compiler_support + #if defined(_MSC_VER) && _MSC_VER < 1940 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + + // no filesystem support before iOS 13 + #if defined(__IPHONE_OS_VERSION_MIN_REQUIRED) && __IPHONE_OS_VERSION_MIN_REQUIRED < 130000 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + + // no filesystem support before macOS Catalina + #if defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && __MAC_OS_X_VERSION_MIN_REQUIRED < 101500 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + #endif +#endif + +#ifndef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #define JSON_HAS_EXPERIMENTAL_FILESYSTEM 0 +#endif + +#ifndef JSON_HAS_FILESYSTEM + #define JSON_HAS_FILESYSTEM 0 +#endif + +// disable documentation warnings on clang +#if defined(__clang__) + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wdocumentation" + #pragma clang diagnostic ignored "-Wdocumentation-unknown-command" +#endif + +// allow disabling exceptions +#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION) + #define JSON_THROW(exception) throw exception + #define JSON_TRY try + #define JSON_CATCH(exception) catch(exception) + #define JSON_INTERNAL_CATCH(exception) catch(exception) +#else + #include + #define JSON_THROW(exception) std::abort() + #define JSON_TRY if(true) + #define JSON_CATCH(exception) if(false) + #define JSON_INTERNAL_CATCH(exception) if(false) +#endif + +// override exception macros +#if defined(JSON_THROW_USER) + #undef JSON_THROW + #define JSON_THROW JSON_THROW_USER +#endif +#if defined(JSON_TRY_USER) + #undef JSON_TRY + #define JSON_TRY JSON_TRY_USER +#endif +#if defined(JSON_CATCH_USER) + #undef JSON_CATCH + #define JSON_CATCH JSON_CATCH_USER + #undef JSON_INTERNAL_CATCH + #define JSON_INTERNAL_CATCH JSON_CATCH_USER +#endif +#if defined(JSON_INTERNAL_CATCH_USER) + #undef JSON_INTERNAL_CATCH + #define JSON_INTERNAL_CATCH JSON_INTERNAL_CATCH_USER +#endif + +// allow overriding assert +#if !defined(JSON_ASSERT) + #include // assert + #define JSON_ASSERT(x) assert(x) +#endif + +// allow to access some private functions (needed by the test suite) +#if defined(JSON_TESTS_PRIVATE) + #define JSON_PRIVATE_UNLESS_TESTED public +#else + #define JSON_PRIVATE_UNLESS_TESTED private +#endif + +/*! +@brief macro to briefly define a mapping between an enum and JSON +@def NLOHMANN_JSON_SERIALIZE_ENUM +@since version 3.4.0 +*/ +#define NLOHMANN_JSON_SERIALIZE_ENUM(ENUM_TYPE, ...) \ + template \ + inline void to_json(BasicJsonType& j, const ENUM_TYPE& e) \ + { \ + static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ + static const std::pair m[] = __VA_ARGS__; \ + auto it = std::find_if(std::begin(m), std::end(m), \ + [e](const std::pair& ej_pair) -> bool \ + { \ + return ej_pair.first == e; \ + }); \ + j = ((it != std::end(m)) ? it : std::begin(m))->second; \ + } \ + template \ + inline void from_json(const BasicJsonType& j, ENUM_TYPE& e) \ + { \ + static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ + static const std::pair m[] = __VA_ARGS__; \ + auto it = std::find_if(std::begin(m), std::end(m), \ + [&j](const std::pair& ej_pair) -> bool \ + { \ + return ej_pair.second == j; \ + }); \ + e = ((it != std::end(m)) ? it : std::begin(m))->first; \ + } + +// Ugly macros to avoid uglier copy-paste when specializing basic_json. They +// may be removed in the future once the class is split. + +#define NLOHMANN_BASIC_JSON_TPL_DECLARATION \ + template class ObjectType, \ + template class ArrayType, \ + class StringType, class BooleanType, class NumberIntegerType, \ + class NumberUnsignedType, class NumberFloatType, \ + template class AllocatorType, \ + template class JSONSerializer, \ + class BinaryType> + +#define NLOHMANN_BASIC_JSON_TPL \ + basic_json + +// Macros to simplify conversion from/to types + +#define NLOHMANN_JSON_EXPAND( x ) x +#define NLOHMANN_JSON_GET_MACRO(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, _16, _17, _18, _19, _20, _21, _22, _23, _24, _25, _26, _27, _28, _29, _30, _31, _32, _33, _34, _35, _36, _37, _38, _39, _40, _41, _42, _43, _44, _45, _46, _47, _48, _49, _50, _51, _52, _53, _54, _55, _56, _57, _58, _59, _60, _61, _62, _63, _64, NAME,...) NAME +#define NLOHMANN_JSON_PASTE(...) NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_GET_MACRO(__VA_ARGS__, \ + NLOHMANN_JSON_PASTE64, \ + NLOHMANN_JSON_PASTE63, \ + NLOHMANN_JSON_PASTE62, \ + NLOHMANN_JSON_PASTE61, \ + NLOHMANN_JSON_PASTE60, \ + NLOHMANN_JSON_PASTE59, \ + NLOHMANN_JSON_PASTE58, \ + NLOHMANN_JSON_PASTE57, \ + NLOHMANN_JSON_PASTE56, \ + NLOHMANN_JSON_PASTE55, \ + NLOHMANN_JSON_PASTE54, \ + NLOHMANN_JSON_PASTE53, \ + NLOHMANN_JSON_PASTE52, \ + NLOHMANN_JSON_PASTE51, \ + NLOHMANN_JSON_PASTE50, \ + NLOHMANN_JSON_PASTE49, \ + NLOHMANN_JSON_PASTE48, \ + NLOHMANN_JSON_PASTE47, \ + NLOHMANN_JSON_PASTE46, \ + NLOHMANN_JSON_PASTE45, \ + NLOHMANN_JSON_PASTE44, \ + NLOHMANN_JSON_PASTE43, \ + NLOHMANN_JSON_PASTE42, \ + NLOHMANN_JSON_PASTE41, \ + NLOHMANN_JSON_PASTE40, \ + NLOHMANN_JSON_PASTE39, \ + NLOHMANN_JSON_PASTE38, \ + NLOHMANN_JSON_PASTE37, \ + NLOHMANN_JSON_PASTE36, \ + NLOHMANN_JSON_PASTE35, \ + NLOHMANN_JSON_PASTE34, \ + NLOHMANN_JSON_PASTE33, \ + NLOHMANN_JSON_PASTE32, \ + NLOHMANN_JSON_PASTE31, \ + NLOHMANN_JSON_PASTE30, \ + NLOHMANN_JSON_PASTE29, \ + NLOHMANN_JSON_PASTE28, \ + NLOHMANN_JSON_PASTE27, \ + NLOHMANN_JSON_PASTE26, \ + NLOHMANN_JSON_PASTE25, \ + NLOHMANN_JSON_PASTE24, \ + NLOHMANN_JSON_PASTE23, \ + NLOHMANN_JSON_PASTE22, \ + NLOHMANN_JSON_PASTE21, \ + NLOHMANN_JSON_PASTE20, \ + NLOHMANN_JSON_PASTE19, \ + NLOHMANN_JSON_PASTE18, \ + NLOHMANN_JSON_PASTE17, \ + NLOHMANN_JSON_PASTE16, \ + NLOHMANN_JSON_PASTE15, \ + NLOHMANN_JSON_PASTE14, \ + NLOHMANN_JSON_PASTE13, \ + NLOHMANN_JSON_PASTE12, \ + NLOHMANN_JSON_PASTE11, \ + NLOHMANN_JSON_PASTE10, \ + NLOHMANN_JSON_PASTE9, \ + NLOHMANN_JSON_PASTE8, \ + NLOHMANN_JSON_PASTE7, \ + NLOHMANN_JSON_PASTE6, \ + NLOHMANN_JSON_PASTE5, \ + NLOHMANN_JSON_PASTE4, \ + NLOHMANN_JSON_PASTE3, \ + NLOHMANN_JSON_PASTE2, \ + NLOHMANN_JSON_PASTE1)(__VA_ARGS__)) +#define NLOHMANN_JSON_PASTE2(func, v1) func(v1) +#define NLOHMANN_JSON_PASTE3(func, v1, v2) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE2(func, v2) +#define NLOHMANN_JSON_PASTE4(func, v1, v2, v3) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE3(func, v2, v3) +#define NLOHMANN_JSON_PASTE5(func, v1, v2, v3, v4) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE4(func, v2, v3, v4) +#define NLOHMANN_JSON_PASTE6(func, v1, v2, v3, v4, v5) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE5(func, v2, v3, v4, v5) +#define NLOHMANN_JSON_PASTE7(func, v1, v2, v3, v4, v5, v6) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE6(func, v2, v3, v4, v5, v6) +#define NLOHMANN_JSON_PASTE8(func, v1, v2, v3, v4, v5, v6, v7) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE7(func, v2, v3, v4, v5, v6, v7) +#define NLOHMANN_JSON_PASTE9(func, v1, v2, v3, v4, v5, v6, v7, v8) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE8(func, v2, v3, v4, v5, v6, v7, v8) +#define NLOHMANN_JSON_PASTE10(func, v1, v2, v3, v4, v5, v6, v7, v8, v9) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE9(func, v2, v3, v4, v5, v6, v7, v8, v9) +#define NLOHMANN_JSON_PASTE11(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE10(func, v2, v3, v4, v5, v6, v7, v8, v9, v10) +#define NLOHMANN_JSON_PASTE12(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE11(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11) +#define NLOHMANN_JSON_PASTE13(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE12(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12) +#define NLOHMANN_JSON_PASTE14(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE13(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13) +#define NLOHMANN_JSON_PASTE15(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE14(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14) +#define NLOHMANN_JSON_PASTE16(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE15(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15) +#define NLOHMANN_JSON_PASTE17(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE16(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16) +#define NLOHMANN_JSON_PASTE18(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE17(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17) +#define NLOHMANN_JSON_PASTE19(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE18(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18) +#define NLOHMANN_JSON_PASTE20(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE19(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19) +#define NLOHMANN_JSON_PASTE21(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE20(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20) +#define NLOHMANN_JSON_PASTE22(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE21(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21) +#define NLOHMANN_JSON_PASTE23(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE22(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22) +#define NLOHMANN_JSON_PASTE24(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE23(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23) +#define NLOHMANN_JSON_PASTE25(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE24(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24) +#define NLOHMANN_JSON_PASTE26(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE25(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25) +#define NLOHMANN_JSON_PASTE27(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE26(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26) +#define NLOHMANN_JSON_PASTE28(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE27(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27) +#define NLOHMANN_JSON_PASTE29(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE28(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28) +#define NLOHMANN_JSON_PASTE30(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE29(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29) +#define NLOHMANN_JSON_PASTE31(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE30(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30) +#define NLOHMANN_JSON_PASTE32(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE31(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31) +#define NLOHMANN_JSON_PASTE33(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE32(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32) +#define NLOHMANN_JSON_PASTE34(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE33(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33) +#define NLOHMANN_JSON_PASTE35(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE34(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34) +#define NLOHMANN_JSON_PASTE36(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE35(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35) +#define NLOHMANN_JSON_PASTE37(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE36(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36) +#define NLOHMANN_JSON_PASTE38(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE37(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37) +#define NLOHMANN_JSON_PASTE39(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE38(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38) +#define NLOHMANN_JSON_PASTE40(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE39(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39) +#define NLOHMANN_JSON_PASTE41(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE40(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40) +#define NLOHMANN_JSON_PASTE42(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE41(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41) +#define NLOHMANN_JSON_PASTE43(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE42(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42) +#define NLOHMANN_JSON_PASTE44(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE43(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43) +#define NLOHMANN_JSON_PASTE45(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE44(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44) +#define NLOHMANN_JSON_PASTE46(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE45(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45) +#define NLOHMANN_JSON_PASTE47(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE46(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46) +#define NLOHMANN_JSON_PASTE48(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE47(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47) +#define NLOHMANN_JSON_PASTE49(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE48(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48) +#define NLOHMANN_JSON_PASTE50(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE49(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49) +#define NLOHMANN_JSON_PASTE51(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE50(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50) +#define NLOHMANN_JSON_PASTE52(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE51(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51) +#define NLOHMANN_JSON_PASTE53(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE52(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52) +#define NLOHMANN_JSON_PASTE54(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE53(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53) +#define NLOHMANN_JSON_PASTE55(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE54(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54) +#define NLOHMANN_JSON_PASTE56(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE55(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55) +#define NLOHMANN_JSON_PASTE57(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE56(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56) +#define NLOHMANN_JSON_PASTE58(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE57(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57) +#define NLOHMANN_JSON_PASTE59(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE58(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58) +#define NLOHMANN_JSON_PASTE60(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE59(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59) +#define NLOHMANN_JSON_PASTE61(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE60(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60) +#define NLOHMANN_JSON_PASTE62(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE61(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61) +#define NLOHMANN_JSON_PASTE63(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE62(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62) +#define NLOHMANN_JSON_PASTE64(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62, v63) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE63(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62, v63) + +#define NLOHMANN_JSON_TO(v1) nlohmann_json_j[#v1] = nlohmann_json_t.v1; +#define NLOHMANN_JSON_FROM(v1) nlohmann_json_j.at(#v1).get_to(nlohmann_json_t.v1); + +/*! +@brief macro +@def NLOHMANN_DEFINE_TYPE_INTRUSIVE +@since version 3.9.0 +*/ +#define NLOHMANN_DEFINE_TYPE_INTRUSIVE(Type, ...) \ + friend void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \ + friend void from_json(const nlohmann::json& nlohmann_json_j, Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM, __VA_ARGS__)) } + +/*! +@brief macro +@def NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE +@since version 3.9.0 +*/ +#define NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Type, ...) \ + inline void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \ + inline void from_json(const nlohmann::json& nlohmann_json_j, Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM, __VA_ARGS__)) } + + +// inspired from https://stackoverflow.com/a/26745591 +// allows to call any std function as if (e.g. with begin): +// using std::begin; begin(x); +// +// it allows using the detected idiom to retrieve the return type +// of such an expression +#define NLOHMANN_CAN_CALL_STD_FUNC_IMPL(std_name) \ + namespace detail { \ + using std::std_name; \ + \ + template \ + using result_of_##std_name = decltype(std_name(std::declval()...)); \ + } \ + \ + namespace detail2 { \ + struct std_name##_tag \ + { \ + }; \ + \ + template \ + std_name##_tag std_name(T&&...); \ + \ + template \ + using result_of_##std_name = decltype(std_name(std::declval()...)); \ + \ + template \ + struct would_call_std_##std_name \ + { \ + static constexpr auto const value = ::nlohmann::detail:: \ + is_detected_exact::value; \ + }; \ + } /* namespace detail2 */ \ + \ + template \ + struct would_call_std_##std_name : detail2::would_call_std_##std_name \ + { \ + } + +#ifndef JSON_USE_IMPLICIT_CONVERSIONS + #define JSON_USE_IMPLICIT_CONVERSIONS 1 +#endif + +#if JSON_USE_IMPLICIT_CONVERSIONS + #define JSON_EXPLICIT +#else + #define JSON_EXPLICIT explicit +#endif + +#ifndef JSON_DIAGNOSTICS + #define JSON_DIAGNOSTICS 0 +#endif + + +namespace nlohmann +{ +namespace detail +{ + +/*! +@brief replace all occurrences of a substring by another string + +@param[in,out] s the string to manipulate; changed so that all + occurrences of @a f are replaced with @a t +@param[in] f the substring to replace with @a t +@param[in] t the string to replace @a f + +@pre The search string @a f must not be empty. **This precondition is +enforced with an assertion.** + +@since version 2.0.0 +*/ +inline void replace_substring(std::string& s, const std::string& f, + const std::string& t) +{ + JSON_ASSERT(!f.empty()); + for (auto pos = s.find(f); // find first occurrence of f + pos != std::string::npos; // make sure f was found + s.replace(pos, f.size(), t), // replace with t, and + pos = s.find(f, pos + t.size())) // find next occurrence of f + {} +} + +/*! + * @brief string escaping as described in RFC 6901 (Sect. 4) + * @param[in] s string to escape + * @return escaped string + * + * Note the order of escaping "~" to "~0" and "/" to "~1" is important. + */ +inline std::string escape(std::string s) +{ + replace_substring(s, "~", "~0"); + replace_substring(s, "/", "~1"); + return s; +} + +/*! + * @brief string unescaping as described in RFC 6901 (Sect. 4) + * @param[in] s string to unescape + * @return unescaped string + * + * Note the order of escaping "~1" to "/" and "~0" to "~" is important. + */ +static void unescape(std::string& s) +{ + replace_substring(s, "~1", "/"); + replace_substring(s, "~0", "~"); +} + +} // namespace detail +} // namespace nlohmann + +// #include + + +#include // size_t + +namespace nlohmann +{ +namespace detail +{ +/// struct to capture the start position of the current token +struct position_t +{ + /// the total number of characters read + std::size_t chars_read_total = 0; + /// the number of characters read in the current line + std::size_t chars_read_current_line = 0; + /// the number of lines read + std::size_t lines_read = 0; + + /// conversion to size_t to preserve SAX interface + constexpr operator size_t() const + { + return chars_read_total; + } +}; + +} // namespace detail +} // namespace nlohmann + +// #include + + +namespace nlohmann +{ +namespace detail +{ +//////////////// +// exceptions // +//////////////// + +/// @brief general exception of the @ref basic_json class +/// @sa https://json.nlohmann.me/api/basic_json/exception/ +class exception : public std::exception +{ + public: + /// returns the explanatory string + const char* what() const noexcept override + { + return m.what(); + } + + /// the id of the exception + const int id; // NOLINT(cppcoreguidelines-non-private-member-variables-in-classes) + + protected: + JSON_HEDLEY_NON_NULL(3) + exception(int id_, const char* what_arg) : id(id_), m(what_arg) {} // NOLINT(bugprone-throw-keyword-missing) + + static std::string name(const std::string& ename, int id_) + { + return "[json.exception." + ename + "." + std::to_string(id_) + "] "; + } + + template + static std::string diagnostics(const BasicJsonType& leaf_element) + { +#if JSON_DIAGNOSTICS + std::vector tokens; + for (const auto* current = &leaf_element; current->m_parent != nullptr; current = current->m_parent) + { + switch (current->m_parent->type()) + { + case value_t::array: + { + for (std::size_t i = 0; i < current->m_parent->m_value.array->size(); ++i) + { + if (¤t->m_parent->m_value.array->operator[](i) == current) + { + tokens.emplace_back(std::to_string(i)); + break; + } + } + break; + } + + case value_t::object: + { + for (const auto& element : *current->m_parent->m_value.object) + { + if (&element.second == current) + { + tokens.emplace_back(element.first.c_str()); + break; + } + } + break; + } + + case value_t::null: // LCOV_EXCL_LINE + case value_t::string: // LCOV_EXCL_LINE + case value_t::boolean: // LCOV_EXCL_LINE + case value_t::number_integer: // LCOV_EXCL_LINE + case value_t::number_unsigned: // LCOV_EXCL_LINE + case value_t::number_float: // LCOV_EXCL_LINE + case value_t::binary: // LCOV_EXCL_LINE + case value_t::discarded: // LCOV_EXCL_LINE + default: // LCOV_EXCL_LINE + break; // LCOV_EXCL_LINE + } + } + + if (tokens.empty()) + { + return ""; + } + + return "(" + std::accumulate(tokens.rbegin(), tokens.rend(), std::string{}, + [](const std::string & a, const std::string & b) + { + return a + "/" + detail::escape(b); + }) + ") "; +#else + static_cast(leaf_element); + return ""; +#endif + } + + private: + /// an exception object as storage for error messages + std::runtime_error m; +}; + +/// @brief exception indicating a parse error +/// @sa https://json.nlohmann.me/api/basic_json/parse_error/ +class parse_error : public exception +{ + public: + /*! + @brief create a parse error exception + @param[in] id_ the id of the exception + @param[in] pos the position where the error occurred (or with + chars_read_total=0 if the position cannot be + determined) + @param[in] what_arg the explanatory string + @return parse_error object + */ + template + static parse_error create(int id_, const position_t& pos, const std::string& what_arg, const BasicJsonType& context) + { + std::string w = exception::name("parse_error", id_) + "parse error" + + position_string(pos) + ": " + exception::diagnostics(context) + what_arg; + return {id_, pos.chars_read_total, w.c_str()}; + } + + template + static parse_error create(int id_, std::size_t byte_, const std::string& what_arg, const BasicJsonType& context) + { + std::string w = exception::name("parse_error", id_) + "parse error" + + (byte_ != 0 ? (" at byte " + std::to_string(byte_)) : "") + + ": " + exception::diagnostics(context) + what_arg; + return {id_, byte_, w.c_str()}; + } + + /*! + @brief byte index of the parse error + + The byte index of the last read character in the input file. + + @note For an input with n bytes, 1 is the index of the first character and + n+1 is the index of the terminating null byte or the end of file. + This also holds true when reading a byte vector (CBOR or MessagePack). + */ + const std::size_t byte; + + private: + parse_error(int id_, std::size_t byte_, const char* what_arg) + : exception(id_, what_arg), byte(byte_) {} + + static std::string position_string(const position_t& pos) + { + return " at line " + std::to_string(pos.lines_read + 1) + + ", column " + std::to_string(pos.chars_read_current_line); + } +}; + +/// @brief exception indicating errors with iterators +/// @sa https://json.nlohmann.me/api/basic_json/invalid_iterator/ +class invalid_iterator : public exception +{ + public: + template + static invalid_iterator create(int id_, const std::string& what_arg, const BasicJsonType& context) + { + std::string w = exception::name("invalid_iterator", id_) + exception::diagnostics(context) + what_arg; + return {id_, w.c_str()}; + } + + private: + JSON_HEDLEY_NON_NULL(3) + invalid_iterator(int id_, const char* what_arg) + : exception(id_, what_arg) {} +}; + +/// @brief exception indicating executing a member function with a wrong type +/// @sa https://json.nlohmann.me/api/basic_json/type_error/ +class type_error : public exception +{ + public: + template + static type_error create(int id_, const std::string& what_arg, const BasicJsonType& context) + { + std::string w = exception::name("type_error", id_) + exception::diagnostics(context) + what_arg; + return {id_, w.c_str()}; + } + + private: + JSON_HEDLEY_NON_NULL(3) + type_error(int id_, const char* what_arg) : exception(id_, what_arg) {} +}; + +/// @brief exception indicating access out of the defined range +/// @sa https://json.nlohmann.me/api/basic_json/out_of_range/ +class out_of_range : public exception +{ + public: + template + static out_of_range create(int id_, const std::string& what_arg, const BasicJsonType& context) + { + std::string w = exception::name("out_of_range", id_) + exception::diagnostics(context) + what_arg; + return {id_, w.c_str()}; + } + + private: + JSON_HEDLEY_NON_NULL(3) + out_of_range(int id_, const char* what_arg) : exception(id_, what_arg) {} +}; + +/// @brief exception indicating other library errors +/// @sa https://json.nlohmann.me/api/basic_json/other_error/ +class other_error : public exception +{ + public: + template + static other_error create(int id_, const std::string& what_arg, const BasicJsonType& context) + { + std::string w = exception::name("other_error", id_) + exception::diagnostics(context) + what_arg; + return {id_, w.c_str()}; + } + + private: + JSON_HEDLEY_NON_NULL(3) + other_error(int id_, const char* what_arg) : exception(id_, what_arg) {} +}; + +} // namespace detail +} // namespace nlohmann + +// #include + +// #include + + +#include // size_t +#include // conditional, enable_if, false_type, integral_constant, is_constructible, is_integral, is_same, remove_cv, remove_reference, true_type +#include // index_sequence, make_index_sequence, index_sequence_for + +// #include + + +namespace nlohmann +{ +namespace detail +{ + +template +using uncvref_t = typename std::remove_cv::type>::type; + +#ifdef JSON_HAS_CPP_14 + +// the following utilities are natively available in C++14 +using std::enable_if_t; +using std::index_sequence; +using std::make_index_sequence; +using std::index_sequence_for; + +#else + +// alias templates to reduce boilerplate +template +using enable_if_t = typename std::enable_if::type; + +// The following code is taken from https://github.com/abseil/abseil-cpp/blob/10cb35e459f5ecca5b2ff107635da0bfa41011b4/absl/utility/utility.h +// which is part of Google Abseil (https://github.com/abseil/abseil-cpp), licensed under the Apache License 2.0. + +//// START OF CODE FROM GOOGLE ABSEIL + +// integer_sequence +// +// Class template representing a compile-time integer sequence. An instantiation +// of `integer_sequence` has a sequence of integers encoded in its +// type through its template arguments (which is a common need when +// working with C++11 variadic templates). `absl::integer_sequence` is designed +// to be a drop-in replacement for C++14's `std::integer_sequence`. +// +// Example: +// +// template< class T, T... Ints > +// void user_function(integer_sequence); +// +// int main() +// { +// // user_function's `T` will be deduced to `int` and `Ints...` +// // will be deduced to `0, 1, 2, 3, 4`. +// user_function(make_integer_sequence()); +// } +template +struct integer_sequence +{ + using value_type = T; + static constexpr std::size_t size() noexcept + { + return sizeof...(Ints); + } +}; + +// index_sequence +// +// A helper template for an `integer_sequence` of `size_t`, +// `absl::index_sequence` is designed to be a drop-in replacement for C++14's +// `std::index_sequence`. +template +using index_sequence = integer_sequence; + +namespace utility_internal +{ + +template +struct Extend; + +// Note that SeqSize == sizeof...(Ints). It's passed explicitly for efficiency. +template +struct Extend, SeqSize, 0> +{ + using type = integer_sequence < T, Ints..., (Ints + SeqSize)... >; +}; + +template +struct Extend, SeqSize, 1> +{ + using type = integer_sequence < T, Ints..., (Ints + SeqSize)..., 2 * SeqSize >; +}; + +// Recursion helper for 'make_integer_sequence'. +// 'Gen::type' is an alias for 'integer_sequence'. +template +struct Gen +{ + using type = + typename Extend < typename Gen < T, N / 2 >::type, N / 2, N % 2 >::type; +}; + +template +struct Gen +{ + using type = integer_sequence; +}; + +} // namespace utility_internal + +// Compile-time sequences of integers + +// make_integer_sequence +// +// This template alias is equivalent to +// `integer_sequence`, and is designed to be a drop-in +// replacement for C++14's `std::make_integer_sequence`. +template +using make_integer_sequence = typename utility_internal::Gen::type; + +// make_index_sequence +// +// This template alias is equivalent to `index_sequence<0, 1, ..., N-1>`, +// and is designed to be a drop-in replacement for C++14's +// `std::make_index_sequence`. +template +using make_index_sequence = make_integer_sequence; + +// index_sequence_for +// +// Converts a typename pack into an index sequence of the same length, and +// is designed to be a drop-in replacement for C++14's +// `std::index_sequence_for()` +template +using index_sequence_for = make_index_sequence; + +//// END OF CODE FROM GOOGLE ABSEIL + +#endif + +// dispatch utility (taken from ranges-v3) +template struct priority_tag : priority_tag < N - 1 > {}; +template<> struct priority_tag<0> {}; + +// taken from ranges-v3 +template +struct static_const +{ + static constexpr T value{}; +}; + +template +constexpr T static_const::value; // NOLINT(readability-redundant-declaration) + +} // namespace detail +} // namespace nlohmann + +// #include + + +namespace nlohmann +{ +namespace detail +{ +// dispatching helper struct +template struct identity_tag {}; +} // namespace detail +} // namespace nlohmann + +// #include + + +#include // numeric_limits +#include // false_type, is_constructible, is_integral, is_same, true_type +#include // declval +#include // tuple + +// #include + + +// #include + + +#include // random_access_iterator_tag + +// #include + +// #include + + +namespace nlohmann +{ +namespace detail +{ +template +struct iterator_types {}; + +template +struct iterator_types < + It, + void_t> +{ + using difference_type = typename It::difference_type; + using value_type = typename It::value_type; + using pointer = typename It::pointer; + using reference = typename It::reference; + using iterator_category = typename It::iterator_category; +}; + +// This is required as some compilers implement std::iterator_traits in a way that +// doesn't work with SFINAE. See https://github.com/nlohmann/json/issues/1341. +template +struct iterator_traits +{ +}; + +template +struct iterator_traits < T, enable_if_t < !std::is_pointer::value >> + : iterator_types +{ +}; + +template +struct iterator_traits::value>> +{ + using iterator_category = std::random_access_iterator_tag; + using value_type = T; + using difference_type = ptrdiff_t; + using pointer = T*; + using reference = T&; +}; +} // namespace detail +} // namespace nlohmann + +// #include + + +// #include + + +namespace nlohmann +{ +NLOHMANN_CAN_CALL_STD_FUNC_IMPL(begin); +} // namespace nlohmann + +// #include + + +// #include + + +namespace nlohmann +{ +NLOHMANN_CAN_CALL_STD_FUNC_IMPL(end); +} // namespace nlohmann + +// #include + +// #include + +// #include +#ifndef INCLUDE_NLOHMANN_JSON_FWD_HPP_ +#define INCLUDE_NLOHMANN_JSON_FWD_HPP_ + +#include // int64_t, uint64_t +#include // map +#include // allocator +#include // string +#include // vector + +/*! +@brief namespace for Niels Lohmann +@see https://github.com/nlohmann +@since version 1.0.0 +*/ +namespace nlohmann +{ +/*! +@brief default JSONSerializer template argument + +This serializer ignores the template arguments and uses ADL +([argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl)) +for serialization. +*/ +template +struct adl_serializer; + +/// a class to store JSON values +/// @sa https://json.nlohmann.me/api/basic_json/ +template class ObjectType = + std::map, + template class ArrayType = std::vector, + class StringType = std::string, class BooleanType = bool, + class NumberIntegerType = std::int64_t, + class NumberUnsignedType = std::uint64_t, + class NumberFloatType = double, + template class AllocatorType = std::allocator, + template class JSONSerializer = + adl_serializer, + class BinaryType = std::vector> +class basic_json; + +/// @brief JSON Pointer defines a string syntax for identifying a specific value within a JSON document +/// @sa https://json.nlohmann.me/api/json_pointer/ +template +class json_pointer; + +/*! +@brief default specialization +@sa https://json.nlohmann.me/api/json/ +*/ +using json = basic_json<>; + +/// @brief a minimal map-like container that preserves insertion order +/// @sa https://json.nlohmann.me/api/ordered_map/ +template +struct ordered_map; + +/// @brief specialization that maintains the insertion order of object keys +/// @sa https://json.nlohmann.me/api/ordered_json/ +using ordered_json = basic_json; + +} // namespace nlohmann + +#endif // INCLUDE_NLOHMANN_JSON_FWD_HPP_ + + +namespace nlohmann +{ +/*! +@brief detail namespace with internal helper functions + +This namespace collects functions that should not be exposed, +implementations of some @ref basic_json methods, and meta-programming helpers. + +@since version 2.1.0 +*/ +namespace detail +{ +///////////// +// helpers // +///////////// + +// Note to maintainers: +// +// Every trait in this file expects a non CV-qualified type. +// The only exceptions are in the 'aliases for detected' section +// (i.e. those of the form: decltype(T::member_function(std::declval()))) +// +// In this case, T has to be properly CV-qualified to constraint the function arguments +// (e.g. to_json(BasicJsonType&, const T&)) + +template struct is_basic_json : std::false_type {}; + +NLOHMANN_BASIC_JSON_TPL_DECLARATION +struct is_basic_json : std::true_type {}; + +////////////////////// +// json_ref helpers // +////////////////////// + +template +class json_ref; + +template +struct is_json_ref : std::false_type {}; + +template +struct is_json_ref> : std::true_type {}; + +////////////////////////// +// aliases for detected // +////////////////////////// + +template +using mapped_type_t = typename T::mapped_type; + +template +using key_type_t = typename T::key_type; + +template +using value_type_t = typename T::value_type; + +template +using difference_type_t = typename T::difference_type; + +template +using pointer_t = typename T::pointer; + +template +using reference_t = typename T::reference; + +template +using iterator_category_t = typename T::iterator_category; + +template +using to_json_function = decltype(T::to_json(std::declval()...)); + +template +using from_json_function = decltype(T::from_json(std::declval()...)); + +template +using get_template_function = decltype(std::declval().template get()); + +// trait checking if JSONSerializer::from_json(json const&, udt&) exists +template +struct has_from_json : std::false_type {}; + +// trait checking if j.get is valid +// use this trait instead of std::is_constructible or std::is_convertible, +// both rely on, or make use of implicit conversions, and thus fail when T +// has several constructors/operator= (see https://github.com/nlohmann/json/issues/958) +template +struct is_getable +{ + static constexpr bool value = is_detected::value; +}; + +template +struct has_from_json < BasicJsonType, T, enable_if_t < !is_basic_json::value >> +{ + using serializer = typename BasicJsonType::template json_serializer; + + static constexpr bool value = + is_detected_exact::value; +}; + +// This trait checks if JSONSerializer::from_json(json const&) exists +// this overload is used for non-default-constructible user-defined-types +template +struct has_non_default_from_json : std::false_type {}; + +template +struct has_non_default_from_json < BasicJsonType, T, enable_if_t < !is_basic_json::value >> +{ + using serializer = typename BasicJsonType::template json_serializer; + + static constexpr bool value = + is_detected_exact::value; +}; + +// This trait checks if BasicJsonType::json_serializer::to_json exists +// Do not evaluate the trait when T is a basic_json type, to avoid template instantiation infinite recursion. +template +struct has_to_json : std::false_type {}; + +template +struct has_to_json < BasicJsonType, T, enable_if_t < !is_basic_json::value >> +{ + using serializer = typename BasicJsonType::template json_serializer; + + static constexpr bool value = + is_detected_exact::value; +}; + + +/////////////////// +// is_ functions // +/////////////////// + +// https://en.cppreference.com/w/cpp/types/conjunction +template struct conjunction : std::true_type { }; +template struct conjunction : B1 { }; +template +struct conjunction +: std::conditional, B1>::type {}; + +// https://en.cppreference.com/w/cpp/types/negation +template struct negation : std::integral_constant < bool, !B::value > { }; + +// Reimplementation of is_constructible and is_default_constructible, due to them being broken for +// std::pair and std::tuple until LWG 2367 fix (see https://cplusplus.github.io/LWG/lwg-defects.html#2367). +// This causes compile errors in e.g. clang 3.5 or gcc 4.9. +template +struct is_default_constructible : std::is_default_constructible {}; + +template +struct is_default_constructible> + : conjunction, is_default_constructible> {}; + +template +struct is_default_constructible> + : conjunction, is_default_constructible> {}; + +template +struct is_default_constructible> + : conjunction...> {}; + +template +struct is_default_constructible> + : conjunction...> {}; + + +template +struct is_constructible : std::is_constructible {}; + +template +struct is_constructible> : is_default_constructible> {}; + +template +struct is_constructible> : is_default_constructible> {}; + +template +struct is_constructible> : is_default_constructible> {}; + +template +struct is_constructible> : is_default_constructible> {}; + + +template +struct is_iterator_traits : std::false_type {}; + +template +struct is_iterator_traits> +{ + private: + using traits = iterator_traits; + + public: + static constexpr auto value = + is_detected::value && + is_detected::value && + is_detected::value && + is_detected::value && + is_detected::value; +}; + +template +struct is_range +{ + private: + using t_ref = typename std::add_lvalue_reference::type; + + using iterator = detected_t; + using sentinel = detected_t; + + // to be 100% correct, it should use https://en.cppreference.com/w/cpp/iterator/input_or_output_iterator + // and https://en.cppreference.com/w/cpp/iterator/sentinel_for + // but reimplementing these would be too much work, as a lot of other concepts are used underneath + static constexpr auto is_iterator_begin = + is_iterator_traits>::value; + + public: + static constexpr bool value = !std::is_same::value && !std::is_same::value && is_iterator_begin; +}; + +template +using iterator_t = enable_if_t::value, result_of_begin())>>; + +template +using range_value_t = value_type_t>>; + +// The following implementation of is_complete_type is taken from +// https://blogs.msdn.microsoft.com/vcblog/2015/12/02/partial-support-for-expression-sfinae-in-vs-2015-update-1/ +// and is written by Xiang Fan who agreed to using it in this library. + +template +struct is_complete_type : std::false_type {}; + +template +struct is_complete_type : std::true_type {}; + +template +struct is_compatible_object_type_impl : std::false_type {}; + +template +struct is_compatible_object_type_impl < + BasicJsonType, CompatibleObjectType, + enable_if_t < is_detected::value&& + is_detected::value >> +{ + using object_t = typename BasicJsonType::object_t; + + // macOS's is_constructible does not play well with nonesuch... + static constexpr bool value = + is_constructible::value && + is_constructible::value; +}; + +template +struct is_compatible_object_type + : is_compatible_object_type_impl {}; + +template +struct is_constructible_object_type_impl : std::false_type {}; + +template +struct is_constructible_object_type_impl < + BasicJsonType, ConstructibleObjectType, + enable_if_t < is_detected::value&& + is_detected::value >> +{ + using object_t = typename BasicJsonType::object_t; + + static constexpr bool value = + (is_default_constructible::value && + (std::is_move_assignable::value || + std::is_copy_assignable::value) && + (is_constructible::value && + std::is_same < + typename object_t::mapped_type, + typename ConstructibleObjectType::mapped_type >::value)) || + (has_from_json::value || + has_non_default_from_json < + BasicJsonType, + typename ConstructibleObjectType::mapped_type >::value); +}; + +template +struct is_constructible_object_type + : is_constructible_object_type_impl {}; + +template +struct is_compatible_string_type +{ + static constexpr auto value = + is_constructible::value; +}; + +template +struct is_constructible_string_type +{ + static constexpr auto value = + is_constructible::value; +}; + +template +struct is_compatible_array_type_impl : std::false_type {}; + +template +struct is_compatible_array_type_impl < + BasicJsonType, CompatibleArrayType, + enable_if_t < + is_detected::value&& + is_iterator_traits>>::value&& +// special case for types like std::filesystem::path whose iterator's value_type are themselves +// c.f. https://github.com/nlohmann/json/pull/3073 + !std::is_same>::value >> +{ + static constexpr bool value = + is_constructible>::value; +}; + +template +struct is_compatible_array_type + : is_compatible_array_type_impl {}; + +template +struct is_constructible_array_type_impl : std::false_type {}; + +template +struct is_constructible_array_type_impl < + BasicJsonType, ConstructibleArrayType, + enable_if_t::value >> + : std::true_type {}; + +template +struct is_constructible_array_type_impl < + BasicJsonType, ConstructibleArrayType, + enable_if_t < !std::is_same::value&& + !is_compatible_string_type::value&& + is_default_constructible::value&& +(std::is_move_assignable::value || + std::is_copy_assignable::value)&& +is_detected::value&& +is_iterator_traits>>::value&& +is_detected::value&& +// special case for types like std::filesystem::path whose iterator's value_type are themselves +// c.f. https://github.com/nlohmann/json/pull/3073 +!std::is_same>::value&& + is_complete_type < + detected_t>::value >> +{ + using value_type = range_value_t; + + static constexpr bool value = + std::is_same::value || + has_from_json::value || + has_non_default_from_json < + BasicJsonType, + value_type >::value; +}; + +template +struct is_constructible_array_type + : is_constructible_array_type_impl {}; + +template +struct is_compatible_integer_type_impl : std::false_type {}; + +template +struct is_compatible_integer_type_impl < + RealIntegerType, CompatibleNumberIntegerType, + enable_if_t < std::is_integral::value&& + std::is_integral::value&& + !std::is_same::value >> +{ + // is there an assert somewhere on overflows? + using RealLimits = std::numeric_limits; + using CompatibleLimits = std::numeric_limits; + + static constexpr auto value = + is_constructible::value && + CompatibleLimits::is_integer && + RealLimits::is_signed == CompatibleLimits::is_signed; +}; + +template +struct is_compatible_integer_type + : is_compatible_integer_type_impl {}; + +template +struct is_compatible_type_impl: std::false_type {}; + +template +struct is_compatible_type_impl < + BasicJsonType, CompatibleType, + enable_if_t::value >> +{ + static constexpr bool value = + has_to_json::value; +}; + +template +struct is_compatible_type + : is_compatible_type_impl {}; + +template +struct is_constructible_tuple : std::false_type {}; + +template +struct is_constructible_tuple> : conjunction...> {}; + +// a naive helper to check if a type is an ordered_map (exploits the fact that +// ordered_map inherits capacity() from std::vector) +template +struct is_ordered_map +{ + using one = char; + + struct two + { + char x[2]; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + }; + + template static one test( decltype(&C::capacity) ) ; + template static two test(...); + + enum { value = sizeof(test(nullptr)) == sizeof(char) }; // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg) +}; + +// to avoid useless casts (see https://github.com/nlohmann/json/issues/2893#issuecomment-889152324) +template < typename T, typename U, enable_if_t < !std::is_same::value, int > = 0 > +T conditional_static_cast(U value) +{ + return static_cast(value); +} + +template::value, int> = 0> +T conditional_static_cast(U value) +{ + return value; +} + +} // namespace detail +} // namespace nlohmann + +// #include + + +#if JSON_HAS_EXPERIMENTAL_FILESYSTEM +#include +namespace nlohmann::detail +{ +namespace std_fs = std::experimental::filesystem; +} // namespace nlohmann::detail +#elif JSON_HAS_FILESYSTEM +#include +namespace nlohmann::detail +{ +namespace std_fs = std::filesystem; +} // namespace nlohmann::detail +#endif + +namespace nlohmann +{ +namespace detail +{ +template +void from_json(const BasicJsonType& j, typename std::nullptr_t& n) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_null())) + { + JSON_THROW(type_error::create(302, "type must be null, but is " + std::string(j.type_name()), j)); + } + n = nullptr; +} + +// overloads for basic_json template parameters +template < typename BasicJsonType, typename ArithmeticType, + enable_if_t < std::is_arithmetic::value&& + !std::is_same::value, + int > = 0 > +void get_arithmetic_value(const BasicJsonType& j, ArithmeticType& val) +{ + switch (static_cast(j)) + { + case value_t::number_unsigned: + { + val = static_cast(*j.template get_ptr()); + break; + } + case value_t::number_integer: + { + val = static_cast(*j.template get_ptr()); + break; + } + case value_t::number_float: + { + val = static_cast(*j.template get_ptr()); + break; + } + + case value_t::null: + case value_t::object: + case value_t::array: + case value_t::string: + case value_t::boolean: + case value_t::binary: + case value_t::discarded: + default: + JSON_THROW(type_error::create(302, "type must be number, but is " + std::string(j.type_name()), j)); + } +} + +template +void from_json(const BasicJsonType& j, typename BasicJsonType::boolean_t& b) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_boolean())) + { + JSON_THROW(type_error::create(302, "type must be boolean, but is " + std::string(j.type_name()), j)); + } + b = *j.template get_ptr(); +} + +template +void from_json(const BasicJsonType& j, typename BasicJsonType::string_t& s) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_string())) + { + JSON_THROW(type_error::create(302, "type must be string, but is " + std::string(j.type_name()), j)); + } + s = *j.template get_ptr(); +} + +template < + typename BasicJsonType, typename ConstructibleStringType, + enable_if_t < + is_constructible_string_type::value&& + !std::is_same::value, + int > = 0 > +void from_json(const BasicJsonType& j, ConstructibleStringType& s) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_string())) + { + JSON_THROW(type_error::create(302, "type must be string, but is " + std::string(j.type_name()), j)); + } + + s = *j.template get_ptr(); +} + +template +void from_json(const BasicJsonType& j, typename BasicJsonType::number_float_t& val) +{ + get_arithmetic_value(j, val); +} + +template +void from_json(const BasicJsonType& j, typename BasicJsonType::number_unsigned_t& val) +{ + get_arithmetic_value(j, val); +} + +template +void from_json(const BasicJsonType& j, typename BasicJsonType::number_integer_t& val) +{ + get_arithmetic_value(j, val); +} + +template::value, int> = 0> +void from_json(const BasicJsonType& j, EnumType& e) +{ + typename std::underlying_type::type val; + get_arithmetic_value(j, val); + e = static_cast(val); +} + +// forward_list doesn't have an insert method +template::value, int> = 0> +void from_json(const BasicJsonType& j, std::forward_list& l) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()), j)); + } + l.clear(); + std::transform(j.rbegin(), j.rend(), + std::front_inserter(l), [](const BasicJsonType & i) + { + return i.template get(); + }); +} + +// valarray doesn't have an insert method +template::value, int> = 0> +void from_json(const BasicJsonType& j, std::valarray& l) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()), j)); + } + l.resize(j.size()); + std::transform(j.begin(), j.end(), std::begin(l), + [](const BasicJsonType & elem) + { + return elem.template get(); + }); +} + +template +auto from_json(const BasicJsonType& j, T (&arr)[N]) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) +-> decltype(j.template get(), void()) +{ + for (std::size_t i = 0; i < N; ++i) + { + arr[i] = j.at(i).template get(); + } +} + +template +void from_json_array_impl(const BasicJsonType& j, typename BasicJsonType::array_t& arr, priority_tag<3> /*unused*/) +{ + arr = *j.template get_ptr(); +} + +template +auto from_json_array_impl(const BasicJsonType& j, std::array& arr, + priority_tag<2> /*unused*/) +-> decltype(j.template get(), void()) +{ + for (std::size_t i = 0; i < N; ++i) + { + arr[i] = j.at(i).template get(); + } +} + +template::value, + int> = 0> +auto from_json_array_impl(const BasicJsonType& j, ConstructibleArrayType& arr, priority_tag<1> /*unused*/) +-> decltype( + arr.reserve(std::declval()), + j.template get(), + void()) +{ + using std::end; + + ConstructibleArrayType ret; + ret.reserve(j.size()); + std::transform(j.begin(), j.end(), + std::inserter(ret, end(ret)), [](const BasicJsonType & i) + { + // get() returns *this, this won't call a from_json + // method when value_type is BasicJsonType + return i.template get(); + }); + arr = std::move(ret); +} + +template::value, + int> = 0> +void from_json_array_impl(const BasicJsonType& j, ConstructibleArrayType& arr, + priority_tag<0> /*unused*/) +{ + using std::end; + + ConstructibleArrayType ret; + std::transform( + j.begin(), j.end(), std::inserter(ret, end(ret)), + [](const BasicJsonType & i) + { + // get() returns *this, this won't call a from_json + // method when value_type is BasicJsonType + return i.template get(); + }); + arr = std::move(ret); +} + +template < typename BasicJsonType, typename ConstructibleArrayType, + enable_if_t < + is_constructible_array_type::value&& + !is_constructible_object_type::value&& + !is_constructible_string_type::value&& + !std::is_same::value&& + !is_basic_json::value, + int > = 0 > +auto from_json(const BasicJsonType& j, ConstructibleArrayType& arr) +-> decltype(from_json_array_impl(j, arr, priority_tag<3> {}), +j.template get(), +void()) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()), j)); + } + + from_json_array_impl(j, arr, priority_tag<3> {}); +} + +template < typename BasicJsonType, typename T, std::size_t... Idx > +std::array from_json_inplace_array_impl(BasicJsonType&& j, + identity_tag> /*unused*/, index_sequence /*unused*/) +{ + return { { std::forward(j).at(Idx).template get()... } }; +} + +template < typename BasicJsonType, typename T, std::size_t N > +auto from_json(BasicJsonType&& j, identity_tag> tag) +-> decltype(from_json_inplace_array_impl(std::forward(j), tag, make_index_sequence {})) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()), j)); + } + + return from_json_inplace_array_impl(std::forward(j), tag, make_index_sequence {}); +} + +template +void from_json(const BasicJsonType& j, typename BasicJsonType::binary_t& bin) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_binary())) + { + JSON_THROW(type_error::create(302, "type must be binary, but is " + std::string(j.type_name()), j)); + } + + bin = *j.template get_ptr(); +} + +template::value, int> = 0> +void from_json(const BasicJsonType& j, ConstructibleObjectType& obj) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_object())) + { + JSON_THROW(type_error::create(302, "type must be object, but is " + std::string(j.type_name()), j)); + } + + ConstructibleObjectType ret; + const auto* inner_object = j.template get_ptr(); + using value_type = typename ConstructibleObjectType::value_type; + std::transform( + inner_object->begin(), inner_object->end(), + std::inserter(ret, ret.begin()), + [](typename BasicJsonType::object_t::value_type const & p) + { + return value_type(p.first, p.second.template get()); + }); + obj = std::move(ret); +} + +// overload for arithmetic types, not chosen for basic_json template arguments +// (BooleanType, etc..); note: Is it really necessary to provide explicit +// overloads for boolean_t etc. in case of a custom BooleanType which is not +// an arithmetic type? +template < typename BasicJsonType, typename ArithmeticType, + enable_if_t < + std::is_arithmetic::value&& + !std::is_same::value&& + !std::is_same::value&& + !std::is_same::value&& + !std::is_same::value, + int > = 0 > +void from_json(const BasicJsonType& j, ArithmeticType& val) +{ + switch (static_cast(j)) + { + case value_t::number_unsigned: + { + val = static_cast(*j.template get_ptr()); + break; + } + case value_t::number_integer: + { + val = static_cast(*j.template get_ptr()); + break; + } + case value_t::number_float: + { + val = static_cast(*j.template get_ptr()); + break; + } + case value_t::boolean: + { + val = static_cast(*j.template get_ptr()); + break; + } + + case value_t::null: + case value_t::object: + case value_t::array: + case value_t::string: + case value_t::binary: + case value_t::discarded: + default: + JSON_THROW(type_error::create(302, "type must be number, but is " + std::string(j.type_name()), j)); + } +} + +template +std::tuple from_json_tuple_impl_base(BasicJsonType&& j, index_sequence /*unused*/) +{ + return std::make_tuple(std::forward(j).at(Idx).template get()...); +} + +template < typename BasicJsonType, class A1, class A2 > +std::pair from_json_tuple_impl(BasicJsonType&& j, identity_tag> /*unused*/, priority_tag<0> /*unused*/) +{ + return {std::forward(j).at(0).template get(), + std::forward(j).at(1).template get()}; +} + +template +void from_json_tuple_impl(BasicJsonType&& j, std::pair& p, priority_tag<1> /*unused*/) +{ + p = from_json_tuple_impl(std::forward(j), identity_tag> {}, priority_tag<0> {}); +} + +template +std::tuple from_json_tuple_impl(BasicJsonType&& j, identity_tag> /*unused*/, priority_tag<2> /*unused*/) +{ + return from_json_tuple_impl_base(std::forward(j), index_sequence_for {}); +} + +template +void from_json_tuple_impl(BasicJsonType&& j, std::tuple& t, priority_tag<3> /*unused*/) +{ + t = from_json_tuple_impl_base(std::forward(j), index_sequence_for {}); +} + +template +auto from_json(BasicJsonType&& j, TupleRelated&& t) +-> decltype(from_json_tuple_impl(std::forward(j), std::forward(t), priority_tag<3> {})) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()), j)); + } + + return from_json_tuple_impl(std::forward(j), std::forward(t), priority_tag<3> {}); +} + +template < typename BasicJsonType, typename Key, typename Value, typename Compare, typename Allocator, + typename = enable_if_t < !std::is_constructible < + typename BasicJsonType::string_t, Key >::value >> +void from_json(const BasicJsonType& j, std::map& m) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()), j)); + } + m.clear(); + for (const auto& p : j) + { + if (JSON_HEDLEY_UNLIKELY(!p.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(p.type_name()), j)); + } + m.emplace(p.at(0).template get(), p.at(1).template get()); + } +} + +template < typename BasicJsonType, typename Key, typename Value, typename Hash, typename KeyEqual, typename Allocator, + typename = enable_if_t < !std::is_constructible < + typename BasicJsonType::string_t, Key >::value >> +void from_json(const BasicJsonType& j, std::unordered_map& m) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()), j)); + } + m.clear(); + for (const auto& p : j) + { + if (JSON_HEDLEY_UNLIKELY(!p.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(p.type_name()), j)); + } + m.emplace(p.at(0).template get(), p.at(1).template get()); + } +} + +#if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM +template +void from_json(const BasicJsonType& j, std_fs::path& p) +{ + if (JSON_HEDLEY_UNLIKELY(!j.is_string())) + { + JSON_THROW(type_error::create(302, "type must be string, but is " + std::string(j.type_name()), j)); + } + p = *j.template get_ptr(); +} +#endif + +struct from_json_fn +{ + template + auto operator()(const BasicJsonType& j, T&& val) const + noexcept(noexcept(from_json(j, std::forward(val)))) + -> decltype(from_json(j, std::forward(val))) + { + return from_json(j, std::forward(val)); + } +}; +} // namespace detail + +/// namespace to hold default `from_json` function +/// to see why this is required: +/// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2015/n4381.html +namespace // NOLINT(cert-dcl59-cpp,fuchsia-header-anon-namespaces,google-build-namespaces) +{ +constexpr const auto& from_json = detail::static_const::value; // NOLINT(misc-definitions-in-headers) +} // namespace +} // namespace nlohmann + +// #include + + +#include // copy +#include // begin, end +#include // string +#include // tuple, get +#include // is_same, is_constructible, is_floating_point, is_enum, underlying_type +#include // move, forward, declval, pair +#include // valarray +#include // vector + +// #include + +// #include + + +#include // size_t +#include // input_iterator_tag +#include // string, to_string +#include // tuple_size, get, tuple_element +#include // move + +// #include + +// #include + + +namespace nlohmann +{ +namespace detail +{ +template +void int_to_string( string_type& target, std::size_t value ) +{ + // For ADL + using std::to_string; + target = to_string(value); +} +template class iteration_proxy_value +{ + public: + using difference_type = std::ptrdiff_t; + using value_type = iteration_proxy_value; + using pointer = value_type * ; + using reference = value_type & ; + using iterator_category = std::input_iterator_tag; + using string_type = typename std::remove_cv< typename std::remove_reference().key() ) >::type >::type; + + private: + /// the iterator + IteratorType anchor; + /// an index for arrays (used to create key names) + std::size_t array_index = 0; + /// last stringified array index + mutable std::size_t array_index_last = 0; + /// a string representation of the array index + mutable string_type array_index_str = "0"; + /// an empty string (to return a reference for primitive values) + const string_type empty_str{}; + + public: + explicit iteration_proxy_value(IteratorType it) noexcept + : anchor(std::move(it)) + {} + + /// dereference operator (needed for range-based for) + iteration_proxy_value& operator*() + { + return *this; + } + + /// increment operator (needed for range-based for) + iteration_proxy_value& operator++() + { + ++anchor; + ++array_index; + + return *this; + } + + /// equality operator (needed for InputIterator) + bool operator==(const iteration_proxy_value& o) const + { + return anchor == o.anchor; + } + + /// inequality operator (needed for range-based for) + bool operator!=(const iteration_proxy_value& o) const + { + return anchor != o.anchor; + } + + /// return key of the iterator + const string_type& key() const + { + JSON_ASSERT(anchor.m_object != nullptr); + + switch (anchor.m_object->type()) + { + // use integer array index as key + case value_t::array: + { + if (array_index != array_index_last) + { + int_to_string( array_index_str, array_index ); + array_index_last = array_index; + } + return array_index_str; + } + + // use key from the object + case value_t::object: + return anchor.key(); + + // use an empty key for all primitive types + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + return empty_str; + } + } + + /// return value of the iterator + typename IteratorType::reference value() const + { + return anchor.value(); + } +}; + +/// proxy class for the items() function +template class iteration_proxy +{ + private: + /// the container to iterate + typename IteratorType::reference container; + + public: + /// construct iteration proxy from a container + explicit iteration_proxy(typename IteratorType::reference cont) noexcept + : container(cont) {} + + /// return iterator begin (needed for range-based for) + iteration_proxy_value begin() noexcept + { + return iteration_proxy_value(container.begin()); + } + + /// return iterator end (needed for range-based for) + iteration_proxy_value end() noexcept + { + return iteration_proxy_value(container.end()); + } +}; +// Structured Bindings Support +// For further reference see https://blog.tartanllama.xyz/structured-bindings/ +// And see https://github.com/nlohmann/json/pull/1391 +template = 0> +auto get(const nlohmann::detail::iteration_proxy_value& i) -> decltype(i.key()) +{ + return i.key(); +} +// Structured Bindings Support +// For further reference see https://blog.tartanllama.xyz/structured-bindings/ +// And see https://github.com/nlohmann/json/pull/1391 +template = 0> +auto get(const nlohmann::detail::iteration_proxy_value& i) -> decltype(i.value()) +{ + return i.value(); +} +} // namespace detail +} // namespace nlohmann + +// The Addition to the STD Namespace is required to add +// Structured Bindings Support to the iteration_proxy_value class +// For further reference see https://blog.tartanllama.xyz/structured-bindings/ +// And see https://github.com/nlohmann/json/pull/1391 +namespace std +{ +#if defined(__clang__) + // Fix: https://github.com/nlohmann/json/issues/1401 + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wmismatched-tags" +#endif +template +class tuple_size<::nlohmann::detail::iteration_proxy_value> + : public std::integral_constant {}; + +template +class tuple_element> +{ + public: + using type = decltype( + get(std::declval < + ::nlohmann::detail::iteration_proxy_value> ())); +}; +#if defined(__clang__) + #pragma clang diagnostic pop +#endif +} // namespace std + +// #include + +// #include + +// #include + + +#if JSON_HAS_EXPERIMENTAL_FILESYSTEM +#include +namespace nlohmann::detail +{ +namespace std_fs = std::experimental::filesystem; +} // namespace nlohmann::detail +#elif JSON_HAS_FILESYSTEM +#include +namespace nlohmann::detail +{ +namespace std_fs = std::filesystem; +} // namespace nlohmann::detail +#endif + +namespace nlohmann +{ +namespace detail +{ +////////////////// +// constructors // +////////////////// + +/* + * Note all external_constructor<>::construct functions need to call + * j.m_value.destroy(j.m_type) to avoid a memory leak in case j contains an + * allocated value (e.g., a string). See bug issue + * https://github.com/nlohmann/json/issues/2865 for more information. + */ + +template struct external_constructor; + +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, typename BasicJsonType::boolean_t b) noexcept + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::boolean; + j.m_value = b; + j.assert_invariant(); + } +}; + +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, const typename BasicJsonType::string_t& s) + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::string; + j.m_value = s; + j.assert_invariant(); + } + + template + static void construct(BasicJsonType& j, typename BasicJsonType::string_t&& s) + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::string; + j.m_value = std::move(s); + j.assert_invariant(); + } + + template < typename BasicJsonType, typename CompatibleStringType, + enable_if_t < !std::is_same::value, + int > = 0 > + static void construct(BasicJsonType& j, const CompatibleStringType& str) + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::string; + j.m_value.string = j.template create(str); + j.assert_invariant(); + } +}; + +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, const typename BasicJsonType::binary_t& b) + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::binary; + j.m_value = typename BasicJsonType::binary_t(b); + j.assert_invariant(); + } + + template + static void construct(BasicJsonType& j, typename BasicJsonType::binary_t&& b) + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::binary; + j.m_value = typename BasicJsonType::binary_t(std::move(b)); + j.assert_invariant(); + } +}; + +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, typename BasicJsonType::number_float_t val) noexcept + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::number_float; + j.m_value = val; + j.assert_invariant(); + } +}; + +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, typename BasicJsonType::number_unsigned_t val) noexcept + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::number_unsigned; + j.m_value = val; + j.assert_invariant(); + } +}; + +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, typename BasicJsonType::number_integer_t val) noexcept + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::number_integer; + j.m_value = val; + j.assert_invariant(); + } +}; + +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, const typename BasicJsonType::array_t& arr) + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::array; + j.m_value = arr; + j.set_parents(); + j.assert_invariant(); + } + + template + static void construct(BasicJsonType& j, typename BasicJsonType::array_t&& arr) + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::array; + j.m_value = std::move(arr); + j.set_parents(); + j.assert_invariant(); + } + + template < typename BasicJsonType, typename CompatibleArrayType, + enable_if_t < !std::is_same::value, + int > = 0 > + static void construct(BasicJsonType& j, const CompatibleArrayType& arr) + { + using std::begin; + using std::end; + + j.m_value.destroy(j.m_type); + j.m_type = value_t::array; + j.m_value.array = j.template create(begin(arr), end(arr)); + j.set_parents(); + j.assert_invariant(); + } + + template + static void construct(BasicJsonType& j, const std::vector& arr) + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::array; + j.m_value = value_t::array; + j.m_value.array->reserve(arr.size()); + for (const bool x : arr) + { + j.m_value.array->push_back(x); + j.set_parent(j.m_value.array->back()); + } + j.assert_invariant(); + } + + template::value, int> = 0> + static void construct(BasicJsonType& j, const std::valarray& arr) + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::array; + j.m_value = value_t::array; + j.m_value.array->resize(arr.size()); + if (arr.size() > 0) + { + std::copy(std::begin(arr), std::end(arr), j.m_value.array->begin()); + } + j.set_parents(); + j.assert_invariant(); + } +}; + +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, const typename BasicJsonType::object_t& obj) + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::object; + j.m_value = obj; + j.set_parents(); + j.assert_invariant(); + } + + template + static void construct(BasicJsonType& j, typename BasicJsonType::object_t&& obj) + { + j.m_value.destroy(j.m_type); + j.m_type = value_t::object; + j.m_value = std::move(obj); + j.set_parents(); + j.assert_invariant(); + } + + template < typename BasicJsonType, typename CompatibleObjectType, + enable_if_t < !std::is_same::value, int > = 0 > + static void construct(BasicJsonType& j, const CompatibleObjectType& obj) + { + using std::begin; + using std::end; + + j.m_value.destroy(j.m_type); + j.m_type = value_t::object; + j.m_value.object = j.template create(begin(obj), end(obj)); + j.set_parents(); + j.assert_invariant(); + } +}; + +///////////// +// to_json // +///////////// + +template::value, int> = 0> +void to_json(BasicJsonType& j, T b) noexcept +{ + external_constructor::construct(j, b); +} + +template::value, int> = 0> +void to_json(BasicJsonType& j, const CompatibleString& s) +{ + external_constructor::construct(j, s); +} + +template +void to_json(BasicJsonType& j, typename BasicJsonType::string_t&& s) +{ + external_constructor::construct(j, std::move(s)); +} + +template::value, int> = 0> +void to_json(BasicJsonType& j, FloatType val) noexcept +{ + external_constructor::construct(j, static_cast(val)); +} + +template::value, int> = 0> +void to_json(BasicJsonType& j, CompatibleNumberUnsignedType val) noexcept +{ + external_constructor::construct(j, static_cast(val)); +} + +template::value, int> = 0> +void to_json(BasicJsonType& j, CompatibleNumberIntegerType val) noexcept +{ + external_constructor::construct(j, static_cast(val)); +} + +template::value, int> = 0> +void to_json(BasicJsonType& j, EnumType e) noexcept +{ + using underlying_type = typename std::underlying_type::type; + external_constructor::construct(j, static_cast(e)); +} + +template +void to_json(BasicJsonType& j, const std::vector& e) +{ + external_constructor::construct(j, e); +} + +template < typename BasicJsonType, typename CompatibleArrayType, + enable_if_t < is_compatible_array_type::value&& + !is_compatible_object_type::value&& + !is_compatible_string_type::value&& + !std::is_same::value&& + !is_basic_json::value, + int > = 0 > +void to_json(BasicJsonType& j, const CompatibleArrayType& arr) +{ + external_constructor::construct(j, arr); +} + +template +void to_json(BasicJsonType& j, const typename BasicJsonType::binary_t& bin) +{ + external_constructor::construct(j, bin); +} + +template::value, int> = 0> +void to_json(BasicJsonType& j, const std::valarray& arr) +{ + external_constructor::construct(j, std::move(arr)); +} + +template +void to_json(BasicJsonType& j, typename BasicJsonType::array_t&& arr) +{ + external_constructor::construct(j, std::move(arr)); +} + +template < typename BasicJsonType, typename CompatibleObjectType, + enable_if_t < is_compatible_object_type::value&& !is_basic_json::value, int > = 0 > +void to_json(BasicJsonType& j, const CompatibleObjectType& obj) +{ + external_constructor::construct(j, obj); +} + +template +void to_json(BasicJsonType& j, typename BasicJsonType::object_t&& obj) +{ + external_constructor::construct(j, std::move(obj)); +} + +template < + typename BasicJsonType, typename T, std::size_t N, + enable_if_t < !std::is_constructible::value, // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + int > = 0 > +void to_json(BasicJsonType& j, const T(&arr)[N]) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) +{ + external_constructor::construct(j, arr); +} + +template < typename BasicJsonType, typename T1, typename T2, enable_if_t < std::is_constructible::value&& std::is_constructible::value, int > = 0 > +void to_json(BasicJsonType& j, const std::pair& p) +{ + j = { p.first, p.second }; +} + +// for https://github.com/nlohmann/json/pull/1134 +template>::value, int> = 0> +void to_json(BasicJsonType& j, const T& b) +{ + j = { {b.key(), b.value()} }; +} + +template +void to_json_tuple_impl(BasicJsonType& j, const Tuple& t, index_sequence /*unused*/) +{ + j = { std::get(t)... }; +} + +template::value, int > = 0> +void to_json(BasicJsonType& j, const T& t) +{ + to_json_tuple_impl(j, t, make_index_sequence::value> {}); +} + +#if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM +template +void to_json(BasicJsonType& j, const std_fs::path& p) +{ + j = p.string(); +} +#endif + +struct to_json_fn +{ + template + auto operator()(BasicJsonType& j, T&& val) const noexcept(noexcept(to_json(j, std::forward(val)))) + -> decltype(to_json(j, std::forward(val)), void()) + { + return to_json(j, std::forward(val)); + } +}; +} // namespace detail + +/// namespace to hold default `to_json` function +/// to see why this is required: +/// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2015/n4381.html +namespace // NOLINT(cert-dcl59-cpp,fuchsia-header-anon-namespaces,google-build-namespaces) +{ +constexpr const auto& to_json = detail::static_const::value; // NOLINT(misc-definitions-in-headers) +} // namespace +} // namespace nlohmann + +// #include + +// #include + + +namespace nlohmann +{ + +/// @sa https://json.nlohmann.me/api/adl_serializer/ +template +struct adl_serializer +{ + /// @brief convert a JSON value to any value type + /// @sa https://json.nlohmann.me/api/adl_serializer/from_json/ + template + static auto from_json(BasicJsonType && j, TargetType& val) noexcept( + noexcept(::nlohmann::from_json(std::forward(j), val))) + -> decltype(::nlohmann::from_json(std::forward(j), val), void()) + { + ::nlohmann::from_json(std::forward(j), val); + } + + /// @brief convert a JSON value to any value type + /// @sa https://json.nlohmann.me/api/adl_serializer/from_json/ + template + static auto from_json(BasicJsonType && j) noexcept( + noexcept(::nlohmann::from_json(std::forward(j), detail::identity_tag {}))) + -> decltype(::nlohmann::from_json(std::forward(j), detail::identity_tag {})) + { + return ::nlohmann::from_json(std::forward(j), detail::identity_tag {}); + } + + /// @brief convert any value type to a JSON value + /// @sa https://json.nlohmann.me/api/adl_serializer/to_json/ + template + static auto to_json(BasicJsonType& j, TargetType && val) noexcept( + noexcept(::nlohmann::to_json(j, std::forward(val)))) + -> decltype(::nlohmann::to_json(j, std::forward(val)), void()) + { + ::nlohmann::to_json(j, std::forward(val)); + } +}; +} // namespace nlohmann + +// #include + + +#include // uint8_t, uint64_t +#include // tie +#include // move + +namespace nlohmann +{ + +/// @brief an internal type for a backed binary type +/// @sa https://json.nlohmann.me/api/byte_container_with_subtype/ +template +class byte_container_with_subtype : public BinaryType +{ + public: + using container_type = BinaryType; + using subtype_type = std::uint64_t; + + /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/byte_container_with_subtype/ + byte_container_with_subtype() noexcept(noexcept(container_type())) + : container_type() + {} + + /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/byte_container_with_subtype/ + byte_container_with_subtype(const container_type& b) noexcept(noexcept(container_type(b))) + : container_type(b) + {} + + /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/byte_container_with_subtype/ + byte_container_with_subtype(container_type&& b) noexcept(noexcept(container_type(std::move(b)))) + : container_type(std::move(b)) + {} + + /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/byte_container_with_subtype/ + byte_container_with_subtype(const container_type& b, subtype_type subtype_) noexcept(noexcept(container_type(b))) + : container_type(b) + , m_subtype(subtype_) + , m_has_subtype(true) + {} + + /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/byte_container_with_subtype/ + byte_container_with_subtype(container_type&& b, subtype_type subtype_) noexcept(noexcept(container_type(std::move(b)))) + : container_type(std::move(b)) + , m_subtype(subtype_) + , m_has_subtype(true) + {} + + bool operator==(const byte_container_with_subtype& rhs) const + { + return std::tie(static_cast(*this), m_subtype, m_has_subtype) == + std::tie(static_cast(rhs), rhs.m_subtype, rhs.m_has_subtype); + } + + bool operator!=(const byte_container_with_subtype& rhs) const + { + return !(rhs == *this); + } + + /// @brief sets the binary subtype + /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/set_subtype/ + void set_subtype(subtype_type subtype_) noexcept + { + m_subtype = subtype_; + m_has_subtype = true; + } + + /// @brief return the binary subtype + /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/subtype/ + constexpr subtype_type subtype() const noexcept + { + return m_has_subtype ? m_subtype : static_cast(-1); + } + + /// @brief return whether the value has a subtype + /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/has_subtype/ + constexpr bool has_subtype() const noexcept + { + return m_has_subtype; + } + + /// @brief clears the binary subtype + /// @sa https://json.nlohmann.me/api/byte_container_with_subtype/clear_subtype/ + void clear_subtype() noexcept + { + m_subtype = 0; + m_has_subtype = false; + } + + private: + subtype_type m_subtype = 0; + bool m_has_subtype = false; +}; + +} // namespace nlohmann + +// #include + +// #include + +// #include + +// #include + + +#include // uint8_t +#include // size_t +#include // hash + +// #include + +// #include + + +namespace nlohmann +{ +namespace detail +{ + +// boost::hash_combine +inline std::size_t combine(std::size_t seed, std::size_t h) noexcept +{ + seed ^= h + 0x9e3779b9 + (seed << 6U) + (seed >> 2U); + return seed; +} + +/*! +@brief hash a JSON value + +The hash function tries to rely on std::hash where possible. Furthermore, the +type of the JSON value is taken into account to have different hash values for +null, 0, 0U, and false, etc. + +@tparam BasicJsonType basic_json specialization +@param j JSON value to hash +@return hash value of j +*/ +template +std::size_t hash(const BasicJsonType& j) +{ + using string_t = typename BasicJsonType::string_t; + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + + const auto type = static_cast(j.type()); + switch (j.type()) + { + case BasicJsonType::value_t::null: + case BasicJsonType::value_t::discarded: + { + return combine(type, 0); + } + + case BasicJsonType::value_t::object: + { + auto seed = combine(type, j.size()); + for (const auto& element : j.items()) + { + const auto h = std::hash {}(element.key()); + seed = combine(seed, h); + seed = combine(seed, hash(element.value())); + } + return seed; + } + + case BasicJsonType::value_t::array: + { + auto seed = combine(type, j.size()); + for (const auto& element : j) + { + seed = combine(seed, hash(element)); + } + return seed; + } + + case BasicJsonType::value_t::string: + { + const auto h = std::hash {}(j.template get_ref()); + return combine(type, h); + } + + case BasicJsonType::value_t::boolean: + { + const auto h = std::hash {}(j.template get()); + return combine(type, h); + } + + case BasicJsonType::value_t::number_integer: + { + const auto h = std::hash {}(j.template get()); + return combine(type, h); + } + + case BasicJsonType::value_t::number_unsigned: + { + const auto h = std::hash {}(j.template get()); + return combine(type, h); + } + + case BasicJsonType::value_t::number_float: + { + const auto h = std::hash {}(j.template get()); + return combine(type, h); + } + + case BasicJsonType::value_t::binary: + { + auto seed = combine(type, j.get_binary().size()); + const auto h = std::hash {}(j.get_binary().has_subtype()); + seed = combine(seed, h); + seed = combine(seed, static_cast(j.get_binary().subtype())); + for (const auto byte : j.get_binary()) + { + seed = combine(seed, std::hash {}(byte)); + } + return seed; + } + + default: // LCOV_EXCL_LINE + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE + return 0; // LCOV_EXCL_LINE + } +} + +} // namespace detail +} // namespace nlohmann + +// #include + + +#include // generate_n +#include // array +#include // ldexp +#include // size_t +#include // uint8_t, uint16_t, uint32_t, uint64_t +#include // snprintf +#include // memcpy +#include // back_inserter +#include // numeric_limits +#include // char_traits, string +#include // make_pair, move +#include // vector + +// #include + +// #include + + +#include // array +#include // size_t +#include // strlen +#include // begin, end, iterator_traits, random_access_iterator_tag, distance, next +#include // shared_ptr, make_shared, addressof +#include // accumulate +#include // string, char_traits +#include // enable_if, is_base_of, is_pointer, is_integral, remove_pointer +#include // pair, declval + +#ifndef JSON_NO_IO + #include // FILE * + #include // istream +#endif // JSON_NO_IO + +// #include + +// #include + + +namespace nlohmann +{ +namespace detail +{ +/// the supported input formats +enum class input_format_t { json, cbor, msgpack, ubjson, bson }; + +//////////////////// +// input adapters // +//////////////////// + +#ifndef JSON_NO_IO +/*! +Input adapter for stdio file access. This adapter read only 1 byte and do not use any + buffer. This adapter is a very low level adapter. +*/ +class file_input_adapter +{ + public: + using char_type = char; + + JSON_HEDLEY_NON_NULL(2) + explicit file_input_adapter(std::FILE* f) noexcept + : m_file(f) + {} + + // make class move-only + file_input_adapter(const file_input_adapter&) = delete; + file_input_adapter(file_input_adapter&&) noexcept = default; + file_input_adapter& operator=(const file_input_adapter&) = delete; + file_input_adapter& operator=(file_input_adapter&&) = delete; + ~file_input_adapter() = default; + + std::char_traits::int_type get_character() noexcept + { + return std::fgetc(m_file); + } + + private: + /// the file pointer to read from + std::FILE* m_file; +}; + + +/*! +Input adapter for a (caching) istream. Ignores a UFT Byte Order Mark at +beginning of input. Does not support changing the underlying std::streambuf +in mid-input. Maintains underlying std::istream and std::streambuf to support +subsequent use of standard std::istream operations to process any input +characters following those used in parsing the JSON input. Clears the +std::istream flags; any input errors (e.g., EOF) will be detected by the first +subsequent call for input from the std::istream. +*/ +class input_stream_adapter +{ + public: + using char_type = char; + + ~input_stream_adapter() + { + // clear stream flags; we use underlying streambuf I/O, do not + // maintain ifstream flags, except eof + if (is != nullptr) + { + is->clear(is->rdstate() & std::ios::eofbit); + } + } + + explicit input_stream_adapter(std::istream& i) + : is(&i), sb(i.rdbuf()) + {} + + // delete because of pointer members + input_stream_adapter(const input_stream_adapter&) = delete; + input_stream_adapter& operator=(input_stream_adapter&) = delete; + input_stream_adapter& operator=(input_stream_adapter&&) = delete; + + input_stream_adapter(input_stream_adapter&& rhs) noexcept + : is(rhs.is), sb(rhs.sb) + { + rhs.is = nullptr; + rhs.sb = nullptr; + } + + // std::istream/std::streambuf use std::char_traits::to_int_type, to + // ensure that std::char_traits::eof() and the character 0xFF do not + // end up as the same value, e.g. 0xFFFFFFFF. + std::char_traits::int_type get_character() + { + auto res = sb->sbumpc(); + // set eof manually, as we don't use the istream interface. + if (JSON_HEDLEY_UNLIKELY(res == std::char_traits::eof())) + { + is->clear(is->rdstate() | std::ios::eofbit); + } + return res; + } + + private: + /// the associated input stream + std::istream* is = nullptr; + std::streambuf* sb = nullptr; +}; +#endif // JSON_NO_IO + +// General-purpose iterator-based adapter. It might not be as fast as +// theoretically possible for some containers, but it is extremely versatile. +template +class iterator_input_adapter +{ + public: + using char_type = typename std::iterator_traits::value_type; + + iterator_input_adapter(IteratorType first, IteratorType last) + : current(std::move(first)), end(std::move(last)) + {} + + typename std::char_traits::int_type get_character() + { + if (JSON_HEDLEY_LIKELY(current != end)) + { + auto result = std::char_traits::to_int_type(*current); + std::advance(current, 1); + return result; + } + + return std::char_traits::eof(); + } + + private: + IteratorType current; + IteratorType end; + + template + friend struct wide_string_input_helper; + + bool empty() const + { + return current == end; + } +}; + + +template +struct wide_string_input_helper; + +template +struct wide_string_input_helper +{ + // UTF-32 + static void fill_buffer(BaseInputAdapter& input, + std::array::int_type, 4>& utf8_bytes, + size_t& utf8_bytes_index, + size_t& utf8_bytes_filled) + { + utf8_bytes_index = 0; + + if (JSON_HEDLEY_UNLIKELY(input.empty())) + { + utf8_bytes[0] = std::char_traits::eof(); + utf8_bytes_filled = 1; + } + else + { + // get the current character + const auto wc = input.get_character(); + + // UTF-32 to UTF-8 encoding + if (wc < 0x80) + { + utf8_bytes[0] = static_cast::int_type>(wc); + utf8_bytes_filled = 1; + } + else if (wc <= 0x7FF) + { + utf8_bytes[0] = static_cast::int_type>(0xC0u | ((static_cast(wc) >> 6u) & 0x1Fu)); + utf8_bytes[1] = static_cast::int_type>(0x80u | (static_cast(wc) & 0x3Fu)); + utf8_bytes_filled = 2; + } + else if (wc <= 0xFFFF) + { + utf8_bytes[0] = static_cast::int_type>(0xE0u | ((static_cast(wc) >> 12u) & 0x0Fu)); + utf8_bytes[1] = static_cast::int_type>(0x80u | ((static_cast(wc) >> 6u) & 0x3Fu)); + utf8_bytes[2] = static_cast::int_type>(0x80u | (static_cast(wc) & 0x3Fu)); + utf8_bytes_filled = 3; + } + else if (wc <= 0x10FFFF) + { + utf8_bytes[0] = static_cast::int_type>(0xF0u | ((static_cast(wc) >> 18u) & 0x07u)); + utf8_bytes[1] = static_cast::int_type>(0x80u | ((static_cast(wc) >> 12u) & 0x3Fu)); + utf8_bytes[2] = static_cast::int_type>(0x80u | ((static_cast(wc) >> 6u) & 0x3Fu)); + utf8_bytes[3] = static_cast::int_type>(0x80u | (static_cast(wc) & 0x3Fu)); + utf8_bytes_filled = 4; + } + else + { + // unknown character + utf8_bytes[0] = static_cast::int_type>(wc); + utf8_bytes_filled = 1; + } + } + } +}; + +template +struct wide_string_input_helper +{ + // UTF-16 + static void fill_buffer(BaseInputAdapter& input, + std::array::int_type, 4>& utf8_bytes, + size_t& utf8_bytes_index, + size_t& utf8_bytes_filled) + { + utf8_bytes_index = 0; + + if (JSON_HEDLEY_UNLIKELY(input.empty())) + { + utf8_bytes[0] = std::char_traits::eof(); + utf8_bytes_filled = 1; + } + else + { + // get the current character + const auto wc = input.get_character(); + + // UTF-16 to UTF-8 encoding + if (wc < 0x80) + { + utf8_bytes[0] = static_cast::int_type>(wc); + utf8_bytes_filled = 1; + } + else if (wc <= 0x7FF) + { + utf8_bytes[0] = static_cast::int_type>(0xC0u | ((static_cast(wc) >> 6u))); + utf8_bytes[1] = static_cast::int_type>(0x80u | (static_cast(wc) & 0x3Fu)); + utf8_bytes_filled = 2; + } + else if (0xD800 > wc || wc >= 0xE000) + { + utf8_bytes[0] = static_cast::int_type>(0xE0u | ((static_cast(wc) >> 12u))); + utf8_bytes[1] = static_cast::int_type>(0x80u | ((static_cast(wc) >> 6u) & 0x3Fu)); + utf8_bytes[2] = static_cast::int_type>(0x80u | (static_cast(wc) & 0x3Fu)); + utf8_bytes_filled = 3; + } + else + { + if (JSON_HEDLEY_UNLIKELY(!input.empty())) + { + const auto wc2 = static_cast(input.get_character()); + const auto charcode = 0x10000u + (((static_cast(wc) & 0x3FFu) << 10u) | (wc2 & 0x3FFu)); + utf8_bytes[0] = static_cast::int_type>(0xF0u | (charcode >> 18u)); + utf8_bytes[1] = static_cast::int_type>(0x80u | ((charcode >> 12u) & 0x3Fu)); + utf8_bytes[2] = static_cast::int_type>(0x80u | ((charcode >> 6u) & 0x3Fu)); + utf8_bytes[3] = static_cast::int_type>(0x80u | (charcode & 0x3Fu)); + utf8_bytes_filled = 4; + } + else + { + utf8_bytes[0] = static_cast::int_type>(wc); + utf8_bytes_filled = 1; + } + } + } + } +}; + +// Wraps another input apdater to convert wide character types into individual bytes. +template +class wide_string_input_adapter +{ + public: + using char_type = char; + + wide_string_input_adapter(BaseInputAdapter base) + : base_adapter(base) {} + + typename std::char_traits::int_type get_character() noexcept + { + // check if buffer needs to be filled + if (utf8_bytes_index == utf8_bytes_filled) + { + fill_buffer(); + + JSON_ASSERT(utf8_bytes_filled > 0); + JSON_ASSERT(utf8_bytes_index == 0); + } + + // use buffer + JSON_ASSERT(utf8_bytes_filled > 0); + JSON_ASSERT(utf8_bytes_index < utf8_bytes_filled); + return utf8_bytes[utf8_bytes_index++]; + } + + private: + BaseInputAdapter base_adapter; + + template + void fill_buffer() + { + wide_string_input_helper::fill_buffer(base_adapter, utf8_bytes, utf8_bytes_index, utf8_bytes_filled); + } + + /// a buffer for UTF-8 bytes + std::array::int_type, 4> utf8_bytes = {{0, 0, 0, 0}}; + + /// index to the utf8_codes array for the next valid byte + std::size_t utf8_bytes_index = 0; + /// number of valid bytes in the utf8_codes array + std::size_t utf8_bytes_filled = 0; +}; + + +template +struct iterator_input_adapter_factory +{ + using iterator_type = IteratorType; + using char_type = typename std::iterator_traits::value_type; + using adapter_type = iterator_input_adapter; + + static adapter_type create(IteratorType first, IteratorType last) + { + return adapter_type(std::move(first), std::move(last)); + } +}; + +template +struct is_iterator_of_multibyte +{ + using value_type = typename std::iterator_traits::value_type; + enum + { + value = sizeof(value_type) > 1 + }; +}; + +template +struct iterator_input_adapter_factory::value>> +{ + using iterator_type = IteratorType; + using char_type = typename std::iterator_traits::value_type; + using base_adapter_type = iterator_input_adapter; + using adapter_type = wide_string_input_adapter; + + static adapter_type create(IteratorType first, IteratorType last) + { + return adapter_type(base_adapter_type(std::move(first), std::move(last))); + } +}; + +// General purpose iterator-based input +template +typename iterator_input_adapter_factory::adapter_type input_adapter(IteratorType first, IteratorType last) +{ + using factory_type = iterator_input_adapter_factory; + return factory_type::create(first, last); +} + +// Convenience shorthand from container to iterator +// Enables ADL on begin(container) and end(container) +// Encloses the using declarations in namespace for not to leak them to outside scope + +namespace container_input_adapter_factory_impl +{ + +using std::begin; +using std::end; + +template +struct container_input_adapter_factory {}; + +template +struct container_input_adapter_factory< ContainerType, + void_t()), end(std::declval()))>> + { + using adapter_type = decltype(input_adapter(begin(std::declval()), end(std::declval()))); + + static adapter_type create(const ContainerType& container) +{ + return input_adapter(begin(container), end(container)); +} + }; + +} // namespace container_input_adapter_factory_impl + +template +typename container_input_adapter_factory_impl::container_input_adapter_factory::adapter_type input_adapter(const ContainerType& container) +{ + return container_input_adapter_factory_impl::container_input_adapter_factory::create(container); +} + +#ifndef JSON_NO_IO +// Special cases with fast paths +inline file_input_adapter input_adapter(std::FILE* file) +{ + return file_input_adapter(file); +} + +inline input_stream_adapter input_adapter(std::istream& stream) +{ + return input_stream_adapter(stream); +} + +inline input_stream_adapter input_adapter(std::istream&& stream) +{ + return input_stream_adapter(stream); +} +#endif // JSON_NO_IO + +using contiguous_bytes_input_adapter = decltype(input_adapter(std::declval(), std::declval())); + +// Null-delimited strings, and the like. +template < typename CharT, + typename std::enable_if < + std::is_pointer::value&& + !std::is_array::value&& + std::is_integral::type>::value&& + sizeof(typename std::remove_pointer::type) == 1, + int >::type = 0 > +contiguous_bytes_input_adapter input_adapter(CharT b) +{ + auto length = std::strlen(reinterpret_cast(b)); + const auto* ptr = reinterpret_cast(b); + return input_adapter(ptr, ptr + length); +} + +template +auto input_adapter(T (&array)[N]) -> decltype(input_adapter(array, array + N)) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) +{ + return input_adapter(array, array + N); +} + +// This class only handles inputs of input_buffer_adapter type. +// It's required so that expressions like {ptr, len} can be implicitly cast +// to the correct adapter. +class span_input_adapter +{ + public: + template < typename CharT, + typename std::enable_if < + std::is_pointer::value&& + std::is_integral::type>::value&& + sizeof(typename std::remove_pointer::type) == 1, + int >::type = 0 > + span_input_adapter(CharT b, std::size_t l) + : ia(reinterpret_cast(b), reinterpret_cast(b) + l) {} + + template::iterator_category, std::random_access_iterator_tag>::value, + int>::type = 0> + span_input_adapter(IteratorType first, IteratorType last) + : ia(input_adapter(first, last)) {} + + contiguous_bytes_input_adapter&& get() + { + return std::move(ia); // NOLINT(hicpp-move-const-arg,performance-move-const-arg) + } + + private: + contiguous_bytes_input_adapter ia; +}; +} // namespace detail +} // namespace nlohmann + +// #include + + +#include +#include // string +#include // move +#include // vector + +// #include + +// #include + + +namespace nlohmann +{ + +/*! +@brief SAX interface + +This class describes the SAX interface used by @ref nlohmann::json::sax_parse. +Each function is called in different situations while the input is parsed. The +boolean return value informs the parser whether to continue processing the +input. +*/ +template +struct json_sax +{ + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using binary_t = typename BasicJsonType::binary_t; + + /*! + @brief a null value was read + @return whether parsing should proceed + */ + virtual bool null() = 0; + + /*! + @brief a boolean value was read + @param[in] val boolean value + @return whether parsing should proceed + */ + virtual bool boolean(bool val) = 0; + + /*! + @brief an integer number was read + @param[in] val integer value + @return whether parsing should proceed + */ + virtual bool number_integer(number_integer_t val) = 0; + + /*! + @brief an unsigned integer number was read + @param[in] val unsigned integer value + @return whether parsing should proceed + */ + virtual bool number_unsigned(number_unsigned_t val) = 0; + + /*! + @brief a floating-point number was read + @param[in] val floating-point value + @param[in] s raw token value + @return whether parsing should proceed + */ + virtual bool number_float(number_float_t val, const string_t& s) = 0; + + /*! + @brief a string value was read + @param[in] val string value + @return whether parsing should proceed + @note It is safe to move the passed string value. + */ + virtual bool string(string_t& val) = 0; + + /*! + @brief a binary value was read + @param[in] val binary value + @return whether parsing should proceed + @note It is safe to move the passed binary value. + */ + virtual bool binary(binary_t& val) = 0; + + /*! + @brief the beginning of an object was read + @param[in] elements number of object elements or -1 if unknown + @return whether parsing should proceed + @note binary formats may report the number of elements + */ + virtual bool start_object(std::size_t elements) = 0; + + /*! + @brief an object key was read + @param[in] val object key + @return whether parsing should proceed + @note It is safe to move the passed string. + */ + virtual bool key(string_t& val) = 0; + + /*! + @brief the end of an object was read + @return whether parsing should proceed + */ + virtual bool end_object() = 0; + + /*! + @brief the beginning of an array was read + @param[in] elements number of array elements or -1 if unknown + @return whether parsing should proceed + @note binary formats may report the number of elements + */ + virtual bool start_array(std::size_t elements) = 0; + + /*! + @brief the end of an array was read + @return whether parsing should proceed + */ + virtual bool end_array() = 0; + + /*! + @brief a parse error occurred + @param[in] position the position in the input where the error occurs + @param[in] last_token the last read token + @param[in] ex an exception object describing the error + @return whether parsing should proceed (must return false) + */ + virtual bool parse_error(std::size_t position, + const std::string& last_token, + const detail::exception& ex) = 0; + + json_sax() = default; + json_sax(const json_sax&) = default; + json_sax(json_sax&&) noexcept = default; + json_sax& operator=(const json_sax&) = default; + json_sax& operator=(json_sax&&) noexcept = default; + virtual ~json_sax() = default; +}; + + +namespace detail +{ +/*! +@brief SAX implementation to create a JSON value from SAX events + +This class implements the @ref json_sax interface and processes the SAX events +to create a JSON value which makes it basically a DOM parser. The structure or +hierarchy of the JSON value is managed by the stack `ref_stack` which contains +a pointer to the respective array or object for each recursion depth. + +After successful parsing, the value that is passed by reference to the +constructor contains the parsed value. + +@tparam BasicJsonType the JSON type +*/ +template +class json_sax_dom_parser +{ + public: + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using binary_t = typename BasicJsonType::binary_t; + + /*! + @param[in,out] r reference to a JSON value that is manipulated while + parsing + @param[in] allow_exceptions_ whether parse errors yield exceptions + */ + explicit json_sax_dom_parser(BasicJsonType& r, const bool allow_exceptions_ = true) + : root(r), allow_exceptions(allow_exceptions_) + {} + + // make class move-only + json_sax_dom_parser(const json_sax_dom_parser&) = delete; + json_sax_dom_parser(json_sax_dom_parser&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor) + json_sax_dom_parser& operator=(const json_sax_dom_parser&) = delete; + json_sax_dom_parser& operator=(json_sax_dom_parser&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor) + ~json_sax_dom_parser() = default; + + bool null() + { + handle_value(nullptr); + return true; + } + + bool boolean(bool val) + { + handle_value(val); + return true; + } + + bool number_integer(number_integer_t val) + { + handle_value(val); + return true; + } + + bool number_unsigned(number_unsigned_t val) + { + handle_value(val); + return true; + } + + bool number_float(number_float_t val, const string_t& /*unused*/) + { + handle_value(val); + return true; + } + + bool string(string_t& val) + { + handle_value(val); + return true; + } + + bool binary(binary_t& val) + { + handle_value(std::move(val)); + return true; + } + + bool start_object(std::size_t len) + { + ref_stack.push_back(handle_value(BasicJsonType::value_t::object)); + + if (JSON_HEDLEY_UNLIKELY(len != static_cast(-1) && len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, "excessive object size: " + std::to_string(len), *ref_stack.back())); + } + + return true; + } + + bool key(string_t& val) + { + // add null at given key and store the reference for later + object_element = &(ref_stack.back()->m_value.object->operator[](val)); + return true; + } + + bool end_object() + { + ref_stack.back()->set_parents(); + ref_stack.pop_back(); + return true; + } + + bool start_array(std::size_t len) + { + ref_stack.push_back(handle_value(BasicJsonType::value_t::array)); + + if (JSON_HEDLEY_UNLIKELY(len != static_cast(-1) && len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, "excessive array size: " + std::to_string(len), *ref_stack.back())); + } + + return true; + } + + bool end_array() + { + ref_stack.back()->set_parents(); + ref_stack.pop_back(); + return true; + } + + template + bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, + const Exception& ex) + { + errored = true; + static_cast(ex); + if (allow_exceptions) + { + JSON_THROW(ex); + } + return false; + } + + constexpr bool is_errored() const + { + return errored; + } + + private: + /*! + @invariant If the ref stack is empty, then the passed value will be the new + root. + @invariant If the ref stack contains a value, then it is an array or an + object to which we can add elements + */ + template + JSON_HEDLEY_RETURNS_NON_NULL + BasicJsonType* handle_value(Value&& v) + { + if (ref_stack.empty()) + { + root = BasicJsonType(std::forward(v)); + return &root; + } + + JSON_ASSERT(ref_stack.back()->is_array() || ref_stack.back()->is_object()); + + if (ref_stack.back()->is_array()) + { + ref_stack.back()->m_value.array->emplace_back(std::forward(v)); + return &(ref_stack.back()->m_value.array->back()); + } + + JSON_ASSERT(ref_stack.back()->is_object()); + JSON_ASSERT(object_element); + *object_element = BasicJsonType(std::forward(v)); + return object_element; + } + + /// the parsed JSON value + BasicJsonType& root; + /// stack to model hierarchy of values + std::vector ref_stack {}; + /// helper to hold the reference for the next object element + BasicJsonType* object_element = nullptr; + /// whether a syntax error occurred + bool errored = false; + /// whether to throw exceptions in case of errors + const bool allow_exceptions = true; +}; + +template +class json_sax_dom_callback_parser +{ + public: + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using binary_t = typename BasicJsonType::binary_t; + using parser_callback_t = typename BasicJsonType::parser_callback_t; + using parse_event_t = typename BasicJsonType::parse_event_t; + + json_sax_dom_callback_parser(BasicJsonType& r, + const parser_callback_t cb, + const bool allow_exceptions_ = true) + : root(r), callback(cb), allow_exceptions(allow_exceptions_) + { + keep_stack.push_back(true); + } + + // make class move-only + json_sax_dom_callback_parser(const json_sax_dom_callback_parser&) = delete; + json_sax_dom_callback_parser(json_sax_dom_callback_parser&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor) + json_sax_dom_callback_parser& operator=(const json_sax_dom_callback_parser&) = delete; + json_sax_dom_callback_parser& operator=(json_sax_dom_callback_parser&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor) + ~json_sax_dom_callback_parser() = default; + + bool null() + { + handle_value(nullptr); + return true; + } + + bool boolean(bool val) + { + handle_value(val); + return true; + } + + bool number_integer(number_integer_t val) + { + handle_value(val); + return true; + } + + bool number_unsigned(number_unsigned_t val) + { + handle_value(val); + return true; + } + + bool number_float(number_float_t val, const string_t& /*unused*/) + { + handle_value(val); + return true; + } + + bool string(string_t& val) + { + handle_value(val); + return true; + } + + bool binary(binary_t& val) + { + handle_value(std::move(val)); + return true; + } + + bool start_object(std::size_t len) + { + // check callback for object start + const bool keep = callback(static_cast(ref_stack.size()), parse_event_t::object_start, discarded); + keep_stack.push_back(keep); + + auto val = handle_value(BasicJsonType::value_t::object, true); + ref_stack.push_back(val.second); + + // check object limit + if (ref_stack.back() && JSON_HEDLEY_UNLIKELY(len != static_cast(-1) && len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, "excessive object size: " + std::to_string(len), *ref_stack.back())); + } + + return true; + } + + bool key(string_t& val) + { + BasicJsonType k = BasicJsonType(val); + + // check callback for key + const bool keep = callback(static_cast(ref_stack.size()), parse_event_t::key, k); + key_keep_stack.push_back(keep); + + // add discarded value at given key and store the reference for later + if (keep && ref_stack.back()) + { + object_element = &(ref_stack.back()->m_value.object->operator[](val) = discarded); + } + + return true; + } + + bool end_object() + { + if (ref_stack.back()) + { + if (!callback(static_cast(ref_stack.size()) - 1, parse_event_t::object_end, *ref_stack.back())) + { + // discard object + *ref_stack.back() = discarded; + } + else + { + ref_stack.back()->set_parents(); + } + } + + JSON_ASSERT(!ref_stack.empty()); + JSON_ASSERT(!keep_stack.empty()); + ref_stack.pop_back(); + keep_stack.pop_back(); + + if (!ref_stack.empty() && ref_stack.back() && ref_stack.back()->is_structured()) + { + // remove discarded value + for (auto it = ref_stack.back()->begin(); it != ref_stack.back()->end(); ++it) + { + if (it->is_discarded()) + { + ref_stack.back()->erase(it); + break; + } + } + } + + return true; + } + + bool start_array(std::size_t len) + { + const bool keep = callback(static_cast(ref_stack.size()), parse_event_t::array_start, discarded); + keep_stack.push_back(keep); + + auto val = handle_value(BasicJsonType::value_t::array, true); + ref_stack.push_back(val.second); + + // check array limit + if (ref_stack.back() && JSON_HEDLEY_UNLIKELY(len != static_cast(-1) && len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, "excessive array size: " + std::to_string(len), *ref_stack.back())); + } + + return true; + } + + bool end_array() + { + bool keep = true; + + if (ref_stack.back()) + { + keep = callback(static_cast(ref_stack.size()) - 1, parse_event_t::array_end, *ref_stack.back()); + if (keep) + { + ref_stack.back()->set_parents(); + } + else + { + // discard array + *ref_stack.back() = discarded; + } + } + + JSON_ASSERT(!ref_stack.empty()); + JSON_ASSERT(!keep_stack.empty()); + ref_stack.pop_back(); + keep_stack.pop_back(); + + // remove discarded value + if (!keep && !ref_stack.empty() && ref_stack.back()->is_array()) + { + ref_stack.back()->m_value.array->pop_back(); + } + + return true; + } + + template + bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, + const Exception& ex) + { + errored = true; + static_cast(ex); + if (allow_exceptions) + { + JSON_THROW(ex); + } + return false; + } + + constexpr bool is_errored() const + { + return errored; + } + + private: + /*! + @param[in] v value to add to the JSON value we build during parsing + @param[in] skip_callback whether we should skip calling the callback + function; this is required after start_array() and + start_object() SAX events, because otherwise we would call the + callback function with an empty array or object, respectively. + + @invariant If the ref stack is empty, then the passed value will be the new + root. + @invariant If the ref stack contains a value, then it is an array or an + object to which we can add elements + + @return pair of boolean (whether value should be kept) and pointer (to the + passed value in the ref_stack hierarchy; nullptr if not kept) + */ + template + std::pair handle_value(Value&& v, const bool skip_callback = false) + { + JSON_ASSERT(!keep_stack.empty()); + + // do not handle this value if we know it would be added to a discarded + // container + if (!keep_stack.back()) + { + return {false, nullptr}; + } + + // create value + auto value = BasicJsonType(std::forward(v)); + + // check callback + const bool keep = skip_callback || callback(static_cast(ref_stack.size()), parse_event_t::value, value); + + // do not handle this value if we just learnt it shall be discarded + if (!keep) + { + return {false, nullptr}; + } + + if (ref_stack.empty()) + { + root = std::move(value); + return {true, &root}; + } + + // skip this value if we already decided to skip the parent + // (https://github.com/nlohmann/json/issues/971#issuecomment-413678360) + if (!ref_stack.back()) + { + return {false, nullptr}; + } + + // we now only expect arrays and objects + JSON_ASSERT(ref_stack.back()->is_array() || ref_stack.back()->is_object()); + + // array + if (ref_stack.back()->is_array()) + { + ref_stack.back()->m_value.array->emplace_back(std::move(value)); + return {true, &(ref_stack.back()->m_value.array->back())}; + } + + // object + JSON_ASSERT(ref_stack.back()->is_object()); + // check if we should store an element for the current key + JSON_ASSERT(!key_keep_stack.empty()); + const bool store_element = key_keep_stack.back(); + key_keep_stack.pop_back(); + + if (!store_element) + { + return {false, nullptr}; + } + + JSON_ASSERT(object_element); + *object_element = std::move(value); + return {true, object_element}; + } + + /// the parsed JSON value + BasicJsonType& root; + /// stack to model hierarchy of values + std::vector ref_stack {}; + /// stack to manage which values to keep + std::vector keep_stack {}; + /// stack to manage which object keys to keep + std::vector key_keep_stack {}; + /// helper to hold the reference for the next object element + BasicJsonType* object_element = nullptr; + /// whether a syntax error occurred + bool errored = false; + /// callback function + const parser_callback_t callback = nullptr; + /// whether to throw exceptions in case of errors + const bool allow_exceptions = true; + /// a discarded value for the callback + BasicJsonType discarded = BasicJsonType::value_t::discarded; +}; + +template +class json_sax_acceptor +{ + public: + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using binary_t = typename BasicJsonType::binary_t; + + bool null() + { + return true; + } + + bool boolean(bool /*unused*/) + { + return true; + } + + bool number_integer(number_integer_t /*unused*/) + { + return true; + } + + bool number_unsigned(number_unsigned_t /*unused*/) + { + return true; + } + + bool number_float(number_float_t /*unused*/, const string_t& /*unused*/) + { + return true; + } + + bool string(string_t& /*unused*/) + { + return true; + } + + bool binary(binary_t& /*unused*/) + { + return true; + } + + bool start_object(std::size_t /*unused*/ = static_cast(-1)) + { + return true; + } + + bool key(string_t& /*unused*/) + { + return true; + } + + bool end_object() + { + return true; + } + + bool start_array(std::size_t /*unused*/ = static_cast(-1)) + { + return true; + } + + bool end_array() + { + return true; + } + + bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, const detail::exception& /*unused*/) + { + return false; + } +}; +} // namespace detail + +} // namespace nlohmann + +// #include + + +#include // array +#include // localeconv +#include // size_t +#include // snprintf +#include // strtof, strtod, strtold, strtoll, strtoull +#include // initializer_list +#include // char_traits, string +#include // move +#include // vector + +// #include + +// #include + +// #include + + +namespace nlohmann +{ +namespace detail +{ +/////////// +// lexer // +/////////// + +template +class lexer_base +{ + public: + /// token types for the parser + enum class token_type + { + uninitialized, ///< indicating the scanner is uninitialized + literal_true, ///< the `true` literal + literal_false, ///< the `false` literal + literal_null, ///< the `null` literal + value_string, ///< a string -- use get_string() for actual value + value_unsigned, ///< an unsigned integer -- use get_number_unsigned() for actual value + value_integer, ///< a signed integer -- use get_number_integer() for actual value + value_float, ///< an floating point number -- use get_number_float() for actual value + begin_array, ///< the character for array begin `[` + begin_object, ///< the character for object begin `{` + end_array, ///< the character for array end `]` + end_object, ///< the character for object end `}` + name_separator, ///< the name separator `:` + value_separator, ///< the value separator `,` + parse_error, ///< indicating a parse error + end_of_input, ///< indicating the end of the input buffer + literal_or_value ///< a literal or the begin of a value (only for diagnostics) + }; + + /// return name of values of type token_type (only used for errors) + JSON_HEDLEY_RETURNS_NON_NULL + JSON_HEDLEY_CONST + static const char* token_type_name(const token_type t) noexcept + { + switch (t) + { + case token_type::uninitialized: + return ""; + case token_type::literal_true: + return "true literal"; + case token_type::literal_false: + return "false literal"; + case token_type::literal_null: + return "null literal"; + case token_type::value_string: + return "string literal"; + case token_type::value_unsigned: + case token_type::value_integer: + case token_type::value_float: + return "number literal"; + case token_type::begin_array: + return "'['"; + case token_type::begin_object: + return "'{'"; + case token_type::end_array: + return "']'"; + case token_type::end_object: + return "'}'"; + case token_type::name_separator: + return "':'"; + case token_type::value_separator: + return "','"; + case token_type::parse_error: + return ""; + case token_type::end_of_input: + return "end of input"; + case token_type::literal_or_value: + return "'[', '{', or a literal"; + // LCOV_EXCL_START + default: // catch non-enum values + return "unknown token"; + // LCOV_EXCL_STOP + } + } +}; +/*! +@brief lexical analysis + +This class organizes the lexical analysis during JSON deserialization. +*/ +template +class lexer : public lexer_base +{ + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using char_type = typename InputAdapterType::char_type; + using char_int_type = typename std::char_traits::int_type; + + public: + using token_type = typename lexer_base::token_type; + + explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false) noexcept + : ia(std::move(adapter)) + , ignore_comments(ignore_comments_) + , decimal_point_char(static_cast(get_decimal_point())) + {} + + // delete because of pointer members + lexer(const lexer&) = delete; + lexer(lexer&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor) + lexer& operator=(lexer&) = delete; + lexer& operator=(lexer&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor) + ~lexer() = default; + + private: + ///////////////////// + // locales + ///////////////////// + + /// return the locale-dependent decimal point + JSON_HEDLEY_PURE + static char get_decimal_point() noexcept + { + const auto* loc = localeconv(); + JSON_ASSERT(loc != nullptr); + return (loc->decimal_point == nullptr) ? '.' : *(loc->decimal_point); + } + + ///////////////////// + // scan functions + ///////////////////// + + /*! + @brief get codepoint from 4 hex characters following `\u` + + For input "\u c1 c2 c3 c4" the codepoint is: + (c1 * 0x1000) + (c2 * 0x0100) + (c3 * 0x0010) + c4 + = (c1 << 12) + (c2 << 8) + (c3 << 4) + (c4 << 0) + + Furthermore, the possible characters '0'..'9', 'A'..'F', and 'a'..'f' + must be converted to the integers 0x0..0x9, 0xA..0xF, 0xA..0xF, resp. The + conversion is done by subtracting the offset (0x30, 0x37, and 0x57) + between the ASCII value of the character and the desired integer value. + + @return codepoint (0x0000..0xFFFF) or -1 in case of an error (e.g. EOF or + non-hex character) + */ + int get_codepoint() + { + // this function only makes sense after reading `\u` + JSON_ASSERT(current == 'u'); + int codepoint = 0; + + const auto factors = { 12u, 8u, 4u, 0u }; + for (const auto factor : factors) + { + get(); + + if (current >= '0' && current <= '9') + { + codepoint += static_cast((static_cast(current) - 0x30u) << factor); + } + else if (current >= 'A' && current <= 'F') + { + codepoint += static_cast((static_cast(current) - 0x37u) << factor); + } + else if (current >= 'a' && current <= 'f') + { + codepoint += static_cast((static_cast(current) - 0x57u) << factor); + } + else + { + return -1; + } + } + + JSON_ASSERT(0x0000 <= codepoint && codepoint <= 0xFFFF); + return codepoint; + } + + /*! + @brief check if the next byte(s) are inside a given range + + Adds the current byte and, for each passed range, reads a new byte and + checks if it is inside the range. If a violation was detected, set up an + error message and return false. Otherwise, return true. + + @param[in] ranges list of integers; interpreted as list of pairs of + inclusive lower and upper bound, respectively + + @pre The passed list @a ranges must have 2, 4, or 6 elements; that is, + 1, 2, or 3 pairs. This precondition is enforced by an assertion. + + @return true if and only if no range violation was detected + */ + bool next_byte_in_range(std::initializer_list ranges) + { + JSON_ASSERT(ranges.size() == 2 || ranges.size() == 4 || ranges.size() == 6); + add(current); + + for (auto range = ranges.begin(); range != ranges.end(); ++range) + { + get(); + if (JSON_HEDLEY_LIKELY(*range <= current && current <= *(++range))) + { + add(current); + } + else + { + error_message = "invalid string: ill-formed UTF-8 byte"; + return false; + } + } + + return true; + } + + /*! + @brief scan a string literal + + This function scans a string according to Sect. 7 of RFC 8259. While + scanning, bytes are escaped and copied into buffer token_buffer. Then the + function returns successfully, token_buffer is *not* null-terminated (as it + may contain \0 bytes), and token_buffer.size() is the number of bytes in the + string. + + @return token_type::value_string if string could be successfully scanned, + token_type::parse_error otherwise + + @note In case of errors, variable error_message contains a textual + description. + */ + token_type scan_string() + { + // reset token_buffer (ignore opening quote) + reset(); + + // we entered the function by reading an open quote + JSON_ASSERT(current == '\"'); + + while (true) + { + // get next character + switch (get()) + { + // end of file while parsing string + case std::char_traits::eof(): + { + error_message = "invalid string: missing closing quote"; + return token_type::parse_error; + } + + // closing quote + case '\"': + { + return token_type::value_string; + } + + // escapes + case '\\': + { + switch (get()) + { + // quotation mark + case '\"': + add('\"'); + break; + // reverse solidus + case '\\': + add('\\'); + break; + // solidus + case '/': + add('/'); + break; + // backspace + case 'b': + add('\b'); + break; + // form feed + case 'f': + add('\f'); + break; + // line feed + case 'n': + add('\n'); + break; + // carriage return + case 'r': + add('\r'); + break; + // tab + case 't': + add('\t'); + break; + + // unicode escapes + case 'u': + { + const int codepoint1 = get_codepoint(); + int codepoint = codepoint1; // start with codepoint1 + + if (JSON_HEDLEY_UNLIKELY(codepoint1 == -1)) + { + error_message = "invalid string: '\\u' must be followed by 4 hex digits"; + return token_type::parse_error; + } + + // check if code point is a high surrogate + if (0xD800 <= codepoint1 && codepoint1 <= 0xDBFF) + { + // expect next \uxxxx entry + if (JSON_HEDLEY_LIKELY(get() == '\\' && get() == 'u')) + { + const int codepoint2 = get_codepoint(); + + if (JSON_HEDLEY_UNLIKELY(codepoint2 == -1)) + { + error_message = "invalid string: '\\u' must be followed by 4 hex digits"; + return token_type::parse_error; + } + + // check if codepoint2 is a low surrogate + if (JSON_HEDLEY_LIKELY(0xDC00 <= codepoint2 && codepoint2 <= 0xDFFF)) + { + // overwrite codepoint + codepoint = static_cast( + // high surrogate occupies the most significant 22 bits + (static_cast(codepoint1) << 10u) + // low surrogate occupies the least significant 15 bits + + static_cast(codepoint2) + // there is still the 0xD800, 0xDC00 and 0x10000 noise + // in the result, so we have to subtract with: + // (0xD800 << 10) + DC00 - 0x10000 = 0x35FDC00 + - 0x35FDC00u); + } + else + { + error_message = "invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF"; + return token_type::parse_error; + } + } + else + { + error_message = "invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF"; + return token_type::parse_error; + } + } + else + { + if (JSON_HEDLEY_UNLIKELY(0xDC00 <= codepoint1 && codepoint1 <= 0xDFFF)) + { + error_message = "invalid string: surrogate U+DC00..U+DFFF must follow U+D800..U+DBFF"; + return token_type::parse_error; + } + } + + // result of the above calculation yields a proper codepoint + JSON_ASSERT(0x00 <= codepoint && codepoint <= 0x10FFFF); + + // translate codepoint into bytes + if (codepoint < 0x80) + { + // 1-byte characters: 0xxxxxxx (ASCII) + add(static_cast(codepoint)); + } + else if (codepoint <= 0x7FF) + { + // 2-byte characters: 110xxxxx 10xxxxxx + add(static_cast(0xC0u | (static_cast(codepoint) >> 6u))); + add(static_cast(0x80u | (static_cast(codepoint) & 0x3Fu))); + } + else if (codepoint <= 0xFFFF) + { + // 3-byte characters: 1110xxxx 10xxxxxx 10xxxxxx + add(static_cast(0xE0u | (static_cast(codepoint) >> 12u))); + add(static_cast(0x80u | ((static_cast(codepoint) >> 6u) & 0x3Fu))); + add(static_cast(0x80u | (static_cast(codepoint) & 0x3Fu))); + } + else + { + // 4-byte characters: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + add(static_cast(0xF0u | (static_cast(codepoint) >> 18u))); + add(static_cast(0x80u | ((static_cast(codepoint) >> 12u) & 0x3Fu))); + add(static_cast(0x80u | ((static_cast(codepoint) >> 6u) & 0x3Fu))); + add(static_cast(0x80u | (static_cast(codepoint) & 0x3Fu))); + } + + break; + } + + // other characters after escape + default: + error_message = "invalid string: forbidden character after backslash"; + return token_type::parse_error; + } + + break; + } + + // invalid control characters + case 0x00: + { + error_message = "invalid string: control character U+0000 (NUL) must be escaped to \\u0000"; + return token_type::parse_error; + } + + case 0x01: + { + error_message = "invalid string: control character U+0001 (SOH) must be escaped to \\u0001"; + return token_type::parse_error; + } + + case 0x02: + { + error_message = "invalid string: control character U+0002 (STX) must be escaped to \\u0002"; + return token_type::parse_error; + } + + case 0x03: + { + error_message = "invalid string: control character U+0003 (ETX) must be escaped to \\u0003"; + return token_type::parse_error; + } + + case 0x04: + { + error_message = "invalid string: control character U+0004 (EOT) must be escaped to \\u0004"; + return token_type::parse_error; + } + + case 0x05: + { + error_message = "invalid string: control character U+0005 (ENQ) must be escaped to \\u0005"; + return token_type::parse_error; + } + + case 0x06: + { + error_message = "invalid string: control character U+0006 (ACK) must be escaped to \\u0006"; + return token_type::parse_error; + } + + case 0x07: + { + error_message = "invalid string: control character U+0007 (BEL) must be escaped to \\u0007"; + return token_type::parse_error; + } + + case 0x08: + { + error_message = "invalid string: control character U+0008 (BS) must be escaped to \\u0008 or \\b"; + return token_type::parse_error; + } + + case 0x09: + { + error_message = "invalid string: control character U+0009 (HT) must be escaped to \\u0009 or \\t"; + return token_type::parse_error; + } + + case 0x0A: + { + error_message = "invalid string: control character U+000A (LF) must be escaped to \\u000A or \\n"; + return token_type::parse_error; + } + + case 0x0B: + { + error_message = "invalid string: control character U+000B (VT) must be escaped to \\u000B"; + return token_type::parse_error; + } + + case 0x0C: + { + error_message = "invalid string: control character U+000C (FF) must be escaped to \\u000C or \\f"; + return token_type::parse_error; + } + + case 0x0D: + { + error_message = "invalid string: control character U+000D (CR) must be escaped to \\u000D or \\r"; + return token_type::parse_error; + } + + case 0x0E: + { + error_message = "invalid string: control character U+000E (SO) must be escaped to \\u000E"; + return token_type::parse_error; + } + + case 0x0F: + { + error_message = "invalid string: control character U+000F (SI) must be escaped to \\u000F"; + return token_type::parse_error; + } + + case 0x10: + { + error_message = "invalid string: control character U+0010 (DLE) must be escaped to \\u0010"; + return token_type::parse_error; + } + + case 0x11: + { + error_message = "invalid string: control character U+0011 (DC1) must be escaped to \\u0011"; + return token_type::parse_error; + } + + case 0x12: + { + error_message = "invalid string: control character U+0012 (DC2) must be escaped to \\u0012"; + return token_type::parse_error; + } + + case 0x13: + { + error_message = "invalid string: control character U+0013 (DC3) must be escaped to \\u0013"; + return token_type::parse_error; + } + + case 0x14: + { + error_message = "invalid string: control character U+0014 (DC4) must be escaped to \\u0014"; + return token_type::parse_error; + } + + case 0x15: + { + error_message = "invalid string: control character U+0015 (NAK) must be escaped to \\u0015"; + return token_type::parse_error; + } + + case 0x16: + { + error_message = "invalid string: control character U+0016 (SYN) must be escaped to \\u0016"; + return token_type::parse_error; + } + + case 0x17: + { + error_message = "invalid string: control character U+0017 (ETB) must be escaped to \\u0017"; + return token_type::parse_error; + } + + case 0x18: + { + error_message = "invalid string: control character U+0018 (CAN) must be escaped to \\u0018"; + return token_type::parse_error; + } + + case 0x19: + { + error_message = "invalid string: control character U+0019 (EM) must be escaped to \\u0019"; + return token_type::parse_error; + } + + case 0x1A: + { + error_message = "invalid string: control character U+001A (SUB) must be escaped to \\u001A"; + return token_type::parse_error; + } + + case 0x1B: + { + error_message = "invalid string: control character U+001B (ESC) must be escaped to \\u001B"; + return token_type::parse_error; + } + + case 0x1C: + { + error_message = "invalid string: control character U+001C (FS) must be escaped to \\u001C"; + return token_type::parse_error; + } + + case 0x1D: + { + error_message = "invalid string: control character U+001D (GS) must be escaped to \\u001D"; + return token_type::parse_error; + } + + case 0x1E: + { + error_message = "invalid string: control character U+001E (RS) must be escaped to \\u001E"; + return token_type::parse_error; + } + + case 0x1F: + { + error_message = "invalid string: control character U+001F (US) must be escaped to \\u001F"; + return token_type::parse_error; + } + + // U+0020..U+007F (except U+0022 (quote) and U+005C (backspace)) + case 0x20: + case 0x21: + case 0x23: + case 0x24: + case 0x25: + case 0x26: + case 0x27: + case 0x28: + case 0x29: + case 0x2A: + case 0x2B: + case 0x2C: + case 0x2D: + case 0x2E: + case 0x2F: + case 0x30: + case 0x31: + case 0x32: + case 0x33: + case 0x34: + case 0x35: + case 0x36: + case 0x37: + case 0x38: + case 0x39: + case 0x3A: + case 0x3B: + case 0x3C: + case 0x3D: + case 0x3E: + case 0x3F: + case 0x40: + case 0x41: + case 0x42: + case 0x43: + case 0x44: + case 0x45: + case 0x46: + case 0x47: + case 0x48: + case 0x49: + case 0x4A: + case 0x4B: + case 0x4C: + case 0x4D: + case 0x4E: + case 0x4F: + case 0x50: + case 0x51: + case 0x52: + case 0x53: + case 0x54: + case 0x55: + case 0x56: + case 0x57: + case 0x58: + case 0x59: + case 0x5A: + case 0x5B: + case 0x5D: + case 0x5E: + case 0x5F: + case 0x60: + case 0x61: + case 0x62: + case 0x63: + case 0x64: + case 0x65: + case 0x66: + case 0x67: + case 0x68: + case 0x69: + case 0x6A: + case 0x6B: + case 0x6C: + case 0x6D: + case 0x6E: + case 0x6F: + case 0x70: + case 0x71: + case 0x72: + case 0x73: + case 0x74: + case 0x75: + case 0x76: + case 0x77: + case 0x78: + case 0x79: + case 0x7A: + case 0x7B: + case 0x7C: + case 0x7D: + case 0x7E: + case 0x7F: + { + add(current); + break; + } + + // U+0080..U+07FF: bytes C2..DF 80..BF + case 0xC2: + case 0xC3: + case 0xC4: + case 0xC5: + case 0xC6: + case 0xC7: + case 0xC8: + case 0xC9: + case 0xCA: + case 0xCB: + case 0xCC: + case 0xCD: + case 0xCE: + case 0xCF: + case 0xD0: + case 0xD1: + case 0xD2: + case 0xD3: + case 0xD4: + case 0xD5: + case 0xD6: + case 0xD7: + case 0xD8: + case 0xD9: + case 0xDA: + case 0xDB: + case 0xDC: + case 0xDD: + case 0xDE: + case 0xDF: + { + if (JSON_HEDLEY_UNLIKELY(!next_byte_in_range({0x80, 0xBF}))) + { + return token_type::parse_error; + } + break; + } + + // U+0800..U+0FFF: bytes E0 A0..BF 80..BF + case 0xE0: + { + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0xA0, 0xBF, 0x80, 0xBF})))) + { + return token_type::parse_error; + } + break; + } + + // U+1000..U+CFFF: bytes E1..EC 80..BF 80..BF + // U+E000..U+FFFF: bytes EE..EF 80..BF 80..BF + case 0xE1: + case 0xE2: + case 0xE3: + case 0xE4: + case 0xE5: + case 0xE6: + case 0xE7: + case 0xE8: + case 0xE9: + case 0xEA: + case 0xEB: + case 0xEC: + case 0xEE: + case 0xEF: + { + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF})))) + { + return token_type::parse_error; + } + break; + } + + // U+D000..U+D7FF: bytes ED 80..9F 80..BF + case 0xED: + { + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x9F, 0x80, 0xBF})))) + { + return token_type::parse_error; + } + break; + } + + // U+10000..U+3FFFF F0 90..BF 80..BF 80..BF + case 0xF0: + { + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF})))) + { + return token_type::parse_error; + } + break; + } + + // U+40000..U+FFFFF F1..F3 80..BF 80..BF 80..BF + case 0xF1: + case 0xF2: + case 0xF3: + { + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF})))) + { + return token_type::parse_error; + } + break; + } + + // U+100000..U+10FFFF F4 80..8F 80..BF 80..BF + case 0xF4: + { + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF})))) + { + return token_type::parse_error; + } + break; + } + + // remaining bytes (80..C1 and F5..FF) are ill-formed + default: + { + error_message = "invalid string: ill-formed UTF-8 byte"; + return token_type::parse_error; + } + } + } + } + + /*! + * @brief scan a comment + * @return whether comment could be scanned successfully + */ + bool scan_comment() + { + switch (get()) + { + // single-line comments skip input until a newline or EOF is read + case '/': + { + while (true) + { + switch (get()) + { + case '\n': + case '\r': + case std::char_traits::eof(): + case '\0': + return true; + + default: + break; + } + } + } + + // multi-line comments skip input until */ is read + case '*': + { + while (true) + { + switch (get()) + { + case std::char_traits::eof(): + case '\0': + { + error_message = "invalid comment; missing closing '*/'"; + return false; + } + + case '*': + { + switch (get()) + { + case '/': + return true; + + default: + { + unget(); + continue; + } + } + } + + default: + continue; + } + } + } + + // unexpected character after reading '/' + default: + { + error_message = "invalid comment; expecting '/' or '*' after '/'"; + return false; + } + } + } + + JSON_HEDLEY_NON_NULL(2) + static void strtof(float& f, const char* str, char** endptr) noexcept + { + f = std::strtof(str, endptr); + } + + JSON_HEDLEY_NON_NULL(2) + static void strtof(double& f, const char* str, char** endptr) noexcept + { + f = std::strtod(str, endptr); + } + + JSON_HEDLEY_NON_NULL(2) + static void strtof(long double& f, const char* str, char** endptr) noexcept + { + f = std::strtold(str, endptr); + } + + /*! + @brief scan a number literal + + This function scans a string according to Sect. 6 of RFC 8259. + + The function is realized with a deterministic finite state machine derived + from the grammar described in RFC 8259. Starting in state "init", the + input is read and used to determined the next state. Only state "done" + accepts the number. State "error" is a trap state to model errors. In the + table below, "anything" means any character but the ones listed before. + + state | 0 | 1-9 | e E | + | - | . | anything + ---------|----------|----------|----------|---------|---------|----------|----------- + init | zero | any1 | [error] | [error] | minus | [error] | [error] + minus | zero | any1 | [error] | [error] | [error] | [error] | [error] + zero | done | done | exponent | done | done | decimal1 | done + any1 | any1 | any1 | exponent | done | done | decimal1 | done + decimal1 | decimal2 | decimal2 | [error] | [error] | [error] | [error] | [error] + decimal2 | decimal2 | decimal2 | exponent | done | done | done | done + exponent | any2 | any2 | [error] | sign | sign | [error] | [error] + sign | any2 | any2 | [error] | [error] | [error] | [error] | [error] + any2 | any2 | any2 | done | done | done | done | done + + The state machine is realized with one label per state (prefixed with + "scan_number_") and `goto` statements between them. The state machine + contains cycles, but any cycle can be left when EOF is read. Therefore, + the function is guaranteed to terminate. + + During scanning, the read bytes are stored in token_buffer. This string is + then converted to a signed integer, an unsigned integer, or a + floating-point number. + + @return token_type::value_unsigned, token_type::value_integer, or + token_type::value_float if number could be successfully scanned, + token_type::parse_error otherwise + + @note The scanner is independent of the current locale. Internally, the + locale's decimal point is used instead of `.` to work with the + locale-dependent converters. + */ + token_type scan_number() // lgtm [cpp/use-of-goto] + { + // reset token_buffer to store the number's bytes + reset(); + + // the type of the parsed number; initially set to unsigned; will be + // changed if minus sign, decimal point or exponent is read + token_type number_type = token_type::value_unsigned; + + // state (init): we just found out we need to scan a number + switch (current) + { + case '-': + { + add(current); + goto scan_number_minus; + } + + case '0': + { + add(current); + goto scan_number_zero; + } + + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + { + add(current); + goto scan_number_any1; + } + + // all other characters are rejected outside scan_number() + default: // LCOV_EXCL_LINE + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE + } + +scan_number_minus: + // state: we just parsed a leading minus sign + number_type = token_type::value_integer; + switch (get()) + { + case '0': + { + add(current); + goto scan_number_zero; + } + + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + { + add(current); + goto scan_number_any1; + } + + default: + { + error_message = "invalid number; expected digit after '-'"; + return token_type::parse_error; + } + } + +scan_number_zero: + // state: we just parse a zero (maybe with a leading minus sign) + switch (get()) + { + case '.': + { + add(decimal_point_char); + goto scan_number_decimal1; + } + + case 'e': + case 'E': + { + add(current); + goto scan_number_exponent; + } + + default: + goto scan_number_done; + } + +scan_number_any1: + // state: we just parsed a number 0-9 (maybe with a leading minus sign) + switch (get()) + { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + { + add(current); + goto scan_number_any1; + } + + case '.': + { + add(decimal_point_char); + goto scan_number_decimal1; + } + + case 'e': + case 'E': + { + add(current); + goto scan_number_exponent; + } + + default: + goto scan_number_done; + } + +scan_number_decimal1: + // state: we just parsed a decimal point + number_type = token_type::value_float; + switch (get()) + { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + { + add(current); + goto scan_number_decimal2; + } + + default: + { + error_message = "invalid number; expected digit after '.'"; + return token_type::parse_error; + } + } + +scan_number_decimal2: + // we just parsed at least one number after a decimal point + switch (get()) + { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + { + add(current); + goto scan_number_decimal2; + } + + case 'e': + case 'E': + { + add(current); + goto scan_number_exponent; + } + + default: + goto scan_number_done; + } + +scan_number_exponent: + // we just parsed an exponent + number_type = token_type::value_float; + switch (get()) + { + case '+': + case '-': + { + add(current); + goto scan_number_sign; + } + + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + { + add(current); + goto scan_number_any2; + } + + default: + { + error_message = + "invalid number; expected '+', '-', or digit after exponent"; + return token_type::parse_error; + } + } + +scan_number_sign: + // we just parsed an exponent sign + switch (get()) + { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + { + add(current); + goto scan_number_any2; + } + + default: + { + error_message = "invalid number; expected digit after exponent sign"; + return token_type::parse_error; + } + } + +scan_number_any2: + // we just parsed a number after the exponent or exponent sign + switch (get()) + { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + { + add(current); + goto scan_number_any2; + } + + default: + goto scan_number_done; + } + +scan_number_done: + // unget the character after the number (we only read it to know that + // we are done scanning a number) + unget(); + + char* endptr = nullptr; // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg) + errno = 0; + + // try to parse integers first and fall back to floats + if (number_type == token_type::value_unsigned) + { + const auto x = std::strtoull(token_buffer.data(), &endptr, 10); + + // we checked the number format before + JSON_ASSERT(endptr == token_buffer.data() + token_buffer.size()); + + if (errno == 0) + { + value_unsigned = static_cast(x); + if (value_unsigned == x) + { + return token_type::value_unsigned; + } + } + } + else if (number_type == token_type::value_integer) + { + const auto x = std::strtoll(token_buffer.data(), &endptr, 10); + + // we checked the number format before + JSON_ASSERT(endptr == token_buffer.data() + token_buffer.size()); + + if (errno == 0) + { + value_integer = static_cast(x); + if (value_integer == x) + { + return token_type::value_integer; + } + } + } + + // this code is reached if we parse a floating-point number or if an + // integer conversion above failed + strtof(value_float, token_buffer.data(), &endptr); + + // we checked the number format before + JSON_ASSERT(endptr == token_buffer.data() + token_buffer.size()); + + return token_type::value_float; + } + + /*! + @param[in] literal_text the literal text to expect + @param[in] length the length of the passed literal text + @param[in] return_type the token type to return on success + */ + JSON_HEDLEY_NON_NULL(2) + token_type scan_literal(const char_type* literal_text, const std::size_t length, + token_type return_type) + { + JSON_ASSERT(std::char_traits::to_char_type(current) == literal_text[0]); + for (std::size_t i = 1; i < length; ++i) + { + if (JSON_HEDLEY_UNLIKELY(std::char_traits::to_char_type(get()) != literal_text[i])) + { + error_message = "invalid literal"; + return token_type::parse_error; + } + } + return return_type; + } + + ///////////////////// + // input management + ///////////////////// + + /// reset token_buffer; current character is beginning of token + void reset() noexcept + { + token_buffer.clear(); + token_string.clear(); + token_string.push_back(std::char_traits::to_char_type(current)); + } + + /* + @brief get next character from the input + + This function provides the interface to the used input adapter. It does + not throw in case the input reached EOF, but returns a + `std::char_traits::eof()` in that case. Stores the scanned characters + for use in error messages. + + @return character read from the input + */ + char_int_type get() + { + ++position.chars_read_total; + ++position.chars_read_current_line; + + if (next_unget) + { + // just reset the next_unget variable and work with current + next_unget = false; + } + else + { + current = ia.get_character(); + } + + if (JSON_HEDLEY_LIKELY(current != std::char_traits::eof())) + { + token_string.push_back(std::char_traits::to_char_type(current)); + } + + if (current == '\n') + { + ++position.lines_read; + position.chars_read_current_line = 0; + } + + return current; + } + + /*! + @brief unget current character (read it again on next get) + + We implement unget by setting variable next_unget to true. The input is not + changed - we just simulate ungetting by modifying chars_read_total, + chars_read_current_line, and token_string. The next call to get() will + behave as if the unget character is read again. + */ + void unget() + { + next_unget = true; + + --position.chars_read_total; + + // in case we "unget" a newline, we have to also decrement the lines_read + if (position.chars_read_current_line == 0) + { + if (position.lines_read > 0) + { + --position.lines_read; + } + } + else + { + --position.chars_read_current_line; + } + + if (JSON_HEDLEY_LIKELY(current != std::char_traits::eof())) + { + JSON_ASSERT(!token_string.empty()); + token_string.pop_back(); + } + } + + /// add a character to token_buffer + void add(char_int_type c) + { + token_buffer.push_back(static_cast(c)); + } + + public: + ///////////////////// + // value getters + ///////////////////// + + /// return integer value + constexpr number_integer_t get_number_integer() const noexcept + { + return value_integer; + } + + /// return unsigned integer value + constexpr number_unsigned_t get_number_unsigned() const noexcept + { + return value_unsigned; + } + + /// return floating-point value + constexpr number_float_t get_number_float() const noexcept + { + return value_float; + } + + /// return current string value (implicitly resets the token; useful only once) + string_t& get_string() + { + return token_buffer; + } + + ///////////////////// + // diagnostics + ///////////////////// + + /// return position of last read token + constexpr position_t get_position() const noexcept + { + return position; + } + + /// return the last read token (for errors only). Will never contain EOF + /// (an arbitrary value that is not a valid char value, often -1), because + /// 255 may legitimately occur. May contain NUL, which should be escaped. + std::string get_token_string() const + { + // escape control characters + std::string result; + for (const auto c : token_string) + { + if (static_cast(c) <= '\x1F') + { + // escape control characters + std::array cs{{}}; + static_cast((std::snprintf)(cs.data(), cs.size(), "", static_cast(c))); // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg) + result += cs.data(); + } + else + { + // add character as is + result.push_back(static_cast(c)); + } + } + + return result; + } + + /// return syntax error message + JSON_HEDLEY_RETURNS_NON_NULL + constexpr const char* get_error_message() const noexcept + { + return error_message; + } + + ///////////////////// + // actual scanner + ///////////////////// + + /*! + @brief skip the UTF-8 byte order mark + @return true iff there is no BOM or the correct BOM has been skipped + */ + bool skip_bom() + { + if (get() == 0xEF) + { + // check if we completely parse the BOM + return get() == 0xBB && get() == 0xBF; + } + + // the first character is not the beginning of the BOM; unget it to + // process is later + unget(); + return true; + } + + void skip_whitespace() + { + do + { + get(); + } + while (current == ' ' || current == '\t' || current == '\n' || current == '\r'); + } + + token_type scan() + { + // initially, skip the BOM + if (position.chars_read_total == 0 && !skip_bom()) + { + error_message = "invalid BOM; must be 0xEF 0xBB 0xBF if given"; + return token_type::parse_error; + } + + // read next character and ignore whitespace + skip_whitespace(); + + // ignore comments + while (ignore_comments && current == '/') + { + if (!scan_comment()) + { + return token_type::parse_error; + } + + // skip following whitespace + skip_whitespace(); + } + + switch (current) + { + // structural characters + case '[': + return token_type::begin_array; + case ']': + return token_type::end_array; + case '{': + return token_type::begin_object; + case '}': + return token_type::end_object; + case ':': + return token_type::name_separator; + case ',': + return token_type::value_separator; + + // literals + case 't': + { + std::array true_literal = {{static_cast('t'), static_cast('r'), static_cast('u'), static_cast('e')}}; + return scan_literal(true_literal.data(), true_literal.size(), token_type::literal_true); + } + case 'f': + { + std::array false_literal = {{static_cast('f'), static_cast('a'), static_cast('l'), static_cast('s'), static_cast('e')}}; + return scan_literal(false_literal.data(), false_literal.size(), token_type::literal_false); + } + case 'n': + { + std::array null_literal = {{static_cast('n'), static_cast('u'), static_cast('l'), static_cast('l')}}; + return scan_literal(null_literal.data(), null_literal.size(), token_type::literal_null); + } + + // string + case '\"': + return scan_string(); + + // number + case '-': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + return scan_number(); + + // end of input (the null byte is needed when parsing from + // string literals) + case '\0': + case std::char_traits::eof(): + return token_type::end_of_input; + + // error + default: + error_message = "invalid literal"; + return token_type::parse_error; + } + } + + private: + /// input adapter + InputAdapterType ia; + + /// whether comments should be ignored (true) or signaled as errors (false) + const bool ignore_comments = false; + + /// the current character + char_int_type current = std::char_traits::eof(); + + /// whether the next get() call should just return current + bool next_unget = false; + + /// the start position of the current token + position_t position {}; + + /// raw input token string (for error messages) + std::vector token_string {}; + + /// buffer for variable-length tokens (numbers, strings) + string_t token_buffer {}; + + /// a description of occurred lexer errors + const char* error_message = ""; + + // number values + number_integer_t value_integer = 0; + number_unsigned_t value_unsigned = 0; + number_float_t value_float = 0; + + /// the decimal point + const char_int_type decimal_point_char = '.'; +}; +} // namespace detail +} // namespace nlohmann + +// #include + +// #include + + +#include // size_t +#include // declval +#include // string + +// #include + +// #include + + +namespace nlohmann +{ +namespace detail +{ +template +using null_function_t = decltype(std::declval().null()); + +template +using boolean_function_t = + decltype(std::declval().boolean(std::declval())); + +template +using number_integer_function_t = + decltype(std::declval().number_integer(std::declval())); + +template +using number_unsigned_function_t = + decltype(std::declval().number_unsigned(std::declval())); + +template +using number_float_function_t = decltype(std::declval().number_float( + std::declval(), std::declval())); + +template +using string_function_t = + decltype(std::declval().string(std::declval())); + +template +using binary_function_t = + decltype(std::declval().binary(std::declval())); + +template +using start_object_function_t = + decltype(std::declval().start_object(std::declval())); + +template +using key_function_t = + decltype(std::declval().key(std::declval())); + +template +using end_object_function_t = decltype(std::declval().end_object()); + +template +using start_array_function_t = + decltype(std::declval().start_array(std::declval())); + +template +using end_array_function_t = decltype(std::declval().end_array()); + +template +using parse_error_function_t = decltype(std::declval().parse_error( + std::declval(), std::declval(), + std::declval())); + +template +struct is_sax +{ + private: + static_assert(is_basic_json::value, + "BasicJsonType must be of type basic_json<...>"); + + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using binary_t = typename BasicJsonType::binary_t; + using exception_t = typename BasicJsonType::exception; + + public: + static constexpr bool value = + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value; +}; + +template +struct is_sax_static_asserts +{ + private: + static_assert(is_basic_json::value, + "BasicJsonType must be of type basic_json<...>"); + + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using binary_t = typename BasicJsonType::binary_t; + using exception_t = typename BasicJsonType::exception; + + public: + static_assert(is_detected_exact::value, + "Missing/invalid function: bool null()"); + static_assert(is_detected_exact::value, + "Missing/invalid function: bool boolean(bool)"); + static_assert(is_detected_exact::value, + "Missing/invalid function: bool boolean(bool)"); + static_assert( + is_detected_exact::value, + "Missing/invalid function: bool number_integer(number_integer_t)"); + static_assert( + is_detected_exact::value, + "Missing/invalid function: bool number_unsigned(number_unsigned_t)"); + static_assert(is_detected_exact::value, + "Missing/invalid function: bool number_float(number_float_t, const string_t&)"); + static_assert( + is_detected_exact::value, + "Missing/invalid function: bool string(string_t&)"); + static_assert( + is_detected_exact::value, + "Missing/invalid function: bool binary(binary_t&)"); + static_assert(is_detected_exact::value, + "Missing/invalid function: bool start_object(std::size_t)"); + static_assert(is_detected_exact::value, + "Missing/invalid function: bool key(string_t&)"); + static_assert(is_detected_exact::value, + "Missing/invalid function: bool end_object()"); + static_assert(is_detected_exact::value, + "Missing/invalid function: bool start_array(std::size_t)"); + static_assert(is_detected_exact::value, + "Missing/invalid function: bool end_array()"); + static_assert( + is_detected_exact::value, + "Missing/invalid function: bool parse_error(std::size_t, const " + "std::string&, const exception&)"); +}; +} // namespace detail +} // namespace nlohmann + +// #include + +// #include + + +namespace nlohmann +{ +namespace detail +{ + +/// how to treat CBOR tags +enum class cbor_tag_handler_t +{ + error, ///< throw a parse_error exception in case of a tag + ignore, ///< ignore tags + store ///< store tags as binary type +}; + +/*! +@brief determine system byte order + +@return true if and only if system's byte order is little endian + +@note from https://stackoverflow.com/a/1001328/266378 +*/ +static inline bool little_endianness(int num = 1) noexcept +{ + return *reinterpret_cast(&num) == 1; +} + + +/////////////////// +// binary reader // +/////////////////// + +/*! +@brief deserialization of CBOR, MessagePack, and UBJSON values +*/ +template> +class binary_reader +{ + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using binary_t = typename BasicJsonType::binary_t; + using json_sax_t = SAX; + using char_type = typename InputAdapterType::char_type; + using char_int_type = typename std::char_traits::int_type; + + public: + /*! + @brief create a binary reader + + @param[in] adapter input adapter to read from + */ + explicit binary_reader(InputAdapterType&& adapter) noexcept : ia(std::move(adapter)) + { + (void)detail::is_sax_static_asserts {}; + } + + // make class move-only + binary_reader(const binary_reader&) = delete; + binary_reader(binary_reader&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor) + binary_reader& operator=(const binary_reader&) = delete; + binary_reader& operator=(binary_reader&&) = default; // NOLINT(hicpp-noexcept-move,performance-noexcept-move-constructor) + ~binary_reader() = default; + + /*! + @param[in] format the binary format to parse + @param[in] sax_ a SAX event processor + @param[in] strict whether to expect the input to be consumed completed + @param[in] tag_handler how to treat CBOR tags + + @return whether parsing was successful + */ + JSON_HEDLEY_NON_NULL(3) + bool sax_parse(const input_format_t format, + json_sax_t* sax_, + const bool strict = true, + const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) + { + sax = sax_; + bool result = false; + + switch (format) + { + case input_format_t::bson: + result = parse_bson_internal(); + break; + + case input_format_t::cbor: + result = parse_cbor_internal(true, tag_handler); + break; + + case input_format_t::msgpack: + result = parse_msgpack_internal(); + break; + + case input_format_t::ubjson: + result = parse_ubjson_internal(); + break; + + case input_format_t::json: // LCOV_EXCL_LINE + default: // LCOV_EXCL_LINE + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE + } + + // strict mode: next byte must be EOF + if (result && strict) + { + if (format == input_format_t::ubjson) + { + get_ignore_noop(); + } + else + { + get(); + } + + if (JSON_HEDLEY_UNLIKELY(current != std::char_traits::eof())) + { + return sax->parse_error(chars_read, get_token_string(), + parse_error::create(110, chars_read, exception_message(format, "expected end of input; last byte: 0x" + get_token_string(), "value"), BasicJsonType())); + } + } + + return result; + } + + private: + ////////// + // BSON // + ////////// + + /*! + @brief Reads in a BSON-object and passes it to the SAX-parser. + @return whether a valid BSON-value was passed to the SAX parser + */ + bool parse_bson_internal() + { + std::int32_t document_size{}; + get_number(input_format_t::bson, document_size); + + if (JSON_HEDLEY_UNLIKELY(!sax->start_object(static_cast(-1)))) + { + return false; + } + + if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_list(/*is_array*/false))) + { + return false; + } + + return sax->end_object(); + } + + /*! + @brief Parses a C-style string from the BSON input. + @param[in,out] result A reference to the string variable where the read + string is to be stored. + @return `true` if the \x00-byte indicating the end of the string was + encountered before the EOF; false` indicates an unexpected EOF. + */ + bool get_bson_cstr(string_t& result) + { + auto out = std::back_inserter(result); + while (true) + { + get(); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bson, "cstring"))) + { + return false; + } + if (current == 0x00) + { + return true; + } + *out++ = static_cast(current); + } + } + + /*! + @brief Parses a zero-terminated string of length @a len from the BSON + input. + @param[in] len The length (including the zero-byte at the end) of the + string to be read. + @param[in,out] result A reference to the string variable where the read + string is to be stored. + @tparam NumberType The type of the length @a len + @pre len >= 1 + @return `true` if the string was successfully parsed + */ + template + bool get_bson_string(const NumberType len, string_t& result) + { + if (JSON_HEDLEY_UNLIKELY(len < 1)) + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::bson, "string length must be at least 1, is " + std::to_string(len), "string"), BasicJsonType())); + } + + return get_string(input_format_t::bson, len - static_cast(1), result) && get() != std::char_traits::eof(); + } + + /*! + @brief Parses a byte array input of length @a len from the BSON input. + @param[in] len The length of the byte array to be read. + @param[in,out] result A reference to the binary variable where the read + array is to be stored. + @tparam NumberType The type of the length @a len + @pre len >= 0 + @return `true` if the byte array was successfully parsed + */ + template + bool get_bson_binary(const NumberType len, binary_t& result) + { + if (JSON_HEDLEY_UNLIKELY(len < 0)) + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::bson, "byte array length cannot be negative, is " + std::to_string(len), "binary"), BasicJsonType())); + } + + // All BSON binary values have a subtype + std::uint8_t subtype{}; + get_number(input_format_t::bson, subtype); + result.set_subtype(subtype); + + return get_binary(input_format_t::bson, len, result); + } + + /*! + @brief Read a BSON document element of the given @a element_type. + @param[in] element_type The BSON element type, c.f. http://bsonspec.org/spec.html + @param[in] element_type_parse_position The position in the input stream, + where the `element_type` was read. + @warning Not all BSON element types are supported yet. An unsupported + @a element_type will give rise to a parse_error.114: + Unsupported BSON record type 0x... + @return whether a valid BSON-object/array was passed to the SAX parser + */ + bool parse_bson_element_internal(const char_int_type element_type, + const std::size_t element_type_parse_position) + { + switch (element_type) + { + case 0x01: // double + { + double number{}; + return get_number(input_format_t::bson, number) && sax->number_float(static_cast(number), ""); + } + + case 0x02: // string + { + std::int32_t len{}; + string_t value; + return get_number(input_format_t::bson, len) && get_bson_string(len, value) && sax->string(value); + } + + case 0x03: // object + { + return parse_bson_internal(); + } + + case 0x04: // array + { + return parse_bson_array(); + } + + case 0x05: // binary + { + std::int32_t len{}; + binary_t value; + return get_number(input_format_t::bson, len) && get_bson_binary(len, value) && sax->binary(value); + } + + case 0x08: // boolean + { + return sax->boolean(get() != 0); + } + + case 0x0A: // null + { + return sax->null(); + } + + case 0x10: // int32 + { + std::int32_t value{}; + return get_number(input_format_t::bson, value) && sax->number_integer(value); + } + + case 0x12: // int64 + { + std::int64_t value{}; + return get_number(input_format_t::bson, value) && sax->number_integer(value); + } + + default: // anything else not supported (yet) + { + std::array cr{{}}; + static_cast((std::snprintf)(cr.data(), cr.size(), "%.2hhX", static_cast(element_type))); // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg) + return sax->parse_error(element_type_parse_position, std::string(cr.data()), parse_error::create(114, element_type_parse_position, "Unsupported BSON record type 0x" + std::string(cr.data()), BasicJsonType())); + } + } + } + + /*! + @brief Read a BSON element list (as specified in the BSON-spec) + + The same binary layout is used for objects and arrays, hence it must be + indicated with the argument @a is_array which one is expected + (true --> array, false --> object). + + @param[in] is_array Determines if the element list being read is to be + treated as an object (@a is_array == false), or as an + array (@a is_array == true). + @return whether a valid BSON-object/array was passed to the SAX parser + */ + bool parse_bson_element_list(const bool is_array) + { + string_t key; + + while (auto element_type = get()) + { + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bson, "element list"))) + { + return false; + } + + const std::size_t element_type_parse_position = chars_read; + if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key))) + { + return false; + } + + if (!is_array && !sax->key(key)) + { + return false; + } + + if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_internal(element_type, element_type_parse_position))) + { + return false; + } + + // get_bson_cstr only appends + key.clear(); + } + + return true; + } + + /*! + @brief Reads an array from the BSON input and passes it to the SAX-parser. + @return whether a valid BSON-array was passed to the SAX parser + */ + bool parse_bson_array() + { + std::int32_t document_size{}; + get_number(input_format_t::bson, document_size); + + if (JSON_HEDLEY_UNLIKELY(!sax->start_array(static_cast(-1)))) + { + return false; + } + + if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_list(/*is_array*/true))) + { + return false; + } + + return sax->end_array(); + } + + ////////// + // CBOR // + ////////// + + /*! + @param[in] get_char whether a new character should be retrieved from the + input (true) or whether the last read character should + be considered instead (false) + @param[in] tag_handler how CBOR tags should be treated + + @return whether a valid CBOR value was passed to the SAX parser + */ + bool parse_cbor_internal(const bool get_char, + const cbor_tag_handler_t tag_handler) + { + switch (get_char ? get() : current) + { + // EOF + case std::char_traits::eof(): + return unexpect_eof(input_format_t::cbor, "value"); + + // Integer 0x00..0x17 (0..23) + case 0x00: + case 0x01: + case 0x02: + case 0x03: + case 0x04: + case 0x05: + case 0x06: + case 0x07: + case 0x08: + case 0x09: + case 0x0A: + case 0x0B: + case 0x0C: + case 0x0D: + case 0x0E: + case 0x0F: + case 0x10: + case 0x11: + case 0x12: + case 0x13: + case 0x14: + case 0x15: + case 0x16: + case 0x17: + return sax->number_unsigned(static_cast(current)); + + case 0x18: // Unsigned integer (one-byte uint8_t follows) + { + std::uint8_t number{}; + return get_number(input_format_t::cbor, number) && sax->number_unsigned(number); + } + + case 0x19: // Unsigned integer (two-byte uint16_t follows) + { + std::uint16_t number{}; + return get_number(input_format_t::cbor, number) && sax->number_unsigned(number); + } + + case 0x1A: // Unsigned integer (four-byte uint32_t follows) + { + std::uint32_t number{}; + return get_number(input_format_t::cbor, number) && sax->number_unsigned(number); + } + + case 0x1B: // Unsigned integer (eight-byte uint64_t follows) + { + std::uint64_t number{}; + return get_number(input_format_t::cbor, number) && sax->number_unsigned(number); + } + + // Negative integer -1-0x00..-1-0x17 (-1..-24) + case 0x20: + case 0x21: + case 0x22: + case 0x23: + case 0x24: + case 0x25: + case 0x26: + case 0x27: + case 0x28: + case 0x29: + case 0x2A: + case 0x2B: + case 0x2C: + case 0x2D: + case 0x2E: + case 0x2F: + case 0x30: + case 0x31: + case 0x32: + case 0x33: + case 0x34: + case 0x35: + case 0x36: + case 0x37: + return sax->number_integer(static_cast(0x20 - 1 - current)); + + case 0x38: // Negative integer (one-byte uint8_t follows) + { + std::uint8_t number{}; + return get_number(input_format_t::cbor, number) && sax->number_integer(static_cast(-1) - number); + } + + case 0x39: // Negative integer -1-n (two-byte uint16_t follows) + { + std::uint16_t number{}; + return get_number(input_format_t::cbor, number) && sax->number_integer(static_cast(-1) - number); + } + + case 0x3A: // Negative integer -1-n (four-byte uint32_t follows) + { + std::uint32_t number{}; + return get_number(input_format_t::cbor, number) && sax->number_integer(static_cast(-1) - number); + } + + case 0x3B: // Negative integer -1-n (eight-byte uint64_t follows) + { + std::uint64_t number{}; + return get_number(input_format_t::cbor, number) && sax->number_integer(static_cast(-1) + - static_cast(number)); + } + + // Binary data (0x00..0x17 bytes follow) + case 0x40: + case 0x41: + case 0x42: + case 0x43: + case 0x44: + case 0x45: + case 0x46: + case 0x47: + case 0x48: + case 0x49: + case 0x4A: + case 0x4B: + case 0x4C: + case 0x4D: + case 0x4E: + case 0x4F: + case 0x50: + case 0x51: + case 0x52: + case 0x53: + case 0x54: + case 0x55: + case 0x56: + case 0x57: + case 0x58: // Binary data (one-byte uint8_t for n follows) + case 0x59: // Binary data (two-byte uint16_t for n follow) + case 0x5A: // Binary data (four-byte uint32_t for n follow) + case 0x5B: // Binary data (eight-byte uint64_t for n follow) + case 0x5F: // Binary data (indefinite length) + { + binary_t b; + return get_cbor_binary(b) && sax->binary(b); + } + + // UTF-8 string (0x00..0x17 bytes follow) + case 0x60: + case 0x61: + case 0x62: + case 0x63: + case 0x64: + case 0x65: + case 0x66: + case 0x67: + case 0x68: + case 0x69: + case 0x6A: + case 0x6B: + case 0x6C: + case 0x6D: + case 0x6E: + case 0x6F: + case 0x70: + case 0x71: + case 0x72: + case 0x73: + case 0x74: + case 0x75: + case 0x76: + case 0x77: + case 0x78: // UTF-8 string (one-byte uint8_t for n follows) + case 0x79: // UTF-8 string (two-byte uint16_t for n follow) + case 0x7A: // UTF-8 string (four-byte uint32_t for n follow) + case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow) + case 0x7F: // UTF-8 string (indefinite length) + { + string_t s; + return get_cbor_string(s) && sax->string(s); + } + + // array (0x00..0x17 data items follow) + case 0x80: + case 0x81: + case 0x82: + case 0x83: + case 0x84: + case 0x85: + case 0x86: + case 0x87: + case 0x88: + case 0x89: + case 0x8A: + case 0x8B: + case 0x8C: + case 0x8D: + case 0x8E: + case 0x8F: + case 0x90: + case 0x91: + case 0x92: + case 0x93: + case 0x94: + case 0x95: + case 0x96: + case 0x97: + return get_cbor_array(static_cast(static_cast(current) & 0x1Fu), tag_handler); + + case 0x98: // array (one-byte uint8_t for n follows) + { + std::uint8_t len{}; + return get_number(input_format_t::cbor, len) && get_cbor_array(static_cast(len), tag_handler); + } + + case 0x99: // array (two-byte uint16_t for n follow) + { + std::uint16_t len{}; + return get_number(input_format_t::cbor, len) && get_cbor_array(static_cast(len), tag_handler); + } + + case 0x9A: // array (four-byte uint32_t for n follow) + { + std::uint32_t len{}; + return get_number(input_format_t::cbor, len) && get_cbor_array(static_cast(len), tag_handler); + } + + case 0x9B: // array (eight-byte uint64_t for n follow) + { + std::uint64_t len{}; + return get_number(input_format_t::cbor, len) && get_cbor_array(detail::conditional_static_cast(len), tag_handler); + } + + case 0x9F: // array (indefinite length) + return get_cbor_array(static_cast(-1), tag_handler); + + // map (0x00..0x17 pairs of data items follow) + case 0xA0: + case 0xA1: + case 0xA2: + case 0xA3: + case 0xA4: + case 0xA5: + case 0xA6: + case 0xA7: + case 0xA8: + case 0xA9: + case 0xAA: + case 0xAB: + case 0xAC: + case 0xAD: + case 0xAE: + case 0xAF: + case 0xB0: + case 0xB1: + case 0xB2: + case 0xB3: + case 0xB4: + case 0xB5: + case 0xB6: + case 0xB7: + return get_cbor_object(static_cast(static_cast(current) & 0x1Fu), tag_handler); + + case 0xB8: // map (one-byte uint8_t for n follows) + { + std::uint8_t len{}; + return get_number(input_format_t::cbor, len) && get_cbor_object(static_cast(len), tag_handler); + } + + case 0xB9: // map (two-byte uint16_t for n follow) + { + std::uint16_t len{}; + return get_number(input_format_t::cbor, len) && get_cbor_object(static_cast(len), tag_handler); + } + + case 0xBA: // map (four-byte uint32_t for n follow) + { + std::uint32_t len{}; + return get_number(input_format_t::cbor, len) && get_cbor_object(static_cast(len), tag_handler); + } + + case 0xBB: // map (eight-byte uint64_t for n follow) + { + std::uint64_t len{}; + return get_number(input_format_t::cbor, len) && get_cbor_object(detail::conditional_static_cast(len), tag_handler); + } + + case 0xBF: // map (indefinite length) + return get_cbor_object(static_cast(-1), tag_handler); + + case 0xC6: // tagged item + case 0xC7: + case 0xC8: + case 0xC9: + case 0xCA: + case 0xCB: + case 0xCC: + case 0xCD: + case 0xCE: + case 0xCF: + case 0xD0: + case 0xD1: + case 0xD2: + case 0xD3: + case 0xD4: + case 0xD8: // tagged item (1 bytes follow) + case 0xD9: // tagged item (2 bytes follow) + case 0xDA: // tagged item (4 bytes follow) + case 0xDB: // tagged item (8 bytes follow) + { + switch (tag_handler) + { + case cbor_tag_handler_t::error: + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::cbor, "invalid byte: 0x" + last_token, "value"), BasicJsonType())); + } + + case cbor_tag_handler_t::ignore: + { + // ignore binary subtype + switch (current) + { + case 0xD8: + { + std::uint8_t subtype_to_ignore{}; + get_number(input_format_t::cbor, subtype_to_ignore); + break; + } + case 0xD9: + { + std::uint16_t subtype_to_ignore{}; + get_number(input_format_t::cbor, subtype_to_ignore); + break; + } + case 0xDA: + { + std::uint32_t subtype_to_ignore{}; + get_number(input_format_t::cbor, subtype_to_ignore); + break; + } + case 0xDB: + { + std::uint64_t subtype_to_ignore{}; + get_number(input_format_t::cbor, subtype_to_ignore); + break; + } + default: + break; + } + return parse_cbor_internal(true, tag_handler); + } + + case cbor_tag_handler_t::store: + { + binary_t b; + // use binary subtype and store in binary container + switch (current) + { + case 0xD8: + { + std::uint8_t subtype{}; + get_number(input_format_t::cbor, subtype); + b.set_subtype(detail::conditional_static_cast(subtype)); + break; + } + case 0xD9: + { + std::uint16_t subtype{}; + get_number(input_format_t::cbor, subtype); + b.set_subtype(detail::conditional_static_cast(subtype)); + break; + } + case 0xDA: + { + std::uint32_t subtype{}; + get_number(input_format_t::cbor, subtype); + b.set_subtype(detail::conditional_static_cast(subtype)); + break; + } + case 0xDB: + { + std::uint64_t subtype{}; + get_number(input_format_t::cbor, subtype); + b.set_subtype(detail::conditional_static_cast(subtype)); + break; + } + default: + return parse_cbor_internal(true, tag_handler); + } + get(); + return get_cbor_binary(b) && sax->binary(b); + } + + default: // LCOV_EXCL_LINE + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE + return false; // LCOV_EXCL_LINE + } + } + + case 0xF4: // false + return sax->boolean(false); + + case 0xF5: // true + return sax->boolean(true); + + case 0xF6: // null + return sax->null(); + + case 0xF9: // Half-Precision Float (two-byte IEEE 754) + { + const auto byte1_raw = get(); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "number"))) + { + return false; + } + const auto byte2_raw = get(); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "number"))) + { + return false; + } + + const auto byte1 = static_cast(byte1_raw); + const auto byte2 = static_cast(byte2_raw); + + // code from RFC 7049, Appendix D, Figure 3: + // As half-precision floating-point numbers were only added + // to IEEE 754 in 2008, today's programming platforms often + // still only have limited support for them. It is very + // easy to include at least decoding support for them even + // without such support. An example of a small decoder for + // half-precision floating-point numbers in the C language + // is shown in Fig. 3. + const auto half = static_cast((byte1 << 8u) + byte2); + const double val = [&half] + { + const int exp = (half >> 10u) & 0x1Fu; + const unsigned int mant = half & 0x3FFu; + JSON_ASSERT(0 <= exp&& exp <= 32); + JSON_ASSERT(mant <= 1024); + switch (exp) + { + case 0: + return std::ldexp(mant, -24); + case 31: + return (mant == 0) + ? std::numeric_limits::infinity() + : std::numeric_limits::quiet_NaN(); + default: + return std::ldexp(mant + 1024, exp - 25); + } + }(); + return sax->number_float((half & 0x8000u) != 0 + ? static_cast(-val) + : static_cast(val), ""); + } + + case 0xFA: // Single-Precision Float (four-byte IEEE 754) + { + float number{}; + return get_number(input_format_t::cbor, number) && sax->number_float(static_cast(number), ""); + } + + case 0xFB: // Double-Precision Float (eight-byte IEEE 754) + { + double number{}; + return get_number(input_format_t::cbor, number) && sax->number_float(static_cast(number), ""); + } + + default: // anything else (0xFF is handled inside the other types) + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::cbor, "invalid byte: 0x" + last_token, "value"), BasicJsonType())); + } + } + } + + /*! + @brief reads a CBOR string + + This function first reads starting bytes to determine the expected + string length and then copies this number of bytes into a string. + Additionally, CBOR's strings with indefinite lengths are supported. + + @param[out] result created string + + @return whether string creation completed + */ + bool get_cbor_string(string_t& result) + { + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "string"))) + { + return false; + } + + switch (current) + { + // UTF-8 string (0x00..0x17 bytes follow) + case 0x60: + case 0x61: + case 0x62: + case 0x63: + case 0x64: + case 0x65: + case 0x66: + case 0x67: + case 0x68: + case 0x69: + case 0x6A: + case 0x6B: + case 0x6C: + case 0x6D: + case 0x6E: + case 0x6F: + case 0x70: + case 0x71: + case 0x72: + case 0x73: + case 0x74: + case 0x75: + case 0x76: + case 0x77: + { + return get_string(input_format_t::cbor, static_cast(current) & 0x1Fu, result); + } + + case 0x78: // UTF-8 string (one-byte uint8_t for n follows) + { + std::uint8_t len{}; + return get_number(input_format_t::cbor, len) && get_string(input_format_t::cbor, len, result); + } + + case 0x79: // UTF-8 string (two-byte uint16_t for n follow) + { + std::uint16_t len{}; + return get_number(input_format_t::cbor, len) && get_string(input_format_t::cbor, len, result); + } + + case 0x7A: // UTF-8 string (four-byte uint32_t for n follow) + { + std::uint32_t len{}; + return get_number(input_format_t::cbor, len) && get_string(input_format_t::cbor, len, result); + } + + case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow) + { + std::uint64_t len{}; + return get_number(input_format_t::cbor, len) && get_string(input_format_t::cbor, len, result); + } + + case 0x7F: // UTF-8 string (indefinite length) + { + while (get() != 0xFF) + { + string_t chunk; + if (!get_cbor_string(chunk)) + { + return false; + } + result.append(chunk); + } + return true; + } + + default: + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::cbor, "expected length specification (0x60-0x7B) or indefinite string type (0x7F); last byte: 0x" + last_token, "string"), BasicJsonType())); + } + } + } + + /*! + @brief reads a CBOR byte array + + This function first reads starting bytes to determine the expected + byte array length and then copies this number of bytes into the byte array. + Additionally, CBOR's byte arrays with indefinite lengths are supported. + + @param[out] result created byte array + + @return whether byte array creation completed + */ + bool get_cbor_binary(binary_t& result) + { + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::cbor, "binary"))) + { + return false; + } + + switch (current) + { + // Binary data (0x00..0x17 bytes follow) + case 0x40: + case 0x41: + case 0x42: + case 0x43: + case 0x44: + case 0x45: + case 0x46: + case 0x47: + case 0x48: + case 0x49: + case 0x4A: + case 0x4B: + case 0x4C: + case 0x4D: + case 0x4E: + case 0x4F: + case 0x50: + case 0x51: + case 0x52: + case 0x53: + case 0x54: + case 0x55: + case 0x56: + case 0x57: + { + return get_binary(input_format_t::cbor, static_cast(current) & 0x1Fu, result); + } + + case 0x58: // Binary data (one-byte uint8_t for n follows) + { + std::uint8_t len{}; + return get_number(input_format_t::cbor, len) && + get_binary(input_format_t::cbor, len, result); + } + + case 0x59: // Binary data (two-byte uint16_t for n follow) + { + std::uint16_t len{}; + return get_number(input_format_t::cbor, len) && + get_binary(input_format_t::cbor, len, result); + } + + case 0x5A: // Binary data (four-byte uint32_t for n follow) + { + std::uint32_t len{}; + return get_number(input_format_t::cbor, len) && + get_binary(input_format_t::cbor, len, result); + } + + case 0x5B: // Binary data (eight-byte uint64_t for n follow) + { + std::uint64_t len{}; + return get_number(input_format_t::cbor, len) && + get_binary(input_format_t::cbor, len, result); + } + + case 0x5F: // Binary data (indefinite length) + { + while (get() != 0xFF) + { + binary_t chunk; + if (!get_cbor_binary(chunk)) + { + return false; + } + result.insert(result.end(), chunk.begin(), chunk.end()); + } + return true; + } + + default: + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::cbor, "expected length specification (0x40-0x5B) or indefinite binary array type (0x5F); last byte: 0x" + last_token, "binary"), BasicJsonType())); + } + } + } + + /*! + @param[in] len the length of the array or static_cast(-1) for an + array of indefinite size + @param[in] tag_handler how CBOR tags should be treated + @return whether array creation completed + */ + bool get_cbor_array(const std::size_t len, + const cbor_tag_handler_t tag_handler) + { + if (JSON_HEDLEY_UNLIKELY(!sax->start_array(len))) + { + return false; + } + + if (len != static_cast(-1)) + { + for (std::size_t i = 0; i < len; ++i) + { + if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(true, tag_handler))) + { + return false; + } + } + } + else + { + while (get() != 0xFF) + { + if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(false, tag_handler))) + { + return false; + } + } + } + + return sax->end_array(); + } + + /*! + @param[in] len the length of the object or static_cast(-1) for an + object of indefinite size + @param[in] tag_handler how CBOR tags should be treated + @return whether object creation completed + */ + bool get_cbor_object(const std::size_t len, + const cbor_tag_handler_t tag_handler) + { + if (JSON_HEDLEY_UNLIKELY(!sax->start_object(len))) + { + return false; + } + + if (len != 0) + { + string_t key; + if (len != static_cast(-1)) + { + for (std::size_t i = 0; i < len; ++i) + { + get(); + if (JSON_HEDLEY_UNLIKELY(!get_cbor_string(key) || !sax->key(key))) + { + return false; + } + + if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(true, tag_handler))) + { + return false; + } + key.clear(); + } + } + else + { + while (get() != 0xFF) + { + if (JSON_HEDLEY_UNLIKELY(!get_cbor_string(key) || !sax->key(key))) + { + return false; + } + + if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(true, tag_handler))) + { + return false; + } + key.clear(); + } + } + } + + return sax->end_object(); + } + + ///////////// + // MsgPack // + ///////////// + + /*! + @return whether a valid MessagePack value was passed to the SAX parser + */ + bool parse_msgpack_internal() + { + switch (get()) + { + // EOF + case std::char_traits::eof(): + return unexpect_eof(input_format_t::msgpack, "value"); + + // positive fixint + case 0x00: + case 0x01: + case 0x02: + case 0x03: + case 0x04: + case 0x05: + case 0x06: + case 0x07: + case 0x08: + case 0x09: + case 0x0A: + case 0x0B: + case 0x0C: + case 0x0D: + case 0x0E: + case 0x0F: + case 0x10: + case 0x11: + case 0x12: + case 0x13: + case 0x14: + case 0x15: + case 0x16: + case 0x17: + case 0x18: + case 0x19: + case 0x1A: + case 0x1B: + case 0x1C: + case 0x1D: + case 0x1E: + case 0x1F: + case 0x20: + case 0x21: + case 0x22: + case 0x23: + case 0x24: + case 0x25: + case 0x26: + case 0x27: + case 0x28: + case 0x29: + case 0x2A: + case 0x2B: + case 0x2C: + case 0x2D: + case 0x2E: + case 0x2F: + case 0x30: + case 0x31: + case 0x32: + case 0x33: + case 0x34: + case 0x35: + case 0x36: + case 0x37: + case 0x38: + case 0x39: + case 0x3A: + case 0x3B: + case 0x3C: + case 0x3D: + case 0x3E: + case 0x3F: + case 0x40: + case 0x41: + case 0x42: + case 0x43: + case 0x44: + case 0x45: + case 0x46: + case 0x47: + case 0x48: + case 0x49: + case 0x4A: + case 0x4B: + case 0x4C: + case 0x4D: + case 0x4E: + case 0x4F: + case 0x50: + case 0x51: + case 0x52: + case 0x53: + case 0x54: + case 0x55: + case 0x56: + case 0x57: + case 0x58: + case 0x59: + case 0x5A: + case 0x5B: + case 0x5C: + case 0x5D: + case 0x5E: + case 0x5F: + case 0x60: + case 0x61: + case 0x62: + case 0x63: + case 0x64: + case 0x65: + case 0x66: + case 0x67: + case 0x68: + case 0x69: + case 0x6A: + case 0x6B: + case 0x6C: + case 0x6D: + case 0x6E: + case 0x6F: + case 0x70: + case 0x71: + case 0x72: + case 0x73: + case 0x74: + case 0x75: + case 0x76: + case 0x77: + case 0x78: + case 0x79: + case 0x7A: + case 0x7B: + case 0x7C: + case 0x7D: + case 0x7E: + case 0x7F: + return sax->number_unsigned(static_cast(current)); + + // fixmap + case 0x80: + case 0x81: + case 0x82: + case 0x83: + case 0x84: + case 0x85: + case 0x86: + case 0x87: + case 0x88: + case 0x89: + case 0x8A: + case 0x8B: + case 0x8C: + case 0x8D: + case 0x8E: + case 0x8F: + return get_msgpack_object(static_cast(static_cast(current) & 0x0Fu)); + + // fixarray + case 0x90: + case 0x91: + case 0x92: + case 0x93: + case 0x94: + case 0x95: + case 0x96: + case 0x97: + case 0x98: + case 0x99: + case 0x9A: + case 0x9B: + case 0x9C: + case 0x9D: + case 0x9E: + case 0x9F: + return get_msgpack_array(static_cast(static_cast(current) & 0x0Fu)); + + // fixstr + case 0xA0: + case 0xA1: + case 0xA2: + case 0xA3: + case 0xA4: + case 0xA5: + case 0xA6: + case 0xA7: + case 0xA8: + case 0xA9: + case 0xAA: + case 0xAB: + case 0xAC: + case 0xAD: + case 0xAE: + case 0xAF: + case 0xB0: + case 0xB1: + case 0xB2: + case 0xB3: + case 0xB4: + case 0xB5: + case 0xB6: + case 0xB7: + case 0xB8: + case 0xB9: + case 0xBA: + case 0xBB: + case 0xBC: + case 0xBD: + case 0xBE: + case 0xBF: + case 0xD9: // str 8 + case 0xDA: // str 16 + case 0xDB: // str 32 + { + string_t s; + return get_msgpack_string(s) && sax->string(s); + } + + case 0xC0: // nil + return sax->null(); + + case 0xC2: // false + return sax->boolean(false); + + case 0xC3: // true + return sax->boolean(true); + + case 0xC4: // bin 8 + case 0xC5: // bin 16 + case 0xC6: // bin 32 + case 0xC7: // ext 8 + case 0xC8: // ext 16 + case 0xC9: // ext 32 + case 0xD4: // fixext 1 + case 0xD5: // fixext 2 + case 0xD6: // fixext 4 + case 0xD7: // fixext 8 + case 0xD8: // fixext 16 + { + binary_t b; + return get_msgpack_binary(b) && sax->binary(b); + } + + case 0xCA: // float 32 + { + float number{}; + return get_number(input_format_t::msgpack, number) && sax->number_float(static_cast(number), ""); + } + + case 0xCB: // float 64 + { + double number{}; + return get_number(input_format_t::msgpack, number) && sax->number_float(static_cast(number), ""); + } + + case 0xCC: // uint 8 + { + std::uint8_t number{}; + return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number); + } + + case 0xCD: // uint 16 + { + std::uint16_t number{}; + return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number); + } + + case 0xCE: // uint 32 + { + std::uint32_t number{}; + return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number); + } + + case 0xCF: // uint 64 + { + std::uint64_t number{}; + return get_number(input_format_t::msgpack, number) && sax->number_unsigned(number); + } + + case 0xD0: // int 8 + { + std::int8_t number{}; + return get_number(input_format_t::msgpack, number) && sax->number_integer(number); + } + + case 0xD1: // int 16 + { + std::int16_t number{}; + return get_number(input_format_t::msgpack, number) && sax->number_integer(number); + } + + case 0xD2: // int 32 + { + std::int32_t number{}; + return get_number(input_format_t::msgpack, number) && sax->number_integer(number); + } + + case 0xD3: // int 64 + { + std::int64_t number{}; + return get_number(input_format_t::msgpack, number) && sax->number_integer(number); + } + + case 0xDC: // array 16 + { + std::uint16_t len{}; + return get_number(input_format_t::msgpack, len) && get_msgpack_array(static_cast(len)); + } + + case 0xDD: // array 32 + { + std::uint32_t len{}; + return get_number(input_format_t::msgpack, len) && get_msgpack_array(static_cast(len)); + } + + case 0xDE: // map 16 + { + std::uint16_t len{}; + return get_number(input_format_t::msgpack, len) && get_msgpack_object(static_cast(len)); + } + + case 0xDF: // map 32 + { + std::uint32_t len{}; + return get_number(input_format_t::msgpack, len) && get_msgpack_object(static_cast(len)); + } + + // negative fixint + case 0xE0: + case 0xE1: + case 0xE2: + case 0xE3: + case 0xE4: + case 0xE5: + case 0xE6: + case 0xE7: + case 0xE8: + case 0xE9: + case 0xEA: + case 0xEB: + case 0xEC: + case 0xED: + case 0xEE: + case 0xEF: + case 0xF0: + case 0xF1: + case 0xF2: + case 0xF3: + case 0xF4: + case 0xF5: + case 0xF6: + case 0xF7: + case 0xF8: + case 0xF9: + case 0xFA: + case 0xFB: + case 0xFC: + case 0xFD: + case 0xFE: + case 0xFF: + return sax->number_integer(static_cast(current)); + + default: // anything else + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::msgpack, "invalid byte: 0x" + last_token, "value"), BasicJsonType())); + } + } + } + + /*! + @brief reads a MessagePack string + + This function first reads starting bytes to determine the expected + string length and then copies this number of bytes into a string. + + @param[out] result created string + + @return whether string creation completed + */ + bool get_msgpack_string(string_t& result) + { + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::msgpack, "string"))) + { + return false; + } + + switch (current) + { + // fixstr + case 0xA0: + case 0xA1: + case 0xA2: + case 0xA3: + case 0xA4: + case 0xA5: + case 0xA6: + case 0xA7: + case 0xA8: + case 0xA9: + case 0xAA: + case 0xAB: + case 0xAC: + case 0xAD: + case 0xAE: + case 0xAF: + case 0xB0: + case 0xB1: + case 0xB2: + case 0xB3: + case 0xB4: + case 0xB5: + case 0xB6: + case 0xB7: + case 0xB8: + case 0xB9: + case 0xBA: + case 0xBB: + case 0xBC: + case 0xBD: + case 0xBE: + case 0xBF: + { + return get_string(input_format_t::msgpack, static_cast(current) & 0x1Fu, result); + } + + case 0xD9: // str 8 + { + std::uint8_t len{}; + return get_number(input_format_t::msgpack, len) && get_string(input_format_t::msgpack, len, result); + } + + case 0xDA: // str 16 + { + std::uint16_t len{}; + return get_number(input_format_t::msgpack, len) && get_string(input_format_t::msgpack, len, result); + } + + case 0xDB: // str 32 + { + std::uint32_t len{}; + return get_number(input_format_t::msgpack, len) && get_string(input_format_t::msgpack, len, result); + } + + default: + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::msgpack, "expected length specification (0xA0-0xBF, 0xD9-0xDB); last byte: 0x" + last_token, "string"), BasicJsonType())); + } + } + } + + /*! + @brief reads a MessagePack byte array + + This function first reads starting bytes to determine the expected + byte array length and then copies this number of bytes into a byte array. + + @param[out] result created byte array + + @return whether byte array creation completed + */ + bool get_msgpack_binary(binary_t& result) + { + // helper function to set the subtype + auto assign_and_return_true = [&result](std::int8_t subtype) + { + result.set_subtype(static_cast(subtype)); + return true; + }; + + switch (current) + { + case 0xC4: // bin 8 + { + std::uint8_t len{}; + return get_number(input_format_t::msgpack, len) && + get_binary(input_format_t::msgpack, len, result); + } + + case 0xC5: // bin 16 + { + std::uint16_t len{}; + return get_number(input_format_t::msgpack, len) && + get_binary(input_format_t::msgpack, len, result); + } + + case 0xC6: // bin 32 + { + std::uint32_t len{}; + return get_number(input_format_t::msgpack, len) && + get_binary(input_format_t::msgpack, len, result); + } + + case 0xC7: // ext 8 + { + std::uint8_t len{}; + std::int8_t subtype{}; + return get_number(input_format_t::msgpack, len) && + get_number(input_format_t::msgpack, subtype) && + get_binary(input_format_t::msgpack, len, result) && + assign_and_return_true(subtype); + } + + case 0xC8: // ext 16 + { + std::uint16_t len{}; + std::int8_t subtype{}; + return get_number(input_format_t::msgpack, len) && + get_number(input_format_t::msgpack, subtype) && + get_binary(input_format_t::msgpack, len, result) && + assign_and_return_true(subtype); + } + + case 0xC9: // ext 32 + { + std::uint32_t len{}; + std::int8_t subtype{}; + return get_number(input_format_t::msgpack, len) && + get_number(input_format_t::msgpack, subtype) && + get_binary(input_format_t::msgpack, len, result) && + assign_and_return_true(subtype); + } + + case 0xD4: // fixext 1 + { + std::int8_t subtype{}; + return get_number(input_format_t::msgpack, subtype) && + get_binary(input_format_t::msgpack, 1, result) && + assign_and_return_true(subtype); + } + + case 0xD5: // fixext 2 + { + std::int8_t subtype{}; + return get_number(input_format_t::msgpack, subtype) && + get_binary(input_format_t::msgpack, 2, result) && + assign_and_return_true(subtype); + } + + case 0xD6: // fixext 4 + { + std::int8_t subtype{}; + return get_number(input_format_t::msgpack, subtype) && + get_binary(input_format_t::msgpack, 4, result) && + assign_and_return_true(subtype); + } + + case 0xD7: // fixext 8 + { + std::int8_t subtype{}; + return get_number(input_format_t::msgpack, subtype) && + get_binary(input_format_t::msgpack, 8, result) && + assign_and_return_true(subtype); + } + + case 0xD8: // fixext 16 + { + std::int8_t subtype{}; + return get_number(input_format_t::msgpack, subtype) && + get_binary(input_format_t::msgpack, 16, result) && + assign_and_return_true(subtype); + } + + default: // LCOV_EXCL_LINE + return false; // LCOV_EXCL_LINE + } + } + + /*! + @param[in] len the length of the array + @return whether array creation completed + */ + bool get_msgpack_array(const std::size_t len) + { + if (JSON_HEDLEY_UNLIKELY(!sax->start_array(len))) + { + return false; + } + + for (std::size_t i = 0; i < len; ++i) + { + if (JSON_HEDLEY_UNLIKELY(!parse_msgpack_internal())) + { + return false; + } + } + + return sax->end_array(); + } + + /*! + @param[in] len the length of the object + @return whether object creation completed + */ + bool get_msgpack_object(const std::size_t len) + { + if (JSON_HEDLEY_UNLIKELY(!sax->start_object(len))) + { + return false; + } + + string_t key; + for (std::size_t i = 0; i < len; ++i) + { + get(); + if (JSON_HEDLEY_UNLIKELY(!get_msgpack_string(key) || !sax->key(key))) + { + return false; + } + + if (JSON_HEDLEY_UNLIKELY(!parse_msgpack_internal())) + { + return false; + } + key.clear(); + } + + return sax->end_object(); + } + + //////////// + // UBJSON // + //////////// + + /*! + @param[in] get_char whether a new character should be retrieved from the + input (true, default) or whether the last read + character should be considered instead + + @return whether a valid UBJSON value was passed to the SAX parser + */ + bool parse_ubjson_internal(const bool get_char = true) + { + return get_ubjson_value(get_char ? get_ignore_noop() : current); + } + + /*! + @brief reads a UBJSON string + + This function is either called after reading the 'S' byte explicitly + indicating a string, or in case of an object key where the 'S' byte can be + left out. + + @param[out] result created string + @param[in] get_char whether a new character should be retrieved from the + input (true, default) or whether the last read + character should be considered instead + + @return whether string creation completed + */ + bool get_ubjson_string(string_t& result, const bool get_char = true) + { + if (get_char) + { + get(); // TODO(niels): may we ignore N here? + } + + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::ubjson, "value"))) + { + return false; + } + + switch (current) + { + case 'U': + { + std::uint8_t len{}; + return get_number(input_format_t::ubjson, len) && get_string(input_format_t::ubjson, len, result); + } + + case 'i': + { + std::int8_t len{}; + return get_number(input_format_t::ubjson, len) && get_string(input_format_t::ubjson, len, result); + } + + case 'I': + { + std::int16_t len{}; + return get_number(input_format_t::ubjson, len) && get_string(input_format_t::ubjson, len, result); + } + + case 'l': + { + std::int32_t len{}; + return get_number(input_format_t::ubjson, len) && get_string(input_format_t::ubjson, len, result); + } + + case 'L': + { + std::int64_t len{}; + return get_number(input_format_t::ubjson, len) && get_string(input_format_t::ubjson, len, result); + } + + default: + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "expected length type specification (U, i, I, l, L); last byte: 0x" + last_token, "string"), BasicJsonType())); + } + } + + /*! + @param[out] result determined size + @return whether size determination completed + */ + bool get_ubjson_size_value(std::size_t& result) + { + switch (get_ignore_noop()) + { + case 'U': + { + std::uint8_t number{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::ubjson, number))) + { + return false; + } + result = static_cast(number); + return true; + } + + case 'i': + { + std::int8_t number{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::ubjson, number))) + { + return false; + } + result = static_cast(number); // NOLINT(bugprone-signed-char-misuse,cert-str34-c): number is not a char + return true; + } + + case 'I': + { + std::int16_t number{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::ubjson, number))) + { + return false; + } + result = static_cast(number); + return true; + } + + case 'l': + { + std::int32_t number{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::ubjson, number))) + { + return false; + } + result = static_cast(number); + return true; + } + + case 'L': + { + std::int64_t number{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::ubjson, number))) + { + return false; + } + result = static_cast(number); + return true; + } + + default: + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "expected length type specification (U, i, I, l, L) after '#'; last byte: 0x" + last_token, "size"), BasicJsonType())); + } + } + } + + /*! + @brief determine the type and size for a container + + In the optimized UBJSON format, a type and a size can be provided to allow + for a more compact representation. + + @param[out] result pair of the size and the type + + @return whether pair creation completed + */ + bool get_ubjson_size_type(std::pair& result) + { + result.first = string_t::npos; // size + result.second = 0; // type + + get_ignore_noop(); + + if (current == '$') + { + result.second = get(); // must not ignore 'N', because 'N' maybe the type + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::ubjson, "type"))) + { + return false; + } + + get_ignore_noop(); + if (JSON_HEDLEY_UNLIKELY(current != '#')) + { + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::ubjson, "value"))) + { + return false; + } + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::ubjson, "expected '#' after type information; last byte: 0x" + last_token, "size"), BasicJsonType())); + } + + return get_ubjson_size_value(result.first); + } + + if (current == '#') + { + return get_ubjson_size_value(result.first); + } + + return true; + } + + /*! + @param prefix the previously read or set type prefix + @return whether value creation completed + */ + bool get_ubjson_value(const char_int_type prefix) + { + switch (prefix) + { + case std::char_traits::eof(): // EOF + return unexpect_eof(input_format_t::ubjson, "value"); + + case 'T': // true + return sax->boolean(true); + case 'F': // false + return sax->boolean(false); + + case 'Z': // null + return sax->null(); + + case 'U': + { + std::uint8_t number{}; + return get_number(input_format_t::ubjson, number) && sax->number_unsigned(number); + } + + case 'i': + { + std::int8_t number{}; + return get_number(input_format_t::ubjson, number) && sax->number_integer(number); + } + + case 'I': + { + std::int16_t number{}; + return get_number(input_format_t::ubjson, number) && sax->number_integer(number); + } + + case 'l': + { + std::int32_t number{}; + return get_number(input_format_t::ubjson, number) && sax->number_integer(number); + } + + case 'L': + { + std::int64_t number{}; + return get_number(input_format_t::ubjson, number) && sax->number_integer(number); + } + + case 'd': + { + float number{}; + return get_number(input_format_t::ubjson, number) && sax->number_float(static_cast(number), ""); + } + + case 'D': + { + double number{}; + return get_number(input_format_t::ubjson, number) && sax->number_float(static_cast(number), ""); + } + + case 'H': + { + return get_ubjson_high_precision_number(); + } + + case 'C': // char + { + get(); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::ubjson, "char"))) + { + return false; + } + if (JSON_HEDLEY_UNLIKELY(current > 127)) + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "byte after 'C' must be in range 0x00..0x7F; last byte: 0x" + last_token, "char"), BasicJsonType())); + } + string_t s(1, static_cast(current)); + return sax->string(s); + } + + case 'S': // string + { + string_t s; + return get_ubjson_string(s) && sax->string(s); + } + + case '[': // array + return get_ubjson_array(); + + case '{': // object + return get_ubjson_object(); + + default: // anything else + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::ubjson, "invalid byte: 0x" + last_token, "value"), BasicJsonType())); + } + } + } + + /*! + @return whether array creation completed + */ + bool get_ubjson_array() + { + std::pair size_and_type; + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_type(size_and_type))) + { + return false; + } + + if (size_and_type.first != string_t::npos) + { + if (JSON_HEDLEY_UNLIKELY(!sax->start_array(size_and_type.first))) + { + return false; + } + + if (size_and_type.second != 0) + { + if (size_and_type.second != 'N') + { + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second))) + { + return false; + } + } + } + } + else + { + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal())) + { + return false; + } + } + } + } + else + { + if (JSON_HEDLEY_UNLIKELY(!sax->start_array(static_cast(-1)))) + { + return false; + } + + while (current != ']') + { + if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal(false))) + { + return false; + } + get_ignore_noop(); + } + } + + return sax->end_array(); + } + + /*! + @return whether object creation completed + */ + bool get_ubjson_object() + { + std::pair size_and_type; + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_type(size_and_type))) + { + return false; + } + + string_t key; + if (size_and_type.first != string_t::npos) + { + if (JSON_HEDLEY_UNLIKELY(!sax->start_object(size_and_type.first))) + { + return false; + } + + if (size_and_type.second != 0) + { + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key))) + { + return false; + } + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second))) + { + return false; + } + key.clear(); + } + } + else + { + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key))) + { + return false; + } + if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal())) + { + return false; + } + key.clear(); + } + } + } + else + { + if (JSON_HEDLEY_UNLIKELY(!sax->start_object(static_cast(-1)))) + { + return false; + } + + while (current != '}') + { + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key, false) || !sax->key(key))) + { + return false; + } + if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal())) + { + return false; + } + get_ignore_noop(); + key.clear(); + } + } + + return sax->end_object(); + } + + // Note, no reader for UBJSON binary types is implemented because they do + // not exist + + bool get_ubjson_high_precision_number() + { + // get size of following number string + std::size_t size{}; + auto res = get_ubjson_size_value(size); + if (JSON_HEDLEY_UNLIKELY(!res)) + { + return res; + } + + // get number string + std::vector number_vector; + for (std::size_t i = 0; i < size; ++i) + { + get(); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::ubjson, "number"))) + { + return false; + } + number_vector.push_back(static_cast(current)); + } + + // parse number string + using ia_type = decltype(detail::input_adapter(number_vector)); + auto number_lexer = detail::lexer(detail::input_adapter(number_vector), false); + const auto result_number = number_lexer.scan(); + const auto number_string = number_lexer.get_token_string(); + const auto result_remainder = number_lexer.scan(); + + using token_type = typename detail::lexer_base::token_type; + + if (JSON_HEDLEY_UNLIKELY(result_remainder != token_type::end_of_input)) + { + return sax->parse_error(chars_read, number_string, parse_error::create(115, chars_read, exception_message(input_format_t::ubjson, "invalid number text: " + number_lexer.get_token_string(), "high-precision number"), BasicJsonType())); + } + + switch (result_number) + { + case token_type::value_integer: + return sax->number_integer(number_lexer.get_number_integer()); + case token_type::value_unsigned: + return sax->number_unsigned(number_lexer.get_number_unsigned()); + case token_type::value_float: + return sax->number_float(number_lexer.get_number_float(), std::move(number_string)); + case token_type::uninitialized: + case token_type::literal_true: + case token_type::literal_false: + case token_type::literal_null: + case token_type::value_string: + case token_type::begin_array: + case token_type::begin_object: + case token_type::end_array: + case token_type::end_object: + case token_type::name_separator: + case token_type::value_separator: + case token_type::parse_error: + case token_type::end_of_input: + case token_type::literal_or_value: + default: + return sax->parse_error(chars_read, number_string, parse_error::create(115, chars_read, exception_message(input_format_t::ubjson, "invalid number text: " + number_lexer.get_token_string(), "high-precision number"), BasicJsonType())); + } + } + + /////////////////////// + // Utility functions // + /////////////////////// + + /*! + @brief get next character from the input + + This function provides the interface to the used input adapter. It does + not throw in case the input reached EOF, but returns a -'ve valued + `std::char_traits::eof()` in that case. + + @return character read from the input + */ + char_int_type get() + { + ++chars_read; + return current = ia.get_character(); + } + + /*! + @return character read from the input after ignoring all 'N' entries + */ + char_int_type get_ignore_noop() + { + do + { + get(); + } + while (current == 'N'); + + return current; + } + + /* + @brief read a number from the input + + @tparam NumberType the type of the number + @param[in] format the current format (for diagnostics) + @param[out] result number of type @a NumberType + + @return whether conversion completed + + @note This function needs to respect the system's endianness, because + bytes in CBOR, MessagePack, and UBJSON are stored in network order + (big endian) and therefore need reordering on little endian systems. + */ + template + bool get_number(const input_format_t format, NumberType& result) + { + // step 1: read input into array with system's byte order + std::array vec{}; + for (std::size_t i = 0; i < sizeof(NumberType); ++i) + { + get(); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(format, "number"))) + { + return false; + } + + // reverse byte order prior to conversion if necessary + if (is_little_endian != InputIsLittleEndian) + { + vec[sizeof(NumberType) - i - 1] = static_cast(current); + } + else + { + vec[i] = static_cast(current); // LCOV_EXCL_LINE + } + } + + // step 2: convert array into number of type T and return + std::memcpy(&result, vec.data(), sizeof(NumberType)); + return true; + } + + /*! + @brief create a string by reading characters from the input + + @tparam NumberType the type of the number + @param[in] format the current format (for diagnostics) + @param[in] len number of characters to read + @param[out] result string created by reading @a len bytes + + @return whether string creation completed + + @note We can not reserve @a len bytes for the result, because @a len + may be too large. Usually, @ref unexpect_eof() detects the end of + the input before we run out of string memory. + */ + template + bool get_string(const input_format_t format, + const NumberType len, + string_t& result) + { + bool success = true; + for (NumberType i = 0; i < len; i++) + { + get(); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(format, "string"))) + { + success = false; + break; + } + result.push_back(static_cast(current)); + } + return success; + } + + /*! + @brief create a byte array by reading bytes from the input + + @tparam NumberType the type of the number + @param[in] format the current format (for diagnostics) + @param[in] len number of bytes to read + @param[out] result byte array created by reading @a len bytes + + @return whether byte array creation completed + + @note We can not reserve @a len bytes for the result, because @a len + may be too large. Usually, @ref unexpect_eof() detects the end of + the input before we run out of memory. + */ + template + bool get_binary(const input_format_t format, + const NumberType len, + binary_t& result) + { + bool success = true; + for (NumberType i = 0; i < len; i++) + { + get(); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(format, "binary"))) + { + success = false; + break; + } + result.push_back(static_cast(current)); + } + return success; + } + + /*! + @param[in] format the current format (for diagnostics) + @param[in] context further context information (for diagnostics) + @return whether the last read character is not EOF + */ + JSON_HEDLEY_NON_NULL(3) + bool unexpect_eof(const input_format_t format, const char* context) const + { + if (JSON_HEDLEY_UNLIKELY(current == std::char_traits::eof())) + { + return sax->parse_error(chars_read, "", + parse_error::create(110, chars_read, exception_message(format, "unexpected end of input", context), BasicJsonType())); + } + return true; + } + + /*! + @return a string representation of the last read byte + */ + std::string get_token_string() const + { + std::array cr{{}}; + static_cast((std::snprintf)(cr.data(), cr.size(), "%.2hhX", static_cast(current))); // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg) + return std::string{cr.data()}; + } + + /*! + @param[in] format the current format + @param[in] detail a detailed error message + @param[in] context further context information + @return a message string to use in the parse_error exceptions + */ + std::string exception_message(const input_format_t format, + const std::string& detail, + const std::string& context) const + { + std::string error_msg = "syntax error while parsing "; + + switch (format) + { + case input_format_t::cbor: + error_msg += "CBOR"; + break; + + case input_format_t::msgpack: + error_msg += "MessagePack"; + break; + + case input_format_t::ubjson: + error_msg += "UBJSON"; + break; + + case input_format_t::bson: + error_msg += "BSON"; + break; + + case input_format_t::json: // LCOV_EXCL_LINE + default: // LCOV_EXCL_LINE + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE + } + + return error_msg + " " + context + ": " + detail; + } + + private: + /// input adapter + InputAdapterType ia; + + /// the current character + char_int_type current = std::char_traits::eof(); + + /// the number of characters read + std::size_t chars_read = 0; + + /// whether we can assume little endianness + const bool is_little_endian = little_endianness(); + + /// the SAX parser + json_sax_t* sax = nullptr; +}; +} // namespace detail +} // namespace nlohmann + +// #include + +// #include + +// #include + + +#include // isfinite +#include // uint8_t +#include // function +#include // string +#include // move +#include // vector + +// #include + +// #include + +// #include + +// #include + +// #include + +// #include + +// #include + + +namespace nlohmann +{ +namespace detail +{ +//////////// +// parser // +//////////// + +enum class parse_event_t : std::uint8_t +{ + /// the parser read `{` and started to process a JSON object + object_start, + /// the parser read `}` and finished processing a JSON object + object_end, + /// the parser read `[` and started to process a JSON array + array_start, + /// the parser read `]` and finished processing a JSON array + array_end, + /// the parser read a key of a value in an object + key, + /// the parser finished reading a JSON value + value +}; + +template +using parser_callback_t = + std::function; + +/*! +@brief syntax analysis + +This class implements a recursive descent parser. +*/ +template +class parser +{ + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using lexer_t = lexer; + using token_type = typename lexer_t::token_type; + + public: + /// a parser reading from an input adapter + explicit parser(InputAdapterType&& adapter, + const parser_callback_t cb = nullptr, + const bool allow_exceptions_ = true, + const bool skip_comments = false) + : callback(cb) + , m_lexer(std::move(adapter), skip_comments) + , allow_exceptions(allow_exceptions_) + { + // read first token + get_token(); + } + + /*! + @brief public parser interface + + @param[in] strict whether to expect the last token to be EOF + @param[in,out] result parsed JSON value + + @throw parse_error.101 in case of an unexpected token + @throw parse_error.102 if to_unicode fails or surrogate error + @throw parse_error.103 if to_unicode fails + */ + void parse(const bool strict, BasicJsonType& result) + { + if (callback) + { + json_sax_dom_callback_parser sdp(result, callback, allow_exceptions); + sax_parse_internal(&sdp); + + // in strict mode, input must be completely read + if (strict && (get_token() != token_type::end_of_input)) + { + sdp.parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::end_of_input, "value"), BasicJsonType())); + } + + // in case of an error, return discarded value + if (sdp.is_errored()) + { + result = value_t::discarded; + return; + } + + // set top-level value to null if it was discarded by the callback + // function + if (result.is_discarded()) + { + result = nullptr; + } + } + else + { + json_sax_dom_parser sdp(result, allow_exceptions); + sax_parse_internal(&sdp); + + // in strict mode, input must be completely read + if (strict && (get_token() != token_type::end_of_input)) + { + sdp.parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_of_input, "value"), BasicJsonType())); + } + + // in case of an error, return discarded value + if (sdp.is_errored()) + { + result = value_t::discarded; + return; + } + } + + result.assert_invariant(); + } + + /*! + @brief public accept interface + + @param[in] strict whether to expect the last token to be EOF + @return whether the input is a proper JSON text + */ + bool accept(const bool strict = true) + { + json_sax_acceptor sax_acceptor; + return sax_parse(&sax_acceptor, strict); + } + + template + JSON_HEDLEY_NON_NULL(2) + bool sax_parse(SAX* sax, const bool strict = true) + { + (void)detail::is_sax_static_asserts {}; + const bool result = sax_parse_internal(sax); + + // strict mode: next byte must be EOF + if (result && strict && (get_token() != token_type::end_of_input)) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_of_input, "value"), BasicJsonType())); + } + + return result; + } + + private: + template + JSON_HEDLEY_NON_NULL(2) + bool sax_parse_internal(SAX* sax) + { + // stack to remember the hierarchy of structured values we are parsing + // true = array; false = object + std::vector states; + // value to avoid a goto (see comment where set to true) + bool skip_to_state_evaluation = false; + + while (true) + { + if (!skip_to_state_evaluation) + { + // invariant: get_token() was called before each iteration + switch (last_token) + { + case token_type::begin_object: + { + if (JSON_HEDLEY_UNLIKELY(!sax->start_object(static_cast(-1)))) + { + return false; + } + + // closing } -> we are done + if (get_token() == token_type::end_object) + { + if (JSON_HEDLEY_UNLIKELY(!sax->end_object())) + { + return false; + } + break; + } + + // parse key + if (JSON_HEDLEY_UNLIKELY(last_token != token_type::value_string)) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::value_string, "object key"), BasicJsonType())); + } + if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string()))) + { + return false; + } + + // parse separator (:) + if (JSON_HEDLEY_UNLIKELY(get_token() != token_type::name_separator)) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::name_separator, "object separator"), BasicJsonType())); + } + + // remember we are now inside an object + states.push_back(false); + + // parse values + get_token(); + continue; + } + + case token_type::begin_array: + { + if (JSON_HEDLEY_UNLIKELY(!sax->start_array(static_cast(-1)))) + { + return false; + } + + // closing ] -> we are done + if (get_token() == token_type::end_array) + { + if (JSON_HEDLEY_UNLIKELY(!sax->end_array())) + { + return false; + } + break; + } + + // remember we are now inside an array + states.push_back(true); + + // parse values (no need to call get_token) + continue; + } + + case token_type::value_float: + { + const auto res = m_lexer.get_number_float(); + + if (JSON_HEDLEY_UNLIKELY(!std::isfinite(res))) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + out_of_range::create(406, "number overflow parsing '" + m_lexer.get_token_string() + "'", BasicJsonType())); + } + + if (JSON_HEDLEY_UNLIKELY(!sax->number_float(res, m_lexer.get_string()))) + { + return false; + } + + break; + } + + case token_type::literal_false: + { + if (JSON_HEDLEY_UNLIKELY(!sax->boolean(false))) + { + return false; + } + break; + } + + case token_type::literal_null: + { + if (JSON_HEDLEY_UNLIKELY(!sax->null())) + { + return false; + } + break; + } + + case token_type::literal_true: + { + if (JSON_HEDLEY_UNLIKELY(!sax->boolean(true))) + { + return false; + } + break; + } + + case token_type::value_integer: + { + if (JSON_HEDLEY_UNLIKELY(!sax->number_integer(m_lexer.get_number_integer()))) + { + return false; + } + break; + } + + case token_type::value_string: + { + if (JSON_HEDLEY_UNLIKELY(!sax->string(m_lexer.get_string()))) + { + return false; + } + break; + } + + case token_type::value_unsigned: + { + if (JSON_HEDLEY_UNLIKELY(!sax->number_unsigned(m_lexer.get_number_unsigned()))) + { + return false; + } + break; + } + + case token_type::parse_error: + { + // using "uninitialized" to avoid "expected" message + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::uninitialized, "value"), BasicJsonType())); + } + + case token_type::uninitialized: + case token_type::end_array: + case token_type::end_object: + case token_type::name_separator: + case token_type::value_separator: + case token_type::end_of_input: + case token_type::literal_or_value: + default: // the last token was unexpected + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::literal_or_value, "value"), BasicJsonType())); + } + } + } + else + { + skip_to_state_evaluation = false; + } + + // we reached this line after we successfully parsed a value + if (states.empty()) + { + // empty stack: we reached the end of the hierarchy: done + return true; + } + + if (states.back()) // array + { + // comma -> next value + if (get_token() == token_type::value_separator) + { + // parse a new value + get_token(); + continue; + } + + // closing ] + if (JSON_HEDLEY_LIKELY(last_token == token_type::end_array)) + { + if (JSON_HEDLEY_UNLIKELY(!sax->end_array())) + { + return false; + } + + // We are done with this array. Before we can parse a + // new value, we need to evaluate the new state first. + // By setting skip_to_state_evaluation to false, we + // are effectively jumping to the beginning of this if. + JSON_ASSERT(!states.empty()); + states.pop_back(); + skip_to_state_evaluation = true; + continue; + } + + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_array, "array"), BasicJsonType())); + } + + // states.back() is false -> object + + // comma -> next value + if (get_token() == token_type::value_separator) + { + // parse key + if (JSON_HEDLEY_UNLIKELY(get_token() != token_type::value_string)) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::value_string, "object key"), BasicJsonType())); + } + + if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string()))) + { + return false; + } + + // parse separator (:) + if (JSON_HEDLEY_UNLIKELY(get_token() != token_type::name_separator)) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::name_separator, "object separator"), BasicJsonType())); + } + + // parse values + get_token(); + continue; + } + + // closing } + if (JSON_HEDLEY_LIKELY(last_token == token_type::end_object)) + { + if (JSON_HEDLEY_UNLIKELY(!sax->end_object())) + { + return false; + } + + // We are done with this object. Before we can parse a + // new value, we need to evaluate the new state first. + // By setting skip_to_state_evaluation to false, we + // are effectively jumping to the beginning of this if. + JSON_ASSERT(!states.empty()); + states.pop_back(); + skip_to_state_evaluation = true; + continue; + } + + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_object, "object"), BasicJsonType())); + } + } + + /// get next token from lexer + token_type get_token() + { + return last_token = m_lexer.scan(); + } + + std::string exception_message(const token_type expected, const std::string& context) + { + std::string error_msg = "syntax error "; + + if (!context.empty()) + { + error_msg += "while parsing " + context + " "; + } + + error_msg += "- "; + + if (last_token == token_type::parse_error) + { + error_msg += std::string(m_lexer.get_error_message()) + "; last read: '" + + m_lexer.get_token_string() + "'"; + } + else + { + error_msg += "unexpected " + std::string(lexer_t::token_type_name(last_token)); + } + + if (expected != token_type::uninitialized) + { + error_msg += "; expected " + std::string(lexer_t::token_type_name(expected)); + } + + return error_msg; + } + + private: + /// callback function + const parser_callback_t callback = nullptr; + /// the type of the last read token + token_type last_token = token_type::uninitialized; + /// the lexer + lexer_t m_lexer; + /// whether to throw exceptions in case of errors + const bool allow_exceptions = true; +}; + +} // namespace detail +} // namespace nlohmann + +// #include + + +// #include + + +#include // ptrdiff_t +#include // numeric_limits + +// #include + + +namespace nlohmann +{ +namespace detail +{ +/* +@brief an iterator for primitive JSON types + +This class models an iterator for primitive JSON types (boolean, number, +string). It's only purpose is to allow the iterator/const_iterator classes +to "iterate" over primitive values. Internally, the iterator is modeled by +a `difference_type` variable. Value begin_value (`0`) models the begin, +end_value (`1`) models past the end. +*/ +class primitive_iterator_t +{ + private: + using difference_type = std::ptrdiff_t; + static constexpr difference_type begin_value = 0; + static constexpr difference_type end_value = begin_value + 1; + + JSON_PRIVATE_UNLESS_TESTED: + /// iterator as signed integer type + difference_type m_it = (std::numeric_limits::min)(); + + public: + constexpr difference_type get_value() const noexcept + { + return m_it; + } + + /// set iterator to a defined beginning + void set_begin() noexcept + { + m_it = begin_value; + } + + /// set iterator to a defined past the end + void set_end() noexcept + { + m_it = end_value; + } + + /// return whether the iterator can be dereferenced + constexpr bool is_begin() const noexcept + { + return m_it == begin_value; + } + + /// return whether the iterator is at end + constexpr bool is_end() const noexcept + { + return m_it == end_value; + } + + friend constexpr bool operator==(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept + { + return lhs.m_it == rhs.m_it; + } + + friend constexpr bool operator<(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept + { + return lhs.m_it < rhs.m_it; + } + + primitive_iterator_t operator+(difference_type n) noexcept + { + auto result = *this; + result += n; + return result; + } + + friend constexpr difference_type operator-(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept + { + return lhs.m_it - rhs.m_it; + } + + primitive_iterator_t& operator++() noexcept + { + ++m_it; + return *this; + } + + primitive_iterator_t const operator++(int) noexcept // NOLINT(readability-const-return-type) + { + auto result = *this; + ++m_it; + return result; + } + + primitive_iterator_t& operator--() noexcept + { + --m_it; + return *this; + } + + primitive_iterator_t const operator--(int) noexcept // NOLINT(readability-const-return-type) + { + auto result = *this; + --m_it; + return result; + } + + primitive_iterator_t& operator+=(difference_type n) noexcept + { + m_it += n; + return *this; + } + + primitive_iterator_t& operator-=(difference_type n) noexcept + { + m_it -= n; + return *this; + } +}; +} // namespace detail +} // namespace nlohmann + + +namespace nlohmann +{ +namespace detail +{ +/*! +@brief an iterator value + +@note This structure could easily be a union, but MSVC currently does not allow +unions members with complex constructors, see https://github.com/nlohmann/json/pull/105. +*/ +template struct internal_iterator +{ + /// iterator for JSON objects + typename BasicJsonType::object_t::iterator object_iterator {}; + /// iterator for JSON arrays + typename BasicJsonType::array_t::iterator array_iterator {}; + /// generic iterator for all other types + primitive_iterator_t primitive_iterator {}; +}; +} // namespace detail +} // namespace nlohmann + +// #include + + +#include // iterator, random_access_iterator_tag, bidirectional_iterator_tag, advance, next +#include // conditional, is_const, remove_const + +// #include + +// #include + +// #include + +// #include + +// #include + +// #include + +// #include + + +namespace nlohmann +{ +namespace detail +{ +// forward declare, to be able to friend it later on +template class iteration_proxy; +template class iteration_proxy_value; + +/*! +@brief a template for a bidirectional iterator for the @ref basic_json class +This class implements a both iterators (iterator and const_iterator) for the +@ref basic_json class. +@note An iterator is called *initialized* when a pointer to a JSON value has + been set (e.g., by a constructor or a copy assignment). If the iterator is + default-constructed, it is *uninitialized* and most methods are undefined. + **The library uses assertions to detect calls on uninitialized iterators.** +@requirement The class satisfies the following concept requirements: +- +[BidirectionalIterator](https://en.cppreference.com/w/cpp/named_req/BidirectionalIterator): + The iterator that can be moved can be moved in both directions (i.e. + incremented and decremented). +@since version 1.0.0, simplified in version 2.0.9, change to bidirectional + iterators in version 3.0.0 (see https://github.com/nlohmann/json/issues/593) +*/ +template +class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-special-member-functions) +{ + /// the iterator with BasicJsonType of different const-ness + using other_iter_impl = iter_impl::value, typename std::remove_const::type, const BasicJsonType>::type>; + /// allow basic_json to access private members + friend other_iter_impl; + friend BasicJsonType; + friend iteration_proxy; + friend iteration_proxy_value; + + using object_t = typename BasicJsonType::object_t; + using array_t = typename BasicJsonType::array_t; + // make sure BasicJsonType is basic_json or const basic_json + static_assert(is_basic_json::type>::value, + "iter_impl only accepts (const) basic_json"); + + public: + + /// The std::iterator class template (used as a base class to provide typedefs) is deprecated in C++17. + /// The C++ Standard has never required user-defined iterators to derive from std::iterator. + /// A user-defined iterator should provide publicly accessible typedefs named + /// iterator_category, value_type, difference_type, pointer, and reference. + /// Note that value_type is required to be non-const, even for constant iterators. + using iterator_category = std::bidirectional_iterator_tag; + + /// the type of the values when the iterator is dereferenced + using value_type = typename BasicJsonType::value_type; + /// a type to represent differences between iterators + using difference_type = typename BasicJsonType::difference_type; + /// defines a pointer to the type iterated over (value_type) + using pointer = typename std::conditional::value, + typename BasicJsonType::const_pointer, + typename BasicJsonType::pointer>::type; + /// defines a reference to the type iterated over (value_type) + using reference = + typename std::conditional::value, + typename BasicJsonType::const_reference, + typename BasicJsonType::reference>::type; + + iter_impl() = default; + ~iter_impl() = default; + iter_impl(iter_impl&&) noexcept = default; + iter_impl& operator=(iter_impl&&) noexcept = default; + + /*! + @brief constructor for a given JSON instance + @param[in] object pointer to a JSON object for this iterator + @pre object != nullptr + @post The iterator is initialized; i.e. `m_object != nullptr`. + */ + explicit iter_impl(pointer object) noexcept : m_object(object) + { + JSON_ASSERT(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + { + m_it.object_iterator = typename object_t::iterator(); + break; + } + + case value_t::array: + { + m_it.array_iterator = typename array_t::iterator(); + break; + } + + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + m_it.primitive_iterator = primitive_iterator_t(); + break; + } + } + } + + /*! + @note The conventional copy constructor and copy assignment are implicitly + defined. Combined with the following converting constructor and + assignment, they support: (1) copy from iterator to iterator, (2) + copy from const iterator to const iterator, and (3) conversion from + iterator to const iterator. However conversion from const iterator + to iterator is not defined. + */ + + /*! + @brief const copy constructor + @param[in] other const iterator to copy from + @note This copy constructor had to be defined explicitly to circumvent a bug + occurring on msvc v19.0 compiler (VS 2015) debug build. For more + information refer to: https://github.com/nlohmann/json/issues/1608 + */ + iter_impl(const iter_impl& other) noexcept + : m_object(other.m_object), m_it(other.m_it) + {} + + /*! + @brief converting assignment + @param[in] other const iterator to copy from + @return const/non-const iterator + @note It is not checked whether @a other is initialized. + */ + iter_impl& operator=(const iter_impl& other) noexcept + { + if (&other != this) + { + m_object = other.m_object; + m_it = other.m_it; + } + return *this; + } + + /*! + @brief converting constructor + @param[in] other non-const iterator to copy from + @note It is not checked whether @a other is initialized. + */ + iter_impl(const iter_impl::type>& other) noexcept + : m_object(other.m_object), m_it(other.m_it) + {} + + /*! + @brief converting assignment + @param[in] other non-const iterator to copy from + @return const/non-const iterator + @note It is not checked whether @a other is initialized. + */ + iter_impl& operator=(const iter_impl::type>& other) noexcept // NOLINT(cert-oop54-cpp) + { + m_object = other.m_object; + m_it = other.m_it; + return *this; + } + + JSON_PRIVATE_UNLESS_TESTED: + /*! + @brief set the iterator to the first value + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + void set_begin() noexcept + { + JSON_ASSERT(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + { + m_it.object_iterator = m_object->m_value.object->begin(); + break; + } + + case value_t::array: + { + m_it.array_iterator = m_object->m_value.array->begin(); + break; + } + + case value_t::null: + { + // set to end so begin()==end() is true: null is empty + m_it.primitive_iterator.set_end(); + break; + } + + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + m_it.primitive_iterator.set_begin(); + break; + } + } + } + + /*! + @brief set the iterator past the last value + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + void set_end() noexcept + { + JSON_ASSERT(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + { + m_it.object_iterator = m_object->m_value.object->end(); + break; + } + + case value_t::array: + { + m_it.array_iterator = m_object->m_value.array->end(); + break; + } + + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + m_it.primitive_iterator.set_end(); + break; + } + } + } + + public: + /*! + @brief return a reference to the value pointed to by the iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + reference operator*() const + { + JSON_ASSERT(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + { + JSON_ASSERT(m_it.object_iterator != m_object->m_value.object->end()); + return m_it.object_iterator->second; + } + + case value_t::array: + { + JSON_ASSERT(m_it.array_iterator != m_object->m_value.array->end()); + return *m_it.array_iterator; + } + + case value_t::null: + JSON_THROW(invalid_iterator::create(214, "cannot get value", *m_object)); + + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + if (JSON_HEDLEY_LIKELY(m_it.primitive_iterator.is_begin())) + { + return *m_object; + } + + JSON_THROW(invalid_iterator::create(214, "cannot get value", *m_object)); + } + } + } + + /*! + @brief dereference the iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + pointer operator->() const + { + JSON_ASSERT(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + { + JSON_ASSERT(m_it.object_iterator != m_object->m_value.object->end()); + return &(m_it.object_iterator->second); + } + + case value_t::array: + { + JSON_ASSERT(m_it.array_iterator != m_object->m_value.array->end()); + return &*m_it.array_iterator; + } + + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + if (JSON_HEDLEY_LIKELY(m_it.primitive_iterator.is_begin())) + { + return m_object; + } + + JSON_THROW(invalid_iterator::create(214, "cannot get value", *m_object)); + } + } + } + + /*! + @brief post-increment (it++) + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl const operator++(int) // NOLINT(readability-const-return-type) + { + auto result = *this; + ++(*this); + return result; + } + + /*! + @brief pre-increment (++it) + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl& operator++() + { + JSON_ASSERT(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + { + std::advance(m_it.object_iterator, 1); + break; + } + + case value_t::array: + { + std::advance(m_it.array_iterator, 1); + break; + } + + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + ++m_it.primitive_iterator; + break; + } + } + + return *this; + } + + /*! + @brief post-decrement (it--) + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl const operator--(int) // NOLINT(readability-const-return-type) + { + auto result = *this; + --(*this); + return result; + } + + /*! + @brief pre-decrement (--it) + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl& operator--() + { + JSON_ASSERT(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + { + std::advance(m_it.object_iterator, -1); + break; + } + + case value_t::array: + { + std::advance(m_it.array_iterator, -1); + break; + } + + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + --m_it.primitive_iterator; + break; + } + } + + return *this; + } + + /*! + @brief comparison: equal + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + template < typename IterImpl, detail::enable_if_t < (std::is_same::value || std::is_same::value), std::nullptr_t > = nullptr > + bool operator==(const IterImpl& other) const + { + // if objects are not the same, the comparison is undefined + if (JSON_HEDLEY_UNLIKELY(m_object != other.m_object)) + { + JSON_THROW(invalid_iterator::create(212, "cannot compare iterators of different containers", *m_object)); + } + + JSON_ASSERT(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + return (m_it.object_iterator == other.m_it.object_iterator); + + case value_t::array: + return (m_it.array_iterator == other.m_it.array_iterator); + + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + return (m_it.primitive_iterator == other.m_it.primitive_iterator); + } + } + + /*! + @brief comparison: not equal + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + template < typename IterImpl, detail::enable_if_t < (std::is_same::value || std::is_same::value), std::nullptr_t > = nullptr > + bool operator!=(const IterImpl& other) const + { + return !operator==(other); + } + + /*! + @brief comparison: smaller + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator<(const iter_impl& other) const + { + // if objects are not the same, the comparison is undefined + if (JSON_HEDLEY_UNLIKELY(m_object != other.m_object)) + { + JSON_THROW(invalid_iterator::create(212, "cannot compare iterators of different containers", *m_object)); + } + + JSON_ASSERT(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + JSON_THROW(invalid_iterator::create(213, "cannot compare order of object iterators", *m_object)); + + case value_t::array: + return (m_it.array_iterator < other.m_it.array_iterator); + + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + return (m_it.primitive_iterator < other.m_it.primitive_iterator); + } + } + + /*! + @brief comparison: less than or equal + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator<=(const iter_impl& other) const + { + return !other.operator < (*this); + } + + /*! + @brief comparison: greater than + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator>(const iter_impl& other) const + { + return !operator<=(other); + } + + /*! + @brief comparison: greater than or equal + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator>=(const iter_impl& other) const + { + return !operator<(other); + } + + /*! + @brief add to iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl& operator+=(difference_type i) + { + JSON_ASSERT(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + JSON_THROW(invalid_iterator::create(209, "cannot use offsets with object iterators", *m_object)); + + case value_t::array: + { + std::advance(m_it.array_iterator, i); + break; + } + + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + m_it.primitive_iterator += i; + break; + } + } + + return *this; + } + + /*! + @brief subtract from iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl& operator-=(difference_type i) + { + return operator+=(-i); + } + + /*! + @brief add to iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl operator+(difference_type i) const + { + auto result = *this; + result += i; + return result; + } + + /*! + @brief addition of distance and iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + friend iter_impl operator+(difference_type i, const iter_impl& it) + { + auto result = it; + result += i; + return result; + } + + /*! + @brief subtract from iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl operator-(difference_type i) const + { + auto result = *this; + result -= i; + return result; + } + + /*! + @brief return difference + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + difference_type operator-(const iter_impl& other) const + { + JSON_ASSERT(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + JSON_THROW(invalid_iterator::create(209, "cannot use offsets with object iterators", *m_object)); + + case value_t::array: + return m_it.array_iterator - other.m_it.array_iterator; + + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + return m_it.primitive_iterator - other.m_it.primitive_iterator; + } + } + + /*! + @brief access to successor + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + reference operator[](difference_type n) const + { + JSON_ASSERT(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + JSON_THROW(invalid_iterator::create(208, "cannot use operator[] for object iterators", *m_object)); + + case value_t::array: + return *std::next(m_it.array_iterator, n); + + case value_t::null: + JSON_THROW(invalid_iterator::create(214, "cannot get value", *m_object)); + + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + if (JSON_HEDLEY_LIKELY(m_it.primitive_iterator.get_value() == -n)) + { + return *m_object; + } + + JSON_THROW(invalid_iterator::create(214, "cannot get value", *m_object)); + } + } + } + + /*! + @brief return the key of an object iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + const typename object_t::key_type& key() const + { + JSON_ASSERT(m_object != nullptr); + + if (JSON_HEDLEY_LIKELY(m_object->is_object())) + { + return m_it.object_iterator->first; + } + + JSON_THROW(invalid_iterator::create(207, "cannot use key() for non-object iterators", *m_object)); + } + + /*! + @brief return the value of an iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + reference value() const + { + return operator*(); + } + + JSON_PRIVATE_UNLESS_TESTED: + /// associated JSON instance + pointer m_object = nullptr; + /// the actual iterator of the associated instance + internal_iterator::type> m_it {}; +}; +} // namespace detail +} // namespace nlohmann + +// #include + +// #include + + +#include // ptrdiff_t +#include // reverse_iterator +#include // declval + +namespace nlohmann +{ +namespace detail +{ +////////////////////// +// reverse_iterator // +////////////////////// + +/*! +@brief a template for a reverse iterator class + +@tparam Base the base iterator type to reverse. Valid types are @ref +iterator (to create @ref reverse_iterator) and @ref const_iterator (to +create @ref const_reverse_iterator). + +@requirement The class satisfies the following concept requirements: +- +[BidirectionalIterator](https://en.cppreference.com/w/cpp/named_req/BidirectionalIterator): + The iterator that can be moved can be moved in both directions (i.e. + incremented and decremented). +- [OutputIterator](https://en.cppreference.com/w/cpp/named_req/OutputIterator): + It is possible to write to the pointed-to element (only if @a Base is + @ref iterator). + +@since version 1.0.0 +*/ +template +class json_reverse_iterator : public std::reverse_iterator +{ + public: + using difference_type = std::ptrdiff_t; + /// shortcut to the reverse iterator adapter + using base_iterator = std::reverse_iterator; + /// the reference type for the pointed-to element + using reference = typename Base::reference; + + /// create reverse iterator from iterator + explicit json_reverse_iterator(const typename base_iterator::iterator_type& it) noexcept + : base_iterator(it) {} + + /// create reverse iterator from base class + explicit json_reverse_iterator(const base_iterator& it) noexcept : base_iterator(it) {} + + /// post-increment (it++) + json_reverse_iterator const operator++(int) // NOLINT(readability-const-return-type) + { + return static_cast(base_iterator::operator++(1)); + } + + /// pre-increment (++it) + json_reverse_iterator& operator++() + { + return static_cast(base_iterator::operator++()); + } + + /// post-decrement (it--) + json_reverse_iterator const operator--(int) // NOLINT(readability-const-return-type) + { + return static_cast(base_iterator::operator--(1)); + } + + /// pre-decrement (--it) + json_reverse_iterator& operator--() + { + return static_cast(base_iterator::operator--()); + } + + /// add to iterator + json_reverse_iterator& operator+=(difference_type i) + { + return static_cast(base_iterator::operator+=(i)); + } + + /// add to iterator + json_reverse_iterator operator+(difference_type i) const + { + return static_cast(base_iterator::operator+(i)); + } + + /// subtract from iterator + json_reverse_iterator operator-(difference_type i) const + { + return static_cast(base_iterator::operator-(i)); + } + + /// return difference + difference_type operator-(const json_reverse_iterator& other) const + { + return base_iterator(*this) - base_iterator(other); + } + + /// access to successor + reference operator[](difference_type n) const + { + return *(this->operator+(n)); + } + + /// return the key of an object iterator + auto key() const -> decltype(std::declval().key()) + { + auto it = --this->base(); + return it.key(); + } + + /// return the value of an iterator + reference value() const + { + auto it = --this->base(); + return it.operator * (); + } +}; +} // namespace detail +} // namespace nlohmann + +// #include + +// #include + + +#include // all_of +#include // isdigit +#include // max +#include // accumulate +#include // string +#include // move +#include // vector + +// #include + +// #include + +// #include + +// #include + + +namespace nlohmann +{ + +/// @brief JSON Pointer defines a string syntax for identifying a specific value within a JSON document +/// @sa https://json.nlohmann.me/api/json_pointer/ +template +class json_pointer +{ + // allow basic_json to access private members + NLOHMANN_BASIC_JSON_TPL_DECLARATION + friend class basic_json; + + public: + /// @brief create JSON pointer + /// @sa https://json.nlohmann.me/api/json_pointer/json_pointer/ + explicit json_pointer(const std::string& s = "") + : reference_tokens(split(s)) + {} + + /// @brief return a string representation of the JSON pointer + /// @sa https://json.nlohmann.me/api/json_pointer/to_string/ + std::string to_string() const + { + return std::accumulate(reference_tokens.begin(), reference_tokens.end(), + std::string{}, + [](const std::string & a, const std::string & b) + { + return a + "/" + detail::escape(b); + }); + } + + /// @brief return a string representation of the JSON pointer + /// @sa https://json.nlohmann.me/api/json_pointer/operator_string/ + operator std::string() const + { + return to_string(); + } + + /// @brief append another JSON pointer at the end of this JSON pointer + /// @sa https://json.nlohmann.me/api/json_pointer/operator_slasheq/ + json_pointer& operator/=(const json_pointer& ptr) + { + reference_tokens.insert(reference_tokens.end(), + ptr.reference_tokens.begin(), + ptr.reference_tokens.end()); + return *this; + } + + /// @brief append an unescaped reference token at the end of this JSON pointer + /// @sa https://json.nlohmann.me/api/json_pointer/operator_slasheq/ + json_pointer& operator/=(std::string token) + { + push_back(std::move(token)); + return *this; + } + + /// @brief append an array index at the end of this JSON pointer + /// @sa https://json.nlohmann.me/api/json_pointer/operator_slasheq/ + json_pointer& operator/=(std::size_t array_idx) + { + return *this /= std::to_string(array_idx); + } + + /// @brief create a new JSON pointer by appending the right JSON pointer at the end of the left JSON pointer + /// @sa https://json.nlohmann.me/api/json_pointer/operator_slash/ + friend json_pointer operator/(const json_pointer& lhs, + const json_pointer& rhs) + { + return json_pointer(lhs) /= rhs; + } + + /// @brief create a new JSON pointer by appending the unescaped token at the end of the JSON pointer + /// @sa https://json.nlohmann.me/api/json_pointer/operator_slash/ + friend json_pointer operator/(const json_pointer& lhs, std::string token) // NOLINT(performance-unnecessary-value-param) + { + return json_pointer(lhs) /= std::move(token); + } + + /// @brief create a new JSON pointer by appending the array-index-token at the end of the JSON pointer + /// @sa https://json.nlohmann.me/api/json_pointer/operator_slash/ + friend json_pointer operator/(const json_pointer& lhs, std::size_t array_idx) + { + return json_pointer(lhs) /= array_idx; + } + + /// @brief returns the parent of this JSON pointer + /// @sa https://json.nlohmann.me/api/json_pointer/parent_pointer/ + json_pointer parent_pointer() const + { + if (empty()) + { + return *this; + } + + json_pointer res = *this; + res.pop_back(); + return res; + } + + /// @brief remove last reference token + /// @sa https://json.nlohmann.me/api/json_pointer/pop_back/ + void pop_back() + { + if (JSON_HEDLEY_UNLIKELY(empty())) + { + JSON_THROW(detail::out_of_range::create(405, "JSON pointer has no parent", BasicJsonType())); + } + + reference_tokens.pop_back(); + } + + /// @brief return last reference token + /// @sa https://json.nlohmann.me/api/json_pointer/back/ + const std::string& back() const + { + if (JSON_HEDLEY_UNLIKELY(empty())) + { + JSON_THROW(detail::out_of_range::create(405, "JSON pointer has no parent", BasicJsonType())); + } + + return reference_tokens.back(); + } + + /// @brief append an unescaped token at the end of the reference pointer + /// @sa https://json.nlohmann.me/api/json_pointer/push_back/ + void push_back(const std::string& token) + { + reference_tokens.push_back(token); + } + + /// @brief append an unescaped token at the end of the reference pointer + /// @sa https://json.nlohmann.me/api/json_pointer/push_back/ + void push_back(std::string&& token) + { + reference_tokens.push_back(std::move(token)); + } + + /// @brief return whether pointer points to the root document + /// @sa https://json.nlohmann.me/api/json_pointer/empty/ + bool empty() const noexcept + { + return reference_tokens.empty(); + } + + private: + /*! + @param[in] s reference token to be converted into an array index + + @return integer representation of @a s + + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index begins not with a digit + @throw out_of_range.404 if string @a s could not be converted to an integer + @throw out_of_range.410 if an array index exceeds size_type + */ + static typename BasicJsonType::size_type array_index(const std::string& s) + { + using size_type = typename BasicJsonType::size_type; + + // error condition (cf. RFC 6901, Sect. 4) + if (JSON_HEDLEY_UNLIKELY(s.size() > 1 && s[0] == '0')) + { + JSON_THROW(detail::parse_error::create(106, 0, "array index '" + s + "' must not begin with '0'", BasicJsonType())); + } + + // error condition (cf. RFC 6901, Sect. 4) + if (JSON_HEDLEY_UNLIKELY(s.size() > 1 && !(s[0] >= '1' && s[0] <= '9'))) + { + JSON_THROW(detail::parse_error::create(109, 0, "array index '" + s + "' is not a number", BasicJsonType())); + } + + std::size_t processed_chars = 0; + unsigned long long res = 0; // NOLINT(runtime/int) + JSON_TRY + { + res = std::stoull(s, &processed_chars); + } + JSON_CATCH(std::out_of_range&) + { + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + s + "'", BasicJsonType())); + } + + // check if the string was completely read + if (JSON_HEDLEY_UNLIKELY(processed_chars != s.size())) + { + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + s + "'", BasicJsonType())); + } + + // only triggered on special platforms (like 32bit), see also + // https://github.com/nlohmann/json/pull/2203 + if (res >= static_cast((std::numeric_limits::max)())) // NOLINT(runtime/int) + { + JSON_THROW(detail::out_of_range::create(410, "array index " + s + " exceeds size_type", BasicJsonType())); // LCOV_EXCL_LINE + } + + return static_cast(res); + } + + JSON_PRIVATE_UNLESS_TESTED: + json_pointer top() const + { + if (JSON_HEDLEY_UNLIKELY(empty())) + { + JSON_THROW(detail::out_of_range::create(405, "JSON pointer has no parent", BasicJsonType())); + } + + json_pointer result = *this; + result.reference_tokens = {reference_tokens[0]}; + return result; + } + + private: + /*! + @brief create and return a reference to the pointed to value + + @complexity Linear in the number of reference tokens. + + @throw parse_error.109 if array index is not a number + @throw type_error.313 if value cannot be unflattened + */ + BasicJsonType& get_and_create(BasicJsonType& j) const + { + auto* result = &j; + + // in case no reference tokens exist, return a reference to the JSON value + // j which will be overwritten by a primitive value + for (const auto& reference_token : reference_tokens) + { + switch (result->type()) + { + case detail::value_t::null: + { + if (reference_token == "0") + { + // start a new array if reference token is 0 + result = &result->operator[](0); + } + else + { + // start a new object otherwise + result = &result->operator[](reference_token); + } + break; + } + + case detail::value_t::object: + { + // create an entry in the object + result = &result->operator[](reference_token); + break; + } + + case detail::value_t::array: + { + // create an entry in the array + result = &result->operator[](array_index(reference_token)); + break; + } + + /* + The following code is only reached if there exists a reference + token _and_ the current value is primitive. In this case, we have + an error situation, because primitive values may only occur as + single value; that is, with an empty list of reference tokens. + */ + case detail::value_t::string: + case detail::value_t::boolean: + case detail::value_t::number_integer: + case detail::value_t::number_unsigned: + case detail::value_t::number_float: + case detail::value_t::binary: + case detail::value_t::discarded: + default: + JSON_THROW(detail::type_error::create(313, "invalid value to unflatten", j)); + } + } + + return *result; + } + + /*! + @brief return a reference to the pointed to value + + @note This version does not throw if a value is not present, but tries to + create nested values instead. For instance, calling this function + with pointer `"/this/that"` on a null value is equivalent to calling + `operator[]("this").operator[]("that")` on that value, effectively + changing the null value to an object. + + @param[in] ptr a JSON value + + @return reference to the JSON value pointed to by the JSON pointer + + @complexity Linear in the length of the JSON pointer. + + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.404 if the JSON pointer can not be resolved + */ + BasicJsonType& get_unchecked(BasicJsonType* ptr) const + { + for (const auto& reference_token : reference_tokens) + { + // convert null values to arrays or objects before continuing + if (ptr->is_null()) + { + // check if reference token is a number + const bool nums = + std::all_of(reference_token.begin(), reference_token.end(), + [](const unsigned char x) + { + return std::isdigit(x); + }); + + // change value to array for numbers or "-" or to object otherwise + *ptr = (nums || reference_token == "-") + ? detail::value_t::array + : detail::value_t::object; + } + + switch (ptr->type()) + { + case detail::value_t::object: + { + // use unchecked object access + ptr = &ptr->operator[](reference_token); + break; + } + + case detail::value_t::array: + { + if (reference_token == "-") + { + // explicitly treat "-" as index beyond the end + ptr = &ptr->operator[](ptr->m_value.array->size()); + } + else + { + // convert array index to number; unchecked access + ptr = &ptr->operator[](array_index(reference_token)); + } + break; + } + + case detail::value_t::null: + case detail::value_t::string: + case detail::value_t::boolean: + case detail::value_t::number_integer: + case detail::value_t::number_unsigned: + case detail::value_t::number_float: + case detail::value_t::binary: + case detail::value_t::discarded: + default: + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'", *ptr)); + } + } + + return *ptr; + } + + /*! + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.402 if the array index '-' is used + @throw out_of_range.404 if the JSON pointer can not be resolved + */ + BasicJsonType& get_checked(BasicJsonType* ptr) const + { + for (const auto& reference_token : reference_tokens) + { + switch (ptr->type()) + { + case detail::value_t::object: + { + // note: at performs range check + ptr = &ptr->at(reference_token); + break; + } + + case detail::value_t::array: + { + if (JSON_HEDLEY_UNLIKELY(reference_token == "-")) + { + // "-" always fails the range check + JSON_THROW(detail::out_of_range::create(402, + "array index '-' (" + std::to_string(ptr->m_value.array->size()) + + ") is out of range", *ptr)); + } + + // note: at performs range check + ptr = &ptr->at(array_index(reference_token)); + break; + } + + case detail::value_t::null: + case detail::value_t::string: + case detail::value_t::boolean: + case detail::value_t::number_integer: + case detail::value_t::number_unsigned: + case detail::value_t::number_float: + case detail::value_t::binary: + case detail::value_t::discarded: + default: + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'", *ptr)); + } + } + + return *ptr; + } + + /*! + @brief return a const reference to the pointed to value + + @param[in] ptr a JSON value + + @return const reference to the JSON value pointed to by the JSON + pointer + + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.402 if the array index '-' is used + @throw out_of_range.404 if the JSON pointer can not be resolved + */ + const BasicJsonType& get_unchecked(const BasicJsonType* ptr) const + { + for (const auto& reference_token : reference_tokens) + { + switch (ptr->type()) + { + case detail::value_t::object: + { + // use unchecked object access + ptr = &ptr->operator[](reference_token); + break; + } + + case detail::value_t::array: + { + if (JSON_HEDLEY_UNLIKELY(reference_token == "-")) + { + // "-" cannot be used for const access + JSON_THROW(detail::out_of_range::create(402, "array index '-' (" + std::to_string(ptr->m_value.array->size()) + ") is out of range", *ptr)); + } + + // use unchecked array access + ptr = &ptr->operator[](array_index(reference_token)); + break; + } + + case detail::value_t::null: + case detail::value_t::string: + case detail::value_t::boolean: + case detail::value_t::number_integer: + case detail::value_t::number_unsigned: + case detail::value_t::number_float: + case detail::value_t::binary: + case detail::value_t::discarded: + default: + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'", *ptr)); + } + } + + return *ptr; + } + + /*! + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.402 if the array index '-' is used + @throw out_of_range.404 if the JSON pointer can not be resolved + */ + const BasicJsonType& get_checked(const BasicJsonType* ptr) const + { + for (const auto& reference_token : reference_tokens) + { + switch (ptr->type()) + { + case detail::value_t::object: + { + // note: at performs range check + ptr = &ptr->at(reference_token); + break; + } + + case detail::value_t::array: + { + if (JSON_HEDLEY_UNLIKELY(reference_token == "-")) + { + // "-" always fails the range check + JSON_THROW(detail::out_of_range::create(402, + "array index '-' (" + std::to_string(ptr->m_value.array->size()) + + ") is out of range", *ptr)); + } + + // note: at performs range check + ptr = &ptr->at(array_index(reference_token)); + break; + } + + case detail::value_t::null: + case detail::value_t::string: + case detail::value_t::boolean: + case detail::value_t::number_integer: + case detail::value_t::number_unsigned: + case detail::value_t::number_float: + case detail::value_t::binary: + case detail::value_t::discarded: + default: + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'", *ptr)); + } + } + + return *ptr; + } + + /*! + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + */ + bool contains(const BasicJsonType* ptr) const + { + for (const auto& reference_token : reference_tokens) + { + switch (ptr->type()) + { + case detail::value_t::object: + { + if (!ptr->contains(reference_token)) + { + // we did not find the key in the object + return false; + } + + ptr = &ptr->operator[](reference_token); + break; + } + + case detail::value_t::array: + { + if (JSON_HEDLEY_UNLIKELY(reference_token == "-")) + { + // "-" always fails the range check + return false; + } + if (JSON_HEDLEY_UNLIKELY(reference_token.size() == 1 && !("0" <= reference_token && reference_token <= "9"))) + { + // invalid char + return false; + } + if (JSON_HEDLEY_UNLIKELY(reference_token.size() > 1)) + { + if (JSON_HEDLEY_UNLIKELY(!('1' <= reference_token[0] && reference_token[0] <= '9'))) + { + // first char should be between '1' and '9' + return false; + } + for (std::size_t i = 1; i < reference_token.size(); i++) + { + if (JSON_HEDLEY_UNLIKELY(!('0' <= reference_token[i] && reference_token[i] <= '9'))) + { + // other char should be between '0' and '9' + return false; + } + } + } + + const auto idx = array_index(reference_token); + if (idx >= ptr->size()) + { + // index out of range + return false; + } + + ptr = &ptr->operator[](idx); + break; + } + + case detail::value_t::null: + case detail::value_t::string: + case detail::value_t::boolean: + case detail::value_t::number_integer: + case detail::value_t::number_unsigned: + case detail::value_t::number_float: + case detail::value_t::binary: + case detail::value_t::discarded: + default: + { + // we do not expect primitive values if there is still a + // reference token to process + return false; + } + } + } + + // no reference token left means we found a primitive value + return true; + } + + /*! + @brief split the string input to reference tokens + + @note This function is only called by the json_pointer constructor. + All exceptions below are documented there. + + @throw parse_error.107 if the pointer is not empty or begins with '/' + @throw parse_error.108 if character '~' is not followed by '0' or '1' + */ + static std::vector split(const std::string& reference_string) + { + std::vector result; + + // special case: empty reference string -> no reference tokens + if (reference_string.empty()) + { + return result; + } + + // check if nonempty reference string begins with slash + if (JSON_HEDLEY_UNLIKELY(reference_string[0] != '/')) + { + JSON_THROW(detail::parse_error::create(107, 1, "JSON pointer must be empty or begin with '/' - was: '" + reference_string + "'", BasicJsonType())); + } + + // extract the reference tokens: + // - slash: position of the last read slash (or end of string) + // - start: position after the previous slash + for ( + // search for the first slash after the first character + std::size_t slash = reference_string.find_first_of('/', 1), + // set the beginning of the first reference token + start = 1; + // we can stop if start == 0 (if slash == std::string::npos) + start != 0; + // set the beginning of the next reference token + // (will eventually be 0 if slash == std::string::npos) + start = (slash == std::string::npos) ? 0 : slash + 1, + // find next slash + slash = reference_string.find_first_of('/', start)) + { + // use the text between the beginning of the reference token + // (start) and the last slash (slash). + auto reference_token = reference_string.substr(start, slash - start); + + // check reference tokens are properly escaped + for (std::size_t pos = reference_token.find_first_of('~'); + pos != std::string::npos; + pos = reference_token.find_first_of('~', pos + 1)) + { + JSON_ASSERT(reference_token[pos] == '~'); + + // ~ must be followed by 0 or 1 + if (JSON_HEDLEY_UNLIKELY(pos == reference_token.size() - 1 || + (reference_token[pos + 1] != '0' && + reference_token[pos + 1] != '1'))) + { + JSON_THROW(detail::parse_error::create(108, 0, "escape character '~' must be followed with '0' or '1'", BasicJsonType())); + } + } + + // finally, store the reference token + detail::unescape(reference_token); + result.push_back(reference_token); + } + + return result; + } + + private: + /*! + @param[in] reference_string the reference string to the current value + @param[in] value the value to consider + @param[in,out] result the result object to insert values to + + @note Empty objects or arrays are flattened to `null`. + */ + static void flatten(const std::string& reference_string, + const BasicJsonType& value, + BasicJsonType& result) + { + switch (value.type()) + { + case detail::value_t::array: + { + if (value.m_value.array->empty()) + { + // flatten empty array as null + result[reference_string] = nullptr; + } + else + { + // iterate array and use index as reference string + for (std::size_t i = 0; i < value.m_value.array->size(); ++i) + { + flatten(reference_string + "/" + std::to_string(i), + value.m_value.array->operator[](i), result); + } + } + break; + } + + case detail::value_t::object: + { + if (value.m_value.object->empty()) + { + // flatten empty object as null + result[reference_string] = nullptr; + } + else + { + // iterate object and use keys as reference string + for (const auto& element : *value.m_value.object) + { + flatten(reference_string + "/" + detail::escape(element.first), element.second, result); + } + } + break; + } + + case detail::value_t::null: + case detail::value_t::string: + case detail::value_t::boolean: + case detail::value_t::number_integer: + case detail::value_t::number_unsigned: + case detail::value_t::number_float: + case detail::value_t::binary: + case detail::value_t::discarded: + default: + { + // add primitive value with its reference string + result[reference_string] = value; + break; + } + } + } + + /*! + @param[in] value flattened JSON + + @return unflattened JSON + + @throw parse_error.109 if array index is not a number + @throw type_error.314 if value is not an object + @throw type_error.315 if object values are not primitive + @throw type_error.313 if value cannot be unflattened + */ + static BasicJsonType + unflatten(const BasicJsonType& value) + { + if (JSON_HEDLEY_UNLIKELY(!value.is_object())) + { + JSON_THROW(detail::type_error::create(314, "only objects can be unflattened", value)); + } + + BasicJsonType result; + + // iterate the JSON object values + for (const auto& element : *value.m_value.object) + { + if (JSON_HEDLEY_UNLIKELY(!element.second.is_primitive())) + { + JSON_THROW(detail::type_error::create(315, "values in object must be primitive", element.second)); + } + + // assign value to reference pointed to by JSON pointer; Note that if + // the JSON pointer is "" (i.e., points to the whole value), function + // get_and_create returns a reference to result itself. An assignment + // will then create a primitive value. + json_pointer(element.first).get_and_create(result) = element.second; + } + + return result; + } + + /*! + @brief compares two JSON pointers for equality + + @param[in] lhs JSON pointer to compare + @param[in] rhs JSON pointer to compare + @return whether @a lhs is equal to @a rhs + + @complexity Linear in the length of the JSON pointer + + @exceptionsafety No-throw guarantee: this function never throws exceptions. + */ + friend bool operator==(json_pointer const& lhs, + json_pointer const& rhs) noexcept + { + return lhs.reference_tokens == rhs.reference_tokens; + } + + /*! + @brief compares two JSON pointers for inequality + + @param[in] lhs JSON pointer to compare + @param[in] rhs JSON pointer to compare + @return whether @a lhs is not equal @a rhs + + @complexity Linear in the length of the JSON pointer + + @exceptionsafety No-throw guarantee: this function never throws exceptions. + */ + friend bool operator!=(json_pointer const& lhs, + json_pointer const& rhs) noexcept + { + return !(lhs == rhs); + } + + /// the reference tokens + std::vector reference_tokens; +}; +} // namespace nlohmann + +// #include + + +#include +#include + +// #include + + +namespace nlohmann +{ +namespace detail +{ +template +class json_ref +{ + public: + using value_type = BasicJsonType; + + json_ref(value_type&& value) + : owned_value(std::move(value)) + {} + + json_ref(const value_type& value) + : value_ref(&value) + {} + + json_ref(std::initializer_list init) + : owned_value(init) + {} + + template < + class... Args, + enable_if_t::value, int> = 0 > + json_ref(Args && ... args) + : owned_value(std::forward(args)...) + {} + + // class should be movable only + json_ref(json_ref&&) noexcept = default; + json_ref(const json_ref&) = delete; + json_ref& operator=(const json_ref&) = delete; + json_ref& operator=(json_ref&&) = delete; + ~json_ref() = default; + + value_type moved_or_copied() const + { + if (value_ref == nullptr) + { + return std::move(owned_value); + } + return *value_ref; + } + + value_type const& operator*() const + { + return value_ref ? *value_ref : owned_value; + } + + value_type const* operator->() const + { + return &** this; + } + + private: + mutable value_type owned_value = nullptr; + value_type const* value_ref = nullptr; +}; +} // namespace detail +} // namespace nlohmann + +// #include + +// #include + +// #include + +// #include + +// #include + + +#include // reverse +#include // array +#include // isnan, isinf +#include // uint8_t, uint16_t, uint32_t, uint64_t +#include // memcpy +#include // numeric_limits +#include // string +#include // move + +// #include + +// #include + +// #include + + +#include // copy +#include // size_t +#include // back_inserter +#include // shared_ptr, make_shared +#include // basic_string +#include // vector + +#ifndef JSON_NO_IO + #include // streamsize + #include // basic_ostream +#endif // JSON_NO_IO + +// #include + + +namespace nlohmann +{ +namespace detail +{ +/// abstract output adapter interface +template struct output_adapter_protocol +{ + virtual void write_character(CharType c) = 0; + virtual void write_characters(const CharType* s, std::size_t length) = 0; + virtual ~output_adapter_protocol() = default; + + output_adapter_protocol() = default; + output_adapter_protocol(const output_adapter_protocol&) = default; + output_adapter_protocol(output_adapter_protocol&&) noexcept = default; + output_adapter_protocol& operator=(const output_adapter_protocol&) = default; + output_adapter_protocol& operator=(output_adapter_protocol&&) noexcept = default; +}; + +/// a type to simplify interfaces +template +using output_adapter_t = std::shared_ptr>; + +/// output adapter for byte vectors +template> +class output_vector_adapter : public output_adapter_protocol +{ + public: + explicit output_vector_adapter(std::vector& vec) noexcept + : v(vec) + {} + + void write_character(CharType c) override + { + v.push_back(c); + } + + JSON_HEDLEY_NON_NULL(2) + void write_characters(const CharType* s, std::size_t length) override + { + std::copy(s, s + length, std::back_inserter(v)); + } + + private: + std::vector& v; +}; + +#ifndef JSON_NO_IO +/// output adapter for output streams +template +class output_stream_adapter : public output_adapter_protocol +{ + public: + explicit output_stream_adapter(std::basic_ostream& s) noexcept + : stream(s) + {} + + void write_character(CharType c) override + { + stream.put(c); + } + + JSON_HEDLEY_NON_NULL(2) + void write_characters(const CharType* s, std::size_t length) override + { + stream.write(s, static_cast(length)); + } + + private: + std::basic_ostream& stream; +}; +#endif // JSON_NO_IO + +/// output adapter for basic_string +template> +class output_string_adapter : public output_adapter_protocol +{ + public: + explicit output_string_adapter(StringType& s) noexcept + : str(s) + {} + + void write_character(CharType c) override + { + str.push_back(c); + } + + JSON_HEDLEY_NON_NULL(2) + void write_characters(const CharType* s, std::size_t length) override + { + str.append(s, length); + } + + private: + StringType& str; +}; + +template> +class output_adapter +{ + public: + template> + output_adapter(std::vector& vec) + : oa(std::make_shared>(vec)) {} + +#ifndef JSON_NO_IO + output_adapter(std::basic_ostream& s) + : oa(std::make_shared>(s)) {} +#endif // JSON_NO_IO + + output_adapter(StringType& s) + : oa(std::make_shared>(s)) {} + + operator output_adapter_t() + { + return oa; + } + + private: + output_adapter_t oa = nullptr; +}; +} // namespace detail +} // namespace nlohmann + + +namespace nlohmann +{ +namespace detail +{ +/////////////////// +// binary writer // +/////////////////// + +/*! +@brief serialization to CBOR and MessagePack values +*/ +template +class binary_writer +{ + using string_t = typename BasicJsonType::string_t; + using binary_t = typename BasicJsonType::binary_t; + using number_float_t = typename BasicJsonType::number_float_t; + + public: + /*! + @brief create a binary writer + + @param[in] adapter output adapter to write to + */ + explicit binary_writer(output_adapter_t adapter) : oa(std::move(adapter)) + { + JSON_ASSERT(oa); + } + + /*! + @param[in] j JSON value to serialize + @pre j.type() == value_t::object + */ + void write_bson(const BasicJsonType& j) + { + switch (j.type()) + { + case value_t::object: + { + write_bson_object(*j.m_value.object); + break; + } + + case value_t::null: + case value_t::array: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + JSON_THROW(type_error::create(317, "to serialize to BSON, top-level type must be object, but is " + std::string(j.type_name()), j)); + } + } + } + + /*! + @param[in] j JSON value to serialize + */ + void write_cbor(const BasicJsonType& j) + { + switch (j.type()) + { + case value_t::null: + { + oa->write_character(to_char_type(0xF6)); + break; + } + + case value_t::boolean: + { + oa->write_character(j.m_value.boolean + ? to_char_type(0xF5) + : to_char_type(0xF4)); + break; + } + + case value_t::number_integer: + { + if (j.m_value.number_integer >= 0) + { + // CBOR does not differentiate between positive signed + // integers and unsigned integers. Therefore, we used the + // code from the value_t::number_unsigned case here. + if (j.m_value.number_integer <= 0x17) + { + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_integer <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x18)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_integer <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x19)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_integer <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x1A)); + write_number(static_cast(j.m_value.number_integer)); + } + else + { + oa->write_character(to_char_type(0x1B)); + write_number(static_cast(j.m_value.number_integer)); + } + } + else + { + // The conversions below encode the sign in the first + // byte, and the value is converted to a positive number. + const auto positive_number = -1 - j.m_value.number_integer; + if (j.m_value.number_integer >= -24) + { + write_number(static_cast(0x20 + positive_number)); + } + else if (positive_number <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x38)); + write_number(static_cast(positive_number)); + } + else if (positive_number <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x39)); + write_number(static_cast(positive_number)); + } + else if (positive_number <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x3A)); + write_number(static_cast(positive_number)); + } + else + { + oa->write_character(to_char_type(0x3B)); + write_number(static_cast(positive_number)); + } + } + break; + } + + case value_t::number_unsigned: + { + if (j.m_value.number_unsigned <= 0x17) + { + write_number(static_cast(j.m_value.number_unsigned)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x18)); + write_number(static_cast(j.m_value.number_unsigned)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x19)); + write_number(static_cast(j.m_value.number_unsigned)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x1A)); + write_number(static_cast(j.m_value.number_unsigned)); + } + else + { + oa->write_character(to_char_type(0x1B)); + write_number(static_cast(j.m_value.number_unsigned)); + } + break; + } + + case value_t::number_float: + { + if (std::isnan(j.m_value.number_float)) + { + // NaN is 0xf97e00 in CBOR + oa->write_character(to_char_type(0xF9)); + oa->write_character(to_char_type(0x7E)); + oa->write_character(to_char_type(0x00)); + } + else if (std::isinf(j.m_value.number_float)) + { + // Infinity is 0xf97c00, -Infinity is 0xf9fc00 + oa->write_character(to_char_type(0xf9)); + oa->write_character(j.m_value.number_float > 0 ? to_char_type(0x7C) : to_char_type(0xFC)); + oa->write_character(to_char_type(0x00)); + } + else + { + write_compact_float(j.m_value.number_float, detail::input_format_t::cbor); + } + break; + } + + case value_t::string: + { + // step 1: write control byte and the string length + const auto N = j.m_value.string->size(); + if (N <= 0x17) + { + write_number(static_cast(0x60 + N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x78)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x79)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x7A)); + write_number(static_cast(N)); + } + // LCOV_EXCL_START + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x7B)); + write_number(static_cast(N)); + } + // LCOV_EXCL_STOP + + // step 2: write the string + oa->write_characters( + reinterpret_cast(j.m_value.string->c_str()), + j.m_value.string->size()); + break; + } + + case value_t::array: + { + // step 1: write control byte and the array size + const auto N = j.m_value.array->size(); + if (N <= 0x17) + { + write_number(static_cast(0x80 + N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x98)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x99)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x9A)); + write_number(static_cast(N)); + } + // LCOV_EXCL_START + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x9B)); + write_number(static_cast(N)); + } + // LCOV_EXCL_STOP + + // step 2: write each element + for (const auto& el : *j.m_value.array) + { + write_cbor(el); + } + break; + } + + case value_t::binary: + { + if (j.m_value.binary->has_subtype()) + { + if (j.m_value.binary->subtype() <= (std::numeric_limits::max)()) + { + write_number(static_cast(0xd8)); + write_number(static_cast(j.m_value.binary->subtype())); + } + else if (j.m_value.binary->subtype() <= (std::numeric_limits::max)()) + { + write_number(static_cast(0xd9)); + write_number(static_cast(j.m_value.binary->subtype())); + } + else if (j.m_value.binary->subtype() <= (std::numeric_limits::max)()) + { + write_number(static_cast(0xda)); + write_number(static_cast(j.m_value.binary->subtype())); + } + else if (j.m_value.binary->subtype() <= (std::numeric_limits::max)()) + { + write_number(static_cast(0xdb)); + write_number(static_cast(j.m_value.binary->subtype())); + } + } + + // step 1: write control byte and the binary array size + const auto N = j.m_value.binary->size(); + if (N <= 0x17) + { + write_number(static_cast(0x40 + N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x58)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x59)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x5A)); + write_number(static_cast(N)); + } + // LCOV_EXCL_START + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x5B)); + write_number(static_cast(N)); + } + // LCOV_EXCL_STOP + + // step 2: write each element + oa->write_characters( + reinterpret_cast(j.m_value.binary->data()), + N); + + break; + } + + case value_t::object: + { + // step 1: write control byte and the object size + const auto N = j.m_value.object->size(); + if (N <= 0x17) + { + write_number(static_cast(0xA0 + N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0xB8)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0xB9)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0xBA)); + write_number(static_cast(N)); + } + // LCOV_EXCL_START + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0xBB)); + write_number(static_cast(N)); + } + // LCOV_EXCL_STOP + + // step 2: write each element + for (const auto& el : *j.m_value.object) + { + write_cbor(el.first); + write_cbor(el.second); + } + break; + } + + case value_t::discarded: + default: + break; + } + } + + /*! + @param[in] j JSON value to serialize + */ + void write_msgpack(const BasicJsonType& j) + { + switch (j.type()) + { + case value_t::null: // nil + { + oa->write_character(to_char_type(0xC0)); + break; + } + + case value_t::boolean: // true and false + { + oa->write_character(j.m_value.boolean + ? to_char_type(0xC3) + : to_char_type(0xC2)); + break; + } + + case value_t::number_integer: + { + if (j.m_value.number_integer >= 0) + { + // MessagePack does not differentiate between positive + // signed integers and unsigned integers. Therefore, we used + // the code from the value_t::number_unsigned case here. + if (j.m_value.number_unsigned < 128) + { + // positive fixnum + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + // uint 8 + oa->write_character(to_char_type(0xCC)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + // uint 16 + oa->write_character(to_char_type(0xCD)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + // uint 32 + oa->write_character(to_char_type(0xCE)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + // uint 64 + oa->write_character(to_char_type(0xCF)); + write_number(static_cast(j.m_value.number_integer)); + } + } + else + { + if (j.m_value.number_integer >= -32) + { + // negative fixnum + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_integer >= (std::numeric_limits::min)() && + j.m_value.number_integer <= (std::numeric_limits::max)()) + { + // int 8 + oa->write_character(to_char_type(0xD0)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_integer >= (std::numeric_limits::min)() && + j.m_value.number_integer <= (std::numeric_limits::max)()) + { + // int 16 + oa->write_character(to_char_type(0xD1)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_integer >= (std::numeric_limits::min)() && + j.m_value.number_integer <= (std::numeric_limits::max)()) + { + // int 32 + oa->write_character(to_char_type(0xD2)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_integer >= (std::numeric_limits::min)() && + j.m_value.number_integer <= (std::numeric_limits::max)()) + { + // int 64 + oa->write_character(to_char_type(0xD3)); + write_number(static_cast(j.m_value.number_integer)); + } + } + break; + } + + case value_t::number_unsigned: + { + if (j.m_value.number_unsigned < 128) + { + // positive fixnum + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + // uint 8 + oa->write_character(to_char_type(0xCC)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + // uint 16 + oa->write_character(to_char_type(0xCD)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + // uint 32 + oa->write_character(to_char_type(0xCE)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + // uint 64 + oa->write_character(to_char_type(0xCF)); + write_number(static_cast(j.m_value.number_integer)); + } + break; + } + + case value_t::number_float: + { + write_compact_float(j.m_value.number_float, detail::input_format_t::msgpack); + break; + } + + case value_t::string: + { + // step 1: write control byte and the string length + const auto N = j.m_value.string->size(); + if (N <= 31) + { + // fixstr + write_number(static_cast(0xA0 | N)); + } + else if (N <= (std::numeric_limits::max)()) + { + // str 8 + oa->write_character(to_char_type(0xD9)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + // str 16 + oa->write_character(to_char_type(0xDA)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + // str 32 + oa->write_character(to_char_type(0xDB)); + write_number(static_cast(N)); + } + + // step 2: write the string + oa->write_characters( + reinterpret_cast(j.m_value.string->c_str()), + j.m_value.string->size()); + break; + } + + case value_t::array: + { + // step 1: write control byte and the array size + const auto N = j.m_value.array->size(); + if (N <= 15) + { + // fixarray + write_number(static_cast(0x90 | N)); + } + else if (N <= (std::numeric_limits::max)()) + { + // array 16 + oa->write_character(to_char_type(0xDC)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + // array 32 + oa->write_character(to_char_type(0xDD)); + write_number(static_cast(N)); + } + + // step 2: write each element + for (const auto& el : *j.m_value.array) + { + write_msgpack(el); + } + break; + } + + case value_t::binary: + { + // step 0: determine if the binary type has a set subtype to + // determine whether or not to use the ext or fixext types + const bool use_ext = j.m_value.binary->has_subtype(); + + // step 1: write control byte and the byte string length + const auto N = j.m_value.binary->size(); + if (N <= (std::numeric_limits::max)()) + { + std::uint8_t output_type{}; + bool fixed = true; + if (use_ext) + { + switch (N) + { + case 1: + output_type = 0xD4; // fixext 1 + break; + case 2: + output_type = 0xD5; // fixext 2 + break; + case 4: + output_type = 0xD6; // fixext 4 + break; + case 8: + output_type = 0xD7; // fixext 8 + break; + case 16: + output_type = 0xD8; // fixext 16 + break; + default: + output_type = 0xC7; // ext 8 + fixed = false; + break; + } + + } + else + { + output_type = 0xC4; // bin 8 + fixed = false; + } + + oa->write_character(to_char_type(output_type)); + if (!fixed) + { + write_number(static_cast(N)); + } + } + else if (N <= (std::numeric_limits::max)()) + { + std::uint8_t output_type = use_ext + ? 0xC8 // ext 16 + : 0xC5; // bin 16 + + oa->write_character(to_char_type(output_type)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + std::uint8_t output_type = use_ext + ? 0xC9 // ext 32 + : 0xC6; // bin 32 + + oa->write_character(to_char_type(output_type)); + write_number(static_cast(N)); + } + + // step 1.5: if this is an ext type, write the subtype + if (use_ext) + { + write_number(static_cast(j.m_value.binary->subtype())); + } + + // step 2: write the byte string + oa->write_characters( + reinterpret_cast(j.m_value.binary->data()), + N); + + break; + } + + case value_t::object: + { + // step 1: write control byte and the object size + const auto N = j.m_value.object->size(); + if (N <= 15) + { + // fixmap + write_number(static_cast(0x80 | (N & 0xF))); + } + else if (N <= (std::numeric_limits::max)()) + { + // map 16 + oa->write_character(to_char_type(0xDE)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + // map 32 + oa->write_character(to_char_type(0xDF)); + write_number(static_cast(N)); + } + + // step 2: write each element + for (const auto& el : *j.m_value.object) + { + write_msgpack(el.first); + write_msgpack(el.second); + } + break; + } + + case value_t::discarded: + default: + break; + } + } + + /*! + @param[in] j JSON value to serialize + @param[in] use_count whether to use '#' prefixes (optimized format) + @param[in] use_type whether to use '$' prefixes (optimized format) + @param[in] add_prefix whether prefixes need to be used for this value + */ + void write_ubjson(const BasicJsonType& j, const bool use_count, + const bool use_type, const bool add_prefix = true) + { + switch (j.type()) + { + case value_t::null: + { + if (add_prefix) + { + oa->write_character(to_char_type('Z')); + } + break; + } + + case value_t::boolean: + { + if (add_prefix) + { + oa->write_character(j.m_value.boolean + ? to_char_type('T') + : to_char_type('F')); + } + break; + } + + case value_t::number_integer: + { + write_number_with_ubjson_prefix(j.m_value.number_integer, add_prefix); + break; + } + + case value_t::number_unsigned: + { + write_number_with_ubjson_prefix(j.m_value.number_unsigned, add_prefix); + break; + } + + case value_t::number_float: + { + write_number_with_ubjson_prefix(j.m_value.number_float, add_prefix); + break; + } + + case value_t::string: + { + if (add_prefix) + { + oa->write_character(to_char_type('S')); + } + write_number_with_ubjson_prefix(j.m_value.string->size(), true); + oa->write_characters( + reinterpret_cast(j.m_value.string->c_str()), + j.m_value.string->size()); + break; + } + + case value_t::array: + { + if (add_prefix) + { + oa->write_character(to_char_type('[')); + } + + bool prefix_required = true; + if (use_type && !j.m_value.array->empty()) + { + JSON_ASSERT(use_count); + const CharType first_prefix = ubjson_prefix(j.front()); + const bool same_prefix = std::all_of(j.begin() + 1, j.end(), + [this, first_prefix](const BasicJsonType & v) + { + return ubjson_prefix(v) == first_prefix; + }); + + if (same_prefix) + { + prefix_required = false; + oa->write_character(to_char_type('$')); + oa->write_character(first_prefix); + } + } + + if (use_count) + { + oa->write_character(to_char_type('#')); + write_number_with_ubjson_prefix(j.m_value.array->size(), true); + } + + for (const auto& el : *j.m_value.array) + { + write_ubjson(el, use_count, use_type, prefix_required); + } + + if (!use_count) + { + oa->write_character(to_char_type(']')); + } + + break; + } + + case value_t::binary: + { + if (add_prefix) + { + oa->write_character(to_char_type('[')); + } + + if (use_type && !j.m_value.binary->empty()) + { + JSON_ASSERT(use_count); + oa->write_character(to_char_type('$')); + oa->write_character('U'); + } + + if (use_count) + { + oa->write_character(to_char_type('#')); + write_number_with_ubjson_prefix(j.m_value.binary->size(), true); + } + + if (use_type) + { + oa->write_characters( + reinterpret_cast(j.m_value.binary->data()), + j.m_value.binary->size()); + } + else + { + for (size_t i = 0; i < j.m_value.binary->size(); ++i) + { + oa->write_character(to_char_type('U')); + oa->write_character(j.m_value.binary->data()[i]); + } + } + + if (!use_count) + { + oa->write_character(to_char_type(']')); + } + + break; + } + + case value_t::object: + { + if (add_prefix) + { + oa->write_character(to_char_type('{')); + } + + bool prefix_required = true; + if (use_type && !j.m_value.object->empty()) + { + JSON_ASSERT(use_count); + const CharType first_prefix = ubjson_prefix(j.front()); + const bool same_prefix = std::all_of(j.begin(), j.end(), + [this, first_prefix](const BasicJsonType & v) + { + return ubjson_prefix(v) == first_prefix; + }); + + if (same_prefix) + { + prefix_required = false; + oa->write_character(to_char_type('$')); + oa->write_character(first_prefix); + } + } + + if (use_count) + { + oa->write_character(to_char_type('#')); + write_number_with_ubjson_prefix(j.m_value.object->size(), true); + } + + for (const auto& el : *j.m_value.object) + { + write_number_with_ubjson_prefix(el.first.size(), true); + oa->write_characters( + reinterpret_cast(el.first.c_str()), + el.first.size()); + write_ubjson(el.second, use_count, use_type, prefix_required); + } + + if (!use_count) + { + oa->write_character(to_char_type('}')); + } + + break; + } + + case value_t::discarded: + default: + break; + } + } + + private: + ////////// + // BSON // + ////////// + + /*! + @return The size of a BSON document entry header, including the id marker + and the entry name size (and its null-terminator). + */ + static std::size_t calc_bson_entry_header_size(const string_t& name, const BasicJsonType& j) + { + const auto it = name.find(static_cast(0)); + if (JSON_HEDLEY_UNLIKELY(it != BasicJsonType::string_t::npos)) + { + JSON_THROW(out_of_range::create(409, "BSON key cannot contain code point U+0000 (at byte " + std::to_string(it) + ")", j)); + static_cast(j); + } + + return /*id*/ 1ul + name.size() + /*zero-terminator*/1u; + } + + /*! + @brief Writes the given @a element_type and @a name to the output adapter + */ + void write_bson_entry_header(const string_t& name, + const std::uint8_t element_type) + { + oa->write_character(to_char_type(element_type)); // boolean + oa->write_characters( + reinterpret_cast(name.c_str()), + name.size() + 1u); + } + + /*! + @brief Writes a BSON element with key @a name and boolean value @a value + */ + void write_bson_boolean(const string_t& name, + const bool value) + { + write_bson_entry_header(name, 0x08); + oa->write_character(value ? to_char_type(0x01) : to_char_type(0x00)); + } + + /*! + @brief Writes a BSON element with key @a name and double value @a value + */ + void write_bson_double(const string_t& name, + const double value) + { + write_bson_entry_header(name, 0x01); + write_number(value); + } + + /*! + @return The size of the BSON-encoded string in @a value + */ + static std::size_t calc_bson_string_size(const string_t& value) + { + return sizeof(std::int32_t) + value.size() + 1ul; + } + + /*! + @brief Writes a BSON element with key @a name and string value @a value + */ + void write_bson_string(const string_t& name, + const string_t& value) + { + write_bson_entry_header(name, 0x02); + + write_number(static_cast(value.size() + 1ul)); + oa->write_characters( + reinterpret_cast(value.c_str()), + value.size() + 1); + } + + /*! + @brief Writes a BSON element with key @a name and null value + */ + void write_bson_null(const string_t& name) + { + write_bson_entry_header(name, 0x0A); + } + + /*! + @return The size of the BSON-encoded integer @a value + */ + static std::size_t calc_bson_integer_size(const std::int64_t value) + { + return (std::numeric_limits::min)() <= value && value <= (std::numeric_limits::max)() + ? sizeof(std::int32_t) + : sizeof(std::int64_t); + } + + /*! + @brief Writes a BSON element with key @a name and integer @a value + */ + void write_bson_integer(const string_t& name, + const std::int64_t value) + { + if ((std::numeric_limits::min)() <= value && value <= (std::numeric_limits::max)()) + { + write_bson_entry_header(name, 0x10); // int32 + write_number(static_cast(value)); + } + else + { + write_bson_entry_header(name, 0x12); // int64 + write_number(static_cast(value)); + } + } + + /*! + @return The size of the BSON-encoded unsigned integer in @a j + */ + static constexpr std::size_t calc_bson_unsigned_size(const std::uint64_t value) noexcept + { + return (value <= static_cast((std::numeric_limits::max)())) + ? sizeof(std::int32_t) + : sizeof(std::int64_t); + } + + /*! + @brief Writes a BSON element with key @a name and unsigned @a value + */ + void write_bson_unsigned(const string_t& name, + const BasicJsonType& j) + { + if (j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + { + write_bson_entry_header(name, 0x10 /* int32 */); + write_number(static_cast(j.m_value.number_unsigned)); + } + else if (j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + { + write_bson_entry_header(name, 0x12 /* int64 */); + write_number(static_cast(j.m_value.number_unsigned)); + } + else + { + JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(j.m_value.number_unsigned) + " cannot be represented by BSON as it does not fit int64", j)); + } + } + + /*! + @brief Writes a BSON element with key @a name and object @a value + */ + void write_bson_object_entry(const string_t& name, + const typename BasicJsonType::object_t& value) + { + write_bson_entry_header(name, 0x03); // object + write_bson_object(value); + } + + /*! + @return The size of the BSON-encoded array @a value + */ + static std::size_t calc_bson_array_size(const typename BasicJsonType::array_t& value) + { + std::size_t array_index = 0ul; + + const std::size_t embedded_document_size = std::accumulate(std::begin(value), std::end(value), static_cast(0), [&array_index](std::size_t result, const typename BasicJsonType::array_t::value_type & el) + { + return result + calc_bson_element_size(std::to_string(array_index++), el); + }); + + return sizeof(std::int32_t) + embedded_document_size + 1ul; + } + + /*! + @return The size of the BSON-encoded binary array @a value + */ + static std::size_t calc_bson_binary_size(const typename BasicJsonType::binary_t& value) + { + return sizeof(std::int32_t) + value.size() + 1ul; + } + + /*! + @brief Writes a BSON element with key @a name and array @a value + */ + void write_bson_array(const string_t& name, + const typename BasicJsonType::array_t& value) + { + write_bson_entry_header(name, 0x04); // array + write_number(static_cast(calc_bson_array_size(value))); + + std::size_t array_index = 0ul; + + for (const auto& el : value) + { + write_bson_element(std::to_string(array_index++), el); + } + + oa->write_character(to_char_type(0x00)); + } + + /*! + @brief Writes a BSON element with key @a name and binary value @a value + */ + void write_bson_binary(const string_t& name, + const binary_t& value) + { + write_bson_entry_header(name, 0x05); + + write_number(static_cast(value.size())); + write_number(value.has_subtype() ? static_cast(value.subtype()) : static_cast(0x00)); + + oa->write_characters(reinterpret_cast(value.data()), value.size()); + } + + /*! + @brief Calculates the size necessary to serialize the JSON value @a j with its @a name + @return The calculated size for the BSON document entry for @a j with the given @a name. + */ + static std::size_t calc_bson_element_size(const string_t& name, + const BasicJsonType& j) + { + const auto header_size = calc_bson_entry_header_size(name, j); + switch (j.type()) + { + case value_t::object: + return header_size + calc_bson_object_size(*j.m_value.object); + + case value_t::array: + return header_size + calc_bson_array_size(*j.m_value.array); + + case value_t::binary: + return header_size + calc_bson_binary_size(*j.m_value.binary); + + case value_t::boolean: + return header_size + 1ul; + + case value_t::number_float: + return header_size + 8ul; + + case value_t::number_integer: + return header_size + calc_bson_integer_size(j.m_value.number_integer); + + case value_t::number_unsigned: + return header_size + calc_bson_unsigned_size(j.m_value.number_unsigned); + + case value_t::string: + return header_size + calc_bson_string_size(*j.m_value.string); + + case value_t::null: + return header_size + 0ul; + + // LCOV_EXCL_START + case value_t::discarded: + default: + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) + return 0ul; + // LCOV_EXCL_STOP + } + } + + /*! + @brief Serializes the JSON value @a j to BSON and associates it with the + key @a name. + @param name The name to associate with the JSON entity @a j within the + current BSON document + */ + void write_bson_element(const string_t& name, + const BasicJsonType& j) + { + switch (j.type()) + { + case value_t::object: + return write_bson_object_entry(name, *j.m_value.object); + + case value_t::array: + return write_bson_array(name, *j.m_value.array); + + case value_t::binary: + return write_bson_binary(name, *j.m_value.binary); + + case value_t::boolean: + return write_bson_boolean(name, j.m_value.boolean); + + case value_t::number_float: + return write_bson_double(name, j.m_value.number_float); + + case value_t::number_integer: + return write_bson_integer(name, j.m_value.number_integer); + + case value_t::number_unsigned: + return write_bson_unsigned(name, j); + + case value_t::string: + return write_bson_string(name, *j.m_value.string); + + case value_t::null: + return write_bson_null(name); + + // LCOV_EXCL_START + case value_t::discarded: + default: + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) + return; + // LCOV_EXCL_STOP + } + } + + /*! + @brief Calculates the size of the BSON serialization of the given + JSON-object @a j. + @param[in] value JSON value to serialize + @pre value.type() == value_t::object + */ + static std::size_t calc_bson_object_size(const typename BasicJsonType::object_t& value) + { + std::size_t document_size = std::accumulate(value.begin(), value.end(), static_cast(0), + [](size_t result, const typename BasicJsonType::object_t::value_type & el) + { + return result += calc_bson_element_size(el.first, el.second); + }); + + return sizeof(std::int32_t) + document_size + 1ul; + } + + /*! + @param[in] value JSON value to serialize + @pre value.type() == value_t::object + */ + void write_bson_object(const typename BasicJsonType::object_t& value) + { + write_number(static_cast(calc_bson_object_size(value))); + + for (const auto& el : value) + { + write_bson_element(el.first, el.second); + } + + oa->write_character(to_char_type(0x00)); + } + + ////////// + // CBOR // + ////////// + + static constexpr CharType get_cbor_float_prefix(float /*unused*/) + { + return to_char_type(0xFA); // Single-Precision Float + } + + static constexpr CharType get_cbor_float_prefix(double /*unused*/) + { + return to_char_type(0xFB); // Double-Precision Float + } + + ///////////// + // MsgPack // + ///////////// + + static constexpr CharType get_msgpack_float_prefix(float /*unused*/) + { + return to_char_type(0xCA); // float 32 + } + + static constexpr CharType get_msgpack_float_prefix(double /*unused*/) + { + return to_char_type(0xCB); // float 64 + } + + //////////// + // UBJSON // + //////////// + + // UBJSON: write number (floating point) + template::value, int>::type = 0> + void write_number_with_ubjson_prefix(const NumberType n, + const bool add_prefix) + { + if (add_prefix) + { + oa->write_character(get_ubjson_float_prefix(n)); + } + write_number(n); + } + + // UBJSON: write number (unsigned integer) + template::value, int>::type = 0> + void write_number_with_ubjson_prefix(const NumberType n, + const bool add_prefix) + { + if (n <= static_cast((std::numeric_limits::max)())) + { + if (add_prefix) + { + oa->write_character(to_char_type('i')); // int8 + } + write_number(static_cast(n)); + } + else if (n <= (std::numeric_limits::max)()) + { + if (add_prefix) + { + oa->write_character(to_char_type('U')); // uint8 + } + write_number(static_cast(n)); + } + else if (n <= static_cast((std::numeric_limits::max)())) + { + if (add_prefix) + { + oa->write_character(to_char_type('I')); // int16 + } + write_number(static_cast(n)); + } + else if (n <= static_cast((std::numeric_limits::max)())) + { + if (add_prefix) + { + oa->write_character(to_char_type('l')); // int32 + } + write_number(static_cast(n)); + } + else if (n <= static_cast((std::numeric_limits::max)())) + { + if (add_prefix) + { + oa->write_character(to_char_type('L')); // int64 + } + write_number(static_cast(n)); + } + else + { + if (add_prefix) + { + oa->write_character(to_char_type('H')); // high-precision number + } + + const auto number = BasicJsonType(n).dump(); + write_number_with_ubjson_prefix(number.size(), true); + for (std::size_t i = 0; i < number.size(); ++i) + { + oa->write_character(to_char_type(static_cast(number[i]))); + } + } + } + + // UBJSON: write number (signed integer) + template < typename NumberType, typename std::enable_if < + std::is_signed::value&& + !std::is_floating_point::value, int >::type = 0 > + void write_number_with_ubjson_prefix(const NumberType n, + const bool add_prefix) + { + if ((std::numeric_limits::min)() <= n && n <= (std::numeric_limits::max)()) + { + if (add_prefix) + { + oa->write_character(to_char_type('i')); // int8 + } + write_number(static_cast(n)); + } + else if (static_cast((std::numeric_limits::min)()) <= n && n <= static_cast((std::numeric_limits::max)())) + { + if (add_prefix) + { + oa->write_character(to_char_type('U')); // uint8 + } + write_number(static_cast(n)); + } + else if ((std::numeric_limits::min)() <= n && n <= (std::numeric_limits::max)()) + { + if (add_prefix) + { + oa->write_character(to_char_type('I')); // int16 + } + write_number(static_cast(n)); + } + else if ((std::numeric_limits::min)() <= n && n <= (std::numeric_limits::max)()) + { + if (add_prefix) + { + oa->write_character(to_char_type('l')); // int32 + } + write_number(static_cast(n)); + } + else if ((std::numeric_limits::min)() <= n && n <= (std::numeric_limits::max)()) + { + if (add_prefix) + { + oa->write_character(to_char_type('L')); // int64 + } + write_number(static_cast(n)); + } + // LCOV_EXCL_START + else + { + if (add_prefix) + { + oa->write_character(to_char_type('H')); // high-precision number + } + + const auto number = BasicJsonType(n).dump(); + write_number_with_ubjson_prefix(number.size(), true); + for (std::size_t i = 0; i < number.size(); ++i) + { + oa->write_character(to_char_type(static_cast(number[i]))); + } + } + // LCOV_EXCL_STOP + } + + /*! + @brief determine the type prefix of container values + */ + CharType ubjson_prefix(const BasicJsonType& j) const noexcept + { + switch (j.type()) + { + case value_t::null: + return 'Z'; + + case value_t::boolean: + return j.m_value.boolean ? 'T' : 'F'; + + case value_t::number_integer: + { + if ((std::numeric_limits::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits::max)()) + { + return 'i'; + } + if ((std::numeric_limits::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits::max)()) + { + return 'U'; + } + if ((std::numeric_limits::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits::max)()) + { + return 'I'; + } + if ((std::numeric_limits::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits::max)()) + { + return 'l'; + } + if ((std::numeric_limits::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits::max)()) + { + return 'L'; + } + // anything else is treated as high-precision number + return 'H'; // LCOV_EXCL_LINE + } + + case value_t::number_unsigned: + { + if (j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + { + return 'i'; + } + if (j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + { + return 'U'; + } + if (j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + { + return 'I'; + } + if (j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + { + return 'l'; + } + if (j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + { + return 'L'; + } + // anything else is treated as high-precision number + return 'H'; // LCOV_EXCL_LINE + } + + case value_t::number_float: + return get_ubjson_float_prefix(j.m_value.number_float); + + case value_t::string: + return 'S'; + + case value_t::array: // fallthrough + case value_t::binary: + return '['; + + case value_t::object: + return '{'; + + case value_t::discarded: + default: // discarded values + return 'N'; + } + } + + static constexpr CharType get_ubjson_float_prefix(float /*unused*/) + { + return 'd'; // float 32 + } + + static constexpr CharType get_ubjson_float_prefix(double /*unused*/) + { + return 'D'; // float 64 + } + + /////////////////////// + // Utility functions // + /////////////////////// + + /* + @brief write a number to output input + @param[in] n number of type @a NumberType + @tparam NumberType the type of the number + @tparam OutputIsLittleEndian Set to true if output data is + required to be little endian + + @note This function needs to respect the system's endianness, because bytes + in CBOR, MessagePack, and UBJSON are stored in network order (big + endian) and therefore need reordering on little endian systems. + */ + template + void write_number(const NumberType n) + { + // step 1: write number to array of length NumberType + std::array vec{}; + std::memcpy(vec.data(), &n, sizeof(NumberType)); + + // step 2: write array to output (with possible reordering) + if (is_little_endian != OutputIsLittleEndian) + { + // reverse byte order prior to conversion if necessary + std::reverse(vec.begin(), vec.end()); + } + + oa->write_characters(vec.data(), sizeof(NumberType)); + } + + void write_compact_float(const number_float_t n, detail::input_format_t format) + { +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wfloat-equal" +#endif + if (static_cast(n) >= static_cast(std::numeric_limits::lowest()) && + static_cast(n) <= static_cast((std::numeric_limits::max)()) && + static_cast(static_cast(n)) == static_cast(n)) + { + oa->write_character(format == detail::input_format_t::cbor + ? get_cbor_float_prefix(static_cast(n)) + : get_msgpack_float_prefix(static_cast(n))); + write_number(static_cast(n)); + } + else + { + oa->write_character(format == detail::input_format_t::cbor + ? get_cbor_float_prefix(n) + : get_msgpack_float_prefix(n)); + write_number(n); + } +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif + } + + public: + // The following to_char_type functions are implement the conversion + // between uint8_t and CharType. In case CharType is not unsigned, + // such a conversion is required to allow values greater than 128. + // See for a discussion. + template < typename C = CharType, + enable_if_t < std::is_signed::value && std::is_signed::value > * = nullptr > + static constexpr CharType to_char_type(std::uint8_t x) noexcept + { + return *reinterpret_cast(&x); + } + + template < typename C = CharType, + enable_if_t < std::is_signed::value && std::is_unsigned::value > * = nullptr > + static CharType to_char_type(std::uint8_t x) noexcept + { + static_assert(sizeof(std::uint8_t) == sizeof(CharType), "size of CharType must be equal to std::uint8_t"); + static_assert(std::is_trivial::value, "CharType must be trivial"); + CharType result; + std::memcpy(&result, &x, sizeof(x)); + return result; + } + + template::value>* = nullptr> + static constexpr CharType to_char_type(std::uint8_t x) noexcept + { + return x; + } + + template < typename InputCharType, typename C = CharType, + enable_if_t < + std::is_signed::value && + std::is_signed::value && + std::is_same::type>::value + > * = nullptr > + static constexpr CharType to_char_type(InputCharType x) noexcept + { + return x; + } + + private: + /// whether we can assume little endianness + const bool is_little_endian = little_endianness(); + + /// the output + output_adapter_t oa = nullptr; +}; +} // namespace detail +} // namespace nlohmann + +// #include + +// #include + + +#include // reverse, remove, fill, find, none_of +#include // array +#include // localeconv, lconv +#include // labs, isfinite, isnan, signbit +#include // size_t, ptrdiff_t +#include // uint8_t +#include // snprintf +#include // numeric_limits +#include // string, char_traits +#include // setfill, setw +#include // stringstream +#include // is_same +#include // move + +// #include + + +#include // array +#include // signbit, isfinite +#include // intN_t, uintN_t +#include // memcpy, memmove +#include // numeric_limits +#include // conditional + +// #include + + +namespace nlohmann +{ +namespace detail +{ + +/*! +@brief implements the Grisu2 algorithm for binary to decimal floating-point +conversion. + +This implementation is a slightly modified version of the reference +implementation which may be obtained from +http://florian.loitsch.com/publications (bench.tar.gz). + +The code is distributed under the MIT license, Copyright (c) 2009 Florian Loitsch. + +For a detailed description of the algorithm see: + +[1] Loitsch, "Printing Floating-Point Numbers Quickly and Accurately with + Integers", Proceedings of the ACM SIGPLAN 2010 Conference on Programming + Language Design and Implementation, PLDI 2010 +[2] Burger, Dybvig, "Printing Floating-Point Numbers Quickly and Accurately", + Proceedings of the ACM SIGPLAN 1996 Conference on Programming Language + Design and Implementation, PLDI 1996 +*/ +namespace dtoa_impl +{ + +template +Target reinterpret_bits(const Source source) +{ + static_assert(sizeof(Target) == sizeof(Source), "size mismatch"); + + Target target; + std::memcpy(&target, &source, sizeof(Source)); + return target; +} + +struct diyfp // f * 2^e +{ + static constexpr int kPrecision = 64; // = q + + std::uint64_t f = 0; + int e = 0; + + constexpr diyfp(std::uint64_t f_, int e_) noexcept : f(f_), e(e_) {} + + /*! + @brief returns x - y + @pre x.e == y.e and x.f >= y.f + */ + static diyfp sub(const diyfp& x, const diyfp& y) noexcept + { + JSON_ASSERT(x.e == y.e); + JSON_ASSERT(x.f >= y.f); + + return {x.f - y.f, x.e}; + } + + /*! + @brief returns x * y + @note The result is rounded. (Only the upper q bits are returned.) + */ + static diyfp mul(const diyfp& x, const diyfp& y) noexcept + { + static_assert(kPrecision == 64, "internal error"); + + // Computes: + // f = round((x.f * y.f) / 2^q) + // e = x.e + y.e + q + + // Emulate the 64-bit * 64-bit multiplication: + // + // p = u * v + // = (u_lo + 2^32 u_hi) (v_lo + 2^32 v_hi) + // = (u_lo v_lo ) + 2^32 ((u_lo v_hi ) + (u_hi v_lo )) + 2^64 (u_hi v_hi ) + // = (p0 ) + 2^32 ((p1 ) + (p2 )) + 2^64 (p3 ) + // = (p0_lo + 2^32 p0_hi) + 2^32 ((p1_lo + 2^32 p1_hi) + (p2_lo + 2^32 p2_hi)) + 2^64 (p3 ) + // = (p0_lo ) + 2^32 (p0_hi + p1_lo + p2_lo ) + 2^64 (p1_hi + p2_hi + p3) + // = (p0_lo ) + 2^32 (Q ) + 2^64 (H ) + // = (p0_lo ) + 2^32 (Q_lo + 2^32 Q_hi ) + 2^64 (H ) + // + // (Since Q might be larger than 2^32 - 1) + // + // = (p0_lo + 2^32 Q_lo) + 2^64 (Q_hi + H) + // + // (Q_hi + H does not overflow a 64-bit int) + // + // = p_lo + 2^64 p_hi + + const std::uint64_t u_lo = x.f & 0xFFFFFFFFu; + const std::uint64_t u_hi = x.f >> 32u; + const std::uint64_t v_lo = y.f & 0xFFFFFFFFu; + const std::uint64_t v_hi = y.f >> 32u; + + const std::uint64_t p0 = u_lo * v_lo; + const std::uint64_t p1 = u_lo * v_hi; + const std::uint64_t p2 = u_hi * v_lo; + const std::uint64_t p3 = u_hi * v_hi; + + const std::uint64_t p0_hi = p0 >> 32u; + const std::uint64_t p1_lo = p1 & 0xFFFFFFFFu; + const std::uint64_t p1_hi = p1 >> 32u; + const std::uint64_t p2_lo = p2 & 0xFFFFFFFFu; + const std::uint64_t p2_hi = p2 >> 32u; + + std::uint64_t Q = p0_hi + p1_lo + p2_lo; + + // The full product might now be computed as + // + // p_hi = p3 + p2_hi + p1_hi + (Q >> 32) + // p_lo = p0_lo + (Q << 32) + // + // But in this particular case here, the full p_lo is not required. + // Effectively we only need to add the highest bit in p_lo to p_hi (and + // Q_hi + 1 does not overflow). + + Q += std::uint64_t{1} << (64u - 32u - 1u); // round, ties up + + const std::uint64_t h = p3 + p2_hi + p1_hi + (Q >> 32u); + + return {h, x.e + y.e + 64}; + } + + /*! + @brief normalize x such that the significand is >= 2^(q-1) + @pre x.f != 0 + */ + static diyfp normalize(diyfp x) noexcept + { + JSON_ASSERT(x.f != 0); + + while ((x.f >> 63u) == 0) + { + x.f <<= 1u; + x.e--; + } + + return x; + } + + /*! + @brief normalize x such that the result has the exponent E + @pre e >= x.e and the upper e - x.e bits of x.f must be zero. + */ + static diyfp normalize_to(const diyfp& x, const int target_exponent) noexcept + { + const int delta = x.e - target_exponent; + + JSON_ASSERT(delta >= 0); + JSON_ASSERT(((x.f << delta) >> delta) == x.f); + + return {x.f << delta, target_exponent}; + } +}; + +struct boundaries +{ + diyfp w; + diyfp minus; + diyfp plus; +}; + +/*! +Compute the (normalized) diyfp representing the input number 'value' and its +boundaries. + +@pre value must be finite and positive +*/ +template +boundaries compute_boundaries(FloatType value) +{ + JSON_ASSERT(std::isfinite(value)); + JSON_ASSERT(value > 0); + + // Convert the IEEE representation into a diyfp. + // + // If v is denormal: + // value = 0.F * 2^(1 - bias) = ( F) * 2^(1 - bias - (p-1)) + // If v is normalized: + // value = 1.F * 2^(E - bias) = (2^(p-1) + F) * 2^(E - bias - (p-1)) + + static_assert(std::numeric_limits::is_iec559, + "internal error: dtoa_short requires an IEEE-754 floating-point implementation"); + + constexpr int kPrecision = std::numeric_limits::digits; // = p (includes the hidden bit) + constexpr int kBias = std::numeric_limits::max_exponent - 1 + (kPrecision - 1); + constexpr int kMinExp = 1 - kBias; + constexpr std::uint64_t kHiddenBit = std::uint64_t{1} << (kPrecision - 1); // = 2^(p-1) + + using bits_type = typename std::conditional::type; + + const auto bits = static_cast(reinterpret_bits(value)); + const std::uint64_t E = bits >> (kPrecision - 1); + const std::uint64_t F = bits & (kHiddenBit - 1); + + const bool is_denormal = E == 0; + const diyfp v = is_denormal + ? diyfp(F, kMinExp) + : diyfp(F + kHiddenBit, static_cast(E) - kBias); + + // Compute the boundaries m- and m+ of the floating-point value + // v = f * 2^e. + // + // Determine v- and v+, the floating-point predecessor and successor if v, + // respectively. + // + // v- = v - 2^e if f != 2^(p-1) or e == e_min (A) + // = v - 2^(e-1) if f == 2^(p-1) and e > e_min (B) + // + // v+ = v + 2^e + // + // Let m- = (v- + v) / 2 and m+ = (v + v+) / 2. All real numbers _strictly_ + // between m- and m+ round to v, regardless of how the input rounding + // algorithm breaks ties. + // + // ---+-------------+-------------+-------------+-------------+--- (A) + // v- m- v m+ v+ + // + // -----------------+------+------+-------------+-------------+--- (B) + // v- m- v m+ v+ + + const bool lower_boundary_is_closer = F == 0 && E > 1; + const diyfp m_plus = diyfp(2 * v.f + 1, v.e - 1); + const diyfp m_minus = lower_boundary_is_closer + ? diyfp(4 * v.f - 1, v.e - 2) // (B) + : diyfp(2 * v.f - 1, v.e - 1); // (A) + + // Determine the normalized w+ = m+. + const diyfp w_plus = diyfp::normalize(m_plus); + + // Determine w- = m- such that e_(w-) = e_(w+). + const diyfp w_minus = diyfp::normalize_to(m_minus, w_plus.e); + + return {diyfp::normalize(v), w_minus, w_plus}; +} + +// Given normalized diyfp w, Grisu needs to find a (normalized) cached +// power-of-ten c, such that the exponent of the product c * w = f * 2^e lies +// within a certain range [alpha, gamma] (Definition 3.2 from [1]) +// +// alpha <= e = e_c + e_w + q <= gamma +// +// or +// +// f_c * f_w * 2^alpha <= f_c 2^(e_c) * f_w 2^(e_w) * 2^q +// <= f_c * f_w * 2^gamma +// +// Since c and w are normalized, i.e. 2^(q-1) <= f < 2^q, this implies +// +// 2^(q-1) * 2^(q-1) * 2^alpha <= c * w * 2^q < 2^q * 2^q * 2^gamma +// +// or +// +// 2^(q - 2 + alpha) <= c * w < 2^(q + gamma) +// +// The choice of (alpha,gamma) determines the size of the table and the form of +// the digit generation procedure. Using (alpha,gamma)=(-60,-32) works out well +// in practice: +// +// The idea is to cut the number c * w = f * 2^e into two parts, which can be +// processed independently: An integral part p1, and a fractional part p2: +// +// f * 2^e = ( (f div 2^-e) * 2^-e + (f mod 2^-e) ) * 2^e +// = (f div 2^-e) + (f mod 2^-e) * 2^e +// = p1 + p2 * 2^e +// +// The conversion of p1 into decimal form requires a series of divisions and +// modulos by (a power of) 10. These operations are faster for 32-bit than for +// 64-bit integers, so p1 should ideally fit into a 32-bit integer. This can be +// achieved by choosing +// +// -e >= 32 or e <= -32 := gamma +// +// In order to convert the fractional part +// +// p2 * 2^e = p2 / 2^-e = d[-1] / 10^1 + d[-2] / 10^2 + ... +// +// into decimal form, the fraction is repeatedly multiplied by 10 and the digits +// d[-i] are extracted in order: +// +// (10 * p2) div 2^-e = d[-1] +// (10 * p2) mod 2^-e = d[-2] / 10^1 + ... +// +// The multiplication by 10 must not overflow. It is sufficient to choose +// +// 10 * p2 < 16 * p2 = 2^4 * p2 <= 2^64. +// +// Since p2 = f mod 2^-e < 2^-e, +// +// -e <= 60 or e >= -60 := alpha + +constexpr int kAlpha = -60; +constexpr int kGamma = -32; + +struct cached_power // c = f * 2^e ~= 10^k +{ + std::uint64_t f; + int e; + int k; +}; + +/*! +For a normalized diyfp w = f * 2^e, this function returns a (normalized) cached +power-of-ten c = f_c * 2^e_c, such that the exponent of the product w * c +satisfies (Definition 3.2 from [1]) + + alpha <= e_c + e + q <= gamma. +*/ +inline cached_power get_cached_power_for_binary_exponent(int e) +{ + // Now + // + // alpha <= e_c + e + q <= gamma (1) + // ==> f_c * 2^alpha <= c * 2^e * 2^q + // + // and since the c's are normalized, 2^(q-1) <= f_c, + // + // ==> 2^(q - 1 + alpha) <= c * 2^(e + q) + // ==> 2^(alpha - e - 1) <= c + // + // If c were an exact power of ten, i.e. c = 10^k, one may determine k as + // + // k = ceil( log_10( 2^(alpha - e - 1) ) ) + // = ceil( (alpha - e - 1) * log_10(2) ) + // + // From the paper: + // "In theory the result of the procedure could be wrong since c is rounded, + // and the computation itself is approximated [...]. In practice, however, + // this simple function is sufficient." + // + // For IEEE double precision floating-point numbers converted into + // normalized diyfp's w = f * 2^e, with q = 64, + // + // e >= -1022 (min IEEE exponent) + // -52 (p - 1) + // -52 (p - 1, possibly normalize denormal IEEE numbers) + // -11 (normalize the diyfp) + // = -1137 + // + // and + // + // e <= +1023 (max IEEE exponent) + // -52 (p - 1) + // -11 (normalize the diyfp) + // = 960 + // + // This binary exponent range [-1137,960] results in a decimal exponent + // range [-307,324]. One does not need to store a cached power for each + // k in this range. For each such k it suffices to find a cached power + // such that the exponent of the product lies in [alpha,gamma]. + // This implies that the difference of the decimal exponents of adjacent + // table entries must be less than or equal to + // + // floor( (gamma - alpha) * log_10(2) ) = 8. + // + // (A smaller distance gamma-alpha would require a larger table.) + + // NB: + // Actually this function returns c, such that -60 <= e_c + e + 64 <= -34. + + constexpr int kCachedPowersMinDecExp = -300; + constexpr int kCachedPowersDecStep = 8; + + static constexpr std::array kCachedPowers = + { + { + { 0xAB70FE17C79AC6CA, -1060, -300 }, + { 0xFF77B1FCBEBCDC4F, -1034, -292 }, + { 0xBE5691EF416BD60C, -1007, -284 }, + { 0x8DD01FAD907FFC3C, -980, -276 }, + { 0xD3515C2831559A83, -954, -268 }, + { 0x9D71AC8FADA6C9B5, -927, -260 }, + { 0xEA9C227723EE8BCB, -901, -252 }, + { 0xAECC49914078536D, -874, -244 }, + { 0x823C12795DB6CE57, -847, -236 }, + { 0xC21094364DFB5637, -821, -228 }, + { 0x9096EA6F3848984F, -794, -220 }, + { 0xD77485CB25823AC7, -768, -212 }, + { 0xA086CFCD97BF97F4, -741, -204 }, + { 0xEF340A98172AACE5, -715, -196 }, + { 0xB23867FB2A35B28E, -688, -188 }, + { 0x84C8D4DFD2C63F3B, -661, -180 }, + { 0xC5DD44271AD3CDBA, -635, -172 }, + { 0x936B9FCEBB25C996, -608, -164 }, + { 0xDBAC6C247D62A584, -582, -156 }, + { 0xA3AB66580D5FDAF6, -555, -148 }, + { 0xF3E2F893DEC3F126, -529, -140 }, + { 0xB5B5ADA8AAFF80B8, -502, -132 }, + { 0x87625F056C7C4A8B, -475, -124 }, + { 0xC9BCFF6034C13053, -449, -116 }, + { 0x964E858C91BA2655, -422, -108 }, + { 0xDFF9772470297EBD, -396, -100 }, + { 0xA6DFBD9FB8E5B88F, -369, -92 }, + { 0xF8A95FCF88747D94, -343, -84 }, + { 0xB94470938FA89BCF, -316, -76 }, + { 0x8A08F0F8BF0F156B, -289, -68 }, + { 0xCDB02555653131B6, -263, -60 }, + { 0x993FE2C6D07B7FAC, -236, -52 }, + { 0xE45C10C42A2B3B06, -210, -44 }, + { 0xAA242499697392D3, -183, -36 }, + { 0xFD87B5F28300CA0E, -157, -28 }, + { 0xBCE5086492111AEB, -130, -20 }, + { 0x8CBCCC096F5088CC, -103, -12 }, + { 0xD1B71758E219652C, -77, -4 }, + { 0x9C40000000000000, -50, 4 }, + { 0xE8D4A51000000000, -24, 12 }, + { 0xAD78EBC5AC620000, 3, 20 }, + { 0x813F3978F8940984, 30, 28 }, + { 0xC097CE7BC90715B3, 56, 36 }, + { 0x8F7E32CE7BEA5C70, 83, 44 }, + { 0xD5D238A4ABE98068, 109, 52 }, + { 0x9F4F2726179A2245, 136, 60 }, + { 0xED63A231D4C4FB27, 162, 68 }, + { 0xB0DE65388CC8ADA8, 189, 76 }, + { 0x83C7088E1AAB65DB, 216, 84 }, + { 0xC45D1DF942711D9A, 242, 92 }, + { 0x924D692CA61BE758, 269, 100 }, + { 0xDA01EE641A708DEA, 295, 108 }, + { 0xA26DA3999AEF774A, 322, 116 }, + { 0xF209787BB47D6B85, 348, 124 }, + { 0xB454E4A179DD1877, 375, 132 }, + { 0x865B86925B9BC5C2, 402, 140 }, + { 0xC83553C5C8965D3D, 428, 148 }, + { 0x952AB45CFA97A0B3, 455, 156 }, + { 0xDE469FBD99A05FE3, 481, 164 }, + { 0xA59BC234DB398C25, 508, 172 }, + { 0xF6C69A72A3989F5C, 534, 180 }, + { 0xB7DCBF5354E9BECE, 561, 188 }, + { 0x88FCF317F22241E2, 588, 196 }, + { 0xCC20CE9BD35C78A5, 614, 204 }, + { 0x98165AF37B2153DF, 641, 212 }, + { 0xE2A0B5DC971F303A, 667, 220 }, + { 0xA8D9D1535CE3B396, 694, 228 }, + { 0xFB9B7CD9A4A7443C, 720, 236 }, + { 0xBB764C4CA7A44410, 747, 244 }, + { 0x8BAB8EEFB6409C1A, 774, 252 }, + { 0xD01FEF10A657842C, 800, 260 }, + { 0x9B10A4E5E9913129, 827, 268 }, + { 0xE7109BFBA19C0C9D, 853, 276 }, + { 0xAC2820D9623BF429, 880, 284 }, + { 0x80444B5E7AA7CF85, 907, 292 }, + { 0xBF21E44003ACDD2D, 933, 300 }, + { 0x8E679C2F5E44FF8F, 960, 308 }, + { 0xD433179D9C8CB841, 986, 316 }, + { 0x9E19DB92B4E31BA9, 1013, 324 }, + } + }; + + // This computation gives exactly the same results for k as + // k = ceil((kAlpha - e - 1) * 0.30102999566398114) + // for |e| <= 1500, but doesn't require floating-point operations. + // NB: log_10(2) ~= 78913 / 2^18 + JSON_ASSERT(e >= -1500); + JSON_ASSERT(e <= 1500); + const int f = kAlpha - e - 1; + const int k = (f * 78913) / (1 << 18) + static_cast(f > 0); + + const int index = (-kCachedPowersMinDecExp + k + (kCachedPowersDecStep - 1)) / kCachedPowersDecStep; + JSON_ASSERT(index >= 0); + JSON_ASSERT(static_cast(index) < kCachedPowers.size()); + + const cached_power cached = kCachedPowers[static_cast(index)]; + JSON_ASSERT(kAlpha <= cached.e + e + 64); + JSON_ASSERT(kGamma >= cached.e + e + 64); + + return cached; +} + +/*! +For n != 0, returns k, such that pow10 := 10^(k-1) <= n < 10^k. +For n == 0, returns 1 and sets pow10 := 1. +*/ +inline int find_largest_pow10(const std::uint32_t n, std::uint32_t& pow10) +{ + // LCOV_EXCL_START + if (n >= 1000000000) + { + pow10 = 1000000000; + return 10; + } + // LCOV_EXCL_STOP + if (n >= 100000000) + { + pow10 = 100000000; + return 9; + } + if (n >= 10000000) + { + pow10 = 10000000; + return 8; + } + if (n >= 1000000) + { + pow10 = 1000000; + return 7; + } + if (n >= 100000) + { + pow10 = 100000; + return 6; + } + if (n >= 10000) + { + pow10 = 10000; + return 5; + } + if (n >= 1000) + { + pow10 = 1000; + return 4; + } + if (n >= 100) + { + pow10 = 100; + return 3; + } + if (n >= 10) + { + pow10 = 10; + return 2; + } + + pow10 = 1; + return 1; +} + +inline void grisu2_round(char* buf, int len, std::uint64_t dist, std::uint64_t delta, + std::uint64_t rest, std::uint64_t ten_k) +{ + JSON_ASSERT(len >= 1); + JSON_ASSERT(dist <= delta); + JSON_ASSERT(rest <= delta); + JSON_ASSERT(ten_k > 0); + + // <--------------------------- delta ----> + // <---- dist ---------> + // --------------[------------------+-------------------]-------------- + // M- w M+ + // + // ten_k + // <------> + // <---- rest ----> + // --------------[------------------+----+--------------]-------------- + // w V + // = buf * 10^k + // + // ten_k represents a unit-in-the-last-place in the decimal representation + // stored in buf. + // Decrement buf by ten_k while this takes buf closer to w. + + // The tests are written in this order to avoid overflow in unsigned + // integer arithmetic. + + while (rest < dist + && delta - rest >= ten_k + && (rest + ten_k < dist || dist - rest > rest + ten_k - dist)) + { + JSON_ASSERT(buf[len - 1] != '0'); + buf[len - 1]--; + rest += ten_k; + } +} + +/*! +Generates V = buffer * 10^decimal_exponent, such that M- <= V <= M+. +M- and M+ must be normalized and share the same exponent -60 <= e <= -32. +*/ +inline void grisu2_digit_gen(char* buffer, int& length, int& decimal_exponent, + diyfp M_minus, diyfp w, diyfp M_plus) +{ + static_assert(kAlpha >= -60, "internal error"); + static_assert(kGamma <= -32, "internal error"); + + // Generates the digits (and the exponent) of a decimal floating-point + // number V = buffer * 10^decimal_exponent in the range [M-, M+]. The diyfp's + // w, M- and M+ share the same exponent e, which satisfies alpha <= e <= gamma. + // + // <--------------------------- delta ----> + // <---- dist ---------> + // --------------[------------------+-------------------]-------------- + // M- w M+ + // + // Grisu2 generates the digits of M+ from left to right and stops as soon as + // V is in [M-,M+]. + + JSON_ASSERT(M_plus.e >= kAlpha); + JSON_ASSERT(M_plus.e <= kGamma); + + std::uint64_t delta = diyfp::sub(M_plus, M_minus).f; // (significand of (M+ - M-), implicit exponent is e) + std::uint64_t dist = diyfp::sub(M_plus, w ).f; // (significand of (M+ - w ), implicit exponent is e) + + // Split M+ = f * 2^e into two parts p1 and p2 (note: e < 0): + // + // M+ = f * 2^e + // = ((f div 2^-e) * 2^-e + (f mod 2^-e)) * 2^e + // = ((p1 ) * 2^-e + (p2 )) * 2^e + // = p1 + p2 * 2^e + + const diyfp one(std::uint64_t{1} << -M_plus.e, M_plus.e); + + auto p1 = static_cast(M_plus.f >> -one.e); // p1 = f div 2^-e (Since -e >= 32, p1 fits into a 32-bit int.) + std::uint64_t p2 = M_plus.f & (one.f - 1); // p2 = f mod 2^-e + + // 1) + // + // Generate the digits of the integral part p1 = d[n-1]...d[1]d[0] + + JSON_ASSERT(p1 > 0); + + std::uint32_t pow10{}; + const int k = find_largest_pow10(p1, pow10); + + // 10^(k-1) <= p1 < 10^k, pow10 = 10^(k-1) + // + // p1 = (p1 div 10^(k-1)) * 10^(k-1) + (p1 mod 10^(k-1)) + // = (d[k-1] ) * 10^(k-1) + (p1 mod 10^(k-1)) + // + // M+ = p1 + p2 * 2^e + // = d[k-1] * 10^(k-1) + (p1 mod 10^(k-1)) + p2 * 2^e + // = d[k-1] * 10^(k-1) + ((p1 mod 10^(k-1)) * 2^-e + p2) * 2^e + // = d[k-1] * 10^(k-1) + ( rest) * 2^e + // + // Now generate the digits d[n] of p1 from left to right (n = k-1,...,0) + // + // p1 = d[k-1]...d[n] * 10^n + d[n-1]...d[0] + // + // but stop as soon as + // + // rest * 2^e = (d[n-1]...d[0] * 2^-e + p2) * 2^e <= delta * 2^e + + int n = k; + while (n > 0) + { + // Invariants: + // M+ = buffer * 10^n + (p1 + p2 * 2^e) (buffer = 0 for n = k) + // pow10 = 10^(n-1) <= p1 < 10^n + // + const std::uint32_t d = p1 / pow10; // d = p1 div 10^(n-1) + const std::uint32_t r = p1 % pow10; // r = p1 mod 10^(n-1) + // + // M+ = buffer * 10^n + (d * 10^(n-1) + r) + p2 * 2^e + // = (buffer * 10 + d) * 10^(n-1) + (r + p2 * 2^e) + // + JSON_ASSERT(d <= 9); + buffer[length++] = static_cast('0' + d); // buffer := buffer * 10 + d + // + // M+ = buffer * 10^(n-1) + (r + p2 * 2^e) + // + p1 = r; + n--; + // + // M+ = buffer * 10^n + (p1 + p2 * 2^e) + // pow10 = 10^n + // + + // Now check if enough digits have been generated. + // Compute + // + // p1 + p2 * 2^e = (p1 * 2^-e + p2) * 2^e = rest * 2^e + // + // Note: + // Since rest and delta share the same exponent e, it suffices to + // compare the significands. + const std::uint64_t rest = (std::uint64_t{p1} << -one.e) + p2; + if (rest <= delta) + { + // V = buffer * 10^n, with M- <= V <= M+. + + decimal_exponent += n; + + // We may now just stop. But instead look if the buffer could be + // decremented to bring V closer to w. + // + // pow10 = 10^n is now 1 ulp in the decimal representation V. + // The rounding procedure works with diyfp's with an implicit + // exponent of e. + // + // 10^n = (10^n * 2^-e) * 2^e = ulp * 2^e + // + const std::uint64_t ten_n = std::uint64_t{pow10} << -one.e; + grisu2_round(buffer, length, dist, delta, rest, ten_n); + + return; + } + + pow10 /= 10; + // + // pow10 = 10^(n-1) <= p1 < 10^n + // Invariants restored. + } + + // 2) + // + // The digits of the integral part have been generated: + // + // M+ = d[k-1]...d[1]d[0] + p2 * 2^e + // = buffer + p2 * 2^e + // + // Now generate the digits of the fractional part p2 * 2^e. + // + // Note: + // No decimal point is generated: the exponent is adjusted instead. + // + // p2 actually represents the fraction + // + // p2 * 2^e + // = p2 / 2^-e + // = d[-1] / 10^1 + d[-2] / 10^2 + ... + // + // Now generate the digits d[-m] of p1 from left to right (m = 1,2,...) + // + // p2 * 2^e = d[-1]d[-2]...d[-m] * 10^-m + // + 10^-m * (d[-m-1] / 10^1 + d[-m-2] / 10^2 + ...) + // + // using + // + // 10^m * p2 = ((10^m * p2) div 2^-e) * 2^-e + ((10^m * p2) mod 2^-e) + // = ( d) * 2^-e + ( r) + // + // or + // 10^m * p2 * 2^e = d + r * 2^e + // + // i.e. + // + // M+ = buffer + p2 * 2^e + // = buffer + 10^-m * (d + r * 2^e) + // = (buffer * 10^m + d) * 10^-m + 10^-m * r * 2^e + // + // and stop as soon as 10^-m * r * 2^e <= delta * 2^e + + JSON_ASSERT(p2 > delta); + + int m = 0; + for (;;) + { + // Invariant: + // M+ = buffer * 10^-m + 10^-m * (d[-m-1] / 10 + d[-m-2] / 10^2 + ...) * 2^e + // = buffer * 10^-m + 10^-m * (p2 ) * 2^e + // = buffer * 10^-m + 10^-m * (1/10 * (10 * p2) ) * 2^e + // = buffer * 10^-m + 10^-m * (1/10 * ((10*p2 div 2^-e) * 2^-e + (10*p2 mod 2^-e)) * 2^e + // + JSON_ASSERT(p2 <= (std::numeric_limits::max)() / 10); + p2 *= 10; + const std::uint64_t d = p2 >> -one.e; // d = (10 * p2) div 2^-e + const std::uint64_t r = p2 & (one.f - 1); // r = (10 * p2) mod 2^-e + // + // M+ = buffer * 10^-m + 10^-m * (1/10 * (d * 2^-e + r) * 2^e + // = buffer * 10^-m + 10^-m * (1/10 * (d + r * 2^e)) + // = (buffer * 10 + d) * 10^(-m-1) + 10^(-m-1) * r * 2^e + // + JSON_ASSERT(d <= 9); + buffer[length++] = static_cast('0' + d); // buffer := buffer * 10 + d + // + // M+ = buffer * 10^(-m-1) + 10^(-m-1) * r * 2^e + // + p2 = r; + m++; + // + // M+ = buffer * 10^-m + 10^-m * p2 * 2^e + // Invariant restored. + + // Check if enough digits have been generated. + // + // 10^-m * p2 * 2^e <= delta * 2^e + // p2 * 2^e <= 10^m * delta * 2^e + // p2 <= 10^m * delta + delta *= 10; + dist *= 10; + if (p2 <= delta) + { + break; + } + } + + // V = buffer * 10^-m, with M- <= V <= M+. + + decimal_exponent -= m; + + // 1 ulp in the decimal representation is now 10^-m. + // Since delta and dist are now scaled by 10^m, we need to do the + // same with ulp in order to keep the units in sync. + // + // 10^m * 10^-m = 1 = 2^-e * 2^e = ten_m * 2^e + // + const std::uint64_t ten_m = one.f; + grisu2_round(buffer, length, dist, delta, p2, ten_m); + + // By construction this algorithm generates the shortest possible decimal + // number (Loitsch, Theorem 6.2) which rounds back to w. + // For an input number of precision p, at least + // + // N = 1 + ceil(p * log_10(2)) + // + // decimal digits are sufficient to identify all binary floating-point + // numbers (Matula, "In-and-Out conversions"). + // This implies that the algorithm does not produce more than N decimal + // digits. + // + // N = 17 for p = 53 (IEEE double precision) + // N = 9 for p = 24 (IEEE single precision) +} + +/*! +v = buf * 10^decimal_exponent +len is the length of the buffer (number of decimal digits) +The buffer must be large enough, i.e. >= max_digits10. +*/ +JSON_HEDLEY_NON_NULL(1) +inline void grisu2(char* buf, int& len, int& decimal_exponent, + diyfp m_minus, diyfp v, diyfp m_plus) +{ + JSON_ASSERT(m_plus.e == m_minus.e); + JSON_ASSERT(m_plus.e == v.e); + + // --------(-----------------------+-----------------------)-------- (A) + // m- v m+ + // + // --------------------(-----------+-----------------------)-------- (B) + // m- v m+ + // + // First scale v (and m- and m+) such that the exponent is in the range + // [alpha, gamma]. + + const cached_power cached = get_cached_power_for_binary_exponent(m_plus.e); + + const diyfp c_minus_k(cached.f, cached.e); // = c ~= 10^-k + + // The exponent of the products is = v.e + c_minus_k.e + q and is in the range [alpha,gamma] + const diyfp w = diyfp::mul(v, c_minus_k); + const diyfp w_minus = diyfp::mul(m_minus, c_minus_k); + const diyfp w_plus = diyfp::mul(m_plus, c_minus_k); + + // ----(---+---)---------------(---+---)---------------(---+---)---- + // w- w w+ + // = c*m- = c*v = c*m+ + // + // diyfp::mul rounds its result and c_minus_k is approximated too. w, w- and + // w+ are now off by a small amount. + // In fact: + // + // w - v * 10^k < 1 ulp + // + // To account for this inaccuracy, add resp. subtract 1 ulp. + // + // --------+---[---------------(---+---)---------------]---+-------- + // w- M- w M+ w+ + // + // Now any number in [M-, M+] (bounds included) will round to w when input, + // regardless of how the input rounding algorithm breaks ties. + // + // And digit_gen generates the shortest possible such number in [M-, M+]. + // Note that this does not mean that Grisu2 always generates the shortest + // possible number in the interval (m-, m+). + const diyfp M_minus(w_minus.f + 1, w_minus.e); + const diyfp M_plus (w_plus.f - 1, w_plus.e ); + + decimal_exponent = -cached.k; // = -(-k) = k + + grisu2_digit_gen(buf, len, decimal_exponent, M_minus, w, M_plus); +} + +/*! +v = buf * 10^decimal_exponent +len is the length of the buffer (number of decimal digits) +The buffer must be large enough, i.e. >= max_digits10. +*/ +template +JSON_HEDLEY_NON_NULL(1) +void grisu2(char* buf, int& len, int& decimal_exponent, FloatType value) +{ + static_assert(diyfp::kPrecision >= std::numeric_limits::digits + 3, + "internal error: not enough precision"); + + JSON_ASSERT(std::isfinite(value)); + JSON_ASSERT(value > 0); + + // If the neighbors (and boundaries) of 'value' are always computed for double-precision + // numbers, all float's can be recovered using strtod (and strtof). However, the resulting + // decimal representations are not exactly "short". + // + // The documentation for 'std::to_chars' (https://en.cppreference.com/w/cpp/utility/to_chars) + // says "value is converted to a string as if by std::sprintf in the default ("C") locale" + // and since sprintf promotes floats to doubles, I think this is exactly what 'std::to_chars' + // does. + // On the other hand, the documentation for 'std::to_chars' requires that "parsing the + // representation using the corresponding std::from_chars function recovers value exactly". That + // indicates that single precision floating-point numbers should be recovered using + // 'std::strtof'. + // + // NB: If the neighbors are computed for single-precision numbers, there is a single float + // (7.0385307e-26f) which can't be recovered using strtod. The resulting double precision + // value is off by 1 ulp. +#if 0 + const boundaries w = compute_boundaries(static_cast(value)); +#else + const boundaries w = compute_boundaries(value); +#endif + + grisu2(buf, len, decimal_exponent, w.minus, w.w, w.plus); +} + +/*! +@brief appends a decimal representation of e to buf +@return a pointer to the element following the exponent. +@pre -1000 < e < 1000 +*/ +JSON_HEDLEY_NON_NULL(1) +JSON_HEDLEY_RETURNS_NON_NULL +inline char* append_exponent(char* buf, int e) +{ + JSON_ASSERT(e > -1000); + JSON_ASSERT(e < 1000); + + if (e < 0) + { + e = -e; + *buf++ = '-'; + } + else + { + *buf++ = '+'; + } + + auto k = static_cast(e); + if (k < 10) + { + // Always print at least two digits in the exponent. + // This is for compatibility with printf("%g"). + *buf++ = '0'; + *buf++ = static_cast('0' + k); + } + else if (k < 100) + { + *buf++ = static_cast('0' + k / 10); + k %= 10; + *buf++ = static_cast('0' + k); + } + else + { + *buf++ = static_cast('0' + k / 100); + k %= 100; + *buf++ = static_cast('0' + k / 10); + k %= 10; + *buf++ = static_cast('0' + k); + } + + return buf; +} + +/*! +@brief prettify v = buf * 10^decimal_exponent + +If v is in the range [10^min_exp, 10^max_exp) it will be printed in fixed-point +notation. Otherwise it will be printed in exponential notation. + +@pre min_exp < 0 +@pre max_exp > 0 +*/ +JSON_HEDLEY_NON_NULL(1) +JSON_HEDLEY_RETURNS_NON_NULL +inline char* format_buffer(char* buf, int len, int decimal_exponent, + int min_exp, int max_exp) +{ + JSON_ASSERT(min_exp < 0); + JSON_ASSERT(max_exp > 0); + + const int k = len; + const int n = len + decimal_exponent; + + // v = buf * 10^(n-k) + // k is the length of the buffer (number of decimal digits) + // n is the position of the decimal point relative to the start of the buffer. + + if (k <= n && n <= max_exp) + { + // digits[000] + // len <= max_exp + 2 + + std::memset(buf + k, '0', static_cast(n) - static_cast(k)); + // Make it look like a floating-point number (#362, #378) + buf[n + 0] = '.'; + buf[n + 1] = '0'; + return buf + (static_cast(n) + 2); + } + + if (0 < n && n <= max_exp) + { + // dig.its + // len <= max_digits10 + 1 + + JSON_ASSERT(k > n); + + std::memmove(buf + (static_cast(n) + 1), buf + n, static_cast(k) - static_cast(n)); + buf[n] = '.'; + return buf + (static_cast(k) + 1U); + } + + if (min_exp < n && n <= 0) + { + // 0.[000]digits + // len <= 2 + (-min_exp - 1) + max_digits10 + + std::memmove(buf + (2 + static_cast(-n)), buf, static_cast(k)); + buf[0] = '0'; + buf[1] = '.'; + std::memset(buf + 2, '0', static_cast(-n)); + return buf + (2U + static_cast(-n) + static_cast(k)); + } + + if (k == 1) + { + // dE+123 + // len <= 1 + 5 + + buf += 1; + } + else + { + // d.igitsE+123 + // len <= max_digits10 + 1 + 5 + + std::memmove(buf + 2, buf + 1, static_cast(k) - 1); + buf[1] = '.'; + buf += 1 + static_cast(k); + } + + *buf++ = 'e'; + return append_exponent(buf, n - 1); +} + +} // namespace dtoa_impl + +/*! +@brief generates a decimal representation of the floating-point number value in [first, last). + +The format of the resulting decimal representation is similar to printf's %g +format. Returns an iterator pointing past-the-end of the decimal representation. + +@note The input number must be finite, i.e. NaN's and Inf's are not supported. +@note The buffer must be large enough. +@note The result is NOT null-terminated. +*/ +template +JSON_HEDLEY_NON_NULL(1, 2) +JSON_HEDLEY_RETURNS_NON_NULL +char* to_chars(char* first, const char* last, FloatType value) +{ + static_cast(last); // maybe unused - fix warning + JSON_ASSERT(std::isfinite(value)); + + // Use signbit(value) instead of (value < 0) since signbit works for -0. + if (std::signbit(value)) + { + value = -value; + *first++ = '-'; + } + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wfloat-equal" +#endif + if (value == 0) // +-0 + { + *first++ = '0'; + // Make it look like a floating-point number (#362, #378) + *first++ = '.'; + *first++ = '0'; + return first; + } +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif + + JSON_ASSERT(last - first >= std::numeric_limits::max_digits10); + + // Compute v = buffer * 10^decimal_exponent. + // The decimal digits are stored in the buffer, which needs to be interpreted + // as an unsigned decimal integer. + // len is the length of the buffer, i.e. the number of decimal digits. + int len = 0; + int decimal_exponent = 0; + dtoa_impl::grisu2(first, len, decimal_exponent, value); + + JSON_ASSERT(len <= std::numeric_limits::max_digits10); + + // Format the buffer like printf("%.*g", prec, value) + constexpr int kMinExp = -4; + // Use digits10 here to increase compatibility with version 2. + constexpr int kMaxExp = std::numeric_limits::digits10; + + JSON_ASSERT(last - first >= kMaxExp + 2); + JSON_ASSERT(last - first >= 2 + (-kMinExp - 1) + std::numeric_limits::max_digits10); + JSON_ASSERT(last - first >= std::numeric_limits::max_digits10 + 6); + + return dtoa_impl::format_buffer(first, len, decimal_exponent, kMinExp, kMaxExp); +} + +} // namespace detail +} // namespace nlohmann + +// #include + +// #include + +// #include + +// #include + +// #include + +// #include + + +namespace nlohmann +{ +namespace detail +{ +/////////////////// +// serialization // +/////////////////// + +/// how to treat decoding errors +enum class error_handler_t +{ + strict, ///< throw a type_error exception in case of invalid UTF-8 + replace, ///< replace invalid UTF-8 sequences with U+FFFD + ignore ///< ignore invalid UTF-8 sequences +}; + +template +class serializer +{ + using string_t = typename BasicJsonType::string_t; + using number_float_t = typename BasicJsonType::number_float_t; + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using binary_char_t = typename BasicJsonType::binary_t::value_type; + static constexpr std::uint8_t UTF8_ACCEPT = 0; + static constexpr std::uint8_t UTF8_REJECT = 1; + + public: + /*! + @param[in] s output stream to serialize to + @param[in] ichar indentation character to use + @param[in] error_handler_ how to react on decoding errors + */ + serializer(output_adapter_t s, const char ichar, + error_handler_t error_handler_ = error_handler_t::strict) + : o(std::move(s)) + , loc(std::localeconv()) + , thousands_sep(loc->thousands_sep == nullptr ? '\0' : std::char_traits::to_char_type(* (loc->thousands_sep))) + , decimal_point(loc->decimal_point == nullptr ? '\0' : std::char_traits::to_char_type(* (loc->decimal_point))) + , indent_char(ichar) + , indent_string(512, indent_char) + , error_handler(error_handler_) + {} + + // delete because of pointer members + serializer(const serializer&) = delete; + serializer& operator=(const serializer&) = delete; + serializer(serializer&&) = delete; + serializer& operator=(serializer&&) = delete; + ~serializer() = default; + + /*! + @brief internal implementation of the serialization function + + This function is called by the public member function dump and organizes + the serialization internally. The indentation level is propagated as + additional parameter. In case of arrays and objects, the function is + called recursively. + + - strings and object keys are escaped using `escape_string()` + - integer numbers are converted implicitly via `operator<<` + - floating-point numbers are converted to a string using `"%g"` format + - binary values are serialized as objects containing the subtype and the + byte array + + @param[in] val value to serialize + @param[in] pretty_print whether the output shall be pretty-printed + @param[in] ensure_ascii If @a ensure_ascii is true, all non-ASCII characters + in the output are escaped with `\uXXXX` sequences, and the result consists + of ASCII characters only. + @param[in] indent_step the indent level + @param[in] current_indent the current indent level (only used internally) + */ + void dump(const BasicJsonType& val, + const bool pretty_print, + const bool ensure_ascii, + const unsigned int indent_step, + const unsigned int current_indent = 0) + { + switch (val.m_type) + { + case value_t::object: + { + if (val.m_value.object->empty()) + { + o->write_characters("{}", 2); + return; + } + + if (pretty_print) + { + o->write_characters("{\n", 2); + + // variable to hold indentation for recursive calls + const auto new_indent = current_indent + indent_step; + if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent)) + { + indent_string.resize(indent_string.size() * 2, ' '); + } + + // first n-1 elements + auto i = val.m_value.object->cbegin(); + for (std::size_t cnt = 0; cnt < val.m_value.object->size() - 1; ++cnt, ++i) + { + o->write_characters(indent_string.c_str(), new_indent); + o->write_character('\"'); + dump_escaped(i->first, ensure_ascii); + o->write_characters("\": ", 3); + dump(i->second, true, ensure_ascii, indent_step, new_indent); + o->write_characters(",\n", 2); + } + + // last element + JSON_ASSERT(i != val.m_value.object->cend()); + JSON_ASSERT(std::next(i) == val.m_value.object->cend()); + o->write_characters(indent_string.c_str(), new_indent); + o->write_character('\"'); + dump_escaped(i->first, ensure_ascii); + o->write_characters("\": ", 3); + dump(i->second, true, ensure_ascii, indent_step, new_indent); + + o->write_character('\n'); + o->write_characters(indent_string.c_str(), current_indent); + o->write_character('}'); + } + else + { + o->write_character('{'); + + // first n-1 elements + auto i = val.m_value.object->cbegin(); + for (std::size_t cnt = 0; cnt < val.m_value.object->size() - 1; ++cnt, ++i) + { + o->write_character('\"'); + dump_escaped(i->first, ensure_ascii); + o->write_characters("\":", 2); + dump(i->second, false, ensure_ascii, indent_step, current_indent); + o->write_character(','); + } + + // last element + JSON_ASSERT(i != val.m_value.object->cend()); + JSON_ASSERT(std::next(i) == val.m_value.object->cend()); + o->write_character('\"'); + dump_escaped(i->first, ensure_ascii); + o->write_characters("\":", 2); + dump(i->second, false, ensure_ascii, indent_step, current_indent); + + o->write_character('}'); + } + + return; + } + + case value_t::array: + { + if (val.m_value.array->empty()) + { + o->write_characters("[]", 2); + return; + } + + if (pretty_print) + { + o->write_characters("[\n", 2); + + // variable to hold indentation for recursive calls + const auto new_indent = current_indent + indent_step; + if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent)) + { + indent_string.resize(indent_string.size() * 2, ' '); + } + + // first n-1 elements + for (auto i = val.m_value.array->cbegin(); + i != val.m_value.array->cend() - 1; ++i) + { + o->write_characters(indent_string.c_str(), new_indent); + dump(*i, true, ensure_ascii, indent_step, new_indent); + o->write_characters(",\n", 2); + } + + // last element + JSON_ASSERT(!val.m_value.array->empty()); + o->write_characters(indent_string.c_str(), new_indent); + dump(val.m_value.array->back(), true, ensure_ascii, indent_step, new_indent); + + o->write_character('\n'); + o->write_characters(indent_string.c_str(), current_indent); + o->write_character(']'); + } + else + { + o->write_character('['); + + // first n-1 elements + for (auto i = val.m_value.array->cbegin(); + i != val.m_value.array->cend() - 1; ++i) + { + dump(*i, false, ensure_ascii, indent_step, current_indent); + o->write_character(','); + } + + // last element + JSON_ASSERT(!val.m_value.array->empty()); + dump(val.m_value.array->back(), false, ensure_ascii, indent_step, current_indent); + + o->write_character(']'); + } + + return; + } + + case value_t::string: + { + o->write_character('\"'); + dump_escaped(*val.m_value.string, ensure_ascii); + o->write_character('\"'); + return; + } + + case value_t::binary: + { + if (pretty_print) + { + o->write_characters("{\n", 2); + + // variable to hold indentation for recursive calls + const auto new_indent = current_indent + indent_step; + if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent)) + { + indent_string.resize(indent_string.size() * 2, ' '); + } + + o->write_characters(indent_string.c_str(), new_indent); + + o->write_characters("\"bytes\": [", 10); + + if (!val.m_value.binary->empty()) + { + for (auto i = val.m_value.binary->cbegin(); + i != val.m_value.binary->cend() - 1; ++i) + { + dump_integer(*i); + o->write_characters(", ", 2); + } + dump_integer(val.m_value.binary->back()); + } + + o->write_characters("],\n", 3); + o->write_characters(indent_string.c_str(), new_indent); + + o->write_characters("\"subtype\": ", 11); + if (val.m_value.binary->has_subtype()) + { + dump_integer(val.m_value.binary->subtype()); + } + else + { + o->write_characters("null", 4); + } + o->write_character('\n'); + o->write_characters(indent_string.c_str(), current_indent); + o->write_character('}'); + } + else + { + o->write_characters("{\"bytes\":[", 10); + + if (!val.m_value.binary->empty()) + { + for (auto i = val.m_value.binary->cbegin(); + i != val.m_value.binary->cend() - 1; ++i) + { + dump_integer(*i); + o->write_character(','); + } + dump_integer(val.m_value.binary->back()); + } + + o->write_characters("],\"subtype\":", 12); + if (val.m_value.binary->has_subtype()) + { + dump_integer(val.m_value.binary->subtype()); + o->write_character('}'); + } + else + { + o->write_characters("null}", 5); + } + } + return; + } + + case value_t::boolean: + { + if (val.m_value.boolean) + { + o->write_characters("true", 4); + } + else + { + o->write_characters("false", 5); + } + return; + } + + case value_t::number_integer: + { + dump_integer(val.m_value.number_integer); + return; + } + + case value_t::number_unsigned: + { + dump_integer(val.m_value.number_unsigned); + return; + } + + case value_t::number_float: + { + dump_float(val.m_value.number_float); + return; + } + + case value_t::discarded: + { + o->write_characters("", 11); + return; + } + + case value_t::null: + { + o->write_characters("null", 4); + return; + } + + default: // LCOV_EXCL_LINE + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE + } + } + + JSON_PRIVATE_UNLESS_TESTED: + /*! + @brief dump escaped string + + Escape a string by replacing certain special characters by a sequence of an + escape character (backslash) and another character and other control + characters by a sequence of "\u" followed by a four-digit hex + representation. The escaped string is written to output stream @a o. + + @param[in] s the string to escape + @param[in] ensure_ascii whether to escape non-ASCII characters with + \uXXXX sequences + + @complexity Linear in the length of string @a s. + */ + void dump_escaped(const string_t& s, const bool ensure_ascii) + { + std::uint32_t codepoint{}; + std::uint8_t state = UTF8_ACCEPT; + std::size_t bytes = 0; // number of bytes written to string_buffer + + // number of bytes written at the point of the last valid byte + std::size_t bytes_after_last_accept = 0; + std::size_t undumped_chars = 0; + + for (std::size_t i = 0; i < s.size(); ++i) + { + const auto byte = static_cast(s[i]); + + switch (decode(state, codepoint, byte)) + { + case UTF8_ACCEPT: // decode found a new code point + { + switch (codepoint) + { + case 0x08: // backspace + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'b'; + break; + } + + case 0x09: // horizontal tab + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 't'; + break; + } + + case 0x0A: // newline + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'n'; + break; + } + + case 0x0C: // formfeed + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'f'; + break; + } + + case 0x0D: // carriage return + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'r'; + break; + } + + case 0x22: // quotation mark + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = '\"'; + break; + } + + case 0x5C: // reverse solidus + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = '\\'; + break; + } + + default: + { + // escape control characters (0x00..0x1F) or, if + // ensure_ascii parameter is used, non-ASCII characters + if ((codepoint <= 0x1F) || (ensure_ascii && (codepoint >= 0x7F))) + { + if (codepoint <= 0xFFFF) + { + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,hicpp-vararg) + static_cast((std::snprintf)(string_buffer.data() + bytes, 7, "\\u%04x", + static_cast(codepoint))); + bytes += 6; + } + else + { + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,hicpp-vararg) + static_cast((std::snprintf)(string_buffer.data() + bytes, 13, "\\u%04x\\u%04x", + static_cast(0xD7C0u + (codepoint >> 10u)), + static_cast(0xDC00u + (codepoint & 0x3FFu)))); + bytes += 12; + } + } + else + { + // copy byte to buffer (all previous bytes + // been copied have in default case above) + string_buffer[bytes++] = s[i]; + } + break; + } + } + + // write buffer and reset index; there must be 13 bytes + // left, as this is the maximal number of bytes to be + // written ("\uxxxx\uxxxx\0") for one code point + if (string_buffer.size() - bytes < 13) + { + o->write_characters(string_buffer.data(), bytes); + bytes = 0; + } + + // remember the byte position of this accept + bytes_after_last_accept = bytes; + undumped_chars = 0; + break; + } + + case UTF8_REJECT: // decode found invalid UTF-8 byte + { + switch (error_handler) + { + case error_handler_t::strict: + { + std::stringstream ss; + ss << std::uppercase << std::setfill('0') << std::setw(2) << std::hex << (byte | 0); + JSON_THROW(type_error::create(316, "invalid UTF-8 byte at index " + std::to_string(i) + ": 0x" + ss.str(), BasicJsonType())); + } + + case error_handler_t::ignore: + case error_handler_t::replace: + { + // in case we saw this character the first time, we + // would like to read it again, because the byte + // may be OK for itself, but just not OK for the + // previous sequence + if (undumped_chars > 0) + { + --i; + } + + // reset length buffer to the last accepted index; + // thus removing/ignoring the invalid characters + bytes = bytes_after_last_accept; + + if (error_handler == error_handler_t::replace) + { + // add a replacement character + if (ensure_ascii) + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'u'; + string_buffer[bytes++] = 'f'; + string_buffer[bytes++] = 'f'; + string_buffer[bytes++] = 'f'; + string_buffer[bytes++] = 'd'; + } + else + { + string_buffer[bytes++] = detail::binary_writer::to_char_type('\xEF'); + string_buffer[bytes++] = detail::binary_writer::to_char_type('\xBF'); + string_buffer[bytes++] = detail::binary_writer::to_char_type('\xBD'); + } + + // write buffer and reset index; there must be 13 bytes + // left, as this is the maximal number of bytes to be + // written ("\uxxxx\uxxxx\0") for one code point + if (string_buffer.size() - bytes < 13) + { + o->write_characters(string_buffer.data(), bytes); + bytes = 0; + } + + bytes_after_last_accept = bytes; + } + + undumped_chars = 0; + + // continue processing the string + state = UTF8_ACCEPT; + break; + } + + default: // LCOV_EXCL_LINE + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE + } + break; + } + + default: // decode found yet incomplete multi-byte code point + { + if (!ensure_ascii) + { + // code point will not be escaped - copy byte to buffer + string_buffer[bytes++] = s[i]; + } + ++undumped_chars; + break; + } + } + } + + // we finished processing the string + if (JSON_HEDLEY_LIKELY(state == UTF8_ACCEPT)) + { + // write buffer + if (bytes > 0) + { + o->write_characters(string_buffer.data(), bytes); + } + } + else + { + // we finish reading, but do not accept: string was incomplete + switch (error_handler) + { + case error_handler_t::strict: + { + std::stringstream ss; + ss << std::uppercase << std::setfill('0') << std::setw(2) << std::hex << (static_cast(s.back()) | 0); + JSON_THROW(type_error::create(316, "incomplete UTF-8 string; last byte: 0x" + ss.str(), BasicJsonType())); + } + + case error_handler_t::ignore: + { + // write all accepted bytes + o->write_characters(string_buffer.data(), bytes_after_last_accept); + break; + } + + case error_handler_t::replace: + { + // write all accepted bytes + o->write_characters(string_buffer.data(), bytes_after_last_accept); + // add a replacement character + if (ensure_ascii) + { + o->write_characters("\\ufffd", 6); + } + else + { + o->write_characters("\xEF\xBF\xBD", 3); + } + break; + } + + default: // LCOV_EXCL_LINE + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE + } + } + } + + private: + /*! + @brief count digits + + Count the number of decimal (base 10) digits for an input unsigned integer. + + @param[in] x unsigned integer number to count its digits + @return number of decimal digits + */ + inline unsigned int count_digits(number_unsigned_t x) noexcept + { + unsigned int n_digits = 1; + for (;;) + { + if (x < 10) + { + return n_digits; + } + if (x < 100) + { + return n_digits + 1; + } + if (x < 1000) + { + return n_digits + 2; + } + if (x < 10000) + { + return n_digits + 3; + } + x = x / 10000u; + n_digits += 4; + } + } + + // templates to avoid warnings about useless casts + template ::value, int> = 0> + bool is_negative_number(NumberType x) + { + return x < 0; + } + + template < typename NumberType, enable_if_t ::value, int > = 0 > + bool is_negative_number(NumberType /*unused*/) + { + return false; + } + + /*! + @brief dump an integer + + Dump a given integer to output stream @a o. Works internally with + @a number_buffer. + + @param[in] x integer number (signed or unsigned) to dump + @tparam NumberType either @a number_integer_t or @a number_unsigned_t + */ + template < typename NumberType, detail::enable_if_t < + std::is_integral::value || + std::is_same::value || + std::is_same::value || + std::is_same::value, + int > = 0 > + void dump_integer(NumberType x) + { + static constexpr std::array, 100> digits_to_99 + { + { + {{'0', '0'}}, {{'0', '1'}}, {{'0', '2'}}, {{'0', '3'}}, {{'0', '4'}}, {{'0', '5'}}, {{'0', '6'}}, {{'0', '7'}}, {{'0', '8'}}, {{'0', '9'}}, + {{'1', '0'}}, {{'1', '1'}}, {{'1', '2'}}, {{'1', '3'}}, {{'1', '4'}}, {{'1', '5'}}, {{'1', '6'}}, {{'1', '7'}}, {{'1', '8'}}, {{'1', '9'}}, + {{'2', '0'}}, {{'2', '1'}}, {{'2', '2'}}, {{'2', '3'}}, {{'2', '4'}}, {{'2', '5'}}, {{'2', '6'}}, {{'2', '7'}}, {{'2', '8'}}, {{'2', '9'}}, + {{'3', '0'}}, {{'3', '1'}}, {{'3', '2'}}, {{'3', '3'}}, {{'3', '4'}}, {{'3', '5'}}, {{'3', '6'}}, {{'3', '7'}}, {{'3', '8'}}, {{'3', '9'}}, + {{'4', '0'}}, {{'4', '1'}}, {{'4', '2'}}, {{'4', '3'}}, {{'4', '4'}}, {{'4', '5'}}, {{'4', '6'}}, {{'4', '7'}}, {{'4', '8'}}, {{'4', '9'}}, + {{'5', '0'}}, {{'5', '1'}}, {{'5', '2'}}, {{'5', '3'}}, {{'5', '4'}}, {{'5', '5'}}, {{'5', '6'}}, {{'5', '7'}}, {{'5', '8'}}, {{'5', '9'}}, + {{'6', '0'}}, {{'6', '1'}}, {{'6', '2'}}, {{'6', '3'}}, {{'6', '4'}}, {{'6', '5'}}, {{'6', '6'}}, {{'6', '7'}}, {{'6', '8'}}, {{'6', '9'}}, + {{'7', '0'}}, {{'7', '1'}}, {{'7', '2'}}, {{'7', '3'}}, {{'7', '4'}}, {{'7', '5'}}, {{'7', '6'}}, {{'7', '7'}}, {{'7', '8'}}, {{'7', '9'}}, + {{'8', '0'}}, {{'8', '1'}}, {{'8', '2'}}, {{'8', '3'}}, {{'8', '4'}}, {{'8', '5'}}, {{'8', '6'}}, {{'8', '7'}}, {{'8', '8'}}, {{'8', '9'}}, + {{'9', '0'}}, {{'9', '1'}}, {{'9', '2'}}, {{'9', '3'}}, {{'9', '4'}}, {{'9', '5'}}, {{'9', '6'}}, {{'9', '7'}}, {{'9', '8'}}, {{'9', '9'}}, + } + }; + + // special case for "0" + if (x == 0) + { + o->write_character('0'); + return; + } + + // use a pointer to fill the buffer + auto buffer_ptr = number_buffer.begin(); // NOLINT(llvm-qualified-auto,readability-qualified-auto,cppcoreguidelines-pro-type-vararg,hicpp-vararg) + + number_unsigned_t abs_value; + + unsigned int n_chars{}; + + if (is_negative_number(x)) + { + *buffer_ptr = '-'; + abs_value = remove_sign(static_cast(x)); + + // account one more byte for the minus sign + n_chars = 1 + count_digits(abs_value); + } + else + { + abs_value = static_cast(x); + n_chars = count_digits(abs_value); + } + + // spare 1 byte for '\0' + JSON_ASSERT(n_chars < number_buffer.size() - 1); + + // jump to the end to generate the string from backward, + // so we later avoid reversing the result + buffer_ptr += n_chars; + + // Fast int2ascii implementation inspired by "Fastware" talk by Andrei Alexandrescu + // See: https://www.youtube.com/watch?v=o4-CwDo2zpg + while (abs_value >= 100) + { + const auto digits_index = static_cast((abs_value % 100)); + abs_value /= 100; + *(--buffer_ptr) = digits_to_99[digits_index][1]; + *(--buffer_ptr) = digits_to_99[digits_index][0]; + } + + if (abs_value >= 10) + { + const auto digits_index = static_cast(abs_value); + *(--buffer_ptr) = digits_to_99[digits_index][1]; + *(--buffer_ptr) = digits_to_99[digits_index][0]; + } + else + { + *(--buffer_ptr) = static_cast('0' + abs_value); + } + + o->write_characters(number_buffer.data(), n_chars); + } + + /*! + @brief dump a floating-point number + + Dump a given floating-point number to output stream @a o. Works internally + with @a number_buffer. + + @param[in] x floating-point number to dump + */ + void dump_float(number_float_t x) + { + // NaN / inf + if (!std::isfinite(x)) + { + o->write_characters("null", 4); + return; + } + + // If number_float_t is an IEEE-754 single or double precision number, + // use the Grisu2 algorithm to produce short numbers which are + // guaranteed to round-trip, using strtof and strtod, resp. + // + // NB: The test below works if == . + static constexpr bool is_ieee_single_or_double + = (std::numeric_limits::is_iec559 && std::numeric_limits::digits == 24 && std::numeric_limits::max_exponent == 128) || + (std::numeric_limits::is_iec559 && std::numeric_limits::digits == 53 && std::numeric_limits::max_exponent == 1024); + + dump_float(x, std::integral_constant()); + } + + void dump_float(number_float_t x, std::true_type /*is_ieee_single_or_double*/) + { + auto* begin = number_buffer.data(); + auto* end = ::nlohmann::detail::to_chars(begin, begin + number_buffer.size(), x); + + o->write_characters(begin, static_cast(end - begin)); + } + + void dump_float(number_float_t x, std::false_type /*is_ieee_single_or_double*/) + { + // get number of digits for a float -> text -> float round-trip + static constexpr auto d = std::numeric_limits::max_digits10; + + // the actual conversion + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg,hicpp-vararg) + std::ptrdiff_t len = (std::snprintf)(number_buffer.data(), number_buffer.size(), "%.*g", d, x); + + // negative value indicates an error + JSON_ASSERT(len > 0); + // check if buffer was large enough + JSON_ASSERT(static_cast(len) < number_buffer.size()); + + // erase thousands separator + if (thousands_sep != '\0') + { + // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::remove returns an iterator, see https://github.com/nlohmann/json/issues/3081 + const auto end = std::remove(number_buffer.begin(), number_buffer.begin() + len, thousands_sep); + std::fill(end, number_buffer.end(), '\0'); + JSON_ASSERT((end - number_buffer.begin()) <= len); + len = (end - number_buffer.begin()); + } + + // convert decimal point to '.' + if (decimal_point != '\0' && decimal_point != '.') + { + // NOLINTNEXTLINE(readability-qualified-auto,llvm-qualified-auto): std::find returns an iterator, see https://github.com/nlohmann/json/issues/3081 + const auto dec_pos = std::find(number_buffer.begin(), number_buffer.end(), decimal_point); + if (dec_pos != number_buffer.end()) + { + *dec_pos = '.'; + } + } + + o->write_characters(number_buffer.data(), static_cast(len)); + + // determine if we need to append ".0" + const bool value_is_int_like = + std::none_of(number_buffer.begin(), number_buffer.begin() + len + 1, + [](char c) + { + return c == '.' || c == 'e'; + }); + + if (value_is_int_like) + { + o->write_characters(".0", 2); + } + } + + /*! + @brief check whether a string is UTF-8 encoded + + The function checks each byte of a string whether it is UTF-8 encoded. The + result of the check is stored in the @a state parameter. The function must + be called initially with state 0 (accept). State 1 means the string must + be rejected, because the current byte is not allowed. If the string is + completely processed, but the state is non-zero, the string ended + prematurely; that is, the last byte indicated more bytes should have + followed. + + @param[in,out] state the state of the decoding + @param[in,out] codep codepoint (valid only if resulting state is UTF8_ACCEPT) + @param[in] byte next byte to decode + @return new state + + @note The function has been edited: a std::array is used. + + @copyright Copyright (c) 2008-2009 Bjoern Hoehrmann + @sa http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ + */ + static std::uint8_t decode(std::uint8_t& state, std::uint32_t& codep, const std::uint8_t byte) noexcept + { + static const std::array utf8d = + { + { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 00..1F + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 20..3F + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 40..5F + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 60..7F + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, // 80..9F + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, // A0..BF + 8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // C0..DF + 0xA, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x4, 0x3, 0x3, // E0..EF + 0xB, 0x6, 0x6, 0x6, 0x5, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, // F0..FF + 0x0, 0x1, 0x2, 0x3, 0x5, 0x8, 0x7, 0x1, 0x1, 0x1, 0x4, 0x6, 0x1, 0x1, 0x1, 0x1, // s0..s0 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, // s1..s2 + 1, 2, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, // s3..s4 + 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, // s5..s6 + 1, 3, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 // s7..s8 + } + }; + + JSON_ASSERT(byte < utf8d.size()); + const std::uint8_t type = utf8d[byte]; + + codep = (state != UTF8_ACCEPT) + ? (byte & 0x3fu) | (codep << 6u) + : (0xFFu >> type) & (byte); + + std::size_t index = 256u + static_cast(state) * 16u + static_cast(type); + JSON_ASSERT(index < 400); + state = utf8d[index]; + return state; + } + + /* + * Overload to make the compiler happy while it is instantiating + * dump_integer for number_unsigned_t. + * Must never be called. + */ + number_unsigned_t remove_sign(number_unsigned_t x) + { + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE + return x; // LCOV_EXCL_LINE + } + + /* + * Helper function for dump_integer + * + * This function takes a negative signed integer and returns its absolute + * value as unsigned integer. The plus/minus shuffling is necessary as we can + * not directly remove the sign of an arbitrary signed integer as the + * absolute values of INT_MIN and INT_MAX are usually not the same. See + * #1708 for details. + */ + inline number_unsigned_t remove_sign(number_integer_t x) noexcept + { + JSON_ASSERT(x < 0 && x < (std::numeric_limits::max)()); // NOLINT(misc-redundant-expression) + return static_cast(-(x + 1)) + 1; + } + + private: + /// the output of the serializer + output_adapter_t o = nullptr; + + /// a (hopefully) large enough character buffer + std::array number_buffer{{}}; + + /// the locale + const std::lconv* loc = nullptr; + /// the locale's thousand separator character + const char thousands_sep = '\0'; + /// the locale's decimal point character + const char decimal_point = '\0'; + + /// string buffer + std::array string_buffer{{}}; + + /// the indentation character + const char indent_char; + /// the indentation string + string_t indent_string; + + /// error_handler how to react on decoding errors + const error_handler_t error_handler; +}; +} // namespace detail +} // namespace nlohmann + +// #include + +// #include + +// #include + + +#include // less +#include // initializer_list +#include // input_iterator_tag, iterator_traits +#include // allocator +#include // for out_of_range +#include // enable_if, is_convertible +#include // pair +#include // vector + +// #include + + +namespace nlohmann +{ + +/// ordered_map: a minimal map-like container that preserves insertion order +/// for use within nlohmann::basic_json +template , + class Allocator = std::allocator>> + struct ordered_map : std::vector, Allocator> +{ + using key_type = Key; + using mapped_type = T; + using Container = std::vector, Allocator>; + using iterator = typename Container::iterator; + using const_iterator = typename Container::const_iterator; + using size_type = typename Container::size_type; + using value_type = typename Container::value_type; + + // Explicit constructors instead of `using Container::Container` + // otherwise older compilers choke on it (GCC <= 5.5, xcode <= 9.4) + ordered_map(const Allocator& alloc = Allocator()) : Container{alloc} {} + template + ordered_map(It first, It last, const Allocator& alloc = Allocator()) + : Container{first, last, alloc} {} + ordered_map(std::initializer_list init, const Allocator& alloc = Allocator() ) + : Container{init, alloc} {} + + std::pair emplace(const key_type& key, T&& t) + { + for (auto it = this->begin(); it != this->end(); ++it) + { + if (it->first == key) + { + return {it, false}; + } + } + Container::emplace_back(key, t); + return {--this->end(), true}; + } + + T& operator[](const Key& key) + { + return emplace(key, T{}).first->second; + } + + const T& operator[](const Key& key) const + { + return at(key); + } + + T& at(const Key& key) + { + for (auto it = this->begin(); it != this->end(); ++it) + { + if (it->first == key) + { + return it->second; + } + } + + JSON_THROW(std::out_of_range("key not found")); + } + + const T& at(const Key& key) const + { + for (auto it = this->begin(); it != this->end(); ++it) + { + if (it->first == key) + { + return it->second; + } + } + + JSON_THROW(std::out_of_range("key not found")); + } + + size_type erase(const Key& key) + { + for (auto it = this->begin(); it != this->end(); ++it) + { + if (it->first == key) + { + // Since we cannot move const Keys, re-construct them in place + for (auto next = it; ++next != this->end(); ++it) + { + it->~value_type(); // Destroy but keep allocation + new (&*it) value_type{std::move(*next)}; + } + Container::pop_back(); + return 1; + } + } + return 0; + } + + iterator erase(iterator pos) + { + return erase(pos, std::next(pos)); + } + + iterator erase(iterator first, iterator last) + { + const auto elements_affected = std::distance(first, last); + const auto offset = std::distance(Container::begin(), first); + + // This is the start situation. We need to delete elements_affected + // elements (3 in this example: e, f, g), and need to return an + // iterator past the last deleted element (h in this example). + // Note that offset is the distance from the start of the vector + // to first. We will need this later. + + // [ a, b, c, d, e, f, g, h, i, j ] + // ^ ^ + // first last + + // Since we cannot move const Keys, we re-construct them in place. + // We start at first and re-construct (viz. copy) the elements from + // the back of the vector. Example for first iteration: + + // ,--------. + // v | destroy e and re-construct with h + // [ a, b, c, d, e, f, g, h, i, j ] + // ^ ^ + // it it + elements_affected + + for (auto it = first; std::next(it, elements_affected) != Container::end(); ++it) + { + it->~value_type(); // destroy but keep allocation + new (&*it) value_type{std::move(*std::next(it, elements_affected))}; // "move" next element to it + } + + // [ a, b, c, d, h, i, j, h, i, j ] + // ^ ^ + // first last + + // remove the unneeded elements at the end of the vector + Container::resize(this->size() - static_cast(elements_affected)); + + // [ a, b, c, d, h, i, j ] + // ^ ^ + // first last + + // first is now pointing past the last deleted element, but we cannot + // use this iterator, because it may have been invalidated by the + // resize call. Instead, we can return begin() + offset. + return Container::begin() + offset; + } + + size_type count(const Key& key) const + { + for (auto it = this->begin(); it != this->end(); ++it) + { + if (it->first == key) + { + return 1; + } + } + return 0; + } + + iterator find(const Key& key) + { + for (auto it = this->begin(); it != this->end(); ++it) + { + if (it->first == key) + { + return it; + } + } + return Container::end(); + } + + const_iterator find(const Key& key) const + { + for (auto it = this->begin(); it != this->end(); ++it) + { + if (it->first == key) + { + return it; + } + } + return Container::end(); + } + + std::pair insert( value_type&& value ) + { + return emplace(value.first, std::move(value.second)); + } + + std::pair insert( const value_type& value ) + { + for (auto it = this->begin(); it != this->end(); ++it) + { + if (it->first == value.first) + { + return {it, false}; + } + } + Container::push_back(value); + return {--this->end(), true}; + } + + template + using require_input_iter = typename std::enable_if::iterator_category, + std::input_iterator_tag>::value>::type; + + template> + void insert(InputIt first, InputIt last) + { + for (auto it = first; it != last; ++it) + { + insert(*it); + } + } +}; + +} // namespace nlohmann + + +#if defined(JSON_HAS_CPP_17) + #include +#endif + +/*! +@brief namespace for Niels Lohmann +@see https://github.com/nlohmann +@since version 1.0.0 +*/ +namespace nlohmann +{ + +/*! +@brief a class to store JSON values + +@internal +@invariant The member variables @a m_value and @a m_type have the following +relationship: +- If `m_type == value_t::object`, then `m_value.object != nullptr`. +- If `m_type == value_t::array`, then `m_value.array != nullptr`. +- If `m_type == value_t::string`, then `m_value.string != nullptr`. +The invariants are checked by member function assert_invariant(). + +@note ObjectType trick from https://stackoverflow.com/a/9860911 +@endinternal + +@since version 1.0.0 + +@nosubgrouping +*/ +NLOHMANN_BASIC_JSON_TPL_DECLARATION +class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-special-member-functions) +{ + private: + template friend struct detail::external_constructor; + friend ::nlohmann::json_pointer; + + template + friend class ::nlohmann::detail::parser; + friend ::nlohmann::detail::serializer; + template + friend class ::nlohmann::detail::iter_impl; + template + friend class ::nlohmann::detail::binary_writer; + template + friend class ::nlohmann::detail::binary_reader; + template + friend class ::nlohmann::detail::json_sax_dom_parser; + template + friend class ::nlohmann::detail::json_sax_dom_callback_parser; + friend class ::nlohmann::detail::exception; + + /// workaround type for MSVC + using basic_json_t = NLOHMANN_BASIC_JSON_TPL; + + JSON_PRIVATE_UNLESS_TESTED: + // convenience aliases for types residing in namespace detail; + using lexer = ::nlohmann::detail::lexer_base; + + template + static ::nlohmann::detail::parser parser( + InputAdapterType adapter, + detail::parser_callback_tcb = nullptr, + const bool allow_exceptions = true, + const bool ignore_comments = false + ) + { + return ::nlohmann::detail::parser(std::move(adapter), + std::move(cb), allow_exceptions, ignore_comments); + } + + private: + using primitive_iterator_t = ::nlohmann::detail::primitive_iterator_t; + template + using internal_iterator = ::nlohmann::detail::internal_iterator; + template + using iter_impl = ::nlohmann::detail::iter_impl; + template + using iteration_proxy = ::nlohmann::detail::iteration_proxy; + template using json_reverse_iterator = ::nlohmann::detail::json_reverse_iterator; + + template + using output_adapter_t = ::nlohmann::detail::output_adapter_t; + + template + using binary_reader = ::nlohmann::detail::binary_reader; + template using binary_writer = ::nlohmann::detail::binary_writer; + + JSON_PRIVATE_UNLESS_TESTED: + using serializer = ::nlohmann::detail::serializer; + + public: + using value_t = detail::value_t; + /// JSON Pointer, see @ref nlohmann::json_pointer + using json_pointer = ::nlohmann::json_pointer; + template + using json_serializer = JSONSerializer; + /// how to treat decoding errors + using error_handler_t = detail::error_handler_t; + /// how to treat CBOR tags + using cbor_tag_handler_t = detail::cbor_tag_handler_t; + /// helper type for initializer lists of basic_json values + using initializer_list_t = std::initializer_list>; + + using input_format_t = detail::input_format_t; + /// SAX interface type, see @ref nlohmann::json_sax + using json_sax_t = json_sax; + + //////////////// + // exceptions // + //////////////// + + /// @name exceptions + /// Classes to implement user-defined exceptions. + /// @{ + + using exception = detail::exception; + using parse_error = detail::parse_error; + using invalid_iterator = detail::invalid_iterator; + using type_error = detail::type_error; + using out_of_range = detail::out_of_range; + using other_error = detail::other_error; + + /// @} + + + ///////////////////// + // container types // + ///////////////////// + + /// @name container types + /// The canonic container types to use @ref basic_json like any other STL + /// container. + /// @{ + + /// the type of elements in a basic_json container + using value_type = basic_json; + + /// the type of an element reference + using reference = value_type&; + /// the type of an element const reference + using const_reference = const value_type&; + + /// a type to represent differences between iterators + using difference_type = std::ptrdiff_t; + /// a type to represent container sizes + using size_type = std::size_t; + + /// the allocator type + using allocator_type = AllocatorType; + + /// the type of an element pointer + using pointer = typename std::allocator_traits::pointer; + /// the type of an element const pointer + using const_pointer = typename std::allocator_traits::const_pointer; + + /// an iterator for a basic_json container + using iterator = iter_impl; + /// a const iterator for a basic_json container + using const_iterator = iter_impl; + /// a reverse iterator for a basic_json container + using reverse_iterator = json_reverse_iterator; + /// a const reverse iterator for a basic_json container + using const_reverse_iterator = json_reverse_iterator; + + /// @} + + + /// @brief returns the allocator associated with the container + /// @sa https://json.nlohmann.me/api/basic_json/get_allocator/ + static allocator_type get_allocator() + { + return allocator_type(); + } + + /// @brief returns version information on the library + /// @sa https://json.nlohmann.me/api/basic_json/meta/ + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json meta() + { + basic_json result; + + result["copyright"] = "(C) 2013-2022 Niels Lohmann"; + result["name"] = "JSON for Modern C++"; + result["url"] = "https://github.com/nlohmann/json"; + result["version"]["string"] = + std::to_string(NLOHMANN_JSON_VERSION_MAJOR) + "." + + std::to_string(NLOHMANN_JSON_VERSION_MINOR) + "." + + std::to_string(NLOHMANN_JSON_VERSION_PATCH); + result["version"]["major"] = NLOHMANN_JSON_VERSION_MAJOR; + result["version"]["minor"] = NLOHMANN_JSON_VERSION_MINOR; + result["version"]["patch"] = NLOHMANN_JSON_VERSION_PATCH; + +#ifdef _WIN32 + result["platform"] = "win32"; +#elif defined __linux__ + result["platform"] = "linux"; +#elif defined __APPLE__ + result["platform"] = "apple"; +#elif defined __unix__ + result["platform"] = "unix"; +#else + result["platform"] = "unknown"; +#endif + +#if defined(__ICC) || defined(__INTEL_COMPILER) + result["compiler"] = {{"family", "icc"}, {"version", __INTEL_COMPILER}}; +#elif defined(__clang__) + result["compiler"] = {{"family", "clang"}, {"version", __clang_version__}}; +#elif defined(__GNUC__) || defined(__GNUG__) + result["compiler"] = {{"family", "gcc"}, {"version", std::to_string(__GNUC__) + "." + std::to_string(__GNUC_MINOR__) + "." + std::to_string(__GNUC_PATCHLEVEL__)}}; +#elif defined(__HP_cc) || defined(__HP_aCC) + result["compiler"] = "hp" +#elif defined(__IBMCPP__) + result["compiler"] = {{"family", "ilecpp"}, {"version", __IBMCPP__}}; +#elif defined(_MSC_VER) + result["compiler"] = {{"family", "msvc"}, {"version", _MSC_VER}}; +#elif defined(__PGI) + result["compiler"] = {{"family", "pgcpp"}, {"version", __PGI}}; +#elif defined(__SUNPRO_CC) + result["compiler"] = {{"family", "sunpro"}, {"version", __SUNPRO_CC}}; +#else + result["compiler"] = {{"family", "unknown"}, {"version", "unknown"}}; +#endif + +#ifdef __cplusplus + result["compiler"]["c++"] = std::to_string(__cplusplus); +#else + result["compiler"]["c++"] = "unknown"; +#endif + return result; + } + + + /////////////////////////// + // JSON value data types // + /////////////////////////// + + /// @name JSON value data types + /// The data types to store a JSON value. These types are derived from + /// the template arguments passed to class @ref basic_json. + /// @{ + + /// @brief object key comparator type + /// @sa https://json.nlohmann.me/api/basic_json/object_comparator_t/ +#if defined(JSON_HAS_CPP_14) + // Use transparent comparator if possible, combined with perfect forwarding + // on find() and count() calls prevents unnecessary string construction. + using object_comparator_t = std::less<>; +#else + using object_comparator_t = std::less; +#endif + + /// @brief a type for an object + /// @sa https://json.nlohmann.me/api/basic_json/object_t/ + using object_t = ObjectType>>; + + /// @brief a type for an array + /// @sa https://json.nlohmann.me/api/basic_json/array_t/ + using array_t = ArrayType>; + + /// @brief a type for a string + /// @sa https://json.nlohmann.me/api/basic_json/string_t/ + using string_t = StringType; + + /// @brief a type for a boolean + /// @sa https://json.nlohmann.me/api/basic_json/boolean_t/ + using boolean_t = BooleanType; + + /// @brief a type for a number (integer) + /// @sa https://json.nlohmann.me/api/basic_json/number_integer_t/ + using number_integer_t = NumberIntegerType; + + /// @brief a type for a number (unsigned) + /// @sa https://json.nlohmann.me/api/basic_json/number_unsigned_t/ + using number_unsigned_t = NumberUnsignedType; + + /// @brief a type for a number (floating-point) + /// @sa https://json.nlohmann.me/api/basic_json/number_float_t/ + using number_float_t = NumberFloatType; + + /// @brief a type for a packed binary type + /// @sa https://json.nlohmann.me/api/basic_json/binary_t/ + using binary_t = nlohmann::byte_container_with_subtype; + + /// @} + + private: + + /// helper for exception-safe object creation + template + JSON_HEDLEY_RETURNS_NON_NULL + static T* create(Args&& ... args) + { + AllocatorType alloc; + using AllocatorTraits = std::allocator_traits>; + + auto deleter = [&](T * obj) + { + AllocatorTraits::deallocate(alloc, obj, 1); + }; + std::unique_ptr obj(AllocatorTraits::allocate(alloc, 1), deleter); + AllocatorTraits::construct(alloc, obj.get(), std::forward(args)...); + JSON_ASSERT(obj != nullptr); + return obj.release(); + } + + //////////////////////// + // JSON value storage // + //////////////////////// + + JSON_PRIVATE_UNLESS_TESTED: + /*! + @brief a JSON value + + The actual storage for a JSON value of the @ref basic_json class. This + union combines the different storage types for the JSON value types + defined in @ref value_t. + + JSON type | value_t type | used type + --------- | --------------- | ------------------------ + object | object | pointer to @ref object_t + array | array | pointer to @ref array_t + string | string | pointer to @ref string_t + boolean | boolean | @ref boolean_t + number | number_integer | @ref number_integer_t + number | number_unsigned | @ref number_unsigned_t + number | number_float | @ref number_float_t + binary | binary | pointer to @ref binary_t + null | null | *no value is stored* + + @note Variable-length types (objects, arrays, and strings) are stored as + pointers. The size of the union should not exceed 64 bits if the default + value types are used. + + @since version 1.0.0 + */ + union json_value + { + /// object (stored with pointer to save storage) + object_t* object; + /// array (stored with pointer to save storage) + array_t* array; + /// string (stored with pointer to save storage) + string_t* string; + /// binary (stored with pointer to save storage) + binary_t* binary; + /// boolean + boolean_t boolean; + /// number (integer) + number_integer_t number_integer; + /// number (unsigned integer) + number_unsigned_t number_unsigned; + /// number (floating-point) + number_float_t number_float; + + /// default constructor (for null values) + json_value() = default; + /// constructor for booleans + json_value(boolean_t v) noexcept : boolean(v) {} + /// constructor for numbers (integer) + json_value(number_integer_t v) noexcept : number_integer(v) {} + /// constructor for numbers (unsigned) + json_value(number_unsigned_t v) noexcept : number_unsigned(v) {} + /// constructor for numbers (floating-point) + json_value(number_float_t v) noexcept : number_float(v) {} + /// constructor for empty values of a given type + json_value(value_t t) + { + switch (t) + { + case value_t::object: + { + object = create(); + break; + } + + case value_t::array: + { + array = create(); + break; + } + + case value_t::string: + { + string = create(""); + break; + } + + case value_t::binary: + { + binary = create(); + break; + } + + case value_t::boolean: + { + boolean = static_cast(false); + break; + } + + case value_t::number_integer: + { + number_integer = static_cast(0); + break; + } + + case value_t::number_unsigned: + { + number_unsigned = static_cast(0); + break; + } + + case value_t::number_float: + { + number_float = static_cast(0.0); + break; + } + + case value_t::null: + { + object = nullptr; // silence warning, see #821 + break; + } + + case value_t::discarded: + default: + { + object = nullptr; // silence warning, see #821 + if (JSON_HEDLEY_UNLIKELY(t == value_t::null)) + { + JSON_THROW(other_error::create(500, "961c151d2e87f2686a955a9be24d316f1362bf21 3.10.5", basic_json())); // LCOV_EXCL_LINE + } + break; + } + } + } + + /// constructor for strings + json_value(const string_t& value) : string(create(value)) {} + + /// constructor for rvalue strings + json_value(string_t&& value) : string(create(std::move(value))) {} + + /// constructor for objects + json_value(const object_t& value) : object(create(value)) {} + + /// constructor for rvalue objects + json_value(object_t&& value) : object(create(std::move(value))) {} + + /// constructor for arrays + json_value(const array_t& value) : array(create(value)) {} + + /// constructor for rvalue arrays + json_value(array_t&& value) : array(create(std::move(value))) {} + + /// constructor for binary arrays + json_value(const typename binary_t::container_type& value) : binary(create(value)) {} + + /// constructor for rvalue binary arrays + json_value(typename binary_t::container_type&& value) : binary(create(std::move(value))) {} + + /// constructor for binary arrays (internal type) + json_value(const binary_t& value) : binary(create(value)) {} + + /// constructor for rvalue binary arrays (internal type) + json_value(binary_t&& value) : binary(create(std::move(value))) {} + + void destroy(value_t t) + { + if (t == value_t::array || t == value_t::object) + { + // flatten the current json_value to a heap-allocated stack + std::vector stack; + + // move the top-level items to stack + if (t == value_t::array) + { + stack.reserve(array->size()); + std::move(array->begin(), array->end(), std::back_inserter(stack)); + } + else + { + stack.reserve(object->size()); + for (auto&& it : *object) + { + stack.push_back(std::move(it.second)); + } + } + + while (!stack.empty()) + { + // move the last item to local variable to be processed + basic_json current_item(std::move(stack.back())); + stack.pop_back(); + + // if current_item is array/object, move + // its children to the stack to be processed later + if (current_item.is_array()) + { + std::move(current_item.m_value.array->begin(), current_item.m_value.array->end(), std::back_inserter(stack)); + + current_item.m_value.array->clear(); + } + else if (current_item.is_object()) + { + for (auto&& it : *current_item.m_value.object) + { + stack.push_back(std::move(it.second)); + } + + current_item.m_value.object->clear(); + } + + // it's now safe that current_item get destructed + // since it doesn't have any children + } + } + + switch (t) + { + case value_t::object: + { + AllocatorType alloc; + std::allocator_traits::destroy(alloc, object); + std::allocator_traits::deallocate(alloc, object, 1); + break; + } + + case value_t::array: + { + AllocatorType alloc; + std::allocator_traits::destroy(alloc, array); + std::allocator_traits::deallocate(alloc, array, 1); + break; + } + + case value_t::string: + { + AllocatorType alloc; + std::allocator_traits::destroy(alloc, string); + std::allocator_traits::deallocate(alloc, string, 1); + break; + } + + case value_t::binary: + { + AllocatorType alloc; + std::allocator_traits::destroy(alloc, binary); + std::allocator_traits::deallocate(alloc, binary, 1); + break; + } + + case value_t::null: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::discarded: + default: + { + break; + } + } + } + }; + + private: + /*! + @brief checks the class invariants + + This function asserts the class invariants. It needs to be called at the + end of every constructor to make sure that created objects respect the + invariant. Furthermore, it has to be called each time the type of a JSON + value is changed, because the invariant expresses a relationship between + @a m_type and @a m_value. + + Furthermore, the parent relation is checked for arrays and objects: If + @a check_parents true and the value is an array or object, then the + container's elements must have the current value as parent. + + @param[in] check_parents whether the parent relation should be checked. + The value is true by default and should only be set to false + during destruction of objects when the invariant does not + need to hold. + */ + void assert_invariant(bool check_parents = true) const noexcept + { + JSON_ASSERT(m_type != value_t::object || m_value.object != nullptr); + JSON_ASSERT(m_type != value_t::array || m_value.array != nullptr); + JSON_ASSERT(m_type != value_t::string || m_value.string != nullptr); + JSON_ASSERT(m_type != value_t::binary || m_value.binary != nullptr); + +#if JSON_DIAGNOSTICS + JSON_TRY + { + // cppcheck-suppress assertWithSideEffect + JSON_ASSERT(!check_parents || !is_structured() || std::all_of(begin(), end(), [this](const basic_json & j) + { + return j.m_parent == this; + })); + } + JSON_CATCH(...) {} // LCOV_EXCL_LINE +#endif + static_cast(check_parents); + } + + void set_parents() + { +#if JSON_DIAGNOSTICS + switch (m_type) + { + case value_t::array: + { + for (auto& element : *m_value.array) + { + element.m_parent = this; + } + break; + } + + case value_t::object: + { + for (auto& element : *m_value.object) + { + element.second.m_parent = this; + } + break; + } + + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + break; + } +#endif + } + + iterator set_parents(iterator it, typename iterator::difference_type count_set_parents) + { +#if JSON_DIAGNOSTICS + for (typename iterator::difference_type i = 0; i < count_set_parents; ++i) + { + (it + i)->m_parent = this; + } +#else + static_cast(count_set_parents); +#endif + return it; + } + + reference set_parent(reference j, std::size_t old_capacity = static_cast(-1)) + { +#if JSON_DIAGNOSTICS + if (old_capacity != static_cast(-1)) + { + // see https://github.com/nlohmann/json/issues/2838 + JSON_ASSERT(type() == value_t::array); + if (JSON_HEDLEY_UNLIKELY(m_value.array->capacity() != old_capacity)) + { + // capacity has changed: update all parents + set_parents(); + return j; + } + } + + // ordered_json uses a vector internally, so pointers could have + // been invalidated; see https://github.com/nlohmann/json/issues/2962 +#ifdef JSON_HEDLEY_MSVC_VERSION +#pragma warning(push ) +#pragma warning(disable : 4127) // ignore warning to replace if with if constexpr +#endif + if (detail::is_ordered_map::value) + { + set_parents(); + return j; + } +#ifdef JSON_HEDLEY_MSVC_VERSION +#pragma warning( pop ) +#endif + + j.m_parent = this; +#else + static_cast(j); + static_cast(old_capacity); +#endif + return j; + } + + public: + ////////////////////////// + // JSON parser callback // + ////////////////////////// + + /// @brief parser event types + /// @sa https://json.nlohmann.me/api/basic_json/parse_event_t/ + using parse_event_t = detail::parse_event_t; + + /// @brief per-element parser callback type + /// @sa https://json.nlohmann.me/api/basic_json/parser_callback_t/ + using parser_callback_t = detail::parser_callback_t; + + ////////////////// + // constructors // + ////////////////// + + /// @name constructors and destructors + /// Constructors of class @ref basic_json, copy/move constructor, copy + /// assignment, static functions creating objects, and the destructor. + /// @{ + + /// @brief create an empty value with a given type + /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ + basic_json(const value_t v) + : m_type(v), m_value(v) + { + assert_invariant(); + } + + /// @brief create a null object + /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ + basic_json(std::nullptr_t = nullptr) noexcept + : basic_json(value_t::null) + { + assert_invariant(); + } + + /// @brief create a JSON value from compatible types + /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ + template < typename CompatibleType, + typename U = detail::uncvref_t, + detail::enable_if_t < + !detail::is_basic_json::value && detail::is_compatible_type::value, int > = 0 > + basic_json(CompatibleType && val) noexcept(noexcept( // NOLINT(bugprone-forwarding-reference-overload,bugprone-exception-escape) + JSONSerializer::to_json(std::declval(), + std::forward(val)))) + { + JSONSerializer::to_json(*this, std::forward(val)); + set_parents(); + assert_invariant(); + } + + /// @brief create a JSON value from an existing one + /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ + template < typename BasicJsonType, + detail::enable_if_t < + detail::is_basic_json::value&& !std::is_same::value, int > = 0 > + basic_json(const BasicJsonType& val) + { + using other_boolean_t = typename BasicJsonType::boolean_t; + using other_number_float_t = typename BasicJsonType::number_float_t; + using other_number_integer_t = typename BasicJsonType::number_integer_t; + using other_number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using other_string_t = typename BasicJsonType::string_t; + using other_object_t = typename BasicJsonType::object_t; + using other_array_t = typename BasicJsonType::array_t; + using other_binary_t = typename BasicJsonType::binary_t; + + switch (val.type()) + { + case value_t::boolean: + JSONSerializer::to_json(*this, val.template get()); + break; + case value_t::number_float: + JSONSerializer::to_json(*this, val.template get()); + break; + case value_t::number_integer: + JSONSerializer::to_json(*this, val.template get()); + break; + case value_t::number_unsigned: + JSONSerializer::to_json(*this, val.template get()); + break; + case value_t::string: + JSONSerializer::to_json(*this, val.template get_ref()); + break; + case value_t::object: + JSONSerializer::to_json(*this, val.template get_ref()); + break; + case value_t::array: + JSONSerializer::to_json(*this, val.template get_ref()); + break; + case value_t::binary: + JSONSerializer::to_json(*this, val.template get_ref()); + break; + case value_t::null: + *this = nullptr; + break; + case value_t::discarded: + m_type = value_t::discarded; + break; + default: // LCOV_EXCL_LINE + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE + } + set_parents(); + assert_invariant(); + } + + /// @brief create a container (array or object) from an initializer list + /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ + basic_json(initializer_list_t init, + bool type_deduction = true, + value_t manual_type = value_t::array) + { + // check if each element is an array with two elements whose first + // element is a string + bool is_an_object = std::all_of(init.begin(), init.end(), + [](const detail::json_ref& element_ref) + { + return element_ref->is_array() && element_ref->size() == 2 && (*element_ref)[0].is_string(); + }); + + // adjust type if type deduction is not wanted + if (!type_deduction) + { + // if array is wanted, do not create an object though possible + if (manual_type == value_t::array) + { + is_an_object = false; + } + + // if object is wanted but impossible, throw an exception + if (JSON_HEDLEY_UNLIKELY(manual_type == value_t::object && !is_an_object)) + { + JSON_THROW(type_error::create(301, "cannot create object from initializer list", basic_json())); + } + } + + if (is_an_object) + { + // the initializer list is a list of pairs -> create object + m_type = value_t::object; + m_value = value_t::object; + + for (auto& element_ref : init) + { + auto element = element_ref.moved_or_copied(); + m_value.object->emplace( + std::move(*((*element.m_value.array)[0].m_value.string)), + std::move((*element.m_value.array)[1])); + } + } + else + { + // the initializer list describes an array -> create array + m_type = value_t::array; + m_value.array = create(init.begin(), init.end()); + } + + set_parents(); + assert_invariant(); + } + + /// @brief explicitly create a binary array (without subtype) + /// @sa https://json.nlohmann.me/api/basic_json/binary/ + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json binary(const typename binary_t::container_type& init) + { + auto res = basic_json(); + res.m_type = value_t::binary; + res.m_value = init; + return res; + } + + /// @brief explicitly create a binary array (with subtype) + /// @sa https://json.nlohmann.me/api/basic_json/binary/ + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json binary(const typename binary_t::container_type& init, typename binary_t::subtype_type subtype) + { + auto res = basic_json(); + res.m_type = value_t::binary; + res.m_value = binary_t(init, subtype); + return res; + } + + /// @brief explicitly create a binary array + /// @sa https://json.nlohmann.me/api/basic_json/binary/ + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json binary(typename binary_t::container_type&& init) + { + auto res = basic_json(); + res.m_type = value_t::binary; + res.m_value = std::move(init); + return res; + } + + /// @brief explicitly create a binary array (with subtype) + /// @sa https://json.nlohmann.me/api/basic_json/binary/ + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json binary(typename binary_t::container_type&& init, typename binary_t::subtype_type subtype) + { + auto res = basic_json(); + res.m_type = value_t::binary; + res.m_value = binary_t(std::move(init), subtype); + return res; + } + + /// @brief explicitly create an array from an initializer list + /// @sa https://json.nlohmann.me/api/basic_json/array/ + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json array(initializer_list_t init = {}) + { + return basic_json(init, false, value_t::array); + } + + /// @brief explicitly create an object from an initializer list + /// @sa https://json.nlohmann.me/api/basic_json/object/ + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json object(initializer_list_t init = {}) + { + return basic_json(init, false, value_t::object); + } + + /// @brief construct an array with count copies of given value + /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ + basic_json(size_type cnt, const basic_json& val) + : m_type(value_t::array) + { + m_value.array = create(cnt, val); + set_parents(); + assert_invariant(); + } + + /// @brief construct a JSON container given an iterator range + /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ + template < class InputIT, typename std::enable_if < + std::is_same::value || + std::is_same::value, int >::type = 0 > + basic_json(InputIT first, InputIT last) + { + JSON_ASSERT(first.m_object != nullptr); + JSON_ASSERT(last.m_object != nullptr); + + // make sure iterator fits the current value + if (JSON_HEDLEY_UNLIKELY(first.m_object != last.m_object)) + { + JSON_THROW(invalid_iterator::create(201, "iterators are not compatible", basic_json())); + } + + // copy type from first iterator + m_type = first.m_object->m_type; + + // check if iterator range is complete for primitive values + switch (m_type) + { + case value_t::boolean: + case value_t::number_float: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::string: + { + if (JSON_HEDLEY_UNLIKELY(!first.m_it.primitive_iterator.is_begin() + || !last.m_it.primitive_iterator.is_end())) + { + JSON_THROW(invalid_iterator::create(204, "iterators out of range", *first.m_object)); + } + break; + } + + case value_t::null: + case value_t::object: + case value_t::array: + case value_t::binary: + case value_t::discarded: + default: + break; + } + + switch (m_type) + { + case value_t::number_integer: + { + m_value.number_integer = first.m_object->m_value.number_integer; + break; + } + + case value_t::number_unsigned: + { + m_value.number_unsigned = first.m_object->m_value.number_unsigned; + break; + } + + case value_t::number_float: + { + m_value.number_float = first.m_object->m_value.number_float; + break; + } + + case value_t::boolean: + { + m_value.boolean = first.m_object->m_value.boolean; + break; + } + + case value_t::string: + { + m_value = *first.m_object->m_value.string; + break; + } + + case value_t::object: + { + m_value.object = create(first.m_it.object_iterator, + last.m_it.object_iterator); + break; + } + + case value_t::array: + { + m_value.array = create(first.m_it.array_iterator, + last.m_it.array_iterator); + break; + } + + case value_t::binary: + { + m_value = *first.m_object->m_value.binary; + break; + } + + case value_t::null: + case value_t::discarded: + default: + JSON_THROW(invalid_iterator::create(206, "cannot construct with iterators from " + std::string(first.m_object->type_name()), *first.m_object)); + } + + set_parents(); + assert_invariant(); + } + + + /////////////////////////////////////// + // other constructors and destructor // + /////////////////////////////////////// + + template, + std::is_same>::value, int> = 0 > + basic_json(const JsonRef& ref) : basic_json(ref.moved_or_copied()) {} + + /// @brief copy constructor + /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ + basic_json(const basic_json& other) + : m_type(other.m_type) + { + // check of passed value is valid + other.assert_invariant(); + + switch (m_type) + { + case value_t::object: + { + m_value = *other.m_value.object; + break; + } + + case value_t::array: + { + m_value = *other.m_value.array; + break; + } + + case value_t::string: + { + m_value = *other.m_value.string; + break; + } + + case value_t::boolean: + { + m_value = other.m_value.boolean; + break; + } + + case value_t::number_integer: + { + m_value = other.m_value.number_integer; + break; + } + + case value_t::number_unsigned: + { + m_value = other.m_value.number_unsigned; + break; + } + + case value_t::number_float: + { + m_value = other.m_value.number_float; + break; + } + + case value_t::binary: + { + m_value = *other.m_value.binary; + break; + } + + case value_t::null: + case value_t::discarded: + default: + break; + } + + set_parents(); + assert_invariant(); + } + + /// @brief move constructor + /// @sa https://json.nlohmann.me/api/basic_json/basic_json/ + basic_json(basic_json&& other) noexcept + : m_type(std::move(other.m_type)), + m_value(std::move(other.m_value)) + { + // check that passed value is valid + other.assert_invariant(false); + + // invalidate payload + other.m_type = value_t::null; + other.m_value = {}; + + set_parents(); + assert_invariant(); + } + + /// @brief copy assignment + /// @sa https://json.nlohmann.me/api/basic_json/operator=/ + basic_json& operator=(basic_json other) noexcept ( + std::is_nothrow_move_constructible::value&& + std::is_nothrow_move_assignable::value&& + std::is_nothrow_move_constructible::value&& + std::is_nothrow_move_assignable::value + ) + { + // check that passed value is valid + other.assert_invariant(); + + using std::swap; + swap(m_type, other.m_type); + swap(m_value, other.m_value); + + set_parents(); + assert_invariant(); + return *this; + } + + /// @brief destructor + /// @sa https://json.nlohmann.me/api/basic_json/~basic_json/ + ~basic_json() noexcept + { + assert_invariant(false); + m_value.destroy(m_type); + } + + /// @} + + public: + /////////////////////// + // object inspection // + /////////////////////// + + /// @name object inspection + /// Functions to inspect the type of a JSON value. + /// @{ + + /// @brief serialization + /// @sa https://json.nlohmann.me/api/basic_json/dump/ + string_t dump(const int indent = -1, + const char indent_char = ' ', + const bool ensure_ascii = false, + const error_handler_t error_handler = error_handler_t::strict) const + { + string_t result; + serializer s(detail::output_adapter(result), indent_char, error_handler); + + if (indent >= 0) + { + s.dump(*this, true, ensure_ascii, static_cast(indent)); + } + else + { + s.dump(*this, false, ensure_ascii, 0); + } + + return result; + } + + /// @brief return the type of the JSON value (explicit) + /// @sa https://json.nlohmann.me/api/basic_json/type/ + constexpr value_t type() const noexcept + { + return m_type; + } + + /// @brief return whether type is primitive + /// @sa https://json.nlohmann.me/api/basic_json/is_primitive/ + constexpr bool is_primitive() const noexcept + { + return is_null() || is_string() || is_boolean() || is_number() || is_binary(); + } + + /// @brief return whether type is structured + /// @sa https://json.nlohmann.me/api/basic_json/is_structured/ + constexpr bool is_structured() const noexcept + { + return is_array() || is_object(); + } + + /// @brief return whether value is null + /// @sa https://json.nlohmann.me/api/basic_json/is_null/ + constexpr bool is_null() const noexcept + { + return m_type == value_t::null; + } + + /// @brief return whether value is a boolean + /// @sa https://json.nlohmann.me/api/basic_json/is_boolean/ + constexpr bool is_boolean() const noexcept + { + return m_type == value_t::boolean; + } + + /// @brief return whether value is a number + /// @sa https://json.nlohmann.me/api/basic_json/is_number/ + constexpr bool is_number() const noexcept + { + return is_number_integer() || is_number_float(); + } + + /// @brief return whether value is an integer number + /// @sa https://json.nlohmann.me/api/basic_json/is_number_integer/ + constexpr bool is_number_integer() const noexcept + { + return m_type == value_t::number_integer || m_type == value_t::number_unsigned; + } + + /// @brief return whether value is an unsigned integer number + /// @sa https://json.nlohmann.me/api/basic_json/is_number_unsigned/ + constexpr bool is_number_unsigned() const noexcept + { + return m_type == value_t::number_unsigned; + } + + /// @brief return whether value is a floating-point number + /// @sa https://json.nlohmann.me/api/basic_json/is_number_float/ + constexpr bool is_number_float() const noexcept + { + return m_type == value_t::number_float; + } + + /// @brief return whether value is an object + /// @sa https://json.nlohmann.me/api/basic_json/is_object/ + constexpr bool is_object() const noexcept + { + return m_type == value_t::object; + } + + /// @brief return whether value is an array + /// @sa https://json.nlohmann.me/api/basic_json/is_array/ + constexpr bool is_array() const noexcept + { + return m_type == value_t::array; + } + + /// @brief return whether value is a string + /// @sa https://json.nlohmann.me/api/basic_json/is_string/ + constexpr bool is_string() const noexcept + { + return m_type == value_t::string; + } + + /// @brief return whether value is a binary array + /// @sa https://json.nlohmann.me/api/basic_json/is_binary/ + constexpr bool is_binary() const noexcept + { + return m_type == value_t::binary; + } + + /// @brief return whether value is discarded + /// @sa https://json.nlohmann.me/api/basic_json/is_discarded/ + constexpr bool is_discarded() const noexcept + { + return m_type == value_t::discarded; + } + + /// @brief return the type of the JSON value (implicit) + /// @sa https://json.nlohmann.me/api/basic_json/operator_value_t/ + constexpr operator value_t() const noexcept + { + return m_type; + } + + /// @} + + private: + ////////////////// + // value access // + ////////////////// + + /// get a boolean (explicit) + boolean_t get_impl(boolean_t* /*unused*/) const + { + if (JSON_HEDLEY_LIKELY(is_boolean())) + { + return m_value.boolean; + } + + JSON_THROW(type_error::create(302, "type must be boolean, but is " + std::string(type_name()), *this)); + } + + /// get a pointer to the value (object) + object_t* get_impl_ptr(object_t* /*unused*/) noexcept + { + return is_object() ? m_value.object : nullptr; + } + + /// get a pointer to the value (object) + constexpr const object_t* get_impl_ptr(const object_t* /*unused*/) const noexcept + { + return is_object() ? m_value.object : nullptr; + } + + /// get a pointer to the value (array) + array_t* get_impl_ptr(array_t* /*unused*/) noexcept + { + return is_array() ? m_value.array : nullptr; + } + + /// get a pointer to the value (array) + constexpr const array_t* get_impl_ptr(const array_t* /*unused*/) const noexcept + { + return is_array() ? m_value.array : nullptr; + } + + /// get a pointer to the value (string) + string_t* get_impl_ptr(string_t* /*unused*/) noexcept + { + return is_string() ? m_value.string : nullptr; + } + + /// get a pointer to the value (string) + constexpr const string_t* get_impl_ptr(const string_t* /*unused*/) const noexcept + { + return is_string() ? m_value.string : nullptr; + } + + /// get a pointer to the value (boolean) + boolean_t* get_impl_ptr(boolean_t* /*unused*/) noexcept + { + return is_boolean() ? &m_value.boolean : nullptr; + } + + /// get a pointer to the value (boolean) + constexpr const boolean_t* get_impl_ptr(const boolean_t* /*unused*/) const noexcept + { + return is_boolean() ? &m_value.boolean : nullptr; + } + + /// get a pointer to the value (integer number) + number_integer_t* get_impl_ptr(number_integer_t* /*unused*/) noexcept + { + return is_number_integer() ? &m_value.number_integer : nullptr; + } + + /// get a pointer to the value (integer number) + constexpr const number_integer_t* get_impl_ptr(const number_integer_t* /*unused*/) const noexcept + { + return is_number_integer() ? &m_value.number_integer : nullptr; + } + + /// get a pointer to the value (unsigned number) + number_unsigned_t* get_impl_ptr(number_unsigned_t* /*unused*/) noexcept + { + return is_number_unsigned() ? &m_value.number_unsigned : nullptr; + } + + /// get a pointer to the value (unsigned number) + constexpr const number_unsigned_t* get_impl_ptr(const number_unsigned_t* /*unused*/) const noexcept + { + return is_number_unsigned() ? &m_value.number_unsigned : nullptr; + } + + /// get a pointer to the value (floating-point number) + number_float_t* get_impl_ptr(number_float_t* /*unused*/) noexcept + { + return is_number_float() ? &m_value.number_float : nullptr; + } + + /// get a pointer to the value (floating-point number) + constexpr const number_float_t* get_impl_ptr(const number_float_t* /*unused*/) const noexcept + { + return is_number_float() ? &m_value.number_float : nullptr; + } + + /// get a pointer to the value (binary) + binary_t* get_impl_ptr(binary_t* /*unused*/) noexcept + { + return is_binary() ? m_value.binary : nullptr; + } + + /// get a pointer to the value (binary) + constexpr const binary_t* get_impl_ptr(const binary_t* /*unused*/) const noexcept + { + return is_binary() ? m_value.binary : nullptr; + } + + /*! + @brief helper function to implement get_ref() + + This function helps to implement get_ref() without code duplication for + const and non-const overloads + + @tparam ThisType will be deduced as `basic_json` or `const basic_json` + + @throw type_error.303 if ReferenceType does not match underlying value + type of the current JSON + */ + template + static ReferenceType get_ref_impl(ThisType& obj) + { + // delegate the call to get_ptr<>() + auto* ptr = obj.template get_ptr::type>(); + + if (JSON_HEDLEY_LIKELY(ptr != nullptr)) + { + return *ptr; + } + + JSON_THROW(type_error::create(303, "incompatible ReferenceType for get_ref, actual type is " + std::string(obj.type_name()), obj)); + } + + public: + /// @name value access + /// Direct access to the stored value of a JSON value. + /// @{ + + /// @brief get a pointer value (implicit) + /// @sa https://json.nlohmann.me/api/basic_json/get_ptr/ + template::value, int>::type = 0> + auto get_ptr() noexcept -> decltype(std::declval().get_impl_ptr(std::declval())) + { + // delegate the call to get_impl_ptr<>() + return get_impl_ptr(static_cast(nullptr)); + } + + /// @brief get a pointer value (implicit) + /// @sa https://json.nlohmann.me/api/basic_json/get_ptr/ + template < typename PointerType, typename std::enable_if < + std::is_pointer::value&& + std::is_const::type>::value, int >::type = 0 > + constexpr auto get_ptr() const noexcept -> decltype(std::declval().get_impl_ptr(std::declval())) + { + // delegate the call to get_impl_ptr<>() const + return get_impl_ptr(static_cast(nullptr)); + } + + private: + /*! + @brief get a value (explicit) + + Explicit type conversion between the JSON value and a compatible value + which is [CopyConstructible](https://en.cppreference.com/w/cpp/named_req/CopyConstructible) + and [DefaultConstructible](https://en.cppreference.com/w/cpp/named_req/DefaultConstructible). + The value is converted by calling the @ref json_serializer + `from_json()` method. + + The function is equivalent to executing + @code {.cpp} + ValueType ret; + JSONSerializer::from_json(*this, ret); + return ret; + @endcode + + This overloads is chosen if: + - @a ValueType is not @ref basic_json, + - @ref json_serializer has a `from_json()` method of the form + `void from_json(const basic_json&, ValueType&)`, and + - @ref json_serializer does not have a `from_json()` method of + the form `ValueType from_json(const basic_json&)` + + @tparam ValueType the returned value type + + @return copy of the JSON value, converted to @a ValueType + + @throw what @ref json_serializer `from_json()` method throws + + @liveexample{The example below shows several conversions from JSON values + to other types. There a few things to note: (1) Floating-point numbers can + be converted to integers\, (2) A JSON array can be converted to a standard + `std::vector`\, (3) A JSON object can be converted to C++ + associative containers such as `std::unordered_map`.,get__ValueType_const} + + @since version 2.1.0 + */ + template < typename ValueType, + detail::enable_if_t < + detail::is_default_constructible::value&& + detail::has_from_json::value, + int > = 0 > + ValueType get_impl(detail::priority_tag<0> /*unused*/) const noexcept(noexcept( + JSONSerializer::from_json(std::declval(), std::declval()))) + { + auto ret = ValueType(); + JSONSerializer::from_json(*this, ret); + return ret; + } + + /*! + @brief get a value (explicit); special case + + Explicit type conversion between the JSON value and a compatible value + which is **not** [CopyConstructible](https://en.cppreference.com/w/cpp/named_req/CopyConstructible) + and **not** [DefaultConstructible](https://en.cppreference.com/w/cpp/named_req/DefaultConstructible). + The value is converted by calling the @ref json_serializer + `from_json()` method. + + The function is equivalent to executing + @code {.cpp} + return JSONSerializer::from_json(*this); + @endcode + + This overloads is chosen if: + - @a ValueType is not @ref basic_json and + - @ref json_serializer has a `from_json()` method of the form + `ValueType from_json(const basic_json&)` + + @note If @ref json_serializer has both overloads of + `from_json()`, this one is chosen. + + @tparam ValueType the returned value type + + @return copy of the JSON value, converted to @a ValueType + + @throw what @ref json_serializer `from_json()` method throws + + @since version 2.1.0 + */ + template < typename ValueType, + detail::enable_if_t < + detail::has_non_default_from_json::value, + int > = 0 > + ValueType get_impl(detail::priority_tag<1> /*unused*/) const noexcept(noexcept( + JSONSerializer::from_json(std::declval()))) + { + return JSONSerializer::from_json(*this); + } + + /*! + @brief get special-case overload + + This overloads converts the current @ref basic_json in a different + @ref basic_json type + + @tparam BasicJsonType == @ref basic_json + + @return a copy of *this, converted into @a BasicJsonType + + @complexity Depending on the implementation of the called `from_json()` + method. + + @since version 3.2.0 + */ + template < typename BasicJsonType, + detail::enable_if_t < + detail::is_basic_json::value, + int > = 0 > + BasicJsonType get_impl(detail::priority_tag<2> /*unused*/) const + { + return *this; + } + + /*! + @brief get special-case overload + + This overloads avoids a lot of template boilerplate, it can be seen as the + identity method + + @tparam BasicJsonType == @ref basic_json + + @return a copy of *this + + @complexity Constant. + + @since version 2.1.0 + */ + template::value, + int> = 0> + basic_json get_impl(detail::priority_tag<3> /*unused*/) const + { + return *this; + } + + /*! + @brief get a pointer value (explicit) + @copydoc get() + */ + template::value, + int> = 0> + constexpr auto get_impl(detail::priority_tag<4> /*unused*/) const noexcept + -> decltype(std::declval().template get_ptr()) + { + // delegate the call to get_ptr + return get_ptr(); + } + + public: + /*! + @brief get a (pointer) value (explicit) + + Performs explicit type conversion between the JSON value and a compatible value if required. + + - If the requested type is a pointer to the internally stored JSON value that pointer is returned. + No copies are made. + + - If the requested type is the current @ref basic_json, or a different @ref basic_json convertible + from the current @ref basic_json. + + - Otherwise the value is converted by calling the @ref json_serializer `from_json()` + method. + + @tparam ValueTypeCV the provided value type + @tparam ValueType the returned value type + + @return copy of the JSON value, converted to @tparam ValueType if necessary + + @throw what @ref json_serializer `from_json()` method throws if conversion is required + + @since version 2.1.0 + */ + template < typename ValueTypeCV, typename ValueType = detail::uncvref_t> +#if defined(JSON_HAS_CPP_14) + constexpr +#endif + auto get() const noexcept( + noexcept(std::declval().template get_impl(detail::priority_tag<4> {}))) + -> decltype(std::declval().template get_impl(detail::priority_tag<4> {})) + { + // we cannot static_assert on ValueTypeCV being non-const, because + // there is support for get(), which is why we + // still need the uncvref + static_assert(!std::is_reference::value, + "get() cannot be used with reference types, you might want to use get_ref()"); + return get_impl(detail::priority_tag<4> {}); + } + + /*! + @brief get a pointer value (explicit) + + Explicit pointer access to the internally stored JSON value. No copies are + made. + + @warning The pointer becomes invalid if the underlying JSON object + changes. + + @tparam PointerType pointer type; must be a pointer to @ref array_t, @ref + object_t, @ref string_t, @ref boolean_t, @ref number_integer_t, + @ref number_unsigned_t, or @ref number_float_t. + + @return pointer to the internally stored JSON value if the requested + pointer type @a PointerType fits to the JSON value; `nullptr` otherwise + + @complexity Constant. + + @liveexample{The example below shows how pointers to internal values of a + JSON value can be requested. Note that no type conversions are made and a + `nullptr` is returned if the value and the requested pointer type does not + match.,get__PointerType} + + @sa see @ref get_ptr() for explicit pointer-member access + + @since version 1.0.0 + */ + template::value, int>::type = 0> + auto get() noexcept -> decltype(std::declval().template get_ptr()) + { + // delegate the call to get_ptr + return get_ptr(); + } + + /// @brief get a value (explicit) + /// @sa https://json.nlohmann.me/api/basic_json/get_to/ + template < typename ValueType, + detail::enable_if_t < + !detail::is_basic_json::value&& + detail::has_from_json::value, + int > = 0 > + ValueType & get_to(ValueType& v) const noexcept(noexcept( + JSONSerializer::from_json(std::declval(), v))) + { + JSONSerializer::from_json(*this, v); + return v; + } + + // specialization to allow calling get_to with a basic_json value + // see https://github.com/nlohmann/json/issues/2175 + template::value, + int> = 0> + ValueType & get_to(ValueType& v) const + { + v = *this; + return v; + } + + template < + typename T, std::size_t N, + typename Array = T (&)[N], // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + detail::enable_if_t < + detail::has_from_json::value, int > = 0 > + Array get_to(T (&v)[N]) const // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + noexcept(noexcept(JSONSerializer::from_json( + std::declval(), v))) + { + JSONSerializer::from_json(*this, v); + return v; + } + + /// @brief get a reference value (implicit) + /// @sa https://json.nlohmann.me/api/basic_json/get_ref/ + template::value, int>::type = 0> + ReferenceType get_ref() + { + // delegate call to get_ref_impl + return get_ref_impl(*this); + } + + /// @brief get a reference value (implicit) + /// @sa https://json.nlohmann.me/api/basic_json/get_ref/ + template < typename ReferenceType, typename std::enable_if < + std::is_reference::value&& + std::is_const::type>::value, int >::type = 0 > + ReferenceType get_ref() const + { + // delegate call to get_ref_impl + return get_ref_impl(*this); + } + + /*! + @brief get a value (implicit) + + Implicit type conversion between the JSON value and a compatible value. + The call is realized by calling @ref get() const. + + @tparam ValueType non-pointer type compatible to the JSON value, for + instance `int` for JSON integer numbers, `bool` for JSON booleans, or + `std::vector` types for JSON arrays. The character type of @ref string_t + as well as an initializer list of this type is excluded to avoid + ambiguities as these types implicitly convert to `std::string`. + + @return copy of the JSON value, converted to type @a ValueType + + @throw type_error.302 in case passed type @a ValueType is incompatible + to the JSON value type (e.g., the JSON value is of type boolean, but a + string is requested); see example below + + @complexity Linear in the size of the JSON value. + + @liveexample{The example below shows several conversions from JSON values + to other types. There a few things to note: (1) Floating-point numbers can + be converted to integers\, (2) A JSON array can be converted to a standard + `std::vector`\, (3) A JSON object can be converted to C++ + associative containers such as `std::unordered_map`.,operator__ValueType} + + @since version 1.0.0 + */ + template < typename ValueType, typename std::enable_if < + detail::conjunction < + detail::negation>, + detail::negation>>, + detail::negation>, + detail::negation>, + detail::negation>>, + +#if defined(JSON_HAS_CPP_17) && (defined(__GNUC__) || (defined(_MSC_VER) && _MSC_VER >= 1910 && _MSC_VER <= 1914)) + detail::negation>, +#endif + detail::is_detected_lazy + >::value, int >::type = 0 > + JSON_EXPLICIT operator ValueType() const + { + // delegate the call to get<>() const + return get(); + } + + /// @brief get a binary value + /// @sa https://json.nlohmann.me/api/basic_json/get_binary/ + binary_t& get_binary() + { + if (!is_binary()) + { + JSON_THROW(type_error::create(302, "type must be binary, but is " + std::string(type_name()), *this)); + } + + return *get_ptr(); + } + + /// @brief get a binary value + /// @sa https://json.nlohmann.me/api/basic_json/get_binary/ + const binary_t& get_binary() const + { + if (!is_binary()) + { + JSON_THROW(type_error::create(302, "type must be binary, but is " + std::string(type_name()), *this)); + } + + return *get_ptr(); + } + + /// @} + + + //////////////////// + // element access // + //////////////////// + + /// @name element access + /// Access to the JSON value. + /// @{ + + /// @brief access specified array element with bounds checking + /// @sa https://json.nlohmann.me/api/basic_json/at/ + reference at(size_type idx) + { + // at only works for arrays + if (JSON_HEDLEY_LIKELY(is_array())) + { + JSON_TRY + { + return set_parent(m_value.array->at(idx)); + } + JSON_CATCH (std::out_of_range&) + { + // create better exception explanation + JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range", *this)); + } + } + else + { + JSON_THROW(type_error::create(304, "cannot use at() with " + std::string(type_name()), *this)); + } + } + + /// @brief access specified array element with bounds checking + /// @sa https://json.nlohmann.me/api/basic_json/at/ + const_reference at(size_type idx) const + { + // at only works for arrays + if (JSON_HEDLEY_LIKELY(is_array())) + { + JSON_TRY + { + return m_value.array->at(idx); + } + JSON_CATCH (std::out_of_range&) + { + // create better exception explanation + JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range", *this)); + } + } + else + { + JSON_THROW(type_error::create(304, "cannot use at() with " + std::string(type_name()), *this)); + } + } + + /// @brief access specified object element with bounds checking + /// @sa https://json.nlohmann.me/api/basic_json/at/ + reference at(const typename object_t::key_type& key) + { + // at only works for objects + if (JSON_HEDLEY_LIKELY(is_object())) + { + JSON_TRY + { + return set_parent(m_value.object->at(key)); + } + JSON_CATCH (std::out_of_range&) + { + // create better exception explanation + JSON_THROW(out_of_range::create(403, "key '" + key + "' not found", *this)); + } + } + else + { + JSON_THROW(type_error::create(304, "cannot use at() with " + std::string(type_name()), *this)); + } + } + + /// @brief access specified object element with bounds checking + /// @sa https://json.nlohmann.me/api/basic_json/at/ + const_reference at(const typename object_t::key_type& key) const + { + // at only works for objects + if (JSON_HEDLEY_LIKELY(is_object())) + { + JSON_TRY + { + return m_value.object->at(key); + } + JSON_CATCH (std::out_of_range&) + { + // create better exception explanation + JSON_THROW(out_of_range::create(403, "key '" + key + "' not found", *this)); + } + } + else + { + JSON_THROW(type_error::create(304, "cannot use at() with " + std::string(type_name()), *this)); + } + } + + /// @brief access specified array element + /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/ + reference operator[](size_type idx) + { + // implicitly convert null value to an empty array + if (is_null()) + { + m_type = value_t::array; + m_value.array = create(); + assert_invariant(); + } + + // operator[] only works for arrays + if (JSON_HEDLEY_LIKELY(is_array())) + { + // fill up array with null values if given idx is outside range + if (idx >= m_value.array->size()) + { +#if JSON_DIAGNOSTICS + // remember array size & capacity before resizing + const auto old_size = m_value.array->size(); + const auto old_capacity = m_value.array->capacity(); +#endif + m_value.array->resize(idx + 1); + +#if JSON_DIAGNOSTICS + if (JSON_HEDLEY_UNLIKELY(m_value.array->capacity() != old_capacity)) + { + // capacity has changed: update all parents + set_parents(); + } + else + { + // set parent for values added above + set_parents(begin() + static_cast(old_size), static_cast(idx + 1 - old_size)); + } +#endif + assert_invariant(); + } + + return m_value.array->operator[](idx); + } + + JSON_THROW(type_error::create(305, "cannot use operator[] with a numeric argument with " + std::string(type_name()), *this)); + } + + /// @brief access specified array element + /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/ + const_reference operator[](size_type idx) const + { + // const operator[] only works for arrays + if (JSON_HEDLEY_LIKELY(is_array())) + { + return m_value.array->operator[](idx); + } + + JSON_THROW(type_error::create(305, "cannot use operator[] with a numeric argument with " + std::string(type_name()), *this)); + } + + /// @brief access specified object element + /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/ + reference operator[](const typename object_t::key_type& key) + { + // implicitly convert null value to an empty object + if (is_null()) + { + m_type = value_t::object; + m_value.object = create(); + assert_invariant(); + } + + // operator[] only works for objects + if (JSON_HEDLEY_LIKELY(is_object())) + { + return set_parent(m_value.object->operator[](key)); + } + + JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name()), *this)); + } + + /// @brief access specified object element + /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/ + const_reference operator[](const typename object_t::key_type& key) const + { + // const operator[] only works for objects + if (JSON_HEDLEY_LIKELY(is_object())) + { + JSON_ASSERT(m_value.object->find(key) != m_value.object->end()); + return m_value.object->find(key)->second; + } + + JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name()), *this)); + } + + /// @brief access specified object element + /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/ + template + JSON_HEDLEY_NON_NULL(2) + reference operator[](T* key) + { + // implicitly convert null to object + if (is_null()) + { + m_type = value_t::object; + m_value = value_t::object; + assert_invariant(); + } + + // at only works for objects + if (JSON_HEDLEY_LIKELY(is_object())) + { + return set_parent(m_value.object->operator[](key)); + } + + JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name()), *this)); + } + + /// @brief access specified object element + /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/ + template + JSON_HEDLEY_NON_NULL(2) + const_reference operator[](T* key) const + { + // at only works for objects + if (JSON_HEDLEY_LIKELY(is_object())) + { + JSON_ASSERT(m_value.object->find(key) != m_value.object->end()); + return m_value.object->find(key)->second; + } + + JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name()), *this)); + } + + /// @brief access specified object element with default value + /// @sa https://json.nlohmann.me/api/basic_json/value/ + /// using std::is_convertible in a std::enable_if will fail when using explicit conversions + template < class ValueType, typename std::enable_if < + detail::is_getable::value + && !std::is_same::value, int >::type = 0 > + ValueType value(const typename object_t::key_type& key, const ValueType& default_value) const + { + // at only works for objects + if (JSON_HEDLEY_LIKELY(is_object())) + { + // if key is found, return value and given default value otherwise + const auto it = find(key); + if (it != end()) + { + return it->template get(); + } + + return default_value; + } + + JSON_THROW(type_error::create(306, "cannot use value() with " + std::string(type_name()), *this)); + } + + /// @brief access specified object element with default value + /// @sa https://json.nlohmann.me/api/basic_json/value/ + /// overload for a default value of type const char* + string_t value(const typename object_t::key_type& key, const char* default_value) const + { + return value(key, string_t(default_value)); + } + + /// @brief access specified object element via JSON Pointer with default value + /// @sa https://json.nlohmann.me/api/basic_json/value/ + template::value, int>::type = 0> + ValueType value(const json_pointer& ptr, const ValueType& default_value) const + { + // at only works for objects + if (JSON_HEDLEY_LIKELY(is_object())) + { + // if pointer resolves a value, return it or use default value + JSON_TRY + { + return ptr.get_checked(this).template get(); + } + JSON_INTERNAL_CATCH (out_of_range&) + { + return default_value; + } + } + + JSON_THROW(type_error::create(306, "cannot use value() with " + std::string(type_name()), *this)); + } + + /// @brief access specified object element via JSON Pointer with default value + /// @sa https://json.nlohmann.me/api/basic_json/value/ + /// overload for a default value of type const char* + JSON_HEDLEY_NON_NULL(3) + string_t value(const json_pointer& ptr, const char* default_value) const + { + return value(ptr, string_t(default_value)); + } + + /// @brief access the first element + /// @sa https://json.nlohmann.me/api/basic_json/front/ + reference front() + { + return *begin(); + } + + /// @brief access the first element + /// @sa https://json.nlohmann.me/api/basic_json/front/ + const_reference front() const + { + return *cbegin(); + } + + /// @brief access the last element + /// @sa https://json.nlohmann.me/api/basic_json/back/ + reference back() + { + auto tmp = end(); + --tmp; + return *tmp; + } + + /// @brief access the last element + /// @sa https://json.nlohmann.me/api/basic_json/back/ + const_reference back() const + { + auto tmp = cend(); + --tmp; + return *tmp; + } + + /// @brief remove element given an iterator + /// @sa https://json.nlohmann.me/api/basic_json/erase/ + template < class IteratorType, typename std::enable_if < + std::is_same::value || + std::is_same::value, int >::type + = 0 > + IteratorType erase(IteratorType pos) + { + // make sure iterator fits the current value + if (JSON_HEDLEY_UNLIKELY(this != pos.m_object)) + { + JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value", *this)); + } + + IteratorType result = end(); + + switch (m_type) + { + case value_t::boolean: + case value_t::number_float: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::string: + case value_t::binary: + { + if (JSON_HEDLEY_UNLIKELY(!pos.m_it.primitive_iterator.is_begin())) + { + JSON_THROW(invalid_iterator::create(205, "iterator out of range", *this)); + } + + if (is_string()) + { + AllocatorType alloc; + std::allocator_traits::destroy(alloc, m_value.string); + std::allocator_traits::deallocate(alloc, m_value.string, 1); + m_value.string = nullptr; + } + else if (is_binary()) + { + AllocatorType alloc; + std::allocator_traits::destroy(alloc, m_value.binary); + std::allocator_traits::deallocate(alloc, m_value.binary, 1); + m_value.binary = nullptr; + } + + m_type = value_t::null; + assert_invariant(); + break; + } + + case value_t::object: + { + result.m_it.object_iterator = m_value.object->erase(pos.m_it.object_iterator); + break; + } + + case value_t::array: + { + result.m_it.array_iterator = m_value.array->erase(pos.m_it.array_iterator); + break; + } + + case value_t::null: + case value_t::discarded: + default: + JSON_THROW(type_error::create(307, "cannot use erase() with " + std::string(type_name()), *this)); + } + + return result; + } + + /// @brief remove elements given an iterator range + /// @sa https://json.nlohmann.me/api/basic_json/erase/ + template < class IteratorType, typename std::enable_if < + std::is_same::value || + std::is_same::value, int >::type + = 0 > + IteratorType erase(IteratorType first, IteratorType last) + { + // make sure iterator fits the current value + if (JSON_HEDLEY_UNLIKELY(this != first.m_object || this != last.m_object)) + { + JSON_THROW(invalid_iterator::create(203, "iterators do not fit current value", *this)); + } + + IteratorType result = end(); + + switch (m_type) + { + case value_t::boolean: + case value_t::number_float: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::string: + case value_t::binary: + { + if (JSON_HEDLEY_LIKELY(!first.m_it.primitive_iterator.is_begin() + || !last.m_it.primitive_iterator.is_end())) + { + JSON_THROW(invalid_iterator::create(204, "iterators out of range", *this)); + } + + if (is_string()) + { + AllocatorType alloc; + std::allocator_traits::destroy(alloc, m_value.string); + std::allocator_traits::deallocate(alloc, m_value.string, 1); + m_value.string = nullptr; + } + else if (is_binary()) + { + AllocatorType alloc; + std::allocator_traits::destroy(alloc, m_value.binary); + std::allocator_traits::deallocate(alloc, m_value.binary, 1); + m_value.binary = nullptr; + } + + m_type = value_t::null; + assert_invariant(); + break; + } + + case value_t::object: + { + result.m_it.object_iterator = m_value.object->erase(first.m_it.object_iterator, + last.m_it.object_iterator); + break; + } + + case value_t::array: + { + result.m_it.array_iterator = m_value.array->erase(first.m_it.array_iterator, + last.m_it.array_iterator); + break; + } + + case value_t::null: + case value_t::discarded: + default: + JSON_THROW(type_error::create(307, "cannot use erase() with " + std::string(type_name()), *this)); + } + + return result; + } + + /// @brief remove element from a JSON object given a key + /// @sa https://json.nlohmann.me/api/basic_json/erase/ + size_type erase(const typename object_t::key_type& key) + { + // this erase only works for objects + if (JSON_HEDLEY_LIKELY(is_object())) + { + return m_value.object->erase(key); + } + + JSON_THROW(type_error::create(307, "cannot use erase() with " + std::string(type_name()), *this)); + } + + /// @brief remove element from a JSON array given an index + /// @sa https://json.nlohmann.me/api/basic_json/erase/ + void erase(const size_type idx) + { + // this erase only works for arrays + if (JSON_HEDLEY_LIKELY(is_array())) + { + if (JSON_HEDLEY_UNLIKELY(idx >= size())) + { + JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range", *this)); + } + + m_value.array->erase(m_value.array->begin() + static_cast(idx)); + } + else + { + JSON_THROW(type_error::create(307, "cannot use erase() with " + std::string(type_name()), *this)); + } + } + + /// @} + + + //////////// + // lookup // + //////////// + + /// @name lookup + /// @{ + + /// @brief find an element in a JSON object + /// @sa https://json.nlohmann.me/api/basic_json/find/ + template + iterator find(KeyT&& key) + { + auto result = end(); + + if (is_object()) + { + result.m_it.object_iterator = m_value.object->find(std::forward(key)); + } + + return result; + } + + /// @brief find an element in a JSON object + /// @sa https://json.nlohmann.me/api/basic_json/find/ + template + const_iterator find(KeyT&& key) const + { + auto result = cend(); + + if (is_object()) + { + result.m_it.object_iterator = m_value.object->find(std::forward(key)); + } + + return result; + } + + /// @brief returns the number of occurrences of a key in a JSON object + /// @sa https://json.nlohmann.me/api/basic_json/count/ + template + size_type count(KeyT&& key) const + { + // return 0 for all nonobject types + return is_object() ? m_value.object->count(std::forward(key)) : 0; + } + + /// @brief check the existence of an element in a JSON object + /// @sa https://json.nlohmann.me/api/basic_json/contains/ + template < typename KeyT, typename std::enable_if < + !std::is_same::type, json_pointer>::value, int >::type = 0 > + bool contains(KeyT && key) const + { + return is_object() && m_value.object->find(std::forward(key)) != m_value.object->end(); + } + + /// @brief check the existence of an element in a JSON object given a JSON pointer + /// @sa https://json.nlohmann.me/api/basic_json/contains/ + bool contains(const json_pointer& ptr) const + { + return ptr.contains(this); + } + + /// @} + + + /////////////// + // iterators // + /////////////// + + /// @name iterators + /// @{ + + /// @brief returns an iterator to the first element + /// @sa https://json.nlohmann.me/api/basic_json/begin/ + iterator begin() noexcept + { + iterator result(this); + result.set_begin(); + return result; + } + + /// @brief returns an iterator to the first element + /// @sa https://json.nlohmann.me/api/basic_json/begin/ + const_iterator begin() const noexcept + { + return cbegin(); + } + + /// @brief returns a const iterator to the first element + /// @sa https://json.nlohmann.me/api/basic_json/cbegin/ + const_iterator cbegin() const noexcept + { + const_iterator result(this); + result.set_begin(); + return result; + } + + /// @brief returns an iterator to one past the last element + /// @sa https://json.nlohmann.me/api/basic_json/end/ + iterator end() noexcept + { + iterator result(this); + result.set_end(); + return result; + } + + /// @brief returns an iterator to one past the last element + /// @sa https://json.nlohmann.me/api/basic_json/end/ + const_iterator end() const noexcept + { + return cend(); + } + + /// @brief returns an iterator to one past the last element + /// @sa https://json.nlohmann.me/api/basic_json/cend/ + const_iterator cend() const noexcept + { + const_iterator result(this); + result.set_end(); + return result; + } + + /// @brief returns an iterator to the reverse-beginning + /// @sa https://json.nlohmann.me/api/basic_json/rbegin/ + reverse_iterator rbegin() noexcept + { + return reverse_iterator(end()); + } + + /// @brief returns an iterator to the reverse-beginning + /// @sa https://json.nlohmann.me/api/basic_json/rbegin/ + const_reverse_iterator rbegin() const noexcept + { + return crbegin(); + } + + /// @brief returns an iterator to the reverse-end + /// @sa https://json.nlohmann.me/api/basic_json/rend/ + reverse_iterator rend() noexcept + { + return reverse_iterator(begin()); + } + + /// @brief returns an iterator to the reverse-end + /// @sa https://json.nlohmann.me/api/basic_json/rend/ + const_reverse_iterator rend() const noexcept + { + return crend(); + } + + /// @brief returns a const reverse iterator to the last element + /// @sa https://json.nlohmann.me/api/basic_json/crbegin/ + const_reverse_iterator crbegin() const noexcept + { + return const_reverse_iterator(cend()); + } + + /// @brief returns a const reverse iterator to one before the first + /// @sa https://json.nlohmann.me/api/basic_json/crend/ + const_reverse_iterator crend() const noexcept + { + return const_reverse_iterator(cbegin()); + } + + public: + /// @brief wrapper to access iterator member functions in range-based for + /// @sa https://json.nlohmann.me/api/basic_json/items/ + /// @deprecated This function is deprecated since 3.1.0 and will be removed in + /// version 4.0.0 of the library. Please use @ref items() instead; + /// that is, replace `json::iterator_wrapper(j)` with `j.items()`. + JSON_HEDLEY_DEPRECATED_FOR(3.1.0, items()) + static iteration_proxy iterator_wrapper(reference ref) noexcept + { + return ref.items(); + } + + /// @brief wrapper to access iterator member functions in range-based for + /// @sa https://json.nlohmann.me/api/basic_json/items/ + /// @deprecated This function is deprecated since 3.1.0 and will be removed in + /// version 4.0.0 of the library. Please use @ref items() instead; + /// that is, replace `json::iterator_wrapper(j)` with `j.items()`. + JSON_HEDLEY_DEPRECATED_FOR(3.1.0, items()) + static iteration_proxy iterator_wrapper(const_reference ref) noexcept + { + return ref.items(); + } + + /// @brief helper to access iterator member functions in range-based for + /// @sa https://json.nlohmann.me/api/basic_json/items/ + iteration_proxy items() noexcept + { + return iteration_proxy(*this); + } + + /// @brief helper to access iterator member functions in range-based for + /// @sa https://json.nlohmann.me/api/basic_json/items/ + iteration_proxy items() const noexcept + { + return iteration_proxy(*this); + } + + /// @} + + + ////////////// + // capacity // + ////////////// + + /// @name capacity + /// @{ + + /// @brief checks whether the container is empty. + /// @sa https://json.nlohmann.me/api/basic_json/empty/ + bool empty() const noexcept + { + switch (m_type) + { + case value_t::null: + { + // null values are empty + return true; + } + + case value_t::array: + { + // delegate call to array_t::empty() + return m_value.array->empty(); + } + + case value_t::object: + { + // delegate call to object_t::empty() + return m_value.object->empty(); + } + + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + // all other types are nonempty + return false; + } + } + } + + /// @brief returns the number of elements + /// @sa https://json.nlohmann.me/api/basic_json/size/ + size_type size() const noexcept + { + switch (m_type) + { + case value_t::null: + { + // null values are empty + return 0; + } + + case value_t::array: + { + // delegate call to array_t::size() + return m_value.array->size(); + } + + case value_t::object: + { + // delegate call to object_t::size() + return m_value.object->size(); + } + + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + // all other types have size 1 + return 1; + } + } + } + + /// @brief returns the maximum possible number of elements + /// @sa https://json.nlohmann.me/api/basic_json/max_size/ + size_type max_size() const noexcept + { + switch (m_type) + { + case value_t::array: + { + // delegate call to array_t::max_size() + return m_value.array->max_size(); + } + + case value_t::object: + { + // delegate call to object_t::max_size() + return m_value.object->max_size(); + } + + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + // all other types have max_size() == size() + return size(); + } + } + } + + /// @} + + + /////////////// + // modifiers // + /////////////// + + /// @name modifiers + /// @{ + + /// @brief clears the contents + /// @sa https://json.nlohmann.me/api/basic_json/clear/ + void clear() noexcept + { + switch (m_type) + { + case value_t::number_integer: + { + m_value.number_integer = 0; + break; + } + + case value_t::number_unsigned: + { + m_value.number_unsigned = 0; + break; + } + + case value_t::number_float: + { + m_value.number_float = 0.0; + break; + } + + case value_t::boolean: + { + m_value.boolean = false; + break; + } + + case value_t::string: + { + m_value.string->clear(); + break; + } + + case value_t::binary: + { + m_value.binary->clear(); + break; + } + + case value_t::array: + { + m_value.array->clear(); + break; + } + + case value_t::object: + { + m_value.object->clear(); + break; + } + + case value_t::null: + case value_t::discarded: + default: + break; + } + } + + /// @brief add an object to an array + /// @sa https://json.nlohmann.me/api/basic_json/push_back/ + void push_back(basic_json&& val) + { + // push_back only works for null objects or arrays + if (JSON_HEDLEY_UNLIKELY(!(is_null() || is_array()))) + { + JSON_THROW(type_error::create(308, "cannot use push_back() with " + std::string(type_name()), *this)); + } + + // transform null object into an array + if (is_null()) + { + m_type = value_t::array; + m_value = value_t::array; + assert_invariant(); + } + + // add element to array (move semantics) + const auto old_capacity = m_value.array->capacity(); + m_value.array->push_back(std::move(val)); + set_parent(m_value.array->back(), old_capacity); + // if val is moved from, basic_json move constructor marks it null, so we do not call the destructor + } + + /// @brief add an object to an array + /// @sa https://json.nlohmann.me/api/basic_json/operator+=/ + reference operator+=(basic_json&& val) + { + push_back(std::move(val)); + return *this; + } + + /// @brief add an object to an array + /// @sa https://json.nlohmann.me/api/basic_json/push_back/ + void push_back(const basic_json& val) + { + // push_back only works for null objects or arrays + if (JSON_HEDLEY_UNLIKELY(!(is_null() || is_array()))) + { + JSON_THROW(type_error::create(308, "cannot use push_back() with " + std::string(type_name()), *this)); + } + + // transform null object into an array + if (is_null()) + { + m_type = value_t::array; + m_value = value_t::array; + assert_invariant(); + } + + // add element to array + const auto old_capacity = m_value.array->capacity(); + m_value.array->push_back(val); + set_parent(m_value.array->back(), old_capacity); + } + + /// @brief add an object to an array + /// @sa https://json.nlohmann.me/api/basic_json/operator+=/ + reference operator+=(const basic_json& val) + { + push_back(val); + return *this; + } + + /// @brief add an object to an object + /// @sa https://json.nlohmann.me/api/basic_json/push_back/ + void push_back(const typename object_t::value_type& val) + { + // push_back only works for null objects or objects + if (JSON_HEDLEY_UNLIKELY(!(is_null() || is_object()))) + { + JSON_THROW(type_error::create(308, "cannot use push_back() with " + std::string(type_name()), *this)); + } + + // transform null object into an object + if (is_null()) + { + m_type = value_t::object; + m_value = value_t::object; + assert_invariant(); + } + + // add element to object + auto res = m_value.object->insert(val); + set_parent(res.first->second); + } + + /// @brief add an object to an object + /// @sa https://json.nlohmann.me/api/basic_json/operator+=/ + reference operator+=(const typename object_t::value_type& val) + { + push_back(val); + return *this; + } + + /// @brief add an object to an object + /// @sa https://json.nlohmann.me/api/basic_json/push_back/ + void push_back(initializer_list_t init) + { + if (is_object() && init.size() == 2 && (*init.begin())->is_string()) + { + basic_json&& key = init.begin()->moved_or_copied(); + push_back(typename object_t::value_type( + std::move(key.get_ref()), (init.begin() + 1)->moved_or_copied())); + } + else + { + push_back(basic_json(init)); + } + } + + /// @brief add an object to an object + /// @sa https://json.nlohmann.me/api/basic_json/operator+=/ + reference operator+=(initializer_list_t init) + { + push_back(init); + return *this; + } + + /// @brief add an object to an array + /// @sa https://json.nlohmann.me/api/basic_json/emplace_back/ + template + reference emplace_back(Args&& ... args) + { + // emplace_back only works for null objects or arrays + if (JSON_HEDLEY_UNLIKELY(!(is_null() || is_array()))) + { + JSON_THROW(type_error::create(311, "cannot use emplace_back() with " + std::string(type_name()), *this)); + } + + // transform null object into an array + if (is_null()) + { + m_type = value_t::array; + m_value = value_t::array; + assert_invariant(); + } + + // add element to array (perfect forwarding) + const auto old_capacity = m_value.array->capacity(); + m_value.array->emplace_back(std::forward(args)...); + return set_parent(m_value.array->back(), old_capacity); + } + + /// @brief add an object to an object if key does not exist + /// @sa https://json.nlohmann.me/api/basic_json/emplace/ + template + std::pair emplace(Args&& ... args) + { + // emplace only works for null objects or arrays + if (JSON_HEDLEY_UNLIKELY(!(is_null() || is_object()))) + { + JSON_THROW(type_error::create(311, "cannot use emplace() with " + std::string(type_name()), *this)); + } + + // transform null object into an object + if (is_null()) + { + m_type = value_t::object; + m_value = value_t::object; + assert_invariant(); + } + + // add element to array (perfect forwarding) + auto res = m_value.object->emplace(std::forward(args)...); + set_parent(res.first->second); + + // create result iterator and set iterator to the result of emplace + auto it = begin(); + it.m_it.object_iterator = res.first; + + // return pair of iterator and boolean + return {it, res.second}; + } + + /// Helper for insertion of an iterator + /// @note: This uses std::distance to support GCC 4.8, + /// see https://github.com/nlohmann/json/pull/1257 + template + iterator insert_iterator(const_iterator pos, Args&& ... args) + { + iterator result(this); + JSON_ASSERT(m_value.array != nullptr); + + auto insert_pos = std::distance(m_value.array->begin(), pos.m_it.array_iterator); + m_value.array->insert(pos.m_it.array_iterator, std::forward(args)...); + result.m_it.array_iterator = m_value.array->begin() + insert_pos; + + // This could have been written as: + // result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, cnt, val); + // but the return value of insert is missing in GCC 4.8, so it is written this way instead. + + set_parents(); + return result; + } + + /// @brief inserts element into array + /// @sa https://json.nlohmann.me/api/basic_json/insert/ + iterator insert(const_iterator pos, const basic_json& val) + { + // insert only works for arrays + if (JSON_HEDLEY_LIKELY(is_array())) + { + // check if iterator pos fits to this JSON value + if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) + { + JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value", *this)); + } + + // insert to array and return iterator + return insert_iterator(pos, val); + } + + JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name()), *this)); + } + + /// @brief inserts element into array + /// @sa https://json.nlohmann.me/api/basic_json/insert/ + iterator insert(const_iterator pos, basic_json&& val) + { + return insert(pos, val); + } + + /// @brief inserts copies of element into array + /// @sa https://json.nlohmann.me/api/basic_json/insert/ + iterator insert(const_iterator pos, size_type cnt, const basic_json& val) + { + // insert only works for arrays + if (JSON_HEDLEY_LIKELY(is_array())) + { + // check if iterator pos fits to this JSON value + if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) + { + JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value", *this)); + } + + // insert to array and return iterator + return insert_iterator(pos, cnt, val); + } + + JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name()), *this)); + } + + /// @brief inserts range of elements into array + /// @sa https://json.nlohmann.me/api/basic_json/insert/ + iterator insert(const_iterator pos, const_iterator first, const_iterator last) + { + // insert only works for arrays + if (JSON_HEDLEY_UNLIKELY(!is_array())) + { + JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name()), *this)); + } + + // check if iterator pos fits to this JSON value + if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) + { + JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value", *this)); + } + + // check if range iterators belong to the same JSON object + if (JSON_HEDLEY_UNLIKELY(first.m_object != last.m_object)) + { + JSON_THROW(invalid_iterator::create(210, "iterators do not fit", *this)); + } + + if (JSON_HEDLEY_UNLIKELY(first.m_object == this)) + { + JSON_THROW(invalid_iterator::create(211, "passed iterators may not belong to container", *this)); + } + + // insert to array and return iterator + return insert_iterator(pos, first.m_it.array_iterator, last.m_it.array_iterator); + } + + /// @brief inserts elements from initializer list into array + /// @sa https://json.nlohmann.me/api/basic_json/insert/ + iterator insert(const_iterator pos, initializer_list_t ilist) + { + // insert only works for arrays + if (JSON_HEDLEY_UNLIKELY(!is_array())) + { + JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name()), *this)); + } + + // check if iterator pos fits to this JSON value + if (JSON_HEDLEY_UNLIKELY(pos.m_object != this)) + { + JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value", *this)); + } + + // insert to array and return iterator + return insert_iterator(pos, ilist.begin(), ilist.end()); + } + + /// @brief inserts range of elements into object + /// @sa https://json.nlohmann.me/api/basic_json/insert/ + void insert(const_iterator first, const_iterator last) + { + // insert only works for objects + if (JSON_HEDLEY_UNLIKELY(!is_object())) + { + JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name()), *this)); + } + + // check if range iterators belong to the same JSON object + if (JSON_HEDLEY_UNLIKELY(first.m_object != last.m_object)) + { + JSON_THROW(invalid_iterator::create(210, "iterators do not fit", *this)); + } + + // passed iterators must belong to objects + if (JSON_HEDLEY_UNLIKELY(!first.m_object->is_object())) + { + JSON_THROW(invalid_iterator::create(202, "iterators first and last must point to objects", *this)); + } + + m_value.object->insert(first.m_it.object_iterator, last.m_it.object_iterator); + } + + /// @brief updates a JSON object from another object, overwriting existing keys + /// @sa https://json.nlohmann.me/api/basic_json/update/ + void update(const_reference j, bool merge_objects = false) + { + update(j.begin(), j.end(), merge_objects); + } + + /// @brief updates a JSON object from another object, overwriting existing keys + /// @sa https://json.nlohmann.me/api/basic_json/update/ + void update(const_iterator first, const_iterator last, bool merge_objects = false) + { + // implicitly convert null value to an empty object + if (is_null()) + { + m_type = value_t::object; + m_value.object = create(); + assert_invariant(); + } + + if (JSON_HEDLEY_UNLIKELY(!is_object())) + { + JSON_THROW(type_error::create(312, "cannot use update() with " + std::string(type_name()), *this)); + } + + // check if range iterators belong to the same JSON object + if (JSON_HEDLEY_UNLIKELY(first.m_object != last.m_object)) + { + JSON_THROW(invalid_iterator::create(210, "iterators do not fit", *this)); + } + + // passed iterators must belong to objects + if (JSON_HEDLEY_UNLIKELY(!first.m_object->is_object())) + { + JSON_THROW(type_error::create(312, "cannot use update() with " + std::string(first.m_object->type_name()), *first.m_object)); + } + + for (auto it = first; it != last; ++it) + { + if (merge_objects && it.value().is_object()) + { + auto it2 = m_value.object->find(it.key()); + if (it2 != m_value.object->end()) + { + it2->second.update(it.value(), true); + continue; + } + } + m_value.object->operator[](it.key()) = it.value(); +#if JSON_DIAGNOSTICS + m_value.object->operator[](it.key()).m_parent = this; +#endif + } + } + + /// @brief exchanges the values + /// @sa https://json.nlohmann.me/api/basic_json/swap/ + void swap(reference other) noexcept ( + std::is_nothrow_move_constructible::value&& + std::is_nothrow_move_assignable::value&& + std::is_nothrow_move_constructible::value&& + std::is_nothrow_move_assignable::value + ) + { + std::swap(m_type, other.m_type); + std::swap(m_value, other.m_value); + + set_parents(); + other.set_parents(); + assert_invariant(); + } + + /// @brief exchanges the values + /// @sa https://json.nlohmann.me/api/basic_json/swap/ + friend void swap(reference left, reference right) noexcept ( + std::is_nothrow_move_constructible::value&& + std::is_nothrow_move_assignable::value&& + std::is_nothrow_move_constructible::value&& + std::is_nothrow_move_assignable::value + ) + { + left.swap(right); + } + + /// @brief exchanges the values + /// @sa https://json.nlohmann.me/api/basic_json/swap/ + void swap(array_t& other) // NOLINT(bugprone-exception-escape) + { + // swap only works for arrays + if (JSON_HEDLEY_LIKELY(is_array())) + { + std::swap(*(m_value.array), other); + } + else + { + JSON_THROW(type_error::create(310, "cannot use swap() with " + std::string(type_name()), *this)); + } + } + + /// @brief exchanges the values + /// @sa https://json.nlohmann.me/api/basic_json/swap/ + void swap(object_t& other) // NOLINT(bugprone-exception-escape) + { + // swap only works for objects + if (JSON_HEDLEY_LIKELY(is_object())) + { + std::swap(*(m_value.object), other); + } + else + { + JSON_THROW(type_error::create(310, "cannot use swap() with " + std::string(type_name()), *this)); + } + } + + /// @brief exchanges the values + /// @sa https://json.nlohmann.me/api/basic_json/swap/ + void swap(string_t& other) // NOLINT(bugprone-exception-escape) + { + // swap only works for strings + if (JSON_HEDLEY_LIKELY(is_string())) + { + std::swap(*(m_value.string), other); + } + else + { + JSON_THROW(type_error::create(310, "cannot use swap() with " + std::string(type_name()), *this)); + } + } + + /// @brief exchanges the values + /// @sa https://json.nlohmann.me/api/basic_json/swap/ + void swap(binary_t& other) // NOLINT(bugprone-exception-escape) + { + // swap only works for strings + if (JSON_HEDLEY_LIKELY(is_binary())) + { + std::swap(*(m_value.binary), other); + } + else + { + JSON_THROW(type_error::create(310, "cannot use swap() with " + std::string(type_name()), *this)); + } + } + + /// @brief exchanges the values + /// @sa https://json.nlohmann.me/api/basic_json/swap/ + void swap(typename binary_t::container_type& other) // NOLINT(bugprone-exception-escape) + { + // swap only works for strings + if (JSON_HEDLEY_LIKELY(is_binary())) + { + std::swap(*(m_value.binary), other); + } + else + { + JSON_THROW(type_error::create(310, "cannot use swap() with " + std::string(type_name()), *this)); + } + } + + /// @} + + public: + ////////////////////////////////////////// + // lexicographical comparison operators // + ////////////////////////////////////////// + + /// @name lexicographical comparison operators + /// @{ + + /// @brief comparison: equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_eq/ + friend bool operator==(const_reference lhs, const_reference rhs) noexcept + { +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wfloat-equal" +#endif + const auto lhs_type = lhs.type(); + const auto rhs_type = rhs.type(); + + if (lhs_type == rhs_type) + { + switch (lhs_type) + { + case value_t::array: + return *lhs.m_value.array == *rhs.m_value.array; + + case value_t::object: + return *lhs.m_value.object == *rhs.m_value.object; + + case value_t::null: + return true; + + case value_t::string: + return *lhs.m_value.string == *rhs.m_value.string; + + case value_t::boolean: + return lhs.m_value.boolean == rhs.m_value.boolean; + + case value_t::number_integer: + return lhs.m_value.number_integer == rhs.m_value.number_integer; + + case value_t::number_unsigned: + return lhs.m_value.number_unsigned == rhs.m_value.number_unsigned; + + case value_t::number_float: + return lhs.m_value.number_float == rhs.m_value.number_float; + + case value_t::binary: + return *lhs.m_value.binary == *rhs.m_value.binary; + + case value_t::discarded: + default: + return false; + } + } + else if (lhs_type == value_t::number_integer && rhs_type == value_t::number_float) + { + return static_cast(lhs.m_value.number_integer) == rhs.m_value.number_float; + } + else if (lhs_type == value_t::number_float && rhs_type == value_t::number_integer) + { + return lhs.m_value.number_float == static_cast(rhs.m_value.number_integer); + } + else if (lhs_type == value_t::number_unsigned && rhs_type == value_t::number_float) + { + return static_cast(lhs.m_value.number_unsigned) == rhs.m_value.number_float; + } + else if (lhs_type == value_t::number_float && rhs_type == value_t::number_unsigned) + { + return lhs.m_value.number_float == static_cast(rhs.m_value.number_unsigned); + } + else if (lhs_type == value_t::number_unsigned && rhs_type == value_t::number_integer) + { + return static_cast(lhs.m_value.number_unsigned) == rhs.m_value.number_integer; + } + else if (lhs_type == value_t::number_integer && rhs_type == value_t::number_unsigned) + { + return lhs.m_value.number_integer == static_cast(rhs.m_value.number_unsigned); + } + + return false; +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif + } + + /// @brief comparison: equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_eq/ + template::value, int>::type = 0> + friend bool operator==(const_reference lhs, ScalarType rhs) noexcept + { + return lhs == basic_json(rhs); + } + + /// @brief comparison: equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_eq/ + template::value, int>::type = 0> + friend bool operator==(ScalarType lhs, const_reference rhs) noexcept + { + return basic_json(lhs) == rhs; + } + + /// @brief comparison: not equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_ne/ + friend bool operator!=(const_reference lhs, const_reference rhs) noexcept + { + return !(lhs == rhs); + } + + /// @brief comparison: not equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_ne/ + template::value, int>::type = 0> + friend bool operator!=(const_reference lhs, ScalarType rhs) noexcept + { + return lhs != basic_json(rhs); + } + + /// @brief comparison: not equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_ne/ + template::value, int>::type = 0> + friend bool operator!=(ScalarType lhs, const_reference rhs) noexcept + { + return basic_json(lhs) != rhs; + } + + /// @brief comparison: less than + /// @sa https://json.nlohmann.me/api/basic_json/operator_lt/ + friend bool operator<(const_reference lhs, const_reference rhs) noexcept + { + const auto lhs_type = lhs.type(); + const auto rhs_type = rhs.type(); + + if (lhs_type == rhs_type) + { + switch (lhs_type) + { + case value_t::array: + // note parentheses are necessary, see + // https://github.com/nlohmann/json/issues/1530 + return (*lhs.m_value.array) < (*rhs.m_value.array); + + case value_t::object: + return (*lhs.m_value.object) < (*rhs.m_value.object); + + case value_t::null: + return false; + + case value_t::string: + return (*lhs.m_value.string) < (*rhs.m_value.string); + + case value_t::boolean: + return (lhs.m_value.boolean) < (rhs.m_value.boolean); + + case value_t::number_integer: + return (lhs.m_value.number_integer) < (rhs.m_value.number_integer); + + case value_t::number_unsigned: + return (lhs.m_value.number_unsigned) < (rhs.m_value.number_unsigned); + + case value_t::number_float: + return (lhs.m_value.number_float) < (rhs.m_value.number_float); + + case value_t::binary: + return (*lhs.m_value.binary) < (*rhs.m_value.binary); + + case value_t::discarded: + default: + return false; + } + } + else if (lhs_type == value_t::number_integer && rhs_type == value_t::number_float) + { + return static_cast(lhs.m_value.number_integer) < rhs.m_value.number_float; + } + else if (lhs_type == value_t::number_float && rhs_type == value_t::number_integer) + { + return lhs.m_value.number_float < static_cast(rhs.m_value.number_integer); + } + else if (lhs_type == value_t::number_unsigned && rhs_type == value_t::number_float) + { + return static_cast(lhs.m_value.number_unsigned) < rhs.m_value.number_float; + } + else if (lhs_type == value_t::number_float && rhs_type == value_t::number_unsigned) + { + return lhs.m_value.number_float < static_cast(rhs.m_value.number_unsigned); + } + else if (lhs_type == value_t::number_integer && rhs_type == value_t::number_unsigned) + { + return lhs.m_value.number_integer < static_cast(rhs.m_value.number_unsigned); + } + else if (lhs_type == value_t::number_unsigned && rhs_type == value_t::number_integer) + { + return static_cast(lhs.m_value.number_unsigned) < rhs.m_value.number_integer; + } + + // We only reach this line if we cannot compare values. In that case, + // we compare types. Note we have to call the operator explicitly, + // because MSVC has problems otherwise. + return operator<(lhs_type, rhs_type); + } + + /// @brief comparison: less than + /// @sa https://json.nlohmann.me/api/basic_json/operator_lt/ + template::value, int>::type = 0> + friend bool operator<(const_reference lhs, ScalarType rhs) noexcept + { + return lhs < basic_json(rhs); + } + + /// @brief comparison: less than + /// @sa https://json.nlohmann.me/api/basic_json/operator_lt/ + template::value, int>::type = 0> + friend bool operator<(ScalarType lhs, const_reference rhs) noexcept + { + return basic_json(lhs) < rhs; + } + + /// @brief comparison: less than or equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_le/ + friend bool operator<=(const_reference lhs, const_reference rhs) noexcept + { + return !(rhs < lhs); + } + + /// @brief comparison: less than or equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_le/ + template::value, int>::type = 0> + friend bool operator<=(const_reference lhs, ScalarType rhs) noexcept + { + return lhs <= basic_json(rhs); + } + + /// @brief comparison: less than or equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_le/ + template::value, int>::type = 0> + friend bool operator<=(ScalarType lhs, const_reference rhs) noexcept + { + return basic_json(lhs) <= rhs; + } + + /// @brief comparison: greater than + /// @sa https://json.nlohmann.me/api/basic_json/operator_gt/ + friend bool operator>(const_reference lhs, const_reference rhs) noexcept + { + return !(lhs <= rhs); + } + + /// @brief comparison: greater than + /// @sa https://json.nlohmann.me/api/basic_json/operator_gt/ + template::value, int>::type = 0> + friend bool operator>(const_reference lhs, ScalarType rhs) noexcept + { + return lhs > basic_json(rhs); + } + + /// @brief comparison: greater than + /// @sa https://json.nlohmann.me/api/basic_json/operator_gt/ + template::value, int>::type = 0> + friend bool operator>(ScalarType lhs, const_reference rhs) noexcept + { + return basic_json(lhs) > rhs; + } + + /// @brief comparison: greater than or equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_ge/ + friend bool operator>=(const_reference lhs, const_reference rhs) noexcept + { + return !(lhs < rhs); + } + + /// @brief comparison: greater than or equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_ge/ + template::value, int>::type = 0> + friend bool operator>=(const_reference lhs, ScalarType rhs) noexcept + { + return lhs >= basic_json(rhs); + } + + /// @brief comparison: greater than or equal + /// @sa https://json.nlohmann.me/api/basic_json/operator_ge/ + template::value, int>::type = 0> + friend bool operator>=(ScalarType lhs, const_reference rhs) noexcept + { + return basic_json(lhs) >= rhs; + } + + /// @} + + /////////////////// + // serialization // + /////////////////// + + /// @name serialization + /// @{ +#ifndef JSON_NO_IO + /// @brief serialize to stream + /// @sa https://json.nlohmann.me/api/basic_json/operator_ltlt/ + friend std::ostream& operator<<(std::ostream& o, const basic_json& j) + { + // read width member and use it as indentation parameter if nonzero + const bool pretty_print = o.width() > 0; + const auto indentation = pretty_print ? o.width() : 0; + + // reset width to 0 for subsequent calls to this stream + o.width(0); + + // do the actual serialization + serializer s(detail::output_adapter(o), o.fill()); + s.dump(j, pretty_print, false, static_cast(indentation)); + return o; + } + + /// @brief serialize to stream + /// @sa https://json.nlohmann.me/api/basic_json/operator_ltlt/ + /// @deprecated This function is deprecated since 3.0.0 and will be removed in + /// version 4.0.0 of the library. Please use + /// operator<<(std::ostream&, const basic_json&) instead; that is, + /// replace calls like `j >> o;` with `o << j;`. + JSON_HEDLEY_DEPRECATED_FOR(3.0.0, operator<<(std::ostream&, const basic_json&)) + friend std::ostream& operator>>(const basic_json& j, std::ostream& o) + { + return o << j; + } +#endif // JSON_NO_IO + /// @} + + + ///////////////////// + // deserialization // + ///////////////////// + + /// @name deserialization + /// @{ + + /// @brief deserialize from a compatible input + /// @sa https://json.nlohmann.me/api/basic_json/parse/ + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json parse(InputType&& i, + const parser_callback_t cb = nullptr, + const bool allow_exceptions = true, + const bool ignore_comments = false) + { + basic_json result; + parser(detail::input_adapter(std::forward(i)), cb, allow_exceptions, ignore_comments).parse(true, result); + return result; + } + + /// @brief deserialize from a pair of character iterators + /// @sa https://json.nlohmann.me/api/basic_json/parse/ + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json parse(IteratorType first, + IteratorType last, + const parser_callback_t cb = nullptr, + const bool allow_exceptions = true, + const bool ignore_comments = false) + { + basic_json result; + parser(detail::input_adapter(std::move(first), std::move(last)), cb, allow_exceptions, ignore_comments).parse(true, result); + return result; + } + + JSON_HEDLEY_WARN_UNUSED_RESULT + JSON_HEDLEY_DEPRECATED_FOR(3.8.0, parse(ptr, ptr + len)) + static basic_json parse(detail::span_input_adapter&& i, + const parser_callback_t cb = nullptr, + const bool allow_exceptions = true, + const bool ignore_comments = false) + { + basic_json result; + parser(i.get(), cb, allow_exceptions, ignore_comments).parse(true, result); + return result; + } + + /// @brief check if the input is valid JSON + /// @sa https://json.nlohmann.me/api/basic_json/accept/ + template + static bool accept(InputType&& i, + const bool ignore_comments = false) + { + return parser(detail::input_adapter(std::forward(i)), nullptr, false, ignore_comments).accept(true); + } + + /// @brief check if the input is valid JSON + /// @sa https://json.nlohmann.me/api/basic_json/accept/ + template + static bool accept(IteratorType first, IteratorType last, + const bool ignore_comments = false) + { + return parser(detail::input_adapter(std::move(first), std::move(last)), nullptr, false, ignore_comments).accept(true); + } + + JSON_HEDLEY_WARN_UNUSED_RESULT + JSON_HEDLEY_DEPRECATED_FOR(3.8.0, accept(ptr, ptr + len)) + static bool accept(detail::span_input_adapter&& i, + const bool ignore_comments = false) + { + return parser(i.get(), nullptr, false, ignore_comments).accept(true); + } + + /// @brief generate SAX events + /// @sa https://json.nlohmann.me/api/basic_json/sax_parse/ + template + JSON_HEDLEY_NON_NULL(2) + static bool sax_parse(InputType&& i, SAX* sax, + input_format_t format = input_format_t::json, + const bool strict = true, + const bool ignore_comments = false) + { + auto ia = detail::input_adapter(std::forward(i)); + return format == input_format_t::json + ? parser(std::move(ia), nullptr, true, ignore_comments).sax_parse(sax, strict) + : detail::binary_reader(std::move(ia)).sax_parse(format, sax, strict); + } + + /// @brief generate SAX events + /// @sa https://json.nlohmann.me/api/basic_json/sax_parse/ + template + JSON_HEDLEY_NON_NULL(3) + static bool sax_parse(IteratorType first, IteratorType last, SAX* sax, + input_format_t format = input_format_t::json, + const bool strict = true, + const bool ignore_comments = false) + { + auto ia = detail::input_adapter(std::move(first), std::move(last)); + return format == input_format_t::json + ? parser(std::move(ia), nullptr, true, ignore_comments).sax_parse(sax, strict) + : detail::binary_reader(std::move(ia)).sax_parse(format, sax, strict); + } + + /// @brief generate SAX events + /// @sa https://json.nlohmann.me/api/basic_json/sax_parse/ + /// @deprecated This function is deprecated since 3.8.0 and will be removed in + /// version 4.0.0 of the library. Please use + /// sax_parse(ptr, ptr + len) instead. + template + JSON_HEDLEY_DEPRECATED_FOR(3.8.0, sax_parse(ptr, ptr + len, ...)) + JSON_HEDLEY_NON_NULL(2) + static bool sax_parse(detail::span_input_adapter&& i, SAX* sax, + input_format_t format = input_format_t::json, + const bool strict = true, + const bool ignore_comments = false) + { + auto ia = i.get(); + return format == input_format_t::json + // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) + ? parser(std::move(ia), nullptr, true, ignore_comments).sax_parse(sax, strict) + // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) + : detail::binary_reader(std::move(ia)).sax_parse(format, sax, strict); + } +#ifndef JSON_NO_IO + /// @brief deserialize from stream + /// @sa https://json.nlohmann.me/api/basic_json/operator_gtgt/ + /// @deprecated This stream operator is deprecated since 3.0.0 and will be removed in + /// version 4.0.0 of the library. Please use + /// operator>>(std::istream&, basic_json&) instead; that is, + /// replace calls like `j << i;` with `i >> j;`. + JSON_HEDLEY_DEPRECATED_FOR(3.0.0, operator>>(std::istream&, basic_json&)) + friend std::istream& operator<<(basic_json& j, std::istream& i) + { + return operator>>(i, j); + } + + /// @brief deserialize from stream + /// @sa https://json.nlohmann.me/api/basic_json/operator_gtgt/ + friend std::istream& operator>>(std::istream& i, basic_json& j) + { + parser(detail::input_adapter(i)).parse(false, j); + return i; + } +#endif // JSON_NO_IO + /// @} + + /////////////////////////// + // convenience functions // + /////////////////////////// + + /// @brief return the type as string + /// @sa https://json.nlohmann.me/api/basic_json/type_name/ + JSON_HEDLEY_RETURNS_NON_NULL + const char* type_name() const noexcept + { + switch (m_type) + { + case value_t::null: + return "null"; + case value_t::object: + return "object"; + case value_t::array: + return "array"; + case value_t::string: + return "string"; + case value_t::boolean: + return "boolean"; + case value_t::binary: + return "binary"; + case value_t::discarded: + return "discarded"; + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + default: + return "number"; + } + } + + + JSON_PRIVATE_UNLESS_TESTED: + ////////////////////// + // member variables // + ////////////////////// + + /// the type of the current element + value_t m_type = value_t::null; + + /// the value of the current element + json_value m_value = {}; + +#if JSON_DIAGNOSTICS + /// a pointer to a parent value (for debugging purposes) + basic_json* m_parent = nullptr; +#endif + + ////////////////////////////////////////// + // binary serialization/deserialization // + ////////////////////////////////////////// + + /// @name binary serialization/deserialization support + /// @{ + + public: + /// @brief create a CBOR serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_cbor/ + static std::vector to_cbor(const basic_json& j) + { + std::vector result; + to_cbor(j, result); + return result; + } + + /// @brief create a CBOR serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_cbor/ + static void to_cbor(const basic_json& j, detail::output_adapter o) + { + binary_writer(o).write_cbor(j); + } + + /// @brief create a CBOR serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_cbor/ + static void to_cbor(const basic_json& j, detail::output_adapter o) + { + binary_writer(o).write_cbor(j); + } + + /// @brief create a MessagePack serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_msgpack/ + static std::vector to_msgpack(const basic_json& j) + { + std::vector result; + to_msgpack(j, result); + return result; + } + + /// @brief create a MessagePack serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_msgpack/ + static void to_msgpack(const basic_json& j, detail::output_adapter o) + { + binary_writer(o).write_msgpack(j); + } + + /// @brief create a MessagePack serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_msgpack/ + static void to_msgpack(const basic_json& j, detail::output_adapter o) + { + binary_writer(o).write_msgpack(j); + } + + /// @brief create a UBJSON serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_ubjson/ + static std::vector to_ubjson(const basic_json& j, + const bool use_size = false, + const bool use_type = false) + { + std::vector result; + to_ubjson(j, result, use_size, use_type); + return result; + } + + /// @brief create a UBJSON serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_ubjson/ + static void to_ubjson(const basic_json& j, detail::output_adapter o, + const bool use_size = false, const bool use_type = false) + { + binary_writer(o).write_ubjson(j, use_size, use_type); + } + + /// @brief create a UBJSON serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_ubjson/ + static void to_ubjson(const basic_json& j, detail::output_adapter o, + const bool use_size = false, const bool use_type = false) + { + binary_writer(o).write_ubjson(j, use_size, use_type); + } + + /// @brief create a BSON serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_bson/ + static std::vector to_bson(const basic_json& j) + { + std::vector result; + to_bson(j, result); + return result; + } + + /// @brief create a BSON serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_bson/ + static void to_bson(const basic_json& j, detail::output_adapter o) + { + binary_writer(o).write_bson(j); + } + + /// @brief create a BSON serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_bson/ + static void to_bson(const basic_json& j, detail::output_adapter o) + { + binary_writer(o).write_bson(j); + } + + /// @brief create a JSON value from an input in CBOR format + /// @sa https://json.nlohmann.me/api/basic_json/from_cbor/ + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json from_cbor(InputType&& i, + const bool strict = true, + const bool allow_exceptions = true, + const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = detail::input_adapter(std::forward(i)); + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); + return res ? result : basic_json(value_t::discarded); + } + + /// @brief create a JSON value from an input in CBOR format + /// @sa https://json.nlohmann.me/api/basic_json/from_cbor/ + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json from_cbor(IteratorType first, IteratorType last, + const bool strict = true, + const bool allow_exceptions = true, + const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = detail::input_adapter(std::move(first), std::move(last)); + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); + return res ? result : basic_json(value_t::discarded); + } + + template + JSON_HEDLEY_WARN_UNUSED_RESULT + JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_cbor(ptr, ptr + len)) + static basic_json from_cbor(const T* ptr, std::size_t len, + const bool strict = true, + const bool allow_exceptions = true, + const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) + { + return from_cbor(ptr, ptr + len, strict, allow_exceptions, tag_handler); + } + + + JSON_HEDLEY_WARN_UNUSED_RESULT + JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_cbor(ptr, ptr + len)) + static basic_json from_cbor(detail::span_input_adapter&& i, + const bool strict = true, + const bool allow_exceptions = true, + const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = i.get(); + // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); + return res ? result : basic_json(value_t::discarded); + } + + /// @brief create a JSON value from an input in MessagePack format + /// @sa https://json.nlohmann.me/api/basic_json/from_msgpack/ + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json from_msgpack(InputType&& i, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = detail::input_adapter(std::forward(i)); + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::msgpack, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + /// @brief create a JSON value from an input in MessagePack format + /// @sa https://json.nlohmann.me/api/basic_json/from_msgpack/ + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json from_msgpack(IteratorType first, IteratorType last, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = detail::input_adapter(std::move(first), std::move(last)); + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::msgpack, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + template + JSON_HEDLEY_WARN_UNUSED_RESULT + JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_msgpack(ptr, ptr + len)) + static basic_json from_msgpack(const T* ptr, std::size_t len, + const bool strict = true, + const bool allow_exceptions = true) + { + return from_msgpack(ptr, ptr + len, strict, allow_exceptions); + } + + JSON_HEDLEY_WARN_UNUSED_RESULT + JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_msgpack(ptr, ptr + len)) + static basic_json from_msgpack(detail::span_input_adapter&& i, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = i.get(); + // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::msgpack, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + /// @brief create a JSON value from an input in UBJSON format + /// @sa https://json.nlohmann.me/api/basic_json/from_ubjson/ + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json from_ubjson(InputType&& i, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = detail::input_adapter(std::forward(i)); + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::ubjson, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + /// @brief create a JSON value from an input in UBJSON format + /// @sa https://json.nlohmann.me/api/basic_json/from_ubjson/ + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json from_ubjson(IteratorType first, IteratorType last, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = detail::input_adapter(std::move(first), std::move(last)); + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::ubjson, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + template + JSON_HEDLEY_WARN_UNUSED_RESULT + JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_ubjson(ptr, ptr + len)) + static basic_json from_ubjson(const T* ptr, std::size_t len, + const bool strict = true, + const bool allow_exceptions = true) + { + return from_ubjson(ptr, ptr + len, strict, allow_exceptions); + } + + JSON_HEDLEY_WARN_UNUSED_RESULT + JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_ubjson(ptr, ptr + len)) + static basic_json from_ubjson(detail::span_input_adapter&& i, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = i.get(); + // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::ubjson, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + /// @brief create a JSON value from an input in BSON format + /// @sa https://json.nlohmann.me/api/basic_json/from_bson/ + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json from_bson(InputType&& i, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = detail::input_adapter(std::forward(i)); + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::bson, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + /// @brief create a JSON value from an input in BSON format + /// @sa https://json.nlohmann.me/api/basic_json/from_bson/ + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json from_bson(IteratorType first, IteratorType last, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = detail::input_adapter(std::move(first), std::move(last)); + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::bson, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + template + JSON_HEDLEY_WARN_UNUSED_RESULT + JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_bson(ptr, ptr + len)) + static basic_json from_bson(const T* ptr, std::size_t len, + const bool strict = true, + const bool allow_exceptions = true) + { + return from_bson(ptr, ptr + len, strict, allow_exceptions); + } + + JSON_HEDLEY_WARN_UNUSED_RESULT + JSON_HEDLEY_DEPRECATED_FOR(3.8.0, from_bson(ptr, ptr + len)) + static basic_json from_bson(detail::span_input_adapter&& i, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = i.get(); + // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) + const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::bson, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + /// @} + + ////////////////////////// + // JSON Pointer support // + ////////////////////////// + + /// @name JSON Pointer functions + /// @{ + + /// @brief access specified element via JSON Pointer + /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/ + reference operator[](const json_pointer& ptr) + { + return ptr.get_unchecked(this); + } + + /// @brief access specified element via JSON Pointer + /// @sa https://json.nlohmann.me/api/basic_json/operator%5B%5D/ + const_reference operator[](const json_pointer& ptr) const + { + return ptr.get_unchecked(this); + } + + /// @brief access specified element via JSON Pointer + /// @sa https://json.nlohmann.me/api/basic_json/at/ + reference at(const json_pointer& ptr) + { + return ptr.get_checked(this); + } + + /// @brief access specified element via JSON Pointer + /// @sa https://json.nlohmann.me/api/basic_json/at/ + const_reference at(const json_pointer& ptr) const + { + return ptr.get_checked(this); + } + + /// @brief return flattened JSON value + /// @sa https://json.nlohmann.me/api/basic_json/flatten/ + basic_json flatten() const + { + basic_json result(value_t::object); + json_pointer::flatten("", *this, result); + return result; + } + + /// @brief unflatten a previously flattened JSON value + /// @sa https://json.nlohmann.me/api/basic_json/unflatten/ + basic_json unflatten() const + { + return json_pointer::unflatten(*this); + } + + /// @} + + ////////////////////////// + // JSON Patch functions // + ////////////////////////// + + /// @name JSON Patch functions + /// @{ + + /// @brief applies a JSON patch + /// @sa https://json.nlohmann.me/api/basic_json/patch/ + basic_json patch(const basic_json& json_patch) const + { + // make a working copy to apply the patch to + basic_json result = *this; + + // the valid JSON Patch operations + enum class patch_operations {add, remove, replace, move, copy, test, invalid}; + + const auto get_op = [](const std::string & op) + { + if (op == "add") + { + return patch_operations::add; + } + if (op == "remove") + { + return patch_operations::remove; + } + if (op == "replace") + { + return patch_operations::replace; + } + if (op == "move") + { + return patch_operations::move; + } + if (op == "copy") + { + return patch_operations::copy; + } + if (op == "test") + { + return patch_operations::test; + } + + return patch_operations::invalid; + }; + + // wrapper for "add" operation; add value at ptr + const auto operation_add = [&result](json_pointer & ptr, basic_json val) + { + // adding to the root of the target document means replacing it + if (ptr.empty()) + { + result = val; + return; + } + + // make sure the top element of the pointer exists + json_pointer top_pointer = ptr.top(); + if (top_pointer != ptr) + { + result.at(top_pointer); + } + + // get reference to parent of JSON pointer ptr + const auto last_path = ptr.back(); + ptr.pop_back(); + basic_json& parent = result[ptr]; + + switch (parent.m_type) + { + case value_t::null: + case value_t::object: + { + // use operator[] to add value + parent[last_path] = val; + break; + } + + case value_t::array: + { + if (last_path == "-") + { + // special case: append to back + parent.push_back(val); + } + else + { + const auto idx = json_pointer::array_index(last_path); + if (JSON_HEDLEY_UNLIKELY(idx > parent.size())) + { + // avoid undefined behavior + JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range", parent)); + } + + // default case: insert add offset + parent.insert(parent.begin() + static_cast(idx), val); + } + break; + } + + // if there exists a parent it cannot be primitive + case value_t::string: // LCOV_EXCL_LINE + case value_t::boolean: // LCOV_EXCL_LINE + case value_t::number_integer: // LCOV_EXCL_LINE + case value_t::number_unsigned: // LCOV_EXCL_LINE + case value_t::number_float: // LCOV_EXCL_LINE + case value_t::binary: // LCOV_EXCL_LINE + case value_t::discarded: // LCOV_EXCL_LINE + default: // LCOV_EXCL_LINE + JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE + } + }; + + // wrapper for "remove" operation; remove value at ptr + const auto operation_remove = [this, &result](json_pointer & ptr) + { + // get reference to parent of JSON pointer ptr + const auto last_path = ptr.back(); + ptr.pop_back(); + basic_json& parent = result.at(ptr); + + // remove child + if (parent.is_object()) + { + // perform range check + auto it = parent.find(last_path); + if (JSON_HEDLEY_LIKELY(it != parent.end())) + { + parent.erase(it); + } + else + { + JSON_THROW(out_of_range::create(403, "key '" + last_path + "' not found", *this)); + } + } + else if (parent.is_array()) + { + // note erase performs range check + parent.erase(json_pointer::array_index(last_path)); + } + }; + + // type check: top level value must be an array + if (JSON_HEDLEY_UNLIKELY(!json_patch.is_array())) + { + JSON_THROW(parse_error::create(104, 0, "JSON patch must be an array of objects", json_patch)); + } + + // iterate and apply the operations + for (const auto& val : json_patch) + { + // wrapper to get a value for an operation + const auto get_value = [&val](const std::string & op, + const std::string & member, + bool string_type) -> basic_json & + { + // find value + auto it = val.m_value.object->find(member); + + // context-sensitive error message + const auto error_msg = (op == "op") ? "operation" : "operation '" + op + "'"; + + // check if desired value is present + if (JSON_HEDLEY_UNLIKELY(it == val.m_value.object->end())) + { + // NOLINTNEXTLINE(performance-inefficient-string-concatenation) + JSON_THROW(parse_error::create(105, 0, error_msg + " must have member '" + member + "'", val)); + } + + // check if result is of type string + if (JSON_HEDLEY_UNLIKELY(string_type && !it->second.is_string())) + { + // NOLINTNEXTLINE(performance-inefficient-string-concatenation) + JSON_THROW(parse_error::create(105, 0, error_msg + " must have string member '" + member + "'", val)); + } + + // no error: return value + return it->second; + }; + + // type check: every element of the array must be an object + if (JSON_HEDLEY_UNLIKELY(!val.is_object())) + { + JSON_THROW(parse_error::create(104, 0, "JSON patch must be an array of objects", val)); + } + + // collect mandatory members + const auto op = get_value("op", "op", true).template get(); + const auto path = get_value(op, "path", true).template get(); + json_pointer ptr(path); + + switch (get_op(op)) + { + case patch_operations::add: + { + operation_add(ptr, get_value("add", "value", false)); + break; + } + + case patch_operations::remove: + { + operation_remove(ptr); + break; + } + + case patch_operations::replace: + { + // the "path" location must exist - use at() + result.at(ptr) = get_value("replace", "value", false); + break; + } + + case patch_operations::move: + { + const auto from_path = get_value("move", "from", true).template get(); + json_pointer from_ptr(from_path); + + // the "from" location must exist - use at() + basic_json v = result.at(from_ptr); + + // The move operation is functionally identical to a + // "remove" operation on the "from" location, followed + // immediately by an "add" operation at the target + // location with the value that was just removed. + operation_remove(from_ptr); + operation_add(ptr, v); + break; + } + + case patch_operations::copy: + { + const auto from_path = get_value("copy", "from", true).template get(); + const json_pointer from_ptr(from_path); + + // the "from" location must exist - use at() + basic_json v = result.at(from_ptr); + + // The copy is functionally identical to an "add" + // operation at the target location using the value + // specified in the "from" member. + operation_add(ptr, v); + break; + } + + case patch_operations::test: + { + bool success = false; + JSON_TRY + { + // check if "value" matches the one at "path" + // the "path" location must exist - use at() + success = (result.at(ptr) == get_value("test", "value", false)); + } + JSON_INTERNAL_CATCH (out_of_range&) + { + // ignore out of range errors: success remains false + } + + // throw an exception if test fails + if (JSON_HEDLEY_UNLIKELY(!success)) + { + JSON_THROW(other_error::create(501, "unsuccessful: " + val.dump(), val)); + } + + break; + } + + case patch_operations::invalid: + default: + { + // op must be "add", "remove", "replace", "move", "copy", or + // "test" + JSON_THROW(parse_error::create(105, 0, "operation value '" + op + "' is invalid", val)); + } + } + } + + return result; + } + + /// @brief creates a diff as a JSON patch + /// @sa https://json.nlohmann.me/api/basic_json/diff/ + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json diff(const basic_json& source, const basic_json& target, + const std::string& path = "") + { + // the patch + basic_json result(value_t::array); + + // if the values are the same, return empty patch + if (source == target) + { + return result; + } + + if (source.type() != target.type()) + { + // different types: replace value + result.push_back( + { + {"op", "replace"}, {"path", path}, {"value", target} + }); + return result; + } + + switch (source.type()) + { + case value_t::array: + { + // first pass: traverse common elements + std::size_t i = 0; + while (i < source.size() && i < target.size()) + { + // recursive call to compare array values at index i + auto temp_diff = diff(source[i], target[i], path + "/" + std::to_string(i)); + result.insert(result.end(), temp_diff.begin(), temp_diff.end()); + ++i; + } + + // We now reached the end of at least one array + // in a second pass, traverse the remaining elements + + // remove my remaining elements + const auto end_index = static_cast(result.size()); + while (i < source.size()) + { + // add operations in reverse order to avoid invalid + // indices + result.insert(result.begin() + end_index, object( + { + {"op", "remove"}, + {"path", path + "/" + std::to_string(i)} + })); + ++i; + } + + // add other remaining elements + while (i < target.size()) + { + result.push_back( + { + {"op", "add"}, + {"path", path + "/-"}, + {"value", target[i]} + }); + ++i; + } + + break; + } + + case value_t::object: + { + // first pass: traverse this object's elements + for (auto it = source.cbegin(); it != source.cend(); ++it) + { + // escape the key name to be used in a JSON patch + const auto path_key = path + "/" + detail::escape(it.key()); + + if (target.find(it.key()) != target.end()) + { + // recursive call to compare object values at key it + auto temp_diff = diff(it.value(), target[it.key()], path_key); + result.insert(result.end(), temp_diff.begin(), temp_diff.end()); + } + else + { + // found a key that is not in o -> remove it + result.push_back(object( + { + {"op", "remove"}, {"path", path_key} + })); + } + } + + // second pass: traverse other object's elements + for (auto it = target.cbegin(); it != target.cend(); ++it) + { + if (source.find(it.key()) == source.end()) + { + // found a key that is not in this -> add it + const auto path_key = path + "/" + detail::escape(it.key()); + result.push_back( + { + {"op", "add"}, {"path", path_key}, + {"value", it.value()} + }); + } + } + + break; + } + + case value_t::null: + case value_t::string: + case value_t::boolean: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::number_float: + case value_t::binary: + case value_t::discarded: + default: + { + // both primitive type: replace value + result.push_back( + { + {"op", "replace"}, {"path", path}, {"value", target} + }); + break; + } + } + + return result; + } + + /// @} + + //////////////////////////////// + // JSON Merge Patch functions // + //////////////////////////////// + + /// @name JSON Merge Patch functions + /// @{ + + /// @brief applies a JSON Merge Patch + /// @sa https://json.nlohmann.me/api/basic_json/merge_patch/ + void merge_patch(const basic_json& apply_patch) + { + if (apply_patch.is_object()) + { + if (!is_object()) + { + *this = object(); + } + for (auto it = apply_patch.begin(); it != apply_patch.end(); ++it) + { + if (it.value().is_null()) + { + erase(it.key()); + } + else + { + operator[](it.key()).merge_patch(it.value()); + } + } + } + else + { + *this = apply_patch; + } + } + + /// @} +}; + +/// @brief user-defined to_string function for JSON values +/// @sa https://json.nlohmann.me/api/basic_json/to_string/ +NLOHMANN_BASIC_JSON_TPL_DECLARATION +std::string to_string(const NLOHMANN_BASIC_JSON_TPL& j) +{ + return j.dump(); +} + +} // namespace nlohmann + +/////////////////////// +// nonmember support // +/////////////////////// + +namespace std // NOLINT(cert-dcl58-cpp) +{ + +/// @brief hash value for JSON objects +/// @sa https://json.nlohmann.me/api/basic_json/std_hash/ +NLOHMANN_BASIC_JSON_TPL_DECLARATION +struct hash +{ + std::size_t operator()(const nlohmann::NLOHMANN_BASIC_JSON_TPL& j) const + { + return nlohmann::detail::hash(j); + } +}; + +// specialization for std::less +template<> +struct less< ::nlohmann::detail::value_t> // do not remove the space after '<', see https://github.com/nlohmann/json/pull/679 +{ + /*! + @brief compare two value_t enum values + @since version 3.0.0 + */ + bool operator()(nlohmann::detail::value_t lhs, + nlohmann::detail::value_t rhs) const noexcept + { + return nlohmann::detail::operator<(lhs, rhs); + } +}; + +// C++20 prohibit function specialization in the std namespace. +#ifndef JSON_HAS_CPP_20 + +/// @brief exchanges the values of two JSON objects +/// @sa https://json.nlohmann.me/api/basic_json/std_swap/ +NLOHMANN_BASIC_JSON_TPL_DECLARATION +inline void swap(nlohmann::NLOHMANN_BASIC_JSON_TPL& j1, nlohmann::NLOHMANN_BASIC_JSON_TPL& j2) noexcept( // NOLINT(readability-inconsistent-declaration-parameter-name) + is_nothrow_move_constructible::value&& // NOLINT(misc-redundant-expression) + is_nothrow_move_assignable::value) +{ + j1.swap(j2); +} + +#endif + +} // namespace std + +/// @brief user-defined string literal for JSON values +/// @sa https://json.nlohmann.me/api/basic_json/operator_literal_json/ +JSON_HEDLEY_NON_NULL(1) +inline nlohmann::json operator "" _json(const char* s, std::size_t n) +{ + return nlohmann::json::parse(s, s + n); +} + +/// @brief user-defined string literal for JSON pointer +/// @sa https://json.nlohmann.me/api/basic_json/operator_literal_json_pointer/ +JSON_HEDLEY_NON_NULL(1) +inline nlohmann::json::json_pointer operator "" _json_pointer(const char* s, std::size_t n) +{ + return nlohmann::json::json_pointer(std::string(s, n)); +} + +// #include + + +// restore clang diagnostic settings +#if defined(__clang__) + #pragma clang diagnostic pop +#endif + +// clean up +#undef JSON_ASSERT +#undef JSON_INTERNAL_CATCH +#undef JSON_CATCH +#undef JSON_THROW +#undef JSON_TRY +#undef JSON_PRIVATE_UNLESS_TESTED +#undef JSON_HAS_CPP_11 +#undef JSON_HAS_CPP_14 +#undef JSON_HAS_CPP_17 +#undef JSON_HAS_CPP_20 +#undef JSON_HAS_FILESYSTEM +#undef JSON_HAS_EXPERIMENTAL_FILESYSTEM +#undef NLOHMANN_BASIC_JSON_TPL_DECLARATION +#undef NLOHMANN_BASIC_JSON_TPL +#undef JSON_EXPLICIT +#undef NLOHMANN_CAN_CALL_STD_FUNC_IMPL + +// #include + + +#undef JSON_HEDLEY_ALWAYS_INLINE +#undef JSON_HEDLEY_ARM_VERSION +#undef JSON_HEDLEY_ARM_VERSION_CHECK +#undef JSON_HEDLEY_ARRAY_PARAM +#undef JSON_HEDLEY_ASSUME +#undef JSON_HEDLEY_BEGIN_C_DECLS +#undef JSON_HEDLEY_CLANG_HAS_ATTRIBUTE +#undef JSON_HEDLEY_CLANG_HAS_BUILTIN +#undef JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE +#undef JSON_HEDLEY_CLANG_HAS_DECLSPEC_DECLSPEC_ATTRIBUTE +#undef JSON_HEDLEY_CLANG_HAS_EXTENSION +#undef JSON_HEDLEY_CLANG_HAS_FEATURE +#undef JSON_HEDLEY_CLANG_HAS_WARNING +#undef JSON_HEDLEY_COMPCERT_VERSION +#undef JSON_HEDLEY_COMPCERT_VERSION_CHECK +#undef JSON_HEDLEY_CONCAT +#undef JSON_HEDLEY_CONCAT3 +#undef JSON_HEDLEY_CONCAT3_EX +#undef JSON_HEDLEY_CONCAT_EX +#undef JSON_HEDLEY_CONST +#undef JSON_HEDLEY_CONSTEXPR +#undef JSON_HEDLEY_CONST_CAST +#undef JSON_HEDLEY_CPP_CAST +#undef JSON_HEDLEY_CRAY_VERSION +#undef JSON_HEDLEY_CRAY_VERSION_CHECK +#undef JSON_HEDLEY_C_DECL +#undef JSON_HEDLEY_DEPRECATED +#undef JSON_HEDLEY_DEPRECATED_FOR +#undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL +#undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_ +#undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED +#undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES +#undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS +#undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION +#undef JSON_HEDLEY_DIAGNOSTIC_POP +#undef JSON_HEDLEY_DIAGNOSTIC_PUSH +#undef JSON_HEDLEY_DMC_VERSION +#undef JSON_HEDLEY_DMC_VERSION_CHECK +#undef JSON_HEDLEY_EMPTY_BASES +#undef JSON_HEDLEY_EMSCRIPTEN_VERSION +#undef JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK +#undef JSON_HEDLEY_END_C_DECLS +#undef JSON_HEDLEY_FLAGS +#undef JSON_HEDLEY_FLAGS_CAST +#undef JSON_HEDLEY_GCC_HAS_ATTRIBUTE +#undef JSON_HEDLEY_GCC_HAS_BUILTIN +#undef JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE +#undef JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE +#undef JSON_HEDLEY_GCC_HAS_EXTENSION +#undef JSON_HEDLEY_GCC_HAS_FEATURE +#undef JSON_HEDLEY_GCC_HAS_WARNING +#undef JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK +#undef JSON_HEDLEY_GCC_VERSION +#undef JSON_HEDLEY_GCC_VERSION_CHECK +#undef JSON_HEDLEY_GNUC_HAS_ATTRIBUTE +#undef JSON_HEDLEY_GNUC_HAS_BUILTIN +#undef JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE +#undef JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE +#undef JSON_HEDLEY_GNUC_HAS_EXTENSION +#undef JSON_HEDLEY_GNUC_HAS_FEATURE +#undef JSON_HEDLEY_GNUC_HAS_WARNING +#undef JSON_HEDLEY_GNUC_VERSION +#undef JSON_HEDLEY_GNUC_VERSION_CHECK +#undef JSON_HEDLEY_HAS_ATTRIBUTE +#undef JSON_HEDLEY_HAS_BUILTIN +#undef JSON_HEDLEY_HAS_CPP_ATTRIBUTE +#undef JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS +#undef JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE +#undef JSON_HEDLEY_HAS_EXTENSION +#undef JSON_HEDLEY_HAS_FEATURE +#undef JSON_HEDLEY_HAS_WARNING +#undef JSON_HEDLEY_IAR_VERSION +#undef JSON_HEDLEY_IAR_VERSION_CHECK +#undef JSON_HEDLEY_IBM_VERSION +#undef JSON_HEDLEY_IBM_VERSION_CHECK +#undef JSON_HEDLEY_IMPORT +#undef JSON_HEDLEY_INLINE +#undef JSON_HEDLEY_INTEL_CL_VERSION +#undef JSON_HEDLEY_INTEL_CL_VERSION_CHECK +#undef JSON_HEDLEY_INTEL_VERSION +#undef JSON_HEDLEY_INTEL_VERSION_CHECK +#undef JSON_HEDLEY_IS_CONSTANT +#undef JSON_HEDLEY_IS_CONSTEXPR_ +#undef JSON_HEDLEY_LIKELY +#undef JSON_HEDLEY_MALLOC +#undef JSON_HEDLEY_MCST_LCC_VERSION +#undef JSON_HEDLEY_MCST_LCC_VERSION_CHECK +#undef JSON_HEDLEY_MESSAGE +#undef JSON_HEDLEY_MSVC_VERSION +#undef JSON_HEDLEY_MSVC_VERSION_CHECK +#undef JSON_HEDLEY_NEVER_INLINE +#undef JSON_HEDLEY_NON_NULL +#undef JSON_HEDLEY_NO_ESCAPE +#undef JSON_HEDLEY_NO_RETURN +#undef JSON_HEDLEY_NO_THROW +#undef JSON_HEDLEY_NULL +#undef JSON_HEDLEY_PELLES_VERSION +#undef JSON_HEDLEY_PELLES_VERSION_CHECK +#undef JSON_HEDLEY_PGI_VERSION +#undef JSON_HEDLEY_PGI_VERSION_CHECK +#undef JSON_HEDLEY_PREDICT +#undef JSON_HEDLEY_PRINTF_FORMAT +#undef JSON_HEDLEY_PRIVATE +#undef JSON_HEDLEY_PUBLIC +#undef JSON_HEDLEY_PURE +#undef JSON_HEDLEY_REINTERPRET_CAST +#undef JSON_HEDLEY_REQUIRE +#undef JSON_HEDLEY_REQUIRE_CONSTEXPR +#undef JSON_HEDLEY_REQUIRE_MSG +#undef JSON_HEDLEY_RESTRICT +#undef JSON_HEDLEY_RETURNS_NON_NULL +#undef JSON_HEDLEY_SENTINEL +#undef JSON_HEDLEY_STATIC_ASSERT +#undef JSON_HEDLEY_STATIC_CAST +#undef JSON_HEDLEY_STRINGIFY +#undef JSON_HEDLEY_STRINGIFY_EX +#undef JSON_HEDLEY_SUNPRO_VERSION +#undef JSON_HEDLEY_SUNPRO_VERSION_CHECK +#undef JSON_HEDLEY_TINYC_VERSION +#undef JSON_HEDLEY_TINYC_VERSION_CHECK +#undef JSON_HEDLEY_TI_ARMCL_VERSION +#undef JSON_HEDLEY_TI_ARMCL_VERSION_CHECK +#undef JSON_HEDLEY_TI_CL2000_VERSION +#undef JSON_HEDLEY_TI_CL2000_VERSION_CHECK +#undef JSON_HEDLEY_TI_CL430_VERSION +#undef JSON_HEDLEY_TI_CL430_VERSION_CHECK +#undef JSON_HEDLEY_TI_CL6X_VERSION +#undef JSON_HEDLEY_TI_CL6X_VERSION_CHECK +#undef JSON_HEDLEY_TI_CL7X_VERSION +#undef JSON_HEDLEY_TI_CL7X_VERSION_CHECK +#undef JSON_HEDLEY_TI_CLPRU_VERSION +#undef JSON_HEDLEY_TI_CLPRU_VERSION_CHECK +#undef JSON_HEDLEY_TI_VERSION +#undef JSON_HEDLEY_TI_VERSION_CHECK +#undef JSON_HEDLEY_UNAVAILABLE +#undef JSON_HEDLEY_UNLIKELY +#undef JSON_HEDLEY_UNPREDICTABLE +#undef JSON_HEDLEY_UNREACHABLE +#undef JSON_HEDLEY_UNREACHABLE_RETURN +#undef JSON_HEDLEY_VERSION +#undef JSON_HEDLEY_VERSION_DECODE_MAJOR +#undef JSON_HEDLEY_VERSION_DECODE_MINOR +#undef JSON_HEDLEY_VERSION_DECODE_REVISION +#undef JSON_HEDLEY_VERSION_ENCODE +#undef JSON_HEDLEY_WARNING +#undef JSON_HEDLEY_WARN_UNUSED_RESULT +#undef JSON_HEDLEY_WARN_UNUSED_RESULT_MSG +#undef JSON_HEDLEY_FALL_THROUGH + + + +#endif // INCLUDE_NLOHMANN_JSON_HPP_ diff --git a/liberty/lib/SLAMP/SLAMPstats/slamp_hooks.h b/liberty/lib/SLAMP/SLAMPstats/slamp_hooks.h new file mode 100644 index 00000000..df546117 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPstats/slamp_hooks.h @@ -0,0 +1,371 @@ +#ifndef SLAMPLIB_HOOKS_SLAMP_HOOKS_H +#define SLAMPLIB_HOOKS_SLAMP_HOOKS_H + +// FIXME: inline tweak actually make things worse in sequential and better with +// 16x, so turn it on at all time before understanding why +#define ATTRIBUTE(x) __attribute__((x)) +// #ifdef ITO_ENABLE +// // #define ATTRIBUTE(x) +// #define ATTRIBUTE(x) __attribute__((x)) +// #else +// #define ATTRIBUTE(x) +// #endif + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +void SLAMP_dbggv(int id); +void SLAMP_dbggvstr(char* str); + +// SLAMP measure functions +void SLAMP_measure_init(); +void SLAMP_measure_fini(); +void SLAMP_measure_load(uint32_t id, uint64_t size); +void SLAMP_measure_store(uint32_t id, uint64_t size); +static void* SLAMP_measure_malloc_hook(size_t size, const void *caller); +static void SLAMP_measure_free_hook(void *ptr, const void *caller); + +void SLAMP_init(uint32_t fn_id, uint32_t loop_id); +void SLAMP_fini(const char* filename); + +void SLAMP_allocated(uint64_t addr); +void SLAMP_init_global_vars(const char *name, uint64_t addr, size_t size); +void SLAMP_main_entry(uint32_t argc, char** argv, char** env); + +void SLAMP_enter_fcn(uint32_t id); +void SLAMP_exit_fcn(uint32_t id); +void SLAMP_enter_loop(uint32_t id); +void SLAMP_exit_loop(uint32_t id); +void SLAMP_loop_iter_ctx(uint32_t id); +void SLAMP_loop_invocation(); +void SLAMP_loop_iteration(); +void SLAMP_loop_exit(); + +void SLAMP_report_base_pointer_arg(uint32_t, uint32_t, void *ptr); +void SLAMP_report_base_pointer_inst(uint32_t, void *ptr); +void SLAMP_callback_stack_alloca(uint64_t, uint64_t, uint32_t, uint64_t); +void SLAMP_callback_stack_free(void); + +void SLAMP_ext_push(const uint32_t instr); +void SLAMP_ext_pop(); + +void SLAMP_push(const uint32_t instr); +void SLAMP_pop(); + +void SLAMP_load1(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_load2(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_load4(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_load8(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_loadn(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, size_t n) ATTRIBUTE(always_inline);; + +void SLAMP_load1_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_load2_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_load4_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_load8_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_loadn_ext(const uint64_t addr, const uint32_t bare_instr, size_t n) ATTRIBUTE(always_inline);; + +void SLAMP_store1(uint32_t instr, const uint64_t addr) ATTRIBUTE(always_inline); +void SLAMP_store2(uint32_t instr, const uint64_t addr) ATTRIBUTE(always_inline); +void SLAMP_store4(uint32_t instr, const uint64_t addr) ATTRIBUTE(always_inline); +void SLAMP_store8(uint32_t instr, const uint64_t addr) ATTRIBUTE(always_inline); +void SLAMP_storen(uint32_t instr, const uint64_t addr, size_t n) ATTRIBUTE(always_inline);; + +void SLAMP_store1_ext(const uint64_t addr, const uint32_t bare_inst) ATTRIBUTE(always_inline); +void SLAMP_store2_ext(const uint64_t addr, const uint32_t bare_inst) ATTRIBUTE(always_inline); +void SLAMP_store4_ext(const uint64_t addr, const uint32_t bare_inst) ATTRIBUTE(always_inline); +void SLAMP_store8_ext(const uint64_t addr, const uint32_t bare_inst) ATTRIBUTE(always_inline); +void SLAMP_storen_ext(const uint64_t addr, const uint32_t bare_inst, size_t n) ATTRIBUTE(always_inline);; + +/* wrappers */ +static void* SLAMP_malloc_hook(size_t size, const void *caller); +static void SLAMP_free_hook(void *ptr, const void *caller); +static void* SLAMP_memalign_hook(size_t alignment, size_t size, const void *caller); +void* SLAMP_malloc(size_t size, uint32_t instr=0, size_t alignment=16); + +void* SLAMP_calloc(size_t nelem, size_t elsize); +void* SLAMP_realloc(void* ptr, size_t size); +void* SLAMP__Znam(size_t size); +void* SLAMP__Znwm(size_t size); + +char* SLAMP_strdup(const char *s1); +char* SLAMP___strdup(const char *s1); +void SLAMP_free(void* ptr); +void SLAMP_cfree(void* ptr); +void SLAMP__ZdlPv(void* ptr); +void SLAMP__ZdaPv(void* ptr); +int SLAMP_brk(void *end_data_segment); +void* SLAMP_sbrk(intptr_t increment); + +/* llvm memory intrinsics */ +void SLAMP_llvm_memcpy_p0i8_p0i8_i32(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint32_t sizeBytes); +void SLAMP_llvm_memcpy_p0i8_p0i8_i64(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint64_t sizeBytes); +void SLAMP_llvm_memmove_p0i8_p0i8_i32(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint32_t sizeBytes); +void SLAMP_llvm_memmove_p0i8_p0i8_i64(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint64_t sizeBytes); +void SLAMP_llvm_memset_p0i8_i32(const uint8_t* dstAddr, const uint32_t len); +void SLAMP_llvm_memset_p0i8_i64(const uint8_t* dstAddr, const uint64_t len); + +// void SLAMP_llvm_lifetime_start_p0i8(uint64_t size, uint8_t* ptr); +// void SLAMP_llvm_lifetime_end_p0i8(uint64_t size, uint8_t* ptr); + +/* String functions */ +size_t SLAMP_strlen(const char *str); +char* SLAMP_strchr(char *s, int c); +char* SLAMP_strrchr(char *s, int c); +int SLAMP_strcmp(const char *s1, const char *s2); +int SLAMP_strncmp(const char *s1, const char *s2, size_t n); +char* SLAMP_strcpy(char *dest, const char *src); +char* SLAMP_strncpy(char *dest, const char *src, size_t n); +char* SLAMP_strcat(char *s1, const char *s2); +char* SLAMP_strncat(char *s1, const char *s2, size_t n); +char* SLAMP_strstr(char *s1, char *s2); +size_t SLAMP_strspn(const char *s1, const char *s2); +size_t SLAMP_strcspn(const char *s1, const char *s2); +char* SLAMP_strtok(char *s, const char *delim); +double SLAMP_strtod(const char *nptr, char **endptr); +long int SLAMP_strtol(const char *nptr, char **endptr, int base); +char* SLAMP_strpbrk(char *s1, char *s2); + +/* Mem* and b* functions */ +void *SLAMP_memset (void *dest, int c, size_t n); +void *SLAMP_memcpy (void *dest, const void *src, size_t n); +void *SLAMP___builtin_memcpy (void *dest, const void *src, size_t n); +void *SLAMP_memmove (void *dest, const void *src, size_t n); +int SLAMP_memcmp(const void *s1, const void *s2, size_t n); +void* SLAMP_memchr(void* ptr, int value, size_t num); +void* SLAMP___rawmemchr(void* ptr, int value); + +void SLAMP_bzero(void *s, size_t n); +void SLAMP_bcopy(const void *s1, void *s2, size_t n); + +/* IO */ +ssize_t SLAMP_read(int fd, void *buf, size_t count); +int SLAMP_open(const char *pathname, int flags, mode_t mode); +int SLAMP_close(int fd); +ssize_t SLAMP_write(int fd, const void *buf, size_t count); +off_t SLAMP_lseek(int fildes, off_t offset, int whence); + +FILE * SLAMP_fopen(const char *path, const char *mode); +FILE * SLAMP_fopen64(const char *path, const char *mode); +FILE * SLAMP_freopen(const char *path, const char *mode, FILE* stream); +int SLAMP_fflush(FILE *stream); +int SLAMP_fclose(FILE *stream); +int SLAMP_ferror(FILE *stream); +int SLAMP_feof(FILE *stream); +long SLAMP_ftell(FILE *stream); +size_t SLAMP_fread(void * ptr, size_t size, size_t nitems, FILE *stream); +size_t SLAMP_fwrite(const void *ptr, size_t size, size_t nitems, FILE *stream); +int SLAMP_fseek(FILE *stream, long offset, int whence); +void SLAMP_rewind(FILE *stream); + +int SLAMP_fgetc(FILE *stream); +int SLAMP_fputc(int c, FILE *stream); +char * SLAMP_fgets(char *s, int n, FILE *stream); +int SLAMP_fputs(const char *s, FILE *stream); + +int SLAMP_ungetc(int c, FILE *stream); +int SLAMP_putchar(int c); +int SLAMP_getchar(void); + +int SLAMP_fileno(FILE *stream); +char * SLAMP_gets(char *s); +int SLAMP_puts(const char *s); + +int SLAMP_select(int n, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, struct timeval *timeout); +int SLAMP_remove(const char *path); + +void SLAMP_setbuf(FILE * stream, char * buf); +void SLAMP_setvbuf(FILE * stream, char * buf, int mode, size_t size); +char * SLAMP_tmpnam(char *s); +FILE* SLAMP_tmpfile(void); +char * SLAMP_ttyname(int fildes); + +FILE * SLAMP_fdopen(int fildes, const char *mode); +void SLAMP_clearerr(FILE *stream); + +int SLAMP_truncate(const char *path, off_t length); +int SLAMP_ftruncate(int fildes, off_t length); + +int SLAMP_dup(int oldfd); +int SLAMP_dup2(int oldfd, int newfd); +int SLAMP_pipe(int filedes[2]); + +int SLAMP_chmod(const char *path, mode_t mode); +int SLAMP_fchmod(int fildes, mode_t mode); +int SLAMP_fchown(int fd, uid_t owner, gid_t group); +int SLAMP_access(const char *pathname, int mode); +long SLAMP_pathconf(char *path, int name); +int SLAMP_mkdir(const char *pathname, mode_t mode); +int SLAMP_rmdir(const char *pathname); +mode_t SLAMP_umask(mode_t mask); +int SLAMP_fcntl(int fd, int cmd, struct flock *lock); + +DIR* SLAMP_opendir(const char* name); +struct dirent* SLAMP_readdir(DIR *dirp); +struct dirent64* SLAMP_readdir64(DIR *dirp); +int SLAMP_closedir(DIR* dirp); + +/* Printf */ +int SLAMP_printf(const char *format, ...); +int SLAMP_fprintf(FILE *stream, const char *format, ...); +int SLAMP_sprintf(char *str, const char *format, ...); +int SLAMP_snprintf(char *str, size_t size, const char *format, ...); + +int SLAMP_vprintf(const char *format, va_list ap); +int SLAMP_vfprintf(FILE *stream, const char *format, va_list ap); +int SLAMP_vsprintf(char *str, const char *format, va_list ap); +int SLAMP_vsnprintf(char *str, size_t size, const char *format, va_list ap); + +/* Scanf */ +int SLAMP_fscanf(FILE *stream, const char *format, ... ); +int SLAMP_scanf(const char *format, ... ); +int SLAMP_sscanf(const char *s, const char *format, ... ); +int SLAMP___isoc99_sscanf(const char *s, const char *format, ... ); + +int SLAMP_vfscanf(FILE *stream, const char *format, va_list ap); +int SLAMP_vscanf(const char *format, va_list ap); +int SLAMP_vsscanf(const char *s, const char *format, va_list ap); + +/* Time */ +time_t SLAMP_time(time_t *t); +struct tm *SLAMP_localtime(const time_t *timer); +struct lconv* SLAMP_localeconv(); +struct tm *SLAMP_gmtime(const time_t *timer); +int SLAMP_gettimeofday(struct timeval *tv, struct timezone *tz); + +/* Math */ +double SLAMP_ldexp(double x, int exp); +float SLAMP_ldexpf(float x, int exp); +long double SLAMP_ldexpl(long double x, int exp); +double SLAMP_log10(double x); +float SLAMP_log10f(float x); +long double SLAMP_log10l(long double x); +double SLAMP_log(double x); +float SLAMP_logf(float x); +long double SLAMP_logl(long double x); + +double SLAMP_exp(double x); +float SLAMP_expf(float x); +long double SLAMP_expl(long double x); + +double SLAMP_cos(double x); +float SLAMP_cosf(float x); +long double SLAMP_cosl(long double x); +double SLAMP_sin(double x); +double SLAMP_tan(double x); +float SLAMP_sinf(float x); +long double SLAMP_sinl(long double x); + +double SLAMP_atan(double x); +float SLAMP_atanf(float x); +long double SLAMP_atanl(long double x); + +double SLAMP_floor(double x); +float SLAMP_floorf(float x); +long double SLAMP_floorl(long double x); +double SLAMP_ceil(double x); +float SLAMP_ceilf(float x); +long double SLAMP_ceill(long double x); + +double SLAMP_atan2(double y, double x); +float SLAMP_atan2f(float y, float x); +long double SLAMP_atan2l(long double y, long double x); + +double SLAMP_sqrt(double x); +float SLAMP_sqrtf(float x); +long double SLAMP_sqrtl(long double x); + +double SLAMP_pow(double x, double y); +float SLAMP_powf(float x, float y); +long double SLAMP_powl(long double x, long double y); + +double SLAMP_fabs(double x); +float SLAMP_fabsf(float x); +long double SLAMP_fabsl(long double x); + +double SLAMP_modf(double x, double *iptr); +float SLAMP_modff(float x, float *iptr); +long double SLAMP_modfl(long double x, long double *iptr); + +double SLAMP_fmod(double x, double y); + +double SLAMP_frexp(double num, int *exp); +float SLAMP_frexpf(float num, int *exp); +long double SLAMP_frexpl(long double num, int *exp); + +int SLAMP_isnan(); + +/* MISC */ +char *SLAMP_getenv(const char *name); +int SLAMP_putenv(char* string); +char *SLAMP_getcwd(char *buf, size_t size); +char* SLAMP_strerror(int errnum); +void SLAMP_exit(int status); +void SLAMP__exit(int status); +int SLAMP_link(const char *oldpath, const char *newpath); +int SLAMP_unlink(const char *pathname); +int SLAMP_isatty(int desc); +int SLAMP_setuid(uid_t uid); +uid_t SLAMP_getuid(void); +uid_t SLAMP_geteuid(void); +int SLAMP_setgid(gid_t gid); +gid_t SLAMP_getgid(void); +gid_t SLAMP_getegid(void); +pid_t SLAMP_getpid(void); +int SLAMP_chdir(const char *path); +int SLAMP_execl(const char *path, const char *arg0, ... /*, (char *)0 */); +int SLAMP_execv(const char *path, char *const argv[]); +int SLAMP_execvp(const char *file, char *const argv[]); +int SLAMP_kill(pid_t pid, int sig); +pid_t SLAMP_fork(void); +sighandler_t SLAMP___sysv_signal(int signum, sighandler_t handler); +pid_t SLAMP_waitpid(pid_t pid, int* status, int options); +void SLAMP_qsort(void* base, size_t nmemb, size_t size, int(*compar)(const void *, const void *)); +int SLAMP_ioctl(int d, int request, ...); +unsigned int SLAMP_sleep(unsigned int seconds); +char* SLAMP_gcvt(double number, size_t ndigit, char* buf); +char* SLAMP_nl_langinfo(nl_item item); + +/* Compiler/Glibc Internals */ +void SLAMP___assert_fail(const char * assertion, const char * file, unsigned int line, const char * function); +const unsigned short int **SLAMP___ctype_b_loc(void); +int SLAMP__IO_getc(_IO_FILE * __fp); +int SLAMP__IO_putc(int __c, _IO_FILE *__fp); +int* SLAMP___errno_location (void); + +int SLAMP___fxstat (int __ver, int __fildes, struct stat *__stat_buf); +int SLAMP___xstat (int __ver, __const char *__filename, struct stat *__stat_buf); + +#ifdef __cplusplus +} +#endif + +#endif /* SLAMP_HOOKS_H */ From 37de55b1636b355efea173528d59331ae8297b01 Mon Sep 17 00:00:00 2001 From: Ziyang Xu Date: Thu, 6 Oct 2022 17:58:10 -0400 Subject: [PATCH 15/97] fix slamp bug: wrong users added slamp_ext_push/pop; invoke target might fail with phi value in get underlying obj --- liberty/lib/SLAMP/SLAMP.cpp | 78 +++++++++++++++++++++++++------------ 1 file changed, 53 insertions(+), 25 deletions(-) diff --git a/liberty/lib/SLAMP/SLAMP.cpp b/liberty/lib/SLAMP/SLAMP.cpp index 39959975..df987bc6 100644 --- a/liberty/lib/SLAMP/SLAMP.cpp +++ b/liberty/lib/SLAMP/SLAMP.cpp @@ -431,8 +431,8 @@ bool SLAMP::runOnModule(Module &m) { std::sort(elidedLoopInstsId.begin(), elidedLoopInstsId.end()); errs() << "Elided Hash: " << elidedHash(elidedLoopInstsId) << "\n"; - // replace external function calls to wrapper function calls - // replaceExternalFunctionCalls(m); + //// replace external function calls to wrapper function calls + replaceExternalFunctionCalls(m); auto setGlobalModule = [&m](string name, bool value) { @@ -473,11 +473,12 @@ bool SLAMP::runOnModule(Module &m) { instrumentGlobalVars(m, ctor); } - if (UsePointsToModule){ + //// FIXME: temporarily instrument alloca and base pointer for all cases + // if (UsePointsToModule){ instrumentAllocas(m); // instrument all base pointer creation instrumentBasePointer(m, this->target_loop); - } + // } instrumentFunctionStartStop(m); instrumentMainFunction(m); @@ -716,6 +717,15 @@ void SLAMP::replaceExternalFunctionCalls(Module &m) { if (inst == nullptr) continue; + // make sure it's a call to the function + if (!isa(inst)) + continue; + auto *cb = dyn_cast(inst); + if (cb->getCalledFunction() != func) + continue; + + + // FIXME: duplicated code as instrumentLoopInst auto id = Namer:: getInstrId(inst); if (id == -1) { @@ -726,9 +736,6 @@ void SLAMP::replaceExternalFunctionCalls(Module &m) { InstInsertPt pt = InstInsertPt::Before(inst); pt << updateDebugInfo(CallInst::Create(push, args), pt.getPosition(), m); - errs() << "Malloc ID " << id << " : " - << getInstructionName(inst) << "\n"; - if (isa(inst)) { pt = InstInsertPt::After(inst); pt << updateDebugInfo(CallInst::Create(pop), pt.getPosition(), m); @@ -753,26 +760,28 @@ void SLAMP::replaceExternalFunctionCalls(Module &m) { assert(false && "Call but not CallInst nor InvokeInst"); } } - if (externs.find(name) == externs.end()) { - // check if the function argument is `readnone`, then it's pure - if (func->hasFnAttribute(llvm::Attribute::AttrKind::ReadNone)) { - continue; - } - errs() << "WARNING: Wrapper for external function " << name - << " not implemented.\n"; - hasUnrecognizedFunction = true; + //// FIXME: temporarily turn off the replacement + // if (externs.find(name) == externs.end()) { + // // check if the function argument is `readnone`, then it's pure + // if (func->hasFnAttribute(llvm::Attribute::AttrKind::ReadNone)) { + // continue; + // } - } else { - string wrapper_name = "SLAMP_" + name; + // errs() << "WARNING: Wrapper for external function " << name + // << " not implemented.\n"; + // hasUnrecognizedFunction = true; + + // } else { + // string wrapper_name = "SLAMP_" + name; /* Function* wrapper = cast( m.getOrInsertFunction(wrapper_name, * func->getFunctionType() ) ); */ - FunctionCallee wrapper = - m.getOrInsertFunction(wrapper_name, func->getFunctionType()); + // FunctionCallee wrapper = + // m.getOrInsertFunction(wrapper_name, func->getFunctionType()); - // replace 'func' to 'wrapper' in uses - func->replaceAllUsesWith(wrapper.getCallee()); - } + // // replace 'func' to 'wrapper' in uses + // func->replaceAllUsesWith(wrapper.getCallee()); + // } } @@ -1029,10 +1038,29 @@ void SLAMP::instrumentBasePointer(Module &m, Loop* l) { InstInsertPt where; if (auto *invoke = dyn_cast(inst)) { auto entry = invoke->getNormalDest(); - if (isa(entry->getFirstNonPHI())) + if (isa(entry->getFirstNonPHI())) { where = InstInsertPt::After(entry->getFirstNonPHI()); - else + } + else { + // iterate all the phi node in the entry block + // if inst is one of the incoming value of the phi node + // then convert the cast to the phi node + for (auto &phi : entry->phis()) { + bool found = false; + for (auto &incoming : phi.incoming_values()) { + if (incoming == inst) { + // replace the cast with the phi node + cast->deleteValue(); + cast = new BitCastInst(&phi, I8Ptr); + found = true; + break; + } + } + if (found) + break; + } where = InstInsertPt::Before(entry->getFirstNonPHI()); + } } else if (auto *phi = dyn_cast(inst)) { // Don't accidentally insert instrumentation before // later PHIs or landing pad instructions. @@ -1229,7 +1257,7 @@ void SLAMP::instrumentLoopStartStopForAll(Module &m) { // TODO: check setjmp/longjmp BasicBlock *header = loop->getHeader(); - unsigned loopId = Namer::getBlkId(header); + auto loopId = Namer::getBlkId(header); if (loopId == -1) { assert(false && "Loop header has no id"); } From 556e8a39e939318c58d480bfcf8c47200b23ee51 Mon Sep 17 00:00:00 2001 From: Ziyang Xu Date: Thu, 6 Oct 2022 17:58:34 -0400 Subject: [PATCH 16/97] update .clangd --- liberty/.clangd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/liberty/.clangd b/liberty/.clangd index 9e4ac78e..0f1417b8 100644 --- a/liberty/.clangd +++ b/liberty/.clangd @@ -1,5 +1,5 @@ CompileFlags: # Tweak the parse settings - Add: [-Wold-style-cast] # treat all files as C++, enable more warnings + Add: [-Wold-style-cast, -Wunused-variable, -Wsigned-compare] # treat all files as C++, enable more warnings Diagnostics: ClangTidy: Add: bugprone-* From 31bcb7b9e92ed62a3665a9b7524d431dea77d710 Mon Sep 17 00:00:00 2001 From: Ziyang Xu Date: Thu, 6 Oct 2022 22:58:34 -0400 Subject: [PATCH 17/97] slamp-driver for stats --- tests/scripts/slamp-driver | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/scripts/slamp-driver b/tests/scripts/slamp-driver index 7fe157e9..0a3097a7 100755 --- a/tests/scripts/slamp-driver +++ b/tests/scripts/slamp-driver @@ -45,7 +45,7 @@ function drive { -load $LIBERTY_LIBS_DIR/libSLAMP.so -basic-loop-aa -scev-loop-aa -auto-restrict-aa -intrinsic-aa -global-malloc-aa -pure-fun-aa -semi-local-fun-aa -phi-maze-aa -no-capture-global-aa -no-capture-src-aa -type-aa -no-escape-fields-aa -acyclic-aa -disjoint-fields-aa -field-malloc-aa -loop-variant-allocation-aa -std-in-out-err-aa -array-of-structures-aa -kill-flow-aa -callsite-depth-combinator-aa -unique-access-paths-aa $EXTRA -llvm-aa-results -pdgbuilder " - local SLAMP_HOOKS="$LIBERTY_LIBS_DIR/libslamp_hooks.a" + local SLAMP_HOOKS="$LIBERTY_LIBS_DIR/libslamp_hooks_stats.a" local SLAMP_OUTFILE="$2-$3.result.slamp.profile" local OPTS="-slamp-insts -slamp-target-fn=$2 -slamp-target-loop=$3 $EXTRA_FLAGS -slamp-outfile=$SLAMP_OUTFILE" @@ -192,8 +192,8 @@ while IFS=$'\n' read -r line_data; do done < __targets.txt let i=0 -# while (( i < 2 )); do -while (( ${#lines[@]} > i )); do +while (( i < 2 )); do +# while (( ${#lines[@]} > i )); do IFS=' ' read -a array <<< ${lines[i++]} if [ ${array[0]} == "-" ]; then drive $1 ${array[1]} ${array[3]} From 7b9ed9e8b38fbaa3456b8a2d81f839225bc7d50c Mon Sep 17 00:00:00 2001 From: Ziyang Xu Date: Mon, 10 Oct 2022 18:31:50 -0400 Subject: [PATCH 18/97] queue supported dep mod --- liberty/lib/SLAMP/SLAMPboost/BoostSend.cpp | 229 +++++++++++------- .../lib/SLAMP/SLAMPboost/consumer/.gitignore | 1 + .../SLAMP/SLAMPboost/consumer/CMakeLists.txt | 2 +- .../ProfilingModules/DependenceModule.cpp | 117 +++++++++ .../ProfilingModules/DependenceModule.h | 23 ++ .../ProfilingModules/slamp_bound_malloc.h | 22 ++ .../consumer/ProfilingModules/slamp_logger.h | 110 +++++++++ .../ProfilingModules/slamp_shadow_mem.h | 197 +++++++++++++++ .../ProfilingModules/slamp_timestamp.h | 22 ++ .../SLAMP/SLAMPboost/consumer/consumer.cpp | 93 +++++-- 10 files changed, 719 insertions(+), 97 deletions(-) create mode 100644 liberty/lib/SLAMP/SLAMPboost/consumer/.gitignore create mode 100644 liberty/lib/SLAMP/SLAMPboost/consumer/ProfilingModules/DependenceModule.cpp create mode 100644 liberty/lib/SLAMP/SLAMPboost/consumer/ProfilingModules/DependenceModule.h create mode 100644 liberty/lib/SLAMP/SLAMPboost/consumer/ProfilingModules/slamp_bound_malloc.h create mode 100644 liberty/lib/SLAMP/SLAMPboost/consumer/ProfilingModules/slamp_logger.h create mode 100644 liberty/lib/SLAMP/SLAMPboost/consumer/ProfilingModules/slamp_shadow_mem.h create mode 100644 liberty/lib/SLAMP/SLAMPboost/consumer/ProfilingModules/slamp_timestamp.h diff --git a/liberty/lib/SLAMP/SLAMPboost/BoostSend.cpp b/liberty/lib/SLAMP/SLAMPboost/BoostSend.cpp index 08e73b6a..818116da 100644 --- a/liberty/lib/SLAMP/SLAMPboost/BoostSend.cpp +++ b/liberty/lib/SLAMP/SLAMPboost/BoostSend.cpp @@ -4,54 +4,79 @@ #include #include #include +#include #include +#include "malloc.h" #define LOCAL_BUFFER_SIZE 256 namespace bip = boost::interprocess; namespace shm { - typedef bip::allocator char_alloc; - typedef bip::basic_string, char_alloc > shared_string; + // typedef bip::allocator char_alloc; + // typedef bip::basic_string, char_alloc > shared_string; // using ring_buffer = boost::lockfree::spsc_queue>; - using ring_buffer = boost::lockfree::spsc_queue>; + using ring_buffer = boost::lockfree::spsc_queue>; } #include +static void *(*old_malloc_hook)(size_t, const void *); +static void (*old_free_hook)(void *, const void *); +static void *(*old_memalign_hook)(size_t, size_t, const void *); // create segment and corresponding allocator bip::managed_shared_memory *segment; -shm::char_alloc *char_alloc; // Ringbuffer fully constructed in shared memory. The element strings are // also allocated from the same shared memory segment. This vector can be // safely accessed from other processes. shm::ring_buffer *queue; -unsigned long counter4 = 0; -unsigned long counter8 = 0; -unsigned long counter4_ext = 0; -unsigned long counter8_ext = 0; +unsigned long counter_load = 0; +unsigned long counter_store = 0; +unsigned long counter_ctx = 0; +unsigned long counter_alloc = 0; char local_buffer[LOCAL_BUFFER_SIZE]; unsigned buffer_counter = 0; +enum DepModAction: uint64_t +{ + INIT = 0, + LOAD, + STORE, + ALLOC, + LOOP_INVOC, + LOOP_ITER, + FINISHED +}; + void SLAMP_init(uint32_t fn_id, uint32_t loop_id) { segment = new bip::managed_shared_memory(bip::open_or_create, "MySharedMemory", 65536UL*1600); - char_alloc = new shm::char_alloc(segment->get_segment_manager()); queue = segment->find_or_construct("queue")(); - // send a msg with "fn_id, loop_id" - char msg[100]; - sprintf(msg, "%d,%d", fn_id, loop_id); - // queue->push(shm::shared_string(msg, *char_alloc)); - queue->push(fn_id); + // char msg[100]; + // sprintf(msg, "%d,%d", fn_id, loop_id); + // queue->push(shm::shared_string(msg, *char_alloc)); + while(!queue->push(INIT)); + queue->push((uint64_t)loop_id); + auto pid = getpid(); + printf("SLAMP_init: %d, %d, %d\n", fn_id, loop_id, pid); + queue->push(pid); + + old_malloc_hook = __malloc_hook; + // old_free_hook = __free_hook; + // old_memalign_hook = __memalign_hook; + __malloc_hook = SLAMP_malloc_hook; + // __free_hook = SLAMP_free_hook; + // __memalign_hook = SLAMP_memalign_hook; } void SLAMP_fini(const char* filename){ // send a msg with "fini" // queue->push(shm::shared_string("fini", *char_alloc)); - std::cout << counter4 << " " << counter8 << " " << counter4_ext << " " << counter8_ext << std::endl; + std::cout << counter_load << " " << counter_store << " " << counter_ctx << std::endl; + while(!queue->push(FINISHED)); } void SLAMP_allocated(uint64_t addr){} @@ -78,93 +103,133 @@ void SLAMP_ext_pop(){} void SLAMP_push(const uint32_t instr){} void SLAMP_pop(){} +void SLAMP_load(const uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline) { + // send a msg with "load, instr, addr, bare_instr, value" + // char msg[100]; + // sprintf(msg, "load,%d,%lu,%d,%lu", instr, addr, bare_instr, value); + // queue->push(shm::shared_string(msg, *char_alloc)); + while(!queue->push(LOAD)); + while(!queue->push(instr)); + while(!queue->push(addr)); + while(!queue->push(bare_instr)); + while(!queue->push(value)); + counter_load++; +} + void SLAMP_load1(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + SLAMP_load(instr, addr, bare_instr, value); } void SLAMP_load2(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + SLAMP_load(instr, addr, bare_instr, value); } void SLAMP_load4(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value){ - counter4++; - - queue->push(4); - queue->push(instr); - - // local_buffer[buffer_counter++] = 4; - // local_buffer[buffer_counter++] = instr & 0xFF; - - // // // sprintf(msg, "load8,%d,%lu,%d,%lu", instr, addr, bare_instr, value); - // if (buffer_counter == LOCAL_BUFFER_SIZE) { - // queue->push(shm::shared_string(local_buffer, LOCAL_BUFFER_SIZE, *char_alloc)); - // buffer_counter = 0; - // } + SLAMP_load(instr, addr, bare_instr, value); } void SLAMP_load8(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value){ - counter8++; - - queue->push(8); - queue->push(instr); - // local_buffer[buffer_counter++] = 8; - // local_buffer[buffer_counter++] = instr & 0xFF; - - // // // sprintf(msg, "load8,%d,%lu,%d,%lu", instr, addr, bare_instr, value); - // if (buffer_counter == LOCAL_BUFFER_SIZE) { - // queue->push(shm::shared_string(local_buffer, LOCAL_BUFFER_SIZE, *char_alloc)); - // buffer_counter = 0; - // } + SLAMP_load(instr, addr, bare_instr, value); } -void SLAMP_loadn(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, size_t n){} +void SLAMP_loadn(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, size_t n){ + SLAMP_load(instr, addr, bare_instr, 0); +} -void SLAMP_load1_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value){} -void SLAMP_load2_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value){} +void SLAMP_load1_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + SLAMP_load1(bare_instr, addr, bare_instr, value); +} +void SLAMP_load2_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + SLAMP_load2(bare_instr, addr, bare_instr, value); +} void SLAMP_load4_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value){ - counter4_ext++; + SLAMP_load4(bare_instr, addr, bare_instr, value); +} +void SLAMP_load8_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + SLAMP_load8(bare_instr, addr, bare_instr, value); +} +void SLAMP_loadn_ext(const uint64_t addr, const uint32_t bare_instr, size_t n){ + SLAMP_loadn(bare_instr, addr, bare_instr, n); +} - queue->push(14); - queue->push(addr & 0xFFFFFFFF); - // local_buffer[buffer_counter++] = 14; - // local_buffer[buffer_counter++] = addr & 0xFF; +void SLAMP_store(const uint32_t instr, const uint64_t addr, const uint32_t bare_instr) ATTRIBUTE(always_inline) { + // send a msg with "store, instr, addr, bare_instr, value" + // char msg[100]; + // sprintf(msg, "store,%d,%lu,%d,%lu", instr, addr, bare_instr, value); + // queue->push(shm::shared_string(msg, *char_alloc)); + while(!queue->push(STORE)); + while(!queue->push(instr)); + while(!queue->push(bare_instr)); + while(!queue->push(addr)); + counter_store++; +} - // // // sprintf(msg, "load8,%d,%lu,%d,%lu", instr, addr, bare_instr, value); - // if (buffer_counter == LOCAL_BUFFER_SIZE) { - // queue->push(shm::shared_string(local_buffer, LOCAL_BUFFER_SIZE, *char_alloc)); - // buffer_counter = 0; - // } +void SLAMP_store1(uint32_t instr, const uint64_t addr){ + SLAMP_store(instr, addr, instr); +} +void SLAMP_store2(uint32_t instr, const uint64_t addr){ + SLAMP_store(instr, addr, instr); } -void SLAMP_load8_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value){ - counter8_ext++; - - queue->push(18); - queue->push(addr & 0xFFFFFFFF); - // local_buffer[buffer_counter++] = 18; - // local_buffer[buffer_counter++] = addr & 0xFF; - - // // // sprintf(msg, "load8,%d,%lu,%d,%lu", instr, addr, bare_instr, value); - // if (buffer_counter == LOCAL_BUFFER_SIZE) { - // queue->push(shm::shared_string(local_buffer, LOCAL_BUFFER_SIZE, *char_alloc)); - // buffer_counter = 0; - // } +void SLAMP_store4(uint32_t instr, const uint64_t addr){ + SLAMP_store(instr, addr, instr); +} +void SLAMP_store8(uint32_t instr, const uint64_t addr){ + SLAMP_store(instr, addr, instr); +} +void SLAMP_storen(uint32_t instr, const uint64_t addr, size_t n){ + SLAMP_store(instr, addr, instr); } -void SLAMP_loadn_ext(const uint64_t addr, const uint32_t bare_instr, size_t n){} - -void SLAMP_store1(uint32_t instr, const uint64_t addr){} -void SLAMP_store2(uint32_t instr, const uint64_t addr){} -void SLAMP_store4(uint32_t instr, const uint64_t addr){} -void SLAMP_store8(uint32_t instr, const uint64_t addr){} -void SLAMP_storen(uint32_t instr, const uint64_t addr, size_t n){} -void SLAMP_store1_ext(const uint64_t addr, const uint32_t bare_inst){} -void SLAMP_store2_ext(const uint64_t addr, const uint32_t bare_inst){} -void SLAMP_store4_ext(const uint64_t addr, const uint32_t bare_inst){} -void SLAMP_store8_ext(const uint64_t addr, const uint32_t bare_inst){} -void SLAMP_storen_ext(const uint64_t addr, const uint32_t bare_inst, size_t n){} +void SLAMP_store1_ext(const uint64_t addr, const uint32_t bare_inst){ + SLAMP_store1(bare_inst, addr); +} +void SLAMP_store2_ext(const uint64_t addr, const uint32_t bare_inst){ + SLAMP_store2(bare_inst, addr); +} +void SLAMP_store4_ext(const uint64_t addr, const uint32_t bare_inst){ + SLAMP_store4(bare_inst, addr); +} +void SLAMP_store8_ext(const uint64_t addr, const uint32_t bare_inst){ + SLAMP_store8(bare_inst, addr); +} +void SLAMP_storen_ext(const uint64_t addr, const uint32_t bare_inst, size_t n){ + SLAMP_storen(bare_inst, addr, n); +} /* wrappers */ -static void* SLAMP_malloc_hook(size_t size, const void *caller){} -static void SLAMP_free_hook(void *ptr, const void *caller){} -static void* SLAMP_memalign_hook(size_t alignment, size_t size, const void *caller){} -void* SLAMP_malloc(size_t size, uint32_t instr, size_t alignment){} +static void* SLAMP_malloc_hook(size_t size, const void *caller){ + __malloc_hook = old_malloc_hook; + void* ptr = malloc(size); + while(!queue->push(ALLOC)); + queue->push(reinterpret_cast(ptr)); + queue->push(size); + printf("malloc %lu at %p", size, ptr); + counter_alloc++; + __malloc_hook = SLAMP_malloc_hook; + return ptr; +} +static void SLAMP_free_hook(void *ptr, const void *caller){ + old_free_hook(ptr, caller); +} +static void* SLAMP_memalign_hook(size_t alignment, size_t size, const void *caller){ + void* ptr = old_memalign_hook(alignment, size, caller); + while(!queue->push(ALLOC)); + queue->push(reinterpret_cast(ptr)); + queue->push(size); + counter_alloc++; + return ptr; +} +void* SLAMP_malloc(size_t size, uint32_t instr, size_t alignment){ + // void* ptr = malloc(size); + // // send a msg with "malloc, instr, ptr, size, alignment" + // // char msg[100]; + // // sprintf(msg, "malloc,%d,%lu,%lu,%lu", instr, ptr, size, alignment); + // // queue->push(shm::shared_string(msg, *char_alloc)); + // queue->push(ALLOC); + // queue->push(reinterpret_cast(ptr)); + // queue->push(size); + // counter_alloc++; + // return ptr; +} void* SLAMP_calloc(size_t nelem, size_t elsize){} void* SLAMP_realloc(void* ptr, size_t size){} diff --git a/liberty/lib/SLAMP/SLAMPboost/consumer/.gitignore b/liberty/lib/SLAMP/SLAMPboost/consumer/.gitignore new file mode 100644 index 00000000..378eac25 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPboost/consumer/.gitignore @@ -0,0 +1 @@ +build diff --git a/liberty/lib/SLAMP/SLAMPboost/consumer/CMakeLists.txt b/liberty/lib/SLAMP/SLAMPboost/consumer/CMakeLists.txt index cdf39bea..0c305bc8 100644 --- a/liberty/lib/SLAMP/SLAMPboost/consumer/CMakeLists.txt +++ b/liberty/lib/SLAMP/SLAMPboost/consumer/CMakeLists.txt @@ -9,7 +9,7 @@ find_package(Threads REQUIRED) find_package(Boost 1.80.0 REQUIRED COMPONENTS system) include_directories(${Boost_INCLUDE_DIRS}) -add_executable (consumer consumer.cpp) +add_executable (consumer consumer.cpp ./ProfilingModules/DependenceModule.cpp) target_link_libraries(consumer LINK_PUBLIC ${Boost_LIBRARIES} rt diff --git a/liberty/lib/SLAMP/SLAMPboost/consumer/ProfilingModules/DependenceModule.cpp b/liberty/lib/SLAMP/SLAMPboost/consumer/ProfilingModules/DependenceModule.cpp new file mode 100644 index 00000000..b06fabfe --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPboost/consumer/ProfilingModules/DependenceModule.cpp @@ -0,0 +1,117 @@ +#include +#include +#include +#include + +#include "slamp_timestamp.h" +#include "slamp_logger.h" +#include "slamp_shadow_mem.h" +#include "DependenceModule.h" +#define SIZE_8M 0x800000 + + +static slamp::MemoryMap* smmap = nullptr; +static std::unordered_set *deplog_set; +static uint64_t slamp_iteration = 0; +static uint64_t slamp_invocation = 0; +static uint32_t target_loop_id = 0; + +namespace DepMod { +// init: setup the shadow memory +void init(uint32_t loop_id, uint64_t pid) { + + target_loop_id = loop_id; + smmap = new slamp::MemoryMap(TIMESTAMP_SIZE_IN_BYTES); + deplog_set = new std::unordered_set(); + + smmap->init_stack(SIZE_8M, pid); + smmap->allocate((void*)&errno, sizeof(errno)); + smmap->allocate((void*)&stdin, sizeof(stdin)); + smmap->allocate((void*)&stdout, sizeof(stdout)); + smmap->allocate((void*)&stderr, sizeof(stderr)); + smmap->allocate((void*)&sys_nerr, sizeof(sys_nerr)); + + { + const unsigned short int* ctype_ptr = (*__ctype_b_loc()) - 128; + smmap->allocate((void*)ctype_ptr, 384 * sizeof(*ctype_ptr)); + } + { + const int32_t* itype_ptr = (*__ctype_tolower_loc()) - 128; + smmap->allocate((void*)itype_ptr, 384 * sizeof(*itype_ptr)); + } + { + const int32_t* itype_ptr = (*__ctype_toupper_loc()) - 128; + smmap->allocate((void*)itype_ptr, 384 * sizeof(*itype_ptr)); + } +} + +void fini(const char *filename) { + std::ofstream of(filename); + of << target_loop_id << " " << 0 << " " << 0 << " " + << 0 << " " << 0 << " " << 0 << "\n"; + + std::set ordered(deplog_set->begin(), deplog_set->end()); + for (auto &k: ordered) { + of << target_loop_id << " " << k.src << " " << k.dst << " " << k.dst_bare << " " + << (k.cross ? 1 : 0) << " " << 1 << " "; + of << "\n"; + } + + delete smmap; + delete deplog_set; +} + +void allocate(void *addr, uint64_t size) { + smmap->allocate(addr, size); + std::cout << "allocate " << addr << " " << size << std::endl; +} + +// void log(TS ts, const uint32_t dst_inst, TS *pts, const uint32_t bare_inst, + // uint64_t addr, uint64_t value, uint8_t size) { +void log(TS ts, const uint32_t dst_inst, const uint32_t bare_inst){ + + uint32_t src_inst = GET_INSTR(ts); + uint64_t src_iter = GET_ITER(ts); + + // uint64_t src_invoc = GET_INVOC(ts); + + slamp::KEY key(src_inst, dst_inst, bare_inst, src_iter != slamp_iteration); + + // std::cout << "src_inst: " << src_inst << " dst_inst: " << dst_inst << " bare_inst: " << bare_inst << " src_iter: " << src_iter << " slamp_iteration: " << slamp_iteration << " src_iter != slamp_iteration: " << (src_iter != slamp_iteration) << std::endl; + deplog_set->insert(key); +} + +// template +void load(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + TS* s = (TS*)GET_SHADOW(addr, TIMESTAMP_SIZE_IN_POWER_OF_TWO); + TS tss = s[0]; + log(tss, instr, bare_instr); +} + +// template +void store(uint32_t instr, uint32_t bare_instr, const uint64_t addr) { + TS *s = (TS *)GET_SHADOW(addr, TIMESTAMP_SIZE_IN_POWER_OF_TWO); + if (!smmap->is_allocated(s)) { + // std::cout << "store not allocated: " << instr << " " << bare_instr << " " << addr << std::endl; + + } + TS ts = CREATE_TS(instr, slamp_iteration, slamp_invocation); + + // TODO: handle output dependence. ignore it as of now. + // if (ASSUME_ONE_ADDR) { + s[0] = ts; + // } else { + // for (auto i = 0; i < size; i++) + // s[i] = ts; + // } +} + +void loop_invoc() { + slamp_iteration = 0; + slamp_invocation++; +} + +void loop_iter() { + slamp_iteration++; +} +} // namespace DepMod diff --git a/liberty/lib/SLAMP/SLAMPboost/consumer/ProfilingModules/DependenceModule.h b/liberty/lib/SLAMP/SLAMPboost/consumer/ProfilingModules/DependenceModule.h new file mode 100644 index 00000000..aa439db7 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPboost/consumer/ProfilingModules/DependenceModule.h @@ -0,0 +1,23 @@ +#include + +namespace DepMod { +void init(uint32_t loop_id, uint64_t pid); +void fini(const char *filename); +void load(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value); +void store(uint32_t instr, uint32_t bare_instr, const uint64_t addr); +void allocate(void *addr, uint64_t size); +void loop_invoc(); +void loop_iter(); + +enum DepModAction: uint64_t +{ + INIT = 0, + LOAD, + STORE, + ALLOC, + LOOP_INVOC, + LOOP_ITER, + FINISHED +}; + +} // namespace DepMod diff --git a/liberty/lib/SLAMP/SLAMPboost/consumer/ProfilingModules/slamp_bound_malloc.h b/liberty/lib/SLAMP/SLAMPboost/consumer/ProfilingModules/slamp_bound_malloc.h new file mode 100644 index 00000000..0d6b9410 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPboost/consumer/ProfilingModules/slamp_bound_malloc.h @@ -0,0 +1,22 @@ +#ifndef SLAMPLIB_HOOKS_SLAMP_BOUND_MALLOC_H +#define SLAMPLIB_HOOKS_SLAMP_BOUND_MALLOC_H + +#include +#include +namespace slamp +{ + +void init_bound_malloc(void* heap_bound); +void fini_bound_malloc(); + +size_t get_object_size(void* ptr); + +void* bound_malloc(size_t size, size_t alignment=16); +bool bound_free(void *ptr, uint64_t &starting_page, unsigned &purge_cnt); +void* bound_calloc(size_t num, size_t size); +void* bound_realloc(void* ptr, size_t size); +void bound_discard_page(); + +} + +#endif diff --git a/liberty/lib/SLAMP/SLAMPboost/consumer/ProfilingModules/slamp_logger.h b/liberty/lib/SLAMP/SLAMPboost/consumer/ProfilingModules/slamp_logger.h new file mode 100644 index 00000000..89641a03 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPboost/consumer/ProfilingModules/slamp_logger.h @@ -0,0 +1,110 @@ +#ifndef SLAMPLIB_HOOKS_SLAMP_LOGGER +#define SLAMPLIB_HOOKS_SLAMP_LOGGER + +#include +#include +#include +#include + +#include "slamp_timestamp.h" + +namespace slamp +{ + +struct KEY +{ + uint32_t src; + uint32_t dst; + uint32_t dst_bare; + uint32_t cross; + + KEY() {} + KEY(uint32_t s, uint32_t d, uint32_t b, uint32_t c) : src(s), dst(d), dst_bare(b), cross(c) {} +}; + +struct KEYHash +{ + size_t operator()(const KEY& key) const + { + static_assert(sizeof(size_t) == sizeof(uint64_t), "Should be 64bit address"); + std::hash hash_fn; + + // hash(32-32) ^ hash(32-1) + // FIXME: find a better hash function + return hash_fn(((uint64_t)key.src << 32) | key.dst) ^ hash_fn(((uint64_t)key.dst_bare << 1) | (key.cross & 0x1)); + } +}; + +struct KEYComp +{ + bool operator()(const KEY& key1, const KEY& key2) const + { + uint32_t src1 = key1.src; + uint32_t src2 = key2.src; + + if (src1 < src2) + return true; + else if (src1 > src2) + return false; + + uint32_t dst1 = key1.dst; + uint32_t dst2 = key2.dst; + + if (dst1 < dst2) + return true; + else if (dst1 > dst2) + return false; + + uint32_t dst_bare1 = key1.dst_bare; + uint32_t dst_bare2 = key2.dst_bare; + + if (dst_bare1 < dst_bare2) + return true; + else if (dst_bare1 > dst_bare2) + return false; + + uint32_t cross1 = key1.cross; + uint32_t cross2 = key2.cross; + + return cross1 < cross2; + } +}; + +struct KEYEqual +{ + bool operator()(const KEY& key1, const KEY& key2) const + { + uint32_t src1 = key1.src; + uint32_t src2 = key2.src; + + if ( src1 != src2 ) return false; + + uint32_t dst1 = key1.dst; + uint32_t dst2 = key2.dst; + + if ( dst1 != dst2 ) return false; + + uint32_t dst_bare1 = key1.dst_bare; + uint32_t dst_bare2 = key2.dst_bare; + + if ( dst_bare1 != dst_bare2 ) return false; + + uint32_t cross1 = key1.cross; + uint32_t cross2 = key2.cross; + + if ( cross1 != cross2 ) return false; + + return true; + } +}; + + +void init_logger(uint32_t fn_id, uint32_t loop_id); +void fini_logger(const char* filename); + +uint32_t log(TS ts, const uint32_t dst_instr, TS* pts, const uint32_t bare_inst, uint64_t addr, uint64_t value, uint8_t size); +void print_log(const char* filename); + +} + +#endif diff --git a/liberty/lib/SLAMP/SLAMPboost/consumer/ProfilingModules/slamp_shadow_mem.h b/liberty/lib/SLAMP/SLAMPboost/consumer/ProfilingModules/slamp_shadow_mem.h new file mode 100644 index 00000000..ef07201b --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPboost/consumer/ProfilingModules/slamp_shadow_mem.h @@ -0,0 +1,197 @@ +#ifndef SLAMPLIB_HOOKS_SLAMP_SHADOW_MEM_H +#define SLAMPLIB_HOOKS_SLAMP_SHADOW_MEM_H + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +// higher half of canonical region cannot be used + +/// left shift by `shift`, mask 47 LSB, toggle #45 bit? +#define MASK1 0x00007fffffffffffL +#define MASK2 0x0000200000000000L +#define GET_SHADOW(addr, shift) \ + (((((uint64_t)(addr)) << (shift)) & MASK1) ^ MASK2) + +namespace slamp { + +class MemoryMap { +public: + uint64_t heapStart = 0; + MemoryMap(unsigned r) : ratio(r), ratio_shift(0) { + // ratio expected to be a power of 2 + assert((r & (r - 1)) == 0); + + // log(r) + unsigned n = r; + while ((n & 1) == 0) { + this->ratio_shift += 1; + n = n >> 1; + } + + // get the page size of the host system + pagesize = getpagesize(); + pagemask = ~(pagesize - 1); + } + + ~MemoryMap() { + // freeing all remaining shadow addresses + for (auto page : pages) { + uint64_t s = GET_SHADOW(page, ratio_shift); + munmap(reinterpret_cast(s), pagesize * ratio); + } + } + + unsigned get_ratio() { return ratio; } + + bool is_allocated(void *addr) { + auto a = reinterpret_cast(addr); + uint64_t page = a & pagemask; + if (pages.find(page) != pages.end()) + return true; + else + return false; + } + + /// allocate shadow page if not exist + void *allocate(void *addr, size_t size) { + auto a = reinterpret_cast(addr); + uint64_t pagebegin = a & pagemask; + uint64_t pageend = (a + size - 1) & pagemask; + + // try mmap + + std::set shadow_pages; + bool success = true; + + for (uint64_t page = pagebegin; page <= pageend; page += pagesize) { + if (pages.find(page) != pages.end()) + continue; + + uint64_t s = GET_SHADOW(page, ratio_shift); + // create a shadow page for the page + void *p = mmap(reinterpret_cast(s), pagesize * ratio, + PROT_WRITE | PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (p == MAP_FAILED) { + int err = errno; + printf("mmap failed: %lx errno: %d\n", s, err); + raise(SIGINT); + + success = false; + break; + } else { + shadow_pages.insert(p); + } + } + + if (success) { + for (uint64_t page = pagebegin; page <= pageend; page += pagesize) + pages.insert(page); + uint64_t pagebegin = a & pagemask; + uint64_t pageend = (a + size - 1) & pagemask; + + // return shadow_mem + auto *shadow_addr = (uint64_t *)GET_SHADOW(a, ratio_shift); + return (void *)(shadow_addr); + } else { + // cleanup + for (auto shadow_page : shadow_pages) + munmap(shadow_page, pagesize * ratio); + return nullptr; + } + } + + // free the shadow pages + void deallocate_pages(uint64_t page, unsigned cnt) { + munmap(reinterpret_cast(GET_SHADOW(page, ratio_shift)), + pagesize * ratio * cnt); + + // fprintf(stderr, "deallocate_pages: %lx %d\n", GET_SHADOW(page, ratio_shift), cnt); + + for (auto i = 0; i < cnt; i++, page += pagesize) + pages.erase(page); + } + + /// for realloc; the dependence carries over + void copy(void *dst, void *src, size_t size) { + size_t shadow_size = size * ratio; + void *shadow_dst = (void *)GET_SHADOW(dst, ratio_shift); + void *shadow_src = (void *)GET_SHADOW(src, ratio_shift); + memcpy(shadow_dst, shadow_src, shadow_size); + } + + void init_heap(void *addr) { + heapStart = reinterpret_cast(addr); + } + + // init stack size to be fixed 8MB + // the stack in /proc/$pid/maps changes in runtime, use it to find the end + void init_stack(uint64_t stack_size, uint64_t pid) { + char filename[256]; + char buf[5000]; + sprintf(filename, "/proc/%lu/maps", pid); + // sprintf(filename, "/proc/%u/maps", getpid()); + + FILE *fp = fopen(filename, "r"); + if (!fp) { + perror(filename); + exit(EXIT_FAILURE); + } + + bool allocated = false; + + while (fgets(buf, sizeof(buf), fp) != nullptr) { + uint64_t start, end; + char name[5000]; + + int n = sscanf(buf, "%lx-%lx %*c%*c%*c%*c %*llx %*x:%*x %*lu %s", &start, + &end, name); + + if (n != 3) { + continue; + } + + // FIXME: get heap start addr + if (!strcmp(name, "[heap]")) { + heapStart = start; + } + + + if (!strcmp(name, "[stack]")) { + // stack grow from end (big address) backwards + // print the stack start and end + printf("stack: %lx %lx\n", end - stack_size, end); + allocate(reinterpret_cast(end - stack_size), stack_size); + allocated = true; + break; + } + } + + if (!allocated) { + fprintf(stderr, "Error: failed to allocate shadow for stack"); + exit(EXIT_FAILURE); + } + } + +private: + std::set pages; // page table + + unsigned ratio; // (size of metadata) / (size of real data) + unsigned ratio_shift; + uint64_t pagesize; + uint64_t pagemask; +}; + +} // namespace slamp +#endif diff --git a/liberty/lib/SLAMP/SLAMPboost/consumer/ProfilingModules/slamp_timestamp.h b/liberty/lib/SLAMP/SLAMPboost/consumer/ProfilingModules/slamp_timestamp.h new file mode 100644 index 00000000..fc9c1428 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPboost/consumer/ProfilingModules/slamp_timestamp.h @@ -0,0 +1,22 @@ +#ifndef SLAMPLIB_HOOKS_SLAMP_TIMESTAMP +#define SLAMPLIB_HOOKS_SLAMP_TIMESTAMP + +#include + +typedef uint64_t TS; // first 20 bits for instr and following 44 bits for iter +#define TIMESTAMP_SIZE_IN_BYTES 8 +#define TIMESTAMP_SIZE_IN_POWER_OF_TWO 3 +#define ITERATION_SIZE 40 +#define INVOCATION_SIZE 4 // 44-40 +#define CREATE_TS(instr, iter, invoc) ( ((TS)instr << 44) | (((TS)iter & (TS)0xffffffffff) << INVOCATION_SIZE) | ((TS)invoc & (TS)0xf)) +#define CREATE_TS_HASH(instr, hash, iter, invoc) \ + (((TS)instr << 44) | \ + (((TS)hash & (TS)0xfffffffff) << (INVOCATION_SIZE + 4)) | \ + (((TS)iter & (TS)0xf) << INVOCATION_SIZE) | ((TS)invoc & (TS)0xf)) +#define GET_INSTR(ts) ((ts >> 44) & 0xfffff) +#define GET_HASH(ts) ( (ts >> 8) & 0xfffffffff) +#define GET_ITER(ts) ( (ts >> INVOCATION_SIZE) & 0xffffffffff) +#define GET_INVOC(ts) ( ts & 0xf) + + +#endif diff --git a/liberty/lib/SLAMP/SLAMPboost/consumer/consumer.cpp b/liberty/lib/SLAMP/SLAMPboost/consumer/consumer.cpp index 6df09897..01a12811 100644 --- a/liberty/lib/SLAMP/SLAMPboost/consumer/consumer.cpp +++ b/liberty/lib/SLAMP/SLAMPboost/consumer/consumer.cpp @@ -3,38 +3,103 @@ #include #include #include +#include +#include "ProfilingModules/DependenceModule.h" +#include +#include + namespace bip = boost::interprocess; namespace shm { - typedef bip::allocator char_alloc; - typedef bip::basic_string, char_alloc > shared_string; + // typedef bip::allocator char_alloc; + // typedef bip::basic_string, char_alloc > shared_string; // shared_string, - using ring_buffer = boost::lockfree::spsc_queue>; + using ring_buffer = boost::lockfree::spsc_queue>; } -#include + int main() { // create segment and corresponding allocator bip::managed_shared_memory segment(bip::open_or_create, "MySharedMemory", 104857600); - shm::char_alloc char_alloc(segment.get_segment_manager()); + // shm::char_alloc char_alloc(segment.get_segment_manager()); shm::ring_buffer *queue = segment.find_or_construct("queue")(); auto counter = 0; // shm::shared_string v(char_alloc); - unsigned int v; - while (true) - { - if (queue->pop(v)) { - counter++; - - if (counter % 1000000 == 0) { - std::cout << "counter: " << counter << std::endl; - } + + using Action=DepMod::DepModAction; + + uint32_t loop_id; + while (true) { + uint64_t v; + if (queue->pop(v)) { + counter++; + auto action = static_cast(v); + + uint64_t v1, v2, v3, v4; + switch (action) { + case Action::INIT: { + while (!queue->pop(v1)); + while (!queue->pop(v2)); + std::cout << "INIT " << v1 << " " << v2 << std::endl; + loop_id = v1; + DepMod::init(v1, v2); + break; + }; + case Action::LOAD: { + while (!queue->pop(v1)) + ; + while (!queue->pop(v2)) + ; + while (!queue->pop(v3)) + ; + while (!queue->pop(v4)) + ; + // std::cout << "LOAD " << v1 << " " << v2 << " " << v3 << " " << v4 << std::endl; + DepMod::load(v1, v2, v3, v4); + + break; + }; + case Action::STORE: { + while (!queue->pop(v1)); + while (!queue->pop(v2)); + while (!queue->pop(v3)); + // std::cout << "STORE " << v1 << " " << v2 << " " << v3 << std::endl; + DepMod::store(v1, v2, v3); + break; + }; + case Action::ALLOC: { + while (!queue->pop(v1)); + while (!queue->pop(v2)); + DepMod::allocate(reinterpret_cast(v1), v2); + break; + }; + case Action::LOOP_INVOC: { + DepMod::loop_invoc(); + break; + }; + case Action::LOOP_ITER: { + DepMod::loop_iter(); + break; + }; + case Action::FINISHED: { + std::stringstream ss; + ss << "deplog-" << loop_id << ".txt"; + DepMod::fini(ss.str().c_str()); + std::cout << "Finished loop: " << loop_id << " after " << counter + << " events" << std::endl; + break; + }; + } + + if (counter % 1000000 == 0) { + std::cout << "Processed " << counter << " events" << std::endl; } + } } } From 520312314792a90d324144a6c8cc6c316376aba3 Mon Sep 17 00:00:00 2001 From: Ziyang Xu Date: Tue, 11 Oct 2022 23:30:32 -0400 Subject: [PATCH 19/97] get nab working --- liberty/lib/SLAMP/SLAMPboost/BoostSend.cpp | 110 +++++++++++++----- .../ProfilingModules/DependenceModule.cpp | 6 +- .../SLAMP/SLAMPboost/consumer/consumer.cpp | 27 ++++- 3 files changed, 104 insertions(+), 39 deletions(-) diff --git a/liberty/lib/SLAMP/SLAMPboost/BoostSend.cpp b/liberty/lib/SLAMP/SLAMPboost/BoostSend.cpp index 818116da..7404c32d 100644 --- a/liberty/lib/SLAMP/SLAMPboost/BoostSend.cpp +++ b/liberty/lib/SLAMP/SLAMPboost/BoostSend.cpp @@ -8,7 +8,7 @@ #include #include "malloc.h" -#define LOCAL_BUFFER_SIZE 256 +#define LOCAL_BUFFER_SIZE 32768 namespace bip = boost::interprocess; namespace shm @@ -36,8 +36,38 @@ unsigned long counter_load = 0; unsigned long counter_store = 0; unsigned long counter_ctx = 0; unsigned long counter_alloc = 0; -char local_buffer[LOCAL_BUFFER_SIZE]; -unsigned buffer_counter = 0; +// char local_buffer[LOCAL_BUFFER_SIZE]; +// unsigned buffer_counter = 0; +bool onProfiling = false; + +template +struct LocalBuffer { + T buffer[LOCAL_BUFFER_SIZE]; + shm::ring_buffer *queue; + unsigned counter = 0; + + LocalBuffer(shm::ring_buffer *queue) : queue(queue) {} + + LocalBuffer *push(T value) { + buffer[counter++] = value; + if (counter == LOCAL_BUFFER_SIZE) { + flush(); + } + return this; + } + + void flush() { + // for (int i = 0; i < counter; i++) { + auto pushed = 0; + while (pushed < counter) { + pushed += queue->push(buffer + pushed, counter - pushed); + } + // } + counter = 0; + } +}; + +LocalBuffer *local_buffer; enum DepModAction: uint64_t { @@ -53,34 +83,41 @@ enum DepModAction: uint64_t void SLAMP_init(uint32_t fn_id, uint32_t loop_id) { segment = new bip::managed_shared_memory(bip::open_or_create, "MySharedMemory", 65536UL*1600); queue = segment->find_or_construct("queue")(); + local_buffer = new LocalBuffer(queue); // send a msg with "fn_id, loop_id" // char msg[100]; // sprintf(msg, "%d,%d", fn_id, loop_id); // queue->push(shm::shared_string(msg, *char_alloc)); - while(!queue->push(INIT)); - queue->push((uint64_t)loop_id); + + local_buffer->push(INIT); + local_buffer->push(loop_id); auto pid = getpid(); printf("SLAMP_init: %d, %d, %d\n", fn_id, loop_id, pid); - queue->push(pid); + local_buffer->push(pid); old_malloc_hook = __malloc_hook; // old_free_hook = __free_hook; - // old_memalign_hook = __memalign_hook; + old_memalign_hook = __memalign_hook; __malloc_hook = SLAMP_malloc_hook; + __free_hook = nullptr; + __realloc_hook = nullptr; // __free_hook = SLAMP_free_hook; - // __memalign_hook = SLAMP_memalign_hook; + __memalign_hook = SLAMP_memalign_hook; } void SLAMP_fini(const char* filename){ // send a msg with "fini" // queue->push(shm::shared_string("fini", *char_alloc)); std::cout << counter_load << " " << counter_store << " " << counter_ctx << std::endl; - while(!queue->push(FINISHED)); + local_buffer->push(FINISHED); + local_buffer->flush(); } void SLAMP_allocated(uint64_t addr){} -void SLAMP_init_global_vars(const char *name, uint64_t addr, size_t size){} +void SLAMP_init_global_vars(const char *name, uint64_t addr, size_t size){ + local_buffer->push(ALLOC)->push(addr)->push(size); +} void SLAMP_main_entry(uint32_t argc, char** argv, char** env){} void SLAMP_enter_fcn(uint32_t id){} @@ -88,9 +125,21 @@ void SLAMP_exit_fcn(uint32_t id){} void SLAMP_enter_loop(uint32_t id){} void SLAMP_exit_loop(uint32_t id){} void SLAMP_loop_iter_ctx(uint32_t id){} -void SLAMP_loop_invocation(){} -void SLAMP_loop_iteration(){} -void SLAMP_loop_exit(){} +void SLAMP_loop_invocation(){ + // send a msg with "loop_invocation" + local_buffer->push(LOOP_INVOC); + + counter_ctx++; + onProfiling = true; +} +void SLAMP_loop_iteration(){ + local_buffer->push(LOOP_ITER); + counter_ctx++; +} + +void SLAMP_loop_exit(){ + onProfiling = false; +} void SLAMP_report_base_pointer_arg(uint32_t, uint32_t, void *ptr){} void SLAMP_report_base_pointer_inst(uint32_t, void *ptr){} @@ -108,12 +157,11 @@ void SLAMP_load(const uint32_t instr, const uint64_t addr, const uint32_t bare_i // char msg[100]; // sprintf(msg, "load,%d,%lu,%d,%lu", instr, addr, bare_instr, value); // queue->push(shm::shared_string(msg, *char_alloc)); - while(!queue->push(LOAD)); - while(!queue->push(instr)); - while(!queue->push(addr)); - while(!queue->push(bare_instr)); - while(!queue->push(value)); - counter_load++; + // + if (onProfiling) { + local_buffer->push(LOAD)->push(instr)->push(addr)->push(bare_instr)->push(value); + counter_load++; + } } void SLAMP_load1(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value){ @@ -155,11 +203,10 @@ void SLAMP_store(const uint32_t instr, const uint64_t addr, const uint32_t bare_ // char msg[100]; // sprintf(msg, "store,%d,%lu,%d,%lu", instr, addr, bare_instr, value); // queue->push(shm::shared_string(msg, *char_alloc)); - while(!queue->push(STORE)); - while(!queue->push(instr)); - while(!queue->push(bare_instr)); - while(!queue->push(addr)); - counter_store++; + if (onProfiling) { + local_buffer->push(STORE)->push(instr)->push(bare_instr)->push(addr); + counter_store++; + } } void SLAMP_store1(uint32_t instr, const uint64_t addr){ @@ -199,10 +246,8 @@ void SLAMP_storen_ext(const uint64_t addr, const uint32_t bare_inst, size_t n){ static void* SLAMP_malloc_hook(size_t size, const void *caller){ __malloc_hook = old_malloc_hook; void* ptr = malloc(size); - while(!queue->push(ALLOC)); - queue->push(reinterpret_cast(ptr)); - queue->push(size); - printf("malloc %lu at %p", size, ptr); + local_buffer->push(ALLOC)->push((uint64_t)ptr)->push(size); + // printf("malloc %lu at %p\n", size, ptr); counter_alloc++; __malloc_hook = SLAMP_malloc_hook; return ptr; @@ -211,11 +256,12 @@ static void SLAMP_free_hook(void *ptr, const void *caller){ old_free_hook(ptr, caller); } static void* SLAMP_memalign_hook(size_t alignment, size_t size, const void *caller){ - void* ptr = old_memalign_hook(alignment, size, caller); - while(!queue->push(ALLOC)); - queue->push(reinterpret_cast(ptr)); - queue->push(size); + old_memalign_hook = __memalign_hook; + void* ptr = memalign(alignment, size); + local_buffer->push(ALLOC)->push((uint64_t)ptr)->push(size); + // printf("memalign %lu at %p\n", size, ptr); counter_alloc++; + __memalign_hook = SLAMP_memalign_hook; return ptr; } void* SLAMP_malloc(size_t size, uint32_t instr, size_t alignment){ diff --git a/liberty/lib/SLAMP/SLAMPboost/consumer/ProfilingModules/DependenceModule.cpp b/liberty/lib/SLAMP/SLAMPboost/consumer/ProfilingModules/DependenceModule.cpp index b06fabfe..0a9d95d0 100644 --- a/liberty/lib/SLAMP/SLAMPboost/consumer/ProfilingModules/DependenceModule.cpp +++ b/liberty/lib/SLAMP/SLAMPboost/consumer/ProfilingModules/DependenceModule.cpp @@ -63,7 +63,7 @@ void fini(const char *filename) { void allocate(void *addr, uint64_t size) { smmap->allocate(addr, size); - std::cout << "allocate " << addr << " " << size << std::endl; + // std::cout << "allocate " << addr << " " << size << std::endl; } // void log(TS ts, const uint32_t dst_inst, TS *pts, const uint32_t bare_inst, @@ -85,7 +85,9 @@ void log(TS ts, const uint32_t dst_inst, const uint32_t bare_inst){ void load(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value){ TS* s = (TS*)GET_SHADOW(addr, TIMESTAMP_SIZE_IN_POWER_OF_TWO); TS tss = s[0]; - log(tss, instr, bare_instr); + if (tss != 0) { + log(tss, instr, bare_instr); + } } // template diff --git a/liberty/lib/SLAMP/SLAMPboost/consumer/consumer.cpp b/liberty/lib/SLAMP/SLAMPboost/consumer/consumer.cpp index 01a12811..5b81cff3 100644 --- a/liberty/lib/SLAMP/SLAMPboost/consumer/consumer.cpp +++ b/liberty/lib/SLAMP/SLAMPboost/consumer/consumer.cpp @@ -8,6 +8,8 @@ #include #include +#define DEBUG 0 + namespace bip = boost::interprocess; namespace shm @@ -46,7 +48,9 @@ int main() case Action::INIT: { while (!queue->pop(v1)); while (!queue->pop(v2)); - std::cout << "INIT " << v1 << " " << v2 << std::endl; + if (DEBUG) { + std::cout << "INIT " << v1 << " " << v2 << std::endl; + } loop_id = v1; DepMod::init(v1, v2); break; @@ -60,7 +64,9 @@ int main() ; while (!queue->pop(v4)) ; - // std::cout << "LOAD " << v1 << " " << v2 << " " << v3 << " " << v4 << std::endl; + if (DEBUG) { + std::cout << "LOAD " << v1 << " " << v2 << " " << v3 << " " << v4 << std::endl; + } DepMod::load(v1, v2, v3, v4); break; @@ -69,7 +75,9 @@ int main() while (!queue->pop(v1)); while (!queue->pop(v2)); while (!queue->pop(v3)); - // std::cout << "STORE " << v1 << " " << v2 << " " << v3 << std::endl; + if (DEBUG) { + std::cout << "STORE " << v1 << " " << v2 << " " << v3 << std::endl; + } DepMod::store(v1, v2, v3); break; }; @@ -77,14 +85,23 @@ int main() while (!queue->pop(v1)); while (!queue->pop(v2)); DepMod::allocate(reinterpret_cast(v1), v2); + if (DEBUG) { + std::cout << "ALLOC " << v1 << " " << v2 << std::endl; + } break; }; case Action::LOOP_INVOC: { DepMod::loop_invoc(); + if (DEBUG) { + std::cout << "LOOP_INVOC" << std::endl; + } break; }; case Action::LOOP_ITER: { DepMod::loop_iter(); + if (DEBUG) { + std::cout << "LOOP_ITER" << std::endl; + } break; }; case Action::FINISHED: { @@ -97,8 +114,8 @@ int main() }; } - if (counter % 1000000 == 0) { - std::cout << "Processed " << counter << " events" << std::endl; + if (counter % 100000000 == 0) { + std::cout << "Processed " << counter / 1000000 << "M events" << std::endl; } } } From 2b5e2ef8c7f357e536e01f038a2f4e0367eb5016 Mon Sep 17 00:00:00 2001 From: Ziyang Xu Date: Thu, 13 Oct 2022 17:08:43 -0400 Subject: [PATCH 20/97] add smtx version --- liberty/lib/SLAMP/CMakeLists.txt | 1 + liberty/lib/SLAMP/SLAMPboost/BoostSend.cpp | 33 +- .../SLAMP/SLAMPboost/consumer/CMakeLists.txt | 7 + .../ProfilingModules/DependenceModule.cpp | 11 +- .../ProfilingModules/DependenceModule.h | 4 +- .../SLAMP/SLAMPboost/consumer/consumer.cpp | 236 +++++--- liberty/lib/SLAMP/SLAMPsmtxq/CMakeLists.txt | 37 ++ liberty/lib/SLAMP/SLAMPsmtxq/SmtxSend.cpp | 544 ++++++++++++++++++ liberty/lib/SLAMP/SLAMPsmtxq/bitcast.h | 29 + .../lib/SLAMP/SLAMPsmtxq/consumer/.gitignore | 1 + .../SLAMP/SLAMPsmtxq/consumer/CMakeLists.txt | 24 + .../ProfilingModules/DependenceModule.cpp | 118 ++++ .../ProfilingModules/DependenceModule.h | 23 + .../ProfilingModules/slamp_bound_malloc.h | 22 + .../consumer/ProfilingModules/slamp_logger.h | 110 ++++ .../ProfilingModules/slamp_shadow_mem.h | 197 +++++++ .../ProfilingModules/slamp_timestamp.h | 22 + .../SLAMP/SLAMPsmtxq/consumer/consumer.cpp | 163 ++++++ liberty/lib/SLAMP/SLAMPsmtxq/inline.h | 12 + liberty/lib/SLAMP/SLAMPsmtxq/slamp_hooks.h | 371 ++++++++++++ liberty/lib/SLAMP/SLAMPsmtxq/sw_queue.c | 103 ++++ .../lib/SLAMP/SLAMPsmtxq/sw_queue_astream.h | 383 ++++++++++++ 22 files changed, 2351 insertions(+), 100 deletions(-) create mode 100644 liberty/lib/SLAMP/SLAMPsmtxq/CMakeLists.txt create mode 100644 liberty/lib/SLAMP/SLAMPsmtxq/SmtxSend.cpp create mode 100644 liberty/lib/SLAMP/SLAMPsmtxq/bitcast.h create mode 100644 liberty/lib/SLAMP/SLAMPsmtxq/consumer/.gitignore create mode 100644 liberty/lib/SLAMP/SLAMPsmtxq/consumer/CMakeLists.txt create mode 100644 liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/DependenceModule.cpp create mode 100644 liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/DependenceModule.h create mode 100644 liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/slamp_bound_malloc.h create mode 100644 liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/slamp_logger.h create mode 100644 liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/slamp_shadow_mem.h create mode 100644 liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/slamp_timestamp.h create mode 100644 liberty/lib/SLAMP/SLAMPsmtxq/consumer/consumer.cpp create mode 100644 liberty/lib/SLAMP/SLAMPsmtxq/inline.h create mode 100644 liberty/lib/SLAMP/SLAMPsmtxq/slamp_hooks.h create mode 100644 liberty/lib/SLAMP/SLAMPsmtxq/sw_queue.c create mode 100644 liberty/lib/SLAMP/SLAMPsmtxq/sw_queue_astream.h diff --git a/liberty/lib/SLAMP/CMakeLists.txt b/liberty/lib/SLAMP/CMakeLists.txt index c96bcfd7..cea20aee 100644 --- a/liberty/lib/SLAMP/CMakeLists.txt +++ b/liberty/lib/SLAMP/CMakeLists.txt @@ -23,3 +23,4 @@ add_subdirectory(SLAMPnng) add_subdirectory(SLAMPboost) add_subdirectory(SLAMPatomicq) add_subdirectory(SLAMPstats) +add_subdirectory(SLAMPsmtxq) diff --git a/liberty/lib/SLAMP/SLAMPboost/BoostSend.cpp b/liberty/lib/SLAMP/SLAMPboost/BoostSend.cpp index 7404c32d..98395d67 100644 --- a/liberty/lib/SLAMP/SLAMPboost/BoostSend.cpp +++ b/liberty/lib/SLAMP/SLAMPboost/BoostSend.cpp @@ -8,7 +8,8 @@ #include #include "malloc.h" -#define LOCAL_BUFFER_SIZE 32768 +// 2MB +#define LOCAL_BUFFER_SIZE 2097152 namespace bip = boost::interprocess; namespace shm @@ -17,7 +18,8 @@ namespace shm // typedef bip::basic_string, char_alloc > shared_string; // using ring_buffer = boost::lockfree::spsc_queue>; - using ring_buffer = boost::lockfree::spsc_queue>; + // 4MB + using ring_buffer = boost::lockfree::spsc_queue>; } #include @@ -40,25 +42,30 @@ unsigned long counter_alloc = 0; // unsigned buffer_counter = 0; bool onProfiling = false; -template +// template struct LocalBuffer { - T buffer[LOCAL_BUFFER_SIZE]; + char buffer[LOCAL_BUFFER_SIZE]; shm::ring_buffer *queue; unsigned counter = 0; LocalBuffer(shm::ring_buffer *queue) : queue(queue) {} - LocalBuffer *push(T value) { - buffer[counter++] = value; - if (counter == LOCAL_BUFFER_SIZE) { + template + LocalBuffer *push(T value) ATTRIBUTE(always_inline) { + size_t size = sizeof(T); + if (counter + size > LOCAL_BUFFER_SIZE) { flush(); } + for (unsigned i = 0; i < size; i++) { + // lsb first + buffer[counter++] = (value >> (i << 3)) & 0xFF; + } return this; } void flush() { // for (int i = 0; i < counter; i++) { - auto pushed = 0; + unsigned long pushed = 0; while (pushed < counter) { pushed += queue->push(buffer + pushed, counter - pushed); } @@ -67,9 +74,9 @@ struct LocalBuffer { } }; -LocalBuffer *local_buffer; +LocalBuffer *local_buffer; -enum DepModAction: uint64_t +enum DepModAction: char { INIT = 0, LOAD, @@ -83,7 +90,7 @@ enum DepModAction: uint64_t void SLAMP_init(uint32_t fn_id, uint32_t loop_id) { segment = new bip::managed_shared_memory(bip::open_or_create, "MySharedMemory", 65536UL*1600); queue = segment->find_or_construct("queue")(); - local_buffer = new LocalBuffer(queue); + local_buffer = new LocalBuffer(queue); // send a msg with "fn_id, loop_id" // char msg[100]; @@ -92,7 +99,7 @@ void SLAMP_init(uint32_t fn_id, uint32_t loop_id) { local_buffer->push(INIT); local_buffer->push(loop_id); - auto pid = getpid(); + uint32_t pid = getpid(); printf("SLAMP_init: %d, %d, %d\n", fn_id, loop_id, pid); local_buffer->push(pid); @@ -159,7 +166,7 @@ void SLAMP_load(const uint32_t instr, const uint64_t addr, const uint32_t bare_i // queue->push(shm::shared_string(msg, *char_alloc)); // if (onProfiling) { - local_buffer->push(LOAD)->push(instr)->push(addr)->push(bare_instr)->push(value); + local_buffer->push(LOAD)->push(instr)->push(addr)->push(bare_instr); //->push(value); counter_load++; } } diff --git a/liberty/lib/SLAMP/SLAMPboost/consumer/CMakeLists.txt b/liberty/lib/SLAMP/SLAMPboost/consumer/CMakeLists.txt index 0c305bc8..9ad2d772 100644 --- a/liberty/lib/SLAMP/SLAMPboost/consumer/CMakeLists.txt +++ b/liberty/lib/SLAMP/SLAMPboost/consumer/CMakeLists.txt @@ -1,10 +1,17 @@ cmake_minimum_required (VERSION 3.6.2 FATAL_ERROR) +# set(CMAKE_C_COMPILER "clang") +# set(CMAKE_CXX_COMPILER "clang++") +# set(CMAKE_LINKER "ld.gold") + set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} "/u/ziyangx/test/boost/boost_1_80_0/install/lib/cmake") set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) +# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -flto") +# set(LINK_FLAGS "${LINK_FLAGS} -flto") + find_package(Boost 1.80.0 REQUIRED COMPONENTS system) include_directories(${Boost_INCLUDE_DIRS}) diff --git a/liberty/lib/SLAMP/SLAMPboost/consumer/ProfilingModules/DependenceModule.cpp b/liberty/lib/SLAMP/SLAMPboost/consumer/ProfilingModules/DependenceModule.cpp index 0a9d95d0..a78b3487 100644 --- a/liberty/lib/SLAMP/SLAMPboost/consumer/ProfilingModules/DependenceModule.cpp +++ b/liberty/lib/SLAMP/SLAMPboost/consumer/ProfilingModules/DependenceModule.cpp @@ -18,7 +18,7 @@ static uint32_t target_loop_id = 0; namespace DepMod { // init: setup the shadow memory -void init(uint32_t loop_id, uint64_t pid) { +void init(uint32_t loop_id, uint32_t pid) { target_loop_id = loop_id; smmap = new slamp::MemoryMap(TIMESTAMP_SIZE_IN_BYTES); @@ -78,7 +78,7 @@ void log(TS ts, const uint32_t dst_inst, const uint32_t bare_inst){ slamp::KEY key(src_inst, dst_inst, bare_inst, src_iter != slamp_iteration); // std::cout << "src_inst: " << src_inst << " dst_inst: " << dst_inst << " bare_inst: " << bare_inst << " src_iter: " << src_iter << " slamp_iteration: " << slamp_iteration << " src_iter != slamp_iteration: " << (src_iter != slamp_iteration) << std::endl; - deplog_set->insert(key); + // deplog_set->insert(key); } // template @@ -93,10 +93,9 @@ void load(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64 // template void store(uint32_t instr, uint32_t bare_instr, const uint64_t addr) { TS *s = (TS *)GET_SHADOW(addr, TIMESTAMP_SIZE_IN_POWER_OF_TWO); - if (!smmap->is_allocated(s)) { - // std::cout << "store not allocated: " << instr << " " << bare_instr << " " << addr << std::endl; - - } + // if (!smmap->is_allocated(s)) { + // std::cout << "store not allocated: " << instr << " " << bare_instr << " " << addr << std::endl; + // } TS ts = CREATE_TS(instr, slamp_iteration, slamp_invocation); // TODO: handle output dependence. ignore it as of now. diff --git a/liberty/lib/SLAMP/SLAMPboost/consumer/ProfilingModules/DependenceModule.h b/liberty/lib/SLAMP/SLAMPboost/consumer/ProfilingModules/DependenceModule.h index aa439db7..a2664cdc 100644 --- a/liberty/lib/SLAMP/SLAMPboost/consumer/ProfilingModules/DependenceModule.h +++ b/liberty/lib/SLAMP/SLAMPboost/consumer/ProfilingModules/DependenceModule.h @@ -1,7 +1,7 @@ #include namespace DepMod { -void init(uint32_t loop_id, uint64_t pid); +void init(uint32_t loop_id, uint32_t pid); void fini(const char *filename); void load(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value); void store(uint32_t instr, uint32_t bare_instr, const uint64_t addr); @@ -9,7 +9,7 @@ void allocate(void *addr, uint64_t size); void loop_invoc(); void loop_iter(); -enum DepModAction: uint64_t +enum DepModAction: char { INIT = 0, LOAD, diff --git a/liberty/lib/SLAMP/SLAMPboost/consumer/consumer.cpp b/liberty/lib/SLAMP/SLAMPboost/consumer/consumer.cpp index 5b81cff3..c86dc291 100644 --- a/liberty/lib/SLAMP/SLAMPboost/consumer/consumer.cpp +++ b/liberty/lib/SLAMP/SLAMPboost/consumer/consumer.cpp @@ -9,6 +9,7 @@ #include #define DEBUG 0 +#define ACTION 0 namespace bip = boost::interprocess; @@ -18,10 +19,52 @@ namespace shm // typedef bip::basic_string, char_alloc > shared_string; // shared_string, - using ring_buffer = boost::lockfree::spsc_queue>; + using ring_buffer = boost::lockfree::spsc_queue>; } +// 2MB +#define LOCAL_BUFFER_SIZE 4194304 + +struct LocalReceiveBuffer { + shm::ring_buffer *queue; + + char buffer[LOCAL_BUFFER_SIZE]; + unsigned counter = 0; + unsigned buffer_size = 0; + + LocalReceiveBuffer(shm::ring_buffer *queue) : queue(queue) {} + + template + void pop(T &t) { + size_t size = sizeof(T); + T value = 0; + uint8_t current_byte = 0; + + while (current_byte < size) { + // load the value from the buffer + while (counter < buffer_size && current_byte < size) { + // lsb first + // auto v = (uint64_t)(uint8_t)buffer[counter]; + // value |= (v << (8 * current_byte)); + counter++; + current_byte++; + } + + // load the buffer up + if (counter == buffer_size) { + buffer_size = queue->pop(buffer, LOCAL_BUFFER_SIZE); + counter = 0; + } + } + + // std::cout << "pop " << (uint64_t)value << std::endl; + + // set the value + t = value; + } +}; + int main() { @@ -30,93 +73,128 @@ int main() // shm::char_alloc char_alloc(segment.get_segment_manager()); shm::ring_buffer *queue = segment.find_or_construct("queue")(); + LocalReceiveBuffer buffer(queue); - auto counter = 0; + uint64_t counter = 0; // shm::shared_string v(char_alloc); using Action=DepMod::DepModAction; uint32_t loop_id; - while (true) { - uint64_t v; - if (queue->pop(v)) { - counter++; - auto action = static_cast(v); - - uint64_t v1, v2, v3, v4; - switch (action) { - case Action::INIT: { - while (!queue->pop(v1)); - while (!queue->pop(v2)); - if (DEBUG) { - std::cout << "INIT " << v1 << " " << v2 << std::endl; - } - loop_id = v1; - DepMod::init(v1, v2); - break; - }; - case Action::LOAD: { - while (!queue->pop(v1)) - ; - while (!queue->pop(v2)) - ; - while (!queue->pop(v3)) - ; - while (!queue->pop(v4)) - ; - if (DEBUG) { - std::cout << "LOAD " << v1 << " " << v2 << " " << v3 << " " << v4 << std::endl; - } - DepMod::load(v1, v2, v3, v4); - - break; - }; - case Action::STORE: { - while (!queue->pop(v1)); - while (!queue->pop(v2)); - while (!queue->pop(v3)); - if (DEBUG) { - std::cout << "STORE " << v1 << " " << v2 << " " << v3 << std::endl; - } - DepMod::store(v1, v2, v3); - break; - }; - case Action::ALLOC: { - while (!queue->pop(v1)); - while (!queue->pop(v2)); - DepMod::allocate(reinterpret_cast(v1), v2); - if (DEBUG) { - std::cout << "ALLOC " << v1 << " " << v2 << std::endl; - } - break; - }; - case Action::LOOP_INVOC: { - DepMod::loop_invoc(); - if (DEBUG) { - std::cout << "LOOP_INVOC" << std::endl; - } - break; - }; - case Action::LOOP_ITER: { - DepMod::loop_iter(); - if (DEBUG) { - std::cout << "LOOP_ITER" << std::endl; - } - break; - }; - case Action::FINISHED: { - std::stringstream ss; - ss << "deplog-" << loop_id << ".txt"; - DepMod::fini(ss.str().c_str()); - std::cout << "Finished loop: " << loop_id << " after " << counter - << " events" << std::endl; - break; - }; + // while (true) { + while (false) { + char v; + buffer.pop(v); + counter++; + + switch (v) { + case Action::INIT: { + uint32_t pid; + buffer.pop(loop_id); + buffer.pop(pid); + + if (DEBUG) { + std::cout << "INIT: " << loop_id << " " << pid << std::endl; + } +#if ACTION + DepMod::init(loop_id, pid); +#endif + break; + }; + case Action::LOAD: { + uint32_t instr; + uint64_t addr; + uint32_t bare_instr; + uint64_t value = 0; + buffer.pop(instr); + buffer.pop(addr); + buffer.pop(bare_instr); + // buffer.pop(value); + if (DEBUG) { + std::cout << "LOAD: " << instr << " " << addr << " " << bare_instr << " " << value << std::endl; + } +#if ACTION + DepMod::load(instr, addr, bare_instr, value); + // DepMod::load(instr, addr, bare_instr, value); +#endif + + break; + }; + case Action::STORE: { + uint32_t instr; + uint32_t bare_instr; + uint64_t addr; + buffer.pop(instr); + buffer.pop(bare_instr); + buffer.pop(addr); + if (DEBUG) { + std::cout << "STORE: " << instr << " " << bare_instr << " " << addr << std::endl; + } +#if ACTION + DepMod::store(instr, bare_instr, addr); +#endif + break; + }; + case Action::ALLOC: { + uint64_t addr; + uint64_t size; + buffer.pop(addr); + buffer.pop(size); + if (DEBUG) { + std::cout << "ALLOC: " << addr << " " << size << std::endl; + } +#if ACTION + DepMod::allocate(reinterpret_cast(addr), size); +#endif + break; + }; + case Action::LOOP_INVOC: { +#if ACTION + DepMod::loop_invoc(); +#endif + + if (DEBUG) { + std::cout << "LOOP_INVOC" << std::endl; } + break; + }; + case Action::LOOP_ITER: { +#if ACTION + DepMod::loop_iter(); +#endif - if (counter % 100000000 == 0) { - std::cout << "Processed " << counter / 1000000 << "M events" << std::endl; + if (DEBUG) { + std::cout << "LOOP_ITER" << std::endl; } + break; + }; + case Action::FINISHED: { + std::stringstream ss; + ss << "deplog-" << loop_id << ".txt"; +#if ACTION + DepMod::fini(ss.str().c_str()); +#endif + std::cout << "Finished loop: " << loop_id << " after " << counter + << " events" << std::endl; + break; + }; + default: + std::cout << "Unknown action: " << v << std::endl; + exit(-1); + } + + if (counter % 100000000 == 0) { + std::cout << "Processed " << counter / 1000000 << "M events" << std::endl; + } + } + + while (true) { + char v; + while (!queue->pop(v)); + counter++; + + if (counter % 100000000 == 0) { + std::cout << "Processed " << counter / 1000000 << "M events" << std::endl; } } } diff --git a/liberty/lib/SLAMP/SLAMPsmtxq/CMakeLists.txt b/liberty/lib/SLAMP/SLAMPsmtxq/CMakeLists.txt new file mode 100644 index 00000000..a8fcb5c0 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPsmtxq/CMakeLists.txt @@ -0,0 +1,37 @@ +file(GLOB SRCS + "*.cpp" + "*.c" +) +set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} "/u/ziyangx/test/boost/boost_1_80_0/install/lib/cmake") + +# add CMAKE_PREFIX_PATH +set(CMAKE_C_COMPILER "clang") +set(CMAKE_CXX_COMPILER "clang++") + +# Compilation flags +# set_source_files_properties(${SRCS} PROPERTIES COMPILE_FLAGS "-Wl,-save-temps -std=c++17 -Wno-inline -O3 -fexceptions")# -emit-llvm") +# set C++ flags +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17") +set_source_files_properties(${SRCS} PROPERTIES COMPILE_FLAGS "-flto -Wno-inline -O3 -fexceptions")# -emit-llvm") +set(PassName "slamp_hooks_smtx") + +set(THREADS_PREFER_PTHREAD_FLAG ON) +find_package(Boost 1.80.0 REQUIRED COMPONENTS system) +include_directories(${Boost_INCLUDE_DIRS}) + + +# list(APPEND CMAKE_MODULE_PATH "${LLVM_CMAKE_DIR}") +#include(HandleLLVMOptions) +# include(AddLLVM) + + +include_directories(./ + /u/ziyangx/test/boost/boost_1_80_0/install/include) + +add_library(${PassName} STATIC ${SRCS}) +target_link_libraries(${PassName} ${Boost_LIBRARIES}) +# target_link_libraries(${PassName} nng::nng) +# add_llvm_library(${PassName}_shared SHARED ${SRCS}) +# set_target_properties(${PassName}_shared PROPERTIES OUTPUT_NAME ${PassName}) +# set_property(TARGET ${PassName} PROPERTY INTERPROCEDURAL_OPTIMIZATION TRUE) +#add_llvm_library(${PassName} SHARED ${SRCS}) # This is to generate libxxx.so diff --git a/liberty/lib/SLAMP/SLAMPsmtxq/SmtxSend.cpp b/liberty/lib/SLAMP/SLAMPsmtxq/SmtxSend.cpp new file mode 100644 index 00000000..391d9a9d --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPsmtxq/SmtxSend.cpp @@ -0,0 +1,544 @@ +#include "slamp_hooks.h" +#include +#include +#include "malloc.h" + +#include +#include "sw_queue_astream.h" + +#include +#include + +namespace bip = boost::interprocess; + +static void *(*old_malloc_hook)(size_t, const void *); +static void (*old_free_hook)(void *, const void *); +static void *(*old_memalign_hook)(size_t, size_t, const void *); +// create segment and corresponding allocator +bip::managed_shared_memory *segment; +static SW_Queue the_queue; + +#define CONSUME sq_consume(the_queue); +#define PRODUCE(x) sq_produce(the_queue,(uint64_t)x); + +#define PRODUCE_2(x,y) PRODUCE( (((uint64_t)x)<<32) | (uint32_t)(y) ) +#define CONSUME_2(x,y) do { uint64_t tmp = CONSUME; x = (uint32_t)(tmp>>32); y = (uint32_t) tmp; } while(0) + +// Ringbuffer fully constructed in shared memory. The element strings are +// also allocated from the same shared memory segment. This vector can be +// safely accessed from other processes. +unsigned long counter_load = 0; +unsigned long counter_store = 0; +unsigned long counter_ctx = 0; +unsigned long counter_alloc = 0; +// char local_buffer[LOCAL_BUFFER_SIZE]; +// unsigned buffer_counter = 0; +bool onProfiling = false; + +enum DepModAction: char +{ + INIT = 0, + LOAD, + STORE, + ALLOC, + LOOP_INVOC, + LOOP_ITER, + FINISHED +}; + +void SLAMP_init(uint32_t fn_id, uint32_t loop_id) { + + segment = new bip::managed_shared_memory(bip::open_or_create, "MySharedMemory", sizeof(uint64_t) *QSIZE *2); + // auto a_queue = new atomic_queue::AtomicQueueB(65536); + the_queue = static_cast(segment->find_or_construct("MyQueue")()); + auto data = static_cast(segment->find_or_construct("smtx_queue_data")[QSIZE]()); + if (the_queue == nullptr) { + std::cout << "Error: could not create queue" << std::endl; + exit(-1); + } + the_queue->data = data; + if (the_queue->data == nullptr) { + std::cout << "Error: could not create queue data" << std::endl; + exit(-1); + } + + /* Initialize the queue data structure */ + the_queue->p_data = (uint64_t) the_queue->data; + the_queue->c_inx = 0; + the_queue->c_margin = 0; + the_queue->p_glb_inx = 0; + the_queue->c_glb_inx = 0; + the_queue->ptr_c_glb_inx = &(the_queue->c_glb_inx); + the_queue->ptr_p_glb_inx = &(the_queue->p_glb_inx); + // local_buffer = new LocalBuffer(queue); + + // send a msg with "fn_id, loop_id" + // char msg[100]; + // sprintf(msg, "%d,%d", fn_id, loop_id); + // queue->push(shm::shared_string(msg, *char_alloc)); + + // local_buffer->push(INIT); + // local_buffer->push(loop_id); + PRODUCE(INIT); + PRODUCE(loop_id); + uint32_t pid = getpid(); + printf("SLAMP_init: %d, %d, %d\n", fn_id, loop_id, pid); + // local_buffer->push(pid); + PRODUCE(pid); + + old_malloc_hook = __malloc_hook; + // old_free_hook = __free_hook; + old_memalign_hook = __memalign_hook; + __malloc_hook = SLAMP_malloc_hook; + __free_hook = nullptr; + __realloc_hook = nullptr; + // __free_hook = SLAMP_free_hook; + __memalign_hook = SLAMP_memalign_hook; +} + +void SLAMP_fini(const char* filename){ + // send a msg with "fini" + // queue->push(shm::shared_string("fini", *char_alloc)); + std::cout << counter_load << " " << counter_store << " " << counter_ctx << std::endl; + // local_buffer->push(FINISHED); + // local_buffer->flush(); + PRODUCE(FINISHED); +} + +void SLAMP_allocated(uint64_t addr){} +void SLAMP_init_global_vars(const char *name, uint64_t addr, size_t size){ + // local_buffer->push(ALLOC)->push(addr)->push(size); + PRODUCE(ALLOC); + PRODUCE(addr); + PRODUCE(size); +} +void SLAMP_main_entry(uint32_t argc, char** argv, char** env){} + +void SLAMP_enter_fcn(uint32_t id){} +void SLAMP_exit_fcn(uint32_t id){} +void SLAMP_enter_loop(uint32_t id){} +void SLAMP_exit_loop(uint32_t id){} +void SLAMP_loop_iter_ctx(uint32_t id){} +void SLAMP_loop_invocation(){ + // send a msg with "loop_invocation" + // local_buffer->push(LOOP_INVOC); + PRODUCE(LOOP_INVOC); + + counter_ctx++; + onProfiling = true; +} +void SLAMP_loop_iteration(){ + // local_buffer->push(LOOP_ITER); + PRODUCE(LOOP_ITER); + counter_ctx++; +} + +void SLAMP_loop_exit(){ + onProfiling = false; +} + +void SLAMP_report_base_pointer_arg(uint32_t, uint32_t, void *ptr){} +void SLAMP_report_base_pointer_inst(uint32_t, void *ptr){} +void SLAMP_callback_stack_alloca(uint64_t, uint64_t, uint32_t, uint64_t){} +void SLAMP_callback_stack_free(){} + +void SLAMP_ext_push(const uint32_t instr){} +void SLAMP_ext_pop(){} + +void SLAMP_push(const uint32_t instr){} +void SLAMP_pop(){} + +void SLAMP_load(const uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline) { + // send a msg with "load, instr, addr, bare_instr, value" + // char msg[100]; + // sprintf(msg, "load,%d,%lu,%d,%lu", instr, addr, bare_instr, value); + // queue->push(shm::shared_string(msg, *char_alloc)); + // + if (onProfiling) { + // local_buffer->push(LOAD)->push(instr)->push(addr)->push(bare_instr); //->push(value); + PRODUCE(LOAD); + PRODUCE(instr); + PRODUCE(addr); + PRODUCE(bare_instr); + // PRODUCE(value); + counter_load++; + } +} + +void SLAMP_load1(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + SLAMP_load(instr, addr, bare_instr, value); +} +void SLAMP_load2(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + SLAMP_load(instr, addr, bare_instr, value); +} +void SLAMP_load4(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + SLAMP_load(instr, addr, bare_instr, value); +} + +void SLAMP_load8(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + SLAMP_load(instr, addr, bare_instr, value); +} + +void SLAMP_loadn(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, size_t n){ + SLAMP_load(instr, addr, bare_instr, 0); +} + +void SLAMP_load1_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + SLAMP_load1(bare_instr, addr, bare_instr, value); +} +void SLAMP_load2_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + SLAMP_load2(bare_instr, addr, bare_instr, value); +} +void SLAMP_load4_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + SLAMP_load4(bare_instr, addr, bare_instr, value); +} +void SLAMP_load8_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + SLAMP_load8(bare_instr, addr, bare_instr, value); +} +void SLAMP_loadn_ext(const uint64_t addr, const uint32_t bare_instr, size_t n){ + SLAMP_loadn(bare_instr, addr, bare_instr, n); +} + +void SLAMP_store(const uint32_t instr, const uint64_t addr, const uint32_t bare_instr) ATTRIBUTE(always_inline) { + // send a msg with "store, instr, addr, bare_instr, value" + // char msg[100]; + // sprintf(msg, "store,%d,%lu,%d,%lu", instr, addr, bare_instr, value); + // queue->push(shm::shared_string(msg, *char_alloc)); + if (onProfiling) { + // local_buffer->push(STORE)->push(instr)->push(bare_instr)->push(addr); + PRODUCE(STORE); + PRODUCE(instr); + PRODUCE(bare_instr); + PRODUCE(addr); + counter_store++; + } +} + +void SLAMP_store1(uint32_t instr, const uint64_t addr){ + SLAMP_store(instr, addr, instr); +} + +void SLAMP_store2(uint32_t instr, const uint64_t addr){ + SLAMP_store(instr, addr, instr); +} +void SLAMP_store4(uint32_t instr, const uint64_t addr){ + SLAMP_store(instr, addr, instr); +} +void SLAMP_store8(uint32_t instr, const uint64_t addr){ + SLAMP_store(instr, addr, instr); +} +void SLAMP_storen(uint32_t instr, const uint64_t addr, size_t n){ + SLAMP_store(instr, addr, instr); +} + +void SLAMP_store1_ext(const uint64_t addr, const uint32_t bare_inst){ + SLAMP_store1(bare_inst, addr); +} +void SLAMP_store2_ext(const uint64_t addr, const uint32_t bare_inst){ + SLAMP_store2(bare_inst, addr); +} +void SLAMP_store4_ext(const uint64_t addr, const uint32_t bare_inst){ + SLAMP_store4(bare_inst, addr); +} +void SLAMP_store8_ext(const uint64_t addr, const uint32_t bare_inst){ + SLAMP_store8(bare_inst, addr); +} +void SLAMP_storen_ext(const uint64_t addr, const uint32_t bare_inst, size_t n){ + SLAMP_storen(bare_inst, addr, n); +} + +/* wrappers */ +static void* SLAMP_malloc_hook(size_t size, const void *caller){ + __malloc_hook = old_malloc_hook; + void* ptr = malloc(size); + // local_buffer->push(ALLOC)->push((uint64_t)ptr)->push(size); + PRODUCE(ALLOC); + PRODUCE((uint64_t)ptr); + PRODUCE(size); + // printf("malloc %lu at %p\n", size, ptr); + counter_alloc++; + __malloc_hook = SLAMP_malloc_hook; + return ptr; +} +static void SLAMP_free_hook(void *ptr, const void *caller){ + old_free_hook(ptr, caller); +} +static void* SLAMP_memalign_hook(size_t alignment, size_t size, const void *caller){ + old_memalign_hook = __memalign_hook; + void* ptr = memalign(alignment, size); + // local_buffer->push(ALLOC)->push((uint64_t)ptr)->push(size); + PRODUCE(ALLOC); + PRODUCE((uint64_t)ptr); + PRODUCE(size); + + // printf("memalign %lu at %p\n", size, ptr); + counter_alloc++; + __memalign_hook = SLAMP_memalign_hook; + return ptr; +} +void* SLAMP_malloc(size_t size, uint32_t instr, size_t alignment){ + // void* ptr = malloc(size); + // // send a msg with "malloc, instr, ptr, size, alignment" + // // char msg[100]; + // // sprintf(msg, "malloc,%d,%lu,%lu,%lu", instr, ptr, size, alignment); + // // queue->push(shm::shared_string(msg, *char_alloc)); + // queue->push(ALLOC); + // queue->push(reinterpret_cast(ptr)); + // queue->push(size); + // counter_alloc++; + // return ptr; +} + +void* SLAMP_calloc(size_t nelem, size_t elsize){} +void* SLAMP_realloc(void* ptr, size_t size){} +void* SLAMP__Znam(size_t size){} +void* SLAMP__Znwm(size_t size){} + + + +char* SLAMP_strdup(const char *s1){} +char* SLAMP___strdup(const char *s1){} +void SLAMP_free(void* ptr){} +void SLAMP_cfree(void* ptr){} +void SLAMP__ZdlPv(void* ptr){} +void SLAMP__ZdaPv(void* ptr){} +int SLAMP_brk(void *end_data_segment){} +void* SLAMP_sbrk(intptr_t increment){} + +/* llvm memory intrinsics */ +void SLAMP_llvm_memcpy_p0i8_p0i8_i32(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint32_t sizeBytes){} +void SLAMP_llvm_memcpy_p0i8_p0i8_i64(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint64_t sizeBytes){} +void SLAMP_llvm_memmove_p0i8_p0i8_i32(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint32_t sizeBytes){} +void SLAMP_llvm_memmove_p0i8_p0i8_i64(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint64_t sizeBytes){} +void SLAMP_llvm_memset_p0i8_i32(const uint8_t* dstAddr, const uint32_t len){} +void SLAMP_llvm_memset_p0i8_i64(const uint8_t* dstAddr, const uint64_t len){} + +// void SLAMP_llvm_lifetime_start_p0i8(uint64_t size, uint8_t* ptr){} +// void SLAMP_llvm_lifetime_end_p0i8(uint64_t size, uint8_t* ptr){} + +/* String functions */ +size_t SLAMP_strlen(const char *str){} +char* SLAMP_strchr(char *s, int c){} +char* SLAMP_strrchr(char *s, int c){} +int SLAMP_strcmp(const char *s1, const char *s2){} +int SLAMP_strncmp(const char *s1, const char *s2, size_t n){} +char* SLAMP_strcpy(char *dest, const char *src){} +char* SLAMP_strncpy(char *dest, const char *src, size_t n){} +char* SLAMP_strcat(char *s1, const char *s2){} +char* SLAMP_strncat(char *s1, const char *s2, size_t n){} +char* SLAMP_strstr(char *s1, char *s2){} +size_t SLAMP_strspn(const char *s1, const char *s2){} +size_t SLAMP_strcspn(const char *s1, const char *s2){} +char* SLAMP_strtok(char *s, const char *delim){} +double SLAMP_strtod(const char *nptr, char **endptr){} +long int SLAMP_strtol(const char *nptr, char **endptr, int base){} +char* SLAMP_strpbrk(char *s1, char *s2){} + +/* Mem* and b* functions */ +void *SLAMP_memset (void *dest, int c, size_t n){} +void *SLAMP_memcpy (void *dest, const void *src, size_t n){} +void *SLAMP___builtin_memcpy (void *dest, const void *src, size_t n){} +void *SLAMP_memmove (void *dest, const void *src, size_t n){} +int SLAMP_memcmp(const void *s1, const void *s2, size_t n){} +void* SLAMP_memchr(void* ptr, int value, size_t num){} +void* SLAMP___rawmemchr(void* ptr, int value){} + +void SLAMP_bzero(void *s, size_t n){} +void SLAMP_bcopy(const void *s1, void *s2, size_t n){} + +/* IO */ +ssize_t SLAMP_read(int fd, void *buf, size_t count){} +int SLAMP_open(const char *pathname, int flags, mode_t mode){} +int SLAMP_close(int fd){} +ssize_t SLAMP_write(int fd, const void *buf, size_t count){} +off_t SLAMP_lseek(int fildes, off_t offset, int whence){} + +FILE * SLAMP_fopen(const char *path, const char *mode){} +FILE * SLAMP_fopen64(const char *path, const char *mode){} +FILE * SLAMP_freopen(const char *path, const char *mode, FILE* stream){} +int SLAMP_fflush(FILE *stream){} +int SLAMP_fclose(FILE *stream){} +int SLAMP_ferror(FILE *stream){} +int SLAMP_feof(FILE *stream){} +long SLAMP_ftell(FILE *stream){} +size_t SLAMP_fread(void * ptr, size_t size, size_t nitems, FILE *stream){} +size_t SLAMP_fwrite(const void *ptr, size_t size, size_t nitems, FILE *stream){} +int SLAMP_fseek(FILE *stream, long offset, int whence){} +void SLAMP_rewind(FILE *stream){} + +int SLAMP_fgetc(FILE *stream){} +int SLAMP_fputc(int c, FILE *stream){} +char * SLAMP_fgets(char *s, int n, FILE *stream){} +int SLAMP_fputs(const char *s, FILE *stream){} + +int SLAMP_ungetc(int c, FILE *stream){} +int SLAMP_putchar(int c){} +int SLAMP_getchar(void){} + +int SLAMP_fileno(FILE *stream){} +char * SLAMP_gets(char *s){} +int SLAMP_puts(const char *s){} + +int SLAMP_select(int n, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, struct timeval *timeout){} +int SLAMP_remove(const char *path){} + +void SLAMP_setbuf(FILE * stream, char * buf){} +void SLAMP_setvbuf(FILE * stream, char * buf, int mode, size_t size){} +char * SLAMP_tmpnam(char *s){} +FILE* SLAMP_tmpfile(void){} +char * SLAMP_ttyname(int fildes){} + +FILE * SLAMP_fdopen(int fildes, const char *mode){} +void SLAMP_clearerr(FILE *stream){} + +int SLAMP_truncate(const char *path, off_t length){} +int SLAMP_ftruncate(int fildes, off_t length){} + +int SLAMP_dup(int oldfd){} +int SLAMP_dup2(int oldfd, int newfd){} +int SLAMP_pipe(int filedes[2]){} + +int SLAMP_chmod(const char *path, mode_t mode){} +int SLAMP_fchmod(int fildes, mode_t mode){} +int SLAMP_fchown(int fd, uid_t owner, gid_t group){} +int SLAMP_access(const char *pathname, int mode){} +long SLAMP_pathconf(char *path, int name){} +int SLAMP_mkdir(const char *pathname, mode_t mode){} +int SLAMP_rmdir(const char *pathname){} +mode_t SLAMP_umask(mode_t mask){} +int SLAMP_fcntl(int fd, int cmd, struct flock *lock){} + +DIR* SLAMP_opendir(const char* name){} +struct dirent* SLAMP_readdir(DIR *dirp){} +struct dirent64* SLAMP_readdir64(DIR *dirp){} +int SLAMP_closedir(DIR* dirp){} + +/* Printf */ +int SLAMP_printf(const char *format, ...){} +int SLAMP_fprintf(FILE *stream, const char *format, ...){} +int SLAMP_sprintf(char *str, const char *format, ...){} +int SLAMP_snprintf(char *str, size_t size, const char *format, ...){} + +int SLAMP_vprintf(const char *format, va_list ap){} +int SLAMP_vfprintf(FILE *stream, const char *format, va_list ap){} +int SLAMP_vsprintf(char *str, const char *format, va_list ap){} +int SLAMP_vsnprintf(char *str, size_t size, const char *format, va_list ap){} + +/* Scanf */ +int SLAMP_fscanf(FILE *stream, const char *format, ... ){} +int SLAMP_scanf(const char *format, ... ){} +int SLAMP_sscanf(const char *s, const char *format, ... ){} +int SLAMP___isoc99_sscanf(const char *s, const char *format, ... ){} + +int SLAMP_vfscanf(FILE *stream, const char *format, va_list ap){} +int SLAMP_vscanf(const char *format, va_list ap){} +int SLAMP_vsscanf(const char *s, const char *format, va_list ap){} + +/* Time */ +time_t SLAMP_time(time_t *t){} +struct tm *SLAMP_localtime(const time_t *timer){} +struct lconv* SLAMP_localeconv(){} +struct tm *SLAMP_gmtime(const time_t *timer){} +int SLAMP_gettimeofday(struct timeval *tv, struct timezone *tz){} + +/* Math */ +double SLAMP_ldexp(double x, int exp){} +float SLAMP_ldexpf(float x, int exp){} +long double SLAMP_ldexpl(long double x, int exp){} +double SLAMP_log10(double x){} +float SLAMP_log10f(float x){} +long double SLAMP_log10l(long double x){} +double SLAMP_log(double x){} +float SLAMP_logf(float x){} +long double SLAMP_logl(long double x){} + +double SLAMP_exp(double x){} +float SLAMP_expf(float x){} +long double SLAMP_expl(long double x){} + +double SLAMP_cos(double x){} +float SLAMP_cosf(float x){} +long double SLAMP_cosl(long double x){} +double SLAMP_sin(double x){} +double SLAMP_tan(double x){} +float SLAMP_sinf(float x){} +long double SLAMP_sinl(long double x){} + +double SLAMP_atan(double x){} +float SLAMP_atanf(float x){} +long double SLAMP_atanl(long double x){} + +double SLAMP_floor(double x){} +float SLAMP_floorf(float x){} +long double SLAMP_floorl(long double x){} +double SLAMP_ceil(double x){} +float SLAMP_ceilf(float x){} +long double SLAMP_ceill(long double x){} + +double SLAMP_atan2(double y, double x){} +float SLAMP_atan2f(float y, float x){} +long double SLAMP_atan2l(long double y, long double x){} + +double SLAMP_sqrt(double x){} +float SLAMP_sqrtf(float x){} +long double SLAMP_sqrtl(long double x){} + +double SLAMP_pow(double x, double y){} +float SLAMP_powf(float x, float y){} +long double SLAMP_powl(long double x, long double y){} + +double SLAMP_fabs(double x){} +float SLAMP_fabsf(float x){} +long double SLAMP_fabsl(long double x){} + +double SLAMP_modf(double x, double *iptr){} +float SLAMP_modff(float x, float *iptr){} +long double SLAMP_modfl(long double x, long double *iptr){} + +double SLAMP_fmod(double x, double y){} + +double SLAMP_frexp(double num, int *exp){} +float SLAMP_frexpf(float num, int *exp){} +long double SLAMP_frexpl(long double num, int *exp){} + +int SLAMP_isnan(){} + +/* MISC */ +char *SLAMP_getenv(const char *name){} +int SLAMP_putenv(char* string){} +char *SLAMP_getcwd(char *buf, size_t size){} +char* SLAMP_strerror(int errnum){} +void SLAMP_exit(int status){} +void SLAMP__exit(int status){} +int SLAMP_link(const char *oldpath, const char *newpath){} +int SLAMP_unlink(const char *pathname){} +int SLAMP_isatty(int desc){} +int SLAMP_setuid(uid_t uid){} +uid_t SLAMP_getuid(void){} +uid_t SLAMP_geteuid(void){} +int SLAMP_setgid(gid_t gid){} +gid_t SLAMP_getgid(void){} +gid_t SLAMP_getegid(void){} +pid_t SLAMP_getpid(void){} +int SLAMP_chdir(const char *path){} +int SLAMP_execl(const char *path, const char *arg0, ... /*, (char *)0 */){} +int SLAMP_execv(const char *path, char *const argv[]){} +int SLAMP_execvp(const char *file, char *const argv[]){} +int SLAMP_kill(pid_t pid, int sig){} +pid_t SLAMP_fork(void){} +sighandler_t SLAMP___sysv_signal(int signum, sighandler_t handler){} +pid_t SLAMP_waitpid(pid_t pid, int* status, int options){} +void SLAMP_qsort(void* base, size_t nmemb, size_t size, int(*compar)(const void *, const void *)){} +int SLAMP_ioctl(int d, int request, ...){} +unsigned int SLAMP_sleep(unsigned int seconds){} +char* SLAMP_gcvt(double number, size_t ndigit, char* buf){} +char* SLAMP_nl_langinfo(nl_item item){} + +/* Compiler/Glibc Internals */ +void SLAMP___assert_fail(const char * assertion, const char * file, unsigned int line, const char * function){} +const unsigned short int **SLAMP___ctype_b_loc(void){} +int SLAMP__IO_getc(_IO_FILE * __fp){} +int SLAMP__IO_putc(int __c, _IO_FILE *__fp){} + +int SLAMP___fxstat (int __ver, int __fildes, struct stat *__stat_buf){} +int SLAMP___xstat (int __ver, __const char *__filename, struct stat *__stat_buf){} diff --git a/liberty/lib/SLAMP/SLAMPsmtxq/bitcast.h b/liberty/lib/SLAMP/SLAMPsmtxq/bitcast.h new file mode 100644 index 00000000..404a9a99 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPsmtxq/bitcast.h @@ -0,0 +1,29 @@ +#ifndef BITCAST_H +#define BITCAST_H + +typedef union { double d; int64_t i; } SQ_DI; +typedef union { float f; int32_t i; } SQ_FI; + +Inline int64_t doubleToInt(double d) { + SQ_DI conv; + conv.d = d; + return conv.i; +} +Inline int64_t floatToInt(float f) { + SQ_FI conv; + conv.f = f; + return conv.i; +} +Inline double intToDouble(int64_t i) { + SQ_DI conv; + conv.i = (int32_t)i; + return conv.d; +} + +Inline float intToFloat(int64_t i) { + SQ_FI conv; + conv.i = (int32_t)i; + return conv.f; +} + +#endif /* BITCAST_H */ diff --git a/liberty/lib/SLAMP/SLAMPsmtxq/consumer/.gitignore b/liberty/lib/SLAMP/SLAMPsmtxq/consumer/.gitignore new file mode 100644 index 00000000..378eac25 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPsmtxq/consumer/.gitignore @@ -0,0 +1 @@ +build diff --git a/liberty/lib/SLAMP/SLAMPsmtxq/consumer/CMakeLists.txt b/liberty/lib/SLAMP/SLAMPsmtxq/consumer/CMakeLists.txt new file mode 100644 index 00000000..a043216a --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPsmtxq/consumer/CMakeLists.txt @@ -0,0 +1,24 @@ +cmake_minimum_required (VERSION 3.6.2 FATAL_ERROR) + +# set(CMAKE_C_COMPILER "clang") +# set(CMAKE_CXX_COMPILER "clang++") +# set(CMAKE_LINKER "ld.gold") + +set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} "/u/ziyangx/test/boost/boost_1_80_0/install/lib/cmake") +set(THREADS_PREFER_PTHREAD_FLAG ON) +find_package(Threads REQUIRED) + +# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -flto") +# set(LINK_FLAGS "${LINK_FLAGS} -flto") + +find_package(Boost 1.80.0 REQUIRED COMPONENTS system) +include_directories(${Boost_INCLUDE_DIRS}) +include_directories(./ + ../) + +add_executable (consumer consumer.cpp ../sw_queue.c ProfilingModules/DependenceModule.cpp) +target_link_libraries(consumer LINK_PUBLIC + ${Boost_LIBRARIES} + rt + Threads::Threads +) diff --git a/liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/DependenceModule.cpp b/liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/DependenceModule.cpp new file mode 100644 index 00000000..a78b3487 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/DependenceModule.cpp @@ -0,0 +1,118 @@ +#include +#include +#include +#include + +#include "slamp_timestamp.h" +#include "slamp_logger.h" +#include "slamp_shadow_mem.h" +#include "DependenceModule.h" +#define SIZE_8M 0x800000 + + +static slamp::MemoryMap* smmap = nullptr; +static std::unordered_set *deplog_set; +static uint64_t slamp_iteration = 0; +static uint64_t slamp_invocation = 0; +static uint32_t target_loop_id = 0; + +namespace DepMod { +// init: setup the shadow memory +void init(uint32_t loop_id, uint32_t pid) { + + target_loop_id = loop_id; + smmap = new slamp::MemoryMap(TIMESTAMP_SIZE_IN_BYTES); + deplog_set = new std::unordered_set(); + + smmap->init_stack(SIZE_8M, pid); + smmap->allocate((void*)&errno, sizeof(errno)); + smmap->allocate((void*)&stdin, sizeof(stdin)); + smmap->allocate((void*)&stdout, sizeof(stdout)); + smmap->allocate((void*)&stderr, sizeof(stderr)); + smmap->allocate((void*)&sys_nerr, sizeof(sys_nerr)); + + { + const unsigned short int* ctype_ptr = (*__ctype_b_loc()) - 128; + smmap->allocate((void*)ctype_ptr, 384 * sizeof(*ctype_ptr)); + } + { + const int32_t* itype_ptr = (*__ctype_tolower_loc()) - 128; + smmap->allocate((void*)itype_ptr, 384 * sizeof(*itype_ptr)); + } + { + const int32_t* itype_ptr = (*__ctype_toupper_loc()) - 128; + smmap->allocate((void*)itype_ptr, 384 * sizeof(*itype_ptr)); + } +} + +void fini(const char *filename) { + std::ofstream of(filename); + of << target_loop_id << " " << 0 << " " << 0 << " " + << 0 << " " << 0 << " " << 0 << "\n"; + + std::set ordered(deplog_set->begin(), deplog_set->end()); + for (auto &k: ordered) { + of << target_loop_id << " " << k.src << " " << k.dst << " " << k.dst_bare << " " + << (k.cross ? 1 : 0) << " " << 1 << " "; + of << "\n"; + } + + delete smmap; + delete deplog_set; +} + +void allocate(void *addr, uint64_t size) { + smmap->allocate(addr, size); + // std::cout << "allocate " << addr << " " << size << std::endl; +} + +// void log(TS ts, const uint32_t dst_inst, TS *pts, const uint32_t bare_inst, + // uint64_t addr, uint64_t value, uint8_t size) { +void log(TS ts, const uint32_t dst_inst, const uint32_t bare_inst){ + + uint32_t src_inst = GET_INSTR(ts); + uint64_t src_iter = GET_ITER(ts); + + // uint64_t src_invoc = GET_INVOC(ts); + + slamp::KEY key(src_inst, dst_inst, bare_inst, src_iter != slamp_iteration); + + // std::cout << "src_inst: " << src_inst << " dst_inst: " << dst_inst << " bare_inst: " << bare_inst << " src_iter: " << src_iter << " slamp_iteration: " << slamp_iteration << " src_iter != slamp_iteration: " << (src_iter != slamp_iteration) << std::endl; + // deplog_set->insert(key); +} + +// template +void load(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + TS* s = (TS*)GET_SHADOW(addr, TIMESTAMP_SIZE_IN_POWER_OF_TWO); + TS tss = s[0]; + if (tss != 0) { + log(tss, instr, bare_instr); + } +} + +// template +void store(uint32_t instr, uint32_t bare_instr, const uint64_t addr) { + TS *s = (TS *)GET_SHADOW(addr, TIMESTAMP_SIZE_IN_POWER_OF_TWO); + // if (!smmap->is_allocated(s)) { + // std::cout << "store not allocated: " << instr << " " << bare_instr << " " << addr << std::endl; + // } + TS ts = CREATE_TS(instr, slamp_iteration, slamp_invocation); + + // TODO: handle output dependence. ignore it as of now. + // if (ASSUME_ONE_ADDR) { + s[0] = ts; + // } else { + // for (auto i = 0; i < size; i++) + // s[i] = ts; + // } +} + +void loop_invoc() { + slamp_iteration = 0; + slamp_invocation++; +} + +void loop_iter() { + slamp_iteration++; +} +} // namespace DepMod diff --git a/liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/DependenceModule.h b/liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/DependenceModule.h new file mode 100644 index 00000000..a2664cdc --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/DependenceModule.h @@ -0,0 +1,23 @@ +#include + +namespace DepMod { +void init(uint32_t loop_id, uint32_t pid); +void fini(const char *filename); +void load(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value); +void store(uint32_t instr, uint32_t bare_instr, const uint64_t addr); +void allocate(void *addr, uint64_t size); +void loop_invoc(); +void loop_iter(); + +enum DepModAction: char +{ + INIT = 0, + LOAD, + STORE, + ALLOC, + LOOP_INVOC, + LOOP_ITER, + FINISHED +}; + +} // namespace DepMod diff --git a/liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/slamp_bound_malloc.h b/liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/slamp_bound_malloc.h new file mode 100644 index 00000000..0d6b9410 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/slamp_bound_malloc.h @@ -0,0 +1,22 @@ +#ifndef SLAMPLIB_HOOKS_SLAMP_BOUND_MALLOC_H +#define SLAMPLIB_HOOKS_SLAMP_BOUND_MALLOC_H + +#include +#include +namespace slamp +{ + +void init_bound_malloc(void* heap_bound); +void fini_bound_malloc(); + +size_t get_object_size(void* ptr); + +void* bound_malloc(size_t size, size_t alignment=16); +bool bound_free(void *ptr, uint64_t &starting_page, unsigned &purge_cnt); +void* bound_calloc(size_t num, size_t size); +void* bound_realloc(void* ptr, size_t size); +void bound_discard_page(); + +} + +#endif diff --git a/liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/slamp_logger.h b/liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/slamp_logger.h new file mode 100644 index 00000000..89641a03 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/slamp_logger.h @@ -0,0 +1,110 @@ +#ifndef SLAMPLIB_HOOKS_SLAMP_LOGGER +#define SLAMPLIB_HOOKS_SLAMP_LOGGER + +#include +#include +#include +#include + +#include "slamp_timestamp.h" + +namespace slamp +{ + +struct KEY +{ + uint32_t src; + uint32_t dst; + uint32_t dst_bare; + uint32_t cross; + + KEY() {} + KEY(uint32_t s, uint32_t d, uint32_t b, uint32_t c) : src(s), dst(d), dst_bare(b), cross(c) {} +}; + +struct KEYHash +{ + size_t operator()(const KEY& key) const + { + static_assert(sizeof(size_t) == sizeof(uint64_t), "Should be 64bit address"); + std::hash hash_fn; + + // hash(32-32) ^ hash(32-1) + // FIXME: find a better hash function + return hash_fn(((uint64_t)key.src << 32) | key.dst) ^ hash_fn(((uint64_t)key.dst_bare << 1) | (key.cross & 0x1)); + } +}; + +struct KEYComp +{ + bool operator()(const KEY& key1, const KEY& key2) const + { + uint32_t src1 = key1.src; + uint32_t src2 = key2.src; + + if (src1 < src2) + return true; + else if (src1 > src2) + return false; + + uint32_t dst1 = key1.dst; + uint32_t dst2 = key2.dst; + + if (dst1 < dst2) + return true; + else if (dst1 > dst2) + return false; + + uint32_t dst_bare1 = key1.dst_bare; + uint32_t dst_bare2 = key2.dst_bare; + + if (dst_bare1 < dst_bare2) + return true; + else if (dst_bare1 > dst_bare2) + return false; + + uint32_t cross1 = key1.cross; + uint32_t cross2 = key2.cross; + + return cross1 < cross2; + } +}; + +struct KEYEqual +{ + bool operator()(const KEY& key1, const KEY& key2) const + { + uint32_t src1 = key1.src; + uint32_t src2 = key2.src; + + if ( src1 != src2 ) return false; + + uint32_t dst1 = key1.dst; + uint32_t dst2 = key2.dst; + + if ( dst1 != dst2 ) return false; + + uint32_t dst_bare1 = key1.dst_bare; + uint32_t dst_bare2 = key2.dst_bare; + + if ( dst_bare1 != dst_bare2 ) return false; + + uint32_t cross1 = key1.cross; + uint32_t cross2 = key2.cross; + + if ( cross1 != cross2 ) return false; + + return true; + } +}; + + +void init_logger(uint32_t fn_id, uint32_t loop_id); +void fini_logger(const char* filename); + +uint32_t log(TS ts, const uint32_t dst_instr, TS* pts, const uint32_t bare_inst, uint64_t addr, uint64_t value, uint8_t size); +void print_log(const char* filename); + +} + +#endif diff --git a/liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/slamp_shadow_mem.h b/liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/slamp_shadow_mem.h new file mode 100644 index 00000000..ef07201b --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/slamp_shadow_mem.h @@ -0,0 +1,197 @@ +#ifndef SLAMPLIB_HOOKS_SLAMP_SHADOW_MEM_H +#define SLAMPLIB_HOOKS_SLAMP_SHADOW_MEM_H + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +// higher half of canonical region cannot be used + +/// left shift by `shift`, mask 47 LSB, toggle #45 bit? +#define MASK1 0x00007fffffffffffL +#define MASK2 0x0000200000000000L +#define GET_SHADOW(addr, shift) \ + (((((uint64_t)(addr)) << (shift)) & MASK1) ^ MASK2) + +namespace slamp { + +class MemoryMap { +public: + uint64_t heapStart = 0; + MemoryMap(unsigned r) : ratio(r), ratio_shift(0) { + // ratio expected to be a power of 2 + assert((r & (r - 1)) == 0); + + // log(r) + unsigned n = r; + while ((n & 1) == 0) { + this->ratio_shift += 1; + n = n >> 1; + } + + // get the page size of the host system + pagesize = getpagesize(); + pagemask = ~(pagesize - 1); + } + + ~MemoryMap() { + // freeing all remaining shadow addresses + for (auto page : pages) { + uint64_t s = GET_SHADOW(page, ratio_shift); + munmap(reinterpret_cast(s), pagesize * ratio); + } + } + + unsigned get_ratio() { return ratio; } + + bool is_allocated(void *addr) { + auto a = reinterpret_cast(addr); + uint64_t page = a & pagemask; + if (pages.find(page) != pages.end()) + return true; + else + return false; + } + + /// allocate shadow page if not exist + void *allocate(void *addr, size_t size) { + auto a = reinterpret_cast(addr); + uint64_t pagebegin = a & pagemask; + uint64_t pageend = (a + size - 1) & pagemask; + + // try mmap + + std::set shadow_pages; + bool success = true; + + for (uint64_t page = pagebegin; page <= pageend; page += pagesize) { + if (pages.find(page) != pages.end()) + continue; + + uint64_t s = GET_SHADOW(page, ratio_shift); + // create a shadow page for the page + void *p = mmap(reinterpret_cast(s), pagesize * ratio, + PROT_WRITE | PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (p == MAP_FAILED) { + int err = errno; + printf("mmap failed: %lx errno: %d\n", s, err); + raise(SIGINT); + + success = false; + break; + } else { + shadow_pages.insert(p); + } + } + + if (success) { + for (uint64_t page = pagebegin; page <= pageend; page += pagesize) + pages.insert(page); + uint64_t pagebegin = a & pagemask; + uint64_t pageend = (a + size - 1) & pagemask; + + // return shadow_mem + auto *shadow_addr = (uint64_t *)GET_SHADOW(a, ratio_shift); + return (void *)(shadow_addr); + } else { + // cleanup + for (auto shadow_page : shadow_pages) + munmap(shadow_page, pagesize * ratio); + return nullptr; + } + } + + // free the shadow pages + void deallocate_pages(uint64_t page, unsigned cnt) { + munmap(reinterpret_cast(GET_SHADOW(page, ratio_shift)), + pagesize * ratio * cnt); + + // fprintf(stderr, "deallocate_pages: %lx %d\n", GET_SHADOW(page, ratio_shift), cnt); + + for (auto i = 0; i < cnt; i++, page += pagesize) + pages.erase(page); + } + + /// for realloc; the dependence carries over + void copy(void *dst, void *src, size_t size) { + size_t shadow_size = size * ratio; + void *shadow_dst = (void *)GET_SHADOW(dst, ratio_shift); + void *shadow_src = (void *)GET_SHADOW(src, ratio_shift); + memcpy(shadow_dst, shadow_src, shadow_size); + } + + void init_heap(void *addr) { + heapStart = reinterpret_cast(addr); + } + + // init stack size to be fixed 8MB + // the stack in /proc/$pid/maps changes in runtime, use it to find the end + void init_stack(uint64_t stack_size, uint64_t pid) { + char filename[256]; + char buf[5000]; + sprintf(filename, "/proc/%lu/maps", pid); + // sprintf(filename, "/proc/%u/maps", getpid()); + + FILE *fp = fopen(filename, "r"); + if (!fp) { + perror(filename); + exit(EXIT_FAILURE); + } + + bool allocated = false; + + while (fgets(buf, sizeof(buf), fp) != nullptr) { + uint64_t start, end; + char name[5000]; + + int n = sscanf(buf, "%lx-%lx %*c%*c%*c%*c %*llx %*x:%*x %*lu %s", &start, + &end, name); + + if (n != 3) { + continue; + } + + // FIXME: get heap start addr + if (!strcmp(name, "[heap]")) { + heapStart = start; + } + + + if (!strcmp(name, "[stack]")) { + // stack grow from end (big address) backwards + // print the stack start and end + printf("stack: %lx %lx\n", end - stack_size, end); + allocate(reinterpret_cast(end - stack_size), stack_size); + allocated = true; + break; + } + } + + if (!allocated) { + fprintf(stderr, "Error: failed to allocate shadow for stack"); + exit(EXIT_FAILURE); + } + } + +private: + std::set pages; // page table + + unsigned ratio; // (size of metadata) / (size of real data) + unsigned ratio_shift; + uint64_t pagesize; + uint64_t pagemask; +}; + +} // namespace slamp +#endif diff --git a/liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/slamp_timestamp.h b/liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/slamp_timestamp.h new file mode 100644 index 00000000..fc9c1428 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/slamp_timestamp.h @@ -0,0 +1,22 @@ +#ifndef SLAMPLIB_HOOKS_SLAMP_TIMESTAMP +#define SLAMPLIB_HOOKS_SLAMP_TIMESTAMP + +#include + +typedef uint64_t TS; // first 20 bits for instr and following 44 bits for iter +#define TIMESTAMP_SIZE_IN_BYTES 8 +#define TIMESTAMP_SIZE_IN_POWER_OF_TWO 3 +#define ITERATION_SIZE 40 +#define INVOCATION_SIZE 4 // 44-40 +#define CREATE_TS(instr, iter, invoc) ( ((TS)instr << 44) | (((TS)iter & (TS)0xffffffffff) << INVOCATION_SIZE) | ((TS)invoc & (TS)0xf)) +#define CREATE_TS_HASH(instr, hash, iter, invoc) \ + (((TS)instr << 44) | \ + (((TS)hash & (TS)0xfffffffff) << (INVOCATION_SIZE + 4)) | \ + (((TS)iter & (TS)0xf) << INVOCATION_SIZE) | ((TS)invoc & (TS)0xf)) +#define GET_INSTR(ts) ((ts >> 44) & 0xfffff) +#define GET_HASH(ts) ( (ts >> 8) & 0xfffffffff) +#define GET_ITER(ts) ( (ts >> INVOCATION_SIZE) & 0xffffffffff) +#define GET_INVOC(ts) ( ts & 0xf) + + +#endif diff --git a/liberty/lib/SLAMP/SLAMPsmtxq/consumer/consumer.cpp b/liberty/lib/SLAMP/SLAMPsmtxq/consumer/consumer.cpp new file mode 100644 index 00000000..eb679486 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPsmtxq/consumer/consumer.cpp @@ -0,0 +1,163 @@ +#include +#include "ProfilingModules/DependenceModule.h" +#include +#include +#include "sw_queue_astream.h" + + +#include +#include + +namespace bip = boost::interprocess; + +#define DEBUG 0 +#define ACTION 0 + +// create segment and corresponding allocator +bip::managed_shared_memory *segment; +static SW_Queue the_queue; + +#define CONSUME sq_consume(the_queue); +#define PRODUCE(x) sq_produce(the_queue,(uint64_t)x); + +#define PRODUCE_2(x,y) PRODUCE( (((uint64_t)x)<<32) | (uint32_t)(y) ) +#define CONSUME_2(x,y) do { uint64_t tmp = CONSUME; x = (uint32_t)(tmp>>32); y = (uint32_t) tmp; } while(0) + +int main() { + segment = new bip::managed_shared_memory(bip::open_or_create, "MySharedMemory", sizeof(uint64_t) *QSIZE *2); + // auto a_queue = new atomic_queue::AtomicQueueB(65536); + the_queue = static_cast(segment->find_or_construct("MyQueue")()); + auto data = static_cast(segment->find_or_construct("smtx_queue_data")[QSIZE]()); + if (the_queue == nullptr) { + std::cout << "Error: could not create queue" << std::endl; + return 1; + } + the_queue->data = data; + if (the_queue->data == nullptr) { + std::cout << "Error: could not create queue data" << std::endl; + return 1; + } + /* Initialize the queue data structure */ + the_queue->p_data = (uint64_t) the_queue->data; + the_queue->c_inx = 0; + the_queue->c_margin = 0; + the_queue->p_glb_inx = 0; + the_queue->c_glb_inx = 0; + the_queue->ptr_c_glb_inx = &(the_queue->c_glb_inx); + the_queue->ptr_p_glb_inx = &(the_queue->p_glb_inx); + + + uint64_t counter = 0; + // shm::shared_string v(char_alloc); + + using Action = DepMod::DepModAction; + + uint32_t loop_id; + while (true) { + char v; + v = (char)CONSUME; + counter++; + + switch (v) { + case Action::INIT: { + uint32_t pid; + loop_id = (uint32_t)CONSUME; + pid = (uint32_t)CONSUME; + + if (DEBUG) { + std::cout << "INIT: " << loop_id << " " << pid << std::endl; + } +#if ACTION + DepMod::init(loop_id, pid); +#endif + break; + }; + case Action::LOAD: { + uint32_t instr; + uint64_t addr; + uint32_t bare_instr; + uint64_t value = 0; + instr = (uint32_t)CONSUME; + addr = CONSUME; + bare_instr = (uint32_t)CONSUME; + // value = CONSUME; + if (DEBUG) { + std::cout << "LOAD: " << instr << " " << addr << " " << bare_instr + << " " << value << std::endl; + } +#if ACTION + DepMod::load(instr, addr, bare_instr, value); + // DepMod::load(instr, addr, bare_instr, value); +#endif + + break; + }; + case Action::STORE: { + uint32_t instr; + uint32_t bare_instr; + uint64_t addr; + instr = (uint32_t)CONSUME; + bare_instr = (uint32_t)CONSUME; + addr = CONSUME; + if (DEBUG) { + std::cout << "STORE: " << instr << " " << bare_instr << " " << addr + << std::endl; + } +#if ACTION + DepMod::store(instr, bare_instr, addr); +#endif + break; + }; + case Action::ALLOC: { + uint64_t addr; + uint64_t size; + addr = CONSUME; + size = CONSUME; + if (DEBUG) { + std::cout << "ALLOC: " << addr << " " << size << std::endl; + } +#if ACTION + DepMod::allocate(reinterpret_cast(addr), size); +#endif + break; + }; + case Action::LOOP_INVOC: { +#if ACTION + DepMod::loop_invoc(); +#endif + + if (DEBUG) { + std::cout << "LOOP_INVOC" << std::endl; + } + break; + }; + case Action::LOOP_ITER: { +#if ACTION + DepMod::loop_iter(); +#endif + + if (DEBUG) { + std::cout << "LOOP_ITER" << std::endl; + } + break; + }; + case Action::FINISHED: { + std::stringstream ss; + ss << "deplog-" << loop_id << ".txt"; +#if ACTION + DepMod::fini(ss.str().c_str()); +#endif + std::cout << "Finished loop: " << loop_id << " after " << counter + << " events" << std::endl; + break; + }; + default: + std::cout << "Unknown action: " << v << std::endl; + exit(-1); + } + + if (counter % 100000000 == 0) { + std::cout << "Processed " << counter / 1000000 << "M events" << std::endl; + } + } +} diff --git a/liberty/lib/SLAMP/SLAMPsmtxq/inline.h b/liberty/lib/SLAMP/SLAMPsmtxq/inline.h new file mode 100644 index 00000000..7b16986f --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPsmtxq/inline.h @@ -0,0 +1,12 @@ +#ifndef INLINE_H +#define INLINE_H + +#define Weak __attribute__((weak)) + +#ifndef NO_INLINE +#define Inline static inline +#else /* NO_INLINE */ +#define Inline Weak +#endif/* NO_INLINE */ + +#endif /* INLINE_H */ diff --git a/liberty/lib/SLAMP/SLAMPsmtxq/slamp_hooks.h b/liberty/lib/SLAMP/SLAMPsmtxq/slamp_hooks.h new file mode 100644 index 00000000..df546117 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPsmtxq/slamp_hooks.h @@ -0,0 +1,371 @@ +#ifndef SLAMPLIB_HOOKS_SLAMP_HOOKS_H +#define SLAMPLIB_HOOKS_SLAMP_HOOKS_H + +// FIXME: inline tweak actually make things worse in sequential and better with +// 16x, so turn it on at all time before understanding why +#define ATTRIBUTE(x) __attribute__((x)) +// #ifdef ITO_ENABLE +// // #define ATTRIBUTE(x) +// #define ATTRIBUTE(x) __attribute__((x)) +// #else +// #define ATTRIBUTE(x) +// #endif + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +void SLAMP_dbggv(int id); +void SLAMP_dbggvstr(char* str); + +// SLAMP measure functions +void SLAMP_measure_init(); +void SLAMP_measure_fini(); +void SLAMP_measure_load(uint32_t id, uint64_t size); +void SLAMP_measure_store(uint32_t id, uint64_t size); +static void* SLAMP_measure_malloc_hook(size_t size, const void *caller); +static void SLAMP_measure_free_hook(void *ptr, const void *caller); + +void SLAMP_init(uint32_t fn_id, uint32_t loop_id); +void SLAMP_fini(const char* filename); + +void SLAMP_allocated(uint64_t addr); +void SLAMP_init_global_vars(const char *name, uint64_t addr, size_t size); +void SLAMP_main_entry(uint32_t argc, char** argv, char** env); + +void SLAMP_enter_fcn(uint32_t id); +void SLAMP_exit_fcn(uint32_t id); +void SLAMP_enter_loop(uint32_t id); +void SLAMP_exit_loop(uint32_t id); +void SLAMP_loop_iter_ctx(uint32_t id); +void SLAMP_loop_invocation(); +void SLAMP_loop_iteration(); +void SLAMP_loop_exit(); + +void SLAMP_report_base_pointer_arg(uint32_t, uint32_t, void *ptr); +void SLAMP_report_base_pointer_inst(uint32_t, void *ptr); +void SLAMP_callback_stack_alloca(uint64_t, uint64_t, uint32_t, uint64_t); +void SLAMP_callback_stack_free(void); + +void SLAMP_ext_push(const uint32_t instr); +void SLAMP_ext_pop(); + +void SLAMP_push(const uint32_t instr); +void SLAMP_pop(); + +void SLAMP_load1(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_load2(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_load4(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_load8(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_loadn(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, size_t n) ATTRIBUTE(always_inline);; + +void SLAMP_load1_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_load2_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_load4_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_load8_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_loadn_ext(const uint64_t addr, const uint32_t bare_instr, size_t n) ATTRIBUTE(always_inline);; + +void SLAMP_store1(uint32_t instr, const uint64_t addr) ATTRIBUTE(always_inline); +void SLAMP_store2(uint32_t instr, const uint64_t addr) ATTRIBUTE(always_inline); +void SLAMP_store4(uint32_t instr, const uint64_t addr) ATTRIBUTE(always_inline); +void SLAMP_store8(uint32_t instr, const uint64_t addr) ATTRIBUTE(always_inline); +void SLAMP_storen(uint32_t instr, const uint64_t addr, size_t n) ATTRIBUTE(always_inline);; + +void SLAMP_store1_ext(const uint64_t addr, const uint32_t bare_inst) ATTRIBUTE(always_inline); +void SLAMP_store2_ext(const uint64_t addr, const uint32_t bare_inst) ATTRIBUTE(always_inline); +void SLAMP_store4_ext(const uint64_t addr, const uint32_t bare_inst) ATTRIBUTE(always_inline); +void SLAMP_store8_ext(const uint64_t addr, const uint32_t bare_inst) ATTRIBUTE(always_inline); +void SLAMP_storen_ext(const uint64_t addr, const uint32_t bare_inst, size_t n) ATTRIBUTE(always_inline);; + +/* wrappers */ +static void* SLAMP_malloc_hook(size_t size, const void *caller); +static void SLAMP_free_hook(void *ptr, const void *caller); +static void* SLAMP_memalign_hook(size_t alignment, size_t size, const void *caller); +void* SLAMP_malloc(size_t size, uint32_t instr=0, size_t alignment=16); + +void* SLAMP_calloc(size_t nelem, size_t elsize); +void* SLAMP_realloc(void* ptr, size_t size); +void* SLAMP__Znam(size_t size); +void* SLAMP__Znwm(size_t size); + +char* SLAMP_strdup(const char *s1); +char* SLAMP___strdup(const char *s1); +void SLAMP_free(void* ptr); +void SLAMP_cfree(void* ptr); +void SLAMP__ZdlPv(void* ptr); +void SLAMP__ZdaPv(void* ptr); +int SLAMP_brk(void *end_data_segment); +void* SLAMP_sbrk(intptr_t increment); + +/* llvm memory intrinsics */ +void SLAMP_llvm_memcpy_p0i8_p0i8_i32(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint32_t sizeBytes); +void SLAMP_llvm_memcpy_p0i8_p0i8_i64(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint64_t sizeBytes); +void SLAMP_llvm_memmove_p0i8_p0i8_i32(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint32_t sizeBytes); +void SLAMP_llvm_memmove_p0i8_p0i8_i64(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint64_t sizeBytes); +void SLAMP_llvm_memset_p0i8_i32(const uint8_t* dstAddr, const uint32_t len); +void SLAMP_llvm_memset_p0i8_i64(const uint8_t* dstAddr, const uint64_t len); + +// void SLAMP_llvm_lifetime_start_p0i8(uint64_t size, uint8_t* ptr); +// void SLAMP_llvm_lifetime_end_p0i8(uint64_t size, uint8_t* ptr); + +/* String functions */ +size_t SLAMP_strlen(const char *str); +char* SLAMP_strchr(char *s, int c); +char* SLAMP_strrchr(char *s, int c); +int SLAMP_strcmp(const char *s1, const char *s2); +int SLAMP_strncmp(const char *s1, const char *s2, size_t n); +char* SLAMP_strcpy(char *dest, const char *src); +char* SLAMP_strncpy(char *dest, const char *src, size_t n); +char* SLAMP_strcat(char *s1, const char *s2); +char* SLAMP_strncat(char *s1, const char *s2, size_t n); +char* SLAMP_strstr(char *s1, char *s2); +size_t SLAMP_strspn(const char *s1, const char *s2); +size_t SLAMP_strcspn(const char *s1, const char *s2); +char* SLAMP_strtok(char *s, const char *delim); +double SLAMP_strtod(const char *nptr, char **endptr); +long int SLAMP_strtol(const char *nptr, char **endptr, int base); +char* SLAMP_strpbrk(char *s1, char *s2); + +/* Mem* and b* functions */ +void *SLAMP_memset (void *dest, int c, size_t n); +void *SLAMP_memcpy (void *dest, const void *src, size_t n); +void *SLAMP___builtin_memcpy (void *dest, const void *src, size_t n); +void *SLAMP_memmove (void *dest, const void *src, size_t n); +int SLAMP_memcmp(const void *s1, const void *s2, size_t n); +void* SLAMP_memchr(void* ptr, int value, size_t num); +void* SLAMP___rawmemchr(void* ptr, int value); + +void SLAMP_bzero(void *s, size_t n); +void SLAMP_bcopy(const void *s1, void *s2, size_t n); + +/* IO */ +ssize_t SLAMP_read(int fd, void *buf, size_t count); +int SLAMP_open(const char *pathname, int flags, mode_t mode); +int SLAMP_close(int fd); +ssize_t SLAMP_write(int fd, const void *buf, size_t count); +off_t SLAMP_lseek(int fildes, off_t offset, int whence); + +FILE * SLAMP_fopen(const char *path, const char *mode); +FILE * SLAMP_fopen64(const char *path, const char *mode); +FILE * SLAMP_freopen(const char *path, const char *mode, FILE* stream); +int SLAMP_fflush(FILE *stream); +int SLAMP_fclose(FILE *stream); +int SLAMP_ferror(FILE *stream); +int SLAMP_feof(FILE *stream); +long SLAMP_ftell(FILE *stream); +size_t SLAMP_fread(void * ptr, size_t size, size_t nitems, FILE *stream); +size_t SLAMP_fwrite(const void *ptr, size_t size, size_t nitems, FILE *stream); +int SLAMP_fseek(FILE *stream, long offset, int whence); +void SLAMP_rewind(FILE *stream); + +int SLAMP_fgetc(FILE *stream); +int SLAMP_fputc(int c, FILE *stream); +char * SLAMP_fgets(char *s, int n, FILE *stream); +int SLAMP_fputs(const char *s, FILE *stream); + +int SLAMP_ungetc(int c, FILE *stream); +int SLAMP_putchar(int c); +int SLAMP_getchar(void); + +int SLAMP_fileno(FILE *stream); +char * SLAMP_gets(char *s); +int SLAMP_puts(const char *s); + +int SLAMP_select(int n, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, struct timeval *timeout); +int SLAMP_remove(const char *path); + +void SLAMP_setbuf(FILE * stream, char * buf); +void SLAMP_setvbuf(FILE * stream, char * buf, int mode, size_t size); +char * SLAMP_tmpnam(char *s); +FILE* SLAMP_tmpfile(void); +char * SLAMP_ttyname(int fildes); + +FILE * SLAMP_fdopen(int fildes, const char *mode); +void SLAMP_clearerr(FILE *stream); + +int SLAMP_truncate(const char *path, off_t length); +int SLAMP_ftruncate(int fildes, off_t length); + +int SLAMP_dup(int oldfd); +int SLAMP_dup2(int oldfd, int newfd); +int SLAMP_pipe(int filedes[2]); + +int SLAMP_chmod(const char *path, mode_t mode); +int SLAMP_fchmod(int fildes, mode_t mode); +int SLAMP_fchown(int fd, uid_t owner, gid_t group); +int SLAMP_access(const char *pathname, int mode); +long SLAMP_pathconf(char *path, int name); +int SLAMP_mkdir(const char *pathname, mode_t mode); +int SLAMP_rmdir(const char *pathname); +mode_t SLAMP_umask(mode_t mask); +int SLAMP_fcntl(int fd, int cmd, struct flock *lock); + +DIR* SLAMP_opendir(const char* name); +struct dirent* SLAMP_readdir(DIR *dirp); +struct dirent64* SLAMP_readdir64(DIR *dirp); +int SLAMP_closedir(DIR* dirp); + +/* Printf */ +int SLAMP_printf(const char *format, ...); +int SLAMP_fprintf(FILE *stream, const char *format, ...); +int SLAMP_sprintf(char *str, const char *format, ...); +int SLAMP_snprintf(char *str, size_t size, const char *format, ...); + +int SLAMP_vprintf(const char *format, va_list ap); +int SLAMP_vfprintf(FILE *stream, const char *format, va_list ap); +int SLAMP_vsprintf(char *str, const char *format, va_list ap); +int SLAMP_vsnprintf(char *str, size_t size, const char *format, va_list ap); + +/* Scanf */ +int SLAMP_fscanf(FILE *stream, const char *format, ... ); +int SLAMP_scanf(const char *format, ... ); +int SLAMP_sscanf(const char *s, const char *format, ... ); +int SLAMP___isoc99_sscanf(const char *s, const char *format, ... ); + +int SLAMP_vfscanf(FILE *stream, const char *format, va_list ap); +int SLAMP_vscanf(const char *format, va_list ap); +int SLAMP_vsscanf(const char *s, const char *format, va_list ap); + +/* Time */ +time_t SLAMP_time(time_t *t); +struct tm *SLAMP_localtime(const time_t *timer); +struct lconv* SLAMP_localeconv(); +struct tm *SLAMP_gmtime(const time_t *timer); +int SLAMP_gettimeofday(struct timeval *tv, struct timezone *tz); + +/* Math */ +double SLAMP_ldexp(double x, int exp); +float SLAMP_ldexpf(float x, int exp); +long double SLAMP_ldexpl(long double x, int exp); +double SLAMP_log10(double x); +float SLAMP_log10f(float x); +long double SLAMP_log10l(long double x); +double SLAMP_log(double x); +float SLAMP_logf(float x); +long double SLAMP_logl(long double x); + +double SLAMP_exp(double x); +float SLAMP_expf(float x); +long double SLAMP_expl(long double x); + +double SLAMP_cos(double x); +float SLAMP_cosf(float x); +long double SLAMP_cosl(long double x); +double SLAMP_sin(double x); +double SLAMP_tan(double x); +float SLAMP_sinf(float x); +long double SLAMP_sinl(long double x); + +double SLAMP_atan(double x); +float SLAMP_atanf(float x); +long double SLAMP_atanl(long double x); + +double SLAMP_floor(double x); +float SLAMP_floorf(float x); +long double SLAMP_floorl(long double x); +double SLAMP_ceil(double x); +float SLAMP_ceilf(float x); +long double SLAMP_ceill(long double x); + +double SLAMP_atan2(double y, double x); +float SLAMP_atan2f(float y, float x); +long double SLAMP_atan2l(long double y, long double x); + +double SLAMP_sqrt(double x); +float SLAMP_sqrtf(float x); +long double SLAMP_sqrtl(long double x); + +double SLAMP_pow(double x, double y); +float SLAMP_powf(float x, float y); +long double SLAMP_powl(long double x, long double y); + +double SLAMP_fabs(double x); +float SLAMP_fabsf(float x); +long double SLAMP_fabsl(long double x); + +double SLAMP_modf(double x, double *iptr); +float SLAMP_modff(float x, float *iptr); +long double SLAMP_modfl(long double x, long double *iptr); + +double SLAMP_fmod(double x, double y); + +double SLAMP_frexp(double num, int *exp); +float SLAMP_frexpf(float num, int *exp); +long double SLAMP_frexpl(long double num, int *exp); + +int SLAMP_isnan(); + +/* MISC */ +char *SLAMP_getenv(const char *name); +int SLAMP_putenv(char* string); +char *SLAMP_getcwd(char *buf, size_t size); +char* SLAMP_strerror(int errnum); +void SLAMP_exit(int status); +void SLAMP__exit(int status); +int SLAMP_link(const char *oldpath, const char *newpath); +int SLAMP_unlink(const char *pathname); +int SLAMP_isatty(int desc); +int SLAMP_setuid(uid_t uid); +uid_t SLAMP_getuid(void); +uid_t SLAMP_geteuid(void); +int SLAMP_setgid(gid_t gid); +gid_t SLAMP_getgid(void); +gid_t SLAMP_getegid(void); +pid_t SLAMP_getpid(void); +int SLAMP_chdir(const char *path); +int SLAMP_execl(const char *path, const char *arg0, ... /*, (char *)0 */); +int SLAMP_execv(const char *path, char *const argv[]); +int SLAMP_execvp(const char *file, char *const argv[]); +int SLAMP_kill(pid_t pid, int sig); +pid_t SLAMP_fork(void); +sighandler_t SLAMP___sysv_signal(int signum, sighandler_t handler); +pid_t SLAMP_waitpid(pid_t pid, int* status, int options); +void SLAMP_qsort(void* base, size_t nmemb, size_t size, int(*compar)(const void *, const void *)); +int SLAMP_ioctl(int d, int request, ...); +unsigned int SLAMP_sleep(unsigned int seconds); +char* SLAMP_gcvt(double number, size_t ndigit, char* buf); +char* SLAMP_nl_langinfo(nl_item item); + +/* Compiler/Glibc Internals */ +void SLAMP___assert_fail(const char * assertion, const char * file, unsigned int line, const char * function); +const unsigned short int **SLAMP___ctype_b_loc(void); +int SLAMP__IO_getc(_IO_FILE * __fp); +int SLAMP__IO_putc(int __c, _IO_FILE *__fp); +int* SLAMP___errno_location (void); + +int SLAMP___fxstat (int __ver, int __fildes, struct stat *__stat_buf); +int SLAMP___xstat (int __ver, __const char *__filename, struct stat *__stat_buf); + +#ifdef __cplusplus +} +#endif + +#endif /* SLAMP_HOOKS_H */ diff --git a/liberty/lib/SLAMP/SLAMPsmtxq/sw_queue.c b/liberty/lib/SLAMP/SLAMPsmtxq/sw_queue.c new file mode 100644 index 00000000..33199929 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPsmtxq/sw_queue.c @@ -0,0 +1,103 @@ +#include +#include + +#include "sw_queue_astream.h" + +/* + * Create and initialize a queue. + */ +SW_Queue sq_createQueue(void) +{ + return sq_initQueue(sq_createQueueBlock(1)); +} + +/* + * Creates, but does not initialize a block of queues. + */ +SW_Queue sq_createQueueBlock(unsigned queues) { + + /* Allocate the queue data structure */ + sw_queue_t *queue = (sw_queue_t *) mmap((void *) (1UL << 32), + queues * sizeof(sw_queue_t), + PROT_WRITE | PROT_READ, + MAP_SHARED | MAP_ANONYMOUS, + -1, + (off_t) 0); + if(queue == (sw_queue_t *) -1) { + perror("sq_createQueue"); + exit(1); + } + + return queue; +} + +/* + * Initialize a queue. sq_createQueue does this automatically. + */ +SW_Queue sq_initQueue(SW_Queue queue) { + + /* Allocate the queue */ + queue->data = (uint64_t *) mmap(0, + sizeof(uint64_t) * QSIZE, + PROT_WRITE | PROT_READ, + MAP_SHARED | MAP_ANONYMOUS | MAP_32BIT, + -1, + (off_t) 0); + + if(queue->data == (void *) -1) { + perror("sq_initQueue"); + exit(1); + } + + /* Initialize the queue data structure */ + queue->p_data = (uint64_t) queue->data; + queue->c_inx = 0; + queue->c_margin = 0; + queue->p_glb_inx = 0; + queue->c_glb_inx = 0; + queue->ptr_c_glb_inx = &(queue->c_glb_inx); + queue->ptr_p_glb_inx = &(queue->p_glb_inx); + +#ifdef INSTRUMENT + queue->total_produces = 0; + queue->total_consumes = 0; +#endif + + return queue; +} + +void sq_freeQueueBlock(SW_Queue queue, unsigned size) { + + for(unsigned i = 0; i < size; ++i) { + + /* This queue may never have been initialized. */ + if(queue[i].data) { + +#ifdef INSTRUMENT + fprintf(stderr, "Queue %p: %ld produces, %ld consumes\n", + (void*)queue, + queue->total_produces, + queue->total_consumes); +#endif + + if(munmap(queue[i].data, sizeof(uint64_t) * QSIZE)) { + perror("sq_freeQueueBlock"); + exit(-1); + } + } + } + + if(munmap(queue, sizeof(sw_queue_t) * size)) { + perror("sq_freeQueueBlock"); + exit(-1); + } +} + +/* + * Free a queue. + */ +void sq_freeQueue(SW_Queue queue) +{ + sq_freeQueueBlock(queue, 1); +} + diff --git a/liberty/lib/SLAMP/SLAMPsmtxq/sw_queue_astream.h b/liberty/lib/SLAMP/SLAMPsmtxq/sw_queue_astream.h new file mode 100644 index 00000000..59ea0182 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPsmtxq/sw_queue_astream.h @@ -0,0 +1,383 @@ +/** ***********************************************/ +/** *** SW Queue with Supporting Variable Size ****/ +/** ***********************************************/ +#ifndef SW_QUEUE_H +#define SW_QUEUE_H + +#include +#include +#include +#include +#include + +#include "inline.h" +#include "bitcast.h" +#ifndef CACHELINE_SIZE +#define CACHELINE_SIZE 64 +/*#define CACHELINE_SIZE 64 */ +/* Cache line size of glacier, as reported by lmbench. Other tests, + * namely the smtx ones, vindicate this. */ +#endif /* CACHELINE_SIZE */ + +#ifndef CHUNK_SIZE +#ifdef DUALCORE +#define CHUNK_SIZE (1 << 8) +#else /* DUALCORE */ +#define CHUNK_SIZE (1 << 14) +#endif /* DUALCORE */ +#endif /* CHUNK_SIZE */ + +#define HIGH_CHUNKMASK ((((uint64_t) (CHUNK_SIZE - 1)) << 32)) + +#ifndef STREAM +#ifdef DUALCORE +#define STREAM (false) +#else /* DUALCORE */ +#define STREAM (true) +#endif /* DUALCORE */ +#endif /* STREAM */ + +#if !STREAM +#define sq_write sq_stdWrite +#else /* STREAM */ +#define sq_write sq_streamWrite +#endif /* STREAM */ + +#ifndef QMARGIN +#define QMARGIN 4 +#endif /* QMARGIN */ + +#ifndef QSIZE +#define QSIZE (1 << 21) +#endif /* QSIZE */ + +#ifndef QPREFETCH +#define QPREFETCH (1 << 7) +#endif /* QPREFETCH */ + +#define QMASK (QSIZE - 1) +#define HIGH_QMASK (((uint64_t) QMASK) << 32 | (uint32_t) ~0) + +#define PAD(suffix, size) char padding ## suffix [CACHELINE_SIZE - (size)] + +#ifdef __cplusplus +extern "C" { +#endif +typedef struct { + + uint64_t p_data; + PAD(1, sizeof(uint64_t)); + + volatile uint32_t *ptr_c_glb_inx; + uint32_t p_glb_inx; + PAD(2, sizeof(volatile uint32_t *) + sizeof(uint32_t)); + + uint32_t c_margin; + uint32_t c_inx; + PAD(3, sizeof(uint32_t) * 2); + + volatile uint32_t *ptr_p_glb_inx; + uint32_t c_glb_inx; + PAD(4, sizeof(volatile uint32_t *) + sizeof(uint32_t)); + + uint64_t *data; + + uint64_t total_produces; + uint64_t total_consumes; + + PAD(5, sizeof(volatile uint64_t *) + sizeof(uint64_t) * 2); + +} sw_queue_t, *SW_Queue; + +typedef void *sq_cbData; +typedef void (*sq_callback)(sq_cbData); + +/* ***************************************** */ +/* ******* Detail Implementation *********** */ +/* ***************************************** */ + +SW_Queue sq_createQueue(void); +SW_Queue sq_createQueueBlock(unsigned queues); +SW_Queue sq_initQueue(SW_Queue queue); +void sq_freeQueueBlock(SW_Queue queue, unsigned size); +void sq_freeQueue(SW_Queue q); + +/* ******************************************************** */ +/* *********** Preallocate Method *********************** */ +/* ******************************************************** */ + +/* + * This method is faster on multi-processor machines as it bypasses the L2 + * cache. + */ +Inline void sq_streamWrite(uint64_t *addr, uint64_t value) { + + /* __m64 mmxReg = _mm_set_pi64x((int64_t) value); */ + /* _mm_stream_pi((__m64 *) addr, mmxReg); */ + + __asm ( + "movntiq %1, (%0)\n" + : + : "r" (addr), "r" (value) + ); +} + +/* + * This method is faster on multi-core machines as it exploits the common L2 + * cache. + */ +Inline void sq_stdWrite(uint64_t *addr, uint64_t value) { + *addr = value; +} + +/* + * Modulo subtraction + */ +Inline uint32_t sq_modSub(uint32_t minuend, + uint32_t subtrahend, + uint32_t mask) { + return (minuend - subtrahend) & mask; +} + +Inline uint32_t sq_pInx(SW_Queue q) { + return (uint32_t) (q->p_data >> 32); +} + +/* + * Flush all produce values. + * + * sfence is slow, but necessary for streaming writes. sfence MUST proceded + * updating p_glb_inx, otherwise there will be a race condition. + */ +Inline void sq_flushQueue(SW_Queue q) +{ +#if STREAM + _mm_sfence(); +#endif + + *q->ptr_p_glb_inx = sq_pInx(q); +} + +/* + * Wait on the consumer. + * + * Pausing while spinning improves performance on NetBurst and improves energy + * efficiency. + */ +Inline void sq_waitConsumer(SW_Queue q, sq_callback cb, sq_cbData cbd) +{ +#ifndef NO_CON + uint32_t threshold = (QMARGIN - 1) * QSIZE / QMARGIN; + if(sq_modSub(sq_pInx(q), *q->ptr_c_glb_inx, QMASK) > threshold) { + + /* Blocking path */ + cb(cbd); + + while(sq_modSub(sq_pInx(q), *q->ptr_c_glb_inx, QMASK) > threshold) { + usleep(10); + } + } else { + + /* Fast path */ + sq_flushQueue(q); + } +#endif /* NO_CON */ +} + +/* + * Produce a value to a queue. + */ +Inline void sq_produce(SW_Queue q, uint64_t value, + sq_callback cb, sq_cbData cbd) +{ +#ifdef INSTRUMENT + q->total_produces++; +#endif + + uint64_t pDataRaw = q->p_data; + uint32_t pInx = sq_pInx(q); + uint64_t *pData = (uint64_t *) (size_t) (uint32_t) q->p_data; + uint64_t *ptr = pInx + pData; + + sq_write(ptr, value); + q->p_data = (pDataRaw + (1ULL << 32)) & HIGH_QMASK; + + if(!(q->p_data & HIGH_CHUNKMASK)) { + sq_waitConsumer(q, cb, cbd); + } +} + +#define sq_produce(Q,V) sq_produce(Q, V, (sq_callback) sq_flushQueue, Q) + +/* + * Produce two values to a queue. Do not intermingle with sq_produce. + */ +Inline void sq_produce2(SW_Queue q, uint64_t a, uint64_t b, + sq_callback cb, sq_cbData cbd) +{ +#ifdef INSTRUMENT + q->total_produces += 2; +#endif + + /* Otherwise, gcc couldn't constant propagate. */ + uint64_t pDataRaw = q->p_data; + uint32_t pInx = sq_pInx(q); + uint64_t *pData = (uint64_t *) (size_t) (uint32_t) q->p_data; + uint64_t *ptr = pInx + pData; + + sq_write(ptr + 0, a); + sq_write(ptr + 1, b); + + q->p_data = (pDataRaw + (2ULL << 32)) & HIGH_QMASK; + + if(!(q->p_data & HIGH_CHUNKMASK)) { + sq_waitConsumer(q, cb, cbd); + } +} + +#define sq_produce2(Q,A,B) sq_produce2(Q, A, B, (sq_callback) sq_flushQueue, Q) + + +Inline void sq_produceDouble(SW_Queue q, double value, sq_callback cb, sq_cbData cbd) { + uint64_t pDataRaw = q->p_data; + uint32_t pInx = sq_pInx(q); + uint64_t *pData = (uint64_t *) (size_t) (uint32_t) q->p_data; + uint64_t *ptr = pInx + pData; + uint64_t valueInt = (uint64_t)doubleToInt(value); + sq_write(ptr, valueInt); + q->p_data = (pDataRaw + (1ULL << 32)) & HIGH_QMASK; + + if(!(q->p_data & HIGH_CHUNKMASK)) { + sq_waitConsumer(q, cb, cbd); + } +} + +#define sq_produceDouble(Q, V) sq_produceDouble(Q, V, (sq_callback) sq_flushQueue, Q) + +/* Functions for Consumer */ + +/* + * Makes reads globally visible + */ + +Inline void sq_reverseFlush(const SW_Queue q) { + *q->ptr_c_glb_inx = q->c_inx; +} + +/* + * Wait for a produce + */ +Inline uint32_t sq_waitAllocated(const SW_Queue q, sq_callback cb, sq_cbData cbd) +{ + if(*q->ptr_p_glb_inx == q->c_inx) { + + /* Blocking path */ + cb(cbd); + while(*q->ptr_p_glb_inx == q->c_inx) usleep(10); + + } else { + /* Fast path */ + sq_reverseFlush(q); + } + + return *q->ptr_p_glb_inx; +} + +/* + * Consume a 64-bit value from the queue. + */ +Inline uint64_t sq_consume(SW_Queue q, sq_callback cb, sq_cbData cbd) +{ +#ifdef INSTRUMENT + q->total_consumes++; +#endif + + if(q->c_inx == q->c_margin) { + q->c_margin = sq_waitAllocated(q, cb, cbd); + } + + uint64_t val = q->data[q->c_inx]; + + if(QPREFETCH) { + _mm_prefetch(q->data + q->c_inx + QPREFETCH, _MM_HINT_T0); + } + + q->c_inx++; + q->c_inx &= QMASK; + return val; +} + +#define sq_consume(Q) sq_consume(Q, (sq_callback) sq_reverseFlush, Q) + + +Inline double sq_consumeDouble(SW_Queue q, sq_callback cb, sq_cbData cbd) +{ +#ifdef INSTRUMENT + q->total_consumes++; +#endif + + if(q->c_inx == q->c_margin) { + q->c_margin = sq_waitAllocated(q, cb, cbd); + } + + uint64_t val = q->data[q->c_inx]; + + if(QPREFETCH) { + _mm_prefetch(q->data + q->c_inx + QPREFETCH, _MM_HINT_T0); + } + + q->c_inx++; + q->c_inx &= QMASK; + double valDouble = intToDouble((int64_t)val); + return valDouble; +} + +#define sq_consumeDouble(Q) sq_consumeDouble(Q, (sq_callback) sq_reverseFlush, Q) + +Inline bool sq_canConsume(const SW_Queue q) { + return q->c_margin != q->c_inx || *q->ptr_p_glb_inx != q->c_inx; +} + +/* + * Empty the queue. + * + * Races with producer methods. + */ +Inline void sq_emptyQueue(SW_Queue q) { + q->c_margin = q->c_inx = *q->ptr_p_glb_inx; + sq_reverseFlush(q); +} + +/* + * Modulo arithmetic (llvmism) + */ +Inline bool sq_selectProducer(unsigned prevChoice, + SW_Queue *queues, + unsigned numQueues) { + (void) queues; + + if( ++prevChoice >= numQueues ) + prevChoice = 0; + + return prevChoice; +} + +/* + * Modulo arithmetic (llvmism) + */ +Inline bool sq_selectConsumer(unsigned prevChoice, + SW_Queue *queues, + unsigned numQueues) { + + (void) queues; + + if( ++prevChoice >= numQueues ) + prevChoice = 0; + + return prevChoice; +} + +#ifdef __cplusplus +} +#endif +#endif /* SW_QUEUE_H */ From 4d1ab5caba84d67c10cb6a869932980326a1cffb Mon Sep 17 00:00:00 2001 From: Ziyang Xu Date: Thu, 13 Oct 2022 23:37:48 -0400 Subject: [PATCH 21/97] parallelize log; use smtx queue --- liberty/lib/SLAMP/SLAMPsmtxq/SmtxSend.cpp | 13 ++++- .../ProfilingModules/DependenceModule.cpp | 55 ++++++++++++++++++- .../SLAMP/SLAMPsmtxq/consumer/consumer.cpp | 15 +++-- .../lib/SLAMP/SLAMPsmtxq/sw_queue_astream.h | 2 + 4 files changed, 77 insertions(+), 8 deletions(-) diff --git a/liberty/lib/SLAMP/SLAMPsmtxq/SmtxSend.cpp b/liberty/lib/SLAMP/SLAMPsmtxq/SmtxSend.cpp index 391d9a9d..c434a846 100644 --- a/liberty/lib/SLAMP/SLAMPsmtxq/SmtxSend.cpp +++ b/liberty/lib/SLAMP/SLAMPsmtxq/SmtxSend.cpp @@ -1,4 +1,7 @@ #include "slamp_hooks.h" +#include +#include +#include #include #include #include "malloc.h" @@ -15,7 +18,8 @@ static void *(*old_malloc_hook)(size_t, const void *); static void (*old_free_hook)(void *, const void *); static void *(*old_memalign_hook)(size_t, size_t, const void *); // create segment and corresponding allocator -bip::managed_shared_memory *segment; +bip::fixed_managed_shared_memory *segment; +bip::fixed_managed_shared_memory *segment2; static SW_Queue the_queue; #define CONSUME sq_consume(the_queue); @@ -48,10 +52,12 @@ enum DepModAction: char void SLAMP_init(uint32_t fn_id, uint32_t loop_id) { - segment = new bip::managed_shared_memory(bip::open_or_create, "MySharedMemory", sizeof(uint64_t) *QSIZE *2); + segment = new bip::fixed_managed_shared_memory(bip::open_or_create, "MySharedMemory", sizeof(uint64_t) *QSIZE *2, (void*)(1UL << 32)); + segment2 = new bip::fixed_managed_shared_memory(bip::open_or_create, "MySharedMemory2", sizeof(uint64_t) *QSIZE *2, (void*)(1UL << 28)); + // managed_shared_memory(bip::open_or_create, "MySharedMemory", sizeof(uint64_t) *QSIZE *2); // auto a_queue = new atomic_queue::AtomicQueueB(65536); the_queue = static_cast(segment->find_or_construct("MyQueue")()); - auto data = static_cast(segment->find_or_construct("smtx_queue_data")[QSIZE]()); + auto data = static_cast(segment2->find_or_construct("smtx_queue_data")[QSIZE]()); if (the_queue == nullptr) { std::cout << "Error: could not create queue" << std::endl; exit(-1); @@ -103,6 +109,7 @@ void SLAMP_fini(const char* filename){ // local_buffer->push(FINISHED); // local_buffer->flush(); PRODUCE(FINISHED); + sq_flushQueue(the_queue); } void SLAMP_allocated(uint64_t addr){} diff --git a/liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/DependenceModule.cpp b/liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/DependenceModule.cpp index a78b3487..bc6b27cf 100644 --- a/liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/DependenceModule.cpp +++ b/liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/DependenceModule.cpp @@ -1,5 +1,7 @@ #include #include +#include +#include #include #include @@ -7,15 +9,56 @@ #include "slamp_logger.h" #include "slamp_shadow_mem.h" #include "DependenceModule.h" +#include +std::mutex m; + #define SIZE_8M 0x800000 +#define DEPLOG_VEC_SIZE 100000000 static slamp::MemoryMap* smmap = nullptr; static std::unordered_set *deplog_set; +static std::vector *deplog_vec; +static unsigned long deplog_vec_counter = 0; static uint64_t slamp_iteration = 0; static uint64_t slamp_invocation = 0; static uint32_t target_loop_id = 0; +static void convertVectorToSet() { + + // launch 56 threads to convert the vector to set independently, chunking + constexpr auto THREADS = 56; + std::thread t[THREADS]; + for (unsigned long i = 0; i < THREADS; i++) { + t[i] = std::thread( + [&](int id) { + m.lock(); + // take the chunk and convert to a set and return + auto *deplog_set_chunk = + new std::unordered_set(); + deplog_set_chunk->reserve(DEPLOG_VEC_SIZE / THREADS + 1); + auto begin = id * (DEPLOG_VEC_SIZE / THREADS); + auto end = (id + 1) * (DEPLOG_VEC_SIZE / THREADS); + deplog_set_chunk->insert(deplog_vec->begin() + begin, + deplog_vec->begin() + end); + + // lock the global set and insert the chunk + deplog_set->insert(deplog_set_chunk->begin(), + deplog_set_chunk->end()); + m.unlock(); + delete deplog_set_chunk; + }, + i); + } + // join the threads + for (auto &i : t) { + i.join(); + } + + std::cout << "Merging vec to set, set length " << deplog_set->size() + << std::endl; +} namespace DepMod { // init: setup the shadow memory void init(uint32_t loop_id, uint32_t pid) { @@ -23,6 +66,8 @@ void init(uint32_t loop_id, uint32_t pid) { target_loop_id = loop_id; smmap = new slamp::MemoryMap(TIMESTAMP_SIZE_IN_BYTES); deplog_set = new std::unordered_set(); + deplog_vec = new std::vector(); + deplog_vec->reserve(DEPLOG_VEC_SIZE); smmap->init_stack(SIZE_8M, pid); smmap->allocate((void*)&errno, sizeof(errno)); @@ -46,6 +91,7 @@ void init(uint32_t loop_id, uint32_t pid) { } void fini(const char *filename) { + convertVectorToSet(); std::ofstream of(filename); of << target_loop_id << " " << 0 << " " << 0 << " " << 0 << " " << 0 << " " << 0 << "\n"; @@ -66,6 +112,7 @@ void allocate(void *addr, uint64_t size) { // std::cout << "allocate " << addr << " " << size << std::endl; } + // void log(TS ts, const uint32_t dst_inst, TS *pts, const uint32_t bare_inst, // uint64_t addr, uint64_t value, uint8_t size) { void log(TS ts, const uint32_t dst_inst, const uint32_t bare_inst){ @@ -78,7 +125,13 @@ void log(TS ts, const uint32_t dst_inst, const uint32_t bare_inst){ slamp::KEY key(src_inst, dst_inst, bare_inst, src_iter != slamp_iteration); // std::cout << "src_inst: " << src_inst << " dst_inst: " << dst_inst << " bare_inst: " << bare_inst << " src_iter: " << src_iter << " slamp_iteration: " << slamp_iteration << " src_iter != slamp_iteration: " << (src_iter != slamp_iteration) << std::endl; - // deplog_set->insert(key); + deplog_vec->emplace_back(key); + deplog_vec_counter++; + if (deplog_vec_counter == DEPLOG_VEC_SIZE - 1) { + convertVectorToSet(); + deplog_vec->resize(0); + deplog_vec_counter = 0; + } } // template diff --git a/liberty/lib/SLAMP/SLAMPsmtxq/consumer/consumer.cpp b/liberty/lib/SLAMP/SLAMPsmtxq/consumer/consumer.cpp index eb679486..bec6ec2d 100644 --- a/liberty/lib/SLAMP/SLAMPsmtxq/consumer/consumer.cpp +++ b/liberty/lib/SLAMP/SLAMPsmtxq/consumer/consumer.cpp @@ -1,3 +1,4 @@ +#include #include #include "ProfilingModules/DependenceModule.h" #include @@ -11,10 +12,10 @@ namespace bip = boost::interprocess; #define DEBUG 0 -#define ACTION 0 +#define ACTION 1 // create segment and corresponding allocator -bip::managed_shared_memory *segment; +bip::fixed_managed_shared_memory *segment, *segment2; static SW_Queue the_queue; #define CONSUME sq_consume(the_queue); @@ -24,10 +25,16 @@ static SW_Queue the_queue; #define CONSUME_2(x,y) do { uint64_t tmp = CONSUME; x = (uint32_t)(tmp>>32); y = (uint32_t) tmp; } while(0) int main() { - segment = new bip::managed_shared_memory(bip::open_or_create, "MySharedMemory", sizeof(uint64_t) *QSIZE *2); + segment = new bip::fixed_managed_shared_memory(bip::open_or_create, "MySharedMemory", sizeof(uint64_t) *QSIZE *2, (void*)(1UL << 32)); + segment2 = new bip::fixed_managed_shared_memory(bip::open_or_create, "MySharedMemory2", sizeof(uint64_t) *QSIZE *2, (void*)(1UL << 28)); + // managed_shared_memory(bip::open_or_create, "MySharedMemory", sizeof(uint64_t) *QSIZE *2); // auto a_queue = new atomic_queue::AtomicQueueB(65536); the_queue = static_cast(segment->find_or_construct("MyQueue")()); - auto data = static_cast(segment->find_or_construct("smtx_queue_data")[QSIZE]()); + auto data = static_cast(segment2->find_or_construct("smtx_queue_data")[QSIZE]()); + // segment = new bip::managed_shared_memory(bip::open_or_create, "MySharedMemory", sizeof(uint64_t) *QSIZE *2); + // auto a_queue = new atomic_queue::AtomicQueueB(65536); + // the_queue = static_cast(segment->find_or_construct("MyQueue")()); + // auto data = static_cast(segment->find_or_construct("smtx_queue_data")[QSIZE]()); if (the_queue == nullptr) { std::cout << "Error: could not create queue" << std::endl; return 1; diff --git a/liberty/lib/SLAMP/SLAMPsmtxq/sw_queue_astream.h b/liberty/lib/SLAMP/SLAMPsmtxq/sw_queue_astream.h index 59ea0182..8fd981e4 100644 --- a/liberty/lib/SLAMP/SLAMPsmtxq/sw_queue_astream.h +++ b/liberty/lib/SLAMP/SLAMPsmtxq/sw_queue_astream.h @@ -4,6 +4,8 @@ #ifndef SW_QUEUE_H #define SW_QUEUE_H +#define DUALCORE + #include #include #include From 270316920de36c1f4bc87b848d8520ba560d7a20 Mon Sep 17 00:00:00 2001 From: Ziyang Xu Date: Fri, 14 Oct 2022 14:41:49 -0400 Subject: [PATCH 22/97] fix allocation; mcf working --- liberty/lib/SLAMP/SLAMPsmtxq/SmtxSend.cpp | 46 +++++++++++++++---- .../SLAMP/SLAMPsmtxq/consumer/consumer.cpp | 6 +-- liberty/lib/SLAMP/SLAMPsmtxq/slamp_hooks.h | 1 + 3 files changed, 40 insertions(+), 13 deletions(-) diff --git a/liberty/lib/SLAMP/SLAMPsmtxq/SmtxSend.cpp b/liberty/lib/SLAMP/SLAMPsmtxq/SmtxSend.cpp index c434a846..6d9f545f 100644 --- a/liberty/lib/SLAMP/SLAMPsmtxq/SmtxSend.cpp +++ b/liberty/lib/SLAMP/SLAMPsmtxq/SmtxSend.cpp @@ -15,6 +15,7 @@ namespace bip = boost::interprocess; static void *(*old_malloc_hook)(size_t, const void *); +static void *(*old_realloc_hook)(void *, size_t, const void *); static void (*old_free_hook)(void *, const void *); static void *(*old_memalign_hook)(size_t, size_t, const void *); // create segment and corresponding allocator @@ -28,6 +29,17 @@ static SW_Queue the_queue; #define PRODUCE_2(x,y) PRODUCE( (((uint64_t)x)<<32) | (uint32_t)(y) ) #define CONSUME_2(x,y) do { uint64_t tmp = CONSUME; x = (uint32_t)(tmp>>32); y = (uint32_t) tmp; } while(0) +#define TURN_ON_HOOKS \ + __malloc_hook = SLAMP_malloc_hook;\ + __realloc_hook = SLAMP_realloc_hook;\ + __memalign_hook = SLAMP_memalign_hook; + + +#define TURN_OFF_HOOKS \ + __malloc_hook = old_malloc_hook; \ + __realloc_hook = old_realloc_hook; \ + __memalign_hook = old_memalign_hook; + // Ringbuffer fully constructed in shared memory. The element strings are // also allocated from the same shared memory segment. This vector can be // safely accessed from other processes. @@ -95,9 +107,10 @@ void SLAMP_init(uint32_t fn_id, uint32_t loop_id) { old_malloc_hook = __malloc_hook; // old_free_hook = __free_hook; old_memalign_hook = __memalign_hook; + old_realloc_hook = __realloc_hook; __malloc_hook = SLAMP_malloc_hook; __free_hook = nullptr; - __realloc_hook = nullptr; + __realloc_hook = SLAMP_realloc_hook; // __free_hook = SLAMP_free_hook; __memalign_hook = SLAMP_memalign_hook; } @@ -164,9 +177,9 @@ void SLAMP_load(const uint32_t instr, const uint64_t addr, const uint32_t bare_i if (onProfiling) { // local_buffer->push(LOAD)->push(instr)->push(addr)->push(bare_instr); //->push(value); PRODUCE(LOAD); - PRODUCE(instr); + // PRODUCE(instr); + PRODUCE_2(instr, bare_instr); PRODUCE(addr); - PRODUCE(bare_instr); // PRODUCE(value); counter_load++; } @@ -214,8 +227,9 @@ void SLAMP_store(const uint32_t instr, const uint64_t addr, const uint32_t bare_ if (onProfiling) { // local_buffer->push(STORE)->push(instr)->push(bare_instr)->push(addr); PRODUCE(STORE); - PRODUCE(instr); - PRODUCE(bare_instr); + // PRODUCE(instr); + // PRODUCE(bare_instr); + PRODUCE_2(instr, bare_instr); PRODUCE(addr); counter_store++; } @@ -256,7 +270,7 @@ void SLAMP_storen_ext(const uint64_t addr, const uint32_t bare_inst, size_t n){ /* wrappers */ static void* SLAMP_malloc_hook(size_t size, const void *caller){ - __malloc_hook = old_malloc_hook; + TURN_OFF_HOOKS void* ptr = malloc(size); // local_buffer->push(ALLOC)->push((uint64_t)ptr)->push(size); PRODUCE(ALLOC); @@ -264,14 +278,28 @@ static void* SLAMP_malloc_hook(size_t size, const void *caller){ PRODUCE(size); // printf("malloc %lu at %p\n", size, ptr); counter_alloc++; - __malloc_hook = SLAMP_malloc_hook; + TURN_ON_HOOKS return ptr; } + +static void* SLAMP_realloc_hook(void* ptr, size_t size, const void *caller){ + TURN_OFF_HOOKS + void* new_ptr = realloc(ptr, size); + // local_buffer->push(REALLOC)->push((uint64_t)ptr)->push((uint64_t)new_ptr)->push(size); + PRODUCE(ALLOC); + PRODUCE((uint64_t)new_ptr); + PRODUCE(size); + // printf("realloc %p to %lu at %p", ptr, size, new_ptr); + counter_alloc++; + TURN_ON_HOOKS + return new_ptr; +} + static void SLAMP_free_hook(void *ptr, const void *caller){ old_free_hook(ptr, caller); } static void* SLAMP_memalign_hook(size_t alignment, size_t size, const void *caller){ - old_memalign_hook = __memalign_hook; + TURN_OFF_HOOKS void* ptr = memalign(alignment, size); // local_buffer->push(ALLOC)->push((uint64_t)ptr)->push(size); PRODUCE(ALLOC); @@ -280,7 +308,7 @@ static void* SLAMP_memalign_hook(size_t alignment, size_t size, const void *call // printf("memalign %lu at %p\n", size, ptr); counter_alloc++; - __memalign_hook = SLAMP_memalign_hook; + TURN_ON_HOOKS return ptr; } void* SLAMP_malloc(size_t size, uint32_t instr, size_t alignment){ diff --git a/liberty/lib/SLAMP/SLAMPsmtxq/consumer/consumer.cpp b/liberty/lib/SLAMP/SLAMPsmtxq/consumer/consumer.cpp index bec6ec2d..730f126c 100644 --- a/liberty/lib/SLAMP/SLAMPsmtxq/consumer/consumer.cpp +++ b/liberty/lib/SLAMP/SLAMPsmtxq/consumer/consumer.cpp @@ -84,9 +84,8 @@ int main() { uint64_t addr; uint32_t bare_instr; uint64_t value = 0; - instr = (uint32_t)CONSUME; + CONSUME_2(instr, bare_instr); addr = CONSUME; - bare_instr = (uint32_t)CONSUME; // value = CONSUME; if (DEBUG) { std::cout << "LOAD: " << instr << " " << addr << " " << bare_instr @@ -103,8 +102,7 @@ int main() { uint32_t instr; uint32_t bare_instr; uint64_t addr; - instr = (uint32_t)CONSUME; - bare_instr = (uint32_t)CONSUME; + CONSUME_2(instr, bare_instr); addr = CONSUME; if (DEBUG) { std::cout << "STORE: " << instr << " " << bare_instr << " " << addr diff --git a/liberty/lib/SLAMP/SLAMPsmtxq/slamp_hooks.h b/liberty/lib/SLAMP/SLAMPsmtxq/slamp_hooks.h index df546117..b18efd38 100644 --- a/liberty/lib/SLAMP/SLAMPsmtxq/slamp_hooks.h +++ b/liberty/lib/SLAMP/SLAMPsmtxq/slamp_hooks.h @@ -107,6 +107,7 @@ void SLAMP_storen_ext(const uint64_t addr, const uint32_t bare_inst, size_t n) A /* wrappers */ static void* SLAMP_malloc_hook(size_t size, const void *caller); +static void* SLAMP_realloc_hook(void *ptr, size_t size, const void *caller); static void SLAMP_free_hook(void *ptr, const void *caller); static void* SLAMP_memalign_hook(size_t alignment, size_t size, const void *caller); void* SLAMP_malloc(size_t size, uint32_t instr=0, size_t alignment=16); From 76c00918453e1b4c3aebacd9ca6bc0635da7db58 Mon Sep 17 00:00:00 2001 From: Ziyang Xu Date: Tue, 18 Oct 2022 11:03:28 -0400 Subject: [PATCH 23/97] update initialization alloc --- liberty/lib/SLAMP/SLAMPsmtxq/SmtxSend.cpp | 20 +++++++++++++++++++ .../ProfilingModules/DependenceModule.cpp | 18 ----------------- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/liberty/lib/SLAMP/SLAMPsmtxq/SmtxSend.cpp b/liberty/lib/SLAMP/SLAMPsmtxq/SmtxSend.cpp index 6d9f545f..9ca692f5 100644 --- a/liberty/lib/SLAMP/SLAMPsmtxq/SmtxSend.cpp +++ b/liberty/lib/SLAMP/SLAMPsmtxq/SmtxSend.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include "malloc.h" @@ -104,6 +105,25 @@ void SLAMP_init(uint32_t fn_id, uint32_t loop_id) { // local_buffer->push(pid); PRODUCE(pid); + auto allocateLibcReqs = [](void *addr, size_t size) { + PRODUCE(ALLOC); + PRODUCE((uint64_t)addr); + PRODUCE(size); + }; + + allocateLibcReqs((void*)&errno, sizeof(errno)); + allocateLibcReqs((void*)&stdin, sizeof(stdin)); + allocateLibcReqs((void*)&stdout, sizeof(stdout)); + allocateLibcReqs((void*)&stderr, sizeof(stderr)); + allocateLibcReqs((void*)&sys_nerr, sizeof(sys_nerr)); + + const unsigned short int* ctype_ptr = (*__ctype_b_loc()) - 128; + allocateLibcReqs((void*)ctype_ptr, 384 * sizeof(*ctype_ptr)); + const int32_t* itype_ptr = (*__ctype_tolower_loc()) - 128; + allocateLibcReqs((void*)itype_ptr, 384 * sizeof(*itype_ptr)); + itype_ptr = (*__ctype_toupper_loc()) - 128; + allocateLibcReqs((void*)itype_ptr, 384 * sizeof(*itype_ptr)); + old_malloc_hook = __malloc_hook; // old_free_hook = __free_hook; old_memalign_hook = __memalign_hook; diff --git a/liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/DependenceModule.cpp b/liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/DependenceModule.cpp index bc6b27cf..7253b81a 100644 --- a/liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/DependenceModule.cpp +++ b/liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/DependenceModule.cpp @@ -70,24 +70,6 @@ void init(uint32_t loop_id, uint32_t pid) { deplog_vec->reserve(DEPLOG_VEC_SIZE); smmap->init_stack(SIZE_8M, pid); - smmap->allocate((void*)&errno, sizeof(errno)); - smmap->allocate((void*)&stdin, sizeof(stdin)); - smmap->allocate((void*)&stdout, sizeof(stdout)); - smmap->allocate((void*)&stderr, sizeof(stderr)); - smmap->allocate((void*)&sys_nerr, sizeof(sys_nerr)); - - { - const unsigned short int* ctype_ptr = (*__ctype_b_loc()) - 128; - smmap->allocate((void*)ctype_ptr, 384 * sizeof(*ctype_ptr)); - } - { - const int32_t* itype_ptr = (*__ctype_tolower_loc()) - 128; - smmap->allocate((void*)itype_ptr, 384 * sizeof(*itype_ptr)); - } - { - const int32_t* itype_ptr = (*__ctype_toupper_loc()) - 128; - smmap->allocate((void*)itype_ptr, 384 * sizeof(*itype_ptr)); - } } void fini(const char *filename) { From ec231c219d1712ddc4cecaf125ad0f9c9f7628aa Mon Sep 17 00:00:00 2001 From: Ziyang Xu Date: Wed, 19 Oct 2022 19:34:03 -0400 Subject: [PATCH 24/97] add a load store buffer and able to proces load store in multiple threads --- .../ProfilingModules/DependenceModule.cpp | 184 ++++++++++++++---- 1 file changed, 148 insertions(+), 36 deletions(-) diff --git a/liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/DependenceModule.cpp b/liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/DependenceModule.cpp index 7253b81a..edef1fbe 100644 --- a/liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/DependenceModule.cpp +++ b/liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/DependenceModule.cpp @@ -10,21 +10,39 @@ #include "slamp_shadow_mem.h" #include "DependenceModule.h" #include + std::mutex m; +std::mutex mutex_process_load_store; + +constexpr unsigned LOCALWRITE_THREADS = 1; +std::mutex localwrite_mutexes[LOCALWRITE_THREADS]; #define SIZE_8M 0x800000 -#define DEPLOG_VEC_SIZE 100000000 +#define DEPLOG_VEC_SIZE 100'000'000 static slamp::MemoryMap* smmap = nullptr; static std::unordered_set *deplog_set; -static std::vector *deplog_vec; -static unsigned long deplog_vec_counter = 0; +static std::vector *deplog_vec[LOCALWRITE_THREADS]; static uint64_t slamp_iteration = 0; static uint64_t slamp_invocation = 0; static uint32_t target_loop_id = 0; -static void convertVectorToSet() { +struct LoadStoreEvent { + bool isLoad; + uint64_t addr; + // uint64_t value; + TS ts; + // uint32_t instr; + // uint32_t bare_instr; + // uint64_t invocation; + // uint64_t iteration; +}; +constexpr unsigned MAX_EVENTS = 100'000'000; +static std::vector *loadstore_vec, *loadstore_vec0, *loadstore_vec1; +static uint64_t loadstore_vec_counter = 0; + +static void convertVectorToSet(const unsigned thread_id) { // launch 56 threads to convert the vector to set independently, chunking constexpr auto THREADS = 56; @@ -32,7 +50,6 @@ static void convertVectorToSet() { for (unsigned long i = 0; i < THREADS; i++) { t[i] = std::thread( [&](int id) { - m.lock(); // take the chunk and convert to a set and return auto *deplog_set_chunk = new std::unordered_setreserve(DEPLOG_VEC_SIZE / THREADS + 1); auto begin = id * (DEPLOG_VEC_SIZE / THREADS); auto end = (id + 1) * (DEPLOG_VEC_SIZE / THREADS); - deplog_set_chunk->insert(deplog_vec->begin() + begin, - deplog_vec->begin() + end); + deplog_set_chunk->insert(deplog_vec[thread_id]->begin() + begin, + deplog_vec[thread_id]->begin() + end); + m.lock(); // lock the global set and insert the chunk deplog_set->insert(deplog_set_chunk->begin(), deplog_set_chunk->end()); @@ -59,6 +77,8 @@ static void convertVectorToSet() { std::cout << "Merging vec to set, set length " << deplog_set->size() << std::endl; } + + namespace DepMod { // init: setup the shadow memory void init(uint32_t loop_id, uint32_t pid) { @@ -66,14 +86,29 @@ void init(uint32_t loop_id, uint32_t pid) { target_loop_id = loop_id; smmap = new slamp::MemoryMap(TIMESTAMP_SIZE_IN_BYTES); deplog_set = new std::unordered_set(); - deplog_vec = new std::vector(); - deplog_vec->reserve(DEPLOG_VEC_SIZE); + for (auto & i : deplog_vec) { + i = new std::vector(); + i->reserve(DEPLOG_VEC_SIZE); + } + + loadstore_vec0 = new std::vector(); + loadstore_vec1 = new std::vector(); + loadstore_vec = loadstore_vec0; // double buffering + loadstore_vec0->reserve(MAX_EVENTS); + loadstore_vec1->reserve(MAX_EVENTS); smmap->init_stack(SIZE_8M, pid); } +void handleLoadAndStore(std::vector *loadstore_vec); + void fini(const char *filename) { - convertVectorToSet(); + + mutex_process_load_store.lock(); + handleLoadAndStore(loadstore_vec); + for (int i = 0; i < LOCALWRITE_THREADS; i++) { + convertVectorToSet(i); + } std::ofstream of(filename); of << target_loop_id << " " << 0 << " " << 0 << " " << 0 << " " << 0 << " " << 0 << "\n"; @@ -97,49 +132,126 @@ void allocate(void *addr, uint64_t size) { // void log(TS ts, const uint32_t dst_inst, TS *pts, const uint32_t bare_inst, // uint64_t addr, uint64_t value, uint8_t size) { -void log(TS ts, const uint32_t dst_inst, const uint32_t bare_inst){ +void log(const unsigned thread_id, TS ts, const uint32_t dst_inst, const uint32_t bare_inst, const uint64_t load_invocation, const uint64_t load_iteration){ uint32_t src_inst = GET_INSTR(ts); + + uint64_t src_invoc = GET_INVOC(ts); uint64_t src_iter = GET_ITER(ts); + if (src_invoc != GET_INVOC(load_invocation)) { + return; + } + // uint64_t src_invoc = GET_INVOC(ts); - slamp::KEY key(src_inst, dst_inst, bare_inst, src_iter != slamp_iteration); + slamp::KEY key(src_inst, dst_inst, bare_inst, src_iter != load_iteration); // std::cout << "src_inst: " << src_inst << " dst_inst: " << dst_inst << " bare_inst: " << bare_inst << " src_iter: " << src_iter << " slamp_iteration: " << slamp_iteration << " src_iter != slamp_iteration: " << (src_iter != slamp_iteration) << std::endl; - deplog_vec->emplace_back(key); - deplog_vec_counter++; - if (deplog_vec_counter == DEPLOG_VEC_SIZE - 1) { - convertVectorToSet(); - deplog_vec->resize(0); - deplog_vec_counter = 0; + deplog_vec[thread_id]->emplace_back(key); + if (deplog_vec[thread_id]->size() == DEPLOG_VEC_SIZE - 1) { + convertVectorToSet(thread_id); + deplog_vec[thread_id]->resize(0); } } +void handleLoadAndStore(std::vector *loadstore_vec) { + + // put in LOCALWRITE_THREADS threads to handle the loadstore_vec + std::thread t[LOCALWRITE_THREADS]; + + for (auto i = 0; i < LOCALWRITE_THREADS; i++) { + t[i] = std::thread( + [&](const unsigned thread_id) { + for (auto &e : *loadstore_vec) { + const auto &addr = e.addr; + const auto &ts = e.ts; + const auto invoc = GET_INVOC(ts); + const auto iter = GET_ITER(ts); + const auto instr = GET_INSTR(ts); + unsigned addr_mod = (addr >> 3) % LOCALWRITE_THREADS; + if (thread_id != addr_mod) { + continue; + } + + TS *s = (TS *)GET_SHADOW(addr, TIMESTAMP_SIZE_IN_POWER_OF_TWO); + if (e.isLoad) { + // load + // std::cout << "load " << e.addr << " " << e.value << " " << + // e.instr + // << " " << e.bare_instr << std::endl; smmap->load(e.addr, + // e.value, e.instr, e.bare_instr); + + // std::cout << "load " << instr << " " << addr << " " << + // bare_instr + // << " " + // << value << std::endl; + TS tss = s[0]; + if (tss != 0) { + log(thread_id, tss, instr, instr, invoc, iter); + } + } else { + // store + // std::cout << "store " << e.addr << " " << e.value << " " << + // e.instr + // << " " << e.bare_instr << std::endl; smmap->store(e.addr, + // e.value, e.instr, e.bare_instr); + // if (!smmap->is_allocated(s)) { + // std::cout << "store not allocated: " << instr << " " << + // bare_instr << " " << addr << std::endl; + // } + // TODO: handle output dependence. ignore it as of now. + // if (ASSUME_ONE_ADDR) { + s[0] = ts; + // } else { + // for (auto i = 0; i < size; i++) + // s[i] = ts; + // } + } + } + }, + i); + } + + for (auto & i : t) { + i.join(); + } + mutex_process_load_store.unlock(); +} + // template -void load(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value){ - TS* s = (TS*)GET_SHADOW(addr, TIMESTAMP_SIZE_IN_POWER_OF_TWO); - TS tss = s[0]; - if (tss != 0) { - log(tss, instr, bare_instr); +void load(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value) { + loadstore_vec->emplace_back(LoadStoreEvent{true, addr, CREATE_TS(instr, slamp_iteration, slamp_invocation)}); + loadstore_vec_counter++; + if (loadstore_vec_counter == MAX_EVENTS - 1) { + + // run this asynchrously + mutex_process_load_store.lock(); + std::thread t = std::thread(handleLoadAndStore, loadstore_vec); + t.detach(); + + // swap double buffer + loadstore_vec = loadstore_vec == loadstore_vec0 ? loadstore_vec1 : loadstore_vec0; + + loadstore_vec->resize(0); + loadstore_vec_counter = 0; } + } // template void store(uint32_t instr, uint32_t bare_instr, const uint64_t addr) { - TS *s = (TS *)GET_SHADOW(addr, TIMESTAMP_SIZE_IN_POWER_OF_TWO); - // if (!smmap->is_allocated(s)) { - // std::cout << "store not allocated: " << instr << " " << bare_instr << " " << addr << std::endl; - // } - TS ts = CREATE_TS(instr, slamp_iteration, slamp_invocation); - - // TODO: handle output dependence. ignore it as of now. - // if (ASSUME_ONE_ADDR) { - s[0] = ts; - // } else { - // for (auto i = 0; i < size; i++) - // s[i] = ts; - // } + loadstore_vec->emplace_back(LoadStoreEvent{false, addr, CREATE_TS(instr, slamp_iteration, slamp_invocation)}); + loadstore_vec_counter++; + if (loadstore_vec_counter == MAX_EVENTS - 1) { + mutex_process_load_store.lock(); + std::thread t = std::thread(handleLoadAndStore, loadstore_vec); + t.detach(); + loadstore_vec = loadstore_vec == loadstore_vec0 ? loadstore_vec1 : loadstore_vec0; + + loadstore_vec->resize(0); + loadstore_vec_counter = 0; + } } void loop_invoc() { From 1e4dade9e025dfc162a874b964e0b1cc7d3230d0 Mon Sep 17 00:00:00 2001 From: Ziyang Xu Date: Wed, 19 Oct 2022 19:34:19 -0400 Subject: [PATCH 25/97] have more bits for invocation --- .../ProfilingModules/slamp_timestamp.h | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/slamp_timestamp.h b/liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/slamp_timestamp.h index fc9c1428..b8ec4673 100644 --- a/liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/slamp_timestamp.h +++ b/liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/slamp_timestamp.h @@ -6,17 +6,17 @@ typedef uint64_t TS; // first 20 bits for instr and following 44 bits for iter #define TIMESTAMP_SIZE_IN_BYTES 8 #define TIMESTAMP_SIZE_IN_POWER_OF_TWO 3 -#define ITERATION_SIZE 40 -#define INVOCATION_SIZE 4 // 44-40 -#define CREATE_TS(instr, iter, invoc) ( ((TS)instr << 44) | (((TS)iter & (TS)0xffffffffff) << INVOCATION_SIZE) | ((TS)invoc & (TS)0xf)) -#define CREATE_TS_HASH(instr, hash, iter, invoc) \ - (((TS)instr << 44) | \ - (((TS)hash & (TS)0xfffffffff) << (INVOCATION_SIZE + 4)) | \ - (((TS)iter & (TS)0xf) << INVOCATION_SIZE) | ((TS)invoc & (TS)0xf)) +#define ITERATION_SIZE 28 +#define INVOCATION_SIZE 16 // 44-40 +#define CREATE_TS(instr, iter, invoc) ( ((TS)instr << 44) | (((TS)iter & (TS)0xfffffff) << INVOCATION_SIZE) | ((TS)invoc & (TS)0xffff)) +// #define CREATE_TS_HASH(instr, hash, iter, invoc) \ + // (((TS)instr << 44) | \ + // (((TS)hash & (TS)0xfffffffff) << (INVOCATION_SIZE + 4)) | \ + // (((TS)iter & (TS)0xf) << INVOCATION_SIZE) | ((TS)invoc & (TS)0xf)) #define GET_INSTR(ts) ((ts >> 44) & 0xfffff) #define GET_HASH(ts) ( (ts >> 8) & 0xfffffffff) -#define GET_ITER(ts) ( (ts >> INVOCATION_SIZE) & 0xffffffffff) -#define GET_INVOC(ts) ( ts & 0xf) +#define GET_ITER(ts) ( (ts >> INVOCATION_SIZE) & 0xfffffff) +#define GET_INVOC(ts) ( ts & 0xffff) #endif From f26828544bb7b7cfc86e3ce6c59258d274bc113b Mon Sep 17 00:00:00 2001 From: Ziyang Xu Date: Tue, 25 Oct 2022 18:06:07 -0400 Subject: [PATCH 26/97] add a custom double buffering design --- liberty/lib/SLAMP/CMakeLists.txt | 1 + liberty/lib/SLAMP/SLAMPcustom/CMakeLists.txt | 37 + liberty/lib/SLAMP/SLAMPcustom/CustomSend.cpp | 670 ++++++++++++++++++ liberty/lib/SLAMP/SLAMPcustom/bitcast.h | 29 + .../lib/SLAMP/SLAMPcustom/consumer/.gitignore | 1 + .../SLAMP/SLAMPcustom/consumer/CMakeLists.txt | 24 + .../ProfilingModules/DependenceModule.cpp | 300 ++++++++ .../ProfilingModules/DependenceModule.h | 23 + .../ProfilingModules/slamp_bound_malloc.h | 22 + .../consumer/ProfilingModules/slamp_logger.h | 110 +++ .../ProfilingModules/slamp_shadow_mem.h | 197 +++++ .../ProfilingModules/slamp_timestamp.h | 22 + .../SLAMP/SLAMPcustom/consumer/consumer.cpp | 262 +++++++ liberty/lib/SLAMP/SLAMPcustom/inline.h | 12 + liberty/lib/SLAMP/SLAMPcustom/slamp_hooks.h | 372 ++++++++++ liberty/lib/SLAMP/SLAMPcustom/sw_queue.c | 103 +++ .../lib/SLAMP/SLAMPcustom/sw_queue_astream.h | 402 +++++++++++ 17 files changed, 2587 insertions(+) create mode 100644 liberty/lib/SLAMP/SLAMPcustom/CMakeLists.txt create mode 100644 liberty/lib/SLAMP/SLAMPcustom/CustomSend.cpp create mode 100644 liberty/lib/SLAMP/SLAMPcustom/bitcast.h create mode 100644 liberty/lib/SLAMP/SLAMPcustom/consumer/.gitignore create mode 100644 liberty/lib/SLAMP/SLAMPcustom/consumer/CMakeLists.txt create mode 100644 liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.cpp create mode 100644 liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.h create mode 100644 liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/slamp_bound_malloc.h create mode 100644 liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/slamp_logger.h create mode 100644 liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/slamp_shadow_mem.h create mode 100644 liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/slamp_timestamp.h create mode 100644 liberty/lib/SLAMP/SLAMPcustom/consumer/consumer.cpp create mode 100644 liberty/lib/SLAMP/SLAMPcustom/inline.h create mode 100644 liberty/lib/SLAMP/SLAMPcustom/slamp_hooks.h create mode 100644 liberty/lib/SLAMP/SLAMPcustom/sw_queue.c create mode 100644 liberty/lib/SLAMP/SLAMPcustom/sw_queue_astream.h diff --git a/liberty/lib/SLAMP/CMakeLists.txt b/liberty/lib/SLAMP/CMakeLists.txt index cea20aee..2ca86ba5 100644 --- a/liberty/lib/SLAMP/CMakeLists.txt +++ b/liberty/lib/SLAMP/CMakeLists.txt @@ -24,3 +24,4 @@ add_subdirectory(SLAMPboost) add_subdirectory(SLAMPatomicq) add_subdirectory(SLAMPstats) add_subdirectory(SLAMPsmtxq) +add_subdirectory(SLAMPcustom) diff --git a/liberty/lib/SLAMP/SLAMPcustom/CMakeLists.txt b/liberty/lib/SLAMP/SLAMPcustom/CMakeLists.txt new file mode 100644 index 00000000..656d5f43 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPcustom/CMakeLists.txt @@ -0,0 +1,37 @@ +file(GLOB SRCS + "*.cpp" + # "*.c" +) +set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} "/u/ziyangx/test/boost/boost_1_80_0/install/lib/cmake") + +# add CMAKE_PREFIX_PATH +set(CMAKE_C_COMPILER "clang") +set(CMAKE_CXX_COMPILER "clang++") + +# Compilation flags +# set_source_files_properties(${SRCS} PROPERTIES COMPILE_FLAGS "-Wl,-save-temps -std=c++17 -Wno-inline -O3 -fexceptions")# -emit-llvm") +# set C++ flags +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17") +set_source_files_properties(${SRCS} PROPERTIES COMPILE_FLAGS "-flto -Wno-inline -O3 -fexceptions")# -emit-llvm") +set(PassName "slamp_hooks_custom") + +set(THREADS_PREFER_PTHREAD_FLAG ON) +find_package(Boost 1.80.0 REQUIRED COMPONENTS system) +include_directories(${Boost_INCLUDE_DIRS}) + + +# list(APPEND CMAKE_MODULE_PATH "${LLVM_CMAKE_DIR}") +#include(HandleLLVMOptions) +# include(AddLLVM) + + +include_directories(./ + /u/ziyangx/test/boost/boost_1_80_0/install/include) + +add_library(${PassName} STATIC ${SRCS}) +target_link_libraries(${PassName} ${Boost_LIBRARIES}) +# target_link_libraries(${PassName} nng::nng) +# add_llvm_library(${PassName}_shared SHARED ${SRCS}) +# set_target_properties(${PassName}_shared PROPERTIES OUTPUT_NAME ${PassName}) +# set_property(TARGET ${PassName} PROPERTY INTERPROCEDURAL_OPTIMIZATION TRUE) +#add_llvm_library(${PassName} SHARED ${SRCS}) # This is to generate libxxx.so diff --git a/liberty/lib/SLAMP/SLAMPcustom/CustomSend.cpp b/liberty/lib/SLAMP/SLAMPcustom/CustomSend.cpp new file mode 100644 index 00000000..48b07519 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPcustom/CustomSend.cpp @@ -0,0 +1,670 @@ +#include "slamp_hooks.h" +#include +#include +#include +#include +#include +#include +#include "malloc.h" + +#include +#include "sw_queue_astream.h" + +#include +#include + +namespace bip = boost::interprocess; + +static void *(*old_malloc_hook)(size_t, const void *); +static void *(*old_realloc_hook)(void *, size_t, const void *); +static void (*old_free_hook)(void *, const void *); +static void *(*old_memalign_hook)(size_t, size_t, const void *); +// create segment and corresponding allocator +bip::fixed_managed_shared_memory *segment; +bip::fixed_managed_shared_memory *segment2; +static SW_Queue the_queue; +static double_queue_p dqA, dqB, dq, dq_other; +static uint64_t dq_index = 0; +static uint64_t *dq_data; +static void swap(){ + if(dq == dqA){ + dq = dqB; + dq_other = dqA; + }else{ + dq = dqA; + dq_other = dqB; + } + dq_data = dq->data; +} + +static void produce_wait() ATTRIBUTE(noinline){ + dq->size = dq_index; + dq->ready_to_read = true; + while (!dq_other->ready_to_write){ + // spin + usleep(10); + } + swap(); + dq->ready_to_read = false; + dq_index = 0; +} + +static void produce(uint64_t x) {// ATTRIBUTE(always_inline) { + if (dq_index == QSIZE){ + produce_wait(); + } + dq_data[dq_index] = x; + dq_index++; + // _mm_prefetch(&dq_data[dq_index] + QPREFETCH, _MM_HINT_T0); +} + +static void produce_2(uint64_t x, uint64_t y) {// ATTRIBUTE(always_inline) { + if (dq_index + 2 >= QSIZE){ + produce_wait(); + } + dq_data[dq_index] = x; + dq_data[dq_index+1] = y; + dq_index += 2; + // _mm_prefetch(&dq_data[dq_index] + QPREFETCH, _MM_HINT_T0); +} + +static void produce_3(uint64_t x, uint64_t y, uint64_t z) {// ATTRIBUTE(always_inline) { + if (dq_index + 3 >= QSIZE){ + produce_wait(); + } + dq_data[dq_index] = x; + dq_data[dq_index+1] = y; + dq_data[dq_index+2] = z; + dq_index += 3; + // _mm_prefetch(&dq_data[dq_index] + QPREFETCH, _MM_HINT_T0); +} + +static void produce_4(uint64_t x, uint64_t y, uint64_t z, uint64_t w) {// ATTRIBUTE(always_inline) { + if (dq_index + 4 >= QSIZE){ + produce_wait(); + } + dq_data[dq_index] = x; + dq_data[dq_index+1] = y; + dq_data[dq_index+2] = z; + dq_data[dq_index+3] = w; + dq_index += 4; +} + +// #define CONSUME sq_consume(the_queue); +// #define PRODUCE(x) sq_produce(the_queue,(uint64_t)x); +#define PRODUCE(x) produce((uint64_t)x); + +#define COMBINE_2_32(x,y) ((((uint64_t)x)<<32) | (uint32_t)(y)) +#define CONSUME_2(x,y) do { uint64_t tmp = CONSUME; x = (uint32_t)(tmp>>32); y = (uint32_t) tmp; } while(0) + +#define TURN_ON_HOOKS \ + __malloc_hook = SLAMP_malloc_hook;\ + __realloc_hook = SLAMP_realloc_hook;\ + __memalign_hook = SLAMP_memalign_hook; + + +#define TURN_OFF_HOOKS \ + __malloc_hook = old_malloc_hook; \ + __realloc_hook = old_realloc_hook; \ + __memalign_hook = old_memalign_hook; + +// Ringbuffer fully constructed in shared memory. The element strings are +// also allocated from the same shared memory segment. This vector can be +// safely accessed from other processes. +unsigned long counter_load = 0; +unsigned long counter_store = 0; +unsigned long counter_ctx = 0; +unsigned long counter_alloc = 0; +// char local_buffer[LOCAL_BUFFER_SIZE]; +// unsigned buffer_counter = 0; +bool onProfiling = false; + +enum DepModAction: char +{ + INIT = 0, + LOAD, + STORE, + ALLOC, + LOOP_INVOC, + LOOP_ITER, + FINISHED +}; + +void SLAMP_init(uint32_t fn_id, uint32_t loop_id) { + + segment = new bip::fixed_managed_shared_memory(bip::open_or_create, "MySharedMemory", sizeof(uint64_t) *QSIZE *4, (void*)(1UL << 32)); + // segment2 = new bip::fixed_managed_shared_memory(bip::open_or_create, "MySharedMemory2", sizeof(uint64_t) *QSIZE *2, (void*)(1UL << 28)); + + dqA = segment->find("DQ_A").first; + dqB = segment->find("DQ_B").first; + dq = dqA; + dq_other = dqB; + dq_index = 0; + dq_data = dq->data; + + // managed_shared_memory(bip::open_or_create, "MySharedMemory", sizeof(uint64_t) *QSIZE *2); + // auto a_queue = new atomic_queue::AtomicQueueB(65536); + + // the_queue = static_cast(segment->find_or_construct("MyQueue")()); + // auto data = static_cast(segment2->find_or_construct("smtx_queue_data")[QSIZE]()); + // if (the_queue == nullptr) { + // std::cout << "Error: could not create queue" << std::endl; + // exit(-1); + // } + // the_queue->data = data; + // if (the_queue->data == nullptr) { + // std::cout << "Error: could not create queue data" << std::endl; + // exit(-1); + // } + + // [> Initialize the queue data structure <] + // the_queue->p_data = (uint64_t) the_queue->data; + // the_queue->c_inx = 0; + // the_queue->c_margin = 0; + // the_queue->p_glb_inx = 0; + // the_queue->c_glb_inx = 0; + // the_queue->ptr_c_glb_inx = &(the_queue->c_glb_inx); + // the_queue->ptr_p_glb_inx = &(the_queue->p_glb_inx); + // // local_buffer = new LocalBuffer(queue); + + // // send a msg with "fn_id, loop_id" + // // char msg[100]; + // // sprintf(msg, "%d,%d", fn_id, loop_id); + // // queue->push(shm::shared_string(msg, *char_alloc)); + + // local_buffer->push(INIT); + // local_buffer->push(loop_id); + uint32_t pid = getpid(); + printf("SLAMP_init: %d, %d, %d\n", fn_id, loop_id, pid); + // local_buffer->push(pid); + produce_3(INIT, loop_id, pid); + + auto allocateLibcReqs = [](void *addr, size_t size) { + produce_3(ALLOC, (uint64_t)addr, size); + }; + + allocateLibcReqs((void*)&errno, sizeof(errno)); + allocateLibcReqs((void*)&stdin, sizeof(stdin)); + allocateLibcReqs((void*)&stdout, sizeof(stdout)); + allocateLibcReqs((void*)&stderr, sizeof(stderr)); + allocateLibcReqs((void*)&sys_nerr, sizeof(sys_nerr)); + + const unsigned short int* ctype_ptr = (*__ctype_b_loc()) - 128; + allocateLibcReqs((void*)ctype_ptr, 384 * sizeof(*ctype_ptr)); + const int32_t* itype_ptr = (*__ctype_tolower_loc()) - 128; + allocateLibcReqs((void*)itype_ptr, 384 * sizeof(*itype_ptr)); + itype_ptr = (*__ctype_toupper_loc()) - 128; + allocateLibcReqs((void*)itype_ptr, 384 * sizeof(*itype_ptr)); + + old_malloc_hook = __malloc_hook; + // old_free_hook = __free_hook; + old_memalign_hook = __memalign_hook; + old_realloc_hook = __realloc_hook; + __malloc_hook = SLAMP_malloc_hook; + __free_hook = nullptr; + __realloc_hook = SLAMP_realloc_hook; + // __free_hook = SLAMP_free_hook; + __memalign_hook = SLAMP_memalign_hook; +} + +void SLAMP_fini(const char* filename){ + // send a msg with "fini" + // queue->push(shm::shared_string("fini", *char_alloc)); + std::cout << counter_load << " " << counter_store << " " << counter_ctx << std::endl; + // local_buffer->push(FINISHED); + // local_buffer->flush(); + PRODUCE(FINISHED); + // sq_flushQueue(the_queue); + dq->size = dq_index; + dq->ready_to_read = true; +} + +void SLAMP_allocated(uint64_t addr){} +void SLAMP_init_global_vars(const char *name, uint64_t addr, size_t size){ + // local_buffer->push(ALLOC)->push(addr)->push(size); + produce_3(ALLOC, addr, size); +} +void SLAMP_main_entry(uint32_t argc, char** argv, char** env){} + +void SLAMP_enter_fcn(uint32_t id){} +void SLAMP_exit_fcn(uint32_t id){} +void SLAMP_enter_loop(uint32_t id){} +void SLAMP_exit_loop(uint32_t id){} +void SLAMP_loop_iter_ctx(uint32_t id){} +void SLAMP_loop_invocation(){ + // send a msg with "loop_invocation" + // local_buffer->push(LOOP_INVOC); + PRODUCE(LOOP_INVOC); + + counter_ctx++; + onProfiling = true; +} +void SLAMP_loop_iteration(){ + // local_buffer->push(LOOP_ITER); + PRODUCE(LOOP_ITER); + counter_ctx++; +} + +void SLAMP_loop_exit(){ + onProfiling = false; +} + +void SLAMP_report_base_pointer_arg(uint32_t, uint32_t, void *ptr){} +void SLAMP_report_base_pointer_inst(uint32_t, void *ptr){} +void SLAMP_callback_stack_alloca(uint64_t, uint64_t, uint32_t, uint64_t){} +void SLAMP_callback_stack_free(){} + +void SLAMP_ext_push(const uint32_t instr){} +void SLAMP_ext_pop(){} + +void SLAMP_push(const uint32_t instr){} +void SLAMP_pop(){} + +void SLAMP_load(const uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline) { + // send a msg with "load, instr, addr, bare_instr, value" + // char msg[100]; + // sprintf(msg, "load,%d,%lu,%d,%lu", instr, addr, bare_instr, value); + // queue->push(shm::shared_string(msg, *char_alloc)); + // + if (onProfiling) { + // local_buffer->push(LOAD)->push(instr)->push(addr)->push(bare_instr); //->push(value); + produce_3(LOAD, COMBINE_2_32(instr, bare_instr), addr); + if (instr == 12356 && addr == 140724580372352) { + printf("SLAMP_load: %d, %lu, %d, %lu\n", instr, addr, bare_instr, value); + exit(-1); + } + // PRODUCE(value); + counter_load++; + } +} + +void SLAMP_load1(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + SLAMP_load(instr, addr, bare_instr, value); +} +void SLAMP_load2(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + SLAMP_load(instr, addr, bare_instr, value); +} +void SLAMP_load4(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + SLAMP_load(instr, addr, bare_instr, value); +} + +void SLAMP_load8(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + SLAMP_load(instr, addr, bare_instr, value); +} + +void SLAMP_loadn(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, size_t n){ + SLAMP_load(instr, addr, bare_instr, 0); +} + +void SLAMP_load1_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + SLAMP_load1(bare_instr, addr, bare_instr, value); +} +void SLAMP_load2_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + SLAMP_load2(bare_instr, addr, bare_instr, value); +} +void SLAMP_load4_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + SLAMP_load4(bare_instr, addr, bare_instr, value); +} +void SLAMP_load8_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value){ + SLAMP_load8(bare_instr, addr, bare_instr, value); +} +void SLAMP_loadn_ext(const uint64_t addr, const uint32_t bare_instr, size_t n){ + SLAMP_loadn(bare_instr, addr, bare_instr, n); +} + +void SLAMP_store(const uint32_t instr, const uint64_t addr, const uint32_t bare_instr) ATTRIBUTE(always_inline) { + // send a msg with "store, instr, addr, bare_instr, value" + // char msg[100]; + // sprintf(msg, "store,%d,%lu,%d,%lu", instr, addr, bare_instr, value); + // queue->push(shm::shared_string(msg, *char_alloc)); + if (onProfiling) { + // local_buffer->push(STORE)->push(instr)->push(bare_instr)->push(addr); + // PRODUCE(instr); + // PRODUCE(bare_instr); + produce_3(STORE, COMBINE_2_32(instr, bare_instr), addr); + counter_store++; + } +} + +void SLAMP_store1(uint32_t instr, const uint64_t addr){ + SLAMP_store(instr, addr, instr); +} + +void SLAMP_store2(uint32_t instr, const uint64_t addr){ + SLAMP_store(instr, addr, instr); +} +void SLAMP_store4(uint32_t instr, const uint64_t addr){ + SLAMP_store(instr, addr, instr); +} +void SLAMP_store8(uint32_t instr, const uint64_t addr){ + SLAMP_store(instr, addr, instr); +} +void SLAMP_storen(uint32_t instr, const uint64_t addr, size_t n){ + SLAMP_store(instr, addr, instr); +} + +void SLAMP_store1_ext(const uint64_t addr, const uint32_t bare_inst){ + SLAMP_store1(bare_inst, addr); +} +void SLAMP_store2_ext(const uint64_t addr, const uint32_t bare_inst){ + SLAMP_store2(bare_inst, addr); +} +void SLAMP_store4_ext(const uint64_t addr, const uint32_t bare_inst){ + SLAMP_store4(bare_inst, addr); +} +void SLAMP_store8_ext(const uint64_t addr, const uint32_t bare_inst){ + SLAMP_store8(bare_inst, addr); +} +void SLAMP_storen_ext(const uint64_t addr, const uint32_t bare_inst, size_t n){ + SLAMP_storen(bare_inst, addr, n); +} + +/* wrappers */ +static void* SLAMP_malloc_hook(size_t size, const void *caller){ + TURN_OFF_HOOKS + void* ptr = malloc(size); + // local_buffer->push(ALLOC)->push((uint64_t)ptr)->push(size); + PRODUCE(ALLOC); + PRODUCE((uint64_t)ptr); + PRODUCE(size); + // printf("malloc %lu at %p\n", size, ptr); + counter_alloc++; + TURN_ON_HOOKS + return ptr; +} + +static void* SLAMP_realloc_hook(void* ptr, size_t size, const void *caller){ + TURN_OFF_HOOKS + void* new_ptr = realloc(ptr, size); + // local_buffer->push(REALLOC)->push((uint64_t)ptr)->push((uint64_t)new_ptr)->push(size); + PRODUCE(ALLOC); + PRODUCE((uint64_t)new_ptr); + PRODUCE(size); + // printf("realloc %p to %lu at %p", ptr, size, new_ptr); + counter_alloc++; + TURN_ON_HOOKS + return new_ptr; +} + +static void SLAMP_free_hook(void *ptr, const void *caller){ + old_free_hook(ptr, caller); +} +static void* SLAMP_memalign_hook(size_t alignment, size_t size, const void *caller){ + TURN_OFF_HOOKS + void* ptr = memalign(alignment, size); + // local_buffer->push(ALLOC)->push((uint64_t)ptr)->push(size); + PRODUCE(ALLOC); + PRODUCE((uint64_t)ptr); + PRODUCE(size); + + // printf("memalign %lu at %p\n", size, ptr); + counter_alloc++; + TURN_ON_HOOKS + return ptr; +} +void* SLAMP_malloc(size_t size, uint32_t instr, size_t alignment){ + // void* ptr = malloc(size); + // // send a msg with "malloc, instr, ptr, size, alignment" + // // char msg[100]; + // // sprintf(msg, "malloc,%d,%lu,%lu,%lu", instr, ptr, size, alignment); + // // queue->push(shm::shared_string(msg, *char_alloc)); + // queue->push(ALLOC); + // queue->push(reinterpret_cast(ptr)); + // queue->push(size); + // counter_alloc++; + // return ptr; +} + +void* SLAMP_calloc(size_t nelem, size_t elsize){} +void* SLAMP_realloc(void* ptr, size_t size){} +void* SLAMP__Znam(size_t size){} +void* SLAMP__Znwm(size_t size){} + + + +char* SLAMP_strdup(const char *s1){} +char* SLAMP___strdup(const char *s1){} +void SLAMP_free(void* ptr){} +void SLAMP_cfree(void* ptr){} +void SLAMP__ZdlPv(void* ptr){} +void SLAMP__ZdaPv(void* ptr){} +int SLAMP_brk(void *end_data_segment){} +void* SLAMP_sbrk(intptr_t increment){} + +/* llvm memory intrinsics */ +void SLAMP_llvm_memcpy_p0i8_p0i8_i32(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint32_t sizeBytes){} +void SLAMP_llvm_memcpy_p0i8_p0i8_i64(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint64_t sizeBytes){} +void SLAMP_llvm_memmove_p0i8_p0i8_i32(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint32_t sizeBytes){} +void SLAMP_llvm_memmove_p0i8_p0i8_i64(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint64_t sizeBytes){} +void SLAMP_llvm_memset_p0i8_i32(const uint8_t* dstAddr, const uint32_t len){} +void SLAMP_llvm_memset_p0i8_i64(const uint8_t* dstAddr, const uint64_t len){} + +// void SLAMP_llvm_lifetime_start_p0i8(uint64_t size, uint8_t* ptr){} +// void SLAMP_llvm_lifetime_end_p0i8(uint64_t size, uint8_t* ptr){} + +/* String functions */ +size_t SLAMP_strlen(const char *str){} +char* SLAMP_strchr(char *s, int c){} +char* SLAMP_strrchr(char *s, int c){} +int SLAMP_strcmp(const char *s1, const char *s2){} +int SLAMP_strncmp(const char *s1, const char *s2, size_t n){} +char* SLAMP_strcpy(char *dest, const char *src){} +char* SLAMP_strncpy(char *dest, const char *src, size_t n){} +char* SLAMP_strcat(char *s1, const char *s2){} +char* SLAMP_strncat(char *s1, const char *s2, size_t n){} +char* SLAMP_strstr(char *s1, char *s2){} +size_t SLAMP_strspn(const char *s1, const char *s2){} +size_t SLAMP_strcspn(const char *s1, const char *s2){} +char* SLAMP_strtok(char *s, const char *delim){} +double SLAMP_strtod(const char *nptr, char **endptr){} +long int SLAMP_strtol(const char *nptr, char **endptr, int base){} +char* SLAMP_strpbrk(char *s1, char *s2){} + +/* Mem* and b* functions */ +void *SLAMP_memset (void *dest, int c, size_t n){} +void *SLAMP_memcpy (void *dest, const void *src, size_t n){} +void *SLAMP___builtin_memcpy (void *dest, const void *src, size_t n){} +void *SLAMP_memmove (void *dest, const void *src, size_t n){} +int SLAMP_memcmp(const void *s1, const void *s2, size_t n){} +void* SLAMP_memchr(void* ptr, int value, size_t num){} +void* SLAMP___rawmemchr(void* ptr, int value){} + +void SLAMP_bzero(void *s, size_t n){} +void SLAMP_bcopy(const void *s1, void *s2, size_t n){} + +/* IO */ +ssize_t SLAMP_read(int fd, void *buf, size_t count){} +int SLAMP_open(const char *pathname, int flags, mode_t mode){} +int SLAMP_close(int fd){} +ssize_t SLAMP_write(int fd, const void *buf, size_t count){} +off_t SLAMP_lseek(int fildes, off_t offset, int whence){} + +FILE * SLAMP_fopen(const char *path, const char *mode){} +FILE * SLAMP_fopen64(const char *path, const char *mode){} +FILE * SLAMP_freopen(const char *path, const char *mode, FILE* stream){} +int SLAMP_fflush(FILE *stream){} +int SLAMP_fclose(FILE *stream){} +int SLAMP_ferror(FILE *stream){} +int SLAMP_feof(FILE *stream){} +long SLAMP_ftell(FILE *stream){} +size_t SLAMP_fread(void * ptr, size_t size, size_t nitems, FILE *stream){} +size_t SLAMP_fwrite(const void *ptr, size_t size, size_t nitems, FILE *stream){} +int SLAMP_fseek(FILE *stream, long offset, int whence){} +void SLAMP_rewind(FILE *stream){} + +int SLAMP_fgetc(FILE *stream){} +int SLAMP_fputc(int c, FILE *stream){} +char * SLAMP_fgets(char *s, int n, FILE *stream){} +int SLAMP_fputs(const char *s, FILE *stream){} + +int SLAMP_ungetc(int c, FILE *stream){} +int SLAMP_putchar(int c){} +int SLAMP_getchar(void){} + +int SLAMP_fileno(FILE *stream){} +char * SLAMP_gets(char *s){} +int SLAMP_puts(const char *s){} + +int SLAMP_select(int n, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, struct timeval *timeout){} +int SLAMP_remove(const char *path){} + +void SLAMP_setbuf(FILE * stream, char * buf){} +void SLAMP_setvbuf(FILE * stream, char * buf, int mode, size_t size){} +char * SLAMP_tmpnam(char *s){} +FILE* SLAMP_tmpfile(void){} +char * SLAMP_ttyname(int fildes){} + +FILE * SLAMP_fdopen(int fildes, const char *mode){} +void SLAMP_clearerr(FILE *stream){} + +int SLAMP_truncate(const char *path, off_t length){} +int SLAMP_ftruncate(int fildes, off_t length){} + +int SLAMP_dup(int oldfd){} +int SLAMP_dup2(int oldfd, int newfd){} +int SLAMP_pipe(int filedes[2]){} + +int SLAMP_chmod(const char *path, mode_t mode){} +int SLAMP_fchmod(int fildes, mode_t mode){} +int SLAMP_fchown(int fd, uid_t owner, gid_t group){} +int SLAMP_access(const char *pathname, int mode){} +long SLAMP_pathconf(char *path, int name){} +int SLAMP_mkdir(const char *pathname, mode_t mode){} +int SLAMP_rmdir(const char *pathname){} +mode_t SLAMP_umask(mode_t mask){} +int SLAMP_fcntl(int fd, int cmd, struct flock *lock){} + +DIR* SLAMP_opendir(const char* name){} +struct dirent* SLAMP_readdir(DIR *dirp){} +struct dirent64* SLAMP_readdir64(DIR *dirp){} +int SLAMP_closedir(DIR* dirp){} + +/* Printf */ +int SLAMP_printf(const char *format, ...){} +int SLAMP_fprintf(FILE *stream, const char *format, ...){} +int SLAMP_sprintf(char *str, const char *format, ...){} +int SLAMP_snprintf(char *str, size_t size, const char *format, ...){} + +int SLAMP_vprintf(const char *format, va_list ap){} +int SLAMP_vfprintf(FILE *stream, const char *format, va_list ap){} +int SLAMP_vsprintf(char *str, const char *format, va_list ap){} +int SLAMP_vsnprintf(char *str, size_t size, const char *format, va_list ap){} + +/* Scanf */ +int SLAMP_fscanf(FILE *stream, const char *format, ... ){} +int SLAMP_scanf(const char *format, ... ){} +int SLAMP_sscanf(const char *s, const char *format, ... ){} +int SLAMP___isoc99_sscanf(const char *s, const char *format, ... ){} + +int SLAMP_vfscanf(FILE *stream, const char *format, va_list ap){} +int SLAMP_vscanf(const char *format, va_list ap){} +int SLAMP_vsscanf(const char *s, const char *format, va_list ap){} + +/* Time */ +time_t SLAMP_time(time_t *t){} +struct tm *SLAMP_localtime(const time_t *timer){} +struct lconv* SLAMP_localeconv(){} +struct tm *SLAMP_gmtime(const time_t *timer){} +int SLAMP_gettimeofday(struct timeval *tv, struct timezone *tz){} + +/* Math */ +double SLAMP_ldexp(double x, int exp){} +float SLAMP_ldexpf(float x, int exp){} +long double SLAMP_ldexpl(long double x, int exp){} +double SLAMP_log10(double x){} +float SLAMP_log10f(float x){} +long double SLAMP_log10l(long double x){} +double SLAMP_log(double x){} +float SLAMP_logf(float x){} +long double SLAMP_logl(long double x){} + +double SLAMP_exp(double x){} +float SLAMP_expf(float x){} +long double SLAMP_expl(long double x){} + +double SLAMP_cos(double x){} +float SLAMP_cosf(float x){} +long double SLAMP_cosl(long double x){} +double SLAMP_sin(double x){} +double SLAMP_tan(double x){} +float SLAMP_sinf(float x){} +long double SLAMP_sinl(long double x){} + +double SLAMP_atan(double x){} +float SLAMP_atanf(float x){} +long double SLAMP_atanl(long double x){} + +double SLAMP_floor(double x){} +float SLAMP_floorf(float x){} +long double SLAMP_floorl(long double x){} +double SLAMP_ceil(double x){} +float SLAMP_ceilf(float x){} +long double SLAMP_ceill(long double x){} + +double SLAMP_atan2(double y, double x){} +float SLAMP_atan2f(float y, float x){} +long double SLAMP_atan2l(long double y, long double x){} + +double SLAMP_sqrt(double x){} +float SLAMP_sqrtf(float x){} +long double SLAMP_sqrtl(long double x){} + +double SLAMP_pow(double x, double y){} +float SLAMP_powf(float x, float y){} +long double SLAMP_powl(long double x, long double y){} + +double SLAMP_fabs(double x){} +float SLAMP_fabsf(float x){} +long double SLAMP_fabsl(long double x){} + +double SLAMP_modf(double x, double *iptr){} +float SLAMP_modff(float x, float *iptr){} +long double SLAMP_modfl(long double x, long double *iptr){} + +double SLAMP_fmod(double x, double y){} + +double SLAMP_frexp(double num, int *exp){} +float SLAMP_frexpf(float num, int *exp){} +long double SLAMP_frexpl(long double num, int *exp){} + +int SLAMP_isnan(){} + +/* MISC */ +char *SLAMP_getenv(const char *name){} +int SLAMP_putenv(char* string){} +char *SLAMP_getcwd(char *buf, size_t size){} +char* SLAMP_strerror(int errnum){} +void SLAMP_exit(int status){} +void SLAMP__exit(int status){} +int SLAMP_link(const char *oldpath, const char *newpath){} +int SLAMP_unlink(const char *pathname){} +int SLAMP_isatty(int desc){} +int SLAMP_setuid(uid_t uid){} +uid_t SLAMP_getuid(void){} +uid_t SLAMP_geteuid(void){} +int SLAMP_setgid(gid_t gid){} +gid_t SLAMP_getgid(void){} +gid_t SLAMP_getegid(void){} +pid_t SLAMP_getpid(void){} +int SLAMP_chdir(const char *path){} +int SLAMP_execl(const char *path, const char *arg0, ... /*, (char *)0 */){} +int SLAMP_execv(const char *path, char *const argv[]){} +int SLAMP_execvp(const char *file, char *const argv[]){} +int SLAMP_kill(pid_t pid, int sig){} +pid_t SLAMP_fork(void){} +sighandler_t SLAMP___sysv_signal(int signum, sighandler_t handler){} +pid_t SLAMP_waitpid(pid_t pid, int* status, int options){} +void SLAMP_qsort(void* base, size_t nmemb, size_t size, int(*compar)(const void *, const void *)){} +int SLAMP_ioctl(int d, int request, ...){} +unsigned int SLAMP_sleep(unsigned int seconds){} +char* SLAMP_gcvt(double number, size_t ndigit, char* buf){} +char* SLAMP_nl_langinfo(nl_item item){} + +/* Compiler/Glibc Internals */ +void SLAMP___assert_fail(const char * assertion, const char * file, unsigned int line, const char * function){} +const unsigned short int **SLAMP___ctype_b_loc(void){} +int SLAMP__IO_getc(_IO_FILE * __fp){} +int SLAMP__IO_putc(int __c, _IO_FILE *__fp){} + +int SLAMP___fxstat (int __ver, int __fildes, struct stat *__stat_buf){} +int SLAMP___xstat (int __ver, __const char *__filename, struct stat *__stat_buf){} diff --git a/liberty/lib/SLAMP/SLAMPcustom/bitcast.h b/liberty/lib/SLAMP/SLAMPcustom/bitcast.h new file mode 100644 index 00000000..404a9a99 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPcustom/bitcast.h @@ -0,0 +1,29 @@ +#ifndef BITCAST_H +#define BITCAST_H + +typedef union { double d; int64_t i; } SQ_DI; +typedef union { float f; int32_t i; } SQ_FI; + +Inline int64_t doubleToInt(double d) { + SQ_DI conv; + conv.d = d; + return conv.i; +} +Inline int64_t floatToInt(float f) { + SQ_FI conv; + conv.f = f; + return conv.i; +} +Inline double intToDouble(int64_t i) { + SQ_DI conv; + conv.i = (int32_t)i; + return conv.d; +} + +Inline float intToFloat(int64_t i) { + SQ_FI conv; + conv.i = (int32_t)i; + return conv.f; +} + +#endif /* BITCAST_H */ diff --git a/liberty/lib/SLAMP/SLAMPcustom/consumer/.gitignore b/liberty/lib/SLAMP/SLAMPcustom/consumer/.gitignore new file mode 100644 index 00000000..378eac25 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPcustom/consumer/.gitignore @@ -0,0 +1 @@ +build diff --git a/liberty/lib/SLAMP/SLAMPcustom/consumer/CMakeLists.txt b/liberty/lib/SLAMP/SLAMPcustom/consumer/CMakeLists.txt new file mode 100644 index 00000000..c28d61e2 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPcustom/consumer/CMakeLists.txt @@ -0,0 +1,24 @@ +cmake_minimum_required (VERSION 3.6.2 FATAL_ERROR) + +# set(CMAKE_C_COMPILER "clang") +# set(CMAKE_CXX_COMPILER "clang++") +# set(CMAKE_LINKER "ld.gold") + +set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} "/u/ziyangx/test/boost/boost_1_80_0/install/lib/cmake") +set(THREADS_PREFER_PTHREAD_FLAG ON) +find_package(Threads REQUIRED) + +# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -flto") +# set(LINK_FLAGS "${LINK_FLAGS} -flto") + +find_package(Boost 1.80.0 REQUIRED COMPONENTS system) +include_directories(${Boost_INCLUDE_DIRS}) +include_directories(./ + ../) + +add_executable (consumer consumer.cpp ProfilingModules/DependenceModule.cpp) +target_link_libraries(consumer LINK_PUBLIC + ${Boost_LIBRARIES} + rt + Threads::Threads +) diff --git a/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.cpp b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.cpp new file mode 100644 index 00000000..a5045b13 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.cpp @@ -0,0 +1,300 @@ +#include +#include +#include +#include +#include +#include + +#include "slamp_timestamp.h" +#include "slamp_logger.h" +#include "slamp_shadow_mem.h" +#include "DependenceModule.h" +#include + +std::mutex m; +std::mutex mutex_process_load_store; + +constexpr unsigned LOCALWRITE_THREADS = 1; +std::mutex localwrite_mutexes[LOCALWRITE_THREADS]; + +#define SIZE_8M 0x800000 + +#define DEPLOG_VEC_SIZE 1'000'000 + +static slamp::MemoryMap* smmap = nullptr; +static std::unordered_set *deplog_set; +static std::vector *deplog_vec[LOCALWRITE_THREADS]; +static uint64_t slamp_iteration = 0; +static uint64_t slamp_invocation = 0; +static uint32_t target_loop_id = 0; + +struct LoadStoreEvent { + bool isLoad; + uint64_t addr; + // uint64_t value; + TS ts; + // uint32_t instr; + // uint32_t bare_instr; + // uint64_t invocation; + // uint64_t iteration; +}; +// constexpr unsigned MAX_EVENTS = 100'000'000; +constexpr unsigned MAX_EVENTS = 100'000; +static std::vector *loadstore_vec, *loadstore_vec0, *loadstore_vec1; +static uint64_t loadstore_vec_counter = 0; + +static std::chrono::duration total_time_off_critical_path(0); +static std::chrono::duration loadstore_vec_time(0); + +static void convertVectorToSet(const unsigned thread_id) { + + // launch 56 threads to convert the vector to set independently, chunking + constexpr auto THREADS = 56; + std::thread t[THREADS]; + for (unsigned long i = 0; i < THREADS; i++) { + t[i] = std::thread( + [&](int id) { + // take the chunk and convert to a set and return + auto *deplog_set_chunk = + new std::unordered_set(); + deplog_set_chunk->reserve(DEPLOG_VEC_SIZE / THREADS + 1); + auto begin = id * (DEPLOG_VEC_SIZE / THREADS); + auto end = (id + 1) * (DEPLOG_VEC_SIZE / THREADS); + deplog_set_chunk->insert(deplog_vec[thread_id]->begin() + begin, + deplog_vec[thread_id]->begin() + end); + + m.lock(); + // lock the global set and insert the chunk + deplog_set->insert(deplog_set_chunk->begin(), + deplog_set_chunk->end()); + m.unlock(); + delete deplog_set_chunk; + }, + i); + } + // join the threads + for (auto &i : t) { + i.join(); + } + + // std::cout << "Merging vec to set, set length " << deplog_set->size() + // << std::endl; +} + + +namespace DepMod { +// init: setup the shadow memory +void init(uint32_t loop_id, uint32_t pid) { + + target_loop_id = loop_id; + smmap = new slamp::MemoryMap(TIMESTAMP_SIZE_IN_BYTES); + deplog_set = new std::unordered_set(); + for (auto & i : deplog_vec) { + i = new std::vector(); + i->reserve(DEPLOG_VEC_SIZE); + } + + loadstore_vec0 = new std::vector(); + loadstore_vec1 = new std::vector(); + loadstore_vec = loadstore_vec0; // double buffering + loadstore_vec0->reserve(MAX_EVENTS); + loadstore_vec1->reserve(MAX_EVENTS); + + smmap->init_stack(SIZE_8M, pid); +} + +void handleLoadAndStore(std::vector *loadstore_vec); + +void fini(const char *filename) { + // show in seconds + std::cout << "Total time off critical path: " + << total_time_off_critical_path.count() / 1000000.0 << "s" + << std::endl; + + std::cout << "Loadstore vec time: " + << loadstore_vec_time.count() / 1000000.0 << "s" + << std::endl; + + mutex_process_load_store.lock(); + handleLoadAndStore(loadstore_vec); + for (int i = 0; i < LOCALWRITE_THREADS; i++) { + convertVectorToSet(i); + } + std::ofstream of(filename); + of << target_loop_id << " " << 0 << " " << 0 << " " + << 0 << " " << 0 << " " << 0 << "\n"; + + std::set ordered(deplog_set->begin(), deplog_set->end()); + for (auto &k: ordered) { + of << target_loop_id << " " << k.src << " " << k.dst << " " << k.dst_bare << " " + << (k.cross ? 1 : 0) << " " << 1 << " "; + of << "\n"; + } + + delete smmap; + delete deplog_set; +} + +void allocate(void *addr, uint64_t size) { + smmap->allocate(addr, size); + // std::cout << "allocate " << addr << " " << size << std::endl; +} + + +// void log(TS ts, const uint32_t dst_inst, TS *pts, const uint32_t bare_inst, + // uint64_t addr, uint64_t value, uint8_t size) { +void log(const unsigned thread_id, TS ts, const uint32_t dst_inst, const uint32_t bare_inst, const uint64_t load_invocation, const uint64_t load_iteration){ + + uint32_t src_inst = GET_INSTR(ts); + + uint64_t src_invoc = GET_INVOC(ts); + uint64_t src_iter = GET_ITER(ts); + + if (src_invoc != GET_INVOC(load_invocation)) { + return; + } + + // uint64_t src_invoc = GET_INVOC(ts); + + slamp::KEY key(src_inst, dst_inst, bare_inst, src_iter != load_iteration); + + // std::cout << "src_inst: " << src_inst << " dst_inst: " << dst_inst << " bare_inst: " << bare_inst << " src_iter: " << src_iter << " slamp_iteration: " << slamp_iteration << " src_iter != slamp_iteration: " << (src_iter != slamp_iteration) << std::endl; + deplog_vec[thread_id]->emplace_back(key); + if (deplog_vec[thread_id]->size() == DEPLOG_VEC_SIZE - 1) { + convertVectorToSet(thread_id); + deplog_vec[thread_id]->resize(0); + } +} + +void handleLoadAndStore(std::vector *loadstore_vec) { + + // put in LOCALWRITE_THREADS threads to handle the loadstore_vec + std::thread t[LOCALWRITE_THREADS]; + + for (auto i = 0; i < LOCALWRITE_THREADS; i++) { + t[i] = std::thread( + [&](const unsigned thread_id) { + for (auto &e : *loadstore_vec) { + const auto &addr = e.addr; + const auto &ts = e.ts; + const auto invoc = GET_INVOC(ts); + const auto iter = GET_ITER(ts); + const auto instr = GET_INSTR(ts); + unsigned addr_mod = (addr >> 3) % LOCALWRITE_THREADS; + if (thread_id != addr_mod) { + continue; + } + + TS *s = (TS *)GET_SHADOW(addr, TIMESTAMP_SIZE_IN_POWER_OF_TWO); + if (e.isLoad) { + // load + // std::cout << "load " << e.addr << " " << e.value << " " << + // e.instr + // << " " << e.bare_instr << std::endl; smmap->load(e.addr, + // e.value, e.instr, e.bare_instr); + + // std::cout << "load " << instr << " " << addr << " " << + // bare_instr + // << " " + // << value << std::endl; + TS tss = s[0]; + if (tss != 0) { + log(thread_id, tss, instr, instr, invoc, iter); + } + } else { + // store + // std::cout << "store " << e.addr << " " << e.value << " " << + // e.instr + // << " " << e.bare_instr << std::endl; smmap->store(e.addr, + // e.value, e.instr, e.bare_instr); + // if (!smmap->is_allocated(s)) { + // std::cout << "store not allocated: " << instr << " " << + // bare_instr << " " << addr << std::endl; + // } + // TODO: handle output dependence. ignore it as of now. + // if (ASSUME_ONE_ADDR) { + s[0] = ts; + // } else { + // for (auto i = 0; i < size; i++) + // s[i] = ts; + // } + } + } + }, + i); + } + + for (auto & i : t) { + i.join(); + } + mutex_process_load_store.unlock(); +} + + +// template +void load(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value) { + + + // auto start = std::chrono::high_resolution_clock::now(); + loadstore_vec->emplace_back(LoadStoreEvent{true, addr, CREATE_TS(instr, slamp_iteration, slamp_invocation)}); + loadstore_vec_counter++; + // auto end = std::chrono::high_resolution_clock::now(); + // auto duration = std::chrono::duration_cast(end - start); + // loadstore_vec_time += duration; + if (loadstore_vec_counter == MAX_EVENTS - 1) { + + // measure time in here + auto start = std::chrono::high_resolution_clock::now(); + + // run this asynchrously + mutex_process_load_store.lock(); + std::thread t = std::thread(handleLoadAndStore, loadstore_vec); + t.detach(); + + // swap double buffer + loadstore_vec = loadstore_vec == loadstore_vec0 ? loadstore_vec1 : loadstore_vec0; + + loadstore_vec->resize(0); + loadstore_vec_counter = 0; + + auto end = std::chrono::high_resolution_clock::now(); + auto duration = std::chrono::duration_cast(end - start); + total_time_off_critical_path += duration; + } + +} + +// template +void store(uint32_t instr, uint32_t bare_instr, const uint64_t addr) { + // auto start = std::chrono::high_resolution_clock::now(); + loadstore_vec->emplace_back(LoadStoreEvent{false, addr, CREATE_TS(instr, slamp_iteration, slamp_invocation)}); + loadstore_vec_counter++; + // auto end = std::chrono::high_resolution_clock::now(); + // auto duration = std::chrono::duration_cast(end - start); + // loadstore_vec_time += duration; + if (loadstore_vec_counter == MAX_EVENTS - 1) { + // measure time in here + auto start = std::chrono::high_resolution_clock::now(); + mutex_process_load_store.lock(); + std::thread t = std::thread(handleLoadAndStore, loadstore_vec); + t.detach(); + loadstore_vec = loadstore_vec == loadstore_vec0 ? loadstore_vec1 : loadstore_vec0; + + loadstore_vec->resize(0); + loadstore_vec_counter = 0; + auto end = std::chrono::high_resolution_clock::now(); + auto duration = std::chrono::duration_cast(end - start); + total_time_off_critical_path += duration; + } +} + +void loop_invoc() { + slamp_iteration = 0; + slamp_invocation++; +} + +void loop_iter() { + slamp_iteration++; +} +} // namespace DepMod diff --git a/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.h b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.h new file mode 100644 index 00000000..a2664cdc --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.h @@ -0,0 +1,23 @@ +#include + +namespace DepMod { +void init(uint32_t loop_id, uint32_t pid); +void fini(const char *filename); +void load(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value); +void store(uint32_t instr, uint32_t bare_instr, const uint64_t addr); +void allocate(void *addr, uint64_t size); +void loop_invoc(); +void loop_iter(); + +enum DepModAction: char +{ + INIT = 0, + LOAD, + STORE, + ALLOC, + LOOP_INVOC, + LOOP_ITER, + FINISHED +}; + +} // namespace DepMod diff --git a/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/slamp_bound_malloc.h b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/slamp_bound_malloc.h new file mode 100644 index 00000000..0d6b9410 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/slamp_bound_malloc.h @@ -0,0 +1,22 @@ +#ifndef SLAMPLIB_HOOKS_SLAMP_BOUND_MALLOC_H +#define SLAMPLIB_HOOKS_SLAMP_BOUND_MALLOC_H + +#include +#include +namespace slamp +{ + +void init_bound_malloc(void* heap_bound); +void fini_bound_malloc(); + +size_t get_object_size(void* ptr); + +void* bound_malloc(size_t size, size_t alignment=16); +bool bound_free(void *ptr, uint64_t &starting_page, unsigned &purge_cnt); +void* bound_calloc(size_t num, size_t size); +void* bound_realloc(void* ptr, size_t size); +void bound_discard_page(); + +} + +#endif diff --git a/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/slamp_logger.h b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/slamp_logger.h new file mode 100644 index 00000000..89641a03 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/slamp_logger.h @@ -0,0 +1,110 @@ +#ifndef SLAMPLIB_HOOKS_SLAMP_LOGGER +#define SLAMPLIB_HOOKS_SLAMP_LOGGER + +#include +#include +#include +#include + +#include "slamp_timestamp.h" + +namespace slamp +{ + +struct KEY +{ + uint32_t src; + uint32_t dst; + uint32_t dst_bare; + uint32_t cross; + + KEY() {} + KEY(uint32_t s, uint32_t d, uint32_t b, uint32_t c) : src(s), dst(d), dst_bare(b), cross(c) {} +}; + +struct KEYHash +{ + size_t operator()(const KEY& key) const + { + static_assert(sizeof(size_t) == sizeof(uint64_t), "Should be 64bit address"); + std::hash hash_fn; + + // hash(32-32) ^ hash(32-1) + // FIXME: find a better hash function + return hash_fn(((uint64_t)key.src << 32) | key.dst) ^ hash_fn(((uint64_t)key.dst_bare << 1) | (key.cross & 0x1)); + } +}; + +struct KEYComp +{ + bool operator()(const KEY& key1, const KEY& key2) const + { + uint32_t src1 = key1.src; + uint32_t src2 = key2.src; + + if (src1 < src2) + return true; + else if (src1 > src2) + return false; + + uint32_t dst1 = key1.dst; + uint32_t dst2 = key2.dst; + + if (dst1 < dst2) + return true; + else if (dst1 > dst2) + return false; + + uint32_t dst_bare1 = key1.dst_bare; + uint32_t dst_bare2 = key2.dst_bare; + + if (dst_bare1 < dst_bare2) + return true; + else if (dst_bare1 > dst_bare2) + return false; + + uint32_t cross1 = key1.cross; + uint32_t cross2 = key2.cross; + + return cross1 < cross2; + } +}; + +struct KEYEqual +{ + bool operator()(const KEY& key1, const KEY& key2) const + { + uint32_t src1 = key1.src; + uint32_t src2 = key2.src; + + if ( src1 != src2 ) return false; + + uint32_t dst1 = key1.dst; + uint32_t dst2 = key2.dst; + + if ( dst1 != dst2 ) return false; + + uint32_t dst_bare1 = key1.dst_bare; + uint32_t dst_bare2 = key2.dst_bare; + + if ( dst_bare1 != dst_bare2 ) return false; + + uint32_t cross1 = key1.cross; + uint32_t cross2 = key2.cross; + + if ( cross1 != cross2 ) return false; + + return true; + } +}; + + +void init_logger(uint32_t fn_id, uint32_t loop_id); +void fini_logger(const char* filename); + +uint32_t log(TS ts, const uint32_t dst_instr, TS* pts, const uint32_t bare_inst, uint64_t addr, uint64_t value, uint8_t size); +void print_log(const char* filename); + +} + +#endif diff --git a/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/slamp_shadow_mem.h b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/slamp_shadow_mem.h new file mode 100644 index 00000000..ef07201b --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/slamp_shadow_mem.h @@ -0,0 +1,197 @@ +#ifndef SLAMPLIB_HOOKS_SLAMP_SHADOW_MEM_H +#define SLAMPLIB_HOOKS_SLAMP_SHADOW_MEM_H + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +// higher half of canonical region cannot be used + +/// left shift by `shift`, mask 47 LSB, toggle #45 bit? +#define MASK1 0x00007fffffffffffL +#define MASK2 0x0000200000000000L +#define GET_SHADOW(addr, shift) \ + (((((uint64_t)(addr)) << (shift)) & MASK1) ^ MASK2) + +namespace slamp { + +class MemoryMap { +public: + uint64_t heapStart = 0; + MemoryMap(unsigned r) : ratio(r), ratio_shift(0) { + // ratio expected to be a power of 2 + assert((r & (r - 1)) == 0); + + // log(r) + unsigned n = r; + while ((n & 1) == 0) { + this->ratio_shift += 1; + n = n >> 1; + } + + // get the page size of the host system + pagesize = getpagesize(); + pagemask = ~(pagesize - 1); + } + + ~MemoryMap() { + // freeing all remaining shadow addresses + for (auto page : pages) { + uint64_t s = GET_SHADOW(page, ratio_shift); + munmap(reinterpret_cast(s), pagesize * ratio); + } + } + + unsigned get_ratio() { return ratio; } + + bool is_allocated(void *addr) { + auto a = reinterpret_cast(addr); + uint64_t page = a & pagemask; + if (pages.find(page) != pages.end()) + return true; + else + return false; + } + + /// allocate shadow page if not exist + void *allocate(void *addr, size_t size) { + auto a = reinterpret_cast(addr); + uint64_t pagebegin = a & pagemask; + uint64_t pageend = (a + size - 1) & pagemask; + + // try mmap + + std::set shadow_pages; + bool success = true; + + for (uint64_t page = pagebegin; page <= pageend; page += pagesize) { + if (pages.find(page) != pages.end()) + continue; + + uint64_t s = GET_SHADOW(page, ratio_shift); + // create a shadow page for the page + void *p = mmap(reinterpret_cast(s), pagesize * ratio, + PROT_WRITE | PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (p == MAP_FAILED) { + int err = errno; + printf("mmap failed: %lx errno: %d\n", s, err); + raise(SIGINT); + + success = false; + break; + } else { + shadow_pages.insert(p); + } + } + + if (success) { + for (uint64_t page = pagebegin; page <= pageend; page += pagesize) + pages.insert(page); + uint64_t pagebegin = a & pagemask; + uint64_t pageend = (a + size - 1) & pagemask; + + // return shadow_mem + auto *shadow_addr = (uint64_t *)GET_SHADOW(a, ratio_shift); + return (void *)(shadow_addr); + } else { + // cleanup + for (auto shadow_page : shadow_pages) + munmap(shadow_page, pagesize * ratio); + return nullptr; + } + } + + // free the shadow pages + void deallocate_pages(uint64_t page, unsigned cnt) { + munmap(reinterpret_cast(GET_SHADOW(page, ratio_shift)), + pagesize * ratio * cnt); + + // fprintf(stderr, "deallocate_pages: %lx %d\n", GET_SHADOW(page, ratio_shift), cnt); + + for (auto i = 0; i < cnt; i++, page += pagesize) + pages.erase(page); + } + + /// for realloc; the dependence carries over + void copy(void *dst, void *src, size_t size) { + size_t shadow_size = size * ratio; + void *shadow_dst = (void *)GET_SHADOW(dst, ratio_shift); + void *shadow_src = (void *)GET_SHADOW(src, ratio_shift); + memcpy(shadow_dst, shadow_src, shadow_size); + } + + void init_heap(void *addr) { + heapStart = reinterpret_cast(addr); + } + + // init stack size to be fixed 8MB + // the stack in /proc/$pid/maps changes in runtime, use it to find the end + void init_stack(uint64_t stack_size, uint64_t pid) { + char filename[256]; + char buf[5000]; + sprintf(filename, "/proc/%lu/maps", pid); + // sprintf(filename, "/proc/%u/maps", getpid()); + + FILE *fp = fopen(filename, "r"); + if (!fp) { + perror(filename); + exit(EXIT_FAILURE); + } + + bool allocated = false; + + while (fgets(buf, sizeof(buf), fp) != nullptr) { + uint64_t start, end; + char name[5000]; + + int n = sscanf(buf, "%lx-%lx %*c%*c%*c%*c %*llx %*x:%*x %*lu %s", &start, + &end, name); + + if (n != 3) { + continue; + } + + // FIXME: get heap start addr + if (!strcmp(name, "[heap]")) { + heapStart = start; + } + + + if (!strcmp(name, "[stack]")) { + // stack grow from end (big address) backwards + // print the stack start and end + printf("stack: %lx %lx\n", end - stack_size, end); + allocate(reinterpret_cast(end - stack_size), stack_size); + allocated = true; + break; + } + } + + if (!allocated) { + fprintf(stderr, "Error: failed to allocate shadow for stack"); + exit(EXIT_FAILURE); + } + } + +private: + std::set pages; // page table + + unsigned ratio; // (size of metadata) / (size of real data) + unsigned ratio_shift; + uint64_t pagesize; + uint64_t pagemask; +}; + +} // namespace slamp +#endif diff --git a/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/slamp_timestamp.h b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/slamp_timestamp.h new file mode 100644 index 00000000..b8ec4673 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/slamp_timestamp.h @@ -0,0 +1,22 @@ +#ifndef SLAMPLIB_HOOKS_SLAMP_TIMESTAMP +#define SLAMPLIB_HOOKS_SLAMP_TIMESTAMP + +#include + +typedef uint64_t TS; // first 20 bits for instr and following 44 bits for iter +#define TIMESTAMP_SIZE_IN_BYTES 8 +#define TIMESTAMP_SIZE_IN_POWER_OF_TWO 3 +#define ITERATION_SIZE 28 +#define INVOCATION_SIZE 16 // 44-40 +#define CREATE_TS(instr, iter, invoc) ( ((TS)instr << 44) | (((TS)iter & (TS)0xfffffff) << INVOCATION_SIZE) | ((TS)invoc & (TS)0xffff)) +// #define CREATE_TS_HASH(instr, hash, iter, invoc) \ + // (((TS)instr << 44) | \ + // (((TS)hash & (TS)0xfffffffff) << (INVOCATION_SIZE + 4)) | \ + // (((TS)iter & (TS)0xf) << INVOCATION_SIZE) | ((TS)invoc & (TS)0xf)) +#define GET_INSTR(ts) ((ts >> 44) & 0xfffff) +#define GET_HASH(ts) ( (ts >> 8) & 0xfffffffff) +#define GET_ITER(ts) ( (ts >> INVOCATION_SIZE) & 0xfffffff) +#define GET_INVOC(ts) ( ts & 0xffff) + + +#endif diff --git a/liberty/lib/SLAMP/SLAMPcustom/consumer/consumer.cpp b/liberty/lib/SLAMP/SLAMPcustom/consumer/consumer.cpp new file mode 100644 index 00000000..31e0604d --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPcustom/consumer/consumer.cpp @@ -0,0 +1,262 @@ +#include +#include +#include "ProfilingModules/DependenceModule.h" +#include +#include +#include "sw_queue_astream.h" +#include + + +#include +#include +#include + +namespace bip = boost::interprocess; + +#define DEBUG 0 +#define ACTION 1 + +// create segment and corresponding allocator +bip::fixed_managed_shared_memory *segment, *segment2; +static SW_Queue the_queue; +static double_queue_p dqA, dqB, dq, dq_other; +static uint64_t dq_index = 0; +static uint64_t dq_size = 0; +static uint64_t *dq_data; +static void swap(){ + if(dq == dqA){ + dq = dqB; + dq_other = dqA; + }else{ + dq = dqA; + dq_other = dqB; + } + dq_data = dq->data; +} + +static void check() { + if (dq_index == dq_size){ + dq->ready_to_write = true; + while (!dq_other->ready_to_read){ + // spin + usleep(10); + } + swap(); + dq->ready_to_write = false; + dq_index = 0; + dq_size = dq->size; + } +} + +static inline uint64_t consume(){ + uint64_t ret = dq_data[dq_index]; + dq_index++; + // _mm_prefetch(&dq_data[dq_index] + QPREFETCH, _MM_HINT_NTA); + return ret; +} + +// #define CONSUME sq_consume(the_queue); +#define CONSUME consume(); +#define PRODUCE(x) sq_produce(the_queue,(uint64_t)x); + +#define PRODUCE_2(x,y) PRODUCE( (((uint64_t)x)<<32) | (uint32_t)(y) ) +#define CONSUME_2(x,y) do { uint64_t tmp = CONSUME; x = (uint32_t)(tmp>>32); y = (uint32_t) tmp; } while(0) + +int main(int argc, char** argv) { + + segment = new bip::fixed_managed_shared_memory(bip::open_or_create, "MySharedMemory", sizeof(uint64_t) *QSIZE *4, (void*)(1UL << 32)); + // segment2 = new bip::fixed_managed_shared_memory(bip::open_or_create, "MySharedMemory2", sizeof(uint64_t) *QSIZE *2, (void*)(1UL << 28)); + // managed_shared_memory(bip::open_or_create, "MySharedMemory", sizeof(uint64_t) *QSIZE *2); + // auto a_queue = new atomic_queue::AtomicQueueB(65536); + + dqA = segment->construct("DQ_A")(double_queue_t()); + dqB = segment->construct("DQ_B")(double_queue_t()); + auto dataA = segment->construct("DQ_Data_A")[QSIZE](uint64_t()); + auto dataB = segment->construct("DQ_Data_B")[QSIZE](uint64_t()); + dqA->init(dataA); + dqB->init(dataB); + dq = dqB; + dq_other = dqA; + dq_data = dq->data; + + + // the_queue = static_cast(segment->find_or_construct("MyQueue")()); + // auto data = static_cast(segment2->find_or_construct("smtx_queue_data")[QSIZE]()); + // segment = new bip::managed_shared_memory(bip::open_or_create, "MySharedMemory", sizeof(uint64_t) *QSIZE *2); + // auto a_queue = new atomic_queue::AtomicQueueB(65536); + // the_queue = static_cast(segment->find_or_construct("MyQueue")()); + // auto data = static_cast(segment->find_or_construct("smtx_queue_data")[QSIZE]()); + + // // measure time + // auto start = std::chrono::high_resolution_clock::now(); + // unsigned long ROUND = 100; + // // parse the ROUND number from cmd + // if (argc > 1) { + // std::stringstream ss; + // ss << argv[1]; + // ss >> ROUND; + // } + // for (int round = 0; round < ROUND; round++) { + // for (unsigned long i = 0; i < QSIZE; i++) { + // data[i] = i; + // } + // } + // auto end = std::chrono::high_resolution_clock::now(); + // std::chrono::duration diff = end-start; + // std::cout << "Time for " << ROUND * QSIZE / 1'000'000 << "M events : " << diff.count() << " s" << std::endl; + // // throughput + // std::cout << "Write Throughput: " << ROUND * QSIZE / diff.count() / 1'000'000 << "M events/s" << std::endl; + + // start = std::chrono::high_resolution_clock::now(); + // unsigned total = 0; + // for (int round = 0; round < ROUND; round++) { + // for (unsigned long i = 0; i < QSIZE; i++) { + // total += data[i]; + // } + // } + // end = std::chrono::high_resolution_clock::now(); + // diff = end-start; + // std::cout << "Total = " << total << std::endl; + // std::cout << "Time for " << ROUND * QSIZE / 1'000'000 << "M events : " << diff.count() << " s" << std::endl; + // // throughput + // std::cout << "Read Throughput: " << ROUND * QSIZE / diff.count() / 1'000'000 << "M events/s" << std::endl; + + + // if (the_queue == nullptr) { + // std::cout << "Error: could not create queue" << std::endl; + // return 1; + // } + // the_queue->data = data; + // if (the_queue->data == nullptr) { + // std::cout << "Error: could not create queue data" << std::endl; + // return 1; + // } + // [> Initialize the queue data structure <] + // the_queue->p_data = (uint64_t) the_queue->data; + // the_queue->c_inx = 0; + // the_queue->c_margin = 0; + // the_queue->p_glb_inx = 0; + // the_queue->c_glb_inx = 0; + // the_queue->ptr_c_glb_inx = &(the_queue->c_glb_inx); + // the_queue->ptr_p_glb_inx = &(the_queue->p_glb_inx); + + + uint64_t counter = 0; + // shm::shared_string v(char_alloc); + + using Action = DepMod::DepModAction; + + uint32_t loop_id; + while (true) { + check(); + char v; + v = (char)CONSUME; + counter++; + + switch (v) { + case Action::INIT: { + uint32_t pid; + loop_id = (uint32_t)CONSUME; + pid = (uint32_t)CONSUME; + + if (DEBUG) { + std::cout << "INIT: " << loop_id << " " << pid << std::endl; + } +#if ACTION + DepMod::init(loop_id, pid); +#endif + break; + }; + case Action::LOAD: { + uint32_t instr; + uint64_t addr; + uint32_t bare_instr; + uint64_t value = 0; + CONSUME_2(instr, bare_instr); + addr = CONSUME; + // value = CONSUME; + if (DEBUG) { + std::cout << "LOAD: " << instr << " " << addr << " " << bare_instr + << " " << value << std::endl; + } +#if ACTION + DepMod::load(instr, addr, bare_instr, value); + // DepMod::load(instr, addr, bare_instr, value); +#endif + + break; + }; + case Action::STORE: { + uint32_t instr; + uint32_t bare_instr; + uint64_t addr; + CONSUME_2(instr, bare_instr); + addr = CONSUME; + if (DEBUG) { + std::cout << "STORE: " << instr << " " << bare_instr << " " << addr + << std::endl; + } +#if ACTION + DepMod::store(instr, bare_instr, addr); +#endif + break; + }; + case Action::ALLOC: { + uint64_t addr; + uint64_t size; + addr = CONSUME; + size = CONSUME; + if (DEBUG) { + std::cout << "ALLOC: " << addr << " " << size << std::endl; + } +#if ACTION + DepMod::allocate(reinterpret_cast(addr), size); +#endif + break; + }; + case Action::LOOP_INVOC: { +#if ACTION + DepMod::loop_invoc(); +#endif + + if (DEBUG) { + std::cout << "LOOP_INVOC" << std::endl; + } + break; + }; + case Action::LOOP_ITER: { +#if ACTION + DepMod::loop_iter(); +#endif + + if (DEBUG) { + std::cout << "LOOP_ITER" << std::endl; + } + break; + }; + case Action::FINISHED: { + std::stringstream ss; + ss << "deplog-" << loop_id << ".txt"; +#if ACTION + DepMod::fini(ss.str().c_str()); +#endif + std::cout << "Finished loop: " << loop_id << " after " << counter + << " events" << std::endl; + break; + }; + default: + std::cout << "Unknown action: " << (uint64_t)v << std::endl; + + std::cout << "Is ready to read?:" << dq->ready_to_read << " " << "Is ready to write?:" << dq->ready_to_write << std::endl; + std::cout << "Index: " << dq_index << " Size:" << dq->size << std::endl; + for (int i = 0; i < 101; i++) { + std::cout << dq->data[dq_index - 100 + i] << " "; + } + exit(-1); + } + + if (counter % 100000000 == 0) { + std::cout << "Processed " << counter / 1000000 << "M events" << std::endl; + } + } +} diff --git a/liberty/lib/SLAMP/SLAMPcustom/inline.h b/liberty/lib/SLAMP/SLAMPcustom/inline.h new file mode 100644 index 00000000..7b16986f --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPcustom/inline.h @@ -0,0 +1,12 @@ +#ifndef INLINE_H +#define INLINE_H + +#define Weak __attribute__((weak)) + +#ifndef NO_INLINE +#define Inline static inline +#else /* NO_INLINE */ +#define Inline Weak +#endif/* NO_INLINE */ + +#endif /* INLINE_H */ diff --git a/liberty/lib/SLAMP/SLAMPcustom/slamp_hooks.h b/liberty/lib/SLAMP/SLAMPcustom/slamp_hooks.h new file mode 100644 index 00000000..b18efd38 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPcustom/slamp_hooks.h @@ -0,0 +1,372 @@ +#ifndef SLAMPLIB_HOOKS_SLAMP_HOOKS_H +#define SLAMPLIB_HOOKS_SLAMP_HOOKS_H + +// FIXME: inline tweak actually make things worse in sequential and better with +// 16x, so turn it on at all time before understanding why +#define ATTRIBUTE(x) __attribute__((x)) +// #ifdef ITO_ENABLE +// // #define ATTRIBUTE(x) +// #define ATTRIBUTE(x) __attribute__((x)) +// #else +// #define ATTRIBUTE(x) +// #endif + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +void SLAMP_dbggv(int id); +void SLAMP_dbggvstr(char* str); + +// SLAMP measure functions +void SLAMP_measure_init(); +void SLAMP_measure_fini(); +void SLAMP_measure_load(uint32_t id, uint64_t size); +void SLAMP_measure_store(uint32_t id, uint64_t size); +static void* SLAMP_measure_malloc_hook(size_t size, const void *caller); +static void SLAMP_measure_free_hook(void *ptr, const void *caller); + +void SLAMP_init(uint32_t fn_id, uint32_t loop_id); +void SLAMP_fini(const char* filename); + +void SLAMP_allocated(uint64_t addr); +void SLAMP_init_global_vars(const char *name, uint64_t addr, size_t size); +void SLAMP_main_entry(uint32_t argc, char** argv, char** env); + +void SLAMP_enter_fcn(uint32_t id); +void SLAMP_exit_fcn(uint32_t id); +void SLAMP_enter_loop(uint32_t id); +void SLAMP_exit_loop(uint32_t id); +void SLAMP_loop_iter_ctx(uint32_t id); +void SLAMP_loop_invocation(); +void SLAMP_loop_iteration(); +void SLAMP_loop_exit(); + +void SLAMP_report_base_pointer_arg(uint32_t, uint32_t, void *ptr); +void SLAMP_report_base_pointer_inst(uint32_t, void *ptr); +void SLAMP_callback_stack_alloca(uint64_t, uint64_t, uint32_t, uint64_t); +void SLAMP_callback_stack_free(void); + +void SLAMP_ext_push(const uint32_t instr); +void SLAMP_ext_pop(); + +void SLAMP_push(const uint32_t instr); +void SLAMP_pop(); + +void SLAMP_load1(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_load2(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_load4(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_load8(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_loadn(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, size_t n) ATTRIBUTE(always_inline);; + +void SLAMP_load1_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_load2_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_load4_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_load8_ext(const uint64_t addr, const uint32_t bare_instr, uint64_t value) ATTRIBUTE(always_inline); +void SLAMP_loadn_ext(const uint64_t addr, const uint32_t bare_instr, size_t n) ATTRIBUTE(always_inline);; + +void SLAMP_store1(uint32_t instr, const uint64_t addr) ATTRIBUTE(always_inline); +void SLAMP_store2(uint32_t instr, const uint64_t addr) ATTRIBUTE(always_inline); +void SLAMP_store4(uint32_t instr, const uint64_t addr) ATTRIBUTE(always_inline); +void SLAMP_store8(uint32_t instr, const uint64_t addr) ATTRIBUTE(always_inline); +void SLAMP_storen(uint32_t instr, const uint64_t addr, size_t n) ATTRIBUTE(always_inline);; + +void SLAMP_store1_ext(const uint64_t addr, const uint32_t bare_inst) ATTRIBUTE(always_inline); +void SLAMP_store2_ext(const uint64_t addr, const uint32_t bare_inst) ATTRIBUTE(always_inline); +void SLAMP_store4_ext(const uint64_t addr, const uint32_t bare_inst) ATTRIBUTE(always_inline); +void SLAMP_store8_ext(const uint64_t addr, const uint32_t bare_inst) ATTRIBUTE(always_inline); +void SLAMP_storen_ext(const uint64_t addr, const uint32_t bare_inst, size_t n) ATTRIBUTE(always_inline);; + +/* wrappers */ +static void* SLAMP_malloc_hook(size_t size, const void *caller); +static void* SLAMP_realloc_hook(void *ptr, size_t size, const void *caller); +static void SLAMP_free_hook(void *ptr, const void *caller); +static void* SLAMP_memalign_hook(size_t alignment, size_t size, const void *caller); +void* SLAMP_malloc(size_t size, uint32_t instr=0, size_t alignment=16); + +void* SLAMP_calloc(size_t nelem, size_t elsize); +void* SLAMP_realloc(void* ptr, size_t size); +void* SLAMP__Znam(size_t size); +void* SLAMP__Znwm(size_t size); + +char* SLAMP_strdup(const char *s1); +char* SLAMP___strdup(const char *s1); +void SLAMP_free(void* ptr); +void SLAMP_cfree(void* ptr); +void SLAMP__ZdlPv(void* ptr); +void SLAMP__ZdaPv(void* ptr); +int SLAMP_brk(void *end_data_segment); +void* SLAMP_sbrk(intptr_t increment); + +/* llvm memory intrinsics */ +void SLAMP_llvm_memcpy_p0i8_p0i8_i32(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint32_t sizeBytes); +void SLAMP_llvm_memcpy_p0i8_p0i8_i64(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint64_t sizeBytes); +void SLAMP_llvm_memmove_p0i8_p0i8_i32(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint32_t sizeBytes); +void SLAMP_llvm_memmove_p0i8_p0i8_i64(const uint8_t* dstAddr, const uint8_t* srcAddr, const uint64_t sizeBytes); +void SLAMP_llvm_memset_p0i8_i32(const uint8_t* dstAddr, const uint32_t len); +void SLAMP_llvm_memset_p0i8_i64(const uint8_t* dstAddr, const uint64_t len); + +// void SLAMP_llvm_lifetime_start_p0i8(uint64_t size, uint8_t* ptr); +// void SLAMP_llvm_lifetime_end_p0i8(uint64_t size, uint8_t* ptr); + +/* String functions */ +size_t SLAMP_strlen(const char *str); +char* SLAMP_strchr(char *s, int c); +char* SLAMP_strrchr(char *s, int c); +int SLAMP_strcmp(const char *s1, const char *s2); +int SLAMP_strncmp(const char *s1, const char *s2, size_t n); +char* SLAMP_strcpy(char *dest, const char *src); +char* SLAMP_strncpy(char *dest, const char *src, size_t n); +char* SLAMP_strcat(char *s1, const char *s2); +char* SLAMP_strncat(char *s1, const char *s2, size_t n); +char* SLAMP_strstr(char *s1, char *s2); +size_t SLAMP_strspn(const char *s1, const char *s2); +size_t SLAMP_strcspn(const char *s1, const char *s2); +char* SLAMP_strtok(char *s, const char *delim); +double SLAMP_strtod(const char *nptr, char **endptr); +long int SLAMP_strtol(const char *nptr, char **endptr, int base); +char* SLAMP_strpbrk(char *s1, char *s2); + +/* Mem* and b* functions */ +void *SLAMP_memset (void *dest, int c, size_t n); +void *SLAMP_memcpy (void *dest, const void *src, size_t n); +void *SLAMP___builtin_memcpy (void *dest, const void *src, size_t n); +void *SLAMP_memmove (void *dest, const void *src, size_t n); +int SLAMP_memcmp(const void *s1, const void *s2, size_t n); +void* SLAMP_memchr(void* ptr, int value, size_t num); +void* SLAMP___rawmemchr(void* ptr, int value); + +void SLAMP_bzero(void *s, size_t n); +void SLAMP_bcopy(const void *s1, void *s2, size_t n); + +/* IO */ +ssize_t SLAMP_read(int fd, void *buf, size_t count); +int SLAMP_open(const char *pathname, int flags, mode_t mode); +int SLAMP_close(int fd); +ssize_t SLAMP_write(int fd, const void *buf, size_t count); +off_t SLAMP_lseek(int fildes, off_t offset, int whence); + +FILE * SLAMP_fopen(const char *path, const char *mode); +FILE * SLAMP_fopen64(const char *path, const char *mode); +FILE * SLAMP_freopen(const char *path, const char *mode, FILE* stream); +int SLAMP_fflush(FILE *stream); +int SLAMP_fclose(FILE *stream); +int SLAMP_ferror(FILE *stream); +int SLAMP_feof(FILE *stream); +long SLAMP_ftell(FILE *stream); +size_t SLAMP_fread(void * ptr, size_t size, size_t nitems, FILE *stream); +size_t SLAMP_fwrite(const void *ptr, size_t size, size_t nitems, FILE *stream); +int SLAMP_fseek(FILE *stream, long offset, int whence); +void SLAMP_rewind(FILE *stream); + +int SLAMP_fgetc(FILE *stream); +int SLAMP_fputc(int c, FILE *stream); +char * SLAMP_fgets(char *s, int n, FILE *stream); +int SLAMP_fputs(const char *s, FILE *stream); + +int SLAMP_ungetc(int c, FILE *stream); +int SLAMP_putchar(int c); +int SLAMP_getchar(void); + +int SLAMP_fileno(FILE *stream); +char * SLAMP_gets(char *s); +int SLAMP_puts(const char *s); + +int SLAMP_select(int n, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, struct timeval *timeout); +int SLAMP_remove(const char *path); + +void SLAMP_setbuf(FILE * stream, char * buf); +void SLAMP_setvbuf(FILE * stream, char * buf, int mode, size_t size); +char * SLAMP_tmpnam(char *s); +FILE* SLAMP_tmpfile(void); +char * SLAMP_ttyname(int fildes); + +FILE * SLAMP_fdopen(int fildes, const char *mode); +void SLAMP_clearerr(FILE *stream); + +int SLAMP_truncate(const char *path, off_t length); +int SLAMP_ftruncate(int fildes, off_t length); + +int SLAMP_dup(int oldfd); +int SLAMP_dup2(int oldfd, int newfd); +int SLAMP_pipe(int filedes[2]); + +int SLAMP_chmod(const char *path, mode_t mode); +int SLAMP_fchmod(int fildes, mode_t mode); +int SLAMP_fchown(int fd, uid_t owner, gid_t group); +int SLAMP_access(const char *pathname, int mode); +long SLAMP_pathconf(char *path, int name); +int SLAMP_mkdir(const char *pathname, mode_t mode); +int SLAMP_rmdir(const char *pathname); +mode_t SLAMP_umask(mode_t mask); +int SLAMP_fcntl(int fd, int cmd, struct flock *lock); + +DIR* SLAMP_opendir(const char* name); +struct dirent* SLAMP_readdir(DIR *dirp); +struct dirent64* SLAMP_readdir64(DIR *dirp); +int SLAMP_closedir(DIR* dirp); + +/* Printf */ +int SLAMP_printf(const char *format, ...); +int SLAMP_fprintf(FILE *stream, const char *format, ...); +int SLAMP_sprintf(char *str, const char *format, ...); +int SLAMP_snprintf(char *str, size_t size, const char *format, ...); + +int SLAMP_vprintf(const char *format, va_list ap); +int SLAMP_vfprintf(FILE *stream, const char *format, va_list ap); +int SLAMP_vsprintf(char *str, const char *format, va_list ap); +int SLAMP_vsnprintf(char *str, size_t size, const char *format, va_list ap); + +/* Scanf */ +int SLAMP_fscanf(FILE *stream, const char *format, ... ); +int SLAMP_scanf(const char *format, ... ); +int SLAMP_sscanf(const char *s, const char *format, ... ); +int SLAMP___isoc99_sscanf(const char *s, const char *format, ... ); + +int SLAMP_vfscanf(FILE *stream, const char *format, va_list ap); +int SLAMP_vscanf(const char *format, va_list ap); +int SLAMP_vsscanf(const char *s, const char *format, va_list ap); + +/* Time */ +time_t SLAMP_time(time_t *t); +struct tm *SLAMP_localtime(const time_t *timer); +struct lconv* SLAMP_localeconv(); +struct tm *SLAMP_gmtime(const time_t *timer); +int SLAMP_gettimeofday(struct timeval *tv, struct timezone *tz); + +/* Math */ +double SLAMP_ldexp(double x, int exp); +float SLAMP_ldexpf(float x, int exp); +long double SLAMP_ldexpl(long double x, int exp); +double SLAMP_log10(double x); +float SLAMP_log10f(float x); +long double SLAMP_log10l(long double x); +double SLAMP_log(double x); +float SLAMP_logf(float x); +long double SLAMP_logl(long double x); + +double SLAMP_exp(double x); +float SLAMP_expf(float x); +long double SLAMP_expl(long double x); + +double SLAMP_cos(double x); +float SLAMP_cosf(float x); +long double SLAMP_cosl(long double x); +double SLAMP_sin(double x); +double SLAMP_tan(double x); +float SLAMP_sinf(float x); +long double SLAMP_sinl(long double x); + +double SLAMP_atan(double x); +float SLAMP_atanf(float x); +long double SLAMP_atanl(long double x); + +double SLAMP_floor(double x); +float SLAMP_floorf(float x); +long double SLAMP_floorl(long double x); +double SLAMP_ceil(double x); +float SLAMP_ceilf(float x); +long double SLAMP_ceill(long double x); + +double SLAMP_atan2(double y, double x); +float SLAMP_atan2f(float y, float x); +long double SLAMP_atan2l(long double y, long double x); + +double SLAMP_sqrt(double x); +float SLAMP_sqrtf(float x); +long double SLAMP_sqrtl(long double x); + +double SLAMP_pow(double x, double y); +float SLAMP_powf(float x, float y); +long double SLAMP_powl(long double x, long double y); + +double SLAMP_fabs(double x); +float SLAMP_fabsf(float x); +long double SLAMP_fabsl(long double x); + +double SLAMP_modf(double x, double *iptr); +float SLAMP_modff(float x, float *iptr); +long double SLAMP_modfl(long double x, long double *iptr); + +double SLAMP_fmod(double x, double y); + +double SLAMP_frexp(double num, int *exp); +float SLAMP_frexpf(float num, int *exp); +long double SLAMP_frexpl(long double num, int *exp); + +int SLAMP_isnan(); + +/* MISC */ +char *SLAMP_getenv(const char *name); +int SLAMP_putenv(char* string); +char *SLAMP_getcwd(char *buf, size_t size); +char* SLAMP_strerror(int errnum); +void SLAMP_exit(int status); +void SLAMP__exit(int status); +int SLAMP_link(const char *oldpath, const char *newpath); +int SLAMP_unlink(const char *pathname); +int SLAMP_isatty(int desc); +int SLAMP_setuid(uid_t uid); +uid_t SLAMP_getuid(void); +uid_t SLAMP_geteuid(void); +int SLAMP_setgid(gid_t gid); +gid_t SLAMP_getgid(void); +gid_t SLAMP_getegid(void); +pid_t SLAMP_getpid(void); +int SLAMP_chdir(const char *path); +int SLAMP_execl(const char *path, const char *arg0, ... /*, (char *)0 */); +int SLAMP_execv(const char *path, char *const argv[]); +int SLAMP_execvp(const char *file, char *const argv[]); +int SLAMP_kill(pid_t pid, int sig); +pid_t SLAMP_fork(void); +sighandler_t SLAMP___sysv_signal(int signum, sighandler_t handler); +pid_t SLAMP_waitpid(pid_t pid, int* status, int options); +void SLAMP_qsort(void* base, size_t nmemb, size_t size, int(*compar)(const void *, const void *)); +int SLAMP_ioctl(int d, int request, ...); +unsigned int SLAMP_sleep(unsigned int seconds); +char* SLAMP_gcvt(double number, size_t ndigit, char* buf); +char* SLAMP_nl_langinfo(nl_item item); + +/* Compiler/Glibc Internals */ +void SLAMP___assert_fail(const char * assertion, const char * file, unsigned int line, const char * function); +const unsigned short int **SLAMP___ctype_b_loc(void); +int SLAMP__IO_getc(_IO_FILE * __fp); +int SLAMP__IO_putc(int __c, _IO_FILE *__fp); +int* SLAMP___errno_location (void); + +int SLAMP___fxstat (int __ver, int __fildes, struct stat *__stat_buf); +int SLAMP___xstat (int __ver, __const char *__filename, struct stat *__stat_buf); + +#ifdef __cplusplus +} +#endif + +#endif /* SLAMP_HOOKS_H */ diff --git a/liberty/lib/SLAMP/SLAMPcustom/sw_queue.c b/liberty/lib/SLAMP/SLAMPcustom/sw_queue.c new file mode 100644 index 00000000..33199929 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPcustom/sw_queue.c @@ -0,0 +1,103 @@ +#include +#include + +#include "sw_queue_astream.h" + +/* + * Create and initialize a queue. + */ +SW_Queue sq_createQueue(void) +{ + return sq_initQueue(sq_createQueueBlock(1)); +} + +/* + * Creates, but does not initialize a block of queues. + */ +SW_Queue sq_createQueueBlock(unsigned queues) { + + /* Allocate the queue data structure */ + sw_queue_t *queue = (sw_queue_t *) mmap((void *) (1UL << 32), + queues * sizeof(sw_queue_t), + PROT_WRITE | PROT_READ, + MAP_SHARED | MAP_ANONYMOUS, + -1, + (off_t) 0); + if(queue == (sw_queue_t *) -1) { + perror("sq_createQueue"); + exit(1); + } + + return queue; +} + +/* + * Initialize a queue. sq_createQueue does this automatically. + */ +SW_Queue sq_initQueue(SW_Queue queue) { + + /* Allocate the queue */ + queue->data = (uint64_t *) mmap(0, + sizeof(uint64_t) * QSIZE, + PROT_WRITE | PROT_READ, + MAP_SHARED | MAP_ANONYMOUS | MAP_32BIT, + -1, + (off_t) 0); + + if(queue->data == (void *) -1) { + perror("sq_initQueue"); + exit(1); + } + + /* Initialize the queue data structure */ + queue->p_data = (uint64_t) queue->data; + queue->c_inx = 0; + queue->c_margin = 0; + queue->p_glb_inx = 0; + queue->c_glb_inx = 0; + queue->ptr_c_glb_inx = &(queue->c_glb_inx); + queue->ptr_p_glb_inx = &(queue->p_glb_inx); + +#ifdef INSTRUMENT + queue->total_produces = 0; + queue->total_consumes = 0; +#endif + + return queue; +} + +void sq_freeQueueBlock(SW_Queue queue, unsigned size) { + + for(unsigned i = 0; i < size; ++i) { + + /* This queue may never have been initialized. */ + if(queue[i].data) { + +#ifdef INSTRUMENT + fprintf(stderr, "Queue %p: %ld produces, %ld consumes\n", + (void*)queue, + queue->total_produces, + queue->total_consumes); +#endif + + if(munmap(queue[i].data, sizeof(uint64_t) * QSIZE)) { + perror("sq_freeQueueBlock"); + exit(-1); + } + } + } + + if(munmap(queue, sizeof(sw_queue_t) * size)) { + perror("sq_freeQueueBlock"); + exit(-1); + } +} + +/* + * Free a queue. + */ +void sq_freeQueue(SW_Queue queue) +{ + sq_freeQueueBlock(queue, 1); +} + diff --git a/liberty/lib/SLAMP/SLAMPcustom/sw_queue_astream.h b/liberty/lib/SLAMP/SLAMPcustom/sw_queue_astream.h new file mode 100644 index 00000000..03ab4074 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPcustom/sw_queue_astream.h @@ -0,0 +1,402 @@ +/** ***********************************************/ +/** *** SW Queue with Supporting Variable Size ****/ +/** ***********************************************/ +#ifndef SW_QUEUE_H +#define SW_QUEUE_H + +#define DUALCORE + +#include +#include +#include +#include +#include + +#include "inline.h" +#include "bitcast.h" +#ifndef CACHELINE_SIZE +#define CACHELINE_SIZE 64 +/*#define CACHELINE_SIZE 64 */ +/* Cache line size of glacier, as reported by lmbench. Other tests, + * namely the smtx ones, vindicate this. */ +#endif /* CACHELINE_SIZE */ + +#ifndef CHUNK_SIZE +#ifdef DUALCORE +#define CHUNK_SIZE (1 << 8) +#else /* DUALCORE */ +#define CHUNK_SIZE (1 << 14) +#endif /* DUALCORE */ +#endif /* CHUNK_SIZE */ + +#define HIGH_CHUNKMASK ((((uint64_t) (CHUNK_SIZE - 1)) << 32)) + +#ifndef STREAM +#ifdef DUALCORE +#define STREAM (false) +#else /* DUALCORE */ +#define STREAM (true) +#endif /* DUALCORE */ +#endif /* STREAM */ + +#if !STREAM +#define sq_write sq_stdWrite +#else /* STREAM */ +#define sq_write sq_streamWrite +#endif /* STREAM */ + +#ifndef QMARGIN +#define QMARGIN 4 +#endif /* QMARGIN */ + +#ifndef QSIZE +#define QSIZE (1 << 21) +#endif /* QSIZE */ + +#ifndef QPREFETCH +#define QPREFETCH (1 << 7) +#endif /* QPREFETCH */ + +#define QMASK (QSIZE - 1) +#define HIGH_QMASK (((uint64_t) QMASK) << 32 | (uint32_t) ~0) + +#define PAD(suffix, size) char padding ## suffix [CACHELINE_SIZE - (size)] + +#ifdef __cplusplus +extern "C" { +#endif +typedef struct { + volatile bool ready_to_read; + PAD(1, sizeof(bool)); + volatile bool ready_to_write; + PAD(2, sizeof(bool)); + uint64_t size; + PAD(3, sizeof(uint64_t)); + uint64_t *data; + + void init(uint64_t *data){ + this->ready_to_read = false; + this->ready_to_write = true; + this->size = 0; + this->data = data; + } +} double_queue_t, *double_queue_p; + +typedef struct { + + uint64_t p_data; + PAD(1, sizeof(uint64_t)); + + volatile uint32_t *ptr_c_glb_inx; + uint32_t p_glb_inx; + PAD(2, sizeof(volatile uint32_t *) + sizeof(uint32_t)); + + uint32_t c_margin; + uint32_t c_inx; + PAD(3, sizeof(uint32_t) * 2); + + volatile uint32_t *ptr_p_glb_inx; + uint32_t c_glb_inx; + PAD(4, sizeof(volatile uint32_t *) + sizeof(uint32_t)); + + uint64_t *data; + + uint64_t total_produces; + uint64_t total_consumes; + + PAD(5, sizeof(volatile uint64_t *) + sizeof(uint64_t) * 2); + +} sw_queue_t, *SW_Queue; + +typedef void *sq_cbData; +typedef void (*sq_callback)(sq_cbData); + +/* ***************************************** */ +/* ******* Detail Implementation *********** */ +/* ***************************************** */ + +SW_Queue sq_createQueue(void); +SW_Queue sq_createQueueBlock(unsigned queues); +SW_Queue sq_initQueue(SW_Queue queue); +void sq_freeQueueBlock(SW_Queue queue, unsigned size); +void sq_freeQueue(SW_Queue q); + +/* ******************************************************** */ +/* *********** Preallocate Method *********************** */ +/* ******************************************************** */ + +/* + * This method is faster on multi-processor machines as it bypasses the L2 + * cache. + */ +Inline void sq_streamWrite(uint64_t *addr, uint64_t value) { + + /* __m64 mmxReg = _mm_set_pi64x((int64_t) value); */ + /* _mm_stream_pi((__m64 *) addr, mmxReg); */ + + __asm ( + "movntiq %1, (%0)\n" + : + : "r" (addr), "r" (value) + ); +} + +/* + * This method is faster on multi-core machines as it exploits the common L2 + * cache. + */ +Inline void sq_stdWrite(uint64_t *addr, uint64_t value) { + *addr = value; +} + +/* + * Modulo subtraction + */ +Inline uint32_t sq_modSub(uint32_t minuend, + uint32_t subtrahend, + uint32_t mask) { + return (minuend - subtrahend) & mask; +} + +Inline uint32_t sq_pInx(SW_Queue q) { + return (uint32_t) (q->p_data >> 32); +} + +/* + * Flush all produce values. + * + * sfence is slow, but necessary for streaming writes. sfence MUST proceded + * updating p_glb_inx, otherwise there will be a race condition. + */ +Inline void sq_flushQueue(SW_Queue q) +{ +#if STREAM + _mm_sfence(); +#endif + + *q->ptr_p_glb_inx = sq_pInx(q); +} + +/* + * Wait on the consumer. + * + * Pausing while spinning improves performance on NetBurst and improves energy + * efficiency. + */ +Inline void sq_waitConsumer(SW_Queue q, sq_callback cb, sq_cbData cbd) +{ +#ifndef NO_CON + uint32_t threshold = (QMARGIN - 1) * QSIZE / QMARGIN; + if(sq_modSub(sq_pInx(q), *q->ptr_c_glb_inx, QMASK) > threshold) { + + /* Blocking path */ + cb(cbd); + + while(sq_modSub(sq_pInx(q), *q->ptr_c_glb_inx, QMASK) > threshold) { + usleep(10); + } + } else { + + /* Fast path */ + sq_flushQueue(q); + } +#endif /* NO_CON */ +} + +/* + * Produce a value to a queue. + */ +Inline void sq_produce(SW_Queue q, uint64_t value, + sq_callback cb, sq_cbData cbd) +{ +#ifdef INSTRUMENT + q->total_produces++; +#endif + + uint64_t pDataRaw = q->p_data; + uint32_t pInx = sq_pInx(q); + uint64_t *pData = (uint64_t *) (size_t) (uint32_t) q->p_data; + uint64_t *ptr = pInx + pData; + + sq_write(ptr, value); + q->p_data = (pDataRaw + (1ULL << 32)) & HIGH_QMASK; + + // if(!(q->p_data & HIGH_CHUNKMASK)) { + // sq_waitConsumer(q, cb, cbd); + // } +} + +#define sq_produce(Q,V) sq_produce(Q, V, (sq_callback) sq_flushQueue, Q) + +/* + * Produce two values to a queue. Do not intermingle with sq_produce. + */ +Inline void sq_produce2(SW_Queue q, uint64_t a, uint64_t b, + sq_callback cb, sq_cbData cbd) +{ +#ifdef INSTRUMENT + q->total_produces += 2; +#endif + + /* Otherwise, gcc couldn't constant propagate. */ + uint64_t pDataRaw = q->p_data; + uint32_t pInx = sq_pInx(q); + uint64_t *pData = (uint64_t *) (size_t) (uint32_t) q->p_data; + uint64_t *ptr = pInx + pData; + + sq_write(ptr + 0, a); + sq_write(ptr + 1, b); + + q->p_data = (pDataRaw + (2ULL << 32)) & HIGH_QMASK; + + // if(!(q->p_data & HIGH_CHUNKMASK)) { + // sq_waitConsumer(q, cb, cbd); + // } +} + +#define sq_produce2(Q,A,B) sq_produce2(Q, A, B, (sq_callback) sq_flushQueue, Q) + + +Inline void sq_produceDouble(SW_Queue q, double value, sq_callback cb, sq_cbData cbd) { + uint64_t pDataRaw = q->p_data; + uint32_t pInx = sq_pInx(q); + uint64_t *pData = (uint64_t *) (size_t) (uint32_t) q->p_data; + uint64_t *ptr = pInx + pData; + uint64_t valueInt = (uint64_t)doubleToInt(value); + sq_write(ptr, valueInt); + q->p_data = (pDataRaw + (1ULL << 32)) & HIGH_QMASK; + + if(!(q->p_data & HIGH_CHUNKMASK)) { + sq_waitConsumer(q, cb, cbd); + } +} + +#define sq_produceDouble(Q, V) sq_produceDouble(Q, V, (sq_callback) sq_flushQueue, Q) + +/* Functions for Consumer */ + +/* + * Makes reads globally visible + */ + +Inline void sq_reverseFlush(const SW_Queue q) { + *q->ptr_c_glb_inx = q->c_inx; +} + +/* + * Wait for a produce + */ +Inline uint32_t sq_waitAllocated(const SW_Queue q, sq_callback cb, sq_cbData cbd) +{ + if(*q->ptr_p_glb_inx == q->c_inx) { + + /* Blocking path */ + cb(cbd); + while(*q->ptr_p_glb_inx == q->c_inx) usleep(10); + + } else { + /* Fast path */ + sq_reverseFlush(q); + } + + return *q->ptr_p_glb_inx; +} + +/* + * Consume a 64-bit value from the queue. + */ +Inline uint64_t sq_consume(SW_Queue q, sq_callback cb, sq_cbData cbd) +{ +#ifdef INSTRUMENT + q->total_consumes++; +#endif + + if(q->c_inx == q->c_margin) { + q->c_margin = sq_waitAllocated(q, cb, cbd); + } + + uint64_t val = q->data[q->c_inx]; + + if(QPREFETCH) { + _mm_prefetch(q->data + q->c_inx + QPREFETCH, _MM_HINT_T0); + } + + q->c_inx++; + q->c_inx &= QMASK; + return val; +} + +#define sq_consume(Q) sq_consume(Q, (sq_callback) sq_reverseFlush, Q) + + +Inline double sq_consumeDouble(SW_Queue q, sq_callback cb, sq_cbData cbd) +{ +#ifdef INSTRUMENT + q->total_consumes++; +#endif + + if(q->c_inx == q->c_margin) { + q->c_margin = sq_waitAllocated(q, cb, cbd); + } + + uint64_t val = q->data[q->c_inx]; + + if(QPREFETCH) { + _mm_prefetch(q->data + q->c_inx + QPREFETCH, _MM_HINT_T0); + } + + q->c_inx++; + q->c_inx &= QMASK; + double valDouble = intToDouble((int64_t)val); + return valDouble; +} + +#define sq_consumeDouble(Q) sq_consumeDouble(Q, (sq_callback) sq_reverseFlush, Q) + +Inline bool sq_canConsume(const SW_Queue q) { + return q->c_margin != q->c_inx || *q->ptr_p_glb_inx != q->c_inx; +} + +/* + * Empty the queue. + * + * Races with producer methods. + */ +Inline void sq_emptyQueue(SW_Queue q) { + q->c_margin = q->c_inx = *q->ptr_p_glb_inx; + sq_reverseFlush(q); +} + +/* + * Modulo arithmetic (llvmism) + */ +Inline bool sq_selectProducer(unsigned prevChoice, + SW_Queue *queues, + unsigned numQueues) { + (void) queues; + + if( ++prevChoice >= numQueues ) + prevChoice = 0; + + return prevChoice; +} + +/* + * Modulo arithmetic (llvmism) + */ +Inline bool sq_selectConsumer(unsigned prevChoice, + SW_Queue *queues, + unsigned numQueues) { + + (void) queues; + + if( ++prevChoice >= numQueues ) + prevChoice = 0; + + return prevChoice; +} + +#ifdef __cplusplus +} +#endif +#endif /* SW_QUEUE_H */ From 60a2fe13f5825a01ddb36a62bc28353d901998ac Mon Sep 17 00:00:00 2001 From: Ziyang Xu Date: Thu, 27 Oct 2022 13:56:13 -0400 Subject: [PATCH 27/97] disable inlining to avoid the bug --- liberty/lib/SLAMP/SLAMPcustom/CustomSend.cpp | 92 ++++++-- .../SLAMP/SLAMPcustom/consumer/CMakeLists.txt | 10 +- .../ProfilingModules/DependenceModule.cpp | 208 ++++++++++-------- .../SLAMP/SLAMPcustom/consumer/consumer.cpp | 1 + 4 files changed, 192 insertions(+), 119 deletions(-) diff --git a/liberty/lib/SLAMP/SLAMPcustom/CustomSend.cpp b/liberty/lib/SLAMP/SLAMPcustom/CustomSend.cpp index 48b07519..4e8fb5e5 100644 --- a/liberty/lib/SLAMP/SLAMPcustom/CustomSend.cpp +++ b/liberty/lib/SLAMP/SLAMPcustom/CustomSend.cpp @@ -15,6 +15,14 @@ namespace bip = boost::interprocess; +static unsigned long counter_load = 0; +static unsigned long counter_store = 0; +static unsigned long counter_ctx = 0; +static unsigned long counter_alloc = 0; +// char local_buffer[LOCAL_BUFFER_SIZE]; +// unsigned buffer_counter = 0; +static bool onProfiling = false; + static void *(*old_malloc_hook)(size_t, const void *); static void *(*old_realloc_hook)(void *, size_t, const void *); static void (*old_free_hook)(void *, const void *); @@ -26,6 +34,8 @@ static SW_Queue the_queue; static double_queue_p dqA, dqB, dq, dq_other; static uint64_t dq_index = 0; static uint64_t *dq_data; +static uint64_t total_pushed = 0; +static uint64_t total_swapped = 0; static void swap(){ if(dq == dqA){ dq = dqB; @@ -40,6 +50,7 @@ static void swap(){ static void produce_wait() ATTRIBUTE(noinline){ dq->size = dq_index; dq->ready_to_read = true; + dq->ready_to_write = false; while (!dq_other->ready_to_write){ // spin usleep(10); @@ -47,28 +58,31 @@ static void produce_wait() ATTRIBUTE(noinline){ swap(); dq->ready_to_read = false; dq_index = 0; + total_swapped++; } -static void produce(uint64_t x) {// ATTRIBUTE(always_inline) { +static void produce(uint64_t x) ATTRIBUTE(noinline) { if (dq_index == QSIZE){ produce_wait(); } dq_data[dq_index] = x; dq_index++; + total_pushed++; // _mm_prefetch(&dq_data[dq_index] + QPREFETCH, _MM_HINT_T0); } -static void produce_2(uint64_t x, uint64_t y) {// ATTRIBUTE(always_inline) { +static void produce_2(uint64_t x, uint64_t y) ATTRIBUTE(noinline) { if (dq_index + 2 >= QSIZE){ produce_wait(); } dq_data[dq_index] = x; dq_data[dq_index+1] = y; dq_index += 2; + total_pushed += 2; // _mm_prefetch(&dq_data[dq_index] + QPREFETCH, _MM_HINT_T0); } -static void produce_3(uint64_t x, uint64_t y, uint64_t z) {// ATTRIBUTE(always_inline) { +static void produce_3(uint64_t x, uint64_t y, uint64_t z) ATTRIBUTE(noinline) { if (dq_index + 3 >= QSIZE){ produce_wait(); } @@ -76,6 +90,7 @@ static void produce_3(uint64_t x, uint64_t y, uint64_t z) {// ATTRIBUTE(always_i dq_data[dq_index+1] = y; dq_data[dq_index+2] = z; dq_index += 3; + total_pushed += 3; // _mm_prefetch(&dq_data[dq_index] + QPREFETCH, _MM_HINT_T0); } @@ -88,6 +103,7 @@ static void produce_4(uint64_t x, uint64_t y, uint64_t z, uint64_t w) {// ATTRIB dq_data[dq_index+2] = z; dq_data[dq_index+3] = w; dq_index += 4; + total_pushed += 4; } // #define CONSUME sq_consume(the_queue); @@ -111,13 +127,6 @@ static void produce_4(uint64_t x, uint64_t y, uint64_t z, uint64_t w) {// ATTRIB // Ringbuffer fully constructed in shared memory. The element strings are // also allocated from the same shared memory segment. This vector can be // safely accessed from other processes. -unsigned long counter_load = 0; -unsigned long counter_store = 0; -unsigned long counter_ctx = 0; -unsigned long counter_alloc = 0; -// char local_buffer[LOCAL_BUFFER_SIZE]; -// unsigned buffer_counter = 0; -bool onProfiling = false; enum DepModAction: char { @@ -269,10 +278,43 @@ void SLAMP_load(const uint32_t instr, const uint64_t addr, const uint32_t bare_i if (onProfiling) { // local_buffer->push(LOAD)->push(instr)->push(addr)->push(bare_instr); //->push(value); produce_3(LOAD, COMBINE_2_32(instr, bare_instr), addr); - if (instr == 12356 && addr == 140724580372352) { - printf("SLAMP_load: %d, %lu, %d, %lu\n", instr, addr, bare_instr, value); - exit(-1); - } + // // if (counter_load % 10000000 == 0) { + // // printf("load: %lu\n", counter_load); + // // } + // // if (counter_load >= 113957060 && counter_load <= 113957064) { + // // printf("SLAMP_load: %d, %lu, %d, %lu\n", instr, addr, bare_instr, value); + // // printf("counter load %d\n", counter_load); + // // printf("counter store%d\n", counter_store); + // // printf("dq_data[%ld - 4] = %lu at addr %p\n", dq_index, dq_data[dq_index - 4], &dq_data[dq_index - 4]); + // // printf("dq_data[%ld - 3] = %lu at addr %p\n", dq_index, dq_data[dq_index - 3], &dq_data[dq_index - 3]); + // // printf("dq_data[%ld - 2] = %lu at addr %p\n", dq_index, dq_data[dq_index - 2], &dq_data[dq_index - 2]); + // // printf("dq_data[%ld - 1] = %lu at addr %p\n", dq_index, dq_data[dq_index - 1], &dq_data[dq_index - 1]); + // // printf("Total pushed: %ld\n", total_pushed); + // // printf("Total swapped: %ld\n", total_swapped); + // // } + + // if (dq_index > 2087456 && dq_index < 2087464) { + // // if (counter_load > 113957060 && counter_load < 113957064) + // // printf("SLAMP_load: %d, %lu, %d, %lu\n", instr, addr, bare_instr, value); + // // if (counter_load == 113957064) { + // // printf("SLAMP_load: %d, %lu, %d, %lu\n", instr, addr, bare_instr, value); + // // printf("counter load %d\n", counter_load); + // // } + // // printf("dq_index: %d\n", dq_index); + // // printf("dq: %p\n", dq); + // // printf("counter load %d\n", counter_load); + // if (instr == 12356 ) { + // printf("SLAMP_load: %d, %lu, %d, %lu\n", instr, addr, bare_instr, value); + // printf("counter load %d\n", counter_load); + // printf("Total pushed: %ld\n", total_pushed); + // printf("Total swapped: %ld\n", total_swapped); + // // printf("dq_data[%ld] = %lu at addr %p\n", dq_index, dq_data[dq_index], &dq_data[dq_index]); + // printf("dq_data[%ld - 4] = %lu at addr %p\n", dq_index, dq_data[dq_index - 4], &dq_data[dq_index - 4]); + // printf("dq_data[%ld - 3] = %lu at addr %p\n", dq_index, dq_data[dq_index - 3], &dq_data[dq_index - 3]); + // printf("dq_data[%ld - 2] = %lu at addr %p\n", dq_index, dq_data[dq_index - 2], &dq_data[dq_index - 2]); + // printf("dq_data[%ld - 1] = %lu at addr %p\n", dq_index, dq_data[dq_index - 1], &dq_data[dq_index - 1]); + // } + // } // PRODUCE(value); counter_load++; } @@ -322,6 +364,16 @@ void SLAMP_store(const uint32_t instr, const uint64_t addr, const uint32_t bare_ // PRODUCE(instr); // PRODUCE(bare_instr); produce_3(STORE, COMBINE_2_32(instr, bare_instr), addr); + // // if (counter_load >= 113957060 && counter_load <= 113957064) { + // if (counter_store == 32816445){ + // printf("SLAMP_store: %d, %lu, %d\n", instr, addr, bare_instr); + // printf("counter load %d\n", counter_load); + // printf("counter store%d\n", counter_store); + // printf("dq_data[%ld - 4] = %lu at addr %p\n", dq_index, dq_data[dq_index - 4], &dq_data[dq_index - 4]); + // printf("dq_data[%ld - 3] = %lu at addr %p\n", dq_index, dq_data[dq_index - 3], &dq_data[dq_index - 3]); + // printf("dq_data[%ld - 2] = %lu at addr %p\n", dq_index, dq_data[dq_index - 2], &dq_data[dq_index - 2]); + // printf("dq_data[%ld - 1] = %lu at addr %p\n", dq_index, dq_data[dq_index - 1], &dq_data[dq_index - 1]); + // } counter_store++; } } @@ -364,9 +416,7 @@ static void* SLAMP_malloc_hook(size_t size, const void *caller){ TURN_OFF_HOOKS void* ptr = malloc(size); // local_buffer->push(ALLOC)->push((uint64_t)ptr)->push(size); - PRODUCE(ALLOC); - PRODUCE((uint64_t)ptr); - PRODUCE(size); + produce_3(ALLOC, (uint64_t)ptr, size); // printf("malloc %lu at %p\n", size, ptr); counter_alloc++; TURN_ON_HOOKS @@ -377,9 +427,7 @@ static void* SLAMP_realloc_hook(void* ptr, size_t size, const void *caller){ TURN_OFF_HOOKS void* new_ptr = realloc(ptr, size); // local_buffer->push(REALLOC)->push((uint64_t)ptr)->push((uint64_t)new_ptr)->push(size); - PRODUCE(ALLOC); - PRODUCE((uint64_t)new_ptr); - PRODUCE(size); + produce_3(ALLOC, (uint64_t)new_ptr, size); // printf("realloc %p to %lu at %p", ptr, size, new_ptr); counter_alloc++; TURN_ON_HOOKS @@ -393,9 +441,7 @@ static void* SLAMP_memalign_hook(size_t alignment, size_t size, const void *call TURN_OFF_HOOKS void* ptr = memalign(alignment, size); // local_buffer->push(ALLOC)->push((uint64_t)ptr)->push(size); - PRODUCE(ALLOC); - PRODUCE((uint64_t)ptr); - PRODUCE(size); + produce_3(ALLOC, (uint64_t)ptr, size); // printf("memalign %lu at %p\n", size, ptr); counter_alloc++; diff --git a/liberty/lib/SLAMP/SLAMPcustom/consumer/CMakeLists.txt b/liberty/lib/SLAMP/SLAMPcustom/consumer/CMakeLists.txt index c28d61e2..2fa3956a 100644 --- a/liberty/lib/SLAMP/SLAMPcustom/consumer/CMakeLists.txt +++ b/liberty/lib/SLAMP/SLAMPcustom/consumer/CMakeLists.txt @@ -1,15 +1,15 @@ cmake_minimum_required (VERSION 3.6.2 FATAL_ERROR) -# set(CMAKE_C_COMPILER "clang") -# set(CMAKE_CXX_COMPILER "clang++") -# set(CMAKE_LINKER "ld.gold") +set(CMAKE_C_COMPILER "clang") +set(CMAKE_CXX_COMPILER "clang++") +set(CMAKE_LINKER "ld.gold") set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} "/u/ziyangx/test/boost/boost_1_80_0/install/lib/cmake") set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) -# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -flto") -# set(LINK_FLAGS "${LINK_FLAGS} -flto") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -flto -Ofast") +set(LINK_FLAGS "${LINK_FLAGS} -flto -O3") find_package(Boost 1.80.0 REQUIRED COMPONENTS system) include_directories(${Boost_INCLUDE_DIRS}) diff --git a/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.cpp b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.cpp index a5045b13..105143f3 100644 --- a/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.cpp +++ b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.cpp @@ -29,7 +29,7 @@ static uint64_t slamp_invocation = 0; static uint32_t target_loop_id = 0; struct LoadStoreEvent { - bool isLoad; + // bool isLoad; uint64_t addr; // uint64_t value; TS ts; @@ -173,120 +173,146 @@ void handleLoadAndStore(std::vector *loadstore_vec) { std::thread t[LOCALWRITE_THREADS]; for (auto i = 0; i < LOCALWRITE_THREADS; i++) { - t[i] = std::thread( - [&](const unsigned thread_id) { - for (auto &e : *loadstore_vec) { + unsigned begin = i * (loadstore_vec_counter / LOCALWRITE_THREADS); + unsigned end = (i + 1) * (loadstore_vec_counter / LOCALWRITE_THREADS); + unsigned thread_id = 0; + // t[i] = std::thread( + // [&](const unsigned thread_id, const unsigned begin, const unsigned end) { + for (auto i = begin; i < end; i++) { + auto &e = (*loadstore_vec)[i]; const auto &addr = e.addr; const auto &ts = e.ts; const auto invoc = GET_INVOC(ts); const auto iter = GET_ITER(ts); const auto instr = GET_INSTR(ts); - unsigned addr_mod = (addr >> 3) % LOCALWRITE_THREADS; - if (thread_id != addr_mod) { - continue; - } TS *s = (TS *)GET_SHADOW(addr, TIMESTAMP_SIZE_IN_POWER_OF_TWO); - if (e.isLoad) { - // load - // std::cout << "load " << e.addr << " " << e.value << " " << - // e.instr - // << " " << e.bare_instr << std::endl; smmap->load(e.addr, - // e.value, e.instr, e.bare_instr); - - // std::cout << "load " << instr << " " << addr << " " << - // bare_instr - // << " " - // << value << std::endl; - TS tss = s[0]; - if (tss != 0) { - log(thread_id, tss, instr, instr, invoc, iter); - } - } else { - // store - // std::cout << "store " << e.addr << " " << e.value << " " << - // e.instr - // << " " << e.bare_instr << std::endl; smmap->store(e.addr, - // e.value, e.instr, e.bare_instr); - // if (!smmap->is_allocated(s)) { - // std::cout << "store not allocated: " << instr << " " << - // bare_instr << " " << addr << std::endl; - // } - // TODO: handle output dependence. ignore it as of now. - // if (ASSUME_ONE_ADDR) { - s[0] = ts; - // } else { - // for (auto i = 0; i < size; i++) - // s[i] = ts; - // } + // if (e.isLoad) { + // load + // std::cout << "load " << e.addr << " " << e.value << " " << + // e.instr + // << " " << e.bare_instr << std::endl; smmap->load(e.addr, + // e.value, e.instr, e.bare_instr); + + // std::cout << "load " << instr << " " << addr << " " << + // bare_instr + // << " " + // << value << std::endl; + TS tss = s[0]; + if (tss != 0) { + log(thread_id, tss, instr, instr, invoc, iter); } + // } + // else { + // // store + // // std::cout << "store " << e.addr << " " << e.value << " " << + // // e.instr + // // << " " << e.bare_instr << std::endl; smmap->store(e.addr, + // // e.value, e.instr, e.bare_instr); + // // if (!smmap->is_allocated(s)) { + // // std::cout << "store not allocated: " << instr << " " << + // // bare_instr << " " << addr << std::endl; + // // } + // // TODO: handle output dependence. ignore it as of now. + // // if (ASSUME_ONE_ADDR) { + // s[0] = ts; + // // } else { + // // for (auto i = 0; i < size; i++) + // // s[i] = ts; + // // } + // } } - }, - i); + // }, i, begin, end); } - for (auto & i : t) { - i.join(); - } - mutex_process_load_store.unlock(); + // for (auto & i : t) { + // i.join(); + // } + // mutex_process_load_store.unlock(); } // template void load(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value) { - - // auto start = std::chrono::high_resolution_clock::now(); - loadstore_vec->emplace_back(LoadStoreEvent{true, addr, CREATE_TS(instr, slamp_iteration, slamp_invocation)}); - loadstore_vec_counter++; - // auto end = std::chrono::high_resolution_clock::now(); - // auto duration = std::chrono::duration_cast(end - start); - // loadstore_vec_time += duration; - if (loadstore_vec_counter == MAX_EVENTS - 1) { - - // measure time in here - auto start = std::chrono::high_resolution_clock::now(); - - // run this asynchrously - mutex_process_load_store.lock(); - std::thread t = std::thread(handleLoadAndStore, loadstore_vec); - t.detach(); - - // swap double buffer - loadstore_vec = loadstore_vec == loadstore_vec0 ? loadstore_vec1 : loadstore_vec0; + TS *s = (TS *)GET_SHADOW(addr, TIMESTAMP_SIZE_IN_POWER_OF_TWO); + // if (e.isLoad) { + // load + // std::cout << "load " << e.addr << " " << e.value << " " << + // e.instr + // << " " << e.bare_instr << std::endl; smmap->load(e.addr, + // e.value, e.instr, e.bare_instr); + + // std::cout << "load " << instr << " " << addr << " " << + // bare_instr + // << " " + // << value << std::endl; + TS tss = s[0]; + if (tss != 0) { + log(0, tss, instr, instr, slamp_invocation, slamp_iteration); + } + // // auto start = std::chrono::high_resolution_clock::now(); + // loadstore_vec->emplace_back(LoadStoreEvent{addr, CREATE_TS(instr, slamp_iteration, slamp_invocation)}); + // loadstore_vec_counter++; + // // auto end = std::chrono::high_resolution_clock::now(); + // // auto duration = std::chrono::duration_cast(end - start); + // // loadstore_vec_time += duration; + // if (loadstore_vec_counter == MAX_EVENTS - 1) { + // handleLoadAndStore(loadstore_vec); + + // // // measure time in here + // // auto start = std::chrono::high_resolution_clock::now(); + + // // // run this asynchrously + // // mutex_process_load_store.lock(); + // // std::thread t = std::thread(handleLoadAndStore, loadstore_vec); + // // t.detach(); + + // // // swap double buffer + // // loadstore_vec = loadstore_vec == loadstore_vec0 ? loadstore_vec1 : loadstore_vec0; - loadstore_vec->resize(0); - loadstore_vec_counter = 0; + // // loadstore_vec->resize(0); + // // loadstore_vec_counter = 0; - auto end = std::chrono::high_resolution_clock::now(); - auto duration = std::chrono::duration_cast(end - start); - total_time_off_critical_path += duration; - } + // // auto end = std::chrono::high_resolution_clock::now(); + // // auto duration = std::chrono::duration_cast(end - start); + // // total_time_off_critical_path += duration; + // } } // template void store(uint32_t instr, uint32_t bare_instr, const uint64_t addr) { - // auto start = std::chrono::high_resolution_clock::now(); - loadstore_vec->emplace_back(LoadStoreEvent{false, addr, CREATE_TS(instr, slamp_iteration, slamp_invocation)}); - loadstore_vec_counter++; - // auto end = std::chrono::high_resolution_clock::now(); - // auto duration = std::chrono::duration_cast(end - start); - // loadstore_vec_time += duration; - if (loadstore_vec_counter == MAX_EVENTS - 1) { - // measure time in here - auto start = std::chrono::high_resolution_clock::now(); - mutex_process_load_store.lock(); - std::thread t = std::thread(handleLoadAndStore, loadstore_vec); - t.detach(); - loadstore_vec = loadstore_vec == loadstore_vec0 ? loadstore_vec1 : loadstore_vec0; - - loadstore_vec->resize(0); - loadstore_vec_counter = 0; - auto end = std::chrono::high_resolution_clock::now(); - auto duration = std::chrono::duration_cast(end - start); - total_time_off_critical_path += duration; - } + // if (loadstore_vec_counter > 0) { + // handleLoadAndStore(loadstore_vec); + // loadstore_vec->resize(0); + // loadstore_vec_counter = 0; + // } + + TS ts = CREATE_TS(instr, slamp_iteration, slamp_invocation); + TS *shadow_addr = (TS *)GET_SHADOW(addr, TIMESTAMP_SIZE_IN_POWER_OF_TWO); + shadow_addr[0] = ts; + + // // auto start = std::chrono::high_resolution_clock::now(); + // loadstore_vec->emplace_back(LoadStoreEvent{false, addr, CREATE_TS(instr, slamp_iteration, slamp_invocation)}); + // loadstore_vec_counter++; + // // auto end = std::chrono::high_resolution_clock::now(); + // // auto duration = std::chrono::duration_cast(end - start); + // // loadstore_vec_time += duration; + // if (loadstore_vec_counter == MAX_EVENTS - 1) { + // // measure time in here + // auto start = std::chrono::high_resolution_clock::now(); + // mutex_process_load_store.lock(); + // std::thread t = std::thread(handleLoadAndStore, loadstore_vec); + // t.detach(); + // loadstore_vec = loadstore_vec == loadstore_vec0 ? loadstore_vec1 : loadstore_vec0; + + // loadstore_vec->resize(0); + // loadstore_vec_counter = 0; + // auto end = std::chrono::high_resolution_clock::now(); + // auto duration = std::chrono::duration_cast(end - start); + // total_time_off_critical_path += duration; + // } } void loop_invoc() { diff --git a/liberty/lib/SLAMP/SLAMPcustom/consumer/consumer.cpp b/liberty/lib/SLAMP/SLAMPcustom/consumer/consumer.cpp index 31e0604d..714a8064 100644 --- a/liberty/lib/SLAMP/SLAMPcustom/consumer/consumer.cpp +++ b/liberty/lib/SLAMP/SLAMPcustom/consumer/consumer.cpp @@ -36,6 +36,7 @@ static void swap(){ static void check() { if (dq_index == dq_size){ + dq->ready_to_read = false; dq->ready_to_write = true; while (!dq_other->ready_to_read){ // spin From d90ffc036c7d4176169b79cee19f0a9f9112f96c Mon Sep 17 00:00:00 2001 From: Ziyang Xu Date: Mon, 31 Oct 2022 13:39:37 -0400 Subject: [PATCH 28/97] update smtx version to match custom --- .../SLAMP/SLAMPsmtxq/consumer/CMakeLists.txt | 10 +- .../ProfilingModules/DependenceModule.cpp | 209 +++++++++++------- .../lib/SLAMP/SLAMPsmtxq/sw_queue_astream.h | 6 +- tests/scripts/slamp-driver | 36 ++- 4 files changed, 172 insertions(+), 89 deletions(-) diff --git a/liberty/lib/SLAMP/SLAMPsmtxq/consumer/CMakeLists.txt b/liberty/lib/SLAMP/SLAMPsmtxq/consumer/CMakeLists.txt index a043216a..74a6c20f 100644 --- a/liberty/lib/SLAMP/SLAMPsmtxq/consumer/CMakeLists.txt +++ b/liberty/lib/SLAMP/SLAMPsmtxq/consumer/CMakeLists.txt @@ -1,15 +1,15 @@ cmake_minimum_required (VERSION 3.6.2 FATAL_ERROR) -# set(CMAKE_C_COMPILER "clang") -# set(CMAKE_CXX_COMPILER "clang++") -# set(CMAKE_LINKER "ld.gold") +set(CMAKE_C_COMPILER "clang") +set(CMAKE_CXX_COMPILER "clang++") +set(CMAKE_LINKER "ld.gold") set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} "/u/ziyangx/test/boost/boost_1_80_0/install/lib/cmake") set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) -# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -flto") -# set(LINK_FLAGS "${LINK_FLAGS} -flto") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -flto -Ofast") +set(LINK_FLAGS "${LINK_FLAGS} -flto -O3") find_package(Boost 1.80.0 REQUIRED COMPONENTS system) include_directories(${Boost_INCLUDE_DIRS}) diff --git a/liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/DependenceModule.cpp b/liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/DependenceModule.cpp index edef1fbe..105143f3 100644 --- a/liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/DependenceModule.cpp +++ b/liberty/lib/SLAMP/SLAMPsmtxq/consumer/ProfilingModules/DependenceModule.cpp @@ -19,7 +19,7 @@ std::mutex localwrite_mutexes[LOCALWRITE_THREADS]; #define SIZE_8M 0x800000 -#define DEPLOG_VEC_SIZE 100'000'000 +#define DEPLOG_VEC_SIZE 1'000'000 static slamp::MemoryMap* smmap = nullptr; static std::unordered_set *deplog_set; @@ -29,7 +29,7 @@ static uint64_t slamp_invocation = 0; static uint32_t target_loop_id = 0; struct LoadStoreEvent { - bool isLoad; + // bool isLoad; uint64_t addr; // uint64_t value; TS ts; @@ -38,10 +38,14 @@ struct LoadStoreEvent { // uint64_t invocation; // uint64_t iteration; }; -constexpr unsigned MAX_EVENTS = 100'000'000; +// constexpr unsigned MAX_EVENTS = 100'000'000; +constexpr unsigned MAX_EVENTS = 100'000; static std::vector *loadstore_vec, *loadstore_vec0, *loadstore_vec1; static uint64_t loadstore_vec_counter = 0; +static std::chrono::duration total_time_off_critical_path(0); +static std::chrono::duration loadstore_vec_time(0); + static void convertVectorToSet(const unsigned thread_id) { // launch 56 threads to convert the vector to set independently, chunking @@ -74,8 +78,8 @@ static void convertVectorToSet(const unsigned thread_id) { i.join(); } - std::cout << "Merging vec to set, set length " << deplog_set->size() - << std::endl; + // std::cout << "Merging vec to set, set length " << deplog_set->size() + // << std::endl; } @@ -103,6 +107,14 @@ void init(uint32_t loop_id, uint32_t pid) { void handleLoadAndStore(std::vector *loadstore_vec); void fini(const char *filename) { + // show in seconds + std::cout << "Total time off critical path: " + << total_time_off_critical_path.count() / 1000000.0 << "s" + << std::endl; + + std::cout << "Loadstore vec time: " + << loadstore_vec_time.count() / 1000000.0 << "s" + << std::endl; mutex_process_load_store.lock(); handleLoadAndStore(loadstore_vec); @@ -161,97 +173,146 @@ void handleLoadAndStore(std::vector *loadstore_vec) { std::thread t[LOCALWRITE_THREADS]; for (auto i = 0; i < LOCALWRITE_THREADS; i++) { - t[i] = std::thread( - [&](const unsigned thread_id) { - for (auto &e : *loadstore_vec) { + unsigned begin = i * (loadstore_vec_counter / LOCALWRITE_THREADS); + unsigned end = (i + 1) * (loadstore_vec_counter / LOCALWRITE_THREADS); + unsigned thread_id = 0; + // t[i] = std::thread( + // [&](const unsigned thread_id, const unsigned begin, const unsigned end) { + for (auto i = begin; i < end; i++) { + auto &e = (*loadstore_vec)[i]; const auto &addr = e.addr; const auto &ts = e.ts; const auto invoc = GET_INVOC(ts); const auto iter = GET_ITER(ts); const auto instr = GET_INSTR(ts); - unsigned addr_mod = (addr >> 3) % LOCALWRITE_THREADS; - if (thread_id != addr_mod) { - continue; - } TS *s = (TS *)GET_SHADOW(addr, TIMESTAMP_SIZE_IN_POWER_OF_TWO); - if (e.isLoad) { - // load - // std::cout << "load " << e.addr << " " << e.value << " " << - // e.instr - // << " " << e.bare_instr << std::endl; smmap->load(e.addr, - // e.value, e.instr, e.bare_instr); - - // std::cout << "load " << instr << " " << addr << " " << - // bare_instr - // << " " - // << value << std::endl; - TS tss = s[0]; - if (tss != 0) { - log(thread_id, tss, instr, instr, invoc, iter); - } - } else { - // store - // std::cout << "store " << e.addr << " " << e.value << " " << - // e.instr - // << " " << e.bare_instr << std::endl; smmap->store(e.addr, - // e.value, e.instr, e.bare_instr); - // if (!smmap->is_allocated(s)) { - // std::cout << "store not allocated: " << instr << " " << - // bare_instr << " " << addr << std::endl; - // } - // TODO: handle output dependence. ignore it as of now. - // if (ASSUME_ONE_ADDR) { - s[0] = ts; - // } else { - // for (auto i = 0; i < size; i++) - // s[i] = ts; - // } + // if (e.isLoad) { + // load + // std::cout << "load " << e.addr << " " << e.value << " " << + // e.instr + // << " " << e.bare_instr << std::endl; smmap->load(e.addr, + // e.value, e.instr, e.bare_instr); + + // std::cout << "load " << instr << " " << addr << " " << + // bare_instr + // << " " + // << value << std::endl; + TS tss = s[0]; + if (tss != 0) { + log(thread_id, tss, instr, instr, invoc, iter); } + // } + // else { + // // store + // // std::cout << "store " << e.addr << " " << e.value << " " << + // // e.instr + // // << " " << e.bare_instr << std::endl; smmap->store(e.addr, + // // e.value, e.instr, e.bare_instr); + // // if (!smmap->is_allocated(s)) { + // // std::cout << "store not allocated: " << instr << " " << + // // bare_instr << " " << addr << std::endl; + // // } + // // TODO: handle output dependence. ignore it as of now. + // // if (ASSUME_ONE_ADDR) { + // s[0] = ts; + // // } else { + // // for (auto i = 0; i < size; i++) + // // s[i] = ts; + // // } + // } } - }, - i); + // }, i, begin, end); } - for (auto & i : t) { - i.join(); - } - mutex_process_load_store.unlock(); + // for (auto & i : t) { + // i.join(); + // } + // mutex_process_load_store.unlock(); } + // template void load(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value) { - loadstore_vec->emplace_back(LoadStoreEvent{true, addr, CREATE_TS(instr, slamp_iteration, slamp_invocation)}); - loadstore_vec_counter++; - if (loadstore_vec_counter == MAX_EVENTS - 1) { - - // run this asynchrously - mutex_process_load_store.lock(); - std::thread t = std::thread(handleLoadAndStore, loadstore_vec); - t.detach(); - // swap double buffer - loadstore_vec = loadstore_vec == loadstore_vec0 ? loadstore_vec1 : loadstore_vec0; - - loadstore_vec->resize(0); - loadstore_vec_counter = 0; + TS *s = (TS *)GET_SHADOW(addr, TIMESTAMP_SIZE_IN_POWER_OF_TWO); + // if (e.isLoad) { + // load + // std::cout << "load " << e.addr << " " << e.value << " " << + // e.instr + // << " " << e.bare_instr << std::endl; smmap->load(e.addr, + // e.value, e.instr, e.bare_instr); + + // std::cout << "load " << instr << " " << addr << " " << + // bare_instr + // << " " + // << value << std::endl; + TS tss = s[0]; + if (tss != 0) { + log(0, tss, instr, instr, slamp_invocation, slamp_iteration); } + // // auto start = std::chrono::high_resolution_clock::now(); + // loadstore_vec->emplace_back(LoadStoreEvent{addr, CREATE_TS(instr, slamp_iteration, slamp_invocation)}); + // loadstore_vec_counter++; + // // auto end = std::chrono::high_resolution_clock::now(); + // // auto duration = std::chrono::duration_cast(end - start); + // // loadstore_vec_time += duration; + // if (loadstore_vec_counter == MAX_EVENTS - 1) { + // handleLoadAndStore(loadstore_vec); + + // // // measure time in here + // // auto start = std::chrono::high_resolution_clock::now(); + + // // // run this asynchrously + // // mutex_process_load_store.lock(); + // // std::thread t = std::thread(handleLoadAndStore, loadstore_vec); + // // t.detach(); + + // // // swap double buffer + // // loadstore_vec = loadstore_vec == loadstore_vec0 ? loadstore_vec1 : loadstore_vec0; + + // // loadstore_vec->resize(0); + // // loadstore_vec_counter = 0; + + // // auto end = std::chrono::high_resolution_clock::now(); + // // auto duration = std::chrono::duration_cast(end - start); + // // total_time_off_critical_path += duration; + // } } // template void store(uint32_t instr, uint32_t bare_instr, const uint64_t addr) { - loadstore_vec->emplace_back(LoadStoreEvent{false, addr, CREATE_TS(instr, slamp_iteration, slamp_invocation)}); - loadstore_vec_counter++; - if (loadstore_vec_counter == MAX_EVENTS - 1) { - mutex_process_load_store.lock(); - std::thread t = std::thread(handleLoadAndStore, loadstore_vec); - t.detach(); - loadstore_vec = loadstore_vec == loadstore_vec0 ? loadstore_vec1 : loadstore_vec0; - - loadstore_vec->resize(0); - loadstore_vec_counter = 0; - } + // if (loadstore_vec_counter > 0) { + // handleLoadAndStore(loadstore_vec); + // loadstore_vec->resize(0); + // loadstore_vec_counter = 0; + // } + + TS ts = CREATE_TS(instr, slamp_iteration, slamp_invocation); + TS *shadow_addr = (TS *)GET_SHADOW(addr, TIMESTAMP_SIZE_IN_POWER_OF_TWO); + shadow_addr[0] = ts; + + // // auto start = std::chrono::high_resolution_clock::now(); + // loadstore_vec->emplace_back(LoadStoreEvent{false, addr, CREATE_TS(instr, slamp_iteration, slamp_invocation)}); + // loadstore_vec_counter++; + // // auto end = std::chrono::high_resolution_clock::now(); + // // auto duration = std::chrono::duration_cast(end - start); + // // loadstore_vec_time += duration; + // if (loadstore_vec_counter == MAX_EVENTS - 1) { + // // measure time in here + // auto start = std::chrono::high_resolution_clock::now(); + // mutex_process_load_store.lock(); + // std::thread t = std::thread(handleLoadAndStore, loadstore_vec); + // t.detach(); + // loadstore_vec = loadstore_vec == loadstore_vec0 ? loadstore_vec1 : loadstore_vec0; + + // loadstore_vec->resize(0); + // loadstore_vec_counter = 0; + // auto end = std::chrono::high_resolution_clock::now(); + // auto duration = std::chrono::duration_cast(end - start); + // total_time_off_critical_path += duration; + // } } void loop_invoc() { diff --git a/liberty/lib/SLAMP/SLAMPsmtxq/sw_queue_astream.h b/liberty/lib/SLAMP/SLAMPsmtxq/sw_queue_astream.h index 8fd981e4..571201fe 100644 --- a/liberty/lib/SLAMP/SLAMPsmtxq/sw_queue_astream.h +++ b/liberty/lib/SLAMP/SLAMPsmtxq/sw_queue_astream.h @@ -189,7 +189,8 @@ Inline void sq_waitConsumer(SW_Queue q, sq_callback cb, sq_cbData cbd) /* * Produce a value to a queue. */ -Inline void sq_produce(SW_Queue q, uint64_t value, +// Inline void sq_produce(SW_Queue q, uint64_t value, +void sq_produce(SW_Queue q, uint64_t value, sq_callback cb, sq_cbData cbd) { #ifdef INSTRUMENT @@ -214,7 +215,8 @@ Inline void sq_produce(SW_Queue q, uint64_t value, /* * Produce two values to a queue. Do not intermingle with sq_produce. */ -Inline void sq_produce2(SW_Queue q, uint64_t a, uint64_t b, +// Inline void sq_produce2(SW_Queue q, uint64_t a, uint64_t b, +void sq_produce2(SW_Queue q, uint64_t a, uint64_t b, sq_callback cb, sq_cbData cbd) { #ifdef INSTRUMENT diff --git a/tests/scripts/slamp-driver b/tests/scripts/slamp-driver index 0a3097a7..4df1455b 100755 --- a/tests/scripts/slamp-driver +++ b/tests/scripts/slamp-driver @@ -45,7 +45,8 @@ function drive { -load $LIBERTY_LIBS_DIR/libSLAMP.so -basic-loop-aa -scev-loop-aa -auto-restrict-aa -intrinsic-aa -global-malloc-aa -pure-fun-aa -semi-local-fun-aa -phi-maze-aa -no-capture-global-aa -no-capture-src-aa -type-aa -no-escape-fields-aa -acyclic-aa -disjoint-fields-aa -field-malloc-aa -loop-variant-allocation-aa -std-in-out-err-aa -array-of-structures-aa -kill-flow-aa -callsite-depth-combinator-aa -unique-access-paths-aa $EXTRA -llvm-aa-results -pdgbuilder " - local SLAMP_HOOKS="$LIBERTY_LIBS_DIR/libslamp_hooks_stats.a" + # local SLAMP_HOOKS="$LIBERTY_LIBS_DIR/libslamp_hooks_stats.a" + local SLAMP_HOOKS="/u/NAS_SCRATCH/ziyangx/cpf-workspace-9-master/build/cpf-build-release/lib/SLAMP/SLAMPcustom/libslamp_hooks_custom.a" local SLAMP_OUTFILE="$2-$3.result.slamp.profile" local OPTS="-slamp-insts -slamp-target-fn=$2 -slamp-target-loop=$3 $EXTRA_FLAGS -slamp-outfile=$SLAMP_OUTFILE" @@ -76,7 +77,8 @@ function drive { # local CMD3="clang++ -no-pie -O2 $PRELINK_OBJ $SLAMP_HOOKS -o $EXE -g $DEFAULT_LDFLAGS -lunwind $DEFAULT_LIBS -ldl -lutil" #local CMD3="g++ -Og $PRELINK_OBJ $SLAMP_HOOKS -o $EXE -g $DEFAULT_LDFLAGS -lunwind $DEFAULT_LIBS -ldl -lutil" - RAW_CMD3="clang++ -no-pie -O2 $PRELINK_OBJ $SLAMP_HOOKS -g $LINKING_OPTS -lunwind $DEFAULT_LIBS -ldl -lutil" + # RAW_CMD3="clang++ -no-pie -O2 $PRELINK_OBJ $SLAMP_HOOKS -g $LINKING_OPTS -lunwind $DEFAULT_LIBS -ldl -lutil" + RAW_CMD3="clang++ -no-pie -O2 $PRELINK_OBJ $SLAMP_HOOKS -g $LINKING_OPTS -lunwind $DEFAULT_LIBS -ldl -lutil -lpthread -lrt " if [[ x$LTO != x ]]; then RAW_CMD3+=" -flto" fi @@ -93,7 +95,7 @@ function drive { echo -e "#include \nconst char LOCALWRITE_MODULE=0;const size_t LOCALWRITE_MASK=0; const size_t LOCALWRITE_PATTERN=0;" | clang -x c -c -o constants.o - fi # CMD3="clang++ -flto -no-pie -O2 $PRELINK_OBJ $SLAMP_HOOKS constants.o -o $EXE -g $LINKING_OPTS -lunwind $DEFAULT_LIBS -ldl -lutil" - CMD3="clang++ -no-pie -O2 $PRELINK_OBJ $SLAMP_HOOKS -o $EXE -g $LINKING_OPTS -lunwind $DEFAULT_LIBS -ldl -lutil" + CMD3="clang++ -no-pie -O2 $PRELINK_OBJ $SLAMP_HOOKS -o $EXE -g $LINKING_OPTS -lunwind $DEFAULT_LIBS -ldl -lutil -lpthread -lrt" #CMD3="clang++ -no-pie -O2 $PRELINK_OBJ $SLAMP_HOOKS -mllvm -inline-threshold=5000 -o $EXE -g $LINKING_OPTS -lunwind $DEFAULT_LIBS -ldl -lutil" fi @@ -128,11 +130,29 @@ function drive { echo -e "${red} --- Run Simulator...${nc}" echo $CMD4 #time $CMD4 > /dev/null 2>&1 - eval time $CMD4 - if [[ $? -ne 0 ]]; then - echo -e "${red} --- SIMULATION FAILED${nc}" - rm -f result.slamp.profile - exit 1 + + CMD4="taskset --cpu-list 7-8 $CMD4" + rm /dev/shm/MySharedMemory2 + rm /dev/shm/MySharedMemory + taskset --cpu-list 0-6,9-27 /u/NAS_SCRATCH/ziyangx/cpf-workspace-9-master/cpf/liberty/lib/SLAMP/SLAMPcustom/consumer/build/consumer & + CONSUME_PID=$! + RUN_TIMEOUT=14400 # 4 hours + echo "timeout $RUN_TIMEOUT $CMD4" + eval regressions-watchdog $RUN_TIMEOUT slamp-custom.time $CMD4 + # if command fail + if [ $? -ne 0 ]; then + # print in red + kill -9 $CONSUME_PID + echo -e "${red} --- SIMULATION FAILED${nc}" + echo "Timed out or other problems" + rm -f result.slamp.profile + exit 1 + else + # sleep for 20 seconds + sleep 20 + cat deplog*.txt >> result.slamp.profile + rm -f deplog*.txt + kill -9 $CONSUME_PID fi if [[ x$LOCALWRITE_THREADS != x ]]; then From fb1d2744f9277a095f035b5ce8f94e0a103ab690 Mon Sep 17 00:00:00 2001 From: Ziyang Xu Date: Mon, 31 Oct 2022 13:42:00 -0400 Subject: [PATCH 29/97] optimize consumer load value --- .../SLAMP/SLAMPcustom/consumer/consumer.cpp | 33 +++++++++++++++---- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/liberty/lib/SLAMP/SLAMPcustom/consumer/consumer.cpp b/liberty/lib/SLAMP/SLAMPcustom/consumer/consumer.cpp index 714a8064..30cf2550 100644 --- a/liberty/lib/SLAMP/SLAMPcustom/consumer/consumer.cpp +++ b/liberty/lib/SLAMP/SLAMPcustom/consumer/consumer.cpp @@ -56,6 +56,24 @@ static inline uint64_t consume(){ return ret; } +static inline void consume_64_64(uint64_t &x, uint64_t &y){ + x = dq_data[dq_index]; + dq_index++; + y = dq_data[dq_index]; + dq_index++; + // _mm_prefetch(&dq_data[dq_index] + QPREFETCH, _MM_HINT_NTA); +} + +static inline void consume_32_32_64(uint32_t &x, uint32_t &y, uint64_t &z){ + uint64_t tmp = dq_data[dq_index]; + dq_index++; + x = (tmp >> 32) & 0xFFFFFFFF; + y = tmp & 0xFFFFFFFF; + z = dq_data[dq_index]; + dq_index++; + // _mm_prefetch(&dq_data[dq_index] + QPREFETCH, _MM_HINT_NTA); +} + // #define CONSUME sq_consume(the_queue); #define CONSUME consume(); #define PRODUCE(x) sq_produce(the_queue,(uint64_t)x); @@ -173,8 +191,9 @@ int main(int argc, char** argv) { uint64_t addr; uint32_t bare_instr; uint64_t value = 0; - CONSUME_2(instr, bare_instr); - addr = CONSUME; + consume_32_32_64(instr, bare_instr, addr); + // CONSUME_2(instr, bare_instr); + // addr = CONSUME; // value = CONSUME; if (DEBUG) { std::cout << "LOAD: " << instr << " " << addr << " " << bare_instr @@ -191,8 +210,9 @@ int main(int argc, char** argv) { uint32_t instr; uint32_t bare_instr; uint64_t addr; - CONSUME_2(instr, bare_instr); - addr = CONSUME; + consume_32_32_64(instr, bare_instr, addr); + // CONSUME_2(instr, bare_instr); + // addr = CONSUME; if (DEBUG) { std::cout << "STORE: " << instr << " " << bare_instr << " " << addr << std::endl; @@ -205,8 +225,9 @@ int main(int argc, char** argv) { case Action::ALLOC: { uint64_t addr; uint64_t size; - addr = CONSUME; - size = CONSUME; + consume_64_64(addr, size); + // addr = CONSUME; + // size = CONSUME; if (DEBUG) { std::cout << "ALLOC: " << addr << " " << size << std::endl; } From b4ba12a4b377f1678740b6ebd20fdf952d303fd6 Mon Sep 17 00:00:00 2001 From: Ziyang Xu Date: Mon, 31 Oct 2022 13:43:06 -0400 Subject: [PATCH 30/97] add slamp smtx driver --- tests/scripts/slamp-driver | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/scripts/slamp-driver b/tests/scripts/slamp-driver index 4df1455b..03a103ae 100755 --- a/tests/scripts/slamp-driver +++ b/tests/scripts/slamp-driver @@ -47,6 +47,7 @@ function drive { # local SLAMP_HOOKS="$LIBERTY_LIBS_DIR/libslamp_hooks_stats.a" local SLAMP_HOOKS="/u/NAS_SCRATCH/ziyangx/cpf-workspace-9-master/build/cpf-build-release/lib/SLAMP/SLAMPcustom/libslamp_hooks_custom.a" + # local SLAMP_HOOKS="/u/NAS_SCRATCH/ziyangx/cpf-workspace-9-master/build/cpf-build-release/lib/SLAMP/SLAMPsmtxq/libslamp_hooks_smtx.a" local SLAMP_OUTFILE="$2-$3.result.slamp.profile" local OPTS="-slamp-insts -slamp-target-fn=$2 -slamp-target-loop=$3 $EXTRA_FLAGS -slamp-outfile=$SLAMP_OUTFILE" @@ -135,10 +136,12 @@ function drive { rm /dev/shm/MySharedMemory2 rm /dev/shm/MySharedMemory taskset --cpu-list 0-6,9-27 /u/NAS_SCRATCH/ziyangx/cpf-workspace-9-master/cpf/liberty/lib/SLAMP/SLAMPcustom/consumer/build/consumer & + # taskset --cpu-list 0-6,9-27 /u/NAS_SCRATCH/ziyangx/cpf-workspace-9-master/cpf/liberty/lib/SLAMP/SLAMPsmtxq/consumer/build/consumer & CONSUME_PID=$! RUN_TIMEOUT=14400 # 4 hours echo "timeout $RUN_TIMEOUT $CMD4" eval regressions-watchdog $RUN_TIMEOUT slamp-custom.time $CMD4 + # eval regressions-watchdog $RUN_TIMEOUT slamp-smtx.time $CMD4 # if command fail if [ $? -ne 0 ]; then # print in red From eebb835a406e0875a9bddcfe153dd858c663ac02 Mon Sep 17 00:00:00 2001 From: Ziyang Xu Date: Tue, 1 Nov 2022 10:38:03 -0400 Subject: [PATCH 31/97] add mm stream and prefetch to improve queue efficiency; increase the queue size to avoid seemingly cohenrency bug --- liberty/lib/SLAMP/SLAMPcustom/CustomSend.cpp | 24 ++++++++++++------- .../SLAMP/SLAMPcustom/consumer/consumer.cpp | 2 ++ .../lib/SLAMP/SLAMPcustom/sw_queue_astream.h | 2 +- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/liberty/lib/SLAMP/SLAMPcustom/CustomSend.cpp b/liberty/lib/SLAMP/SLAMPcustom/CustomSend.cpp index 4e8fb5e5..c8bea0f2 100644 --- a/liberty/lib/SLAMP/SLAMPcustom/CustomSend.cpp +++ b/liberty/lib/SLAMP/SLAMPcustom/CustomSend.cpp @@ -8,6 +8,7 @@ #include "malloc.h" #include +#include #include "sw_queue_astream.h" #include @@ -34,7 +35,7 @@ static SW_Queue the_queue; static double_queue_p dqA, dqB, dq, dq_other; static uint64_t dq_index = 0; static uint64_t *dq_data; -static uint64_t total_pushed = 0; +// static uint64_t total_pushed = 0; static uint64_t total_swapped = 0; static void swap(){ if(dq == dqA){ @@ -67,7 +68,7 @@ static void produce(uint64_t x) ATTRIBUTE(noinline) { } dq_data[dq_index] = x; dq_index++; - total_pushed++; + // total_pushed++; // _mm_prefetch(&dq_data[dq_index] + QPREFETCH, _MM_HINT_T0); } @@ -78,7 +79,7 @@ static void produce_2(uint64_t x, uint64_t y) ATTRIBUTE(noinline) { dq_data[dq_index] = x; dq_data[dq_index+1] = y; dq_index += 2; - total_pushed += 2; + // total_pushed += 2; // _mm_prefetch(&dq_data[dq_index] + QPREFETCH, _MM_HINT_T0); } @@ -86,11 +87,18 @@ static void produce_3(uint64_t x, uint64_t y, uint64_t z) ATTRIBUTE(noinline) { if (dq_index + 3 >= QSIZE){ produce_wait(); } - dq_data[dq_index] = x; - dq_data[dq_index+1] = y; - dq_data[dq_index+2] = z; + // dq_data[0] = x; + // dq_data[1] = y; + // dq_data[2] = z; + + _mm_stream_pi((__m64*)&dq_data[dq_index], (__m64)x); + _mm_stream_pi((__m64*)&dq_data[dq_index+1], (__m64)y); + _mm_stream_pi((__m64*)&dq_data[dq_index+2], (__m64)z); + // dq_data[dq_index] = x; + // dq_data[dq_index+1] = y; + // dq_data[dq_index+2] = z; dq_index += 3; - total_pushed += 3; + // total_pushed += 3; // _mm_prefetch(&dq_data[dq_index] + QPREFETCH, _MM_HINT_T0); } @@ -103,7 +111,7 @@ static void produce_4(uint64_t x, uint64_t y, uint64_t z, uint64_t w) {// ATTRIB dq_data[dq_index+2] = z; dq_data[dq_index+3] = w; dq_index += 4; - total_pushed += 4; + // total_pushed += 4; } // #define CONSUME sq_consume(the_queue); diff --git a/liberty/lib/SLAMP/SLAMPcustom/consumer/consumer.cpp b/liberty/lib/SLAMP/SLAMPcustom/consumer/consumer.cpp index 30cf2550..93793ceb 100644 --- a/liberty/lib/SLAMP/SLAMPcustom/consumer/consumer.cpp +++ b/liberty/lib/SLAMP/SLAMPcustom/consumer/consumer.cpp @@ -61,6 +61,7 @@ static inline void consume_64_64(uint64_t &x, uint64_t &y){ dq_index++; y = dq_data[dq_index]; dq_index++; + _mm_prefetch(&dq_data[dq_index] + QPREFETCH, _MM_HINT_T0); // _mm_prefetch(&dq_data[dq_index] + QPREFETCH, _MM_HINT_NTA); } @@ -71,6 +72,7 @@ static inline void consume_32_32_64(uint32_t &x, uint32_t &y, uint64_t &z){ y = tmp & 0xFFFFFFFF; z = dq_data[dq_index]; dq_index++; + _mm_prefetch(&dq_data[dq_index] + QPREFETCH, _MM_HINT_T0); // _mm_prefetch(&dq_data[dq_index] + QPREFETCH, _MM_HINT_NTA); } diff --git a/liberty/lib/SLAMP/SLAMPcustom/sw_queue_astream.h b/liberty/lib/SLAMP/SLAMPcustom/sw_queue_astream.h index 03ab4074..6626843d 100644 --- a/liberty/lib/SLAMP/SLAMPcustom/sw_queue_astream.h +++ b/liberty/lib/SLAMP/SLAMPcustom/sw_queue_astream.h @@ -50,7 +50,7 @@ #endif /* QMARGIN */ #ifndef QSIZE -#define QSIZE (1 << 21) +#define QSIZE (1 << 23) #endif /* QSIZE */ #ifndef QPREFETCH From eca9c514b475f6585069fc9736fe626bc8330c3c Mon Sep 17 00:00:00 2001 From: Ziyang Xu Date: Tue, 1 Nov 2022 11:06:47 -0400 Subject: [PATCH 32/97] add history and replay function --- liberty/lib/Repl/ReplParse.hpp | 3 ++ liberty/lib/Repl/repl.cpp | 82 ++++++++++++++++++++++++++-------- 2 files changed, 67 insertions(+), 18 deletions(-) diff --git a/liberty/lib/Repl/ReplParse.hpp b/liberty/lib/Repl/ReplParse.hpp index 5c89af74..717142cc 100644 --- a/liberty/lib/Repl/ReplParse.hpp +++ b/liberty/lib/Repl/ReplParse.hpp @@ -18,6 +18,7 @@ enum ReplAction { RemoveAll, Parallelize, Modref, + Save, Unknown = -1 }; @@ -45,6 +46,7 @@ const map ReplActions = { {"parallelize", ReplAction::Parallelize}, {"p", ReplAction::Parallelize}, {"modref", ReplAction::Modref}, + {"save", ReplAction::Save} }; // a helper to get the vocabulary of Repl, to help the auto completion @@ -70,6 +72,7 @@ const map HelpText = { {RemoveAll, "removeAll/ra $inst_id: \tremove all dependences from and to a instruction from the loop"}, {Parallelize, "paralelize/p: \tparallelize the selected loop with current dependences"}, {Modref, "modref/mr $inst_id1, $inst_id2: \tquery the modref between two instructions"}, + {Save, "save $filename: \tsave the loop to a file"}, {Quit, "quit/q: quit the repl"}, }; diff --git a/liberty/lib/Repl/repl.cpp b/liberty/lib/Repl/repl.cpp index a0803048..344d8b3f 100644 --- a/liberty/lib/Repl/repl.cpp +++ b/liberty/lib/Repl/repl.cpp @@ -30,6 +30,8 @@ using namespace std; using namespace liberty; using namespace llvm::noelle; +cl::opt HistoryFileName("history", cl::desc("Specify command history file name"), cl::init("")); + class OptRepl : public ModulePass { public: static char ID; @@ -179,24 +181,19 @@ bool OptRepl::runOnModule(Module &M) { } rl_attempted_completion_function = completer; - // the main repl while loop - while (true) { - string query; - char *buf = readline("(opt-repl) "); - query = (const char *)(buf); - if (query.size() > 0) { - add_history(buf); - free(buf); // free the buf readline created - } - + int selectLoopId = -1; + bool quit = false; + auto mainLoop = [&](string &query) { // check if it's quit or unknown ReplParser parser(query); - if (parser.getAction() == ReplAction::Quit) - break; + if (parser.getAction() == ReplAction::Quit) { + quit = true; + return; + } if (parser.getAction() == ReplAction::Unknown) { outs() << "Unknown command!\n"; - continue; + return; } // print all loops @@ -210,12 +207,13 @@ bool OptRepl::runOnModule(Module &M) { }; // select one loop - auto selectFn = [&loopIdMap, &parser, &selectedLoop, &selectedPDG, &selectedSCCDAG, &pdgbuilder, &instIdMap, &instIdLookupMap, &noelle]() { + auto selectFn = [&selectLoopId, &loopIdMap, &parser, &selectedLoop, &selectedPDG, &selectedSCCDAG, &pdgbuilder, &instIdMap, &instIdLookupMap, &noelle]() { int loopId = parser.getActionId(); if (loopId == -1) { outs() << "No number specified\n"; return; } + selectLoopId = loopId; if (loopIdMap.find(loopId) == loopIdMap.end()) { outs() << "Loop " << loopId << " does not exist\n"; @@ -256,23 +254,23 @@ bool OptRepl::runOnModule(Module &M) { // early checks for several actions that do not need the loop set if (parser.getAction() == ReplAction::Loops) { loopsFn(); - continue; + return; } if (parser.getAction() == ReplAction::Select) { selectFn(); - continue; + return; } if (parser.getAction() == ReplAction::Help){ helpFn(); - continue; + return; } // after this assume the loop has been selected if (!selectedLoop) { outs() << "No loops selected\n"; - continue; + return; } // dump information about the loop @@ -607,6 +605,19 @@ bool OptRepl::runOnModule(Module &M) { } }; + auto saveFn = [&parser]() { + string fileName = parser.getStringAfterAction(); + if (fileName == "") { + fileName = "repl_command_history.log"; + } + // remove the current command from readline history + remove_history(history_length - 1); + write_history(fileName.c_str()); + outs() << "command history (excluding \"save\" command) has been written " + "into " + << fileName << "\n"; + }; + switch (parser.getAction()) { case ReplAction::Deps: depsFn(); @@ -629,10 +640,45 @@ bool OptRepl::runOnModule(Module &M) { case ReplAction::Modref: modrefFn(); break; + case ReplAction::Save: + saveFn(); + break; default: outs() << "SHOULD NOT HAPPEN\n"; break; } + }; + + // execute command history file if specified + string historyFileName = HistoryFileName; + if (historyFileName != "") { + read_history(historyFileName.c_str()); + // DISCUSSION: the last command won't get executed if using 'i < history_length' + for (int i = history_base; i <= history_length; i++) { + char *buf = history_get(i)->line; + string query = (const char *)(buf); + mainLoop(query); + } + clear_history(); + } + + // the main repl while loop + while (true) { + if (quit) { + break; + } + + stringstream ss; + ss << "(cpf-repl"; + if (selectLoopId != -1) ss << " loop " << selectLoopId; + ss << ") "; + char *buf = readline(ss.str().c_str()); + string query = (const char *)(buf); + if (query.size() > 0) { + add_history(buf); + free(buf); // free the buf readline created + } + mainLoop(query); } return modified; From bc5c53a2b4ab31a4107f98bb112fbbe245ac246e Mon Sep 17 00:00:00 2001 From: Ziyang Xu Date: Thu, 3 Nov 2022 18:57:02 -0400 Subject: [PATCH 33/97] refactoring --- .../ProfilingModules/DependenceModule.cpp | 290 ++++-------------- .../ProfilingModules/DependenceModule.h | 92 ++++-- .../SLAMP/SLAMPcustom/consumer/consumer.cpp | 226 ++++++-------- 3 files changed, 228 insertions(+), 380 deletions(-) diff --git a/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.cpp b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.cpp index 105143f3..cf6e854a 100644 --- a/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.cpp +++ b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -11,62 +12,27 @@ #include "DependenceModule.h" #include -std::mutex m; -std::mutex mutex_process_load_store; -constexpr unsigned LOCALWRITE_THREADS = 1; -std::mutex localwrite_mutexes[LOCALWRITE_THREADS]; - -#define SIZE_8M 0x800000 - -#define DEPLOG_VEC_SIZE 1'000'000 - -static slamp::MemoryMap* smmap = nullptr; -static std::unordered_set *deplog_set; -static std::vector *deplog_vec[LOCALWRITE_THREADS]; -static uint64_t slamp_iteration = 0; -static uint64_t slamp_invocation = 0; -static uint32_t target_loop_id = 0; - -struct LoadStoreEvent { - // bool isLoad; - uint64_t addr; - // uint64_t value; - TS ts; - // uint32_t instr; - // uint32_t bare_instr; - // uint64_t invocation; - // uint64_t iteration; -}; -// constexpr unsigned MAX_EVENTS = 100'000'000; -constexpr unsigned MAX_EVENTS = 100'000; -static std::vector *loadstore_vec, *loadstore_vec0, *loadstore_vec1; -static uint64_t loadstore_vec_counter = 0; - -static std::chrono::duration total_time_off_critical_path(0); -static std::chrono::duration loadstore_vec_time(0); - -static void convertVectorToSet(const unsigned thread_id) { - - // launch 56 threads to convert the vector to set independently, chunking - constexpr auto THREADS = 56; - std::thread t[THREADS]; - for (unsigned long i = 0; i < THREADS; i++) { +// static std::map *inst_count; +void DependenceModule::convertVectorToSet() { + // launch N threads to convert the vector to set independently, chunking + std::thread t[VECTOR_TO_SET_THREADS]; + for (unsigned long i = 0; i < VECTOR_TO_SET_THREADS; i++) { t[i] = std::thread( [&](int id) { // take the chunk and convert to a set and return auto *deplog_set_chunk = new std::unordered_set(); - deplog_set_chunk->reserve(DEPLOG_VEC_SIZE / THREADS + 1); - auto begin = id * (DEPLOG_VEC_SIZE / THREADS); - auto end = (id + 1) * (DEPLOG_VEC_SIZE / THREADS); - deplog_set_chunk->insert(deplog_vec[thread_id]->begin() + begin, - deplog_vec[thread_id]->begin() + end); + deplog_set_chunk->reserve(DEPLOG_VEC_SIZE / VECTOR_TO_SET_THREADS + 1); + auto begin = id * (DEPLOG_VEC_SIZE / VECTOR_TO_SET_THREADS); + auto end = (id + 1) * (DEPLOG_VEC_SIZE / VECTOR_TO_SET_THREADS); + deplog_set_chunk->insert(deplog_vec.begin() + begin, + deplog_vec.begin() + end); m.lock(); // lock the global set and insert the chunk - deplog_set->insert(deplog_set_chunk->begin(), + deplog_set.insert(deplog_set_chunk->begin(), deplog_set_chunk->end()); m.unlock(); delete deplog_set_chunk; @@ -77,74 +43,48 @@ static void convertVectorToSet(const unsigned thread_id) { for (auto &i : t) { i.join(); } - - // std::cout << "Merging vec to set, set length " << deplog_set->size() - // << std::endl; } - -namespace DepMod { // init: setup the shadow memory -void init(uint32_t loop_id, uint32_t pid) { - +void DependenceModule::init(uint32_t loop_id, uint32_t pid) { target_loop_id = loop_id; - smmap = new slamp::MemoryMap(TIMESTAMP_SIZE_IN_BYTES); - deplog_set = new std::unordered_set(); - for (auto & i : deplog_vec) { - i = new std::vector(); - i->reserve(DEPLOG_VEC_SIZE); - } - - loadstore_vec0 = new std::vector(); - loadstore_vec1 = new std::vector(); - loadstore_vec = loadstore_vec0; // double buffering - loadstore_vec0->reserve(MAX_EVENTS); - loadstore_vec1->reserve(MAX_EVENTS); +#define SIZE_8M 0x800000 smmap->init_stack(SIZE_8M, pid); + // inst_count = new std::map(); + } -void handleLoadAndStore(std::vector *loadstore_vec); -void fini(const char *filename) { - // show in seconds - std::cout << "Total time off critical path: " - << total_time_off_critical_path.count() / 1000000.0 << "s" - << std::endl; +void DependenceModule::fini(const char *filename) { + std::cout << "Load count: " << load_count << std::endl; + std::cout << "Store count: " << store_count << std::endl; - std::cout << "Loadstore vec time: " - << loadstore_vec_time.count() / 1000000.0 << "s" - << std::endl; + convertVectorToSet(); - mutex_process_load_store.lock(); - handleLoadAndStore(loadstore_vec); - for (int i = 0; i < LOCALWRITE_THREADS; i++) { - convertVectorToSet(i); - } std::ofstream of(filename); of << target_loop_id << " " << 0 << " " << 0 << " " << 0 << " " << 0 << " " << 0 << "\n"; - std::set ordered(deplog_set->begin(), deplog_set->end()); + std::set ordered(deplog_set.begin(), deplog_set.end()); for (auto &k: ordered) { of << target_loop_id << " " << k.src << " " << k.dst << " " << k.dst_bare << " " << (k.cross ? 1 : 0) << " " << 1 << " "; of << "\n"; } + // for (auto &i : *inst_count) { + // of << target_loop_id << " " << i.first << " " << i.second << "\n"; + // } + delete smmap; - delete deplog_set; } -void allocate(void *addr, uint64_t size) { +void DependenceModule::allocate(void *addr, uint64_t size) { smmap->allocate(addr, size); - // std::cout << "allocate " << addr << " " << size << std::endl; } - -// void log(TS ts, const uint32_t dst_inst, TS *pts, const uint32_t bare_inst, - // uint64_t addr, uint64_t value, uint8_t size) { -void log(const unsigned thread_id, TS ts, const uint32_t dst_inst, const uint32_t bare_inst, const uint64_t load_invocation, const uint64_t load_iteration){ +void DependenceModule::log(TS ts, const uint32_t dst_inst, const uint32_t bare_inst, const uint64_t load_invocation, const uint64_t load_iteration){ uint32_t src_inst = GET_INSTR(ts); @@ -155,172 +95,50 @@ void log(const unsigned thread_id, TS ts, const uint32_t dst_inst, const uint32_ return; } - // uint64_t src_invoc = GET_INVOC(ts); - slamp::KEY key(src_inst, dst_inst, bare_inst, src_iter != load_iteration); - // std::cout << "src_inst: " << src_inst << " dst_inst: " << dst_inst << " bare_inst: " << bare_inst << " src_iter: " << src_iter << " slamp_iteration: " << slamp_iteration << " src_iter != slamp_iteration: " << (src_iter != slamp_iteration) << std::endl; - deplog_vec[thread_id]->emplace_back(key); - if (deplog_vec[thread_id]->size() == DEPLOG_VEC_SIZE - 1) { - convertVectorToSet(thread_id); - deplog_vec[thread_id]->resize(0); + deplog_vec.emplace_back(key); + if (deplog_vec.size() == DEPLOG_VEC_SIZE - 1) { + convertVectorToSet(); + deplog_vec.resize(0); } } -void handleLoadAndStore(std::vector *loadstore_vec) { - - // put in LOCALWRITE_THREADS threads to handle the loadstore_vec - std::thread t[LOCALWRITE_THREADS]; +void DependenceModule::load(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value) { - for (auto i = 0; i < LOCALWRITE_THREADS; i++) { - unsigned begin = i * (loadstore_vec_counter / LOCALWRITE_THREADS); - unsigned end = (i + 1) * (loadstore_vec_counter / LOCALWRITE_THREADS); - unsigned thread_id = 0; - // t[i] = std::thread( - // [&](const unsigned thread_id, const unsigned begin, const unsigned end) { - for (auto i = begin; i < end; i++) { - auto &e = (*loadstore_vec)[i]; - const auto &addr = e.addr; - const auto &ts = e.ts; - const auto invoc = GET_INVOC(ts); - const auto iter = GET_ITER(ts); - const auto instr = GET_INSTR(ts); - - TS *s = (TS *)GET_SHADOW(addr, TIMESTAMP_SIZE_IN_POWER_OF_TWO); - // if (e.isLoad) { - // load - // std::cout << "load " << e.addr << " " << e.value << " " << - // e.instr - // << " " << e.bare_instr << std::endl; smmap->load(e.addr, - // e.value, e.instr, e.bare_instr); - - // std::cout << "load " << instr << " " << addr << " " << - // bare_instr - // << " " - // << value << std::endl; - TS tss = s[0]; - if (tss != 0) { - log(thread_id, tss, instr, instr, invoc, iter); - } - // } - // else { - // // store - // // std::cout << "store " << e.addr << " " << e.value << " " << - // // e.instr - // // << " " << e.bare_instr << std::endl; smmap->store(e.addr, - // // e.value, e.instr, e.bare_instr); - // // if (!smmap->is_allocated(s)) { - // // std::cout << "store not allocated: " << instr << " " << - // // bare_instr << " " << addr << std::endl; - // // } - // // TODO: handle output dependence. ignore it as of now. - // // if (ASSUME_ONE_ADDR) { - // s[0] = ts; - // // } else { - // // for (auto i = 0; i < size; i++) - // // s[i] = ts; - // // } - // } - } - // }, i, begin, end); - } - - // for (auto & i : t) { - // i.join(); + // // increase inst_count for this instr + // if (inst_count->find(instr) == inst_count->end()) { + // inst_count->insert(std::make_pair(instr, 1)); + // } else { + // inst_count->at(instr)++; // } - // mutex_process_load_store.unlock(); -} - - -// template -void load(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value) { - - TS *s = (TS *)GET_SHADOW(addr, TIMESTAMP_SIZE_IN_POWER_OF_TWO); - // if (e.isLoad) { - // load - // std::cout << "load " << e.addr << " " << e.value << " " << - // e.instr - // << " " << e.bare_instr << std::endl; smmap->load(e.addr, - // e.value, e.instr, e.bare_instr); - - // std::cout << "load " << instr << " " << addr << " " << - // bare_instr - // << " " - // << value << std::endl; - TS tss = s[0]; - if (tss != 0) { - log(0, tss, instr, instr, slamp_invocation, slamp_iteration); - } - // // auto start = std::chrono::high_resolution_clock::now(); - // loadstore_vec->emplace_back(LoadStoreEvent{addr, CREATE_TS(instr, slamp_iteration, slamp_invocation)}); - // loadstore_vec_counter++; - // // auto end = std::chrono::high_resolution_clock::now(); - // // auto duration = std::chrono::duration_cast(end - start); - // // loadstore_vec_time += duration; - // if (loadstore_vec_counter == MAX_EVENTS - 1) { - // handleLoadAndStore(loadstore_vec); - // // // measure time in here - // // auto start = std::chrono::high_resolution_clock::now(); - - // // // run this asynchrously - // // mutex_process_load_store.lock(); - // // std::thread t = std::thread(handleLoadAndStore, loadstore_vec); - // // t.detach(); - - // // // swap double buffer - // // loadstore_vec = loadstore_vec == loadstore_vec0 ? loadstore_vec1 : loadstore_vec0; - - // // loadstore_vec->resize(0); - // // loadstore_vec_counter = 0; - - // // auto end = std::chrono::high_resolution_clock::now(); - // // auto duration = std::chrono::duration_cast(end - start); - // // total_time_off_critical_path += duration; - // } + local_write(addr, [&]() { + load_count++; + TS *s = (TS *)GET_SHADOW(addr, TIMESTAMP_SIZE_IN_POWER_OF_TWO); + TS tss = s[0]; + if (tss != 0) { + log(tss, instr, instr, slamp_invocation, slamp_iteration); + } + }); } -// template -void store(uint32_t instr, uint32_t bare_instr, const uint64_t addr) { - // if (loadstore_vec_counter > 0) { - // handleLoadAndStore(loadstore_vec); - // loadstore_vec->resize(0); - // loadstore_vec_counter = 0; - // } - - TS ts = CREATE_TS(instr, slamp_iteration, slamp_invocation); - TS *shadow_addr = (TS *)GET_SHADOW(addr, TIMESTAMP_SIZE_IN_POWER_OF_TWO); - shadow_addr[0] = ts; +void DependenceModule::store(uint32_t instr, uint32_t bare_instr, const uint64_t addr) { - // // auto start = std::chrono::high_resolution_clock::now(); - // loadstore_vec->emplace_back(LoadStoreEvent{false, addr, CREATE_TS(instr, slamp_iteration, slamp_invocation)}); - // loadstore_vec_counter++; - // // auto end = std::chrono::high_resolution_clock::now(); - // // auto duration = std::chrono::duration_cast(end - start); - // // loadstore_vec_time += duration; - // if (loadstore_vec_counter == MAX_EVENTS - 1) { - // // measure time in here - // auto start = std::chrono::high_resolution_clock::now(); - // mutex_process_load_store.lock(); - // std::thread t = std::thread(handleLoadAndStore, loadstore_vec); - // t.detach(); - // loadstore_vec = loadstore_vec == loadstore_vec0 ? loadstore_vec1 : loadstore_vec0; - - // loadstore_vec->resize(0); - // loadstore_vec_counter = 0; - // auto end = std::chrono::high_resolution_clock::now(); - // auto duration = std::chrono::duration_cast(end - start); - // total_time_off_critical_path += duration; - // } + local_write(addr, [&]() { + store_count++; + TS ts = CREATE_TS(instr, slamp_iteration, slamp_invocation); + TS *shadow_addr = (TS *)GET_SHADOW(addr, TIMESTAMP_SIZE_IN_POWER_OF_TWO); + shadow_addr[0] = ts; + }); } -void loop_invoc() { +void DependenceModule::loop_invoc() { slamp_iteration = 0; slamp_invocation++; } -void loop_iter() { +void DependenceModule::loop_iter() { slamp_iteration++; } -} // namespace DepMod diff --git a/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.h b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.h index a2664cdc..449f92eb 100644 --- a/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.h +++ b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.h @@ -1,23 +1,77 @@ #include +#include +#include +#include -namespace DepMod { -void init(uint32_t loop_id, uint32_t pid); -void fini(const char *filename); -void load(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value); -void store(uint32_t instr, uint32_t bare_instr, const uint64_t addr); -void allocate(void *addr, uint64_t size); -void loop_invoc(); -void loop_iter(); - -enum DepModAction: char -{ - INIT = 0, - LOAD, - STORE, - ALLOC, - LOOP_INVOC, - LOOP_ITER, - FINISHED +#include "slamp_timestamp.h" +#include "slamp_shadow_mem.h" +#include "slamp_logger.h" + +enum DepModAction : char { + INIT = 0, + LOAD, + STORE, + ALLOC, + LOOP_INVOC, + LOOP_ITER, + FINISHED +}; + +class LocalWriteModule { + protected: + const uint32_t LOCALWRITE_MASK{}; + const uint32_t LOCALWRITE_PATTERN{}; + static constexpr uint32_t LOCALWRITE_SHIFT = 12; // PAGE SIZE 4096 = 2^12 + // takes in a lambda action and uint64_t addr + template + inline void local_write(uint64_t addr, const F &action) { + if (((addr >> LOCALWRITE_SHIFT) & LOCALWRITE_MASK) == LOCALWRITE_PATTERN) { + action(); + } + } + + public: + LocalWriteModule(uint32_t mask, uint32_t pattern) : LOCALWRITE_MASK(mask), LOCALWRITE_PATTERN(pattern) {} + virtual ~LocalWriteModule() = default; }; -} // namespace DepMod +class DependenceModule : public LocalWriteModule { +private: + static constexpr unsigned DEPLOG_VEC_SIZE = 1'000'000; + static constexpr unsigned VECTOR_TO_SET_THREADS = 56; + + uint64_t slamp_iteration = 0; + uint64_t slamp_invocation = 0; + uint32_t target_loop_id = 0; + + // debugging stats + uint64_t load_count = 0; + uint64_t store_count = 0; + + slamp::MemoryMap *smmap = nullptr; + std::unordered_set deplog_set; + std::vector deplog_vec; + std::mutex m; // mutex for the deplog_set + + void convertVectorToSet(); + void log(TS ts, const uint32_t dst_inst, const uint32_t bare_inst, + const uint64_t load_invocation, const uint64_t load_iteration); + +public: + + DependenceModule(uint32_t mask, uint32_t pattern) + : LocalWriteModule(mask, pattern) { + smmap = new slamp::MemoryMap(TIMESTAMP_SIZE_IN_BYTES); + deplog_vec.reserve(DEPLOG_VEC_SIZE); + } + + void init(uint32_t loop_id, uint32_t pid); + void fini(const char *filename); + // always_inline attribute + void load(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, + uint64_t value); + void store(uint32_t instr, uint32_t bare_instr, const uint64_t addr); + void allocate(void *addr, uint64_t size); + void loop_invoc(); + void loop_iter(); +}; diff --git a/liberty/lib/SLAMP/SLAMPcustom/consumer/consumer.cpp b/liberty/lib/SLAMP/SLAMPcustom/consumer/consumer.cpp index 93793ceb..9e854e86 100644 --- a/liberty/lib/SLAMP/SLAMPcustom/consumer/consumer.cpp +++ b/liberty/lib/SLAMP/SLAMPcustom/consumer/consumer.cpp @@ -12,17 +12,32 @@ #include namespace bip = boost::interprocess; +using Action = DepModAction; + +static inline uint64_t rdtsc() { + uint64_t a, d; + __asm__ volatile("rdtsc" : "=a"(a), "=d"(d)); + return (d << 32) | a; +} #define DEBUG 0 #define ACTION 1 +#define MEASURE_TIME 1 + +// #define CONSUME sq_consume(the_queue); +#define CONSUME consume(); + +static uint64_t load_time(0); +static uint64_t store_time(0); +static uint64_t alloc_time(0); // create segment and corresponding allocator -bip::fixed_managed_shared_memory *segment, *segment2; -static SW_Queue the_queue; +bip::fixed_managed_shared_memory *segment; static double_queue_p dqA, dqB, dq, dq_other; static uint64_t dq_index = 0; static uint64_t dq_size = 0; static uint64_t *dq_data; + static void swap(){ if(dq == dqA){ dq = dqB; @@ -76,98 +91,28 @@ static inline void consume_32_32_64(uint32_t &x, uint32_t &y, uint64_t &z){ // _mm_prefetch(&dq_data[dq_index] + QPREFETCH, _MM_HINT_NTA); } -// #define CONSUME sq_consume(the_queue); -#define CONSUME consume(); -#define PRODUCE(x) sq_produce(the_queue,(uint64_t)x); - -#define PRODUCE_2(x,y) PRODUCE( (((uint64_t)x)<<32) | (uint32_t)(y) ) -#define CONSUME_2(x,y) do { uint64_t tmp = CONSUME; x = (uint32_t)(tmp>>32); y = (uint32_t) tmp; } while(0) - -int main(int argc, char** argv) { - - segment = new bip::fixed_managed_shared_memory(bip::open_or_create, "MySharedMemory", sizeof(uint64_t) *QSIZE *4, (void*)(1UL << 32)); - // segment2 = new bip::fixed_managed_shared_memory(bip::open_or_create, "MySharedMemory2", sizeof(uint64_t) *QSIZE *2, (void*)(1UL << 28)); - // managed_shared_memory(bip::open_or_create, "MySharedMemory", sizeof(uint64_t) *QSIZE *2); - // auto a_queue = new atomic_queue::AtomicQueueB(65536); - - dqA = segment->construct("DQ_A")(double_queue_t()); - dqB = segment->construct("DQ_B")(double_queue_t()); - auto dataA = segment->construct("DQ_Data_A")[QSIZE](uint64_t()); - auto dataB = segment->construct("DQ_Data_B")[QSIZE](uint64_t()); - dqA->init(dataA); - dqB->init(dataB); - dq = dqB; - dq_other = dqA; - dq_data = dq->data; - - - // the_queue = static_cast(segment->find_or_construct("MyQueue")()); - // auto data = static_cast(segment2->find_or_construct("smtx_queue_data")[QSIZE]()); - // segment = new bip::managed_shared_memory(bip::open_or_create, "MySharedMemory", sizeof(uint64_t) *QSIZE *2); - // auto a_queue = new atomic_queue::AtomicQueueB(65536); - // the_queue = static_cast(segment->find_or_construct("MyQueue")()); - // auto data = static_cast(segment->find_or_construct("smtx_queue_data")[QSIZE]()); - - // // measure time - // auto start = std::chrono::high_resolution_clock::now(); - // unsigned long ROUND = 100; - // // parse the ROUND number from cmd - // if (argc > 1) { - // std::stringstream ss; - // ss << argv[1]; - // ss >> ROUND; - // } - // for (int round = 0; round < ROUND; round++) { - // for (unsigned long i = 0; i < QSIZE; i++) { - // data[i] = i; - // } - // } - // auto end = std::chrono::high_resolution_clock::now(); - // std::chrono::duration diff = end-start; - // std::cout << "Time for " << ROUND * QSIZE / 1'000'000 << "M events : " << diff.count() << " s" << std::endl; - // // throughput - // std::cout << "Write Throughput: " << ROUND * QSIZE / diff.count() / 1'000'000 << "M events/s" << std::endl; - - // start = std::chrono::high_resolution_clock::now(); - // unsigned total = 0; - // for (int round = 0; round < ROUND; round++) { - // for (unsigned long i = 0; i < QSIZE; i++) { - // total += data[i]; - // } - // } - // end = std::chrono::high_resolution_clock::now(); - // diff = end-start; - // std::cout << "Total = " << total << std::endl; - // std::cout << "Time for " << ROUND * QSIZE / 1'000'000 << "M events : " << diff.count() << " s" << std::endl; - // // throughput - // std::cout << "Read Throughput: " << ROUND * QSIZE / diff.count() / 1'000'000 << "M events/s" << std::endl; - - - // if (the_queue == nullptr) { - // std::cout << "Error: could not create queue" << std::endl; - // return 1; - // } - // the_queue->data = data; - // if (the_queue->data == nullptr) { - // std::cout << "Error: could not create queue data" << std::endl; - // return 1; - // } - // [> Initialize the queue data structure <] - // the_queue->p_data = (uint64_t) the_queue->data; - // the_queue->c_inx = 0; - // the_queue->c_margin = 0; - // the_queue->p_glb_inx = 0; - // the_queue->c_glb_inx = 0; - // the_queue->ptr_c_glb_inx = &(the_queue->c_glb_inx); - // the_queue->ptr_p_glb_inx = &(the_queue->p_glb_inx); +void consume_loop() { + DependenceModule depMod(0x3, 1); + uint64_t rdtsc_start = 0; uint64_t counter = 0; - // shm::shared_string v(char_alloc); + uint32_t loop_id; - using Action = DepMod::DepModAction; + // measure time with lambda action + auto measure_time = [](uint64_t &time, auto action) { + // measure time with rdtsc + if (MEASURE_TIME) { + uint64_t start = rdtsc(); + action(); + uint64_t end = rdtsc(); + time += end - start; + } + else { + action(); + } + }; - uint32_t loop_id; while (true) { check(); char v; @@ -179,13 +124,14 @@ int main(int argc, char** argv) { uint32_t pid; loop_id = (uint32_t)CONSUME; pid = (uint32_t)CONSUME; + rdtsc_start = rdtsc(); if (DEBUG) { std::cout << "INIT: " << loop_id << " " << pid << std::endl; } -#if ACTION - DepMod::init(loop_id, pid); -#endif + if (ACTION) { + depMod.init(loop_id, pid); + } break; }; case Action::LOAD: { @@ -193,18 +139,17 @@ int main(int argc, char** argv) { uint64_t addr; uint32_t bare_instr; uint64_t value = 0; + consume_32_32_64(instr, bare_instr, addr); - // CONSUME_2(instr, bare_instr); - // addr = CONSUME; - // value = CONSUME; + if (DEBUG) { std::cout << "LOAD: " << instr << " " << addr << " " << bare_instr << " " << value << std::endl; } -#if ACTION - DepMod::load(instr, addr, bare_instr, value); - // DepMod::load(instr, addr, bare_instr, value); -#endif + if (ACTION) { + measure_time(load_time, + [&]() { depMod.load(instr, addr, bare_instr, value); }); + } break; }; @@ -213,74 +158,105 @@ int main(int argc, char** argv) { uint32_t bare_instr; uint64_t addr; consume_32_32_64(instr, bare_instr, addr); - // CONSUME_2(instr, bare_instr); - // addr = CONSUME; + if (DEBUG) { std::cout << "STORE: " << instr << " " << bare_instr << " " << addr << std::endl; } -#if ACTION - DepMod::store(instr, bare_instr, addr); -#endif + if (ACTION) { + measure_time(store_time, + [&]() { depMod.store(instr, bare_instr, addr); }); + } break; }; case Action::ALLOC: { uint64_t addr; uint64_t size; consume_64_64(addr, size); - // addr = CONSUME; - // size = CONSUME; + if (DEBUG) { std::cout << "ALLOC: " << addr << " " << size << std::endl; } -#if ACTION - DepMod::allocate(reinterpret_cast(addr), size); -#endif + if (ACTION) { + measure_time(alloc_time, [&]() { + depMod.allocate(reinterpret_cast(addr), size); + }); + } break; }; case Action::LOOP_INVOC: { -#if ACTION - DepMod::loop_invoc(); -#endif - if (DEBUG) { std::cout << "LOOP_INVOC" << std::endl; } + + if (ACTION) { + depMod.loop_invoc(); + } break; }; case Action::LOOP_ITER: { -#if ACTION - DepMod::loop_iter(); -#endif - if (DEBUG) { std::cout << "LOOP_ITER" << std::endl; } + if (ACTION) { + depMod.loop_iter(); + } break; }; case Action::FINISHED: { - std::stringstream ss; - ss << "deplog-" << loop_id << ".txt"; -#if ACTION - DepMod::fini(ss.str().c_str()); -#endif + if (ACTION) { + std::stringstream ss; + ss << "deplog-" << loop_id << ".txt"; + depMod.fini(ss.str().c_str()); + } + + uint64_t rdtsc_end = rdtsc(); + // total cycles + uint64_t total_cycles = rdtsc_end - rdtsc_start; std::cout << "Finished loop: " << loop_id << " after " << counter << " events" << std::endl; + // print time in seconds + std::cout << "Total time: " << total_cycles / 2.6e9 << " s" << std::endl; + std::cout << "Load time: " << load_time / 2.6e9 << " s" << std::endl; + std::cout << "Store time: " << store_time / 2.6e9 << " s" << std::endl; + std::cout << "Alloc time: " << alloc_time / 2.6e9 << " s" << std::endl; + exit(0); + break; }; default: std::cout << "Unknown action: " << (uint64_t)v << std::endl; - std::cout << "Is ready to read?:" << dq->ready_to_read << " " << "Is ready to write?:" << dq->ready_to_write << std::endl; + std::cout << "Is ready to read?:" << dq->ready_to_read << " " + << "Is ready to write?:" << dq->ready_to_write << std::endl; std::cout << "Index: " << dq_index << " Size:" << dq->size << std::endl; + for (int i = 0; i < 101; i++) { std::cout << dq->data[dq_index - 100 + i] << " "; } exit(-1); } - if (counter % 100000000 == 0) { - std::cout << "Processed " << counter / 1000000 << "M events" << std::endl; + if (counter % 100'000'000 == 0) { + std::cout << "Processed " << counter / 1'000'000 << "M events" << std::endl; } } + +} + +int main(int argc, char** argv) { + segment = new bip::fixed_managed_shared_memory(bip::open_or_create, "MySharedMemory", sizeof(uint64_t) *QSIZE *4, (void*)(1UL << 32)); + + // double buffering + dqA = segment->construct("DQ_A")(double_queue_t()); + dqB = segment->construct("DQ_B")(double_queue_t()); + auto dataA = segment->construct("DQ_Data_A")[QSIZE](uint64_t()); + auto dataB = segment->construct("DQ_Data_B")[QSIZE](uint64_t()); + dqA->init(dataA); + dqB->init(dataB); + dq = dqB; + dq_other = dqA; + dq_data = dq->data; + + consume_loop(); } From bff970d7f82b8fe28fca7a45756fa9fca9701b8f Mon Sep 17 00:00:00 2001 From: Ziyang Xu Date: Fri, 4 Nov 2022 13:15:12 -0400 Subject: [PATCH 34/97] refactoring: isolate High-throughput container --- .../ProfilingModules/DependenceModule.cpp | 54 ++------- .../ProfilingModules/DependenceModule.h | 44 ++++---- .../consumer/ProfilingModules/HTContainer.h | 104 ++++++++++++++++++ 3 files changed, 132 insertions(+), 70 deletions(-) create mode 100644 liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/HTContainer.h diff --git a/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.cpp b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.cpp index cf6e854a..87bdead1 100644 --- a/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.cpp +++ b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.cpp @@ -1,49 +1,17 @@ #include +#include #include #include #include -#include #include -#include +#include -#include "slamp_timestamp.h" +#include "DependenceModule.h" #include "slamp_logger.h" #include "slamp_shadow_mem.h" -#include "DependenceModule.h" -#include - +#include "slamp_timestamp.h" // static std::map *inst_count; -void DependenceModule::convertVectorToSet() { - // launch N threads to convert the vector to set independently, chunking - std::thread t[VECTOR_TO_SET_THREADS]; - for (unsigned long i = 0; i < VECTOR_TO_SET_THREADS; i++) { - t[i] = std::thread( - [&](int id) { - // take the chunk and convert to a set and return - auto *deplog_set_chunk = - new std::unordered_set(); - deplog_set_chunk->reserve(DEPLOG_VEC_SIZE / VECTOR_TO_SET_THREADS + 1); - auto begin = id * (DEPLOG_VEC_SIZE / VECTOR_TO_SET_THREADS); - auto end = (id + 1) * (DEPLOG_VEC_SIZE / VECTOR_TO_SET_THREADS); - deplog_set_chunk->insert(deplog_vec.begin() + begin, - deplog_vec.begin() + end); - - m.lock(); - // lock the global set and insert the chunk - deplog_set.insert(deplog_set_chunk->begin(), - deplog_set_chunk->end()); - m.unlock(); - delete deplog_set_chunk; - }, - i); - } - // join the threads - for (auto &i : t) { - i.join(); - } -} // init: setup the shadow memory void DependenceModule::init(uint32_t loop_id, uint32_t pid) { @@ -60,13 +28,11 @@ void DependenceModule::fini(const char *filename) { std::cout << "Load count: " << load_count << std::endl; std::cout << "Store count: " << store_count << std::endl; - convertVectorToSet(); - std::ofstream of(filename); of << target_loop_id << " " << 0 << " " << 0 << " " << 0 << " " << 0 << " " << 0 << "\n"; - std::set ordered(deplog_set.begin(), deplog_set.end()); + std::set ordered(dep_set.begin(), dep_set.end()); for (auto &k: ordered) { of << target_loop_id << " " << k.src << " " << k.dst << " " << k.dst_bare << " " << (k.cross ? 1 : 0) << " " << 1 << " "; @@ -84,7 +50,7 @@ void DependenceModule::allocate(void *addr, uint64_t size) { smmap->allocate(addr, size); } -void DependenceModule::log(TS ts, const uint32_t dst_inst, const uint32_t bare_inst, const uint64_t load_invocation, const uint64_t load_iteration){ +void DependenceModule::log(TS ts, const uint32_t dst_inst, const uint32_t bare_inst, const uint64_t load_invocation, const uint64_t load_iteration){ uint32_t src_inst = GET_INSTR(ts); @@ -96,12 +62,8 @@ void DependenceModule::log(TS ts, const uint32_t dst_inst, const uint32_t bare_i } slamp::KEY key(src_inst, dst_inst, bare_inst, src_iter != load_iteration); - - deplog_vec.emplace_back(key); - if (deplog_vec.size() == DEPLOG_VEC_SIZE - 1) { - convertVectorToSet(); - deplog_vec.resize(0); - } + + dep_set.emplace_back(key); } void DependenceModule::load(uint32_t instr, const uint64_t addr, const uint32_t bare_instr, uint64_t value) { diff --git a/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.h b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.h index 449f92eb..7c082dbe 100644 --- a/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.h +++ b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.h @@ -3,9 +3,11 @@ #include #include -#include "slamp_timestamp.h" -#include "slamp_shadow_mem.h" #include "slamp_logger.h" +#include "slamp_shadow_mem.h" +#include "slamp_timestamp.h" + +#include "HTContainer.h" enum DepModAction : char { INIT = 0, @@ -18,28 +20,26 @@ enum DepModAction : char { }; class LocalWriteModule { - protected: - const uint32_t LOCALWRITE_MASK{}; - const uint32_t LOCALWRITE_PATTERN{}; - static constexpr uint32_t LOCALWRITE_SHIFT = 12; // PAGE SIZE 4096 = 2^12 - // takes in a lambda action and uint64_t addr - template - inline void local_write(uint64_t addr, const F &action) { - if (((addr >> LOCALWRITE_SHIFT) & LOCALWRITE_MASK) == LOCALWRITE_PATTERN) { - action(); - } +protected: + const uint32_t LOCALWRITE_MASK{}; + const uint32_t LOCALWRITE_PATTERN{}; + static constexpr uint32_t LOCALWRITE_SHIFT = 12; // PAGE SIZE 4096 = 2^12 + // takes in a lambda action and uint64_t addr + template + inline void local_write(uint64_t addr, const F &action) { + if (((addr >> LOCALWRITE_SHIFT) & LOCALWRITE_MASK) == LOCALWRITE_PATTERN) { + action(); } + } - public: - LocalWriteModule(uint32_t mask, uint32_t pattern) : LOCALWRITE_MASK(mask), LOCALWRITE_PATTERN(pattern) {} - virtual ~LocalWriteModule() = default; +public: + LocalWriteModule(uint32_t mask, uint32_t pattern) + : LOCALWRITE_MASK(mask), LOCALWRITE_PATTERN(pattern) {} + virtual ~LocalWriteModule() = default; }; class DependenceModule : public LocalWriteModule { private: - static constexpr unsigned DEPLOG_VEC_SIZE = 1'000'000; - static constexpr unsigned VECTOR_TO_SET_THREADS = 56; - uint64_t slamp_iteration = 0; uint64_t slamp_invocation = 0; uint32_t target_loop_id = 0; @@ -49,20 +49,16 @@ class DependenceModule : public LocalWriteModule { uint64_t store_count = 0; slamp::MemoryMap *smmap = nullptr; - std::unordered_set deplog_set; - std::vector deplog_vec; - std::mutex m; // mutex for the deplog_set - void convertVectorToSet(); + HTSet dep_set; + void log(TS ts, const uint32_t dst_inst, const uint32_t bare_inst, const uint64_t load_invocation, const uint64_t load_iteration); public: - DependenceModule(uint32_t mask, uint32_t pattern) : LocalWriteModule(mask, pattern) { smmap = new slamp::MemoryMap(TIMESTAMP_SIZE_IN_BYTES); - deplog_vec.reserve(DEPLOG_VEC_SIZE); } void init(uint32_t loop_id, uint32_t pid); diff --git a/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/HTContainer.h b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/HTContainer.h new file mode 100644 index 00000000..a35b5e0d --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/HTContainer.h @@ -0,0 +1,104 @@ +/* + * HT (High Throughput) Containers + * Author: Ziyang Xu + * + * Use vector as buffer and use parallelism to improve performance + * This can replace set and map in STL + * + */ +#include +#include +#include +#include + +template , + typename KeyEqual = std::equal_to, + uint32_t MAX_THREAD = 56, + uint32_t BUFFER_SIZE = 1'000'000> +class HTSet { +private: + std::vector buffer; + std::mutex m; + +public: + std::unordered_set set; + HTSet() { buffer.reserve(BUFFER_SIZE); } + + void emplace_back(T &&t) { + buffer.emplace_back(std::move(t)); + + checkBuffer(); + } + + void emplace_back(const T &t) { + buffer.emplace_back(t); + + checkBuffer(); + } + + // iterator begin + auto begin() { + convertVectorToSet(); + return set.begin(); + } + + // iterator end + auto end() { + convertVectorToSet(); + return set.end(); + } + +private: + // TODO: adaptive thread count + const uint32_t getThreadCount() { return MAX_THREAD; } + + inline void checkBuffer() { + if (buffer.size() == BUFFER_SIZE) { + convertVectorToSet(); + buffer.resize(0); + buffer.reserve(BUFFER_SIZE); + } + } + + void convertVectorToSet() { + const uint32_t thread_count = getThreadCount(); + const auto set_size = buffer.size() / thread_count; + const auto buffer_size = buffer.size(); + + if (buffer_size == 0) { + return; + } + + if (thread_count == 1) { + set.insert(buffer.begin(), buffer.end()); + return; + } + + // launch N threads to convert the vector to set independently, chunking + std::thread t[thread_count]; + for (unsigned long i = 0; i < thread_count; i++) { + t[i] = std::thread( + [&](int id) { + // take the chunk and convert to a set and return + auto *set_chunk = new std::unordered_set(); + set_chunk->reserve(set_size); + + auto begin = id * (buffer_size / thread_count); + auto end = (id + 1) * (buffer_size / thread_count); + + set_chunk->insert(buffer.begin() + begin, buffer.begin() + end); + + m.lock(); + // lock the global set and insert the chunk + set.insert(set_chunk->begin(), set_chunk->end()); + m.unlock(); + delete set_chunk; + }, + i); + } + // join the threads + for (auto &i : t) { + i.join(); + } + } +}; From 527566db95b3407cdec671c81275f473b6e4643c Mon Sep 17 00:00:00 2001 From: Ziyang Xu Date: Fri, 4 Nov 2022 18:42:53 -0400 Subject: [PATCH 35/97] localwrite through buffer --- liberty/lib/SLAMP/SLAMPcustom/CustomSend.cpp | 7 +- .../ProfilingModules/DependenceModule.cpp | 30 +- .../ProfilingModules/DependenceModule.h | 4 +- .../ProfilingModules/slamp_shadow_mem.h | 19 +- .../SLAMP/SLAMPcustom/consumer/consumer.cpp | 120 ++---- .../lib/SLAMP/SLAMPcustom/sw_queue_astream.h | 402 +++++------------- tests/scripts/slamp-driver | 2 +- 7 files changed, 183 insertions(+), 401 deletions(-) diff --git a/liberty/lib/SLAMP/SLAMPcustom/CustomSend.cpp b/liberty/lib/SLAMP/SLAMPcustom/CustomSend.cpp index c8bea0f2..3f26b59e 100644 --- a/liberty/lib/SLAMP/SLAMPcustom/CustomSend.cpp +++ b/liberty/lib/SLAMP/SLAMPcustom/CustomSend.cpp @@ -31,8 +31,7 @@ static void *(*old_memalign_hook)(size_t, size_t, const void *); // create segment and corresponding allocator bip::fixed_managed_shared_memory *segment; bip::fixed_managed_shared_memory *segment2; -static SW_Queue the_queue; -static double_queue_p dqA, dqB, dq, dq_other; +static Queue_p dqA, dqB, dq, dq_other; static uint64_t dq_index = 0; static uint64_t *dq_data; // static uint64_t total_pushed = 0; @@ -152,8 +151,8 @@ void SLAMP_init(uint32_t fn_id, uint32_t loop_id) { segment = new bip::fixed_managed_shared_memory(bip::open_or_create, "MySharedMemory", sizeof(uint64_t) *QSIZE *4, (void*)(1UL << 32)); // segment2 = new bip::fixed_managed_shared_memory(bip::open_or_create, "MySharedMemory2", sizeof(uint64_t) *QSIZE *2, (void*)(1UL << 28)); - dqA = segment->find("DQ_A").first; - dqB = segment->find("DQ_B").first; + dqA = segment->find("DQ_A").first; + dqB = segment->find("DQ_B").first; dq = dqA; dq_other = dqB; dq_index = 0; diff --git a/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.cpp b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.cpp index 87bdead1..0eabd25f 100644 --- a/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.cpp +++ b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.cpp @@ -12,6 +12,11 @@ #include "slamp_timestamp.h" // static std::map *inst_count; +static inline uint64_t rdtsc() { + uint64_t a, d; + __asm__ volatile("rdtsc" : "=a"(a), "=d"(d)); + return (d << 32) | a; +} // init: setup the shadow memory void DependenceModule::init(uint32_t loop_id, uint32_t pid) { @@ -23,21 +28,25 @@ void DependenceModule::init(uint32_t loop_id, uint32_t pid) { } +static uint64_t log_time = 0; + void DependenceModule::fini(const char *filename) { std::cout << "Load count: " << load_count << std::endl; std::cout << "Store count: " << store_count << std::endl; - std::ofstream of(filename); - of << target_loop_id << " " << 0 << " " << 0 << " " - << 0 << " " << 0 << " " << 0 << "\n"; + // std::ofstream of(filename); + // of << target_loop_id << " " << 0 << " " << 0 << " " + // << 0 << " " << 0 << " " << 0 << "\n"; + + // std::set ordered(dep_set.begin(), dep_set.end()); + // for (auto &k: ordered) { + // of << target_loop_id << " " << k.src << " " << k.dst << " " << k.dst_bare << " " + // << (k.cross ? 1 : 0) << " " << 1 << " "; + // of << "\n"; + // } - std::set ordered(dep_set.begin(), dep_set.end()); - for (auto &k: ordered) { - of << target_loop_id << " " << k.src << " " << k.dst << " " << k.dst_bare << " " - << (k.cross ? 1 : 0) << " " << 1 << " "; - of << "\n"; - } + std::cout << "Log time: " << log_time/ 2.6e9 << " s" << std::endl; // for (auto &i : *inst_count) { // of << target_loop_id << " " << i.first << " " << i.second << "\n"; @@ -81,7 +90,10 @@ void DependenceModule::load(uint32_t instr, const uint64_t addr, const uint32_t TS tss = s[0]; if (tss != 0) { + uint64_t start = rdtsc(); log(tss, instr, instr, slamp_invocation, slamp_iteration); + uint64_t end = rdtsc(); + log_time += end - start; } }); } diff --git a/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.h b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.h index 7c082dbe..c2776349 100644 --- a/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.h +++ b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.h @@ -50,7 +50,7 @@ class DependenceModule : public LocalWriteModule { slamp::MemoryMap *smmap = nullptr; - HTSet dep_set; + HTSet dep_set; void log(TS ts, const uint32_t dst_inst, const uint32_t bare_inst, const uint64_t load_invocation, const uint64_t load_iteration); @@ -58,7 +58,7 @@ class DependenceModule : public LocalWriteModule { public: DependenceModule(uint32_t mask, uint32_t pattern) : LocalWriteModule(mask, pattern) { - smmap = new slamp::MemoryMap(TIMESTAMP_SIZE_IN_BYTES); + smmap = new slamp::MemoryMap(LOCALWRITE_MASK, LOCALWRITE_PATTERN, TIMESTAMP_SIZE_IN_BYTES); } void init(uint32_t loop_id, uint32_t pid); diff --git a/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/slamp_shadow_mem.h b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/slamp_shadow_mem.h index ef07201b..e18703d3 100644 --- a/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/slamp_shadow_mem.h +++ b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/slamp_shadow_mem.h @@ -26,9 +26,22 @@ namespace slamp { class MemoryMap { +private: + // TODO: fix this redundant info as the LocalWriteModule + const uint32_t LOCALWRITE_MASK{}; + const uint32_t LOCALWRITE_PATTERN{}; + static constexpr uint32_t LOCALWRITE_SHIFT = 12; // PAGE SIZE 4096 = 2^12 + + bool local_write_cond(uint64_t addr) { + if (((addr >> LOCALWRITE_SHIFT) & LOCALWRITE_MASK) == LOCALWRITE_PATTERN) + return true; + return false; + } + public: uint64_t heapStart = 0; - MemoryMap(unsigned r) : ratio(r), ratio_shift(0) { + MemoryMap(uint32_t mask, uint32_t pattern, unsigned r) + : LOCALWRITE_MASK(mask), LOCALWRITE_PATTERN(pattern), ratio(r), ratio_shift(0) { // ratio expected to be a power of 2 assert((r & (r - 1)) == 0); @@ -47,6 +60,8 @@ class MemoryMap { ~MemoryMap() { // freeing all remaining shadow addresses for (auto page : pages) { + if (!local_write_cond(page)) + continue; uint64_t s = GET_SHADOW(page, ratio_shift); munmap(reinterpret_cast(s), pagesize * ratio); } @@ -75,6 +90,8 @@ class MemoryMap { bool success = true; for (uint64_t page = pagebegin; page <= pageend; page += pagesize) { + if (!local_write_cond(page)) + continue; if (pages.find(page) != pages.end()) continue; diff --git a/liberty/lib/SLAMP/SLAMPcustom/consumer/consumer.cpp b/liberty/lib/SLAMP/SLAMPcustom/consumer/consumer.cpp index 9e854e86..037528ed 100644 --- a/liberty/lib/SLAMP/SLAMPcustom/consumer/consumer.cpp +++ b/liberty/lib/SLAMP/SLAMPcustom/consumer/consumer.cpp @@ -22,10 +22,9 @@ static inline uint64_t rdtsc() { #define DEBUG 0 #define ACTION 1 -#define MEASURE_TIME 1 +#define MEASURE_TIME 0 // #define CONSUME sq_consume(the_queue); -#define CONSUME consume(); static uint64_t load_time(0); static uint64_t store_time(0); @@ -33,68 +32,8 @@ static uint64_t alloc_time(0); // create segment and corresponding allocator bip::fixed_managed_shared_memory *segment; -static double_queue_p dqA, dqB, dq, dq_other; -static uint64_t dq_index = 0; -static uint64_t dq_size = 0; -static uint64_t *dq_data; - -static void swap(){ - if(dq == dqA){ - dq = dqB; - dq_other = dqA; - }else{ - dq = dqA; - dq_other = dqB; - } - dq_data = dq->data; -} - -static void check() { - if (dq_index == dq_size){ - dq->ready_to_read = false; - dq->ready_to_write = true; - while (!dq_other->ready_to_read){ - // spin - usleep(10); - } - swap(); - dq->ready_to_write = false; - dq_index = 0; - dq_size = dq->size; - } -} - -static inline uint64_t consume(){ - uint64_t ret = dq_data[dq_index]; - dq_index++; - // _mm_prefetch(&dq_data[dq_index] + QPREFETCH, _MM_HINT_NTA); - return ret; -} - -static inline void consume_64_64(uint64_t &x, uint64_t &y){ - x = dq_data[dq_index]; - dq_index++; - y = dq_data[dq_index]; - dq_index++; - _mm_prefetch(&dq_data[dq_index] + QPREFETCH, _MM_HINT_T0); - // _mm_prefetch(&dq_data[dq_index] + QPREFETCH, _MM_HINT_NTA); -} - -static inline void consume_32_32_64(uint32_t &x, uint32_t &y, uint64_t &z){ - uint64_t tmp = dq_data[dq_index]; - dq_index++; - x = (tmp >> 32) & 0xFFFFFFFF; - y = tmp & 0xFFFFFFFF; - z = dq_data[dq_index]; - dq_index++; - _mm_prefetch(&dq_data[dq_index] + QPREFETCH, _MM_HINT_T0); - // _mm_prefetch(&dq_data[dq_index] + QPREFETCH, _MM_HINT_NTA); -} - -void consume_loop() { - - DependenceModule depMod(0x3, 1); +void consume_loop(DoubleQueue &dq, DependenceModule &depMod) { uint64_t rdtsc_start = 0; uint64_t counter = 0; uint32_t loop_id; @@ -114,16 +53,16 @@ void consume_loop() { }; while (true) { - check(); + dq.check(); char v; - v = (char)CONSUME; + v = (char)dq.consume(); counter++; switch (v) { case Action::INIT: { uint32_t pid; - loop_id = (uint32_t)CONSUME; - pid = (uint32_t)CONSUME; + loop_id = (uint32_t)dq.consume(); + pid = (uint32_t)dq.consume(); rdtsc_start = rdtsc(); if (DEBUG) { @@ -140,7 +79,7 @@ void consume_loop() { uint32_t bare_instr; uint64_t value = 0; - consume_32_32_64(instr, bare_instr, addr); + dq.consume_32_32_64(instr, bare_instr, addr); if (DEBUG) { std::cout << "LOAD: " << instr << " " << addr << " " << bare_instr @@ -157,7 +96,7 @@ void consume_loop() { uint32_t instr; uint32_t bare_instr; uint64_t addr; - consume_32_32_64(instr, bare_instr, addr); + dq.consume_32_32_64(instr, bare_instr, addr); if (DEBUG) { std::cout << "STORE: " << instr << " " << bare_instr << " " << addr @@ -172,7 +111,7 @@ void consume_loop() { case Action::ALLOC: { uint64_t addr; uint64_t size; - consume_64_64(addr, size); + dq.consume_64_64(addr, size); if (DEBUG) { std::cout << "ALLOC: " << addr << " " << size << std::endl; @@ -227,12 +166,12 @@ void consume_loop() { default: std::cout << "Unknown action: " << (uint64_t)v << std::endl; - std::cout << "Is ready to read?:" << dq->ready_to_read << " " - << "Is ready to write?:" << dq->ready_to_write << std::endl; - std::cout << "Index: " << dq_index << " Size:" << dq->size << std::endl; + std::cout << "Is ready to read?:" << dq.qNow->ready_to_read << " " + << "Is ready to write?:" << dq.qNow->ready_to_write << std::endl; + std::cout << "Index: " << dq.index << " Size:" << dq.qNow->size << std::endl; for (int i = 0; i < 101; i++) { - std::cout << dq->data[dq_index - 100 + i] << " "; + std::cout << dq.qNow->data[dq.index - 100 + i] << " "; } exit(-1); } @@ -247,16 +186,37 @@ void consume_loop() { int main(int argc, char** argv) { segment = new bip::fixed_managed_shared_memory(bip::open_or_create, "MySharedMemory", sizeof(uint64_t) *QSIZE *4, (void*)(1UL << 32)); + Queue_p dqA, dqB; // double buffering - dqA = segment->construct("DQ_A")(double_queue_t()); - dqB = segment->construct("DQ_B")(double_queue_t()); + dqA = segment->construct("DQ_A")(Queue()); + dqB = segment->construct("DQ_B")(Queue()); auto dataA = segment->construct("DQ_Data_A")[QSIZE](uint64_t()); auto dataB = segment->construct("DQ_Data_B")[QSIZE](uint64_t()); dqA->init(dataA); dqB->init(dataB); - dq = dqB; - dq_other = dqA; - dq_data = dq->data; - consume_loop(); + // set the thread count + constexpr unsigned THREAD_COUNT = 8; + constexpr unsigned MASK = THREAD_COUNT - 1; + + unsigned running_threads= THREAD_COUNT; + std::mutex m; + std::condition_variable cv; + + std::vector threads; + DoubleQueue *dqs[THREAD_COUNT]; + DependenceModule *depMods[THREAD_COUNT]; + + for (unsigned i = 0; i < THREAD_COUNT; i++) { + dqs[i] = new DoubleQueue(dqA, dqB, true, running_threads, m, cv); + depMods[i] = new DependenceModule(MASK, i); + + threads.emplace_back(std::thread([&](unsigned id) { + consume_loop(*dqs[id], *depMods[id]); + }, i)); + } + + for (auto &t : threads) { + t.join(); + } } diff --git a/liberty/lib/SLAMP/SLAMPcustom/sw_queue_astream.h b/liberty/lib/SLAMP/SLAMPcustom/sw_queue_astream.h index 6626843d..a8fdab9c 100644 --- a/liberty/lib/SLAMP/SLAMPcustom/sw_queue_astream.h +++ b/liberty/lib/SLAMP/SLAMPcustom/sw_queue_astream.h @@ -4,6 +4,7 @@ #ifndef SW_QUEUE_H #define SW_QUEUE_H +#include #define DUALCORE #include @@ -11,6 +12,9 @@ #include #include #include +#include +#include +#include #include "inline.h" #include "bitcast.h" @@ -62,10 +66,7 @@ #define PAD(suffix, size) char padding ## suffix [CACHELINE_SIZE - (size)] -#ifdef __cplusplus -extern "C" { -#endif -typedef struct { +struct UnderlyingQueue { volatile bool ready_to_read; PAD(1, sizeof(bool)); volatile bool ready_to_write; @@ -80,323 +81,116 @@ typedef struct { this->size = 0; this->data = data; } -} double_queue_t, *double_queue_p; +}; -typedef struct { - - uint64_t p_data; - PAD(1, sizeof(uint64_t)); - - volatile uint32_t *ptr_c_glb_inx; - uint32_t p_glb_inx; - PAD(2, sizeof(volatile uint32_t *) + sizeof(uint32_t)); - - uint32_t c_margin; - uint32_t c_inx; - PAD(3, sizeof(uint32_t) * 2); - - volatile uint32_t *ptr_p_glb_inx; - uint32_t c_glb_inx; - PAD(4, sizeof(volatile uint32_t *) + sizeof(uint32_t)); +using Queue = UnderlyingQueue; +using Queue_p = Queue *; +struct DoubleQueue { + Queue_p qA, qB, qNow, qOther; + uint64_t index = 0; + uint64_t size = 0; uint64_t *data; - uint64_t total_produces; - uint64_t total_consumes; - - PAD(5, sizeof(volatile uint64_t *) + sizeof(uint64_t) * 2); - -} sw_queue_t, *SW_Queue; - -typedef void *sq_cbData; -typedef void (*sq_callback)(sq_cbData); - -/* ***************************************** */ -/* ******* Detail Implementation *********** */ -/* ***************************************** */ - -SW_Queue sq_createQueue(void); -SW_Queue sq_createQueueBlock(unsigned queues); -SW_Queue sq_initQueue(SW_Queue queue); -void sq_freeQueueBlock(SW_Queue queue, unsigned size); -void sq_freeQueue(SW_Queue q); - -/* ******************************************************** */ -/* *********** Preallocate Method *********************** */ -/* ******************************************************** */ - -/* - * This method is faster on multi-processor machines as it bypasses the L2 - * cache. - */ -Inline void sq_streamWrite(uint64_t *addr, uint64_t value) { - - /* __m64 mmxReg = _mm_set_pi64x((int64_t) value); */ - /* _mm_stream_pi((__m64 *) addr, mmxReg); */ - - __asm ( - "movntiq %1, (%0)\n" - : - : "r" (addr), "r" (value) - ); -} - -/* - * This method is faster on multi-core machines as it exploits the common L2 - * cache. - */ -Inline void sq_stdWrite(uint64_t *addr, uint64_t value) { - *addr = value; -} - -/* - * Modulo subtraction - */ -Inline uint32_t sq_modSub(uint32_t minuend, - uint32_t subtrahend, - uint32_t mask) { - return (minuend - subtrahend) & mask; -} - -Inline uint32_t sq_pInx(SW_Queue q) { - return (uint32_t) (q->p_data >> 32); -} + std::mutex &m; + std::condition_variable &cv; + unsigned &running_threads; -/* - * Flush all produce values. - * - * sfence is slow, but necessary for streaming writes. sfence MUST proceded - * updating p_glb_inx, otherwise there will be a race condition. - */ -Inline void sq_flushQueue(SW_Queue q) -{ -#if STREAM - _mm_sfence(); -#endif + DoubleQueue(Queue_p dqA, Queue_p dqB, bool isConsumer, unsigned &threads, + std::mutex &m, std::condition_variable &cv) + : qA(dqA), qB(dqB), m(m), cv(cv), running_threads(threads) { + this->qA = dqA; + this->qB = dqB; - *q->ptr_p_glb_inx = sq_pInx(q); -} - -/* - * Wait on the consumer. - * - * Pausing while spinning improves performance on NetBurst and improves energy - * efficiency. - */ -Inline void sq_waitConsumer(SW_Queue q, sq_callback cb, sq_cbData cbd) -{ -#ifndef NO_CON - uint32_t threshold = (QMARGIN - 1) * QSIZE / QMARGIN; - if(sq_modSub(sq_pInx(q), *q->ptr_c_glb_inx, QMASK) > threshold) { - - /* Blocking path */ - cb(cbd); - - while(sq_modSub(sq_pInx(q), *q->ptr_c_glb_inx, QMASK) > threshold) { - usleep(10); + // consumer + if (isConsumer) { + this->qNow = dqB; + this->qOther = dqA; + } + // producer + else { + this->qNow = dqA; + this->qOther = dqB; } - } else { - - /* Fast path */ - sq_flushQueue(q); - } -#endif /* NO_CON */ -} - -/* - * Produce a value to a queue. - */ -Inline void sq_produce(SW_Queue q, uint64_t value, - sq_callback cb, sq_cbData cbd) -{ -#ifdef INSTRUMENT - q->total_produces++; -#endif - - uint64_t pDataRaw = q->p_data; - uint32_t pInx = sq_pInx(q); - uint64_t *pData = (uint64_t *) (size_t) (uint32_t) q->p_data; - uint64_t *ptr = pInx + pData; - - sq_write(ptr, value); - q->p_data = (pDataRaw + (1ULL << 32)) & HIGH_QMASK; - - // if(!(q->p_data & HIGH_CHUNKMASK)) { - // sq_waitConsumer(q, cb, cbd); - // } -} - -#define sq_produce(Q,V) sq_produce(Q, V, (sq_callback) sq_flushQueue, Q) - -/* - * Produce two values to a queue. Do not intermingle with sq_produce. - */ -Inline void sq_produce2(SW_Queue q, uint64_t a, uint64_t b, - sq_callback cb, sq_cbData cbd) -{ -#ifdef INSTRUMENT - q->total_produces += 2; -#endif - - /* Otherwise, gcc couldn't constant propagate. */ - uint64_t pDataRaw = q->p_data; - uint32_t pInx = sq_pInx(q); - uint64_t *pData = (uint64_t *) (size_t) (uint32_t) q->p_data; - uint64_t *ptr = pInx + pData; - - sq_write(ptr + 0, a); - sq_write(ptr + 1, b); - - q->p_data = (pDataRaw + (2ULL << 32)) & HIGH_QMASK; - - // if(!(q->p_data & HIGH_CHUNKMASK)) { - // sq_waitConsumer(q, cb, cbd); - // } -} - -#define sq_produce2(Q,A,B) sq_produce2(Q, A, B, (sq_callback) sq_flushQueue, Q) - - -Inline void sq_produceDouble(SW_Queue q, double value, sq_callback cb, sq_cbData cbd) { - uint64_t pDataRaw = q->p_data; - uint32_t pInx = sq_pInx(q); - uint64_t *pData = (uint64_t *) (size_t) (uint32_t) q->p_data; - uint64_t *ptr = pInx + pData; - uint64_t valueInt = (uint64_t)doubleToInt(value); - sq_write(ptr, valueInt); - q->p_data = (pDataRaw + (1ULL << 32)) & HIGH_QMASK; - if(!(q->p_data & HIGH_CHUNKMASK)) { - sq_waitConsumer(q, cb, cbd); + this->data = qNow->data; } -} - -#define sq_produceDouble(Q, V) sq_produceDouble(Q, V, (sq_callback) sq_flushQueue, Q) - -/* Functions for Consumer */ - -/* - * Makes reads globally visible - */ - -Inline void sq_reverseFlush(const SW_Queue q) { - *q->ptr_c_glb_inx = q->c_inx; -} - -/* - * Wait for a produce - */ -Inline uint32_t sq_waitAllocated(const SW_Queue q, sq_callback cb, sq_cbData cbd) -{ - if(*q->ptr_p_glb_inx == q->c_inx) { - /* Blocking path */ - cb(cbd); - while(*q->ptr_p_glb_inx == q->c_inx) usleep(10); - - } else { - /* Fast path */ - sq_reverseFlush(q); + void swap(){ + if(qNow == qA){ + qNow = qB; + qOther = qA; + }else{ + qNow = qA; + qOther = qB; + } + data = qNow->data; } - return *q->ptr_p_glb_inx; -} - -/* - * Consume a 64-bit value from the queue. - */ -Inline uint64_t sq_consume(SW_Queue q, sq_callback cb, sq_cbData cbd) -{ -#ifdef INSTRUMENT - q->total_consumes++; -#endif - - if(q->c_inx == q->c_margin) { - q->c_margin = sq_waitAllocated(q, cb, cbd); + void check() { + if (index == size) { + // only the last thread one does this + auto lock = std::unique_lock(m); + // lock is locked + if (running_threads == 1) { + qNow->ready_to_read = false; + qNow->ready_to_write = true; + // std::cerr << "Thread " << std::this_thread::get_id() << " is waiting for queue" << std::endl; + while (!qOther->ready_to_read) { + // spin + usleep(10); + } + qOther->ready_to_write = false; + // std::cerr << "Thread " << std::this_thread::get_id() << " ready for queue" << std::endl; + lock.unlock(); + // allow all other threads to continue + cv.notify_all(); + // std::cerr << "Thread " << std::this_thread::get_id() << " is ready to go" << std::endl; + } + else { + // lock is locked + running_threads--; + // std::cerr << "Thread " << std::this_thread::get_id() << " is waiting, threads: " << running_threads << std::endl; + + // wait fo the last thread to finish + cv.wait(lock); // unlocks + // std::cerr << "Thread " << std::this_thread::get_id() << " is unlocked" << std::endl; + // lock reaquires + running_threads++; + lock.unlock(); + // std::cerr << "Thread " << std::this_thread::get_id() << " is ready to go" << std::endl; + } + swap(); + index = 0; + size = qNow->size; + } } - uint64_t val = q->data[q->c_inx]; - - if(QPREFETCH) { - _mm_prefetch(q->data + q->c_inx + QPREFETCH, _MM_HINT_T0); + uint64_t consume() { + uint64_t ret = data[index]; + index++; + // _mm_prefetch(&dq_data[dq_index] + QPREFETCH, _MM_HINT_NTA); + return ret; } - q->c_inx++; - q->c_inx &= QMASK; - return val; -} - -#define sq_consume(Q) sq_consume(Q, (sq_callback) sq_reverseFlush, Q) - - -Inline double sq_consumeDouble(SW_Queue q, sq_callback cb, sq_cbData cbd) -{ -#ifdef INSTRUMENT - q->total_consumes++; -#endif - - if(q->c_inx == q->c_margin) { - q->c_margin = sq_waitAllocated(q, cb, cbd); + void consume_64_64(uint64_t &x, uint64_t &y) { + x = data[index]; + index++; + y = data[index]; + index++; + // _mm_prefetch(&dq_data[dq_index] + QPREFETCH, _MM_HINT_T0); + // _mm_prefetch(&dq_data[dq_index] + QPREFETCH, _MM_HINT_NTA); } - uint64_t val = q->data[q->c_inx]; - - if(QPREFETCH) { - _mm_prefetch(q->data + q->c_inx + QPREFETCH, _MM_HINT_T0); + void consume_32_32_64(uint32_t &x, uint32_t &y, uint64_t &z) { + uint64_t tmp = data[index]; + index++; + x = (tmp >> 32) & 0xFFFFFFFF; + y = tmp & 0xFFFFFFFF; + z = data[index]; + index++; + // _mm_prefetch(&dq_data[dq_index] + QPREFETCH, _MM_HINT_T0); + // _mm_prefetch(&dq_data[dq_index] + QPREFETCH, _MM_HINT_NTA); } - - q->c_inx++; - q->c_inx &= QMASK; - double valDouble = intToDouble((int64_t)val); - return valDouble; -} - -#define sq_consumeDouble(Q) sq_consumeDouble(Q, (sq_callback) sq_reverseFlush, Q) - -Inline bool sq_canConsume(const SW_Queue q) { - return q->c_margin != q->c_inx || *q->ptr_p_glb_inx != q->c_inx; -} - -/* - * Empty the queue. - * - * Races with producer methods. - */ -Inline void sq_emptyQueue(SW_Queue q) { - q->c_margin = q->c_inx = *q->ptr_p_glb_inx; - sq_reverseFlush(q); -} - -/* - * Modulo arithmetic (llvmism) - */ -Inline bool sq_selectProducer(unsigned prevChoice, - SW_Queue *queues, - unsigned numQueues) { - (void) queues; - - if( ++prevChoice >= numQueues ) - prevChoice = 0; - - return prevChoice; -} - -/* - * Modulo arithmetic (llvmism) - */ -Inline bool sq_selectConsumer(unsigned prevChoice, - SW_Queue *queues, - unsigned numQueues) { - - (void) queues; - - if( ++prevChoice >= numQueues ) - prevChoice = 0; - - return prevChoice; -} - -#ifdef __cplusplus -} -#endif +}; #endif /* SW_QUEUE_H */ diff --git a/tests/scripts/slamp-driver b/tests/scripts/slamp-driver index 03a103ae..6baeec20 100755 --- a/tests/scripts/slamp-driver +++ b/tests/scripts/slamp-driver @@ -135,7 +135,7 @@ function drive { CMD4="taskset --cpu-list 7-8 $CMD4" rm /dev/shm/MySharedMemory2 rm /dev/shm/MySharedMemory - taskset --cpu-list 0-6,9-27 /u/NAS_SCRATCH/ziyangx/cpf-workspace-9-master/cpf/liberty/lib/SLAMP/SLAMPcustom/consumer/build/consumer & + taskset --cpu-list 0-6,9-55 /u/NAS_SCRATCH/ziyangx/cpf-workspace-9-master/cpf/liberty/lib/SLAMP/SLAMPcustom/consumer/build/consumer & # taskset --cpu-list 0-6,9-27 /u/NAS_SCRATCH/ziyangx/cpf-workspace-9-master/cpf/liberty/lib/SLAMP/SLAMPsmtxq/consumer/build/consumer & CONSUME_PID=$! RUN_TIMEOUT=14400 # 4 hours From d0c4c21b99a2ae6235ee5f63e8a39767090b1c95 Mon Sep 17 00:00:00 2001 From: Ziyang Xu Date: Mon, 7 Nov 2022 11:34:56 -0500 Subject: [PATCH 36/97] remove static log time update --- .../consumer/ProfilingModules/DependenceModule.cpp | 8 ++++---- .../lib/SLAMP/SLAMPcustom/consumer/consumer.cpp | 14 ++++++++------ 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.cpp b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.cpp index 0eabd25f..8ffbfc8d 100644 --- a/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.cpp +++ b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/DependenceModule.cpp @@ -46,7 +46,7 @@ void DependenceModule::fini(const char *filename) { // of << "\n"; // } - std::cout << "Log time: " << log_time/ 2.6e9 << " s" << std::endl; + // std::cout << "Log time: " << log_time/ 2.6e9 << " s" << std::endl; // for (auto &i : *inst_count) { // of << target_loop_id << " " << i.first << " " << i.second << "\n"; @@ -90,10 +90,10 @@ void DependenceModule::load(uint32_t instr, const uint64_t addr, const uint32_t TS tss = s[0]; if (tss != 0) { - uint64_t start = rdtsc(); + // uint64_t start = rdtsc(); log(tss, instr, instr, slamp_invocation, slamp_iteration); - uint64_t end = rdtsc(); - log_time += end - start; + // uint64_t end = rdtsc(); + // log_time += end - start; } }); } diff --git a/liberty/lib/SLAMP/SLAMPcustom/consumer/consumer.cpp b/liberty/lib/SLAMP/SLAMPcustom/consumer/consumer.cpp index 037528ed..604b3953 100644 --- a/liberty/lib/SLAMP/SLAMPcustom/consumer/consumer.cpp +++ b/liberty/lib/SLAMP/SLAMPcustom/consumer/consumer.cpp @@ -156,9 +156,11 @@ void consume_loop(DoubleQueue &dq, DependenceModule &depMod) { << " events" << std::endl; // print time in seconds std::cout << "Total time: " << total_cycles / 2.6e9 << " s" << std::endl; - std::cout << "Load time: " << load_time / 2.6e9 << " s" << std::endl; - std::cout << "Store time: " << store_time / 2.6e9 << " s" << std::endl; - std::cout << "Alloc time: " << alloc_time / 2.6e9 << " s" << std::endl; + if (MEASURE_TIME) { + std::cout << "Load time: " << load_time / 2.6e9 << " s" << std::endl; + std::cout << "Store time: " << store_time / 2.6e9 << " s" << std::endl; + std::cout << "Alloc time: " << alloc_time / 2.6e9 << " s" << std::endl; + } exit(0); break; @@ -176,9 +178,9 @@ void consume_loop(DoubleQueue &dq, DependenceModule &depMod) { exit(-1); } - if (counter % 100'000'000 == 0) { - std::cout << "Processed " << counter / 1'000'000 << "M events" << std::endl; - } + // if (counter % 100'000'000 == 0) { + // std::cout << "Processed " << counter / 1'000'000 << "M events" << std::endl; + // } } } From 6a9a11a3a3e819fb26385ec375f0e6447a59c3ae Mon Sep 17 00:00:00 2001 From: Ziyang Xu Date: Mon, 7 Nov 2022 13:47:16 -0500 Subject: [PATCH 37/97] fix repl --- liberty/lib/Repl/repl.cpp | 55 +++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/liberty/lib/Repl/repl.cpp b/liberty/lib/Repl/repl.cpp index 344d8b3f..71202685 100644 --- a/liberty/lib/Repl/repl.cpp +++ b/liberty/lib/Repl/repl.cpp @@ -525,40 +525,45 @@ bool OptRepl::runOnModule(Module &M) { // try all loopAA, from only the first one, to all of them, the last one is always NoLoopAA liberty::LoopAA::ModRefResult lastRet[3] = {liberty::LoopAA::ModRef, liberty::LoopAA::ModRef, liberty::LoopAA::ModRef}; - for (auto i = 1; i < numLoopAAs - 1; i++) { + for (auto i = 1; i <= numLoopAAs - 1; i++) { // set the first i loopAA to be enabled(loopAAEnabled[i] = true) // and the rest to be disabled (loopAAEnabled[i] = false) + // [0~i-1] for (auto j = 0; j < i; j++) { loopAAEnabled[j] = true; } - for (auto j = i; j < numLoopAAs; j++) { + + // [i~numLoopAAs-2] + for (auto j = i; j < numLoopAAs - 1; j++) { loopAAEnabled[j] = false; } - - // configure the loop AAs - for (auto j = 0; j < i; j++) { - // if (j >= i) { - // loopAAs[j]->disable(); - // continue; - // } - - // set up the correct prev and next - LoopAA *prev, *next; - if (j == 0) { - prev = nullptr; - } else { - prev = loopAAs[j - 1]; + // the last one is always NoLoopAA and enabled + loopAAEnabled[numLoopAAs - 1] = true; + + // configure the loop AAs prev and next based on the enabled/disabled setting + LoopAA *prev, *cur, *next; + prev = nullptr; + cur = nullptr; + next = nullptr; + // cur is the first enabled, next is the second enabled + for (auto j = 0; j < numLoopAAs; j++) { + if (loopAAEnabled[j]) { + // the first one + if (!cur) { + cur = loopAAs[j]; + continue; + } else { + next = loopAAs[j]; + cur->configure(prev, next); + prev = cur; + cur = next; + } } - if (j == i - 1) { - // NoLoopAA - next = loopAAs[numLoopAAs - 1]; - } else { - next = loopAAs[j + 1]; - } - - loopAAs[j]->configure(prev, next); } - loopAAs[numLoopAAs - 1]->configure(loopAAs[i - 1], nullptr); + + // NoLoopAA is always enabled + assert(cur->getLoopAAName() == "NoLoopAA"); + cur->configure(prev, nullptr); // aa->dump(); auto ret = aa->modref(fromInst, liberty::LoopAA::TemporalRelation::Same, toInst, selectedLoop, remeds); From 03cfbadf8b6e4e0926291723c36e6d5f47113dcb Mon Sep 17 00:00:00 2001 From: Ziyang Xu Date: Mon, 7 Nov 2022 15:37:59 -0500 Subject: [PATCH 38/97] switch to parallel hashmap's unordered set impl --- .../consumer/ProfilingModules/HTContainer.h | 19 +- .../ProfilingModules/parallel_hashmap/btree.h | 4087 +++++++++++++ .../parallel_hashmap/conanfile.py | 37 + .../parallel_hashmap/meminfo.h | 195 + .../ProfilingModules/parallel_hashmap/phmap.h | 5118 ++++++++++++++++ .../parallel_hashmap/phmap_base.h | 5157 +++++++++++++++++ .../parallel_hashmap/phmap_bits.h | 664 +++ .../parallel_hashmap/phmap_config.h | 789 +++ .../parallel_hashmap/phmap_dump.h | 288 + .../parallel_hashmap/phmap_fwd_decl.h | 154 + .../parallel_hashmap/phmap_utils.h | 417 ++ 11 files changed, 16922 insertions(+), 3 deletions(-) create mode 100644 liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/parallel_hashmap/btree.h create mode 100644 liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/parallel_hashmap/conanfile.py create mode 100644 liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/parallel_hashmap/meminfo.h create mode 100644 liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/parallel_hashmap/phmap.h create mode 100644 liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/parallel_hashmap/phmap_base.h create mode 100644 liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/parallel_hashmap/phmap_bits.h create mode 100644 liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/parallel_hashmap/phmap_config.h create mode 100644 liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/parallel_hashmap/phmap_dump.h create mode 100644 liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/parallel_hashmap/phmap_fwd_decl.h create mode 100644 liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/parallel_hashmap/phmap_utils.h diff --git a/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/HTContainer.h b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/HTContainer.h index a35b5e0d..87cacd62 100644 --- a/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/HTContainer.h +++ b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/HTContainer.h @@ -8,9 +8,22 @@ */ #include #include -#include #include +#define PB +#ifdef PB +#include "parallel_hashmap/phmap.h" +#else +#include +#endif + +#ifdef PB +#define hash_set phmap::flat_hash_set +#else +#define hash_set std::unordered_set +#endif + + template , typename KeyEqual = std::equal_to, uint32_t MAX_THREAD = 56, @@ -21,7 +34,7 @@ class HTSet { std::mutex m; public: - std::unordered_set set; + hash_set set; HTSet() { buffer.reserve(BUFFER_SIZE); } void emplace_back(T &&t) { @@ -80,7 +93,7 @@ class HTSet { t[i] = std::thread( [&](int id) { // take the chunk and convert to a set and return - auto *set_chunk = new std::unordered_set(); + auto *set_chunk = new hash_set(); set_chunk->reserve(set_size); auto begin = id * (buffer_size / thread_count); diff --git a/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/parallel_hashmap/btree.h b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/parallel_hashmap/btree.h new file mode 100644 index 00000000..8aee516a --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/parallel_hashmap/btree.h @@ -0,0 +1,4087 @@ +// --------------------------------------------------------------------------- +// Copyright (c) 2019, Gregory Popovitch - greg7mdp@gmail.com +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Includes work from abseil-cpp (https://github.com/abseil/abseil-cpp) +// with modifications. +// +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// --------------------------------------------------------------------------- + +#ifndef PHMAP_BTREE_BTREE_CONTAINER_H_ +#define PHMAP_BTREE_BTREE_CONTAINER_H_ + +#ifdef _MSC_VER + #pragma warning(push) + + #pragma warning(disable : 4127) // conditional expression is constant + #pragma warning(disable : 4324) // structure was padded due to alignment specifier + #pragma warning(disable : 4355) // 'this': used in base member initializer list + #pragma warning(disable : 4365) // conversion from 'int' to 'const unsigned __int64', signed/unsigned mismatch + #pragma warning(disable : 4514) // unreferenced inline function has been removed + #pragma warning(disable : 4623) // default constructor was implicitly defined as deleted + #pragma warning(disable : 4625) // copy constructor was implicitly defined as deleted + #pragma warning(disable : 4626) // assignment operator was implicitly defined as deleted + #pragma warning(disable : 4710) // function not inlined + #pragma warning(disable : 4711) // selected for automatic inline expansion + #pragma warning(disable : 4820) // '6' bytes padding added after data member + #pragma warning(disable : 4868) // compiler may not enforce left-to-right evaluation order in braced initializer list + #pragma warning(disable : 5026) // move constructor was implicitly defined as deleted + #pragma warning(disable : 5027) // move assignment operator was implicitly defined as deleted + #pragma warning(disable : 5045) // Compiler will insert Spectre mitigation for memory load if /Qspectre switch specified +#endif + + +#include +#include +#include +#include +#include + +#include "phmap_fwd_decl.h" +#include "phmap_base.h" + +#if PHMAP_HAVE_STD_STRING_VIEW + #include +#endif + +// MSVC constructibility traits do not detect destructor properties and so our +// implementations should not use them as a source-of-truth. +#if defined(_MSC_VER) && !defined(__clang__) && !defined(__GNUC__) + #define PHMAP_META_INTERNAL_STD_CONSTRUCTION_TRAITS_DONT_CHECK_DESTRUCTION 1 +#endif + +namespace phmap { + + // Defined and documented later on in this file. + template + struct is_trivially_destructible; + + // Defined and documented later on in this file. + template + struct is_trivially_move_assignable; + + namespace type_traits_internal { + + // Silence MSVC warnings about the destructor being defined as deleted. +#if defined(_MSC_VER) && !defined(__GNUC__) + #pragma warning(push) + #pragma warning(disable : 4624) +#endif // defined(_MSC_VER) && !defined(__GNUC__) + + template + union SingleMemberUnion { + T t; + }; + + // Restore the state of the destructor warning that was silenced above. +#if defined(_MSC_VER) && !defined(__GNUC__) + #pragma warning(pop) +#endif // defined(_MSC_VER) && !defined(__GNUC__) + + template + struct IsTriviallyMoveConstructibleObject + : std::integral_constant< + bool, std::is_move_constructible< + type_traits_internal::SingleMemberUnion>::value && + phmap::is_trivially_destructible::value> {}; + + template + struct IsTriviallyCopyConstructibleObject + : std::integral_constant< + bool, std::is_copy_constructible< + type_traits_internal::SingleMemberUnion>::value && + phmap::is_trivially_destructible::value> {}; + + template + struct IsTriviallyMoveAssignableReference : std::false_type {}; + + template + struct IsTriviallyMoveAssignableReference + : phmap::is_trivially_move_assignable::type {}; + + template + struct IsTriviallyMoveAssignableReference + : phmap::is_trivially_move_assignable::type {}; + + } // namespace type_traits_internal + + + template + using void_t = typename type_traits_internal::VoidTImpl::type; + + + template + struct is_function + : std::integral_constant< + bool, !(std::is_reference::value || + std::is_const::type>::value)> {}; + + + namespace type_traits_internal { + + template + class is_trivially_copyable_impl { + using ExtentsRemoved = typename std::remove_all_extents::type; + static constexpr bool kIsCopyOrMoveConstructible = + std::is_copy_constructible::value || + std::is_move_constructible::value; + static constexpr bool kIsCopyOrMoveAssignable = + phmap::is_copy_assignable::value || + phmap::is_move_assignable::value; + + public: + static constexpr bool kValue = + (__has_trivial_copy(ExtentsRemoved) || !kIsCopyOrMoveConstructible) && + (__has_trivial_assign(ExtentsRemoved) || !kIsCopyOrMoveAssignable) && + (kIsCopyOrMoveConstructible || kIsCopyOrMoveAssignable) && + is_trivially_destructible::value && + // We need to check for this explicitly because otherwise we'll say + // references are trivial copyable when compiled by MSVC. + !std::is_reference::value; + }; + + template + struct is_trivially_copyable + : std::integral_constant< + bool, type_traits_internal::is_trivially_copyable_impl::kValue> {}; + } // namespace type_traits_internal + + namespace swap_internal { + + // Necessary for the traits. + using std::swap; + + // This declaration prevents global `swap` and `phmap::swap` overloads from being + // considered unless ADL picks them up. + void swap(); + + template + using IsSwappableImpl = decltype(swap(std::declval(), std::declval())); + + // NOTE: This dance with the default template parameter is for MSVC. + template (), std::declval()))>> + using IsNothrowSwappableImpl = typename std::enable_if::type; + + template + struct IsSwappable + : phmap::type_traits_internal::is_detected {}; + + template + struct IsNothrowSwappable + : phmap::type_traits_internal::is_detected {}; + + template ::value, int> = 0> + void Swap(T& lhs, T& rhs) noexcept(IsNothrowSwappable::value) { + swap(lhs, rhs); + } + + using StdSwapIsUnconstrained = IsSwappable; + + } // namespace swap_internal + + namespace type_traits_internal { + + // Make the swap-related traits/function accessible from this namespace. + using swap_internal::IsNothrowSwappable; + using swap_internal::IsSwappable; + using swap_internal::Swap; + using swap_internal::StdSwapIsUnconstrained; + + } // namespace type_traits_internal + + namespace compare_internal { + + using value_type = int8_t; + + template + struct Fail { + static_assert(sizeof(T) < 0, "Only literal `0` is allowed."); + }; + + template + struct OnlyLiteralZero { + constexpr OnlyLiteralZero(NullPtrT) noexcept {} // NOLINT + + template < + typename T, + typename = typename std::enable_if< + std::is_same::value || + (std::is_integral::value && !std::is_same::value)>::type, + typename = typename Fail::type> + OnlyLiteralZero(T); // NOLINT + }; + + enum class eq : value_type { + equal = 0, + equivalent = equal, + nonequal = 1, + nonequivalent = nonequal, + }; + + enum class ord : value_type { less = -1, greater = 1 }; + + enum class ncmp : value_type { unordered = -127 }; + +#if defined(__cpp_inline_variables) && !defined(_MSC_VER) + +#define PHMAP_COMPARE_INLINE_BASECLASS_DECL(name) + +#define PHMAP_COMPARE_INLINE_SUBCLASS_DECL(type, name) \ + static const type name; + +#define PHMAP_COMPARE_INLINE_INIT(type, name, init) \ + inline constexpr type type::name(init) + +#else // __cpp_inline_variables + +#define PHMAP_COMPARE_INLINE_BASECLASS_DECL(name) \ + static const T name; + +#define PHMAP_COMPARE_INLINE_SUBCLASS_DECL(type, name) + +#define PHMAP_COMPARE_INLINE_INIT(type, name, init) \ + template \ + const T compare_internal::type##_base::name(init) + +#endif // __cpp_inline_variables + + // These template base classes allow for defining the values of the constants + // in the header file (for performance) without using inline variables (which + // aren't available in C++11). + template + struct weak_equality_base { + PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent) + PHMAP_COMPARE_INLINE_BASECLASS_DECL(nonequivalent) + }; + + template + struct strong_equality_base { + PHMAP_COMPARE_INLINE_BASECLASS_DECL(equal) + PHMAP_COMPARE_INLINE_BASECLASS_DECL(nonequal) + PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent) + PHMAP_COMPARE_INLINE_BASECLASS_DECL(nonequivalent) + }; + + template + struct partial_ordering_base { + PHMAP_COMPARE_INLINE_BASECLASS_DECL(less) + PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent) + PHMAP_COMPARE_INLINE_BASECLASS_DECL(greater) + PHMAP_COMPARE_INLINE_BASECLASS_DECL(unordered) + }; + + template + struct weak_ordering_base { + PHMAP_COMPARE_INLINE_BASECLASS_DECL(less) + PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent) + PHMAP_COMPARE_INLINE_BASECLASS_DECL(greater) + }; + + template + struct strong_ordering_base { + PHMAP_COMPARE_INLINE_BASECLASS_DECL(less) + PHMAP_COMPARE_INLINE_BASECLASS_DECL(equal) + PHMAP_COMPARE_INLINE_BASECLASS_DECL(equivalent) + PHMAP_COMPARE_INLINE_BASECLASS_DECL(greater) + }; + + } // namespace compare_internal + + class weak_equality + : public compare_internal::weak_equality_base { + explicit constexpr weak_equality(compare_internal::eq v) noexcept + : value_(static_cast(v)) {} + friend struct compare_internal::weak_equality_base; + + public: + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_equality, equivalent) + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_equality, nonequivalent) + + // Comparisons + friend constexpr bool operator==( + weak_equality v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ == 0; + } + friend constexpr bool operator!=( + weak_equality v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ != 0; + } + friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, + weak_equality v) noexcept { + return 0 == v.value_; + } + friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, + weak_equality v) noexcept { + return 0 != v.value_; + } + + private: + compare_internal::value_type value_; + }; + PHMAP_COMPARE_INLINE_INIT(weak_equality, equivalent, + compare_internal::eq::equivalent); + PHMAP_COMPARE_INLINE_INIT(weak_equality, nonequivalent, + compare_internal::eq::nonequivalent); + + class strong_equality + : public compare_internal::strong_equality_base { + explicit constexpr strong_equality(compare_internal::eq v) noexcept + : value_(static_cast(v)) {} + friend struct compare_internal::strong_equality_base; + + public: + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_equality, equal) + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_equality, nonequal) + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_equality, equivalent) + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_equality, nonequivalent) + + // Conversion + constexpr operator weak_equality() const noexcept { // NOLINT + return value_ == 0 ? weak_equality::equivalent + : weak_equality::nonequivalent; + } + // Comparisons + friend constexpr bool operator==( + strong_equality v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ == 0; + } + friend constexpr bool operator!=( + strong_equality v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ != 0; + } + friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, + strong_equality v) noexcept { + return 0 == v.value_; + } + friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, + strong_equality v) noexcept { + return 0 != v.value_; + } + + private: + compare_internal::value_type value_; + }; + + PHMAP_COMPARE_INLINE_INIT(strong_equality, equal, compare_internal::eq::equal); + PHMAP_COMPARE_INLINE_INIT(strong_equality, nonequal, + compare_internal::eq::nonequal); + PHMAP_COMPARE_INLINE_INIT(strong_equality, equivalent, + compare_internal::eq::equivalent); + PHMAP_COMPARE_INLINE_INIT(strong_equality, nonequivalent, + compare_internal::eq::nonequivalent); + + class partial_ordering + : public compare_internal::partial_ordering_base { + explicit constexpr partial_ordering(compare_internal::eq v) noexcept + : value_(static_cast(v)) {} + explicit constexpr partial_ordering(compare_internal::ord v) noexcept + : value_(static_cast(v)) {} + explicit constexpr partial_ordering(compare_internal::ncmp v) noexcept + : value_(static_cast(v)) {} + friend struct compare_internal::partial_ordering_base; + + constexpr bool is_ordered() const noexcept { + return value_ != + compare_internal::value_type(compare_internal::ncmp::unordered); + } + + public: + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(partial_ordering, less) + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(partial_ordering, equivalent) + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(partial_ordering, greater) + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(partial_ordering, unordered) + + // Conversion + constexpr operator weak_equality() const noexcept { // NOLINT + return value_ == 0 ? weak_equality::equivalent + : weak_equality::nonequivalent; + } + // Comparisons + friend constexpr bool operator==( + partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.is_ordered() && v.value_ == 0; + } + friend constexpr bool operator!=( + partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return !v.is_ordered() || v.value_ != 0; + } + friend constexpr bool operator<( + partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.is_ordered() && v.value_ < 0; + } + friend constexpr bool operator<=( + partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.is_ordered() && v.value_ <= 0; + } + friend constexpr bool operator>( + partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.is_ordered() && v.value_ > 0; + } + friend constexpr bool operator>=( + partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.is_ordered() && v.value_ >= 0; + } + friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, + partial_ordering v) noexcept { + return v.is_ordered() && 0 == v.value_; + } + friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, + partial_ordering v) noexcept { + return !v.is_ordered() || 0 != v.value_; + } + friend constexpr bool operator<(compare_internal::OnlyLiteralZero<>, + partial_ordering v) noexcept { + return v.is_ordered() && 0 < v.value_; + } + friend constexpr bool operator<=(compare_internal::OnlyLiteralZero<>, + partial_ordering v) noexcept { + return v.is_ordered() && 0 <= v.value_; + } + friend constexpr bool operator>(compare_internal::OnlyLiteralZero<>, + partial_ordering v) noexcept { + return v.is_ordered() && 0 > v.value_; + } + friend constexpr bool operator>=(compare_internal::OnlyLiteralZero<>, + partial_ordering v) noexcept { + return v.is_ordered() && 0 >= v.value_; + } + + private: + compare_internal::value_type value_; + }; + + PHMAP_COMPARE_INLINE_INIT(partial_ordering, less, compare_internal::ord::less); + PHMAP_COMPARE_INLINE_INIT(partial_ordering, equivalent, + compare_internal::eq::equivalent); + PHMAP_COMPARE_INLINE_INIT(partial_ordering, greater, + compare_internal::ord::greater); + PHMAP_COMPARE_INLINE_INIT(partial_ordering, unordered, + compare_internal::ncmp::unordered); + + class weak_ordering + : public compare_internal::weak_ordering_base { + explicit constexpr weak_ordering(compare_internal::eq v) noexcept + : value_(static_cast(v)) {} + explicit constexpr weak_ordering(compare_internal::ord v) noexcept + : value_(static_cast(v)) {} + friend struct compare_internal::weak_ordering_base; + + public: + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_ordering, less) + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_ordering, equivalent) + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(weak_ordering, greater) + + // Conversions + constexpr operator weak_equality() const noexcept { // NOLINT + return value_ == 0 ? weak_equality::equivalent + : weak_equality::nonequivalent; + } + constexpr operator partial_ordering() const noexcept { // NOLINT + return value_ == 0 ? partial_ordering::equivalent + : (value_ < 0 ? partial_ordering::less + : partial_ordering::greater); + } + // Comparisons + friend constexpr bool operator==( + weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ == 0; + } + friend constexpr bool operator!=( + weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ != 0; + } + friend constexpr bool operator<( + weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ < 0; + } + friend constexpr bool operator<=( + weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ <= 0; + } + friend constexpr bool operator>( + weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ > 0; + } + friend constexpr bool operator>=( + weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ >= 0; + } + friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, + weak_ordering v) noexcept { + return 0 == v.value_; + } + friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, + weak_ordering v) noexcept { + return 0 != v.value_; + } + friend constexpr bool operator<(compare_internal::OnlyLiteralZero<>, + weak_ordering v) noexcept { + return 0 < v.value_; + } + friend constexpr bool operator<=(compare_internal::OnlyLiteralZero<>, + weak_ordering v) noexcept { + return 0 <= v.value_; + } + friend constexpr bool operator>(compare_internal::OnlyLiteralZero<>, + weak_ordering v) noexcept { + return 0 > v.value_; + } + friend constexpr bool operator>=(compare_internal::OnlyLiteralZero<>, + weak_ordering v) noexcept { + return 0 >= v.value_; + } + + private: + compare_internal::value_type value_; + }; + + PHMAP_COMPARE_INLINE_INIT(weak_ordering, less, compare_internal::ord::less); + PHMAP_COMPARE_INLINE_INIT(weak_ordering, equivalent, + compare_internal::eq::equivalent); + PHMAP_COMPARE_INLINE_INIT(weak_ordering, greater, + compare_internal::ord::greater); + + class strong_ordering + : public compare_internal::strong_ordering_base { + explicit constexpr strong_ordering(compare_internal::eq v) noexcept + : value_(static_cast(v)) {} + explicit constexpr strong_ordering(compare_internal::ord v) noexcept + : value_(static_cast(v)) {} + friend struct compare_internal::strong_ordering_base; + + public: + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_ordering, less) + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_ordering, equal) + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_ordering, equivalent) + PHMAP_COMPARE_INLINE_SUBCLASS_DECL(strong_ordering, greater) + + // Conversions + constexpr operator weak_equality() const noexcept { // NOLINT + return value_ == 0 ? weak_equality::equivalent + : weak_equality::nonequivalent; + } + constexpr operator strong_equality() const noexcept { // NOLINT + return value_ == 0 ? strong_equality::equal : strong_equality::nonequal; + } + constexpr operator partial_ordering() const noexcept { // NOLINT + return value_ == 0 ? partial_ordering::equivalent + : (value_ < 0 ? partial_ordering::less + : partial_ordering::greater); + } + constexpr operator weak_ordering() const noexcept { // NOLINT + return value_ == 0 + ? weak_ordering::equivalent + : (value_ < 0 ? weak_ordering::less : weak_ordering::greater); + } + // Comparisons + friend constexpr bool operator==( + strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ == 0; + } + friend constexpr bool operator!=( + strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ != 0; + } + friend constexpr bool operator<( + strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ < 0; + } + friend constexpr bool operator<=( + strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ <= 0; + } + friend constexpr bool operator>( + strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ > 0; + } + friend constexpr bool operator>=( + strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ >= 0; + } + friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, + strong_ordering v) noexcept { + return 0 == v.value_; + } + friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, + strong_ordering v) noexcept { + return 0 != v.value_; + } + friend constexpr bool operator<(compare_internal::OnlyLiteralZero<>, + strong_ordering v) noexcept { + return 0 < v.value_; + } + friend constexpr bool operator<=(compare_internal::OnlyLiteralZero<>, + strong_ordering v) noexcept { + return 0 <= v.value_; + } + friend constexpr bool operator>(compare_internal::OnlyLiteralZero<>, + strong_ordering v) noexcept { + return 0 > v.value_; + } + friend constexpr bool operator>=(compare_internal::OnlyLiteralZero<>, + strong_ordering v) noexcept { + return 0 >= v.value_; + } + + private: + compare_internal::value_type value_; + }; + PHMAP_COMPARE_INLINE_INIT(strong_ordering, less, compare_internal::ord::less); + PHMAP_COMPARE_INLINE_INIT(strong_ordering, equal, compare_internal::eq::equal); + PHMAP_COMPARE_INLINE_INIT(strong_ordering, equivalent, + compare_internal::eq::equivalent); + PHMAP_COMPARE_INLINE_INIT(strong_ordering, greater, + compare_internal::ord::greater); + +#undef PHMAP_COMPARE_INLINE_BASECLASS_DECL +#undef PHMAP_COMPARE_INLINE_SUBCLASS_DECL +#undef PHMAP_COMPARE_INLINE_INIT + + namespace compare_internal { + // We also provide these comparator adapter functions for internal phmap use. + + // Helper functions to do a boolean comparison of two keys given a boolean + // or three-way comparator. + // SFINAE prevents implicit conversions to bool (such as from int). + template ::value, int> = 0> + constexpr bool compare_result_as_less_than(const BoolType r) { return r; } + constexpr bool compare_result_as_less_than(const phmap::weak_ordering r) { + return r < 0; + } + + template + constexpr bool do_less_than_comparison(const Compare &compare, const K &x, + const LK &y) { + return compare_result_as_less_than(compare(x, y)); + } + + // Helper functions to do a three-way comparison of two keys given a boolean or + // three-way comparator. + // SFINAE prevents implicit conversions to int (such as from bool). + template ::value, int> = 0> + constexpr phmap::weak_ordering compare_result_as_ordering(const Int c) { + return c < 0 ? phmap::weak_ordering::less + : c == 0 ? phmap::weak_ordering::equivalent + : phmap::weak_ordering::greater; + } + constexpr phmap::weak_ordering compare_result_as_ordering( + const phmap::weak_ordering c) { + return c; + } + + template < + typename Compare, typename K, typename LK, + phmap::enable_if_t>::value, + int> = 0> + constexpr phmap::weak_ordering do_three_way_comparison(const Compare &compare, + const K &x, const LK &y) { + return compare_result_as_ordering(compare(x, y)); + } + template < + typename Compare, typename K, typename LK, + phmap::enable_if_t>::value, + int> = 0> + constexpr phmap::weak_ordering do_three_way_comparison(const Compare &compare, + const K &x, const LK &y) { + return compare(x, y) ? phmap::weak_ordering::less + : compare(y, x) ? phmap::weak_ordering::greater + : phmap::weak_ordering::equivalent; + } + + } // namespace compare_internal +} + + +namespace phmap { + +namespace priv { + + // A helper class that indicates if the Compare parameter is a key-compare-to + // comparator. + template + using btree_is_key_compare_to = + std::is_convertible, + phmap::weak_ordering>; + + struct StringBtreeDefaultLess { + using is_transparent = void; + + StringBtreeDefaultLess() = default; + + // Compatibility constructor. + StringBtreeDefaultLess(std::less) {} // NOLINT +#if PHMAP_HAVE_STD_STRING_VIEW + StringBtreeDefaultLess(std::less) {} // NOLINT + StringBtreeDefaultLess(phmap::Less) {} // NOLINT + + phmap::weak_ordering operator()(std::string_view lhs, + std::string_view rhs) const { + return compare_internal::compare_result_as_ordering(lhs.compare(rhs)); + } +#else + phmap::weak_ordering operator()(std::string lhs, + std::string rhs) const { + return compare_internal::compare_result_as_ordering(lhs.compare(rhs)); + } +#endif + }; + + struct StringBtreeDefaultGreater { + using is_transparent = void; + + StringBtreeDefaultGreater() = default; + + StringBtreeDefaultGreater(std::greater) {} // NOLINT +#if PHMAP_HAVE_STD_STRING_VIEW + StringBtreeDefaultGreater(std::greater) {} // NOLINT + + phmap::weak_ordering operator()(std::string_view lhs, + std::string_view rhs) const { + return compare_internal::compare_result_as_ordering(rhs.compare(lhs)); + } +#else + phmap::weak_ordering operator()(std::string lhs, + std::string rhs) const { + return compare_internal::compare_result_as_ordering(rhs.compare(lhs)); + } +#endif + }; + + // A helper class to convert a boolean comparison into a three-way "compare-to" + // comparison that returns a negative value to indicate less-than, zero to + // indicate equality and a positive value to indicate greater-than. This helper + // class is specialized for less, greater, + // less, and greater. + // + // key_compare_to_adapter is provided so that btree users + // automatically get the more efficient compare-to code when using common + // google string types with common comparison functors. + // These string-like specializations also turn on heterogeneous lookup by + // default. + template + struct key_compare_to_adapter { + using type = Compare; + }; + + template <> + struct key_compare_to_adapter> { + using type = StringBtreeDefaultLess; + }; + + template <> + struct key_compare_to_adapter> { + using type = StringBtreeDefaultLess; + }; + + template <> + struct key_compare_to_adapter> { + using type = StringBtreeDefaultGreater; + }; + +#if PHMAP_HAVE_STD_STRING_VIEW + template <> + struct key_compare_to_adapter> { + using type = StringBtreeDefaultLess; + }; + + template <> + struct key_compare_to_adapter> { + using type = StringBtreeDefaultLess; + }; + + template <> + struct key_compare_to_adapter> { + using type = StringBtreeDefaultGreater; + }; +#endif + + template + struct common_params { + // If Compare is a common comparator for a std::string-like type, then we adapt it + // to use heterogeneous lookup and to be a key-compare-to comparator. + using key_compare = typename key_compare_to_adapter::type; + // A type which indicates if we have a key-compare-to functor or a plain old + // key-compare functor. + using is_key_compare_to = btree_is_key_compare_to; + + using allocator_type = Alloc; + using key_type = Key; + using size_type = std::size_t ; + using difference_type = ptrdiff_t; + + // True if this is a multiset or multimap. + using is_multi_container = std::integral_constant; + + using slot_policy = SlotPolicy; + using slot_type = typename slot_policy::slot_type; + using value_type = typename slot_policy::value_type; + using init_type = typename slot_policy::mutable_value_type; + using pointer = value_type *; + using const_pointer = const value_type *; + using reference = value_type &; + using const_reference = const value_type &; + + enum { + kTargetNodeSize = TargetNodeSize, + + // Upper bound for the available space for values. This is largest for leaf + // nodes, which have overhead of at least a pointer + 4 bytes (for storing + // 3 field_types and an enum). + kNodeSlotSpace = + TargetNodeSize - /*minimum overhead=*/(sizeof(void *) + 4), + }; + + // This is an integral type large enough to hold as many + // ValueSize-values as will fit a node of TargetNodeSize bytes. + using node_count_type = + phmap::conditional_t<(kNodeSlotSpace / sizeof(slot_type) > + (std::numeric_limits::max)()), + uint16_t, uint8_t>; // NOLINT + + // The following methods are necessary for passing this struct as PolicyTraits + // for node_handle and/or are used within btree. + static value_type &element(slot_type *slot) { + return slot_policy::element(slot); + } + static const value_type &element(const slot_type *slot) { + return slot_policy::element(slot); + } + template + static void construct(Alloc *alloc, slot_type *slot, Args &&... args) { + slot_policy::construct(alloc, slot, std::forward(args)...); + } + static void construct(Alloc *alloc, slot_type *slot, slot_type *other) { + slot_policy::construct(alloc, slot, other); + } + static void destroy(Alloc *alloc, slot_type *slot) { + slot_policy::destroy(alloc, slot); + } + static void transfer(Alloc *alloc, slot_type *new_slot, slot_type *old_slot) { + construct(alloc, new_slot, old_slot); + destroy(alloc, old_slot); + } + static void swap(Alloc *alloc, slot_type *a, slot_type *b) { + slot_policy::swap(alloc, a, b); + } + static void move(Alloc *alloc, slot_type *src, slot_type *dest) { + slot_policy::move(alloc, src, dest); + } + static void move(Alloc *alloc, slot_type *first, slot_type *last, + slot_type *result) { + slot_policy::move(alloc, first, last, result); + } + }; + + // A parameters structure for holding the type parameters for a btree_map. + // Compare and Alloc should be nothrow copy-constructible. + template + struct map_params : common_params> { + using super_type = typename map_params::common_params; + using mapped_type = Data; + // This type allows us to move keys when it is safe to do so. It is safe + // for maps in which value_type and mutable_value_type are layout compatible. + using slot_policy = typename super_type::slot_policy; + using slot_type = typename super_type::slot_type; + using value_type = typename super_type::value_type; + using init_type = typename super_type::init_type; + + using key_compare = typename super_type::key_compare; + // Inherit from key_compare for empty base class optimization. + struct value_compare : private key_compare { + value_compare() = default; + explicit value_compare(const key_compare &cmp) : key_compare(cmp) {} + + template + auto operator()(const T &left, const U &right) const + -> decltype(std::declval()(left.first, right.first)) { + return key_compare::operator()(left.first, right.first); + } + }; + using is_map_container = std::true_type; + + static const Key &key(const value_type &x) { return x.first; } + static const Key &key(const init_type &x) { return x.first; } + static const Key &key(const slot_type *x) { return slot_policy::key(x); } + static mapped_type &value(value_type *value) { return value->second; } + }; + + // This type implements the necessary functions from the + // btree::priv::slot_type interface. + template + struct set_slot_policy { + using slot_type = Key; + using value_type = Key; + using mutable_value_type = Key; + + static value_type &element(slot_type *slot) { return *slot; } + static const value_type &element(const slot_type *slot) { return *slot; } + + template + static void construct(Alloc *alloc, slot_type *slot, Args &&... args) { + phmap::allocator_traits::construct(*alloc, slot, + std::forward(args)...); + } + + template + static void construct(Alloc *alloc, slot_type *slot, slot_type *other) { + phmap::allocator_traits::construct(*alloc, slot, std::move(*other)); + } + + template + static void destroy(Alloc *alloc, slot_type *slot) { + phmap::allocator_traits::destroy(*alloc, slot); + } + + template + static void swap(Alloc * /*alloc*/, slot_type *a, slot_type *b) { + using std::swap; + swap(*a, *b); + } + + template + static void move(Alloc * /*alloc*/, slot_type *src, slot_type *dest) { + *dest = std::move(*src); + } + + template + static void move(Alloc *alloc, slot_type *first, slot_type *last, + slot_type *result) { + for (slot_type *src = first, *dest = result; src != last; ++src, ++dest) + move(alloc, src, dest); + } + }; + + // A parameters structure for holding the type parameters for a btree_set. + // Compare and Alloc should be nothrow copy-constructible. + template + struct set_params : common_params> { + using value_type = Key; + using slot_type = typename set_params::common_params::slot_type; + using value_compare = typename set_params::common_params::key_compare; + using is_map_container = std::false_type; + + static const Key &key(const value_type &x) { return x; } + static const Key &key(const slot_type *x) { return *x; } + }; + + // An adapter class that converts a lower-bound compare into an upper-bound + // compare. Note: there is no need to make a version of this adapter specialized + // for key-compare-to functors because the upper-bound (the first value greater + // than the input) is never an exact match. + template + struct upper_bound_adapter { + explicit upper_bound_adapter(const Compare &c) : comp(c) {} + template + bool operator()(const K &a, const LK &b) const { + // Returns true when a is not greater than b. + return !phmap::compare_internal::compare_result_as_less_than(comp(b, a)); + } + + private: + Compare comp; + }; + + enum class MatchKind : uint8_t { kEq, kNe }; + + template + struct SearchResult { + V value; + MatchKind match; + + static constexpr bool HasMatch() { return true; } + bool IsEq() const { return match == MatchKind::kEq; } + }; + + // When we don't use CompareTo, `match` is not present. + // This ensures that callers can't use it accidentally when it provides no + // useful information. + template + struct SearchResult { + V value; + + static constexpr bool HasMatch() { return false; } + static constexpr bool IsEq() { return false; } + }; + + // A node in the btree holding. The same node type is used for both internal + // and leaf nodes in the btree, though the nodes are allocated in such a way + // that the children array is only valid in internal nodes. + template + class btree_node { + using is_key_compare_to = typename Params::is_key_compare_to; + using is_multi_container = typename Params::is_multi_container; + using field_type = typename Params::node_count_type; + using allocator_type = typename Params::allocator_type; + using slot_type = typename Params::slot_type; + + public: + using params_type = Params; + using key_type = typename Params::key_type; + using value_type = typename Params::value_type; + using pointer = typename Params::pointer; + using const_pointer = typename Params::const_pointer; + using reference = typename Params::reference; + using const_reference = typename Params::const_reference; + using key_compare = typename Params::key_compare; + using size_type = typename Params::size_type; + using difference_type = typename Params::difference_type; + + // Btree decides whether to use linear node search as follows: + // - If the key is arithmetic and the comparator is std::less or + // std::greater, choose linear. + // - Otherwise, choose binary. + // TODO(ezb): Might make sense to add condition(s) based on node-size. + using use_linear_search = std::integral_constant< + bool, + std::is_arithmetic::value && + (std::is_same, key_compare>::value || + std::is_same, key_compare>::value || + std::is_same, key_compare>::value)>; + + + ~btree_node() = default; + btree_node(btree_node const &) = delete; + btree_node &operator=(btree_node const &) = delete; + + // Public for EmptyNodeType. + constexpr static size_type Alignment() { + static_assert(LeafLayout(1).Alignment() == InternalLayout().Alignment(), + "Alignment of all nodes must be equal."); + return (size_type)InternalLayout().Alignment(); + } + + protected: + btree_node() = default; + + private: + using layout_type = phmap::priv::Layout; + constexpr static size_type SizeWithNValues(size_type n) { + return (size_type)layout_type(/*parent*/ 1, + /*position, start, count, max_count*/ 4, + /*values*/ (size_t)n, + /*children*/ 0) + .AllocSize(); + } + // A lower bound for the overhead of fields other than values in a leaf node. + constexpr static size_type MinimumOverhead() { + return (size_type)(SizeWithNValues(1) - sizeof(value_type)); + } + + // Compute how many values we can fit onto a leaf node taking into account + // padding. + constexpr static size_type NodeTargetValues(const int begin, const int end) { + return begin == end ? begin + : SizeWithNValues((begin + end) / 2 + 1) > + params_type::kTargetNodeSize + ? NodeTargetValues(begin, (begin + end) / 2) + : NodeTargetValues((begin + end) / 2 + 1, end); + } + + enum { + kTargetNodeSize = params_type::kTargetNodeSize, + kNodeTargetValues = NodeTargetValues(0, params_type::kTargetNodeSize), + + // We need a minimum of 3 values per internal node in order to perform + // splitting (1 value for the two nodes involved in the split and 1 value + // propagated to the parent as the delimiter for the split). + kNodeValues = kNodeTargetValues >= 3 ? kNodeTargetValues : 3, + + // The node is internal (i.e. is not a leaf node) if and only if `max_count` + // has this value. + kInternalNodeMaxCount = 0, + }; + + // Leaves can have less than kNodeValues values. + constexpr static layout_type LeafLayout(const int max_values = kNodeValues) { + return layout_type(/*parent*/ 1, + /*position, start, count, max_count*/ 4, + /*values*/ (size_t)max_values, + /*children*/ 0); + } + constexpr static layout_type InternalLayout() { + return layout_type(/*parent*/ 1, + /*position, start, count, max_count*/ 4, + /*values*/ kNodeValues, + /*children*/ kNodeValues + 1); + } + constexpr static size_type LeafSize(const int max_values = kNodeValues) { + return (size_type)LeafLayout(max_values).AllocSize(); + } + constexpr static size_type InternalSize() { + return (size_type)InternalLayout().AllocSize(); + } + + // N is the index of the type in the Layout definition. + // ElementType is the Nth type in the Layout definition. + template + inline typename layout_type::template ElementType *GetField() { + // We assert that we don't read from values that aren't there. + assert(N < 3 || !leaf()); + return InternalLayout().template Pointer(reinterpret_cast(this)); + } + + template + inline const typename layout_type::template ElementType *GetField() const { + assert(N < 3 || !leaf()); + return InternalLayout().template Pointer( + reinterpret_cast(this)); + } + + void set_parent(btree_node *p) { *GetField<0>() = p; } + field_type &mutable_count() { return GetField<1>()[2]; } + slot_type *slot(size_type i) { return &GetField<2>()[i]; } + const slot_type *slot(size_type i) const { return &GetField<2>()[i]; } + void set_position(field_type v) { GetField<1>()[0] = v; } + void set_start(field_type v) { GetField<1>()[1] = v; } + void set_count(field_type v) { GetField<1>()[2] = v; } + void set_max_count(field_type v) { GetField<1>()[3] = v; } + + public: + // Whether this is a leaf node or not. This value doesn't change after the + // node is created. + bool leaf() const { return GetField<1>()[3] != kInternalNodeMaxCount; } + + // Getter for the position of this node in its parent. + field_type position() const { return GetField<1>()[0]; } + + // Getter for the offset of the first value in the `values` array. + field_type start() const { return GetField<1>()[1]; } + + // Getters for the number of values stored in this node. + field_type count() const { return GetField<1>()[2]; } + field_type max_count() const { + // Internal nodes have max_count==kInternalNodeMaxCount. + // Leaf nodes have max_count in [1, kNodeValues]. + const field_type max_cnt = GetField<1>()[3]; + return max_cnt == field_type{kInternalNodeMaxCount} + ? field_type{kNodeValues} + : max_cnt; + } + + // Getter for the parent of this node. + btree_node *parent() const { return *GetField<0>(); } + // Getter for whether the node is the root of the tree. The parent of the + // root of the tree is the leftmost node in the tree which is guaranteed to + // be a leaf. + bool is_root() const { return parent()->leaf(); } + void make_root() { + assert(parent()->is_root()); + set_parent(parent()->parent()); + } + + // Getters for the key/value at position i in the node. + const key_type &key(size_type i) const { return params_type::key(slot(i)); } + reference value(size_type i) { return params_type::element(slot(i)); } + const_reference value(size_type i) const { return params_type::element(slot(i)); } + +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Warray-bounds" +#endif + // Getters/setter for the child at position i in the node. + btree_node *child(size_type i) const { return GetField<3>()[i]; } + btree_node *&mutable_child(size_type i) { return GetField<3>()[i]; } + void clear_child(size_type i) { + phmap::priv::SanitizerPoisonObject(&mutable_child(i)); + } + void set_child(size_type i, btree_node *c) { + phmap::priv::SanitizerUnpoisonObject(&mutable_child(i)); + mutable_child(i) = c; + c->set_position((field_type)i); + } +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic pop +#endif + void init_child(int i, btree_node *c) { + set_child(i, c); + c->set_parent(this); + } + + // Returns the position of the first value whose key is not less than k. + template + SearchResult lower_bound( + const K &k, const key_compare &comp) const { + return use_linear_search::value ? linear_search(k, comp) + : binary_search(k, comp); + } + // Returns the position of the first value whose key is greater than k. + template + int upper_bound(const K &k, const key_compare &comp) const { + auto upper_compare = upper_bound_adapter(comp); + return use_linear_search::value ? linear_search(k, upper_compare).value + : binary_search(k, upper_compare).value; + } + + template + SearchResult::value> + linear_search(const K &k, const Compare &comp) const { + return linear_search_impl(k, 0, count(), comp, + btree_is_key_compare_to()); + } + + template + SearchResult::value> + binary_search(const K &k, const Compare &comp) const { + return binary_search_impl(k, 0, count(), comp, + btree_is_key_compare_to()); + } + + // Returns the position of the first value whose key is not less than k using + // linear search performed using plain compare. + template + SearchResult linear_search_impl( + const K &k, int s, const int e, const Compare &comp, + std::false_type /* IsCompareTo */) const { + while (s < e) { + if (!comp(key(s), k)) { + break; + } + ++s; + } + return {s}; + } + + // Returns the position of the first value whose key is not less than k using + // linear search performed using compare-to. + template + SearchResult linear_search_impl( + const K &k, int s, const int e, const Compare &comp, + std::true_type /* IsCompareTo */) const { + while (s < e) { + const phmap::weak_ordering c = comp(key(s), k); + if (c == 0) { + return {s, MatchKind::kEq}; + } else if (c > 0) { + break; + } + ++s; + } + return {s, MatchKind::kNe}; + } + + // Returns the position of the first value whose key is not less than k using + // binary search performed using plain compare. + template + SearchResult binary_search_impl( + const K &k, int s, int e, const Compare &comp, + std::false_type /* IsCompareTo */) const { + while (s != e) { + const int mid = (s + e) >> 1; + if (comp(key(mid), k)) { + s = mid + 1; + } else { + e = mid; + } + } + return {s}; + } + + // Returns the position of the first value whose key is not less than k using + // binary search performed using compare-to. + template + SearchResult binary_search_impl( + const K &k, int s, int e, const CompareTo &comp, + std::true_type /* IsCompareTo */) const { + if (is_multi_container::value) { + MatchKind exact_match = MatchKind::kNe; + while (s != e) { + const int mid = (s + e) >> 1; + const phmap::weak_ordering c = comp(key(mid), k); + if (c < 0) { + s = mid + 1; + } else { + e = mid; + if (c == 0) { + // Need to return the first value whose key is not less than k, + // which requires continuing the binary search if this is a + // multi-container. + exact_match = MatchKind::kEq; + } + } + } + return {s, exact_match}; + } else { // Not a multi-container. + while (s != e) { + const int mid = (s + e) >> 1; + const phmap::weak_ordering c = comp(key(mid), k); + if (c < 0) { + s = mid + 1; + } else if (c > 0) { + e = mid; + } else { + return {mid, MatchKind::kEq}; + } + } + return {s, MatchKind::kNe}; + } + } + + // Emplaces a value at position i, shifting all existing values and + // children at positions >= i to the right by 1. + template + void emplace_value(size_type i, allocator_type *alloc, Args &&... args); + + // Removes the value at position i, shifting all existing values and children + // at positions > i to the left by 1. + void remove_value(int i, allocator_type *alloc); + + // Removes the values at positions [i, i + to_erase), shifting all values + // after that range to the left by to_erase. Does not change children at all. + void remove_values_ignore_children(int i, size_type to_erase, + allocator_type *alloc); + + // Rebalances a node with its right sibling. + void rebalance_right_to_left(int to_move, btree_node *right, + allocator_type *alloc); + void rebalance_left_to_right(int to_move, btree_node *right, + allocator_type *alloc); + + // Splits a node, moving a portion of the node's values to its right sibling. + void split(int insert_position, btree_node *dest, allocator_type *alloc); + + // Merges a node with its right sibling, moving all of the values and the + // delimiting key in the parent node onto itself. + void merge(btree_node *sibling, allocator_type *alloc); + + // Swap the contents of "this" and "src". + void swap(btree_node *src, allocator_type *alloc); + + // Node allocation/deletion routines. + static btree_node *init_leaf(btree_node *n, btree_node *parent, + int max_cnt) { + n->set_parent(parent); + n->set_position(0); + n->set_start(0); + n->set_count(0); + n->set_max_count((field_type)max_cnt); + phmap::priv::SanitizerPoisonMemoryRegion( + n->slot(0), max_cnt * sizeof(slot_type)); + return n; + } + static btree_node *init_internal(btree_node *n, btree_node *parent) { + init_leaf(n, parent, kNodeValues); + // Set `max_count` to a sentinel value to indicate that this node is + // internal. + n->set_max_count(kInternalNodeMaxCount); + phmap::priv::SanitizerPoisonMemoryRegion( + &n->mutable_child(0), (kNodeValues + 1) * sizeof(btree_node *)); + return n; + } + void destroy(allocator_type *alloc) { + for (int i = 0; i < count(); ++i) { + value_destroy(i, alloc); + } + } + + public: + // Exposed only for tests. + static bool testonly_uses_linear_node_search() { + return use_linear_search::value; + } + + private: + template + void value_init(const size_type i, allocator_type *alloc, Args &&... args) { + phmap::priv::SanitizerUnpoisonObject(slot(i)); + params_type::construct(alloc, slot(i), std::forward(args)...); + } + void value_destroy(const size_type i, allocator_type *alloc) { + params_type::destroy(alloc, slot(i)); + phmap::priv::SanitizerPoisonObject(slot(i)); + } + + // Move n values starting at value i in this node into the values starting at + // value j in node x. + void uninitialized_move_n(const size_type n, const size_type i, + const size_type j, btree_node *x, + allocator_type *alloc) { + phmap::priv::SanitizerUnpoisonMemoryRegion( + x->slot(j), n * sizeof(slot_type)); + for (slot_type *src = slot(i), *end = src + n, *dest = x->slot(j); + src != end; ++src, ++dest) { + params_type::construct(alloc, dest, src); + } + } + + // Destroys a range of n values, starting at index i. + void value_destroy_n(const size_type i, const size_type n, + allocator_type *alloc) { + for (size_type j = 0; j < n; ++j) { + value_destroy(i + j, alloc); + } + } + + template + friend class btree; + template + friend struct btree_iterator; + friend class BtreeNodePeer; + }; + + template + struct btree_iterator { + private: + using key_type = typename Node::key_type; + using size_type = typename Node::size_type; + using params_type = typename Node::params_type; + + using node_type = Node; + using normal_node = typename std::remove_const::type; + using const_node = const Node; + using normal_pointer = typename params_type::pointer; + using normal_reference = typename params_type::reference; + using const_pointer = typename params_type::const_pointer; + using const_reference = typename params_type::const_reference; + using slot_type = typename params_type::slot_type; + + using iterator = + btree_iterator; + using const_iterator = + btree_iterator; + + public: + // These aliases are public for std::iterator_traits. + using difference_type = typename Node::difference_type; + using value_type = typename params_type::value_type; + using pointer = Pointer; + using reference = Reference; + using iterator_category = std::bidirectional_iterator_tag; + + btree_iterator() : node(nullptr), position(-1) {} + btree_iterator(Node *n, int p) : node(n), position(p) {} + + // NOTE: this SFINAE allows for implicit conversions from iterator to + // const_iterator, but it specifically avoids defining copy constructors so + // that btree_iterator can be trivially copyable. This is for performance and + // binary size reasons. + template , iterator>::value && + std::is_same::value, + int> = 0> + btree_iterator(const btree_iterator &x) // NOLINT + : node(x.node), position(x.position) {} + + private: + // This SFINAE allows explicit conversions from const_iterator to + // iterator, but also avoids defining a copy constructor. + // NOTE: the const_cast is safe because this constructor is only called by + // non-const methods and the container owns the nodes. + template , const_iterator>::value && + std::is_same::value, + int> = 0> + explicit btree_iterator(const btree_iterator &x) + : node(const_cast(x.node)), position(x.position) {} + + // Increment/decrement the iterator. + void increment() { + if (node->leaf() && ++position < node->count()) { + return; + } + increment_slow(); + } + void increment_slow(); + + void decrement() { + if (node->leaf() && --position >= 0) { + return; + } + decrement_slow(); + } + void decrement_slow(); + + public: + bool operator==(const const_iterator &x) const { + return node == x.node && position == x.position; + } + bool operator!=(const const_iterator &x) const { + return node != x.node || position != x.position; + } + bool operator==(const iterator &x) const { + return node == x.node && position == x.position; + } + bool operator!=(const iterator &x) const { + return node != x.node || position != x.position; + } + + // Accessors for the key/value the iterator is pointing at. + reference operator*() const { + return node->value(position); + } + pointer operator->() const { + return &node->value(position); + } + + btree_iterator& operator++() { + increment(); + return *this; + } + btree_iterator& operator--() { + decrement(); + return *this; + } + btree_iterator operator++(int) { + btree_iterator tmp = *this; + ++*this; + return tmp; + } + btree_iterator operator--(int) { + btree_iterator tmp = *this; + --*this; + return tmp; + } + + private: + template + friend class btree; + template + friend class btree_container; + template + friend class btree_set_container; + template + friend class btree_map_container; + template + friend class btree_multiset_container; + template + friend struct btree_iterator; + template + friend class base_checker; + + const key_type &key() const { return node->key(position); } + slot_type *slot() { return node->slot(position); } + + // The node in the tree the iterator is pointing at. + Node *node; + // The position within the node of the tree the iterator is pointing at. + // TODO(ezb): make this a field_type + int position; + }; + + template + class btree { + using node_type = btree_node; + using is_key_compare_to = typename Params::is_key_compare_to; + + // We use a static empty node for the root/leftmost/rightmost of empty btrees + // in order to avoid branching in begin()/end(). + struct alignas(node_type::Alignment()) EmptyNodeType : node_type { + using field_type = typename node_type::field_type; + node_type *parent; + field_type position = 0; + field_type start = 0; + field_type count = 0; + // max_count must be != kInternalNodeMaxCount (so that this node is regarded + // as a leaf node). max_count() is never called when the tree is empty. + field_type max_count = node_type::kInternalNodeMaxCount + 1; + +#ifdef _MSC_VER + // MSVC has constexpr code generations bugs here. + EmptyNodeType() : parent(this) {} +#else + constexpr EmptyNodeType(node_type *p) : parent(p) {} +#endif + }; + + static node_type *EmptyNode() { +#ifdef _MSC_VER + static EmptyNodeType empty_node; + // This assert fails on some other construction methods. + assert(empty_node.parent == &empty_node); + return &empty_node; +#else + static constexpr EmptyNodeType empty_node( + const_cast(&empty_node)); + return const_cast(&empty_node); +#endif + } + + enum { + kNodeValues = node_type::kNodeValues, + kMinNodeValues = kNodeValues / 2, + }; + + struct node_stats { + using size_type = typename Params::size_type; + + node_stats(size_type l, size_type i) + : leaf_nodes(l), + internal_nodes(i) { + } + + node_stats& operator+=(const node_stats &x) { + leaf_nodes += x.leaf_nodes; + internal_nodes += x.internal_nodes; + return *this; + } + + size_type leaf_nodes; + size_type internal_nodes; + }; + + public: + using key_type = typename Params::key_type; + using value_type = typename Params::value_type; + using size_type = typename Params::size_type; + using difference_type = typename Params::difference_type; + using key_compare = typename Params::key_compare; + using value_compare = typename Params::value_compare; + using allocator_type = typename Params::allocator_type; + using reference = typename Params::reference; + using const_reference = typename Params::const_reference; + using pointer = typename Params::pointer; + using const_pointer = typename Params::const_pointer; + using iterator = btree_iterator; + using const_iterator = typename iterator::const_iterator; + using reverse_iterator = std::reverse_iterator; + using const_reverse_iterator = std::reverse_iterator; + using node_handle_type = node_handle; + + // Internal types made public for use by btree_container types. + using params_type = Params; + using slot_type = typename Params::slot_type; + + private: + // For use in copy_or_move_values_in_order. + const value_type &maybe_move_from_iterator(const_iterator x) { return *x; } + value_type &&maybe_move_from_iterator(iterator x) { return std::move(*x); } + + // Copies or moves (depending on the template parameter) the values in + // x into this btree in their order in x. This btree must be empty before this + // method is called. This method is used in copy construction, copy + // assignment, and move assignment. + template + void copy_or_move_values_in_order(Btree *x); + + // Validates that various assumptions/requirements are true at compile time. + constexpr static bool static_assert_validation(); + + public: + btree(const key_compare &comp, const allocator_type &alloc); + + btree(const btree &x); + btree(btree &&x) noexcept + : root_(std::move(x.root_)), + rightmost_(phmap::exchange(x.rightmost_, EmptyNode())), + size_(phmap::exchange(x.size_, 0)) { + x.mutable_root() = EmptyNode(); + } + + ~btree() { + // Put static_asserts in destructor to avoid triggering them before the type + // is complete. + static_assert(static_assert_validation(), "This call must be elided."); + clear(); + } + + // Assign the contents of x to *this. + btree &operator=(const btree &x); + btree &operator=(btree &&x) noexcept; + + iterator begin() { + return iterator(leftmost(), 0); + } + const_iterator begin() const { + return const_iterator(leftmost(), 0); + } + iterator end() { return iterator(rightmost_, rightmost_->count()); } + const_iterator end() const { + return const_iterator(rightmost_, rightmost_->count()); + } + reverse_iterator rbegin() { + return reverse_iterator(end()); + } + const_reverse_iterator rbegin() const { + return const_reverse_iterator(end()); + } + reverse_iterator rend() { + return reverse_iterator(begin()); + } + const_reverse_iterator rend() const { + return const_reverse_iterator(begin()); + } + + // Finds the first element whose key is not less than key. + template + iterator lower_bound(const K &key) { + return internal_end(internal_lower_bound(key)); + } + template + const_iterator lower_bound(const K &key) const { + return internal_end(internal_lower_bound(key)); + } + + // Finds the first element whose key is greater than key. + template + iterator upper_bound(const K &key) { + return internal_end(internal_upper_bound(key)); + } + template + const_iterator upper_bound(const K &key) const { + return internal_end(internal_upper_bound(key)); + } + + // Finds the range of values which compare equal to key. The first member of + // the returned pair is equal to lower_bound(key). The second member pair of + // the pair is equal to upper_bound(key). + template + std::pair equal_range(const K &key) { + return {lower_bound(key), upper_bound(key)}; + } + template + std::pair equal_range(const K &key) const { + return {lower_bound(key), upper_bound(key)}; + } + + // Inserts a value into the btree only if it does not already exist. The + // boolean return value indicates whether insertion succeeded or failed. + // Requirement: if `key` already exists in the btree, does not consume `args`. + // Requirement: `key` is never referenced after consuming `args`. + template + std::pair insert_unique(const key_type &key, Args &&... args); + + // Inserts with hint. Checks to see if the value should be placed immediately + // before `position` in the tree. If so, then the insertion will take + // amortized constant time. If not, the insertion will take amortized + // logarithmic time as if a call to insert_unique() were made. + // Requirement: if `key` already exists in the btree, does not consume `args`. + // Requirement: `key` is never referenced after consuming `args`. + template + std::pair insert_hint_unique(iterator position, + const key_type &key, + Args &&... args); + + // Insert a range of values into the btree. + template + void insert_iterator_unique(InputIterator b, InputIterator e); + + // Inserts a value into the btree. + template + iterator insert_multi(const key_type &key, ValueType &&v); + + // Inserts a value into the btree. + template + iterator insert_multi(ValueType &&v) { + return insert_multi(params_type::key(v), std::forward(v)); + } + + // Insert with hint. Check to see if the value should be placed immediately + // before position in the tree. If it does, then the insertion will take + // amortized constant time. If not, the insertion will take amortized + // logarithmic time as if a call to insert_multi(v) were made. + template + iterator insert_hint_multi(iterator position, ValueType &&v); + + // Insert a range of values into the btree. + template + void insert_iterator_multi(InputIterator b, InputIterator e); + + // Erase the specified iterator from the btree. The iterator must be valid + // (i.e. not equal to end()). Return an iterator pointing to the node after + // the one that was erased (or end() if none exists). + // Requirement: does not read the value at `*iter`. + iterator erase(iterator iter); + + // Erases range. Returns the number of keys erased and an iterator pointing + // to the element after the last erased element. + std::pair erase(iterator begin, iterator end); + + // Erases the specified key from the btree. Returns 1 if an element was + // erased and 0 otherwise. + template + size_type erase_unique(const K &key); + + // Erases all of the entries matching the specified key from the + // btree. Returns the number of elements erased. + template + size_type erase_multi(const K &key); + + // Finds the iterator corresponding to a key or returns end() if the key is + // not present. + template + iterator find(const K &key) { + return internal_end(internal_find(key)); + } + template + const_iterator find(const K &key) const { + return internal_end(internal_find(key)); + } + + // Returns a count of the number of times the key appears in the btree. + template + size_type count_unique(const K &key) const { + const iterator beg = internal_find(key); + if (beg.node == nullptr) { + // The key doesn't exist in the tree. + return 0; + } + return 1; + } + // Returns a count of the number of times the key appears in the btree. + template + size_type count_multi(const K &key) const { + const auto range = equal_range(key); + return std::distance(range.first, range.second); + } + + // Clear the btree, deleting all of the values it contains. + void clear(); + + // Swap the contents of *this and x. + void swap(btree &x); + + const key_compare &key_comp() const noexcept { + return root_.template get<0>(); + } + template + bool compare_keys(const K &x, const LK &y) const { + return compare_internal::compare_result_as_less_than(key_comp()(x, y)); + } + + value_compare value_comp() const { return value_compare(key_comp()); } + + // Verifies the structure of the btree. + void verify() const; + + // Size routines. + size_type size() const { return size_; } + size_type max_size() const { return (std::numeric_limits::max)(); } + bool empty() const { return size_ == 0; } + + // The height of the btree. An empty tree will have height 0. + size_type height() const { + size_type h = 0; + if (!empty()) { + // Count the length of the chain from the leftmost node up to the + // root. We actually count from the root back around to the level below + // the root, but the calculation is the same because of the circularity + // of that traversal. + const node_type *n = root(); + do { + ++h; + n = n->parent(); + } while (n != root()); + } + return h; + } + + // The number of internal, leaf and total nodes used by the btree. + size_type leaf_nodes() const { + return internal_stats(root()).leaf_nodes; + } + size_type internal_nodes() const { + return internal_stats(root()).internal_nodes; + } + size_type nodes() const { + node_stats stats = internal_stats(root()); + return stats.leaf_nodes + stats.internal_nodes; + } + + // The total number of bytes used by the btree. + size_type bytes_used() const { + node_stats stats = internal_stats(root()); + if (stats.leaf_nodes == 1 && stats.internal_nodes == 0) { + return sizeof(*this) + + node_type::LeafSize(root()->max_count()); + } else { + return sizeof(*this) + + stats.leaf_nodes * node_type::LeafSize() + + stats.internal_nodes * node_type::InternalSize(); + } + } + + // The average number of bytes used per value stored in the btree. + static double average_bytes_per_value() { + // Returns the number of bytes per value on a leaf node that is 75% + // full. Experimentally, this matches up nicely with the computed number of + // bytes per value in trees that had their values inserted in random order. + return node_type::LeafSize() / (kNodeValues * 0.75); + } + + // The fullness of the btree. Computed as the number of elements in the btree + // divided by the maximum number of elements a tree with the current number + // of nodes could hold. A value of 1 indicates perfect space + // utilization. Smaller values indicate space wastage. + // Returns 0 for empty trees. + double fullness() const { + if (empty()) return 0.0; + return static_cast(size()) / (nodes() * kNodeValues); + } + // The overhead of the btree structure in bytes per node. Computed as the + // total number of bytes used by the btree minus the number of bytes used for + // storing elements divided by the number of elements. + // Returns 0 for empty trees. + double overhead() const { + if (empty()) return 0.0; + return (bytes_used() - size() * sizeof(value_type)) / + static_cast(size()); + } + + // The allocator used by the btree. + allocator_type get_allocator() const { + return allocator(); + } + + private: + // Internal accessor routines. + node_type *root() { return root_.template get<2>(); } + const node_type *root() const { return root_.template get<2>(); } + node_type *&mutable_root() noexcept { return root_.template get<2>(); } + key_compare *mutable_key_comp() noexcept { return &root_.template get<0>(); } + + // The leftmost node is stored as the parent of the root node. + node_type *leftmost() { return root()->parent(); } + const node_type *leftmost() const { return root()->parent(); } + + // Allocator routines. + allocator_type *mutable_allocator() noexcept { + return &root_.template get<1>(); + } + const allocator_type &allocator() const noexcept { + return root_.template get<1>(); + } + + // Allocates a correctly aligned node of at least size bytes using the + // allocator. + node_type *allocate(const size_type sz) { + return reinterpret_cast( + phmap::priv::Allocate( + mutable_allocator(), (size_t)sz)); + } + + // Node creation/deletion routines. + node_type* new_internal_node(node_type *parent) { + node_type *p = allocate(node_type::InternalSize()); + return node_type::init_internal(p, parent); + } + node_type* new_leaf_node(node_type *parent) { + node_type *p = allocate(node_type::LeafSize()); + return node_type::init_leaf(p, parent, kNodeValues); + } + node_type *new_leaf_root_node(const int max_count) { + node_type *p = allocate(node_type::LeafSize(max_count)); + return node_type::init_leaf(p, p, max_count); + } + + // Deletion helper routines. + void erase_same_node(iterator begin, iterator end); + iterator erase_from_leaf_node(iterator begin, size_type to_erase); + iterator rebalance_after_delete(iterator iter); + + // Deallocates a node of a certain size in bytes using the allocator. + void deallocate(const size_type sz, node_type *node) { + phmap::priv::Deallocate( + mutable_allocator(), node, (size_t)sz); + } + + void delete_internal_node(node_type *node) { + node->destroy(mutable_allocator()); + deallocate(node_type::InternalSize(), node); + } + void delete_leaf_node(node_type *node) { + node->destroy(mutable_allocator()); + deallocate(node_type::LeafSize(node->max_count()), node); + } + + // Rebalances or splits the node iter points to. + void rebalance_or_split(iterator *iter); + + // Merges the values of left, right and the delimiting key on their parent + // onto left, removing the delimiting key and deleting right. + void merge_nodes(node_type *left, node_type *right); + + // Tries to merge node with its left or right sibling, and failing that, + // rebalance with its left or right sibling. Returns true if a merge + // occurred, at which point it is no longer valid to access node. Returns + // false if no merging took place. + bool try_merge_or_rebalance(iterator *iter); + + // Tries to shrink the height of the tree by 1. + void try_shrink(); + + iterator internal_end(iterator iter) { + return iter.node != nullptr ? iter : end(); + } + const_iterator internal_end(const_iterator iter) const { + return iter.node != nullptr ? iter : end(); + } + + // Emplaces a value into the btree immediately before iter. Requires that + // key(v) <= iter.key() and (--iter).key() <= key(v). + template + iterator internal_emplace(iterator iter, Args &&... args); + + // Returns an iterator pointing to the first value >= the value "iter" is + // pointing at. Note that "iter" might be pointing to an invalid location as + // iter.position == iter.node->count(). This routine simply moves iter up in + // the tree to a valid location. + // Requires: iter.node is non-null. + template + static IterType internal_last(IterType iter); + + // Returns an iterator pointing to the leaf position at which key would + // reside in the tree. We provide 2 versions of internal_locate. The first + // version uses a less-than comparator and is incapable of distinguishing when + // there is an exact match. The second version is for the key-compare-to + // specialization and distinguishes exact matches. The key-compare-to + // specialization allows the caller to avoid a subsequent comparison to + // determine if an exact match was made, which is important for keys with + // expensive comparison, such as strings. + template + SearchResult internal_locate( + const K &key) const; + + template + SearchResult internal_locate_impl( + const K &key, std::false_type /* IsCompareTo */) const; + + template + SearchResult internal_locate_impl( + const K &key, std::true_type /* IsCompareTo */) const; + + // Internal routine which implements lower_bound(). + template + iterator internal_lower_bound(const K &key) const; + + // Internal routine which implements upper_bound(). + template + iterator internal_upper_bound(const K &key) const; + + // Internal routine which implements find(). + template + iterator internal_find(const K &key) const; + + // Deletes a node and all of its children. + void internal_clear(node_type *node); + + // Verifies the tree structure of node. + size_type internal_verify(const node_type *node, + const key_type *lo, const key_type *hi) const; + + node_stats internal_stats(const node_type *node) const { + // The root can be a static empty node. + if (node == nullptr || (node == root() && empty())) { + return node_stats(0, 0); + } + if (node->leaf()) { + return node_stats(1, 0); + } + node_stats res(0, 1); + for (int i = 0; i <= node->count(); ++i) { + res += internal_stats(node->child(i)); + } + return res; + } + + public: + // Exposed only for tests. + static bool testonly_uses_linear_node_search() { + return node_type::testonly_uses_linear_node_search(); + } + + private: + // We use compressed tuple in order to save space because key_compare and + // allocator_type are usually empty. + phmap::priv::CompressedTuple + root_; + + // A pointer to the rightmost node. Note that the leftmost node is stored as + // the root's parent. + node_type *rightmost_; + + // Number of values. + size_type size_; + }; + + //// + // btree_node methods + template + template + inline void btree_node

::emplace_value(const size_type i, + allocator_type *alloc, + Args &&... args) { + assert(i <= count()); + // Shift old values to create space for new value and then construct it in + // place. + if (i < count()) { + value_init(count(), alloc, slot(count() - 1)); + for (size_type j = count() - 1; j > i; --j) + params_type::move(alloc, slot(j - 1), slot(j)); + value_destroy(i, alloc); + } + value_init(i, alloc, std::forward(args)...); + set_count((field_type)(count() + 1)); + + if (!leaf() && count() > i + 1) { + for (int j = count(); j > (int)(i + 1); --j) { + set_child(j, child(j - 1)); + } + clear_child(i + 1); + } + } + + template + inline void btree_node

::remove_value(const int i, allocator_type *alloc) { + if (!leaf() && count() > i + 1) { + assert(child(i + 1)->count() == 0); + for (size_type j = i + 1; j < count(); ++j) { + set_child(j, child(j + 1)); + } + clear_child(count()); + } + + remove_values_ignore_children(i, /*to_erase=*/1, alloc); + } + + template + inline void btree_node

::remove_values_ignore_children( + int i, size_type to_erase, allocator_type *alloc) { + params_type::move(alloc, slot(i + to_erase), slot(count()), slot(i)); + value_destroy_n(count() - to_erase, to_erase, alloc); + set_count((field_type)(count() - to_erase)); + } + + template + void btree_node

::rebalance_right_to_left(const int to_move, + btree_node *right, + allocator_type *alloc) { + assert(parent() == right->parent()); + assert(position() + 1 == right->position()); + assert(right->count() >= count()); + assert(to_move >= 1); + assert(to_move <= right->count()); + + // 1) Move the delimiting value in the parent to the left node. + value_init(count(), alloc, parent()->slot(position())); + + // 2) Move the (to_move - 1) values from the right node to the left node. + right->uninitialized_move_n(to_move - 1, 0, count() + 1, this, alloc); + + // 3) Move the new delimiting value to the parent from the right node. + params_type::move(alloc, right->slot(to_move - 1), + parent()->slot(position())); + + // 4) Shift the values in the right node to their correct position. + params_type::move(alloc, right->slot(to_move), right->slot(right->count()), + right->slot(0)); + + // 5) Destroy the now-empty to_move entries in the right node. + right->value_destroy_n(right->count() - to_move, to_move, alloc); + + if (!leaf()) { + // Move the child pointers from the right to the left node. + for (int i = 0; i < to_move; ++i) { + init_child(count() + i + 1, right->child(i)); + } + for (int i = 0; i <= right->count() - to_move; ++i) { + assert(i + to_move <= right->max_count()); + right->init_child(i, right->child(i + to_move)); + right->clear_child(i + to_move); + } + } + + // Fixup the counts on the left and right nodes. + set_count((field_type)(count() + to_move)); + right->set_count((field_type)(right->count() - to_move)); + } + + template + void btree_node

::rebalance_left_to_right(const int to_move, + btree_node *right, + allocator_type *alloc) { + assert(parent() == right->parent()); + assert(position() + 1 == right->position()); + assert(count() >= right->count()); + assert(to_move >= 1); + assert(to_move <= count()); + + // Values in the right node are shifted to the right to make room for the + // new to_move values. Then, the delimiting value in the parent and the + // other (to_move - 1) values in the left node are moved into the right node. + // Lastly, a new delimiting value is moved from the left node into the + // parent, and the remaining empty left node entries are destroyed. + + if (right->count() >= to_move) { + // The original location of the right->count() values are sufficient to hold + // the new to_move entries from the parent and left node. + + // 1) Shift existing values in the right node to their correct positions. + right->uninitialized_move_n(to_move, right->count() - to_move, + right->count(), right, alloc); + if (right->count() > to_move) { + for (slot_type *src = right->slot(right->count() - to_move - 1), + *dest = right->slot(right->count() - 1), + *end = right->slot(0); + src >= end; --src, --dest) { + params_type::move(alloc, src, dest); + } + } + + // 2) Move the delimiting value in the parent to the right node. + params_type::move(alloc, parent()->slot(position()), + right->slot(to_move - 1)); + + // 3) Move the (to_move - 1) values from the left node to the right node. + params_type::move(alloc, slot(count() - (to_move - 1)), slot(count()), + right->slot(0)); + } else { + // The right node does not have enough initialized space to hold the new + // to_move entries, so part of them will move to uninitialized space. + + // 1) Shift existing values in the right node to their correct positions. + right->uninitialized_move_n(right->count(), 0, to_move, right, alloc); + + // 2) Move the delimiting value in the parent to the right node. + right->value_init(to_move - 1, alloc, parent()->slot(position())); + + // 3) Move the (to_move - 1) values from the left node to the right node. + const size_type uninitialized_remaining = to_move - right->count() - 1; + uninitialized_move_n(uninitialized_remaining, + count() - uninitialized_remaining, right->count(), + right, alloc); + params_type::move(alloc, slot(count() - (to_move - 1)), + slot(count() - uninitialized_remaining), right->slot(0)); + } + + // 4) Move the new delimiting value to the parent from the left node. + params_type::move(alloc, slot(count() - to_move), parent()->slot(position())); + + // 5) Destroy the now-empty to_move entries in the left node. + value_destroy_n(count() - to_move, to_move, alloc); + + if (!leaf()) { + // Move the child pointers from the left to the right node. + for (int i = right->count(); i >= 0; --i) { + right->init_child(i + to_move, right->child(i)); + right->clear_child(i); + } + for (int i = 1; i <= to_move; ++i) { + right->init_child(i - 1, child(count() - to_move + i)); + clear_child(count() - to_move + i); + } + } + + // Fixup the counts on the left and right nodes. + set_count((field_type)(count() - to_move)); + right->set_count((field_type)(right->count() + to_move)); + } + + template + void btree_node

::split(const int insert_position, btree_node *dest, + allocator_type *alloc) { + assert(dest->count() == 0); + assert(max_count() == kNodeValues); + + // We bias the split based on the position being inserted. If we're + // inserting at the beginning of the left node then bias the split to put + // more values on the right node. If we're inserting at the end of the + // right node then bias the split to put more values on the left node. + if (insert_position == 0) { + dest->set_count((field_type)(count() - 1)); + } else if (insert_position == kNodeValues) { + dest->set_count(0); + } else { + dest->set_count((field_type)(count() / 2)); + } + set_count((field_type)(count() - dest->count())); + assert(count() >= 1); + + // Move values from the left sibling to the right sibling. + uninitialized_move_n(dest->count(), count(), 0, dest, alloc); + + // Destroy the now-empty entries in the left node. + value_destroy_n(count(), dest->count(), alloc); + + // The split key is the largest value in the left sibling. + set_count((field_type)(count() - 1)); + parent()->emplace_value(position(), alloc, slot(count())); + value_destroy(count(), alloc); + parent()->init_child(position() + 1, dest); + + if (!leaf()) { + for (int i = 0; i <= dest->count(); ++i) { + assert(child(count() + i + 1) != nullptr); + dest->init_child(i, child(count() + i + 1)); + clear_child(count() + i + 1); + } + } + } + + template + void btree_node

::merge(btree_node *src, allocator_type *alloc) { + assert(parent() == src->parent()); + assert(position() + 1 == src->position()); + + // Move the delimiting value to the left node. + value_init(count(), alloc, parent()->slot(position())); + + // Move the values from the right to the left node. + src->uninitialized_move_n(src->count(), 0, count() + 1, this, alloc); + + // Destroy the now-empty entries in the right node. + src->value_destroy_n(0, src->count(), alloc); + + if (!leaf()) { + // Move the child pointers from the right to the left node. + for (int i = 0; i <= src->count(); ++i) { + init_child(count() + i + 1, src->child(i)); + src->clear_child(i); + } + } + + // Fixup the counts on the src and dest nodes. + set_count((field_type)(1 + count() + src->count())); + src->set_count(0); + + // Remove the value on the parent node. + parent()->remove_value(position(), alloc); + } + + template + void btree_node

::swap(btree_node *x, allocator_type *alloc) { + using std::swap; + assert(leaf() == x->leaf()); + + // Determine which is the smaller/larger node. + btree_node *smaller = this, *larger = x; + if (smaller->count() > larger->count()) { + swap(smaller, larger); + } + + // Swap the values. + for (slot_type *a = smaller->slot(0), *b = larger->slot(0), + *end = a + smaller->count(); + a != end; ++a, ++b) { + params_type::swap(alloc, a, b); + } + + // Move values that can't be swapped. + const size_type to_move = larger->count() - smaller->count(); + larger->uninitialized_move_n(to_move, smaller->count(), smaller->count(), + smaller, alloc); + larger->value_destroy_n(smaller->count(), to_move, alloc); + + if (!leaf()) { + // Swap the child pointers. + std::swap_ranges(&smaller->mutable_child(0), + &smaller->mutable_child(smaller->count() + 1), + &larger->mutable_child(0)); + // Update swapped children's parent pointers. + int i = 0; + for (; i <= smaller->count(); ++i) { + smaller->child(i)->set_parent(smaller); + larger->child(i)->set_parent(larger); + } + // Move the child pointers that couldn't be swapped. + for (; i <= larger->count(); ++i) { + smaller->init_child(i, larger->child(i)); + larger->clear_child(i); + } + } + + // Swap the counts. + swap(mutable_count(), x->mutable_count()); + } + + //// + // btree_iterator methods + template + void btree_iterator::increment_slow() { + if (node->leaf()) { + assert(position >= node->count()); + btree_iterator save(*this); + while (position == node->count() && !node->is_root()) { + assert(node->parent()->child(node->position()) == node); + position = node->position(); + node = node->parent(); + } + if (position == node->count()) { + *this = save; + } + } else { + assert(position < node->count()); + node = node->child(position + 1); + while (!node->leaf()) { + node = node->child(0); + } + position = 0; + } + } + + template + void btree_iterator::decrement_slow() { + if (node->leaf()) { + assert(position <= -1); + btree_iterator save(*this); + while (position < 0 && !node->is_root()) { + assert(node->parent()->child(node->position()) == node); + position = node->position() - 1; + node = node->parent(); + } + if (position < 0) { + *this = save; + } + } else { + assert(position >= 0); + node = node->child(position); + while (!node->leaf()) { + node = node->child(node->count()); + } + position = node->count() - 1; + } + } + + //// + // btree methods + template + template + void btree

::copy_or_move_values_in_order(Btree *x) { + static_assert(std::is_same::value || + std::is_same::value, + "Btree type must be same or const."); + assert(empty()); + + // We can avoid key comparisons because we know the order of the + // values is the same order we'll store them in. + auto iter = x->begin(); + if (iter == x->end()) return; + insert_multi(maybe_move_from_iterator(iter)); + ++iter; + for (; iter != x->end(); ++iter) { + // If the btree is not empty, we can just insert the new value at the end + // of the tree. + internal_emplace(end(), maybe_move_from_iterator(iter)); + } + } + + template + constexpr bool btree

::static_assert_validation() { + static_assert(std::is_nothrow_copy_constructible::value, + "Key comparison must be nothrow copy constructible"); + static_assert(std::is_nothrow_copy_constructible::value, + "Allocator must be nothrow copy constructible"); + static_assert(type_traits_internal::is_trivially_copyable::value, + "iterator not trivially copyable."); + + // Note: We assert that kTargetValues, which is computed from + // Params::kTargetNodeSize, must fit the node_type::field_type. + static_assert( + kNodeValues < (1 << (8 * sizeof(typename node_type::field_type))), + "target node size too large"); + + // Verify that key_compare returns an phmap::{weak,strong}_ordering or bool. + using compare_result_type = + phmap::invoke_result_t; + static_assert( + std::is_same::value || + std::is_convertible::value, + "key comparison function must return phmap::{weak,strong}_ordering or " + "bool."); + + // Test the assumption made in setting kNodeSlotSpace. + static_assert(node_type::MinimumOverhead() >= sizeof(void *) + 4, + "node space assumption incorrect"); + + return true; + } + + template + btree

::btree(const key_compare &comp, const allocator_type &alloc) + : root_(comp, alloc, EmptyNode()), rightmost_(EmptyNode()), size_(0) {} + + template + btree

::btree(const btree &x) : btree(x.key_comp(), x.allocator()) { + copy_or_move_values_in_order(&x); + } + + template + template + auto btree

::insert_unique(const key_type &key, Args &&... args) + -> std::pair { + if (empty()) { + mutable_root() = rightmost_ = new_leaf_root_node(1); + } + + auto res = internal_locate(key); + iterator &iter = res.value; + + if (res.HasMatch()) { + if (res.IsEq()) { + // The key already exists in the tree, do nothing. + return {iter, false}; + } + } else { + iterator last = internal_last(iter); + if (last.node && !compare_keys(key, last.key())) { + // The key already exists in the tree, do nothing. + return {last, false}; + } + } + return {internal_emplace(iter, std::forward(args)...), true}; + } + + template + template + inline auto btree

::insert_hint_unique(iterator position, const key_type &key, + Args &&... args) + -> std::pair { + if (!empty()) { + if (position == end() || compare_keys(key, position.key())) { + iterator prev = position; + if (position == begin() || compare_keys((--prev).key(), key)) { + // prev.key() < key < position.key() + return {internal_emplace(position, std::forward(args)...), true}; + } + } else if (compare_keys(position.key(), key)) { + ++position; + if (position == end() || compare_keys(key, position.key())) { + // {original `position`}.key() < key < {current `position`}.key() + return {internal_emplace(position, std::forward(args)...), true}; + } + } else { + // position.key() == key + return {position, false}; + } + } + return insert_unique(key, std::forward(args)...); + } + + template + template + void btree

::insert_iterator_unique(InputIterator b, InputIterator e) { + for (; b != e; ++b) { + insert_hint_unique(end(), params_type::key(*b), *b); + } + } + + template + template + auto btree

::insert_multi(const key_type &key, ValueType &&v) -> iterator { + if (empty()) { + mutable_root() = rightmost_ = new_leaf_root_node(1); + } + + iterator iter = internal_upper_bound(key); + if (iter.node == nullptr) { + iter = end(); + } + return internal_emplace(iter, std::forward(v)); + } + + template + template + auto btree

::insert_hint_multi(iterator position, ValueType &&v) -> iterator { + if (!empty()) { + const key_type &key = params_type::key(v); + if (position == end() || !compare_keys(position.key(), key)) { + iterator prev = position; + if (position == begin() || !compare_keys(key, (--prev).key())) { + // prev.key() <= key <= position.key() + return internal_emplace(position, std::forward(v)); + } + } else { + iterator next = position; + ++next; + if (next == end() || !compare_keys(next.key(), key)) { + // position.key() < key <= next.key() + return internal_emplace(next, std::forward(v)); + } + } + } + return insert_multi(std::forward(v)); + } + + template + template + void btree

::insert_iterator_multi(InputIterator b, InputIterator e) { + for (; b != e; ++b) { + insert_hint_multi(end(), *b); + } + } + + template + auto btree

::operator=(const btree &x) -> btree & { + if (this != &x) { + clear(); + + *mutable_key_comp() = x.key_comp(); + if (phmap::allocator_traits< + allocator_type>::propagate_on_container_copy_assignment::value) { + *mutable_allocator() = x.allocator(); + } + + copy_or_move_values_in_order(&x); + } + return *this; + } + + template + auto btree

::operator=(btree &&x) noexcept -> btree & { + if (this != &x) { + clear(); + + using std::swap; + if (phmap::allocator_traits< + allocator_type>::propagate_on_container_copy_assignment::value) { + // Note: `root_` also contains the allocator and the key comparator. + swap(root_, x.root_); + swap(rightmost_, x.rightmost_); + swap(size_, x.size_); + } else { + if (allocator() == x.allocator()) { + swap(mutable_root(), x.mutable_root()); + swap(*mutable_key_comp(), *x.mutable_key_comp()); + swap(rightmost_, x.rightmost_); + swap(size_, x.size_); + } else { + // We aren't allowed to propagate the allocator and the allocator is + // different so we can't take over its memory. We must move each element + // individually. We need both `x` and `this` to have `x`s key comparator + // while moving the values so we can't swap the key comparators. + *mutable_key_comp() = x.key_comp(); + copy_or_move_values_in_order(&x); + } + } + } + return *this; + } + + template + auto btree

::erase(iterator iter) -> iterator { + bool internal_delete = false; + if (!iter.node->leaf()) { + // Deletion of a value on an internal node. First, move the largest value + // from our left child here, then delete that position (in remove_value() + // below). We can get to the largest value from our left child by + // decrementing iter. + iterator internal_iter(iter); + --iter; + assert(iter.node->leaf()); + params_type::move(mutable_allocator(), iter.node->slot(iter.position), + internal_iter.node->slot(internal_iter.position)); + internal_delete = true; + } + + // Delete the key from the leaf. + iter.node->remove_value(iter.position, mutable_allocator()); + --size_; + + // We want to return the next value after the one we just erased. If we + // erased from an internal node (internal_delete == true), then the next + // value is ++(++iter). If we erased from a leaf node (internal_delete == + // false) then the next value is ++iter. Note that ++iter may point to an + // internal node and the value in the internal node may move to a leaf node + // (iter.node) when rebalancing is performed at the leaf level. + + iterator res = rebalance_after_delete(iter); + + // If we erased from an internal node, advance the iterator. + if (internal_delete) { + ++res; + } + return res; + } + + template + auto btree

::rebalance_after_delete(iterator iter) -> iterator { + // Merge/rebalance as we walk back up the tree. + iterator res(iter); + bool first_iteration = true; + for (;;) { + if (iter.node == root()) { + try_shrink(); + if (empty()) { + return end(); + } + break; + } + if (iter.node->count() >= kMinNodeValues) { + break; + } + bool merged = try_merge_or_rebalance(&iter); + // On the first iteration, we should update `res` with `iter` because `res` + // may have been invalidated. + if (first_iteration) { + res = iter; + first_iteration = false; + } + if (!merged) { + break; + } + iter.position = iter.node->position(); + iter.node = iter.node->parent(); + } + + // Adjust our return value. If we're pointing at the end of a node, advance + // the iterator. + if (res.position == res.node->count()) { + res.position = res.node->count() - 1; + ++res; + } + + return res; + } + + template + auto btree

::erase(iterator _begin, iterator _end) + -> std::pair { + difference_type count = std::distance(_begin, _end); + assert(count >= 0); + + if (count == 0) { + return {0, _begin}; + } + + if (count == (difference_type)size_) { + clear(); + return {count, this->end()}; + } + + if (_begin.node == _end.node) { + erase_same_node(_begin, _end); + size_ -= count; + return {count, rebalance_after_delete(_begin)}; + } + + const size_type target_size = size_ - count; + while (size_ > target_size) { + if (_begin.node->leaf()) { + const size_type remaining_to_erase = size_ - target_size; + const size_type remaining_in_node = _begin.node->count() - _begin.position; + _begin = erase_from_leaf_node( + _begin, (std::min)(remaining_to_erase, remaining_in_node)); + } else { + _begin = erase(_begin); + } + } + return {count, _begin}; + } + + template + void btree

::erase_same_node(iterator _begin, iterator _end) { + assert(_begin.node == _end.node); + assert(_end.position > _begin.position); + + node_type *node = _begin.node; + size_type to_erase = _end.position - _begin.position; + if (!node->leaf()) { + // Delete all children between _begin and _end. + for (size_type i = 0; i < to_erase; ++i) { + internal_clear(node->child(_begin.position + i + 1)); + } + // Rotate children after _end into new positions. + for (size_type i = _begin.position + to_erase + 1; i <= node->count(); ++i) { + node->set_child(i - to_erase, node->child(i)); + node->clear_child(i); + } + } + node->remove_values_ignore_children(_begin.position, to_erase, + mutable_allocator()); + + // Do not need to update rightmost_, because + // * either _end == this->end(), and therefore node == rightmost_, and still + // exists + // * or _end != this->end(), and therefore rightmost_ hasn't been erased, since + // it wasn't covered in [_begin, _end) + } + + template + auto btree

::erase_from_leaf_node(iterator _begin, size_type to_erase) + -> iterator { + node_type *node = _begin.node; + assert(node->leaf()); + assert(node->count() > _begin.position); + assert(_begin.position + to_erase <= node->count()); + + node->remove_values_ignore_children(_begin.position, to_erase, + mutable_allocator()); + + size_ -= to_erase; + + return rebalance_after_delete(_begin); + } + + template + template + auto btree

::erase_unique(const K &key) -> size_type { + const iterator iter = internal_find(key); + if (iter.node == nullptr) { + // The key doesn't exist in the tree, return nothing done. + return 0; + } + erase(iter); + return 1; + } + + template + template + auto btree

::erase_multi(const K &key) -> size_type { + const iterator _begin = internal_lower_bound(key); + if (_begin.node == nullptr) { + // The key doesn't exist in the tree, return nothing done. + return 0; + } + // Delete all of the keys between _begin and upper_bound(key). + const iterator _end = internal_end(internal_upper_bound(key)); + return erase(_begin, _end).first; + } + + template + void btree

::clear() { + if (!empty()) { + internal_clear(root()); + } + mutable_root() = EmptyNode(); + rightmost_ = EmptyNode(); + size_ = 0; + } + + template + void btree

::swap(btree &x) { + using std::swap; + if (phmap::allocator_traits< + allocator_type>::propagate_on_container_swap::value) { + // Note: `root_` also contains the allocator and the key comparator. + swap(root_, x.root_); + } else { + // It's undefined behavior if the allocators are unequal here. + assert(allocator() == x.allocator()); + swap(mutable_root(), x.mutable_root()); + swap(*mutable_key_comp(), *x.mutable_key_comp()); + } + swap(rightmost_, x.rightmost_); + swap(size_, x.size_); + } + + template + void btree

::verify() const { + assert(root() != nullptr); + assert(leftmost() != nullptr); + assert(rightmost_ != nullptr); + assert(empty() || size() == internal_verify(root(), nullptr, nullptr)); + assert(leftmost() == (++const_iterator(root(), -1)).node); + assert(rightmost_ == (--const_iterator(root(), root()->count())).node); + assert(leftmost()->leaf()); + assert(rightmost_->leaf()); + } + + template + void btree

::rebalance_or_split(iterator *iter) { + node_type *&node = iter->node; + int &insert_position = iter->position; + assert(node->count() == node->max_count()); + assert(kNodeValues == node->max_count()); + + // First try to make room on the node by rebalancing. + node_type *parent = node->parent(); + if (node != root()) { + if (node->position() > 0) { + // Try rebalancing with our left sibling. + node_type *left = parent->child(node->position() - 1); + assert(left->max_count() == kNodeValues); + if (left->count() < kNodeValues) { + // We bias rebalancing based on the position being inserted. If we're + // inserting at the end of the right node then we bias rebalancing to + // fill up the left node. + int to_move = (kNodeValues - left->count()) / + (1 + (insert_position < kNodeValues)); + to_move = (std::max)(1, to_move); + + if (((insert_position - to_move) >= 0) || + ((left->count() + to_move) < kNodeValues)) { + left->rebalance_right_to_left(to_move, node, mutable_allocator()); + + assert(node->max_count() - node->count() == to_move); + insert_position = insert_position - to_move; + if (insert_position < 0) { + insert_position = insert_position + left->count() + 1; + node = left; + } + + assert(node->count() < node->max_count()); + return; + } + } + } + + if (node->position() < parent->count()) { + // Try rebalancing with our right sibling. + node_type *right = parent->child(node->position() + 1); + assert(right->max_count() == kNodeValues); + if (right->count() < kNodeValues) { + // We bias rebalancing based on the position being inserted. If we're + // inserting at the _beginning of the left node then we bias rebalancing + // to fill up the right node. + int to_move = + (kNodeValues - right->count()) / (1 + (insert_position > 0)); + to_move = (std::max)(1, to_move); + + if ((insert_position <= (node->count() - to_move)) || + ((right->count() + to_move) < kNodeValues)) { + node->rebalance_left_to_right(to_move, right, mutable_allocator()); + + if (insert_position > node->count()) { + insert_position = insert_position - node->count() - 1; + node = right; + } + + assert(node->count() < node->max_count()); + return; + } + } + } + + // Rebalancing failed, make sure there is room on the parent node for a new + // value. + assert(parent->max_count() == kNodeValues); + if (parent->count() == kNodeValues) { + iterator parent_iter(node->parent(), node->position()); + rebalance_or_split(&parent_iter); + } + } else { + // Rebalancing not possible because this is the root node. + // Create a new root node and set the current root node as the child of the + // new root. + parent = new_internal_node(parent); + parent->init_child(0, root()); + mutable_root() = parent; + // If the former root was a leaf node, then it's now the rightmost node. + assert(!parent->child(0)->leaf() || parent->child(0) == rightmost_); + } + + // Split the node. + node_type *split_node; + if (node->leaf()) { + split_node = new_leaf_node(parent); + node->split(insert_position, split_node, mutable_allocator()); + if (rightmost_ == node) rightmost_ = split_node; + } else { + split_node = new_internal_node(parent); + node->split(insert_position, split_node, mutable_allocator()); + } + + if (insert_position > node->count()) { + insert_position = insert_position - node->count() - 1; + node = split_node; + } + } + + template + void btree

::merge_nodes(node_type *left, node_type *right) { + left->merge(right, mutable_allocator()); + if (right->leaf()) { + if (rightmost_ == right) rightmost_ = left; + delete_leaf_node(right); + } else { + delete_internal_node(right); + } + } + + template + bool btree

::try_merge_or_rebalance(iterator *iter) { + node_type *parent = iter->node->parent(); + if (iter->node->position() > 0) { + // Try merging with our left sibling. + node_type *left = parent->child(iter->node->position() - 1); + assert(left->max_count() == kNodeValues); + if ((1 + left->count() + iter->node->count()) <= kNodeValues) { + iter->position += 1 + left->count(); + merge_nodes(left, iter->node); + iter->node = left; + return true; + } + } + if (iter->node->position() < parent->count()) { + // Try merging with our right sibling. + node_type *right = parent->child(iter->node->position() + 1); + assert(right->max_count() == kNodeValues); + if ((1 + iter->node->count() + right->count()) <= kNodeValues) { + merge_nodes(iter->node, right); + return true; + } + // Try rebalancing with our right sibling. We don't perform rebalancing if + // we deleted the first element from iter->node and the node is not + // empty. This is a small optimization for the common pattern of deleting + // from the front of the tree. + if ((right->count() > kMinNodeValues) && + ((iter->node->count() == 0) || + (iter->position > 0))) { + int to_move = (right->count() - iter->node->count()) / 2; + to_move = (std::min)(to_move, right->count() - 1); + iter->node->rebalance_right_to_left(to_move, right, mutable_allocator()); + return false; + } + } + if (iter->node->position() > 0) { + // Try rebalancing with our left sibling. We don't perform rebalancing if + // we deleted the last element from iter->node and the node is not + // empty. This is a small optimization for the common pattern of deleting + // from the back of the tree. + node_type *left = parent->child(iter->node->position() - 1); + if ((left->count() > kMinNodeValues) && + ((iter->node->count() == 0) || + (iter->position < iter->node->count()))) { + int to_move = (left->count() - iter->node->count()) / 2; + to_move = (std::min)(to_move, left->count() - 1); + left->rebalance_left_to_right(to_move, iter->node, mutable_allocator()); + iter->position += to_move; + return false; + } + } + return false; + } + + template + void btree

::try_shrink() { + if (root()->count() > 0) { + return; + } + // Deleted the last item on the root node, shrink the height of the tree. + if (root()->leaf()) { + assert(size() == 0); + delete_leaf_node(root()); + mutable_root() = EmptyNode(); + rightmost_ = EmptyNode(); + } else { + node_type *child = root()->child(0); + child->make_root(); + delete_internal_node(root()); + mutable_root() = child; + } + } + + template + template + inline IterType btree

::internal_last(IterType iter) { + assert(iter.node != nullptr); + while (iter.position == iter.node->count()) { + iter.position = iter.node->position(); + iter.node = iter.node->parent(); + if (iter.node->leaf()) { + iter.node = nullptr; + break; + } + } + return iter; + } + + template + template + inline auto btree

::internal_emplace(iterator iter, Args &&... args) + -> iterator { + if (!iter.node->leaf()) { + // We can't insert on an internal node. Instead, we'll insert after the + // previous value which is guaranteed to be on a leaf node. + --iter; + ++iter.position; + } + const int max_count = iter.node->max_count(); + if (iter.node->count() == max_count) { + // Make room in the leaf for the new item. + if (max_count < kNodeValues) { + // Insertion into the root where the root is smaller than the full node + // size. Simply grow the size of the root node. + assert(iter.node == root()); + iter.node = + new_leaf_root_node((std::min)(kNodeValues, 2 * max_count)); + iter.node->swap(root(), mutable_allocator()); + delete_leaf_node(root()); + mutable_root() = iter.node; + rightmost_ = iter.node; + } else { + rebalance_or_split(&iter); + } + } + iter.node->emplace_value(iter.position, mutable_allocator(), + std::forward(args)...); + ++size_; + return iter; + } + + template + template + inline auto btree

::internal_locate(const K &key) const + -> SearchResult { + return internal_locate_impl(key, is_key_compare_to()); + } + + template + template + inline auto btree

::internal_locate_impl( + const K &key, std::false_type /* IsCompareTo */) const + -> SearchResult { + iterator iter(const_cast(root()), 0); + for (;;) { + iter.position = iter.node->lower_bound(key, key_comp()).value; + // NOTE: we don't need to walk all the way down the tree if the keys are + // equal, but determining equality would require doing an extra comparison + // on each node on the way down, and we will need to go all the way to the + // leaf node in the expected case. + if (iter.node->leaf()) { + break; + } + iter.node = iter.node->child(iter.position); + } + return {iter}; + } + + template + template + inline auto btree

::internal_locate_impl( + const K &key, std::true_type /* IsCompareTo */) const + -> SearchResult { + iterator iter(const_cast(root()), 0); + for (;;) { + SearchResult res = iter.node->lower_bound(key, key_comp()); + iter.position = res.value; + if (res.match == MatchKind::kEq) { + return {iter, MatchKind::kEq}; + } + if (iter.node->leaf()) { + break; + } + iter.node = iter.node->child(iter.position); + } + return {iter, MatchKind::kNe}; + } + + template + template + auto btree

::internal_lower_bound(const K &key) const -> iterator { + iterator iter(const_cast(root()), 0); + for (;;) { + iter.position = iter.node->lower_bound(key, key_comp()).value; + if (iter.node->leaf()) { + break; + } + iter.node = iter.node->child(iter.position); + } + return internal_last(iter); + } + + template + template + auto btree

::internal_upper_bound(const K &key) const -> iterator { + iterator iter(const_cast(root()), 0); + for (;;) { + iter.position = iter.node->upper_bound(key, key_comp()); + if (iter.node->leaf()) { + break; + } + iter.node = iter.node->child(iter.position); + } + return internal_last(iter); + } + + template + template + auto btree

::internal_find(const K &key) const -> iterator { + auto res = internal_locate(key); + if (res.HasMatch()) { + if (res.IsEq()) { + return res.value; + } + } else { + const iterator iter = internal_last(res.value); + if (iter.node != nullptr && !compare_keys(key, iter.key())) { + return iter; + } + } + return {nullptr, 0}; + } + + template + void btree

::internal_clear(node_type *node) { + if (!node->leaf()) { + for (int i = 0; i <= node->count(); ++i) { + internal_clear(node->child(i)); + } + delete_internal_node(node); + } else { + delete_leaf_node(node); + } + } + + template + typename btree

::size_type btree

::internal_verify( + const node_type *node, const key_type *lo, const key_type *hi) const { + assert(node->count() > 0); + assert(node->count() <= node->max_count()); + if (lo) { + assert(!compare_keys(node->key(0), *lo)); + } + if (hi) { + assert(!compare_keys(*hi, node->key(node->count() - 1))); + } + for (int i = 1; i < node->count(); ++i) { + assert(!compare_keys(node->key(i), node->key(i - 1))); + } + size_type count = node->count(); + if (!node->leaf()) { + for (int i = 0; i <= node->count(); ++i) { + assert(node->child(i) != nullptr); + assert(node->child(i)->parent() == node); + assert(node->child(i)->position() == i); + count += internal_verify( + node->child(i), + (i == 0) ? lo : &node->key(i - 1), + (i == node->count()) ? hi : &node->key(i)); + } + } + return count; + } + + // A common base class for btree_set, btree_map, btree_multiset, and btree_multimap. + // --------------------------------------------------------------------------------- + template + class btree_container { + using params_type = typename Tree::params_type; + + protected: + // Alias used for heterogeneous lookup functions. + // `key_arg` evaluates to `K` when the functors are transparent and to + // `key_type` otherwise. It permits template argument deduction on `K` for the + // transparent case. + template + using key_arg = + typename KeyArg::value>:: + template type; + + public: + using key_type = typename Tree::key_type; + using value_type = typename Tree::value_type; + using size_type = typename Tree::size_type; + using difference_type = typename Tree::difference_type; + using key_compare = typename Tree::key_compare; + using value_compare = typename Tree::value_compare; + using allocator_type = typename Tree::allocator_type; + using reference = typename Tree::reference; + using const_reference = typename Tree::const_reference; + using pointer = typename Tree::pointer; + using const_pointer = typename Tree::const_pointer; + using iterator = typename Tree::iterator; + using const_iterator = typename Tree::const_iterator; + using reverse_iterator = typename Tree::reverse_iterator; + using const_reverse_iterator = typename Tree::const_reverse_iterator; + using node_type = typename Tree::node_handle_type; + + // Constructors/assignments. + btree_container() : tree_(key_compare(), allocator_type()) {} + explicit btree_container(const key_compare &comp, + const allocator_type &alloc = allocator_type()) + : tree_(comp, alloc) {} + btree_container(const btree_container &x) = default; + btree_container(btree_container &&x) noexcept = default; + btree_container &operator=(const btree_container &x) = default; + btree_container &operator=(btree_container &&x) noexcept( + std::is_nothrow_move_assignable::value) = default; + + // Iterator routines. + iterator begin() { return tree_.begin(); } + const_iterator begin() const { return tree_.begin(); } + const_iterator cbegin() const { return tree_.begin(); } + iterator end() { return tree_.end(); } + const_iterator end() const { return tree_.end(); } + const_iterator cend() const { return tree_.end(); } + reverse_iterator rbegin() { return tree_.rbegin(); } + const_reverse_iterator rbegin() const { return tree_.rbegin(); } + const_reverse_iterator crbegin() const { return tree_.rbegin(); } + reverse_iterator rend() { return tree_.rend(); } + const_reverse_iterator rend() const { return tree_.rend(); } + const_reverse_iterator crend() const { return tree_.rend(); } + + // Lookup routines. + // ---------------- + template + size_type count(const key_arg &key) const { + auto equal_range = this->equal_range(key); + return std::distance(equal_range.first, equal_range.second); + } + template + iterator find(const key_arg &key) { + return tree_.find(key); + } + template + const_iterator find(const key_arg &key) const { return tree_.find(key); } + + template + bool contains(const key_arg &key) const { return find(key) != end(); } + + template + iterator lower_bound(const key_arg &key) { return tree_.lower_bound(key); } + + template + const_iterator lower_bound(const key_arg &key) const { return tree_.lower_bound(key); } + + template + iterator upper_bound(const key_arg &key) { return tree_.upper_bound(key); } + + template + const_iterator upper_bound(const key_arg &key) const { return tree_.upper_bound(key); } + + template + std::pair equal_range(const key_arg &key) { return tree_.equal_range(key); } + + template + std::pair equal_range( + const key_arg &key) const { + return tree_.equal_range(key); + } + + iterator erase(const_iterator iter) { return tree_.erase(iterator(iter)); } + iterator erase(iterator iter) { return tree_.erase(iter); } + iterator erase(const_iterator first, const_iterator last) { + return tree_.erase(iterator(first), iterator(last)).second; + } + template + size_type erase(const key_arg &key) { + auto equal_range = this->equal_range(key); + return tree_.erase_range(equal_range.first, equal_range.second).first; + } + node_type extract(iterator position) { + // Use Move instead of Transfer, because the rebalancing code expects to + // have a valid object to scribble metadata bits on top of. + auto node = CommonAccess::Move(get_allocator(), position.slot()); + erase(position); + return node; + } + + node_type extract(const_iterator position) { + return extract(iterator(position)); + } + + public: + void clear() { tree_.clear(); } + void swap(btree_container &x) { tree_.swap(x.tree_); } + void verify() const { tree_.verify(); } + + size_type size() const { return tree_.size(); } + size_type max_size() const { return tree_.max_size(); } + bool empty() const { return tree_.empty(); } + + friend bool operator==(const btree_container &x, const btree_container &y) { + if (x.size() != y.size()) return false; + return std::equal(x.begin(), x.end(), y.begin()); + } + + friend bool operator!=(const btree_container &x, const btree_container &y) { return !(x == y); } + + friend bool operator<(const btree_container &x, const btree_container &y) { + return std::lexicographical_compare(x.begin(), x.end(), y.begin(), y.end()); + } + + friend bool operator>(const btree_container &x, const btree_container &y) { return y < x; } + + friend bool operator<=(const btree_container &x, const btree_container &y) { return !(y < x); } + + friend bool operator>=(const btree_container &x, const btree_container &y) { return !(x < y); } + + // The allocator used by the btree. + allocator_type get_allocator() const { return tree_.get_allocator(); } + + // The key comparator used by the btree. + key_compare key_comp() const { return tree_.key_comp(); } + value_compare value_comp() const { return tree_.value_comp(); } + + // Support absl::Hash. + template + friend State AbslHashValue(State h, const btree_container &b) { + for (const auto &v : b) { + h = State::combine(std::move(h), v); + } + return State::combine(std::move(h), b.size()); + } + + protected: + Tree tree_; + }; + + // A common base class for btree_set and btree_map. + // ----------------------------------------------- + template + class btree_set_container : public btree_container { + using super_type = btree_container; + using params_type = typename Tree::params_type; + using init_type = typename params_type::init_type; + using is_key_compare_to = typename params_type::is_key_compare_to; + friend class BtreeNodePeer; + + protected: + template + using key_arg = typename super_type::template key_arg; + + public: + using key_type = typename Tree::key_type; + using value_type = typename Tree::value_type; + using size_type = typename Tree::size_type; + using key_compare = typename Tree::key_compare; + using allocator_type = typename Tree::allocator_type; + using iterator = typename Tree::iterator; + using const_iterator = typename Tree::const_iterator; + using node_type = typename super_type::node_type; + using insert_return_type = InsertReturnType; + using super_type::super_type; + btree_set_container() {} + + template + btree_set_container(InputIterator b, InputIterator e, + const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(comp, alloc) { + insert(b, e); + } + + btree_set_container(std::initializer_list init, + const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : btree_set_container(init.begin(), init.end(), comp, alloc) {} + + btree_set_container(std::initializer_list init, + const allocator_type &alloc) + : btree_set_container(init.begin(), init.end(), alloc) {} + + // Lookup routines. + template + size_type count(const key_arg &key) const { + return this->tree_.count_unique(key); + } + + // Insertion routines. + std::pair insert(const value_type &x) { + return this->tree_.insert_unique(params_type::key(x), x); + } + std::pair insert(value_type &&x) { + return this->tree_.insert_unique(params_type::key(x), std::move(x)); + } + template + std::pair emplace(Args &&... args) { + init_type v(std::forward(args)...); + return this->tree_.insert_unique(params_type::key(v), std::move(v)); + } + iterator insert(const_iterator hint, const value_type &x) { + return this->tree_ + .insert_hint_unique(iterator(hint), params_type::key(x), x) + .first; + } + iterator insert(const_iterator hint, value_type &&x) { + return this->tree_ + .insert_hint_unique(iterator(hint), params_type::key(x), + std::move(x)) + .first; + } + + template + iterator emplace_hint(const_iterator hint, Args &&... args) { + init_type v(std::forward(args)...); + return this->tree_ + .insert_hint_unique(iterator(hint), params_type::key(v), + std::move(v)) + .first; + } + + template + void insert(InputIterator b, InputIterator e) { + this->tree_.insert_iterator_unique(b, e); + } + + void insert(std::initializer_list init) { + this->tree_.insert_iterator_unique(init.begin(), init.end()); + } + + insert_return_type insert(node_type &&node) { + if (!node) return {this->end(), false, node_type()}; + std::pair res = + this->tree_.insert_unique(params_type::key(CommonAccess::GetSlot(node)), + CommonAccess::GetSlot(node)); + if (res.second) { + CommonAccess::Destroy(&node); + return {res.first, true, node_type()}; + } else { + return {res.first, false, std::move(node)}; + } + } + + iterator insert(const_iterator hint, node_type &&node) { + if (!node) return this->end(); + std::pair res = this->tree_.insert_hint_unique( + iterator(hint), params_type::key(CommonAccess::GetSlot(node)), + CommonAccess::GetSlot(node)); + if (res.second) CommonAccess::Destroy(&node); + return res.first; + } + + template + size_type erase(const key_arg &key) { return this->tree_.erase_unique(key); } + using super_type::erase; + + template + node_type extract(const key_arg &key) { + auto it = this->find(key); + return it == this->end() ? node_type() : extract(it); + } + + using super_type::extract; + + // Merge routines. + // Moves elements from `src` into `this`. If the element already exists in + // `this`, it is left unmodified in `src`. + template < + typename T, + typename phmap::enable_if_t< + phmap::conjunction< + std::is_same, + std::is_same, + std::is_same>::value, + int> = 0> + void merge(btree_container &src) { // NOLINT + for (auto src_it = src.begin(); src_it != src.end();) { + if (insert(std::move(*src_it)).second) { + src_it = src.erase(src_it); + } else { + ++src_it; + } + } + } + + template < + typename T, + typename phmap::enable_if_t< + phmap::conjunction< + std::is_same, + std::is_same, + std::is_same>::value, + int> = 0> + void merge(btree_container &&src) { + merge(src); + } + }; + + // Base class for btree_map. + // ------------------------- + template + class btree_map_container : public btree_set_container { + using super_type = btree_set_container; + using params_type = typename Tree::params_type; + + protected: + template + using key_arg = typename super_type::template key_arg; + + public: + using key_type = typename Tree::key_type; + using mapped_type = typename params_type::mapped_type; + using value_type = typename Tree::value_type; + using key_compare = typename Tree::key_compare; + using allocator_type = typename Tree::allocator_type; + using iterator = typename Tree::iterator; + using const_iterator = typename Tree::const_iterator; + + // Inherit constructors. + using super_type::super_type; + btree_map_container() {} + + // Insertion routines. + template + std::pair try_emplace(const key_type &k, Args &&... args) { + return this->tree_.insert_unique( + k, std::piecewise_construct, std::forward_as_tuple(k), + std::forward_as_tuple(std::forward(args)...)); + } + template + std::pair try_emplace(key_type &&k, Args &&... args) { + // Note: `key_ref` exists to avoid a ClangTidy warning about moving from `k` + // and then using `k` unsequenced. This is safe because the move is into a + // forwarding reference and insert_unique guarantees that `key` is never + // referenced after consuming `args`. + const key_type& key_ref = k; + return this->tree_.insert_unique( + key_ref, std::piecewise_construct, std::forward_as_tuple(std::move(k)), + std::forward_as_tuple(std::forward(args)...)); + } + template + iterator try_emplace(const_iterator hint, const key_type &k, + Args &&... args) { + return this->tree_ + .insert_hint_unique(iterator(hint), k, std::piecewise_construct, + std::forward_as_tuple(k), + std::forward_as_tuple(std::forward(args)...)) + .first; + } + template + iterator try_emplace(const_iterator hint, key_type &&k, Args &&... args) { + // Note: `key_ref` exists to avoid a ClangTidy warning about moving from `k` + // and then using `k` unsequenced. This is safe because the move is into a + // forwarding reference and insert_hint_unique guarantees that `key` is + // never referenced after consuming `args`. + const key_type& key_ref = k; + return this->tree_ + .insert_hint_unique(iterator(hint), key_ref, std::piecewise_construct, + std::forward_as_tuple(std::move(k)), + std::forward_as_tuple(std::forward(args)...)) + .first; + } + mapped_type &operator[](const key_type &k) { + return try_emplace(k).first->second; + } + mapped_type &operator[](key_type &&k) { + return try_emplace(std::move(k)).first->second; + } + + template + mapped_type &at(const key_arg &key) { + auto it = this->find(key); + if (it == this->end()) + base_internal::ThrowStdOutOfRange("phmap::btree_map::at"); + return it->second; + } + template + const mapped_type &at(const key_arg &key) const { + auto it = this->find(key); + if (it == this->end()) + base_internal::ThrowStdOutOfRange("phmap::btree_map::at"); + return it->second; + } + }; + + // A common base class for btree_multiset and btree_multimap. + template + class btree_multiset_container : public btree_container { + using super_type = btree_container; + using params_type = typename Tree::params_type; + using init_type = typename params_type::init_type; + using is_key_compare_to = typename params_type::is_key_compare_to; + + template + using key_arg = typename super_type::template key_arg; + + public: + using key_type = typename Tree::key_type; + using value_type = typename Tree::value_type; + using size_type = typename Tree::size_type; + using key_compare = typename Tree::key_compare; + using allocator_type = typename Tree::allocator_type; + using iterator = typename Tree::iterator; + using const_iterator = typename Tree::const_iterator; + using node_type = typename super_type::node_type; + + // Inherit constructors. + using super_type::super_type; + btree_multiset_container() {} + + // Range constructor. + template + btree_multiset_container(InputIterator b, InputIterator e, + const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(comp, alloc) { + insert(b, e); + } + + // Initializer list constructor. + btree_multiset_container(std::initializer_list init, + const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : btree_multiset_container(init.begin(), init.end(), comp, alloc) {} + + // Lookup routines. + template + size_type count(const key_arg &key) const { + return this->tree_.count_multi(key); + } + + // Insertion routines. + iterator insert(const value_type &x) { return this->tree_.insert_multi(x); } + iterator insert(value_type &&x) { + return this->tree_.insert_multi(std::move(x)); + } + iterator insert(const_iterator hint, const value_type &x) { + return this->tree_.insert_hint_multi(iterator(hint), x); + } + iterator insert(const_iterator hint, value_type &&x) { + return this->tree_.insert_hint_multi(iterator(hint), std::move(x)); + } + template + void insert(InputIterator b, InputIterator e) { + this->tree_.insert_iterator_multi(b, e); + } + void insert(std::initializer_list init) { + this->tree_.insert_iterator_multi(init.begin(), init.end()); + } + template + iterator emplace(Args &&... args) { + return this->tree_.insert_multi(init_type(std::forward(args)...)); + } + template + iterator emplace_hint(const_iterator hint, Args &&... args) { + return this->tree_.insert_hint_multi( + iterator(hint), init_type(std::forward(args)...)); + } + iterator insert(node_type &&node) { + if (!node) return this->end(); + iterator res = + this->tree_.insert_multi(params_type::key(CommonAccess::GetSlot(node)), + CommonAccess::GetSlot(node)); + CommonAccess::Destroy(&node); + return res; + } + iterator insert(const_iterator hint, node_type &&node) { + if (!node) return this->end(); + iterator res = this->tree_.insert_hint_multi( + iterator(hint), + std::move(params_type::element(CommonAccess::GetSlot(node)))); + CommonAccess::Destroy(&node); + return res; + } + + // Deletion routines. + template + size_type erase(const key_arg &key) { + return this->tree_.erase_multi(key); + } + using super_type::erase; + + // Node extraction routines. + template + node_type extract(const key_arg &key) { + auto it = this->find(key); + return it == this->end() ? node_type() : extract(it); + } + using super_type::extract; + + // Merge routines. + // Moves all elements from `src` into `this`. + template < + typename T, + typename phmap::enable_if_t< + phmap::conjunction< + std::is_same, + std::is_same, + std::is_same>::value, + int> = 0> + void merge(btree_container &src) { // NOLINT + insert(std::make_move_iterator(src.begin()), + std::make_move_iterator(src.end())); + src.clear(); + } + + template < + typename T, + typename phmap::enable_if_t< + phmap::conjunction< + std::is_same, + std::is_same, + std::is_same>::value, + int> = 0> + void merge(btree_container &&src) { + merge(src); + } + }; + + // A base class for btree_multimap. + template + class btree_multimap_container : public btree_multiset_container { + using super_type = btree_multiset_container; + using params_type = typename Tree::params_type; + + public: + using mapped_type = typename params_type::mapped_type; + + // Inherit constructors. + using super_type::super_type; + btree_multimap_container() {} + }; + +} // namespace priv + + + + // ---------------------------------------------------------------------- + // btree_set - default values in phmap_fwd_decl.h + // ---------------------------------------------------------------------- + template + class btree_set : public priv::btree_set_container< + priv::btree>> + { + using Base = typename btree_set::btree_set_container; + + public: + btree_set() {} + using Base::Base; + using Base::begin; + using Base::cbegin; + using Base::end; + using Base::cend; + using Base::empty; + using Base::max_size; + using Base::size; + using Base::clear; + using Base::erase; + using Base::insert; + using Base::emplace; + using Base::emplace_hint; + using Base::extract; + using Base::merge; + using Base::swap; + using Base::contains; + using Base::count; + using Base::equal_range; + using Base::lower_bound; + using Base::upper_bound; + using Base::find; + using Base::get_allocator; + using Base::key_comp; + using Base::value_comp; + }; + + // Swaps the contents of two `phmap::btree_set` containers. + // ------------------------------------------------------- + template + void swap(btree_set &x, btree_set &y) { + return x.swap(y); + } + + // Erases all elements that satisfy the predicate pred from the container. + // ---------------------------------------------------------------------- + template + void erase_if(btree_set &set, Pred pred) { + for (auto it = set.begin(); it != set.end();) { + if (pred(*it)) { + it = set.erase(it); + } else { + ++it; + } + } + } + + // ---------------------------------------------------------------------- + // btree_multiset - default values in phmap_fwd_decl.h + // ---------------------------------------------------------------------- + template + class btree_multiset : public priv::btree_multiset_container< + priv::btree>> + { + using Base = typename btree_multiset::btree_multiset_container; + + public: + btree_multiset() {} + using Base::Base; + using Base::begin; + using Base::cbegin; + using Base::end; + using Base::cend; + using Base::empty; + using Base::max_size; + using Base::size; + using Base::clear; + using Base::erase; + using Base::insert; + using Base::emplace; + using Base::emplace_hint; + using Base::extract; + using Base::merge; + using Base::swap; + using Base::contains; + using Base::count; + using Base::equal_range; + using Base::lower_bound; + using Base::upper_bound; + using Base::find; + using Base::get_allocator; + using Base::key_comp; + using Base::value_comp; + }; + + // Swaps the contents of two `phmap::btree_multiset` containers. + // ------------------------------------------------------------ + template + void swap(btree_multiset &x, btree_multiset &y) { + return x.swap(y); + } + + // Erases all elements that satisfy the predicate pred from the container. + // ---------------------------------------------------------------------- + template + void erase_if(btree_multiset &set, Pred pred) { + for (auto it = set.begin(); it != set.end();) { + if (pred(*it)) { + it = set.erase(it); + } else { + ++it; + } + } + } + + + // ---------------------------------------------------------------------- + // btree_map - default values in phmap_fwd_decl.h + // ---------------------------------------------------------------------- + template + class btree_map : public priv::btree_map_container< + priv::btree>> + { + using Base = typename btree_map::btree_map_container; + + public: + btree_map() {} + using Base::Base; + using Base::begin; + using Base::cbegin; + using Base::end; + using Base::cend; + using Base::empty; + using Base::max_size; + using Base::size; + using Base::clear; + using Base::erase; + using Base::insert; + using Base::emplace; + using Base::emplace_hint; + using Base::try_emplace; + using Base::extract; + using Base::merge; + using Base::swap; + using Base::at; + using Base::contains; + using Base::count; + using Base::equal_range; + using Base::lower_bound; + using Base::upper_bound; + using Base::find; + using Base::operator[]; + using Base::get_allocator; + using Base::key_comp; + using Base::value_comp; + }; + + // Swaps the contents of two `phmap::btree_map` containers. + // ------------------------------------------------------- + template + void swap(btree_map &x, btree_map &y) { + return x.swap(y); + } + + // ---------------------------------------------------------------------- + template + void erase_if(btree_map &map, Pred pred) { + for (auto it = map.begin(); it != map.end();) { + if (pred(*it)) { + it = map.erase(it); + } else { + ++it; + } + } + } + + // ---------------------------------------------------------------------- + // btree_multimap - default values in phmap_fwd_decl.h + // ---------------------------------------------------------------------- + template + class btree_multimap : public priv::btree_multimap_container< + priv::btree>> + { + using Base = typename btree_multimap::btree_multimap_container; + + public: + btree_multimap() {} + using Base::Base; + using Base::begin; + using Base::cbegin; + using Base::end; + using Base::cend; + using Base::empty; + using Base::max_size; + using Base::size; + using Base::clear; + using Base::erase; + using Base::insert; + using Base::emplace; + using Base::emplace_hint; + using Base::extract; + using Base::merge; + using Base::swap; + using Base::contains; + using Base::count; + using Base::equal_range; + using Base::lower_bound; + using Base::upper_bound; + using Base::find; + using Base::get_allocator; + using Base::key_comp; + using Base::value_comp; + }; + + // Swaps the contents of two `phmap::btree_multimap` containers. + // ------------------------------------------------------------ + template + void swap(btree_multimap &x, btree_multimap &y) { + return x.swap(y); + } + + // Erases all elements that satisfy the predicate pred from the container. + // ---------------------------------------------------------------------- + template + void erase_if(btree_multimap &map, Pred pred) { + for (auto it = map.begin(); it != map.end();) { + if (pred(*it)) { + it = map.erase(it); + } else { + ++it; + } + } + } + + +} // namespace btree + +#ifdef _MSC_VER + #pragma warning(pop) +#endif + + +#endif // PHMAP_BTREE_BTREE_CONTAINER_H_ diff --git a/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/parallel_hashmap/conanfile.py b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/parallel_hashmap/conanfile.py new file mode 100644 index 00000000..42112473 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/parallel_hashmap/conanfile.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from conans import ConanFile, tools +import os + +class SparseppConan(ConanFile): + name = "parallel_hashmap" + version = "1.36" + description = "A header-only, very fast and memory-friendly hash map" + url = "https://github.com/greg7mdp/parallel-hashmap/blob/master/parallel_hashmap/conanfile.py" + + # Indicates License type of the packaged library + license = "https://github.com/greg7mdp/parallel-hashmap/blob/master/LICENSE" + + # Packages the license for the conanfile.py + exports = ["LICENSE"] + + # Custom attributes for Bincrafters recipe conventions + source_subfolder = "source_subfolder" + + def source(self): + source_url = "https://github.com/greg7mdp/parallel-hashmap" + tools.get("{0}/archive/{1}.tar.gz".format(source_url, self.version)) + extracted_dir = self.name + "-" + self.version + + #Rename to "source_folder" is a convention to simplify later steps + os.rename(extracted_dir, self.source_subfolder) + + + def package(self): + include_folder = os.path.join(self.source_subfolder, "parallel_hashmap") + self.copy(pattern="LICENSE") + self.copy(pattern="*", dst="include/parallel_hashmap", src=include_folder) + + def package_id(self): + self.info.header_only() diff --git a/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/parallel_hashmap/meminfo.h b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/parallel_hashmap/meminfo.h new file mode 100644 index 00000000..872f3c69 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/parallel_hashmap/meminfo.h @@ -0,0 +1,195 @@ +#if !defined(spp_memory_h_guard) +#define spp_memory_h_guard + +#include +#include +#include + +#if defined(_WIN32) || defined( __CYGWIN__) + #define SPP_WIN +#endif + +#ifdef SPP_WIN + #include + #include + #undef min + #undef max +#elif defined(__linux__) + #include + #include +#elif defined(__FreeBSD__) + #include + #include + #include + #include + #include + #include +#endif + +namespace spp +{ + uint64_t GetSystemMemory(); + uint64_t GetTotalMemoryUsed(); + uint64_t GetProcessMemoryUsed(); + uint64_t GetPhysicalMemory(); + + uint64_t GetSystemMemory() + { +#ifdef SPP_WIN + MEMORYSTATUSEX memInfo; + memInfo.dwLength = sizeof(MEMORYSTATUSEX); + GlobalMemoryStatusEx(&memInfo); + return static_cast(memInfo.ullTotalPageFile); +#elif defined(__linux__) + struct sysinfo memInfo; + sysinfo (&memInfo); + auto totalVirtualMem = memInfo.totalram; + + totalVirtualMem += memInfo.totalswap; + totalVirtualMem *= memInfo.mem_unit; + return static_cast(totalVirtualMem); +#elif defined(__FreeBSD__) + kvm_t *kd; + u_int pageCnt; + size_t pageCntLen = sizeof(pageCnt); + u_int pageSize; + struct kvm_swap kswap; + uint64_t totalVirtualMem; + + pageSize = static_cast(getpagesize()); + + sysctlbyname("vm.stats.vm.v_page_count", &pageCnt, &pageCntLen, NULL, 0); + totalVirtualMem = pageCnt * pageSize; + + kd = kvm_open(NULL, _PATH_DEVNULL, NULL, O_RDONLY, "kvm_open"); + kvm_getswapinfo(kd, &kswap, 1, 0); + kvm_close(kd); + totalVirtualMem += kswap.ksw_total * pageSize; + + return totalVirtualMem; +#else + return 0; +#endif + } + + uint64_t GetTotalMemoryUsed() + { +#ifdef SPP_WIN + MEMORYSTATUSEX memInfo; + memInfo.dwLength = sizeof(MEMORYSTATUSEX); + GlobalMemoryStatusEx(&memInfo); + return static_cast(memInfo.ullTotalPageFile - memInfo.ullAvailPageFile); +#elif defined(__linux__) + struct sysinfo memInfo; + sysinfo(&memInfo); + auto virtualMemUsed = memInfo.totalram - memInfo.freeram; + + virtualMemUsed += memInfo.totalswap - memInfo.freeswap; + virtualMemUsed *= memInfo.mem_unit; + + return static_cast(virtualMemUsed); +#elif defined(__FreeBSD__) + kvm_t *kd; + u_int pageSize; + u_int pageCnt, freeCnt; + size_t pageCntLen = sizeof(pageCnt); + size_t freeCntLen = sizeof(freeCnt); + struct kvm_swap kswap; + uint64_t virtualMemUsed; + + pageSize = static_cast(getpagesize()); + + sysctlbyname("vm.stats.vm.v_page_count", &pageCnt, &pageCntLen, NULL, 0); + sysctlbyname("vm.stats.vm.v_free_count", &freeCnt, &freeCntLen, NULL, 0); + virtualMemUsed = (pageCnt - freeCnt) * pageSize; + + kd = kvm_open(NULL, _PATH_DEVNULL, NULL, O_RDONLY, "kvm_open"); + kvm_getswapinfo(kd, &kswap, 1, 0); + kvm_close(kd); + virtualMemUsed += kswap.ksw_used * pageSize; + + return virtualMemUsed; +#else + return 0; +#endif + } + + uint64_t GetProcessMemoryUsed() + { +#ifdef SPP_WIN + PROCESS_MEMORY_COUNTERS_EX pmc; + GetProcessMemoryInfo(GetCurrentProcess(), reinterpret_cast(&pmc), sizeof(pmc)); + return static_cast(pmc.PrivateUsage); +#elif defined(__linux__) + auto parseLine = + [](char* line)->int + { + auto i = strlen(line); + + while(*line < '0' || *line > '9') + { + line++; + } + + line[i-3] = '\0'; + i = atoi(line); + return i; + }; + + auto file = fopen("/proc/self/status", "r"); + auto result = -1; + char line[128]; + + while(fgets(line, 128, file) != nullptr) + { + if(strncmp(line, "VmSize:", 7) == 0) + { + result = parseLine(line); + break; + } + } + + fclose(file); + return static_cast(result) * 1024; +#elif defined(__FreeBSD__) + struct kinfo_proc info; + size_t infoLen = sizeof(info); + int mib[] = { CTL_KERN, KERN_PROC, KERN_PROC_PID, getpid() }; + + sysctl(mib, sizeof(mib) / sizeof(*mib), &info, &infoLen, NULL, 0); + return static_cast(info.ki_rssize * getpagesize()); +#else + return 0; +#endif + } + + uint64_t GetPhysicalMemory() + { +#ifdef SPP_WIN + MEMORYSTATUSEX memInfo; + memInfo.dwLength = sizeof(MEMORYSTATUSEX); + GlobalMemoryStatusEx(&memInfo); + return static_cast(memInfo.ullTotalPhys); +#elif defined(__linux__) + struct sysinfo memInfo; + sysinfo(&memInfo); + + auto totalPhysMem = memInfo.totalram; + + totalPhysMem *= memInfo.mem_unit; + return static_cast(totalPhysMem); +#elif defined(__FreeBSD__) + u_long physMem; + size_t physMemLen = sizeof(physMem); + int mib[] = { CTL_HW, HW_PHYSMEM }; + + sysctl(mib, sizeof(mib) / sizeof(*mib), &physMem, &physMemLen, NULL, 0); + return physMem; +#else + return 0; +#endif + } + +} + +#endif // spp_memory_h_guard diff --git a/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/parallel_hashmap/phmap.h b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/parallel_hashmap/phmap.h new file mode 100644 index 00000000..a4215875 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/parallel_hashmap/phmap.h @@ -0,0 +1,5118 @@ +#if !defined(phmap_h_guard_) +#define phmap_h_guard_ + +// --------------------------------------------------------------------------- +// Copyright (c) 2019, Gregory Popovitch - greg7mdp@gmail.com +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Includes work from abseil-cpp (https://github.com/abseil/abseil-cpp) +// with modifications. +// +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// --------------------------------------------------------------------------- + +// --------------------------------------------------------------------------- +// IMPLEMENTATION DETAILS +// +// The table stores elements inline in a slot array. In addition to the slot +// array the table maintains some control state per slot. The extra state is one +// byte per slot and stores empty or deleted marks, or alternatively 7 bits from +// the hash of an occupied slot. The table is split into logical groups of +// slots, like so: +// +// Group 1 Group 2 Group 3 +// +---------------+---------------+---------------+ +// | | | | | | | | | | | | | | | | | | | | | | | | | +// +---------------+---------------+---------------+ +// +// On lookup the hash is split into two parts: +// - H2: 7 bits (those stored in the control bytes) +// - H1: the rest of the bits +// The groups are probed using H1. For each group the slots are matched to H2 in +// parallel. Because H2 is 7 bits (128 states) and the number of slots per group +// is low (8 or 16) in almost all cases a match in H2 is also a lookup hit. +// +// On insert, once the right group is found (as in lookup), its slots are +// filled in order. +// +// On erase a slot is cleared. In case the group did not have any empty slots +// before the erase, the erased slot is marked as deleted. +// +// Groups without empty slots (but maybe with deleted slots) extend the probe +// sequence. The probing algorithm is quadratic. Given N the number of groups, +// the probing function for the i'th probe is: +// +// P(0) = H1 % N +// +// P(i) = (P(i - 1) + i) % N +// +// This probing function guarantees that after N probes, all the groups of the +// table will be probed exactly once. +// +// The control state and slot array are stored contiguously in a shared heap +// allocation. The layout of this allocation is: `capacity()` control bytes, +// one sentinel control byte, `Group::kWidth - 1` cloned control bytes, +// , `capacity()` slots. The sentinel control byte is used in +// iteration so we know when we reach the end of the table. The cloned control +// bytes at the end of the table are cloned from the beginning of the table so +// groups that begin near the end of the table can see a full group. In cases in +// which there are more than `capacity()` cloned control bytes, the extra bytes +// are `kEmpty`, and these ensure that we always see at least one empty slot and +// can stop an unsuccessful search. +// --------------------------------------------------------------------------- + + + +#ifdef _MSC_VER + #pragma warning(push) + + #pragma warning(disable : 4127) // conditional expression is constant + #pragma warning(disable : 4324) // structure was padded due to alignment specifier + #pragma warning(disable : 4514) // unreferenced inline function has been removed + #pragma warning(disable : 4623) // default constructor was implicitly defined as deleted + #pragma warning(disable : 4625) // copy constructor was implicitly defined as deleted + #pragma warning(disable : 4626) // assignment operator was implicitly defined as deleted + #pragma warning(disable : 4710) // function not inlined + #pragma warning(disable : 4711) // selected for automatic inline expansion + #pragma warning(disable : 4820) // '6' bytes padding added after data member + #pragma warning(disable : 4868) // compiler may not enforce left-to-right evaluation order in braced initializer list + #pragma warning(disable : 5027) // move assignment operator was implicitly defined as deleted + #pragma warning(disable : 5045) // Compiler will insert Spectre mitigation for memory load if /Qspectre switch specified +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "phmap_fwd_decl.h" +#include "phmap_utils.h" +#include "phmap_base.h" + +#if PHMAP_HAVE_STD_STRING_VIEW + #include +#endif + +namespace phmap { + +namespace priv { + +// -------------------------------------------------------------------------- +template +void SwapAlloc(AllocType& lhs, AllocType& rhs, + std::true_type /* propagate_on_container_swap */) { + using std::swap; + swap(lhs, rhs); +} +template +void SwapAlloc(AllocType& /*lhs*/, AllocType& /*rhs*/, + std::false_type /* propagate_on_container_swap */) {} + +// -------------------------------------------------------------------------- +template +class probe_seq +{ +public: + probe_seq(size_t hashval, size_t mask) { + assert(((mask + 1) & mask) == 0 && "not a mask"); + mask_ = mask; + offset_ = hashval & mask_; + } + size_t offset() const { return offset_; } + size_t offset(size_t i) const { return (offset_ + i) & mask_; } + + void next() { + index_ += Width; + offset_ += index_; + offset_ &= mask_; + } + // 0-based probe index. The i-th probe in the probe sequence. + size_t getindex() const { return index_; } + +private: + size_t mask_; + size_t offset_; + size_t index_ = 0; +}; + +// -------------------------------------------------------------------------- +template +struct RequireUsableKey +{ + template + std::pair< + decltype(std::declval()(std::declval())), + decltype(std::declval()(std::declval(), + std::declval()))>* + operator()(const PassedKey&, const Args&...) const; +}; + +// -------------------------------------------------------------------------- +template +struct IsDecomposable : std::false_type {}; + +template +struct IsDecomposable< + phmap::void_t(), + std::declval()...))>, + Policy, Hash, Eq, Ts...> : std::true_type {}; + +// TODO(alkis): Switch to std::is_nothrow_swappable when gcc/clang supports it. +// -------------------------------------------------------------------------- +template +constexpr bool IsNoThrowSwappable() { + using std::swap; + return noexcept(swap(std::declval(), std::declval())); +} + +// -------------------------------------------------------------------------- +template +int TrailingZeros(T x) { + PHMAP_IF_CONSTEXPR(sizeof(T) == 8) + return base_internal::CountTrailingZerosNonZero64(static_cast(x)); + else + return base_internal::CountTrailingZerosNonZero32(static_cast(x)); +} + +// -------------------------------------------------------------------------- +template +int LeadingZeros(T x) { + PHMAP_IF_CONSTEXPR(sizeof(T) == 8) + return base_internal::CountLeadingZeros64(static_cast(x)); + else + return base_internal::CountLeadingZeros32(static_cast(x)); +} + +// -------------------------------------------------------------------------- +// An abstraction over a bitmask. It provides an easy way to iterate through the +// indexes of the set bits of a bitmask. When Shift=0 (platforms with SSE), +// this is a true bitmask. On non-SSE, platforms the arithematic used to +// emulate the SSE behavior works in bytes (Shift=3) and leaves each bytes as +// either 0x00 or 0x80. +// +// For example: +// for (int i : BitMask(0x5)) -> yields 0, 2 +// for (int i : BitMask(0x0000000080800000)) -> yields 2, 3 +// -------------------------------------------------------------------------- +template +class BitMask +{ + static_assert(std::is_unsigned::value, ""); + static_assert(Shift == 0 || Shift == 3, ""); + +public: + // These are useful for unit tests (gunit). + using value_type = int; + using iterator = BitMask; + using const_iterator = BitMask; + + explicit BitMask(T mask) : mask_(mask) {} + + BitMask& operator++() { // ++iterator + mask_ &= (mask_ - 1); // clear the least significant bit set + return *this; + } + + explicit operator bool() const { return mask_ != 0; } + uint32_t operator*() const { return LowestBitSet(); } + + uint32_t LowestBitSet() const { + return priv::TrailingZeros(mask_) >> Shift; + } + + uint32_t HighestBitSet() const { + return (sizeof(T) * CHAR_BIT - priv::LeadingZeros(mask_) - 1) >> Shift; + } + + BitMask begin() const { return *this; } + BitMask end() const { return BitMask(0); } + + uint32_t TrailingZeros() const { + return priv::TrailingZeros(mask_) >> Shift; + } + + uint32_t LeadingZeros() const { + constexpr uint32_t total_significant_bits = SignificantBits << Shift; + constexpr uint32_t extra_bits = sizeof(T) * 8 - total_significant_bits; + return priv::LeadingZeros(mask_ << extra_bits) >> Shift; + } + +private: + friend bool operator==(const BitMask& a, const BitMask& b) { + return a.mask_ == b.mask_; + } + friend bool operator!=(const BitMask& a, const BitMask& b) { + return a.mask_ != b.mask_; + } + + T mask_; +}; + +// -------------------------------------------------------------------------- +using ctrl_t = signed char; +using h2_t = uint8_t; + +// -------------------------------------------------------------------------- +// The values here are selected for maximum performance. See the static asserts +// below for details. +// -------------------------------------------------------------------------- +enum Ctrl : ctrl_t +{ + kEmpty = -128, // 0b10000000 or 0x80 + kDeleted = -2, // 0b11111110 or 0xfe + kSentinel = -1, // 0b11111111 or 0xff +}; + +static_assert( + kEmpty & kDeleted & kSentinel & 0x80, + "Special markers need to have the MSB to make checking for them efficient"); +static_assert(kEmpty < kSentinel && kDeleted < kSentinel, + "kEmpty and kDeleted must be smaller than kSentinel to make the " + "SIMD test of IsEmptyOrDeleted() efficient"); +static_assert(kSentinel == -1, + "kSentinel must be -1 to elide loading it from memory into SIMD " + "registers (pcmpeqd xmm, xmm)"); +static_assert(kEmpty == -128, + "kEmpty must be -128 to make the SIMD check for its " + "existence efficient (psignb xmm, xmm)"); +static_assert(~kEmpty & ~kDeleted & kSentinel & 0x7F, + "kEmpty and kDeleted must share an unset bit that is not shared " + "by kSentinel to make the scalar test for MatchEmptyOrDeleted() " + "efficient"); +static_assert(kDeleted == -2, + "kDeleted must be -2 to make the implementation of " + "ConvertSpecialToEmptyAndFullToDeleted efficient"); + +// -------------------------------------------------------------------------- +// A single block of empty control bytes for tables without any slots allocated. +// This enables removing a branch in the hot path of find(). +// -------------------------------------------------------------------------- +inline ctrl_t* EmptyGroup() { + alignas(16) static constexpr ctrl_t empty_group[] = { + kSentinel, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, + kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty}; + return const_cast(empty_group); +} + +// -------------------------------------------------------------------------- +inline size_t HashSeed(const ctrl_t* ctrl) { + // The low bits of the pointer have little or no entropy because of + // alignment. We shift the pointer to try to use higher entropy bits. A + // good number seems to be 12 bits, because that aligns with page size. + return reinterpret_cast(ctrl) >> 12; +} + +#ifdef PHMAP_NON_DETERMINISTIC + +inline size_t H1(size_t hashval, const ctrl_t* ctrl) { + // use ctrl_ pointer to add entropy to ensure + // non-deterministic iteration order. + return (hashval >> 7) ^ HashSeed(ctrl); +} + +#else + +inline size_t H1(size_t hashval, const ctrl_t* ) { + return (hashval >> 7); +} + +#endif + + +inline h2_t H2(size_t hashval) { return (ctrl_t)(hashval & 0x7F); } + +inline bool IsEmpty(ctrl_t c) { return c == kEmpty; } +inline bool IsFull(ctrl_t c) { return c >= static_cast(0); } +inline bool IsDeleted(ctrl_t c) { return c == kDeleted; } +inline bool IsEmptyOrDeleted(ctrl_t c) { return c < kSentinel; } + +#if PHMAP_HAVE_SSE2 + +#ifdef _MSC_VER + #pragma warning(push) + #pragma warning(disable : 4365) // conversion from 'int' to 'T', signed/unsigned mismatch +#endif + +// -------------------------------------------------------------------------- +// https://github.com/abseil/abseil-cpp/issues/209 +// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=87853 +// _mm_cmpgt_epi8 is broken under GCC with -funsigned-char +// Work around this by using the portable implementation of Group +// when using -funsigned-char under GCC. +// -------------------------------------------------------------------------- +inline __m128i _mm_cmpgt_epi8_fixed(__m128i a, __m128i b) { +#if defined(__GNUC__) && !defined(__clang__) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Woverflow" + + if (std::is_unsigned::value) { + const __m128i mask = _mm_set1_epi8(static_cast(0x80)); + const __m128i diff = _mm_subs_epi8(b, a); + return _mm_cmpeq_epi8(_mm_and_si128(diff, mask), mask); + } + + #pragma GCC diagnostic pop +#endif + return _mm_cmpgt_epi8(a, b); +} + +// -------------------------------------------------------------------------- +// -------------------------------------------------------------------------- +struct GroupSse2Impl +{ + enum { kWidth = 16 }; // the number of slots per group + + explicit GroupSse2Impl(const ctrl_t* pos) { + ctrl = _mm_loadu_si128(reinterpret_cast(pos)); + } + + // Returns a bitmask representing the positions of slots that match hash. + // ---------------------------------------------------------------------- + BitMask Match(h2_t hash) const { + auto match = _mm_set1_epi8((char)hash); + return BitMask( + static_cast(_mm_movemask_epi8(_mm_cmpeq_epi8(match, ctrl)))); + } + + // Returns a bitmask representing the positions of empty slots. + // ------------------------------------------------------------ + BitMask MatchEmpty() const { +#if PHMAP_HAVE_SSSE3 + // This only works because kEmpty is -128. + return BitMask( + static_cast(_mm_movemask_epi8(_mm_sign_epi8(ctrl, ctrl)))); +#else + return Match(static_cast(kEmpty)); +#endif + } + +#ifdef __INTEL_COMPILER +#pragma warning push +#pragma warning disable 68 +#endif + // Returns a bitmask representing the positions of empty or deleted slots. + // ----------------------------------------------------------------------- + BitMask MatchEmptyOrDeleted() const { + auto special = _mm_set1_epi8(static_cast(kSentinel)); + return BitMask( + static_cast(_mm_movemask_epi8(_mm_cmpgt_epi8_fixed(special, ctrl)))); + } + + // Returns the number of trailing empty or deleted elements in the group. + // ---------------------------------------------------------------------- + uint32_t CountLeadingEmptyOrDeleted() const { + auto special = _mm_set1_epi8(static_cast(kSentinel)); + return TrailingZeros( + static_cast(_mm_movemask_epi8(_mm_cmpgt_epi8_fixed(special, ctrl)) + 1)); + } +#ifdef __INTEL_COMPILER +#pragma warning pop +#endif + + // ---------------------------------------------------------------------- + void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const { + auto msbs = _mm_set1_epi8(static_cast(-128)); + auto x126 = _mm_set1_epi8(126); +#if PHMAP_HAVE_SSSE3 + auto res = _mm_or_si128(_mm_shuffle_epi8(x126, ctrl), msbs); +#else + auto zero = _mm_setzero_si128(); + auto special_mask = _mm_cmpgt_epi8_fixed(zero, ctrl); + auto res = _mm_or_si128(msbs, _mm_andnot_si128(special_mask, x126)); +#endif + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), res); + } + + __m128i ctrl; +}; + +#ifdef _MSC_VER + #pragma warning(pop) +#endif + +#endif // PHMAP_HAVE_SSE2 + +// -------------------------------------------------------------------------- +// -------------------------------------------------------------------------- +struct GroupPortableImpl +{ + enum { kWidth = 8 }; + + explicit GroupPortableImpl(const ctrl_t* pos) + : ctrl(little_endian::Load64(pos)) {} + + BitMask Match(h2_t hash) const { + // For the technique, see: + // http://graphics.stanford.edu/~seander/bithacks.html##ValueInWord + // (Determine if a word has a byte equal to n). + // + // Caveat: there are false positives but: + // - they only occur if there is a real match + // - they never occur on kEmpty, kDeleted, kSentinel + // - they will be handled gracefully by subsequent checks in code + // + // Example: + // v = 0x1716151413121110 + // hash = 0x12 + // retval = (v - lsbs) & ~v & msbs = 0x0000000080800000 + constexpr uint64_t msbs = 0x8080808080808080ULL; + constexpr uint64_t lsbs = 0x0101010101010101ULL; + auto x = ctrl ^ (lsbs * hash); + return BitMask((x - lsbs) & ~x & msbs); + } + + BitMask MatchEmpty() const { // bit 1 of each byte is 0 for empty (but not for deleted) + constexpr uint64_t msbs = 0x8080808080808080ULL; + return BitMask((ctrl & (~ctrl << 6)) & msbs); + } + + BitMask MatchEmptyOrDeleted() const { // lsb of each byte is 0 for empty or deleted + constexpr uint64_t msbs = 0x8080808080808080ULL; + return BitMask((ctrl & (~ctrl << 7)) & msbs); + } + + uint32_t CountLeadingEmptyOrDeleted() const { + constexpr uint64_t gaps = 0x00FEFEFEFEFEFEFEULL; + return (uint32_t)((TrailingZeros(((~ctrl & (ctrl >> 7)) | gaps) + 1) + 7) >> 3); + } + + void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const { + constexpr uint64_t msbs = 0x8080808080808080ULL; + constexpr uint64_t lsbs = 0x0101010101010101ULL; + auto x = ctrl & msbs; + auto res = (~x + (x >> 7)) & ~lsbs; + little_endian::Store64(dst, res); + } + + uint64_t ctrl; +}; + +#if PHMAP_HAVE_SSE2 + using Group = GroupSse2Impl; +#else + using Group = GroupPortableImpl; +#endif + +// The number of cloned control bytes that we copy from the beginning to the +// end of the control bytes array. +// ------------------------------------------------------------------------- +constexpr size_t NumClonedBytes() { return Group::kWidth - 1; } + +template +class raw_hash_set; + +inline bool IsValidCapacity(size_t n) { return ((n + 1) & n) == 0 && n > 0; } + +// -------------------------------------------------------------------------- +// PRECONDITION: +// IsValidCapacity(capacity) +// ctrl[capacity] == kSentinel +// ctrl[i] != kSentinel for all i < capacity +// Applies mapping for every byte in ctrl: +// DELETED -> EMPTY +// EMPTY -> EMPTY +// FULL -> DELETED +// -------------------------------------------------------------------------- +inline void ConvertDeletedToEmptyAndFullToDeleted( + ctrl_t* ctrl, size_t capacity) +{ + assert(ctrl[capacity] == kSentinel); + assert(IsValidCapacity(capacity)); + for (ctrl_t* pos = ctrl; pos != ctrl + capacity + 1; pos += Group::kWidth) { + Group{pos}.ConvertSpecialToEmptyAndFullToDeleted(pos); + } + // Copy the cloned ctrl bytes. + std::memcpy(ctrl + capacity + 1, ctrl, Group::kWidth); + ctrl[capacity] = kSentinel; +} + +// -------------------------------------------------------------------------- +// Rounds up the capacity to the next power of 2 minus 1, with a minimum of 1. +// -------------------------------------------------------------------------- +inline size_t NormalizeCapacity(size_t n) +{ + return n ? ~size_t{} >> LeadingZeros(n) : 1; +} + +// -------------------------------------------------------------------------- +// We use 7/8th as maximum load factor. +// For 16-wide groups, that gives an average of two empty slots per group. +// -------------------------------------------------------------------------- +inline size_t CapacityToGrowth(size_t capacity) +{ + assert(IsValidCapacity(capacity)); + // `capacity*7/8` + PHMAP_IF_CONSTEXPR (Group::kWidth == 8) { + if (capacity == 7) + { + // x-x/8 does not work when x==7. + return 6; + } + } + return capacity - capacity / 8; +} + +// -------------------------------------------------------------------------- +// From desired "growth" to a lowerbound of the necessary capacity. +// Might not be a valid one and required NormalizeCapacity(). +// -------------------------------------------------------------------------- +inline size_t GrowthToLowerboundCapacity(size_t growth) +{ + // `growth*8/7` + PHMAP_IF_CONSTEXPR (Group::kWidth == 8) { + if (growth == 7) + { + // x+(x-1)/7 does not work when x==7. + return 8; + } + } + return growth + static_cast((static_cast(growth) - 1) / 7); +} + +namespace hashtable_debug_internal { + +// If it is a map, call get<0>(). +using std::get; +template +auto GetKey(const typename T::value_type& pair, int) -> decltype(get<0>(pair)) { + return get<0>(pair); +} + +// If it is not a map, return the value directly. +template +const typename T::key_type& GetKey(const typename T::key_type& key, char) { + return key; +} + +// -------------------------------------------------------------------------- +// Containers should specialize this to provide debug information for that +// container. +// -------------------------------------------------------------------------- +template +struct HashtableDebugAccess +{ + // Returns the number of probes required to find `key` in `c`. The "number of + // probes" is a concept that can vary by container. Implementations should + // return 0 when `key` was found in the minimum number of operations and + // should increment the result for each non-trivial operation required to find + // `key`. + // + // The default implementation uses the bucket api from the standard and thus + // works for `std::unordered_*` containers. + // -------------------------------------------------------------------------- + static size_t GetNumProbes(const Container& c, + const typename Container::key_type& key) { + if (!c.bucket_count()) return {}; + size_t num_probes = 0; + size_t bucket = c.bucket(key); + for (auto it = c.begin(bucket), e = c.end(bucket);; ++it, ++num_probes) { + if (it == e) return num_probes; + if (c.key_eq()(key, GetKey(*it, 0))) return num_probes; + } + } +}; + +} // namespace hashtable_debug_internal + +// ---------------------------------------------------------------------------- +// I N F O Z S T U B S +// ---------------------------------------------------------------------------- +struct HashtablezInfo +{ + void PrepareForSampling() {} +}; + +inline void RecordRehashSlow(HashtablezInfo*, size_t ) {} + +static inline void RecordInsertSlow(HashtablezInfo* , size_t, size_t ) {} + +static inline void RecordEraseSlow(HashtablezInfo*) {} + +static inline HashtablezInfo* SampleSlow(int64_t*) { return nullptr; } +static inline void UnsampleSlow(HashtablezInfo* ) {} + +class HashtablezInfoHandle +{ +public: + inline void RecordStorageChanged(size_t , size_t ) {} + inline void RecordRehash(size_t ) {} + inline void RecordInsert(size_t , size_t ) {} + inline void RecordErase() {} + friend inline void swap(HashtablezInfoHandle& , + HashtablezInfoHandle& ) noexcept {} +}; + +static inline HashtablezInfoHandle Sample() { return HashtablezInfoHandle(); } + +class HashtablezSampler +{ +public: + // Returns a global Sampler. + static HashtablezSampler& Global() { static HashtablezSampler hzs; return hzs; } + HashtablezInfo* Register() { static HashtablezInfo info; return &info; } + void Unregister(HashtablezInfo* ) {} + + using DisposeCallback = void (*)(const HashtablezInfo&); + DisposeCallback SetDisposeCallback(DisposeCallback ) { return nullptr; } + int64_t Iterate(const std::function& ) { return 0; } +}; + +static inline void SetHashtablezEnabled(bool ) {} +static inline void SetHashtablezSampleParameter(int32_t ) {} +static inline void SetHashtablezMaxSamples(int32_t ) {} + + +namespace memory_internal { + +// Constructs T into uninitialized storage pointed by `ptr` using the args +// specified in the tuple. +// ---------------------------------------------------------------------------- +template +void ConstructFromTupleImpl(Alloc* alloc, T* ptr, Tuple&& t, + phmap::index_sequence) { + phmap::allocator_traits::construct( + *alloc, ptr, std::get(std::forward(t))...); +} + +template +struct WithConstructedImplF { + template + decltype(std::declval()(std::declval())) operator()( + Args&&... args) const { + return std::forward(f)(T(std::forward(args)...)); + } + F&& f; +}; + +template +decltype(std::declval()(std::declval())) WithConstructedImpl( + Tuple&& t, phmap::index_sequence, F&& f) { + return WithConstructedImplF{std::forward(f)}( + std::get(std::forward(t))...); +} + +template +auto TupleRefImpl(T&& t, phmap::index_sequence) + -> decltype(std::forward_as_tuple(std::get(std::forward(t))...)) { + return std::forward_as_tuple(std::get(std::forward(t))...); +} + +// Returns a tuple of references to the elements of the input tuple. T must be a +// tuple. +// ---------------------------------------------------------------------------- +template +auto TupleRef(T&& t) -> decltype( + TupleRefImpl(std::forward(t), + phmap::make_index_sequence< + std::tuple_size::type>::value>())) { + return TupleRefImpl( + std::forward(t), + phmap::make_index_sequence< + std::tuple_size::type>::value>()); +} + +template +decltype(std::declval()(std::declval(), std::piecewise_construct, + std::declval>(), std::declval())) +DecomposePairImpl(F&& f, std::pair, V> p) { + const auto& key = std::get<0>(p.first); + return std::forward(f)(key, std::piecewise_construct, std::move(p.first), + std::move(p.second)); +} + +} // namespace memory_internal + + +// ---------------------------------------------------------------------------- +// R A W _ H A S H _ S E T +// ---------------------------------------------------------------------------- +// An open-addressing +// hashtable with quadratic probing. +// +// This is a low level hashtable on top of which different interfaces can be +// implemented, like flat_hash_set, node_hash_set, string_hash_set, etc. +// +// The table interface is similar to that of std::unordered_set. Notable +// differences are that most member functions support heterogeneous keys when +// BOTH the hash and eq functions are marked as transparent. They do so by +// providing a typedef called `is_transparent`. +// +// When heterogeneous lookup is enabled, functions that take key_type act as if +// they have an overload set like: +// +// iterator find(const key_type& key); +// template +// iterator find(const K& key); +// +// size_type erase(const key_type& key); +// template +// size_type erase(const K& key); +// +// std::pair equal_range(const key_type& key); +// template +// std::pair equal_range(const K& key); +// +// When heterogeneous lookup is disabled, only the explicit `key_type` overloads +// exist. +// +// find() also supports passing the hash explicitly: +// +// iterator find(const key_type& key, size_t hash); +// template +// iterator find(const U& key, size_t hash); +// +// In addition the pointer to element and iterator stability guarantees are +// weaker: all iterators and pointers are invalidated after a new element is +// inserted. +// +// IMPLEMENTATION DETAILS +// +// The table stores elements inline in a slot array. In addition to the slot +// array the table maintains some control state per slot. The extra state is one +// byte per slot and stores empty or deleted marks, or alternatively 7 bits from +// the hash of an occupied slot. The table is split into logical groups of +// slots, like so: +// +// Group 1 Group 2 Group 3 +// +---------------+---------------+---------------+ +// | | | | | | | | | | | | | | | | | | | | | | | | | +// +---------------+---------------+---------------+ +// +// On lookup the hash is split into two parts: +// - H2: 7 bits (those stored in the control bytes) +// - H1: the rest of the bits +// The groups are probed using H1. For each group the slots are matched to H2 in +// parallel. Because H2 is 7 bits (128 states) and the number of slots per group +// is low (8 or 16) in almost all cases a match in H2 is also a lookup hit. +// +// On insert, once the right group is found (as in lookup), its slots are +// filled in order. +// +// On erase a slot is cleared. In case the group did not have any empty slots +// before the erase, the erased slot is marked as deleted. +// +// Groups without empty slots (but maybe with deleted slots) extend the probe +// sequence. The probing algorithm is quadratic. Given N the number of groups, +// the probing function for the i'th probe is: +// +// P(0) = H1 % N +// +// P(i) = (P(i - 1) + i) % N +// +// This probing function guarantees that after N probes, all the groups of the +// table will be probed exactly once. +// ---------------------------------------------------------------------------- +template +class raw_hash_set +{ + using PolicyTraits = hash_policy_traits; + using KeyArgImpl = + KeyArg::value && IsTransparent::value>; + +public: + using init_type = typename PolicyTraits::init_type; + using key_type = typename PolicyTraits::key_type; + // TODO(sbenza): Hide slot_type as it is an implementation detail. Needs user + // code fixes! + using slot_type = typename PolicyTraits::slot_type; + using allocator_type = Alloc; + using size_type = size_t; + using difference_type = ptrdiff_t; + using hasher = Hash; + using key_equal = Eq; + using policy_type = Policy; + using value_type = typename PolicyTraits::value_type; + using reference = value_type&; + using const_reference = const value_type&; + using pointer = typename phmap::allocator_traits< + allocator_type>::template rebind_traits::pointer; + using const_pointer = typename phmap::allocator_traits< + allocator_type>::template rebind_traits::const_pointer; + + // Alias used for heterogeneous lookup functions. + // `key_arg` evaluates to `K` when the functors are transparent and to + // `key_type` otherwise. It permits template argument deduction on `K` for the + // transparent case. + template + using key_arg = typename KeyArgImpl::template type; + +private: + // Give an early error when key_type is not hashable/eq. + auto KeyTypeCanBeHashed(const Hash& h, const key_type& k) -> decltype(h(k)); + auto KeyTypeCanBeEq(const Eq& eq, const key_type& k) -> decltype(eq(k, k)); + + using Layout = phmap::priv::Layout; + + static Layout MakeLayout(size_t capacity) { + assert(IsValidCapacity(capacity)); + return Layout(capacity + Group::kWidth + 1, capacity); + } + + using AllocTraits = phmap::allocator_traits; + using SlotAlloc = typename phmap::allocator_traits< + allocator_type>::template rebind_alloc; + using SlotAllocTraits = typename phmap::allocator_traits< + allocator_type>::template rebind_traits; + + static_assert(std::is_lvalue_reference::value, + "Policy::element() must return a reference"); + + template + struct SameAsElementReference + : std::is_same::type>::type, + typename std::remove_cv< + typename std::remove_reference::type>::type> {}; + + // An enabler for insert(T&&): T must be convertible to init_type or be the + // same as [cv] value_type [ref]. + // Note: we separate SameAsElementReference into its own type to avoid using + // reference unless we need to. MSVC doesn't seem to like it in some + // cases. + template + using RequiresInsertable = typename std::enable_if< + phmap::disjunction, + SameAsElementReference>::value, + int>::type; + + // RequiresNotInit is a workaround for gcc prior to 7.1. + // See https://godbolt.org/g/Y4xsUh. + template + using RequiresNotInit = + typename std::enable_if::value, int>::type; + + template + using IsDecomposable = IsDecomposable; + +public: + static_assert(std::is_same::value, + "Allocators with custom pointer types are not supported"); + static_assert(std::is_same::value, + "Allocators with custom pointer types are not supported"); + + class iterator + { + friend class raw_hash_set; + + public: + using iterator_category = std::forward_iterator_tag; + using value_type = typename raw_hash_set::value_type; + using reference = + phmap::conditional_t; + using pointer = phmap::remove_reference_t*; + using difference_type = typename raw_hash_set::difference_type; + + iterator() {} + + // PRECONDITION: not an end() iterator. + reference operator*() const { return PolicyTraits::element(slot_); } + + // PRECONDITION: not an end() iterator. + pointer operator->() const { return &operator*(); } + + // PRECONDITION: not an end() iterator. + iterator& operator++() { + ++ctrl_; + ++slot_; + skip_empty_or_deleted(); + return *this; + } + // PRECONDITION: not an end() iterator. + iterator operator++(int) { + auto tmp = *this; + ++*this; + return tmp; + } + +#if PHMAP_BIDIRECTIONAL + // PRECONDITION: not a begin() iterator. + iterator& operator--() { + assert(ctrl_); + do { + --ctrl_; + --slot_; + } while (IsEmptyOrDeleted(*ctrl_)); + return *this; + } + + // PRECONDITION: not a begin() iterator. + iterator operator--(int) { + auto tmp = *this; + --*this; + return tmp; + } +#endif + + friend bool operator==(const iterator& a, const iterator& b) { + return a.ctrl_ == b.ctrl_; + } + friend bool operator!=(const iterator& a, const iterator& b) { + return !(a == b); + } + + private: + iterator(ctrl_t* ctrl) : ctrl_(ctrl) {} // for end() + iterator(ctrl_t* ctrl, slot_type* slot) : ctrl_(ctrl), slot_(slot) {} + + void skip_empty_or_deleted() { + while (IsEmptyOrDeleted(*ctrl_)) { + // ctrl is not necessarily aligned to Group::kWidth. It is also likely + // to read past the space for ctrl bytes and into slots. This is ok + // because ctrl has sizeof() == 1 and slot has sizeof() >= 1 so there + // is no way to read outside the combined slot array. + uint32_t shift = Group{ctrl_}.CountLeadingEmptyOrDeleted(); + ctrl_ += shift; + slot_ += shift; + } + } + + ctrl_t* ctrl_ = nullptr; + // To avoid uninitialized member warnings, put slot_ in an anonymous union. + // The member is not initialized on singleton and end iterators. + union { + slot_type* slot_; + }; + }; + + class const_iterator + { + friend class raw_hash_set; + + public: + using iterator_category = typename iterator::iterator_category; + using value_type = typename raw_hash_set::value_type; + using reference = typename raw_hash_set::const_reference; + using pointer = typename raw_hash_set::const_pointer; + using difference_type = typename raw_hash_set::difference_type; + + const_iterator() {} + // Implicit construction from iterator. + const_iterator(iterator i) : inner_(std::move(i)) {} + + reference operator*() const { return *inner_; } + pointer operator->() const { return inner_.operator->(); } + + const_iterator& operator++() { + ++inner_; + return *this; + } + const_iterator operator++(int) { return inner_++; } + + friend bool operator==(const const_iterator& a, const const_iterator& b) { + return a.inner_ == b.inner_; + } + friend bool operator!=(const const_iterator& a, const const_iterator& b) { + return !(a == b); + } + + private: + const_iterator(const ctrl_t* ctrl, const slot_type* slot) + : inner_(const_cast(ctrl), const_cast(slot)) {} + + iterator inner_; + }; + + using node_type = node_handle, Alloc>; + using insert_return_type = InsertReturnType; + + raw_hash_set() noexcept( + std::is_nothrow_default_constructible::value&& + std::is_nothrow_default_constructible::value&& + std::is_nothrow_default_constructible::value) {} + + explicit raw_hash_set(size_t bucket_cnt, const hasher& hashfn = hasher(), + const key_equal& eq = key_equal(), + const allocator_type& alloc = allocator_type()) + : ctrl_(EmptyGroup()), settings_(0, hashfn, eq, alloc) { + if (bucket_cnt) { + size_t new_capacity = NormalizeCapacity(bucket_cnt); + reset_growth_left(new_capacity); + initialize_slots(new_capacity); + capacity_ = new_capacity; + } + } + + raw_hash_set(size_t bucket_cnt, const hasher& hashfn, + const allocator_type& alloc) + : raw_hash_set(bucket_cnt, hashfn, key_equal(), alloc) {} + + raw_hash_set(size_t bucket_cnt, const allocator_type& alloc) + : raw_hash_set(bucket_cnt, hasher(), key_equal(), alloc) {} + + explicit raw_hash_set(const allocator_type& alloc) + : raw_hash_set(0, hasher(), key_equal(), alloc) {} + + template + raw_hash_set(InputIter first, InputIter last, size_t bucket_cnt = 0, + const hasher& hashfn = hasher(), const key_equal& eq = key_equal(), + const allocator_type& alloc = allocator_type()) + : raw_hash_set(bucket_cnt, hashfn, eq, alloc) { + insert(first, last); + } + + template + raw_hash_set(InputIter first, InputIter last, size_t bucket_cnt, + const hasher& hashfn, const allocator_type& alloc) + : raw_hash_set(first, last, bucket_cnt, hashfn, key_equal(), alloc) {} + + template + raw_hash_set(InputIter first, InputIter last, size_t bucket_cnt, + const allocator_type& alloc) + : raw_hash_set(first, last, bucket_cnt, hasher(), key_equal(), alloc) {} + + template + raw_hash_set(InputIter first, InputIter last, const allocator_type& alloc) + : raw_hash_set(first, last, 0, hasher(), key_equal(), alloc) {} + + // Instead of accepting std::initializer_list as the first + // argument like std::unordered_set does, we have two overloads + // that accept std::initializer_list and std::initializer_list. + // This is advantageous for performance. + // + // // Turns {"abc", "def"} into std::initializer_list, then + // // copies the strings into the set. + // std::unordered_set s = {"abc", "def"}; + // + // // Turns {"abc", "def"} into std::initializer_list, then + // // copies the strings into the set. + // phmap::flat_hash_set s = {"abc", "def"}; + // + // The same trick is used in insert(). + // + // The enabler is necessary to prevent this constructor from triggering where + // the copy constructor is meant to be called. + // + // phmap::flat_hash_set a, b{a}; + // + // RequiresNotInit is a workaround for gcc prior to 7.1. + template = 0, RequiresInsertable = 0> + raw_hash_set(std::initializer_list init, size_t bucket_cnt = 0, + const hasher& hashfn = hasher(), const key_equal& eq = key_equal(), + const allocator_type& alloc = allocator_type()) + : raw_hash_set(init.begin(), init.end(), bucket_cnt, hashfn, eq, alloc) {} + + raw_hash_set(std::initializer_list init, size_t bucket_cnt = 0, + const hasher& hashfn = hasher(), const key_equal& eq = key_equal(), + const allocator_type& alloc = allocator_type()) + : raw_hash_set(init.begin(), init.end(), bucket_cnt, hashfn, eq, alloc) {} + + template = 0, RequiresInsertable = 0> + raw_hash_set(std::initializer_list init, size_t bucket_cnt, + const hasher& hashfn, const allocator_type& alloc) + : raw_hash_set(init, bucket_cnt, hashfn, key_equal(), alloc) {} + + raw_hash_set(std::initializer_list init, size_t bucket_cnt, + const hasher& hashfn, const allocator_type& alloc) + : raw_hash_set(init, bucket_cnt, hashfn, key_equal(), alloc) {} + + template = 0, RequiresInsertable = 0> + raw_hash_set(std::initializer_list init, size_t bucket_cnt, + const allocator_type& alloc) + : raw_hash_set(init, bucket_cnt, hasher(), key_equal(), alloc) {} + + raw_hash_set(std::initializer_list init, size_t bucket_cnt, + const allocator_type& alloc) + : raw_hash_set(init, bucket_cnt, hasher(), key_equal(), alloc) {} + + template = 0, RequiresInsertable = 0> + raw_hash_set(std::initializer_list init, const allocator_type& alloc) + : raw_hash_set(init, 0, hasher(), key_equal(), alloc) {} + + raw_hash_set(std::initializer_list init, + const allocator_type& alloc) + : raw_hash_set(init, 0, hasher(), key_equal(), alloc) {} + + raw_hash_set(const raw_hash_set& that) + : raw_hash_set(that, AllocTraits::select_on_container_copy_construction( + that.alloc_ref())) {} + + raw_hash_set(const raw_hash_set& that, const allocator_type& a) + : raw_hash_set(0, that.hash_ref(), that.eq_ref(), a) { + rehash(that.capacity()); // operator=() should preserve load_factor + // Because the table is guaranteed to be empty, we can do something faster + // than a full `insert`. + for (const auto& v : that) { + const size_t hashval = PolicyTraits::apply(HashElement{hash_ref()}, v); + auto target = find_first_non_full(hashval); + set_ctrl(target.offset, H2(hashval)); + emplace_at(target.offset, v); + infoz_.RecordInsert(hashval, target.probe_length); + } + size_ = that.size(); + growth_left() -= that.size(); + } + + raw_hash_set(raw_hash_set&& that) noexcept( + std::is_nothrow_copy_constructible::value&& + std::is_nothrow_copy_constructible::value&& + std::is_nothrow_copy_constructible::value) + : ctrl_(phmap::exchange(that.ctrl_, EmptyGroup())), + slots_(phmap::exchange(that.slots_, nullptr)), + size_(phmap::exchange(that.size_, 0)), + capacity_(phmap::exchange(that.capacity_, 0)), + infoz_(phmap::exchange(that.infoz_, HashtablezInfoHandle())), + // Hash, equality and allocator are copied instead of moved because + // `that` must be left valid. If Hash is std::function, moving it + // would create a nullptr functor that cannot be called. + settings_(that.settings_) { + // growth_left was copied above, reset the one from `that`. + that.growth_left() = 0; + } + + raw_hash_set(raw_hash_set&& that, const allocator_type& a) + : ctrl_(EmptyGroup()), + slots_(nullptr), + size_(0), + capacity_(0), + settings_(0, that.hash_ref(), that.eq_ref(), a) { + if (a == that.alloc_ref()) { + std::swap(ctrl_, that.ctrl_); + std::swap(slots_, that.slots_); + std::swap(size_, that.size_); + std::swap(capacity_, that.capacity_); + std::swap(growth_left(), that.growth_left()); + std::swap(infoz_, that.infoz_); + } else { + reserve(that.size()); + // Note: this will copy elements of dense_set and unordered_set instead of + // moving them. This can be fixed if it ever becomes an issue. + for (auto& elem : that) insert(std::move(elem)); + } + } + + raw_hash_set& operator=(const raw_hash_set& that) { + raw_hash_set tmp(that, + AllocTraits::propagate_on_container_copy_assignment::value + ? that.alloc_ref() + : alloc_ref()); + swap(tmp); + return *this; + } + + raw_hash_set& operator=(raw_hash_set&& that) noexcept( + phmap::allocator_traits::is_always_equal::value&& + std::is_nothrow_move_assignable::value&& + std::is_nothrow_move_assignable::value) { + // TODO(sbenza): We should only use the operations from the noexcept clause + // to make sure we actually adhere to that contract. + return move_assign( + std::move(that), + typename AllocTraits::propagate_on_container_move_assignment()); + } + + ~raw_hash_set() { destroy_slots(); } + + iterator begin() { + auto it = iterator_at(0); + it.skip_empty_or_deleted(); + return it; + } + iterator end() + { +#if PHMAP_BIDIRECTIONAL + return iterator_at(capacity_); +#else + return {ctrl_ + capacity_}; +#endif + } + + const_iterator begin() const { + return const_cast(this)->begin(); + } + const_iterator end() const { return const_cast(this)->end(); } + const_iterator cbegin() const { return begin(); } + const_iterator cend() const { return end(); } + + bool empty() const { return !size(); } + size_t size() const { return size_; } + size_t capacity() const { return capacity_; } + size_t max_size() const { return (std::numeric_limits::max)(); } + + PHMAP_ATTRIBUTE_REINITIALIZES void clear() { + // Iterating over this container is O(bucket_count()). When bucket_count() + // is much greater than size(), iteration becomes prohibitively expensive. + // For clear() it is more important to reuse the allocated array when the + // container is small because allocation takes comparatively long time + // compared to destruction of the elements of the container. So we pick the + // largest bucket_count() threshold for which iteration is still fast and + // past that we simply deallocate the array. + if (empty()) + return; + if (capacity_ > 127) { + destroy_slots(); + } else if (capacity_) { + for (size_t i = 0; i != capacity_; ++i) { + if (IsFull(ctrl_[i])) { + PolicyTraits::destroy(&alloc_ref(), slots_ + i); + } + } + size_ = 0; + reset_ctrl(capacity_); + reset_growth_left(capacity_); + } + assert(empty()); + infoz_.RecordStorageChanged(0, capacity_); + } + + // This overload kicks in when the argument is an rvalue of insertable and + // decomposable type other than init_type. + // + // flat_hash_map m; + // m.insert(std::make_pair("abc", 42)); + template = 0, + typename std::enable_if::value, int>::type = 0, + T* = nullptr> + std::pair insert(T&& value) { + return emplace(std::forward(value)); + } + + // This overload kicks in when the argument is a bitfield or an lvalue of + // insertable and decomposable type. + // + // union { int n : 1; }; + // flat_hash_set s; + // s.insert(n); + // + // flat_hash_set s; + // const char* p = "hello"; + // s.insert(p); + // + // TODO(romanp): Once we stop supporting gcc 5.1 and below, replace + // RequiresInsertable with RequiresInsertable. + // We are hitting this bug: https://godbolt.org/g/1Vht4f. + template = 0, + typename std::enable_if::value, int>::type = 0> + std::pair insert(const T& value) { + return emplace(value); + } + + // This overload kicks in when the argument is an rvalue of init_type. Its + // purpose is to handle brace-init-list arguments. + // + // flat_hash_set s; + // s.insert({"abc", 42}); + std::pair insert(init_type&& value) { + return emplace(std::move(value)); + } + + template = 0, + typename std::enable_if::value, int>::type = 0, + T* = nullptr> + iterator insert(const_iterator, T&& value) { + return insert(std::forward(value)).first; + } + + // TODO(romanp): Once we stop supporting gcc 5.1 and below, replace + // RequiresInsertable with RequiresInsertable. + // We are hitting this bug: https://godbolt.org/g/1Vht4f. + template = 0, + typename std::enable_if::value, int>::type = 0> + iterator insert(const_iterator, const T& value) { + return insert(value).first; + } + + iterator insert(const_iterator, init_type&& value) { + return insert(std::move(value)).first; + } + + template + using IsRandomAccess = std::is_same::iterator_category, + std::random_access_iterator_tag>; + + + template + struct has_difference_operator + { + private: + using yes = std::true_type; + using no = std::false_type; + + template static auto test(int) -> decltype(std::declval() - std::declval() == 1, yes()); + template static no test(...); + + public: + static constexpr bool value = std::is_same(0)), yes>::value; + }; + + template ::value, int> = 0> + void insert(InputIt first, InputIt last) { + this->reserve(this->size() + (last - first)); + for (; first != last; ++first) + emplace(*first); + } + + template ::value, int> = 0> + void insert(InputIt first, InputIt last) { + for (; first != last; ++first) + emplace(*first); + } + + template = 0, RequiresInsertable = 0> + void insert(std::initializer_list ilist) { + insert(ilist.begin(), ilist.end()); + } + + void insert(std::initializer_list ilist) { + insert(ilist.begin(), ilist.end()); + } + + insert_return_type insert(node_type&& node) { + if (!node) return {end(), false, node_type()}; + const auto& elem = PolicyTraits::element(CommonAccess::GetSlot(node)); + auto res = PolicyTraits::apply( + InsertSlot{*this, std::move(*CommonAccess::GetSlot(node))}, + elem); + if (res.second) { + CommonAccess::Reset(&node); + return {res.first, true, node_type()}; + } else { + return {res.first, false, std::move(node)}; + } + } + + insert_return_type insert(node_type&& node, size_t hashval) { + if (!node) return {end(), false, node_type()}; + const auto& elem = PolicyTraits::element(CommonAccess::GetSlot(node)); + auto res = PolicyTraits::apply( + InsertSlotWithHash{*this, std::move(*CommonAccess::GetSlot(node)), hashval}, + elem); + if (res.second) { + CommonAccess::Reset(&node); + return {res.first, true, node_type()}; + } else { + return {res.first, false, std::move(node)}; + } + } + + iterator insert(const_iterator, node_type&& node) { + auto res = insert(std::move(node)); + node = std::move(res.node); + return res.position; + } + + // This overload kicks in if we can deduce the key from args. This enables us + // to avoid constructing value_type if an entry with the same key already + // exists. + // + // For example: + // + // flat_hash_map m = {{"abc", "def"}}; + // // Creates no std::string copies and makes no heap allocations. + // m.emplace("abc", "xyz"); + template ::value, int>::type = 0> + std::pair emplace(Args&&... args) { + return PolicyTraits::apply(EmplaceDecomposable{*this}, + std::forward(args)...); + } + + template ::value, int>::type = 0> + std::pair emplace_with_hash(size_t hashval, Args&&... args) { + return PolicyTraits::apply(EmplaceDecomposableHashval{*this, hashval}, std::forward(args)...); + } + + // This overload kicks in if we cannot deduce the key from args. It constructs + // value_type unconditionally and then either moves it into the table or + // destroys. + template ::value, int>::type = 0> + std::pair emplace(Args&&... args) { + typename phmap::aligned_storage::type + raw; + slot_type* slot = reinterpret_cast(&raw); + + PolicyTraits::construct(&alloc_ref(), slot, std::forward(args)...); + const auto& elem = PolicyTraits::element(slot); + return PolicyTraits::apply(InsertSlot{*this, std::move(*slot)}, elem); + } + + template ::value, int>::type = 0> + std::pair emplace_with_hash(size_t hashval, Args&&... args) { + typename phmap::aligned_storage::type raw; + slot_type* slot = reinterpret_cast(&raw); + + PolicyTraits::construct(&alloc_ref(), slot, std::forward(args)...); + const auto& elem = PolicyTraits::element(slot); + return PolicyTraits::apply(InsertSlotWithHash{*this, std::move(*slot), hashval}, elem); + } + + template + iterator emplace_hint(const_iterator, Args&&... args) { + return emplace(std::forward(args)...).first; + } + + template + iterator emplace_hint_with_hash(size_t hashval, const_iterator, Args&&... args) { + return emplace_with_hash(hashval, std::forward(args)...).first; + } + + // Extension API: support for lazy emplace. + // + // Looks up key in the table. If found, returns the iterator to the element. + // Otherwise calls f with one argument of type raw_hash_set::constructor. f + // MUST call raw_hash_set::constructor with arguments as if a + // raw_hash_set::value_type is constructed, otherwise the behavior is + // undefined. + // + // For example: + // + // std::unordered_set s; + // // Makes ArenaStr even if "abc" is in the map. + // s.insert(ArenaString(&arena, "abc")); + // + // flat_hash_set s; + // // Makes ArenaStr only if "abc" is not in the map. + // s.lazy_emplace("abc", [&](const constructor& ctor) { + // ctor(&arena, "abc"); + // }); + // + // WARNING: This API is currently experimental. If there is a way to implement + // the same thing with the rest of the API, prefer that. + class constructor + { + friend class raw_hash_set; + + public: + template + void operator()(Args&&... args) const { + assert(*slot_); + PolicyTraits::construct(alloc_, *slot_, std::forward(args)...); + *slot_ = nullptr; + } + + private: + constructor(allocator_type* a, slot_type** slot) : alloc_(a), slot_(slot) {} + + allocator_type* alloc_; + slot_type** slot_; + }; + + template + iterator lazy_emplace(const key_arg& key, F&& f) { + auto res = find_or_prepare_insert(key); + if (res.second) { + lazy_emplace_at(res.first, std::forward(f)); + } + return iterator_at(res.first); + } + + template + iterator lazy_emplace_with_hash(const key_arg& key, size_t hashval, F&& f) { + auto res = find_or_prepare_insert(key, hashval); + if (res.second) { + lazy_emplace_at(res.first, std::forward(f)); + } + return iterator_at(res.first); + } + + template + void lazy_emplace_at(size_t& idx, F&& f) { + slot_type* slot = slots_ + idx; + std::forward(f)(constructor(&alloc_ref(), &slot)); + assert(!slot); + } + + template + void emplace_single_with_hash(const key_arg& key, size_t hashval, F&& f) { + auto res = find_or_prepare_insert(key, hashval); + if (res.second) + lazy_emplace_at(res.first, std::forward(f)); + else + _erase(iterator_at(res.first)); + } + + + // Extension API: support for heterogeneous keys. + // + // std::unordered_set s; + // // Turns "abc" into std::string. + // s.erase("abc"); + // + // flat_hash_set s; + // // Uses "abc" directly without copying it into std::string. + // s.erase("abc"); + template + size_type erase(const key_arg& key) { + auto it = find(key); + if (it == end()) return 0; + _erase(it); + return 1; + } + + + iterator erase(const_iterator cit) { return erase(cit.inner_); } + + // Erases the element pointed to by `it`. Unlike `std::unordered_set::erase`, + // this method returns void to reduce algorithmic complexity to O(1). In + // order to erase while iterating across a map, use the following idiom (which + // also works for standard containers): + // + // for (auto it = m.begin(), end = m.end(); it != end;) { + // if () { + // m._erase(it++); + // } else { + // ++it; + // } + // } + void _erase(iterator it) { + assert(it != end()); + PolicyTraits::destroy(&alloc_ref(), it.slot_); + erase_meta_only(it); + } + void _erase(const_iterator cit) { _erase(cit.inner_); } + + // This overload is necessary because otherwise erase(const K&) would be + // a better match if non-const iterator is passed as an argument. + iterator erase(iterator it) { + auto res = it; + ++res; + _erase(it); + return res; + } + + iterator erase(const_iterator first, const_iterator last) { + while (first != last) { + _erase(first++); + } + return last.inner_; + } + + // Moves elements from `src` into `this`. + // If the element already exists in `this`, it is left unmodified in `src`. + template + void merge(raw_hash_set& src) { // NOLINT + assert(this != &src); + for (auto it = src.begin(), e = src.end(); it != e; ++it) { + if (PolicyTraits::apply(InsertSlot{*this, std::move(*it.slot_)}, + PolicyTraits::element(it.slot_)) + .second) { + src.erase_meta_only(it); + } + } + } + + template + void merge(raw_hash_set&& src) { + merge(src); + } + + node_type extract(const_iterator position) { + auto node = + CommonAccess::Make(alloc_ref(), position.inner_.slot_); + erase_meta_only(position); + return node; + } + + template < + class K = key_type, + typename std::enable_if::value, int>::type = 0> + node_type extract(const key_arg& key) { + auto it = find(key); + return it == end() ? node_type() : extract(const_iterator{it}); + } + + void swap(raw_hash_set& that) noexcept( + IsNoThrowSwappable() && IsNoThrowSwappable() && + (!AllocTraits::propagate_on_container_swap::value || + IsNoThrowSwappable())) { + using std::swap; + swap(ctrl_, that.ctrl_); + swap(slots_, that.slots_); + swap(size_, that.size_); + swap(capacity_, that.capacity_); + swap(growth_left(), that.growth_left()); + swap(hash_ref(), that.hash_ref()); + swap(eq_ref(), that.eq_ref()); + swap(infoz_, that.infoz_); + if (AllocTraits::propagate_on_container_swap::value) { + swap(alloc_ref(), that.alloc_ref()); + } else { + // If the allocators do not compare equal it is officially undefined + // behavior. We choose to do nothing. + } + } + +#if !defined(PHMAP_NON_DETERMINISTIC) + template + bool phmap_dump(OutputArchive&) const; + + template + bool phmap_load(InputArchive&); +#endif + + void rehash(size_t n) { + if (n == 0 && capacity_ == 0) return; + if (n == 0 && size_ == 0) { + destroy_slots(); + infoz_.RecordStorageChanged(0, 0); + return; + } + // bitor is a faster way of doing `max` here. We will round up to the next + // power-of-2-minus-1, so bitor is good enough. + auto m = NormalizeCapacity((std::max)(n, size())); + // n == 0 unconditionally rehashes as per the standard. + if (n == 0 || m > capacity_) { + resize(m); + } + } + + void reserve(size_t n) { rehash(GrowthToLowerboundCapacity(n)); } + + // Extension API: support for heterogeneous keys. + // + // std::unordered_set s; + // // Turns "abc" into std::string. + // s.count("abc"); + // + // ch_set s; + // // Uses "abc" directly without copying it into std::string. + // s.count("abc"); + template + size_t count(const key_arg& key) const { + return find(key) == end() ? size_t(0) : size_t(1); + } + + // Issues CPU prefetch instructions for the memory needed to find or insert + // a key. Like all lookup functions, this support heterogeneous keys. + // + // NOTE: This is a very low level operation and should not be used without + // specific benchmarks indicating its importance. + void prefetch_hash(size_t hashval) const { + (void)hashval; +#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) + auto seq = probe(hashval); + _mm_prefetch((const char *)(ctrl_ + seq.offset()), _MM_HINT_NTA); + _mm_prefetch((const char *)(slots_ + seq.offset()), _MM_HINT_NTA); +#elif defined(__GNUC__) + auto seq = probe(hashval); + __builtin_prefetch(static_cast(ctrl_ + seq.offset())); + __builtin_prefetch(static_cast(slots_ + seq.offset())); +#endif // __GNUC__ + } + + template + void prefetch(const key_arg& key) const { + prefetch_hash(this->hash(key)); + } + + // The API of find() has two extensions. + // + // 1. The hash can be passed by the user. It must be equal to the hash of the + // key. + // + // 2. The type of the key argument doesn't have to be key_type. This is so + // called heterogeneous key support. + template + iterator find(const key_arg& key, size_t hashval) { + size_t offset; + if (find_impl(key, hashval, offset)) + return iterator_at(offset); + else + return end(); + } + + template + pointer find_ptr(const key_arg& key, size_t hashval) { + size_t offset; + if (find_impl(key, hashval, offset)) + return &PolicyTraits::element(slots_ + offset); + else + return nullptr; + } + + template + iterator find(const key_arg& key) { + return find(key, this->hash(key)); + } + + template + const_iterator find(const key_arg& key, size_t hashval) const { + return const_cast(this)->find(key, hashval); + } + template + const_iterator find(const key_arg& key) const { + return find(key, this->hash(key)); + } + + template + bool contains(const key_arg& key) const { + return find(key) != end(); + } + + template + bool contains(const key_arg& key, size_t hashval) const { + return find(key, hashval) != end(); + } + + template + std::pair equal_range(const key_arg& key) { + auto it = find(key); + if (it != end()) return {it, std::next(it)}; + return {it, it}; + } + template + std::pair equal_range( + const key_arg& key) const { + auto it = find(key); + if (it != end()) return {it, std::next(it)}; + return {it, it}; + } + + size_t bucket_count() const { return capacity_; } + float load_factor() const { + return capacity_ ? static_cast(size()) / capacity_ : 0.0; + } + float max_load_factor() const { return 1.0f; } + void max_load_factor(float) { + // Does nothing. + } + + hasher hash_function() const { return hash_ref(); } // warning: doesn't match internal hash - use hash() member function + key_equal key_eq() const { return eq_ref(); } + allocator_type get_allocator() const { return alloc_ref(); } + + friend bool operator==(const raw_hash_set& a, const raw_hash_set& b) { + if (a.size() != b.size()) return false; + const raw_hash_set* outer = &a; + const raw_hash_set* inner = &b; + if (outer->capacity() > inner->capacity()) + std::swap(outer, inner); + for (const value_type& elem : *outer) + if (!inner->has_element(elem)) return false; + return true; + } + + friend bool operator!=(const raw_hash_set& a, const raw_hash_set& b) { + return !(a == b); + } + + friend void swap(raw_hash_set& a, + raw_hash_set& b) noexcept(noexcept(a.swap(b))) { + a.swap(b); + } + + template + size_t hash(const K& key) const { + return HashElement{hash_ref()}(key); + } + +private: + template + friend struct phmap::priv::hashtable_debug_internal::HashtableDebugAccess; + + template + bool find_impl(const key_arg& key, size_t hashval, size_t& offset) { + auto seq = probe(hashval); + while (true) { + Group g{ ctrl_ + seq.offset() }; + for (uint32_t i : g.Match((h2_t)H2(hashval))) { + offset = seq.offset((size_t)i); + if (PHMAP_PREDICT_TRUE(PolicyTraits::apply( + EqualElement{key, eq_ref()}, + PolicyTraits::element(slots_ + offset)))) + return true; + } + if (PHMAP_PREDICT_TRUE(g.MatchEmpty())) + return false; + seq.next(); + } + } + + struct FindElement + { + template + const_iterator operator()(const K& key, Args&&...) const { + return s.find(key); + } + const raw_hash_set& s; + }; + + struct HashElement + { + template + size_t operator()(const K& key, Args&&...) const { + return phmap_mix()(h(key)); + } + const hasher& h; + }; + + template + struct EqualElement + { + template + bool operator()(const K2& lhs, Args&&...) const { + return eq(lhs, rhs); + } + const K1& rhs; + const key_equal& eq; + }; + + template + std::pair emplace_decomposable(const K& key, size_t hashval, + Args&&... args) + { + auto res = find_or_prepare_insert(key, hashval); + if (res.second) { + emplace_at(res.first, std::forward(args)...); + } + return {iterator_at(res.first), res.second}; + } + + struct EmplaceDecomposable + { + template + std::pair operator()(const K& key, Args&&... args) const { + return s.emplace_decomposable(key, s.hash(key), std::forward(args)...); + } + raw_hash_set& s; + }; + + struct EmplaceDecomposableHashval { + template + std::pair operator()(const K& key, Args&&... args) const { + return s.emplace_decomposable(key, hashval, std::forward(args)...); + } + raw_hash_set& s; + size_t hashval; + }; + + template + struct InsertSlot + { + template + std::pair operator()(const K& key, Args&&...) && { + auto res = s.find_or_prepare_insert(key); + if (res.second) { + PolicyTraits::transfer(&s.alloc_ref(), s.slots_ + res.first, &slot); + } else if (do_destroy) { + PolicyTraits::destroy(&s.alloc_ref(), &slot); + } + return {s.iterator_at(res.first), res.second}; + } + raw_hash_set& s; + // Constructed slot. Either moved into place or destroyed. + slot_type&& slot; + }; + + template + struct InsertSlotWithHash + { + template + std::pair operator()(const K& key, Args&&...) && { + auto res = s.find_or_prepare_insert(key, hashval); + if (res.second) { + PolicyTraits::transfer(&s.alloc_ref(), s.slots_ + res.first, &slot); + } else if (do_destroy) { + PolicyTraits::destroy(&s.alloc_ref(), &slot); + } + return {s.iterator_at(res.first), res.second}; + } + raw_hash_set& s; + // Constructed slot. Either moved into place or destroyed. + slot_type&& slot; + size_t &hashval; + }; + + // "erases" the object from the container, except that it doesn't actually + // destroy the object. It only updates all the metadata of the class. + // This can be used in conjunction with Policy::transfer to move the object to + // another place. + void erase_meta_only(const_iterator it) { + assert(IsFull(*it.inner_.ctrl_) && "erasing a dangling iterator"); + --size_; + const size_t index = (size_t)(it.inner_.ctrl_ - ctrl_); + const size_t index_before = (index - Group::kWidth) & capacity_; + const auto empty_after = Group(it.inner_.ctrl_).MatchEmpty(); + const auto empty_before = Group(ctrl_ + index_before).MatchEmpty(); + + // We count how many consecutive non empties we have to the right and to the + // left of `it`. If the sum is >= kWidth then there is at least one probe + // window that might have seen a full group. + bool was_never_full = + empty_before && empty_after && + static_cast(empty_after.TrailingZeros() + + empty_before.LeadingZeros()) < Group::kWidth; + + set_ctrl(index, was_never_full ? kEmpty : kDeleted); + growth_left() += was_never_full; + infoz_.RecordErase(); + } + + void initialize_slots(size_t new_capacity) { + assert(new_capacity); + if (std::is_same>::value && + slots_ == nullptr) { + infoz_ = Sample(); + } + + auto layout = MakeLayout(new_capacity); + char* mem = static_cast( + Allocate(&alloc_ref(), layout.AllocSize())); + ctrl_ = reinterpret_cast(layout.template Pointer<0>(mem)); + slots_ = layout.template Pointer<1>(mem); + reset_ctrl(new_capacity); + reset_growth_left(new_capacity); + infoz_.RecordStorageChanged(size_, new_capacity); + } + + void destroy_slots() { + if (!capacity_) return; + for (size_t i = 0; i != capacity_; ++i) { + if (IsFull(ctrl_[i])) { + PolicyTraits::destroy(&alloc_ref(), slots_ + i); + } + } + auto layout = MakeLayout(capacity_); + // Unpoison before returning the memory to the allocator. + SanitizerUnpoisonMemoryRegion(slots_, sizeof(slot_type) * capacity_); + Deallocate(&alloc_ref(), ctrl_, layout.AllocSize()); + ctrl_ = EmptyGroup(); + slots_ = nullptr; + size_ = 0; + capacity_ = 0; + growth_left() = 0; + } + + void resize(size_t new_capacity) { + assert(IsValidCapacity(new_capacity)); + auto* old_ctrl = ctrl_; + auto* old_slots = slots_; + const size_t old_capacity = capacity_; + initialize_slots(new_capacity); + capacity_ = new_capacity; + + for (size_t i = 0; i != old_capacity; ++i) { + if (IsFull(old_ctrl[i])) { + size_t hashval = PolicyTraits::apply(HashElement{hash_ref()}, + PolicyTraits::element(old_slots + i)); + auto target = find_first_non_full(hashval); + size_t new_i = target.offset; + set_ctrl(new_i, H2(hashval)); + PolicyTraits::transfer(&alloc_ref(), slots_ + new_i, old_slots + i); + } + } + if (old_capacity) { + SanitizerUnpoisonMemoryRegion(old_slots, + sizeof(slot_type) * old_capacity); + auto layout = MakeLayout(old_capacity); + Deallocate(&alloc_ref(), old_ctrl, + layout.AllocSize()); + } + } + + void drop_deletes_without_resize() PHMAP_ATTRIBUTE_NOINLINE { + assert(IsValidCapacity(capacity_)); + assert(!is_small()); + // Algorithm: + // - mark all DELETED slots as EMPTY + // - mark all FULL slots as DELETED + // - for each slot marked as DELETED + // hash = Hash(element) + // target = find_first_non_full(hash) + // if target is in the same group + // mark slot as FULL + // else if target is EMPTY + // transfer element to target + // mark slot as EMPTY + // mark target as FULL + // else if target is DELETED + // swap current element with target element + // mark target as FULL + // repeat procedure for current slot with moved from element (target) + ConvertDeletedToEmptyAndFullToDeleted(ctrl_, capacity_); + typename phmap::aligned_storage::type + raw; + slot_type* slot = reinterpret_cast(&raw); + for (size_t i = 0; i != capacity_; ++i) { + if (!IsDeleted(ctrl_[i])) continue; + size_t hashval = PolicyTraits::apply(HashElement{hash_ref()}, + PolicyTraits::element(slots_ + i)); + auto target = find_first_non_full(hashval); + size_t new_i = target.offset; + + // Verify if the old and new i fall within the same group wrt the hashval. + // If they do, we don't need to move the object as it falls already in the + // best probe we can. + const auto probe_index = [&](size_t pos) { + return ((pos - probe(hashval).offset()) & capacity_) / Group::kWidth; + }; + + // Element doesn't move. + if (PHMAP_PREDICT_TRUE(probe_index(new_i) == probe_index(i))) { + set_ctrl(i, H2(hashval)); + continue; + } + if (IsEmpty(ctrl_[new_i])) { + // Transfer element to the empty spot. + // set_ctrl poisons/unpoisons the slots so we have to call it at the + // right time. + set_ctrl(new_i, H2(hashval)); + PolicyTraits::transfer(&alloc_ref(), slots_ + new_i, slots_ + i); + set_ctrl(i, kEmpty); + } else { + assert(IsDeleted(ctrl_[new_i])); + set_ctrl(new_i, H2(hashval)); + // Until we are done rehashing, DELETED marks previously FULL slots. + // Swap i and new_i elements. + PolicyTraits::transfer(&alloc_ref(), slot, slots_ + i); + PolicyTraits::transfer(&alloc_ref(), slots_ + i, slots_ + new_i); + PolicyTraits::transfer(&alloc_ref(), slots_ + new_i, slot); + --i; // repeat + } + } + reset_growth_left(capacity_); + } + + void rehash_and_grow_if_necessary() { + if (capacity_ == 0) { + resize(1); + } else if (size() <= CapacityToGrowth(capacity()) / 2) { + // Squash DELETED without growing if there is enough capacity. + drop_deletes_without_resize(); + } else { + // Otherwise grow the container. + resize(capacity_ * 2 + 1); + } + } + + bool has_element(const value_type& elem, size_t hashval) const { + auto seq = probe(hashval); + while (true) { + Group g{ctrl_ + seq.offset()}; + for (uint32_t i : g.Match((h2_t)H2(hashval))) { + if (PHMAP_PREDICT_TRUE(PolicyTraits::element(slots_ + seq.offset((size_t)i)) == + elem)) + return true; + } + if (PHMAP_PREDICT_TRUE(g.MatchEmpty())) return false; + seq.next(); + assert(seq.getindex() < capacity_ && "full table!"); + } + return false; + } + + bool has_element(const value_type& elem) const { + size_t hashval = PolicyTraits::apply(HashElement{hash_ref()}, elem); + return has_element(elem, hashval); + } + + // Probes the raw_hash_set with the probe sequence for hash and returns the + // pointer to the first empty or deleted slot. + // NOTE: this function must work with tables having both kEmpty and kDelete + // in one group. Such tables appears during drop_deletes_without_resize. + // + // This function is very useful when insertions happen and: + // - the input is already a set + // - there are enough slots + // - the element with the hash is not in the table + struct FindInfo + { + size_t offset; + size_t probe_length; + }; + FindInfo find_first_non_full(size_t hashval) { + auto seq = probe(hashval); + while (true) { + Group g{ctrl_ + seq.offset()}; + auto mask = g.MatchEmptyOrDeleted(); + if (mask) { + return {seq.offset((size_t)mask.LowestBitSet()), seq.getindex()}; + } + assert(seq.getindex() < capacity_ && "full table!"); + seq.next(); + } + } + + // TODO(alkis): Optimize this assuming *this and that don't overlap. + raw_hash_set& move_assign(raw_hash_set&& that, std::true_type) { + raw_hash_set tmp(std::move(that)); + swap(tmp); + return *this; + } + raw_hash_set& move_assign(raw_hash_set&& that, std::false_type) { + raw_hash_set tmp(std::move(that), alloc_ref()); + swap(tmp); + return *this; + } + +protected: + template + std::pair find_or_prepare_insert(const K& key, size_t hashval) { + auto seq = probe(hashval); + while (true) { + Group g{ctrl_ + seq.offset()}; + for (uint32_t i : g.Match((h2_t)H2(hashval))) { + if (PHMAP_PREDICT_TRUE(PolicyTraits::apply( + EqualElement{key, eq_ref()}, + PolicyTraits::element(slots_ + seq.offset((size_t)i))))) + return {seq.offset((size_t)i), false}; + } + if (PHMAP_PREDICT_TRUE(g.MatchEmpty())) break; + seq.next(); + } + return {prepare_insert(hashval), true}; + } + + template + std::pair find_or_prepare_insert(const K& key) { + return find_or_prepare_insert(key, this->hash(key)); + } + + size_t prepare_insert(size_t hashval) PHMAP_ATTRIBUTE_NOINLINE { + auto target = find_first_non_full(hashval); + if (PHMAP_PREDICT_FALSE(growth_left() == 0 && + !IsDeleted(ctrl_[target.offset]))) { + rehash_and_grow_if_necessary(); + target = find_first_non_full(hashval); + } + ++size_; + growth_left() -= IsEmpty(ctrl_[target.offset]); + set_ctrl(target.offset, H2(hashval)); + infoz_.RecordInsert(hashval, target.probe_length); + return target.offset; + } + + // Constructs the value in the space pointed by the iterator. This only works + // after an unsuccessful find_or_prepare_insert() and before any other + // modifications happen in the raw_hash_set. + // + // PRECONDITION: i is an index returned from find_or_prepare_insert(k), where + // k is the key decomposed from `forward(args)...`, and the bool + // returned by find_or_prepare_insert(k) was true. + // POSTCONDITION: *m.iterator_at(i) == value_type(forward(args)...). + template + void emplace_at(size_t i, Args&&... args) { + PolicyTraits::construct(&alloc_ref(), slots_ + i, + std::forward(args)...); + +#ifdef PHMAP_CHECK_CONSTRUCTED_VALUE + // this check can be costly, so do it only when requested + assert(PolicyTraits::apply(FindElement{*this}, *iterator_at(i)) == + iterator_at(i) && + "constructed value does not match the lookup key"); +#endif + } + + iterator iterator_at(size_t i) { return {ctrl_ + i, slots_ + i}; } + const_iterator iterator_at(size_t i) const { return {ctrl_ + i, slots_ + i}; } + +private: + friend struct RawHashSetTestOnlyAccess; + + probe_seq probe(size_t hashval) const { + return probe_seq(H1(hashval, ctrl_), capacity_); + } + + // Reset all ctrl bytes back to kEmpty, except the sentinel. + void reset_ctrl(size_t capacity) { + std::memset(ctrl_, kEmpty, capacity + Group::kWidth); + ctrl_[capacity] = kSentinel; + SanitizerPoisonMemoryRegion(slots_, sizeof(slot_type) * capacity); + } + + void reset_growth_left(size_t capacity) { + growth_left() = CapacityToGrowth(capacity) - size_; + } + + // Sets the control byte, and if `i < Group::kWidth`, set the cloned byte at + // the end too. + void set_ctrl(size_t i, ctrl_t h) { + assert(i < capacity_); + + if (IsFull(h)) { + SanitizerUnpoisonObject(slots_ + i); + } else { + SanitizerPoisonObject(slots_ + i); + } + + ctrl_[i] = h; + ctrl_[((i - Group::kWidth) & capacity_) + 1 + + ((Group::kWidth - 1) & capacity_)] = h; + } + + size_t& growth_left() { return settings_.template get<0>(); } + + template class RefSet, + class M, class P, class H, class E, class A> + friend class parallel_hash_set; + + template class RefSet, + class M, class P, class H, class E, class A> + friend class parallel_hash_map; + + // The representation of the object has two modes: + // - small: For capacities < kWidth-1 + // - large: For the rest. + // + // Differences: + // - In small mode we are able to use the whole capacity. The extra control + // bytes give us at least one "empty" control byte to stop the iteration. + // This is important to make 1 a valid capacity. + // + // - In small mode only the first `capacity()` control bytes after the + // sentinel are valid. The rest contain dummy kEmpty values that do not + // represent a real slot. This is important to take into account on + // find_first_non_full(), where we never try ShouldInsertBackwards() for + // small tables. + bool is_small() const { return capacity_ < Group::kWidth - 1; } + + hasher& hash_ref() { return settings_.template get<1>(); } + const hasher& hash_ref() const { return settings_.template get<1>(); } + key_equal& eq_ref() { return settings_.template get<2>(); } + const key_equal& eq_ref() const { return settings_.template get<2>(); } + allocator_type& alloc_ref() { return settings_.template get<3>(); } + const allocator_type& alloc_ref() const { + return settings_.template get<3>(); + } + + // TODO(alkis): Investigate removing some of these fields: + // - ctrl/slots can be derived from each other + // - size can be moved into the slot array + ctrl_t* ctrl_ = EmptyGroup(); // [(capacity + 1) * ctrl_t] + slot_type* slots_ = nullptr; // [capacity * slot_type] + size_t size_ = 0; // number of full slots + size_t capacity_ = 0; // total number of slots + HashtablezInfoHandle infoz_; + phmap::priv::CompressedTuple + settings_{0, hasher{}, key_equal{}, allocator_type{}}; +}; + + +// -------------------------------------------------------------------------- +// -------------------------------------------------------------------------- +template +class raw_hash_map : public raw_hash_set +{ + // P is Policy. It's passed as a template argument to support maps that have + // incomplete types as values, as in unordered_map. + // MappedReference<> may be a non-reference type. + template + using MappedReference = decltype(P::value( + std::addressof(std::declval()))); + + // MappedConstReference<> may be a non-reference type. + template + using MappedConstReference = decltype(P::value( + std::addressof(std::declval()))); + + using KeyArgImpl = + KeyArg::value && IsTransparent::value>; + + using Base = raw_hash_set; + +public: + using key_type = typename Policy::key_type; + using mapped_type = typename Policy::mapped_type; + template + using key_arg = typename KeyArgImpl::template type; + + static_assert(!std::is_reference::value, ""); + + // TODO(b/187807849): Evaluate whether to support reference mapped_type and + // remove this assertion if/when it is supported. + static_assert(!std::is_reference::value, ""); + + using iterator = typename raw_hash_map::raw_hash_set::iterator; + using const_iterator = typename raw_hash_map::raw_hash_set::const_iterator; + + raw_hash_map() {} + using Base::raw_hash_set; // use raw_hash_set constructor + + // The last two template parameters ensure that both arguments are rvalues + // (lvalue arguments are handled by the overloads below). This is necessary + // for supporting bitfield arguments. + // + // union { int n : 1; }; + // flat_hash_map m; + // m.insert_or_assign(n, n); + template + std::pair insert_or_assign(key_arg&& k, V&& v) { + return insert_or_assign_impl(std::forward(k), std::forward(v)); + } + + template + std::pair insert_or_assign(key_arg&& k, const V& v) { + return insert_or_assign_impl(std::forward(k), v); + } + + template + std::pair insert_or_assign(const key_arg& k, V&& v) { + return insert_or_assign_impl(k, std::forward(v)); + } + + template + std::pair insert_or_assign(const key_arg& k, const V& v) { + return insert_or_assign_impl(k, v); + } + + template + iterator insert_or_assign(const_iterator, key_arg&& k, V&& v) { + return insert_or_assign(std::forward(k), std::forward(v)).first; + } + + template + iterator insert_or_assign(const_iterator, key_arg&& k, const V& v) { + return insert_or_assign(std::forward(k), v).first; + } + + template + iterator insert_or_assign(const_iterator, const key_arg& k, V&& v) { + return insert_or_assign(k, std::forward(v)).first; + } + + template + iterator insert_or_assign(const_iterator, const key_arg& k, const V& v) { + return insert_or_assign(k, v).first; + } + + template ::value, int>::type = 0, + K* = nullptr> + std::pair try_emplace(key_arg&& k, Args&&... args) { + return try_emplace_impl(std::forward(k), std::forward(args)...); + } + + template ::value, int>::type = 0> + std::pair try_emplace(const key_arg& k, Args&&... args) { + return try_emplace_impl(k, std::forward(args)...); + } + + template + iterator try_emplace(const_iterator, key_arg&& k, Args&&... args) { + return try_emplace(std::forward(k), std::forward(args)...).first; + } + + template + iterator try_emplace(const_iterator, const key_arg& k, Args&&... args) { + return try_emplace(k, std::forward(args)...).first; + } + + template + MappedReference

at(const key_arg& key) { + auto it = this->find(key); + if (it == this->end()) + phmap::base_internal::ThrowStdOutOfRange("phmap at(): lookup non-existent key"); + return Policy::value(&*it); + } + + template + MappedConstReference

at(const key_arg& key) const { + auto it = this->find(key); + if (it == this->end()) + phmap::base_internal::ThrowStdOutOfRange("phmap at(): lookup non-existent key"); + return Policy::value(&*it); + } + + template + MappedReference

operator[](key_arg&& key) { + return Policy::value(&*try_emplace(std::forward(key)).first); + } + + template + MappedReference

operator[](const key_arg& key) { + return Policy::value(&*try_emplace(key).first); + } + +private: + template + std::pair insert_or_assign_impl(K&& k, V&& v) { + auto res = this->find_or_prepare_insert(k); + if (res.second) + this->emplace_at(res.first, std::forward(k), std::forward(v)); + else + Policy::value(&*this->iterator_at(res.first)) = std::forward(v); + return {this->iterator_at(res.first), res.second}; + } + + template + std::pair try_emplace_impl(K&& k, Args&&... args) { + auto res = this->find_or_prepare_insert(k); + if (res.second) + this->emplace_at(res.first, std::piecewise_construct, + std::forward_as_tuple(std::forward(k)), + std::forward_as_tuple(std::forward(args)...)); + return {this->iterator_at(res.first), res.second}; + } +}; + +// ---------------------------------------------------------------------------- +// ---------------------------------------------------------------------------- +// Returns "random" seed. +inline size_t RandomSeed() +{ +#if PHMAP_HAVE_THREAD_LOCAL + static thread_local size_t counter = 0; + size_t value = ++counter; +#else // PHMAP_HAVE_THREAD_LOCAL + static std::atomic counter(0); + size_t value = counter.fetch_add(1, std::memory_order_relaxed); +#endif // PHMAP_HAVE_THREAD_LOCAL + return value ^ static_cast(reinterpret_cast(&counter)); +} + +// ---------------------------------------------------------------------------- +// ---------------------------------------------------------------------------- +template class RefSet, + class Mtx_, + class Policy, class Hash, class Eq, class Alloc> +class parallel_hash_set +{ + using PolicyTraits = hash_policy_traits; + using KeyArgImpl = + KeyArg::value && IsTransparent::value>; + + static_assert(N <= 12, "N = 12 means 4096 hash tables!"); + constexpr static size_t num_tables = 1 << N; + constexpr static size_t mask = num_tables - 1; + +public: + using EmbeddedSet = RefSet; + using EmbeddedIterator= typename EmbeddedSet::iterator; + using EmbeddedConstIterator= typename EmbeddedSet::const_iterator; + using constructor = typename EmbeddedSet::constructor; + using init_type = typename PolicyTraits::init_type; + using key_type = typename PolicyTraits::key_type; + using slot_type = typename PolicyTraits::slot_type; + using allocator_type = Alloc; + using size_type = size_t; + using difference_type = ptrdiff_t; + using hasher = Hash; + using key_equal = Eq; + using policy_type = Policy; + using value_type = typename PolicyTraits::value_type; + using reference = value_type&; + using const_reference = const value_type&; + using pointer = typename phmap::allocator_traits< + allocator_type>::template rebind_traits::pointer; + using const_pointer = typename phmap::allocator_traits< + allocator_type>::template rebind_traits::const_pointer; + + // Alias used for heterogeneous lookup functions. + // `key_arg` evaluates to `K` when the functors are transparent and to + // `key_type` otherwise. It permits template argument deduction on `K` for the + // transparent case. + // -------------------------------------------------------------------- + template + using key_arg = typename KeyArgImpl::template type; + +protected: + using Lockable = phmap::LockableImpl; + + // -------------------------------------------------------------------- + struct Inner : public Lockable + { + struct Params + { + size_t bucket_cnt; + const hasher& hashfn; + const key_equal& eq; + const allocator_type& alloc; + }; + + Inner() {} + + Inner(Params const &p) : set_(p.bucket_cnt, p.hashfn, p.eq, p.alloc) + {} + + bool operator==(const Inner& o) const + { + typename Lockable::SharedLocks l(const_cast(*this), const_cast(o)); + return set_ == o.set_; + } + + EmbeddedSet set_; + }; + +private: + // Give an early error when key_type is not hashable/eq. + // -------------------------------------------------------------------- + auto KeyTypeCanBeHashed(const Hash& h, const key_type& k) -> decltype(h(k)); + auto KeyTypeCanBeEq(const Eq& eq, const key_type& k) -> decltype(eq(k, k)); + + using AllocTraits = phmap::allocator_traits; + + static_assert(std::is_lvalue_reference::value, + "Policy::element() must return a reference"); + + template + struct SameAsElementReference : std::is_same< + typename std::remove_cv::type>::type, + typename std::remove_cv::type>::type> {}; + + // An enabler for insert(T&&): T must be convertible to init_type or be the + // same as [cv] value_type [ref]. + // Note: we separate SameAsElementReference into its own type to avoid using + // reference unless we need to. MSVC doesn't seem to like it in some + // cases. + // -------------------------------------------------------------------- + template + using RequiresInsertable = typename std::enable_if< + phmap::disjunction, + SameAsElementReference>::value, + int>::type; + + // RequiresNotInit is a workaround for gcc prior to 7.1. + // See https://godbolt.org/g/Y4xsUh. + template + using RequiresNotInit = + typename std::enable_if::value, int>::type; + + template + using IsDecomposable = IsDecomposable; + +public: + static_assert(std::is_same::value, + "Allocators with custom pointer types are not supported"); + static_assert(std::is_same::value, + "Allocators with custom pointer types are not supported"); + + // --------------------- i t e r a t o r ------------------------------ + class iterator + { + friend class parallel_hash_set; + + public: + using iterator_category = std::forward_iterator_tag; + using value_type = typename parallel_hash_set::value_type; + using reference = + phmap::conditional_t; + using pointer = phmap::remove_reference_t*; + using difference_type = typename parallel_hash_set::difference_type; + using Inner = typename parallel_hash_set::Inner; + using EmbeddedSet = typename parallel_hash_set::EmbeddedSet; + using EmbeddedIterator = typename EmbeddedSet::iterator; + + iterator() {} + + reference operator*() const { return *it_; } + pointer operator->() const { return &operator*(); } + + iterator& operator++() { + assert(inner_); // null inner means we are already at the end + ++it_; + skip_empty(); + return *this; + } + + iterator operator++(int) { + assert(inner_); // null inner means we are already at the end + auto tmp = *this; + ++*this; + return tmp; + } + + friend bool operator==(const iterator& a, const iterator& b) { + return a.inner_ == b.inner_ && (!a.inner_ || a.it_ == b.it_); + } + + friend bool operator!=(const iterator& a, const iterator& b) { + return !(a == b); + } + + private: + iterator(Inner *inner, Inner *inner_end, const EmbeddedIterator& it) : + inner_(inner), inner_end_(inner_end), it_(it) { // for begin() and end() + if (inner) + it_end_ = inner->set_.end(); + } + + void skip_empty() { + while (it_ == it_end_) { + ++inner_; + if (inner_ == inner_end_) { + inner_ = nullptr; // marks end() + break; + } + else { + it_ = inner_->set_.begin(); + it_end_ = inner_->set_.end(); + } + } + } + + Inner *inner_ = nullptr; + Inner *inner_end_ = nullptr; + EmbeddedIterator it_, it_end_; + }; + + // --------------------- c o n s t i t e r a t o r ----------------- + class const_iterator + { + friend class parallel_hash_set; + + public: + using iterator_category = typename iterator::iterator_category; + using value_type = typename parallel_hash_set::value_type; + using reference = typename parallel_hash_set::const_reference; + using pointer = typename parallel_hash_set::const_pointer; + using difference_type = typename parallel_hash_set::difference_type; + using Inner = typename parallel_hash_set::Inner; + + const_iterator() {} + // Implicit construction from iterator. + const_iterator(iterator i) : iter_(std::move(i)) {} + + reference operator*() const { return *(iter_); } + pointer operator->() const { return iter_.operator->(); } + + const_iterator& operator++() { + ++iter_; + return *this; + } + const_iterator operator++(int) { return iter_++; } + + friend bool operator==(const const_iterator& a, const const_iterator& b) { + return a.iter_ == b.iter_; + } + friend bool operator!=(const const_iterator& a, const const_iterator& b) { + return !(a == b); + } + + private: + const_iterator(const Inner *inner, const Inner *inner_end, const EmbeddedIterator& it) + : iter_(const_cast(inner), + const_cast(inner_end), + const_cast(it)) {} + + iterator iter_; + }; + + using node_type = node_handle, Alloc>; + using insert_return_type = InsertReturnType; + + // ------------------------- c o n s t r u c t o r s ------------------ + + parallel_hash_set() noexcept( + std::is_nothrow_default_constructible::value&& + std::is_nothrow_default_constructible::value&& + std::is_nothrow_default_constructible::value) {} + +#if (__cplusplus >= 201703L || _MSVC_LANG >= 201402) && (defined(_MSC_VER) || defined(__clang__) || (defined(__GNUC__) && __GNUC__ > 6)) + explicit parallel_hash_set(size_t bucket_cnt, + const hasher& hash_param = hasher(), + const key_equal& eq = key_equal(), + const allocator_type& alloc = allocator_type()) : + parallel_hash_set(typename Inner::Params{bucket_cnt, hash_param, eq, alloc}, + phmap::make_index_sequence{}) + {} + + template + parallel_hash_set(typename Inner::Params const &p, + phmap::index_sequence) : sets_{((void)i, p)...} + {} +#else + explicit parallel_hash_set(size_t bucket_cnt, + const hasher& hash_param = hasher(), + const key_equal& eq = key_equal(), + const allocator_type& alloc = allocator_type()) { + for (auto& inner : sets_) + inner.set_ = EmbeddedSet(bucket_cnt / N, hash_param, eq, alloc); + } +#endif + + parallel_hash_set(size_t bucket_cnt, + const hasher& hash_param, + const allocator_type& alloc) + : parallel_hash_set(bucket_cnt, hash_param, key_equal(), alloc) {} + + parallel_hash_set(size_t bucket_cnt, const allocator_type& alloc) + : parallel_hash_set(bucket_cnt, hasher(), key_equal(), alloc) {} + + explicit parallel_hash_set(const allocator_type& alloc) + : parallel_hash_set(0, hasher(), key_equal(), alloc) {} + + template + parallel_hash_set(InputIter first, InputIter last, size_t bucket_cnt = 0, + const hasher& hash_param = hasher(), const key_equal& eq = key_equal(), + const allocator_type& alloc = allocator_type()) + : parallel_hash_set(bucket_cnt, hash_param, eq, alloc) { + insert(first, last); + } + + template + parallel_hash_set(InputIter first, InputIter last, size_t bucket_cnt, + const hasher& hash_param, const allocator_type& alloc) + : parallel_hash_set(first, last, bucket_cnt, hash_param, key_equal(), alloc) {} + + template + parallel_hash_set(InputIter first, InputIter last, size_t bucket_cnt, + const allocator_type& alloc) + : parallel_hash_set(first, last, bucket_cnt, hasher(), key_equal(), alloc) {} + + template + parallel_hash_set(InputIter first, InputIter last, const allocator_type& alloc) + : parallel_hash_set(first, last, 0, hasher(), key_equal(), alloc) {} + + // Instead of accepting std::initializer_list as the first + // argument like std::unordered_set does, we have two overloads + // that accept std::initializer_list and std::initializer_list. + // This is advantageous for performance. + // + // // Turns {"abc", "def"} into std::initializer_list, then copies + // // the strings into the set. + // std::unordered_set s = {"abc", "def"}; + // + // // Turns {"abc", "def"} into std::initializer_list, then + // // copies the strings into the set. + // phmap::flat_hash_set s = {"abc", "def"}; + // + // The same trick is used in insert(). + // + // The enabler is necessary to prevent this constructor from triggering where + // the copy constructor is meant to be called. + // + // phmap::flat_hash_set a, b{a}; + // + // RequiresNotInit is a workaround for gcc prior to 7.1. + // -------------------------------------------------------------------- + template = 0, RequiresInsertable = 0> + parallel_hash_set(std::initializer_list init, size_t bucket_cnt = 0, + const hasher& hash_param = hasher(), const key_equal& eq = key_equal(), + const allocator_type& alloc = allocator_type()) + : parallel_hash_set(init.begin(), init.end(), bucket_cnt, hash_param, eq, alloc) {} + + parallel_hash_set(std::initializer_list init, size_t bucket_cnt = 0, + const hasher& hash_param = hasher(), const key_equal& eq = key_equal(), + const allocator_type& alloc = allocator_type()) + : parallel_hash_set(init.begin(), init.end(), bucket_cnt, hash_param, eq, alloc) {} + + template = 0, RequiresInsertable = 0> + parallel_hash_set(std::initializer_list init, size_t bucket_cnt, + const hasher& hash_param, const allocator_type& alloc) + : parallel_hash_set(init, bucket_cnt, hash_param, key_equal(), alloc) {} + + parallel_hash_set(std::initializer_list init, size_t bucket_cnt, + const hasher& hash_param, const allocator_type& alloc) + : parallel_hash_set(init, bucket_cnt, hash_param, key_equal(), alloc) {} + + template = 0, RequiresInsertable = 0> + parallel_hash_set(std::initializer_list init, size_t bucket_cnt, + const allocator_type& alloc) + : parallel_hash_set(init, bucket_cnt, hasher(), key_equal(), alloc) {} + + parallel_hash_set(std::initializer_list init, size_t bucket_cnt, + const allocator_type& alloc) + : parallel_hash_set(init, bucket_cnt, hasher(), key_equal(), alloc) {} + + template = 0, RequiresInsertable = 0> + parallel_hash_set(std::initializer_list init, const allocator_type& alloc) + : parallel_hash_set(init, 0, hasher(), key_equal(), alloc) {} + + parallel_hash_set(std::initializer_list init, + const allocator_type& alloc) + : parallel_hash_set(init, 0, hasher(), key_equal(), alloc) {} + + parallel_hash_set(const parallel_hash_set& that) + : parallel_hash_set(that, AllocTraits::select_on_container_copy_construction( + that.alloc_ref())) {} + + parallel_hash_set(const parallel_hash_set& that, const allocator_type& a) + : parallel_hash_set(0, that.hash_ref(), that.eq_ref(), a) { + for (size_t i=0; i::value&& + std::is_nothrow_copy_constructible::value&& + std::is_nothrow_copy_constructible::value) + : parallel_hash_set(std::move(that), that.alloc_ref()) { + } + + parallel_hash_set(parallel_hash_set&& that, const allocator_type& a) + { + for (size_t i=0; i::is_always_equal::value && + std::is_nothrow_move_assignable::value && + std::is_nothrow_move_assignable::value) { + for (size_t i=0; i(this)->begin(); } + const_iterator end() const { return const_cast(this)->end(); } + const_iterator cbegin() const { return begin(); } + const_iterator cend() const { return end(); } + + bool empty() const { return !size(); } + + size_t size() const { + size_t sz = 0; + for (const auto& inner : sets_) + sz += inner.set_.size(); + return sz; + } + + size_t capacity() const { + size_t c = 0; + for (const auto& inner : sets_) + c += inner.set_.capacity(); + return c; + } + + size_t max_size() const { return (std::numeric_limits::max)(); } + + PHMAP_ATTRIBUTE_REINITIALIZES void clear() { + for (auto& inner : sets_) + { + typename Lockable::UniqueLock m(inner); + inner.set_.clear(); + } + } + + // extension - clears only soecified submap + // ---------------------------------------- + void clear(std::size_t submap_index) { + Inner& inner = sets_[submap_index]; + typename Lockable::UniqueLock m(inner); + inner.set_.clear(); + } + + // This overload kicks in when the argument is an rvalue of insertable and + // decomposable type other than init_type. + // + // flat_hash_map m; + // m.insert(std::make_pair("abc", 42)); + // -------------------------------------------------------------------- + template = 0, + typename std::enable_if::value, int>::type = 0, + T* = nullptr> + std::pair insert(T&& value) { + return emplace(std::forward(value)); + } + + // This overload kicks in when the argument is a bitfield or an lvalue of + // insertable and decomposable type. + // + // union { int n : 1; }; + // flat_hash_set s; + // s.insert(n); + // + // flat_hash_set s; + // const char* p = "hello"; + // s.insert(p); + // + // TODO(romanp): Once we stop supporting gcc 5.1 and below, replace + // RequiresInsertable with RequiresInsertable. + // We are hitting this bug: https://godbolt.org/g/1Vht4f. + // -------------------------------------------------------------------- + template < + class T, RequiresInsertable = 0, + typename std::enable_if::value, int>::type = 0> + std::pair insert(const T& value) { + return emplace(value); + } + + // This overload kicks in when the argument is an rvalue of init_type. Its + // purpose is to handle brace-init-list arguments. + // + // flat_hash_set> s; + // s.insert({"abc", 42}); + // -------------------------------------------------------------------- + std::pair insert(init_type&& value) { + return emplace(std::move(value)); + } + + template = 0, + typename std::enable_if::value, int>::type = 0, + T* = nullptr> + iterator insert(const_iterator, T&& value) { + return insert(std::forward(value)).first; + } + + // TODO(romanp): Once we stop supporting gcc 5.1 and below, replace + // RequiresInsertable with RequiresInsertable. + // We are hitting this bug: https://godbolt.org/g/1Vht4f. + // -------------------------------------------------------------------- + template < + class T, RequiresInsertable = 0, + typename std::enable_if::value, int>::type = 0> + iterator insert(const_iterator, const T& value) { + return insert(value).first; + } + + iterator insert(const_iterator, init_type&& value) { + return insert(std::move(value)).first; + } + + template + void insert(InputIt first, InputIt last) { + for (; first != last; ++first) insert(*first); + } + + template = 0, RequiresInsertable = 0> + void insert(std::initializer_list ilist) { + insert(ilist.begin(), ilist.end()); + } + + void insert(std::initializer_list ilist) { + insert(ilist.begin(), ilist.end()); + } + + insert_return_type insert(node_type&& node) { + if (!node) + return {end(), false, node_type()}; + auto& key = node.key(); + size_t hashval = this->hash(key); + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; + + typename Lockable::UniqueLock m(inner); + auto res = set.insert(std::move(node), hashval); + return { make_iterator(&inner, res.position), + res.inserted, + res.inserted ? node_type() : std::move(res.node) }; + } + + iterator insert(const_iterator, node_type&& node) { + return insert(std::move(node)).first; + } + + struct ReturnKey_ + { + template + Key operator()(Key&& k, const Args&...) const { + return std::forward(k); + } + }; + + // -------------------------------------------------------------------- + // phmap extension: emplace_with_hash + // ---------------------------------- + // same as emplace, but hashval is provided + // -------------------------------------------------------------------- + template + std::pair emplace_decomposable_with_hash(const K& key, size_t hashval, Args&&... args) + { + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; + typename Lockable::UniqueLock m(inner); + return make_rv(&inner, set.emplace_decomposable(key, hashval, std::forward(args)...)); + } + + struct EmplaceDecomposableHashval + { + template + std::pair operator()(const K& key, Args&&... args) const { + return s.emplace_decomposable_with_hash(key, hashval, std::forward(args)...); + } + parallel_hash_set& s; + size_t hashval; + }; + + // This overload kicks in if we can deduce the key from args. This enables us + // to avoid constructing value_type if an entry with the same key already + // exists. + // + // For example: + // + // flat_hash_map m = {{"abc", "def"}}; + // // Creates no std::string copies and makes no heap allocations. + // m.emplace("abc", "xyz"); + // -------------------------------------------------------------------- + template ::value, int>::type = 0> + std::pair emplace_with_hash(size_t hashval, Args&&... args) { + return PolicyTraits::apply(EmplaceDecomposableHashval{*this, hashval}, + std::forward(args)...); + } + + // This overload kicks in if we cannot deduce the key from args. It constructs + // value_type unconditionally and then either moves it into the table or + // destroys. + // -------------------------------------------------------------------- + template ::value, int>::type = 0> + std::pair emplace_with_hash(size_t hashval, Args&&... args) { + typename phmap::aligned_storage::type raw; + slot_type* slot = reinterpret_cast(&raw); + + PolicyTraits::construct(&alloc_ref(), slot, std::forward(args)...); + const auto& elem = PolicyTraits::element(slot); + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; + typename Lockable::UniqueLock m(inner); + typename EmbeddedSet::template InsertSlotWithHash f { + inner, std::move(*slot), hashval}; + return make_rv(PolicyTraits::apply(f, elem)); + } + + template + iterator emplace_hint_with_hash(size_t hashval, const_iterator, Args&&... args) { + return emplace_with_hash(hashval, std::forward(args)...).first; + } + + template + iterator lazy_emplace_with_hash(const key_arg& key, size_t hashval, F&& f) { + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; + typename Lockable::UniqueLock m(inner); + return make_iterator(&inner, set.lazy_emplace_with_hash(key, hashval, std::forward(f))); + } + + // -------------------------------------------------------------------- + // end of phmap expension + // -------------------------------------------------------------------- + + template + std::pair emplace_decomposable(const K& key, Args&&... args) + { + size_t hashval = this->hash(key); + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; + typename Lockable::UniqueLock m(inner); + return make_rv(&inner, set.emplace_decomposable(key, hashval, std::forward(args)...)); + } + + struct EmplaceDecomposable + { + template + std::pair operator()(const K& key, Args&&... args) const { + return s.emplace_decomposable(key, std::forward(args)...); + } + parallel_hash_set& s; + }; + + // This overload kicks in if we can deduce the key from args. This enables us + // to avoid constructing value_type if an entry with the same key already + // exists. + // + // For example: + // + // flat_hash_map m = {{"abc", "def"}}; + // // Creates no std::string copies and makes no heap allocations. + // m.emplace("abc", "xyz"); + // -------------------------------------------------------------------- + template ::value, int>::type = 0> + std::pair emplace(Args&&... args) { + return PolicyTraits::apply(EmplaceDecomposable{*this}, + std::forward(args)...); + } + + // This overload kicks in if we cannot deduce the key from args. It constructs + // value_type unconditionally and then either moves it into the table or + // destroys. + // -------------------------------------------------------------------- + template ::value, int>::type = 0> + std::pair emplace(Args&&... args) { + typename phmap::aligned_storage::type raw; + slot_type* slot = reinterpret_cast(&raw); + size_t hashval = this->hash(PolicyTraits::key(slot)); + + PolicyTraits::construct(&alloc_ref(), slot, std::forward(args)...); + const auto& elem = PolicyTraits::element(slot); + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; + typename Lockable::UniqueLock m(inner); + typename EmbeddedSet::template InsertSlotWithHash f { + inner, std::move(*slot), hashval}; + return make_rv(PolicyTraits::apply(f, elem)); + } + + template + iterator emplace_hint(const_iterator, Args&&... args) { + return emplace(std::forward(args)...).first; + } + + iterator make_iterator(Inner* inner, const EmbeddedIterator it) + { + if (it == inner->set_.end()) + return iterator(); + return iterator(inner, &sets_[0] + num_tables, it); + } + + std::pair make_rv(Inner* inner, + const std::pair& res) + { + return {iterator(inner, &sets_[0] + num_tables, res.first), res.second}; + } + + // lazy_emplace + // ------------ + template + iterator lazy_emplace(const key_arg& key, F&& f) { + auto hashval = this->hash(key); + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; + typename Lockable::UniqueLock m(inner); + return make_iterator(&inner, set.lazy_emplace_with_hash(key, hashval, std::forward(f))); + } + + // emplace_single + // -------------- + template + void emplace_single_with_hash(const key_arg& key, size_t hashval, F&& f) { + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; + typename Lockable::UniqueLock m(inner); + set.emplace_single_with_hash(key, hashval, std::forward(f)); + } + + template + void emplace_single(const key_arg& key, F&& f) { + auto hashval = this->hash(key); + emplace_single_with_hash(key, hashval, std::forward(f)); + } + + // if set contains key, lambda is called with the value_type (under read lock protection), + // and if_contains returns true. This is a const API and lambda should not modify the value + // ----------------------------------------------------------------------------------------- + template + bool if_contains(const key_arg& key, F&& f) const { + return const_cast(this)->template + modify_if_impl(key, std::forward(f)); + } + + // if set contains key, lambda is called with the value_type without read lock protection, + // and if_contains_unsafe returns true. This is a const API and lambda should not modify the value + // This should be used only if we know that no other thread may be mutating the set at the time. + // ----------------------------------------------------------------------------------------- + template + bool if_contains_unsafe(const key_arg& key, F&& f) const { + return const_cast(this)->template + modify_if_impl::DoNothing>(key, std::forward(f)); + } + + // if map contains key, lambda is called with the value_type (under write lock protection), + // and modify_if returns true. This is a non-const API and lambda is allowed to modify the mapped value + // ---------------------------------------------------------------------------------------------------- + template + bool modify_if(const key_arg& key, F&& f) { + return modify_if_impl(key, std::forward(f)); + } + + // ----------------------------------------------------------------------------------------- + template + bool modify_if_impl(const key_arg& key, F&& f) { +#if __cplusplus >= 201703L + static_assert(std::is_invocable::value); +#endif + L m; + auto ptr = this->template find_ptr(key, this->hash(key), m); + if (ptr == nullptr) + return false; + std::forward(f)(*ptr); + return true; + } + + // if map contains key, lambda is called with the mapped value (under write lock protection). + // If the lambda returns true, the key is subsequently erased from the map (the write lock + // is only released after erase). + // returns true if key was erased, false otherwise. + // ---------------------------------------------------------------------------------------------------- + template + bool erase_if(const key_arg& key, F&& f) { + return erase_if_impl(key, std::forward(f)); + } + + template + bool erase_if_impl(const key_arg& key, F&& f) { +#if __cplusplus >= 201703L + static_assert(std::is_invocable::value); +#endif + L m; + auto it = this->template find(key, this->hash(key), m); + if (it == this->end()) return false; + if (std::forward(f)(const_cast(*it))) + { + this->erase(it); + return true; + } + return false; + } + + // if map already contains key, the first lambda is called with the mapped value (under + // write lock protection) and can update the mapped value. + // if map does not contains key, the second lambda is called and it should invoke the + // passed constructor to construct the value + // returns true if key was not already present, false otherwise. + // --------------------------------------------------------------------------------------- + template + bool lazy_emplace_l(const key_arg& key, FExists&& fExists, FEmplace&& fEmplace) { + typename Lockable::UniqueLock m; + auto res = this->find_or_prepare_insert(key, m); + Inner* inner = std::get<0>(res); + if (std::get<2>(res)) + inner->set_.lazy_emplace_at(std::get<1>(res), std::forward(fEmplace)); + else { + auto it = this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res))); + std::forward(fExists)(const_cast(*it)); // in case of the set, non "key" part of value_type can be changed + } + return std::get<2>(res); + } + + // Extension API: support iterating over all values + // + // flat_hash_set s; + // s.insert(...); + // s.for_each([](auto const & key) { + // // Safely iterates over all the keys + // }); + template + void for_each(F&& fCallback) const { + for (auto const& inner : sets_) { + typename Lockable::SharedLock m(const_cast(inner)); + std::for_each(inner.set_.begin(), inner.set_.end(), fCallback); + } + } + + // this version allows to modify the values + template + void for_each_m(F&& fCallback) { + for (auto& inner : sets_) { + typename Lockable::UniqueLock m(inner); + std::for_each(inner.set_.begin(), inner.set_.end(), fCallback); + } + } + +#if __cplusplus >= 201703L + template + void for_each(ExecutionPolicy&& policy, F&& fCallback) const { + std::for_each( + std::forward(policy), sets_.begin(), sets_.end(), + [&](auto const& inner) { + typename Lockable::SharedLock m(const_cast(inner)); + std::for_each(inner.set_.begin(), inner.set_.end(), fCallback); + } + ); + } + + template + void for_each_m(ExecutionPolicy&& policy, F&& fCallback) { + std::for_each( + std::forward(policy), sets_.begin(), sets_.end(), + [&](auto& inner) { + typename Lockable::UniqueLock m(inner); + std::for_each(inner.set_.begin(), inner.set_.end(), fCallback); + } + ); + } +#endif + + // Extension API: access internal submaps by index + // under lock protection + // ex: m.with_submap(i, [&](const Map::EmbeddedSet& set) { + // for (auto& p : set) { ...; }}); + // ------------------------------------------------- + template + void with_submap(size_t idx, F&& fCallback) const { + const Inner& inner = sets_[idx]; + const auto& set = inner.set_; + typename Lockable::SharedLock m(const_cast(inner)); + fCallback(set); + } + + template + void with_submap_m(size_t idx, F&& fCallback) { + Inner& inner = sets_[idx]; + auto& set = inner.set_; + typename Lockable::UniqueLock m(inner); + fCallback(set); + } + + // unsafe, for internal use only + Inner& get_inner(size_t idx) { + return sets_[idx]; + } + + // Extension API: support for heterogeneous keys. + // + // std::unordered_set s; + // // Turns "abc" into std::string. + // s.erase("abc"); + // + // flat_hash_set s; + // // Uses "abc" directly without copying it into std::string. + // s.erase("abc"); + // + // -------------------------------------------------------------------- + template + size_type erase(const key_arg& key) { + auto hashval = this->hash(key); + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; + typename Lockable::UpgradeLock m(inner); + auto it = set.find(key, hashval); + if (it == set.end()) + return 0; + + typename Lockable::UpgradeToUnique unique(m); + set._erase(it); + return 1; + } + + // -------------------------------------------------------------------- + iterator erase(const_iterator cit) { return erase(cit.iter_); } + + // Erases the element pointed to by `it`. Unlike `std::unordered_set::erase`, + // this method returns void to reduce algorithmic complexity to O(1). In + // order to erase while iterating across a map, use the following idiom (which + // also works for standard containers): + // + // for (auto it = m.begin(), end = m.end(); it != end;) { + // if () { + // m._erase(it++); + // } else { + // ++it; + // } + // } + // + // Do not use erase APIs taking iterators when accessing the map concurrently + // -------------------------------------------------------------------- + void _erase(iterator it) { + Inner* inner = it.inner_; + assert(inner != nullptr); + auto& set = inner->set_; + // typename Lockable::UniqueLock m(*inner); // don't lock here + + set._erase(it.it_); + } + void _erase(const_iterator cit) { _erase(cit.iter_); } + + // This overload is necessary because otherwise erase(const K&) would be + // a better match if non-const iterator is passed as an argument. + // Do not use erase APIs taking iterators when accessing the map concurrently + // -------------------------------------------------------------------- + iterator erase(iterator it) { _erase(it++); return it; } + + iterator erase(const_iterator first, const_iterator last) { + while (first != last) { + _erase(first++); + } + return last.iter_; + } + + // Moves elements from `src` into `this`. + // If the element already exists in `this`, it is left unmodified in `src`. + // Do not use erase APIs taking iterators when accessing the map concurrently + // -------------------------------------------------------------------- + template + void merge(parallel_hash_set& src) { // NOLINT + assert(this != &src); + if (this != &src) + { + for (size_t i=0; i + void merge(parallel_hash_set&& src) { + merge(src); + } + + node_type extract(const_iterator position) { + return position.iter_.inner_->set_.extract(EmbeddedConstIterator(position.iter_.it_)); + } + + template < + class K = key_type, + typename std::enable_if::value, int>::type = 0> + node_type extract(const key_arg& key) { + auto it = find(key); + return it == end() ? node_type() : extract(const_iterator{it}); + } + + template + void swap(parallel_hash_set& that) + noexcept(IsNoThrowSwappable() && + (!AllocTraits::propagate_on_container_swap::value || + IsNoThrowSwappable())) + { + using std::swap; + using Lockable2 = phmap::LockableImpl; + + for (size_t i=0; i target ? normalized : target); + } + + // Extension API: support for heterogeneous keys. + // + // std::unordered_set s; + // // Turns "abc" into std::string. + // s.count("abc"); + // + // ch_set s; + // // Uses "abc" directly without copying it into std::string. + // s.count("abc"); + // -------------------------------------------------------------------- + template + size_t count(const key_arg& key) const { + return find(key) == end() ? 0 : 1; + } + + // Issues CPU prefetch instructions for the memory needed to find or insert + // a key. Like all lookup functions, this support heterogeneous keys. + // + // NOTE: This is a very low level operation and should not be used without + // specific benchmarks indicating its importance. + // -------------------------------------------------------------------- + void prefetch_hash(size_t hashval) const { + const Inner& inner = sets_[subidx(hashval)]; + const auto& set = inner.set_; + typename Lockable::SharedLock m(const_cast(inner)); + set.prefetch_hash(hashval); + } + + template + void prefetch(const key_arg& key) const { + prefetch_hash(this->hash(key)); + } + + // The API of find() has two extensions. + // + // 1. The hash can be passed by the user. It must be equal to the hash of the + // key. + // + // 2. The type of the key argument doesn't have to be key_type. This is so + // called heterogeneous key support. + // -------------------------------------------------------------------- + template + iterator find(const key_arg& key, size_t hashval) { + typename Lockable::SharedLock m; + return find(key, hashval, m); + } + + template + iterator find(const key_arg& key) { + return find(key, this->hash(key)); + } + + template + const_iterator find(const key_arg& key, size_t hashval) const { + return const_cast(this)->find(key, hashval); + } + + template + const_iterator find(const key_arg& key) const { + return find(key, this->hash(key)); + } + + template + bool contains(const key_arg& key) const { + return find(key) != end(); + } + + template + bool contains(const key_arg& key, size_t hashval) const { + return find(key, hashval) != end(); + } + + template + std::pair equal_range(const key_arg& key) { + auto it = find(key); + if (it != end()) return {it, std::next(it)}; + return {it, it}; + } + + template + std::pair equal_range( + const key_arg& key) const { + auto it = find(key); + if (it != end()) return {it, std::next(it)}; + return {it, it}; + } + + size_t bucket_count() const { + size_t sz = 0; + for (const auto& inner : sets_) + { + typename Lockable::SharedLock m(const_cast(inner)); + sz += inner.set_.bucket_count(); + } + return sz; + } + + float load_factor() const { + size_t _capacity = bucket_count(); + return _capacity ? static_cast(static_cast(size()) / _capacity) : 0; + } + + float max_load_factor() const { return 1.0f; } + void max_load_factor(float) { + // Does nothing. + } + + hasher hash_function() const { return hash_ref(); } // warning: doesn't match internal hash - use hash() member function + key_equal key_eq() const { return eq_ref(); } + allocator_type get_allocator() const { return alloc_ref(); } + + friend bool operator==(const parallel_hash_set& a, const parallel_hash_set& b) { + return std::equal(a.sets_.begin(), a.sets_.end(), b.sets_.begin()); + } + + friend bool operator!=(const parallel_hash_set& a, const parallel_hash_set& b) { + return !(a == b); + } + + template + friend void swap(parallel_hash_set& a, + parallel_hash_set& b) + noexcept(noexcept(a.swap(b))) + { + a.swap(b); + } + + template + size_t hash(const K& key) const { + return HashElement{hash_ref()}(key); + } + +#if !defined(PHMAP_NON_DETERMINISTIC) + template + bool phmap_dump(OutputArchive& ar) const; + + template + bool phmap_load(InputArchive& ar); +#endif + +private: + template + friend struct phmap::priv::hashtable_debug_internal::HashtableDebugAccess; + + struct FindElement + { + template + const_iterator operator()(const K& key, Args&&...) const { + return s.find(key); + } + const parallel_hash_set& s; + }; + + struct HashElement + { + template + size_t operator()(const K& key, Args&&...) const { + return phmap_mix()(h(key)); + } + const hasher& h; + }; + + template + struct EqualElement + { + template + bool operator()(const K2& lhs, Args&&...) const { + return eq(lhs, rhs); + } + const K1& rhs; + const key_equal& eq; + }; + + // "erases" the object from the container, except that it doesn't actually + // destroy the object. It only updates all the metadata of the class. + // This can be used in conjunction with Policy::transfer to move the object to + // another place. + // -------------------------------------------------------------------- + void erase_meta_only(const_iterator cit) { + auto &it = cit.iter_; + assert(it.set_ != nullptr); + it.set_.erase_meta_only(const_iterator(it.it_)); + } + + void drop_deletes_without_resize() PHMAP_ATTRIBUTE_NOINLINE { + for (auto& inner : sets_) + { + typename Lockable::UniqueLock m(inner); + inner.set_.drop_deletes_without_resize(); + } + } + + bool has_element(const value_type& elem) const { + size_t hashval = PolicyTraits::apply(HashElement{hash_ref()}, elem); + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; + typename Lockable::SharedLock m(const_cast(inner)); + return set.has_element(elem, hashval); + } + + // TODO(alkis): Optimize this assuming *this and that don't overlap. + // -------------------------------------------------------------------- + template + parallel_hash_set& move_assign(parallel_hash_set&& that, std::true_type) { + parallel_hash_set tmp(std::move(that)); + swap(tmp); + return *this; + } + + template + parallel_hash_set& move_assign(parallel_hash_set&& that, std::false_type) { + parallel_hash_set tmp(std::move(that), alloc_ref()); + swap(tmp); + return *this; + } + +protected: + template + pointer find_ptr(const key_arg& key, size_t hashval, L& mutexlock) + { + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; + mutexlock = std::move(L(inner)); + return set.find_ptr(key, hashval); + } + + template + iterator find(const key_arg& key, size_t hashval, L& mutexlock) { + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; + mutexlock = std::move(L(inner)); + return make_iterator(&inner, set.find(key, hashval)); + } + + template + std::tuple + find_or_prepare_insert_with_hash(size_t hashval, const K& key, typename Lockable::UniqueLock &mutexlock) { + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; + mutexlock = std::move(typename Lockable::UniqueLock(inner)); + auto p = set.find_or_prepare_insert(key, hashval); // std::pair + return std::make_tuple(&inner, p.first, p.second); + } + + template + std::tuple + find_or_prepare_insert(const K& key, typename Lockable::UniqueLock &mutexlock) { + return find_or_prepare_insert_with_hash(this->hash(key), key, mutexlock); + } + + iterator iterator_at(Inner *inner, + const EmbeddedIterator& it) { + return {inner, &sets_[0] + num_tables, it}; + } + const_iterator iterator_at(Inner *inner, + const EmbeddedIterator& it) const { + return {inner, &sets_[0] + num_tables, it}; + } + + static size_t subidx(size_t hashval) { + return ((hashval >> 8) ^ (hashval >> 16) ^ (hashval >> 24)) & mask; + } + + static size_t subcnt() { + return num_tables; + } + +private: + friend struct RawHashSetTestOnlyAccess; + + size_t growth_left() { + size_t sz = 0; + for (const auto& set : sets_) + sz += set.growth_left(); + return sz; + } + + hasher& hash_ref() { return sets_[0].set_.hash_ref(); } + const hasher& hash_ref() const { return sets_[0].set_.hash_ref(); } + key_equal& eq_ref() { return sets_[0].set_.eq_ref(); } + const key_equal& eq_ref() const { return sets_[0].set_.eq_ref(); } + allocator_type& alloc_ref() { return sets_[0].set_.alloc_ref(); } + const allocator_type& alloc_ref() const { + return sets_[0].set_.alloc_ref(); + } + +protected: // protected in case users want to derive fromm this + std::array sets_; +}; + +// -------------------------------------------------------------------------- +// -------------------------------------------------------------------------- +template class RefSet, + class Mtx_, + class Policy, class Hash, class Eq, class Alloc> +class parallel_hash_map : public parallel_hash_set +{ + // P is Policy. It's passed as a template argument to support maps that have + // incomplete types as values, as in unordered_map. + // MappedReference<> may be a non-reference type. + template + using MappedReference = decltype(P::value( + std::addressof(std::declval()))); + + // MappedConstReference<> may be a non-reference type. + template + using MappedConstReference = decltype(P::value( + std::addressof(std::declval()))); + + using KeyArgImpl = + KeyArg::value && IsTransparent::value>; + + using Base = typename parallel_hash_map::parallel_hash_set; + using Lockable = phmap::LockableImpl; + +public: + using key_type = typename Policy::key_type; + using mapped_type = typename Policy::mapped_type; + using value_type = typename Base::value_type; + template + using key_arg = typename KeyArgImpl::template type; + + static_assert(!std::is_reference::value, ""); + // TODO(alkis): remove this assertion and verify that reference mapped_type is + // supported. + static_assert(!std::is_reference::value, ""); + + using iterator = typename parallel_hash_map::parallel_hash_set::iterator; + using const_iterator = typename parallel_hash_map::parallel_hash_set::const_iterator; + + parallel_hash_map() {} + +#ifdef __INTEL_COMPILER + using Base::parallel_hash_set; +#else + using parallel_hash_map::parallel_hash_set::parallel_hash_set; +#endif + + // The last two template parameters ensure that both arguments are rvalues + // (lvalue arguments are handled by the overloads below). This is necessary + // for supporting bitfield arguments. + // + // union { int n : 1; }; + // flat_hash_map m; + // m.insert_or_assign(n, n); + template + std::pair insert_or_assign(key_arg&& k, V&& v) { + return insert_or_assign_impl(std::forward(k), std::forward(v)); + } + + template + std::pair insert_or_assign(key_arg&& k, const V& v) { + return insert_or_assign_impl(std::forward(k), v); + } + + template + std::pair insert_or_assign(const key_arg& k, V&& v) { + return insert_or_assign_impl(k, std::forward(v)); + } + + template + std::pair insert_or_assign(const key_arg& k, const V& v) { + return insert_or_assign_impl(k, v); + } + + template + iterator insert_or_assign(const_iterator, key_arg&& k, V&& v) { + return insert_or_assign(std::forward(k), std::forward(v)).first; + } + + template + iterator insert_or_assign(const_iterator, key_arg&& k, const V& v) { + return insert_or_assign(std::forward(k), v).first; + } + + template + iterator insert_or_assign(const_iterator, const key_arg& k, V&& v) { + return insert_or_assign(k, std::forward(v)).first; + } + + template + iterator insert_or_assign(const_iterator, const key_arg& k, const V& v) { + return insert_or_assign(k, v).first; + } + + template ::value, int>::type = 0, + K* = nullptr> + std::pair try_emplace(key_arg&& k, Args&&... args) { + return try_emplace_impl(std::forward(k), std::forward(args)...); + } + + template ::value, int>::type = 0> + std::pair try_emplace(const key_arg& k, Args&&... args) { + return try_emplace_impl(k, std::forward(args)...); + } + + template + iterator try_emplace(const_iterator, key_arg&& k, Args&&... args) { + return try_emplace(std::forward(k), std::forward(args)...).first; + } + + template + iterator try_emplace(const_iterator, const key_arg& k, Args&&... args) { + return try_emplace(k, std::forward(args)...).first; + } + + template + MappedReference

at(const key_arg& key) { + auto it = this->find(key); + if (it == this->end()) + phmap::base_internal::ThrowStdOutOfRange("phmap at(): lookup non-existent key"); + return Policy::value(&*it); + } + + template + MappedConstReference

at(const key_arg& key) const { + auto it = this->find(key); + if (it == this->end()) + phmap::base_internal::ThrowStdOutOfRange("phmap at(): lookup non-existent key"); + return Policy::value(&*it); + } + + // ----------- phmap extensions -------------------------- + + template ::value, int>::type = 0, + K* = nullptr> + std::pair try_emplace_with_hash(size_t hashval, key_arg&& k, Args&&... args) { + return try_emplace_impl_with_hash(hashval, std::forward(k), std::forward(args)...); + } + + template ::value, int>::type = 0> + std::pair try_emplace_with_hash(size_t hashval, const key_arg& k, Args&&... args) { + return try_emplace_impl_with_hash(hashval, k, std::forward(args)...); + } + + template + iterator try_emplace_with_hash(size_t hashval, const_iterator, key_arg&& k, Args&&... args) { + return try_emplace_with_hash(hashval, std::forward(k), std::forward(args)...).first; + } + + template + iterator try_emplace_with_hash(size_t hashval, const_iterator, const key_arg& k, Args&&... args) { + return try_emplace_with_hash(hashval, k, std::forward(args)...).first; + } + + // if map does not contains key, it is inserted and the mapped value is value-constructed + // with the provided arguments (if any), as with try_emplace. + // if map already contains key, then the lambda is called with the mapped value (under + // write lock protection) and can update the mapped value. + // returns true if key was not already present, false otherwise. + // --------------------------------------------------------------------------------------- + template + bool try_emplace_l(K&& k, F&& f, Args&&... args) { + typename Lockable::UniqueLock m; + auto res = this->find_or_prepare_insert(k, m); + typename Base::Inner *inner = std::get<0>(res); + if (std::get<2>(res)) + inner->set_.emplace_at(std::get<1>(res), std::piecewise_construct, + std::forward_as_tuple(std::forward(k)), + std::forward_as_tuple(std::forward(args)...)); + else { + auto it = this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res))); + std::forward(f)(const_cast(*it)); // in case of the set, non "key" part of value_type can be changed + } + return std::get<2>(res); + } + + // ----------- end of phmap extensions -------------------------- + + template + MappedReference

operator[](key_arg&& key) { + return Policy::value(&*try_emplace(std::forward(key)).first); + } + + template + MappedReference

operator[](const key_arg& key) { + return Policy::value(&*try_emplace(key).first); + } + +private: + + template + std::pair insert_or_assign_impl(K&& k, V&& v) { + typename Lockable::UniqueLock m; + auto res = this->find_or_prepare_insert(k, m); + typename Base::Inner *inner = std::get<0>(res); + if (std::get<2>(res)) + inner->set_.emplace_at(std::get<1>(res), std::forward(k), std::forward(v)); + else + Policy::value(&*inner->set_.iterator_at(std::get<1>(res))) = std::forward(v); + return {this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res))), + std::get<2>(res)}; + } + + template + std::pair try_emplace_impl(K&& k, Args&&... args) { + typename Lockable::UniqueLock m; + auto res = this->find_or_prepare_insert(k, m); + typename Base::Inner *inner = std::get<0>(res); + if (std::get<2>(res)) + inner->set_.emplace_at(std::get<1>(res), std::piecewise_construct, + std::forward_as_tuple(std::forward(k)), + std::forward_as_tuple(std::forward(args)...)); + return {this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res))), + std::get<2>(res)}; + } + + template + std::pair try_emplace_impl_with_hash(size_t hashval, K&& k, Args&&... args) { + typename Lockable::UniqueLock m; + auto res = this->find_or_prepare_insert_with_hash(hashval, k, m); + typename Base::Inner *inner = std::get<0>(res); + if (std::get<2>(res)) + inner->set_.emplace_at(std::get<1>(res), std::piecewise_construct, + std::forward_as_tuple(std::forward(k)), + std::forward_as_tuple(std::forward(args)...)); + return {this->iterator_at(inner, inner->set_.iterator_at(std::get<1>(res))), + std::get<2>(res)}; + } + + +}; + + +// Constructs T into uninitialized storage pointed by `ptr` using the args +// specified in the tuple. +// ---------------------------------------------------------------------------- +template +void ConstructFromTuple(Alloc* alloc, T* ptr, Tuple&& t) { + memory_internal::ConstructFromTupleImpl( + alloc, ptr, std::forward(t), + phmap::make_index_sequence< + std::tuple_size::type>::value>()); +} + +// Constructs T using the args specified in the tuple and calls F with the +// constructed value. +// ---------------------------------------------------------------------------- +template +decltype(std::declval()(std::declval())) WithConstructed( + Tuple&& t, F&& f) { + return memory_internal::WithConstructedImpl( + std::forward(t), + phmap::make_index_sequence< + std::tuple_size::type>::value>(), + std::forward(f)); +} + +// ---------------------------------------------------------------------------- +// Given arguments of an std::pair's consructor, PairArgs() returns a pair of +// tuples with references to the passed arguments. The tuples contain +// constructor arguments for the first and the second elements of the pair. +// +// The following two snippets are equivalent. +// +// 1. std::pair p(args...); +// +// 2. auto a = PairArgs(args...); +// std::pair p(std::piecewise_construct, +// std::move(p.first), std::move(p.second)); +// ---------------------------------------------------------------------------- +inline std::pair, std::tuple<>> PairArgs() { return {}; } + +template +std::pair, std::tuple> PairArgs(F&& f, S&& s) { + return {std::piecewise_construct, std::forward_as_tuple(std::forward(f)), + std::forward_as_tuple(std::forward(s))}; +} + +template +std::pair, std::tuple> PairArgs( + const std::pair& p) { + return PairArgs(p.first, p.second); +} + +template +std::pair, std::tuple> PairArgs(std::pair&& p) { + return PairArgs(std::forward(p.first), std::forward(p.second)); +} + +template +auto PairArgs(std::piecewise_construct_t, F&& f, S&& s) + -> decltype(std::make_pair(memory_internal::TupleRef(std::forward(f)), + memory_internal::TupleRef(std::forward(s)))) { + return std::make_pair(memory_internal::TupleRef(std::forward(f)), + memory_internal::TupleRef(std::forward(s))); +} + +// A helper function for implementing apply() in map policies. +// ---------------------------------------------------------------------------- +template +auto DecomposePair(F&& f, Args&&... args) + -> decltype(memory_internal::DecomposePairImpl( + std::forward(f), PairArgs(std::forward(args)...))) { + return memory_internal::DecomposePairImpl( + std::forward(f), PairArgs(std::forward(args)...)); +} + +// A helper function for implementing apply() in set policies. +// ---------------------------------------------------------------------------- +template +decltype(std::declval()(std::declval(), std::declval())) +DecomposeValue(F&& f, Arg&& arg) { + const auto& key = arg; + return std::forward(f)(key, std::forward(arg)); +} + + +// -------------------------------------------------------------------------- +// Policy: a policy defines how to perform different operations on +// the slots of the hashtable (see hash_policy_traits.h for the full interface +// of policy). +// +// Hash: a (possibly polymorphic) functor that hashes keys of the hashtable. The +// functor should accept a key and return size_t as hash. For best performance +// it is important that the hash function provides high entropy across all bits +// of the hash. +// +// Eq: a (possibly polymorphic) functor that compares two keys for equality. It +// should accept two (of possibly different type) keys and return a bool: true +// if they are equal, false if they are not. If two keys compare equal, then +// their hash values as defined by Hash MUST be equal. +// +// Allocator: an Allocator [https://devdocs.io/cpp/concept/allocator] with which +// the storage of the hashtable will be allocated and the elements will be +// constructed and destroyed. +// -------------------------------------------------------------------------- +template +struct FlatHashSetPolicy +{ + using slot_type = T; + using key_type = T; + using init_type = T; + using constant_iterators = std::true_type; + + template + static void construct(Allocator* alloc, slot_type* slot, Args&&... args) { + phmap::allocator_traits::construct(*alloc, slot, + std::forward(args)...); + } + + template + static void destroy(Allocator* alloc, slot_type* slot) { + phmap::allocator_traits::destroy(*alloc, slot); + } + + template + static void transfer(Allocator* alloc, slot_type* new_slot, + slot_type* old_slot) { + construct(alloc, new_slot, std::move(*old_slot)); + destroy(alloc, old_slot); + } + + static T& element(slot_type* slot) { return *slot; } + + template + static decltype(phmap::priv::DecomposeValue( + std::declval(), std::declval()...)) + apply(F&& f, Args&&... args) { + return phmap::priv::DecomposeValue( + std::forward(f), std::forward(args)...); + } + + static size_t space_used(const T*) { return 0; } +}; + +// -------------------------------------------------------------------------- +// -------------------------------------------------------------------------- +template +struct FlatHashMapPolicy +{ + using slot_policy = priv::map_slot_policy; + using slot_type = typename slot_policy::slot_type; + using key_type = K; + using mapped_type = V; + using init_type = std::pair; + + template + static void construct(Allocator* alloc, slot_type* slot, Args&&... args) { + slot_policy::construct(alloc, slot, std::forward(args)...); + } + + template + static void destroy(Allocator* alloc, slot_type* slot) { + slot_policy::destroy(alloc, slot); + } + + template + static void transfer(Allocator* alloc, slot_type* new_slot, + slot_type* old_slot) { + slot_policy::transfer(alloc, new_slot, old_slot); + } + + template + static decltype(phmap::priv::DecomposePair( + std::declval(), std::declval()...)) + apply(F&& f, Args&&... args) { + return phmap::priv::DecomposePair(std::forward(f), + std::forward(args)...); + } + + static size_t space_used(const slot_type*) { return 0; } + + static std::pair& element(slot_type* slot) { return slot->value; } + + static V& value(std::pair* kv) { return kv->second; } + static const V& value(const std::pair* kv) { return kv->second; } +}; + +template +struct node_hash_policy { + static_assert(std::is_lvalue_reference::value, ""); + + using slot_type = typename std::remove_cv< + typename std::remove_reference::type>::type*; + + template + static void construct(Alloc* alloc, slot_type* slot, Args&&... args) { + *slot = Policy::new_element(alloc, std::forward(args)...); + } + + template + static void destroy(Alloc* alloc, slot_type* slot) { + Policy::delete_element(alloc, *slot); + } + + template + static void transfer(Alloc*, slot_type* new_slot, slot_type* old_slot) { + *new_slot = *old_slot; + } + + static size_t space_used(const slot_type* slot) { + if (slot == nullptr) return Policy::element_space_used(nullptr); + return Policy::element_space_used(*slot); + } + + static Reference element(slot_type* slot) { return **slot; } + + template + static auto value(T* elem) -> decltype(P::value(elem)) { + return P::value(elem); + } + + template + static auto apply(Ts&&... ts) -> decltype(P::apply(std::forward(ts)...)) { + return P::apply(std::forward(ts)...); + } +}; + +// -------------------------------------------------------------------------- +// -------------------------------------------------------------------------- +template +struct NodeHashSetPolicy + : phmap::priv::node_hash_policy> +{ + using key_type = T; + using init_type = T; + using constant_iterators = std::true_type; + + template + static T* new_element(Allocator* alloc, Args&&... args) { + using ValueAlloc = + typename phmap::allocator_traits::template rebind_alloc; + ValueAlloc value_alloc(*alloc); + T* res = phmap::allocator_traits::allocate(value_alloc, 1); + phmap::allocator_traits::construct(value_alloc, res, + std::forward(args)...); + return res; + } + + template + static void delete_element(Allocator* alloc, T* elem) { + using ValueAlloc = + typename phmap::allocator_traits::template rebind_alloc; + ValueAlloc value_alloc(*alloc); + phmap::allocator_traits::destroy(value_alloc, elem); + phmap::allocator_traits::deallocate(value_alloc, elem, 1); + } + + template + static decltype(phmap::priv::DecomposeValue( + std::declval(), std::declval()...)) + apply(F&& f, Args&&... args) { + return phmap::priv::DecomposeValue( + std::forward(f), std::forward(args)...); + } + + static size_t element_space_used(const T*) { return sizeof(T); } +}; + +// -------------------------------------------------------------------------- +// -------------------------------------------------------------------------- +template +class NodeHashMapPolicy + : public phmap::priv::node_hash_policy< + std::pair&, NodeHashMapPolicy> +{ + using value_type = std::pair; + +public: + using key_type = Key; + using mapped_type = Value; + using init_type = std::pair; + + template + static value_type* new_element(Allocator* alloc, Args&&... args) { + using PairAlloc = typename phmap::allocator_traits< + Allocator>::template rebind_alloc; + PairAlloc pair_alloc(*alloc); + value_type* res = + phmap::allocator_traits::allocate(pair_alloc, 1); + phmap::allocator_traits::construct(pair_alloc, res, + std::forward(args)...); + return res; + } + + template + static void delete_element(Allocator* alloc, value_type* pair) { + using PairAlloc = typename phmap::allocator_traits< + Allocator>::template rebind_alloc; + PairAlloc pair_alloc(*alloc); + phmap::allocator_traits::destroy(pair_alloc, pair); + phmap::allocator_traits::deallocate(pair_alloc, pair, 1); + } + + template + static decltype(phmap::priv::DecomposePair( + std::declval(), std::declval()...)) + apply(F&& f, Args&&... args) { + return phmap::priv::DecomposePair(std::forward(f), + std::forward(args)...); + } + + static size_t element_space_used(const value_type*) { + return sizeof(value_type); + } + + static Value& value(value_type* elem) { return elem->second; } + static const Value& value(const value_type* elem) { return elem->second; } +}; + + +// -------------------------------------------------------------------------- +// hash_default +// -------------------------------------------------------------------------- + +#if PHMAP_HAVE_STD_STRING_VIEW + +// Supports heterogeneous lookup for basic_string-like elements. +template +struct StringHashEqT +{ + struct Hash + { + using is_transparent = void; + + size_t operator()(std::basic_string_view v) const { + std::string_view bv{ + reinterpret_cast(v.data()), v.size() * sizeof(CharT)}; + return std::hash()(bv); + } + }; + + struct Eq { + using is_transparent = void; + + bool operator()(std::basic_string_view lhs, + std::basic_string_view rhs) const { + return lhs == rhs; + } + }; +}; + +template <> +struct HashEq : StringHashEqT {}; + +template <> +struct HashEq : StringHashEqT {}; + +// char16_t +template <> +struct HashEq : StringHashEqT {}; + +template <> +struct HashEq : StringHashEqT {}; + +// wchar_t +template <> +struct HashEq : StringHashEqT {}; + +template <> +struct HashEq : StringHashEqT {}; + +#endif + +// Supports heterogeneous lookup for pointers and smart pointers. +// ------------------------------------------------------------- +template +struct HashEq +{ + struct Hash { + using is_transparent = void; + template + size_t operator()(const U& ptr) const { + // we want phmap::Hash and not phmap::Hash + // so "struct std::hash " override works + return phmap::Hash{}((T*)(uintptr_t)HashEq::ToPtr(ptr)); + } + }; + + struct Eq { + using is_transparent = void; + template + bool operator()(const A& a, const B& b) const { + return HashEq::ToPtr(a) == HashEq::ToPtr(b); + } + }; + +private: + static const T* ToPtr(const T* ptr) { return ptr; } + + template + static const T* ToPtr(const std::unique_ptr& ptr) { + return ptr.get(); + } + + template + static const T* ToPtr(const std::shared_ptr& ptr) { + return ptr.get(); + } +}; + +template +struct HashEq> : HashEq {}; + +template +struct HashEq> : HashEq {}; + +namespace hashtable_debug_internal { + +// -------------------------------------------------------------------------- +// -------------------------------------------------------------------------- + +template +struct has_member_type_raw_hash_set : std::false_type +{}; +template +struct has_member_type_raw_hash_set> : std::true_type +{}; + +template +struct HashtableDebugAccess::value>::type> +{ + using Traits = typename Set::PolicyTraits; + using Slot = typename Traits::slot_type; + + static size_t GetNumProbes(const Set& set, + const typename Set::key_type& key) { + size_t num_probes = 0; + size_t hashval = set.hash(key); + auto seq = set.probe(hashval); + while (true) { + priv::Group g{set.ctrl_ + seq.offset()}; + for (uint32_t i : g.Match(priv::H2(hashval))) { + if (Traits::apply( + typename Set::template EqualElement{ + key, set.eq_ref()}, + Traits::element(set.slots_ + seq.offset((size_t)i)))) + return num_probes; + ++num_probes; + } + if (g.MatchEmpty()) return num_probes; + seq.next(); + ++num_probes; + } + } + + static size_t AllocatedByteSize(const Set& c) { + size_t capacity = c.capacity_; + if (capacity == 0) return 0; + auto layout = Set::MakeLayout(capacity); + size_t m = layout.AllocSize(); + + size_t per_slot = Traits::space_used(static_cast(nullptr)); + if (per_slot != ~size_t{}) { + m += per_slot * c.size(); + } else { + for (size_t i = 0; i != capacity; ++i) { + if (priv::IsFull(c.ctrl_[i])) { + m += Traits::space_used(c.slots_ + i); + } + } + } + return m; + } + + static size_t LowerBoundAllocatedByteSize(size_t size) { + size_t capacity = GrowthToLowerboundCapacity(size); + if (capacity == 0) return 0; + auto layout = Set::MakeLayout(NormalizeCapacity(capacity)); + size_t m = layout.AllocSize(); + size_t per_slot = Traits::space_used(static_cast(nullptr)); + if (per_slot != ~size_t{}) { + m += per_slot * size; + } + return m; + } +}; + + +template +struct has_member_type_EmbeddedSet : std::false_type +{}; +template +struct has_member_type_EmbeddedSet> : std::true_type +{}; + +template +struct HashtableDebugAccess::value>::type> { + using Traits = typename Set::PolicyTraits; + using Slot = typename Traits::slot_type; + using EmbeddedSet = typename Set::EmbeddedSet; + + static size_t GetNumProbes(const Set& set, const typename Set::key_type& key) { + size_t hashval = set.hash(key); + auto& inner = set.sets_[set.subidx(hashval)]; + auto& inner_set = inner.set_; + return HashtableDebugAccess::GetNumProbes(inner_set, key); + } +}; + +} // namespace hashtable_debug_internal +} // namespace priv + +// ----------------------------------------------------------------------------- +// phmap::flat_hash_set +// ----------------------------------------------------------------------------- +// An `phmap::flat_hash_set` is an unordered associative container which has +// been optimized for both speed and memory footprint in most common use cases. +// Its interface is similar to that of `std::unordered_set` with the +// following notable differences: +// +// * Supports heterogeneous lookup, through `find()`, `operator[]()` and +// `insert()`, provided that the set is provided a compatible heterogeneous +// hashing function and equality operator. +// * Invalidates any references and pointers to elements within the table after +// `rehash()`. +// * Contains a `capacity()` member function indicating the number of element +// slots (open, deleted, and empty) within the hash set. +// * Returns `void` from the `_erase(iterator)` overload. +// ----------------------------------------------------------------------------- +template // default values in phmap_fwd_decl.h +class flat_hash_set + : public phmap::priv::raw_hash_set< + phmap::priv::FlatHashSetPolicy, Hash, Eq, Alloc> +{ + using Base = typename flat_hash_set::raw_hash_set; + +public: + flat_hash_set() {} +#ifdef __INTEL_COMPILER + using Base::raw_hash_set; +#else + using Base::Base; +#endif + using Base::begin; + using Base::cbegin; + using Base::cend; + using Base::end; + using Base::capacity; + using Base::empty; + using Base::max_size; + using Base::size; + using Base::clear; // may shrink - To avoid shrinking `erase(begin(), end())` + using Base::erase; + using Base::insert; + using Base::emplace; + using Base::emplace_hint; + using Base::extract; + using Base::merge; + using Base::swap; + using Base::rehash; + using Base::reserve; + using Base::contains; + using Base::count; + using Base::equal_range; + using Base::find; + using Base::bucket_count; + using Base::load_factor; + using Base::max_load_factor; + using Base::get_allocator; + using Base::hash_function; + using Base::hash; + using Base::key_eq; +}; + +// ----------------------------------------------------------------------------- +// phmap::flat_hash_map +// ----------------------------------------------------------------------------- +// +// An `phmap::flat_hash_map` is an unordered associative container which +// has been optimized for both speed and memory footprint in most common use +// cases. Its interface is similar to that of `std::unordered_map` with +// the following notable differences: +// +// * Supports heterogeneous lookup, through `find()`, `operator[]()` and +// `insert()`, provided that the map is provided a compatible heterogeneous +// hashing function and equality operator. +// * Invalidates any references and pointers to elements within the table after +// `rehash()`. +// * Contains a `capacity()` member function indicating the number of element +// slots (open, deleted, and empty) within the hash map. +// * Returns `void` from the `_erase(iterator)` overload. +// ----------------------------------------------------------------------------- +template // default values in phmap_fwd_decl.h +class flat_hash_map : public phmap::priv::raw_hash_map< + phmap::priv::FlatHashMapPolicy, + Hash, Eq, Alloc> { + using Base = typename flat_hash_map::raw_hash_map; + +public: + flat_hash_map() {} +#ifdef __INTEL_COMPILER + using Base::raw_hash_map; +#else + using Base::Base; +#endif + using Base::begin; + using Base::cbegin; + using Base::cend; + using Base::end; + using Base::capacity; + using Base::empty; + using Base::max_size; + using Base::size; + using Base::clear; + using Base::erase; + using Base::insert; + using Base::insert_or_assign; + using Base::emplace; + using Base::emplace_hint; + using Base::try_emplace; + using Base::extract; + using Base::merge; + using Base::swap; + using Base::rehash; + using Base::reserve; + using Base::at; + using Base::contains; + using Base::count; + using Base::equal_range; + using Base::find; + using Base::operator[]; + using Base::bucket_count; + using Base::load_factor; + using Base::max_load_factor; + using Base::get_allocator; + using Base::hash_function; + using Base::hash; + using Base::key_eq; +}; + +// ----------------------------------------------------------------------------- +// phmap::node_hash_set +// ----------------------------------------------------------------------------- +// An `phmap::node_hash_set` is an unordered associative container which +// has been optimized for both speed and memory footprint in most common use +// cases. Its interface is similar to that of `std::unordered_set` with the +// following notable differences: +// +// * Supports heterogeneous lookup, through `find()`, `operator[]()` and +// `insert()`, provided that the map is provided a compatible heterogeneous +// hashing function and equality operator. +// * Contains a `capacity()` member function indicating the number of element +// slots (open, deleted, and empty) within the hash set. +// * Returns `void` from the `erase(iterator)` overload. +// ----------------------------------------------------------------------------- +template // default values in phmap_fwd_decl.h +class node_hash_set + : public phmap::priv::raw_hash_set< + phmap::priv::NodeHashSetPolicy, Hash, Eq, Alloc> +{ + using Base = typename node_hash_set::raw_hash_set; + +public: + node_hash_set() {} +#ifdef __INTEL_COMPILER + using Base::raw_hash_set; +#else + using Base::Base; +#endif + using Base::begin; + using Base::cbegin; + using Base::cend; + using Base::end; + using Base::capacity; + using Base::empty; + using Base::max_size; + using Base::size; + using Base::clear; + using Base::erase; + using Base::insert; + using Base::emplace; + using Base::emplace_hint; + using Base::emplace_with_hash; + using Base::emplace_hint_with_hash; + using Base::extract; + using Base::merge; + using Base::swap; + using Base::rehash; + using Base::reserve; + using Base::contains; + using Base::count; + using Base::equal_range; + using Base::find; + using Base::bucket_count; + using Base::load_factor; + using Base::max_load_factor; + using Base::get_allocator; + using Base::hash_function; + using Base::hash; + using Base::key_eq; + typename Base::hasher hash_funct() { return this->hash_function(); } + void resize(typename Base::size_type hint) { this->rehash(hint); } +}; + +// ----------------------------------------------------------------------------- +// phmap::node_hash_map +// ----------------------------------------------------------------------------- +// +// An `phmap::node_hash_map` is an unordered associative container which +// has been optimized for both speed and memory footprint in most common use +// cases. Its interface is similar to that of `std::unordered_map` with +// the following notable differences: +// +// * Supports heterogeneous lookup, through `find()`, `operator[]()` and +// `insert()`, provided that the map is provided a compatible heterogeneous +// hashing function and equality operator. +// * Contains a `capacity()` member function indicating the number of element +// slots (open, deleted, and empty) within the hash map. +// * Returns `void` from the `erase(iterator)` overload. +// ----------------------------------------------------------------------------- +template // default values in phmap_fwd_decl.h +class node_hash_map + : public phmap::priv::raw_hash_map< + phmap::priv::NodeHashMapPolicy, Hash, Eq, + Alloc> +{ + using Base = typename node_hash_map::raw_hash_map; + +public: + node_hash_map() {} +#ifdef __INTEL_COMPILER + using Base::raw_hash_map; +#else + using Base::Base; +#endif + using Base::begin; + using Base::cbegin; + using Base::cend; + using Base::end; + using Base::capacity; + using Base::empty; + using Base::max_size; + using Base::size; + using Base::clear; + using Base::erase; + using Base::insert; + using Base::insert_or_assign; + using Base::emplace; + using Base::emplace_hint; + using Base::try_emplace; + using Base::extract; + using Base::merge; + using Base::swap; + using Base::rehash; + using Base::reserve; + using Base::at; + using Base::contains; + using Base::count; + using Base::equal_range; + using Base::find; + using Base::operator[]; + using Base::bucket_count; + using Base::load_factor; + using Base::max_load_factor; + using Base::get_allocator; + using Base::hash_function; + using Base::hash; + using Base::key_eq; + typename Base::hasher hash_funct() { return this->hash_function(); } + void resize(typename Base::size_type hint) { this->rehash(hint); } +}; + +// ----------------------------------------------------------------------------- +// phmap::parallel_flat_hash_set +// ----------------------------------------------------------------------------- +template // default values in phmap_fwd_decl.h +class parallel_flat_hash_set + : public phmap::priv::parallel_hash_set< + N, phmap::priv::raw_hash_set, Mtx_, + phmap::priv::FlatHashSetPolicy, + Hash, Eq, Alloc> +{ + using Base = typename parallel_flat_hash_set::parallel_hash_set; + +public: + parallel_flat_hash_set() {} +#ifdef __INTEL_COMPILER + using Base::parallel_hash_set; +#else + using Base::Base; +#endif + using Base::hash; + using Base::subidx; + using Base::subcnt; + using Base::begin; + using Base::cbegin; + using Base::cend; + using Base::end; + using Base::capacity; + using Base::empty; + using Base::max_size; + using Base::size; + using Base::clear; + using Base::erase; + using Base::insert; + using Base::emplace; + using Base::emplace_hint; + using Base::emplace_with_hash; + using Base::emplace_hint_with_hash; + using Base::extract; + using Base::merge; + using Base::swap; + using Base::rehash; + using Base::reserve; + using Base::contains; + using Base::count; + using Base::equal_range; + using Base::find; + using Base::bucket_count; + using Base::load_factor; + using Base::max_load_factor; + using Base::get_allocator; + using Base::hash_function; + using Base::key_eq; +}; + +// ----------------------------------------------------------------------------- +// phmap::parallel_flat_hash_map - default values in phmap_fwd_decl.h +// ----------------------------------------------------------------------------- +template +class parallel_flat_hash_map : public phmap::priv::parallel_hash_map< + N, phmap::priv::raw_hash_set, Mtx_, + phmap::priv::FlatHashMapPolicy, + Hash, Eq, Alloc> +{ + using Base = typename parallel_flat_hash_map::parallel_hash_map; + +public: + parallel_flat_hash_map() {} +#ifdef __INTEL_COMPILER + using Base::parallel_hash_map; +#else + using Base::Base; +#endif + using Base::hash; + using Base::subidx; + using Base::subcnt; + using Base::begin; + using Base::cbegin; + using Base::cend; + using Base::end; + using Base::capacity; + using Base::empty; + using Base::max_size; + using Base::size; + using Base::clear; + using Base::erase; + using Base::insert; + using Base::insert_or_assign; + using Base::emplace; + using Base::emplace_hint; + using Base::try_emplace; + using Base::emplace_with_hash; + using Base::emplace_hint_with_hash; + using Base::try_emplace_with_hash; + using Base::extract; + using Base::merge; + using Base::swap; + using Base::rehash; + using Base::reserve; + using Base::at; + using Base::contains; + using Base::count; + using Base::equal_range; + using Base::find; + using Base::operator[]; + using Base::bucket_count; + using Base::load_factor; + using Base::max_load_factor; + using Base::get_allocator; + using Base::hash_function; + using Base::key_eq; +}; + +// ----------------------------------------------------------------------------- +// phmap::parallel_node_hash_set +// ----------------------------------------------------------------------------- +template +class parallel_node_hash_set + : public phmap::priv::parallel_hash_set< + N, phmap::priv::raw_hash_set, Mtx_, + phmap::priv::NodeHashSetPolicy, Hash, Eq, Alloc> +{ + using Base = typename parallel_node_hash_set::parallel_hash_set; + +public: + parallel_node_hash_set() {} +#ifdef __INTEL_COMPILER + using Base::parallel_hash_set; +#else + using Base::Base; +#endif + using Base::hash; + using Base::subidx; + using Base::subcnt; + using Base::begin; + using Base::cbegin; + using Base::cend; + using Base::end; + using Base::capacity; + using Base::empty; + using Base::max_size; + using Base::size; + using Base::clear; + using Base::erase; + using Base::insert; + using Base::emplace; + using Base::emplace_hint; + using Base::emplace_with_hash; + using Base::emplace_hint_with_hash; + using Base::extract; + using Base::merge; + using Base::swap; + using Base::rehash; + using Base::reserve; + using Base::contains; + using Base::count; + using Base::equal_range; + using Base::find; + using Base::bucket_count; + using Base::load_factor; + using Base::max_load_factor; + using Base::get_allocator; + using Base::hash_function; + using Base::key_eq; + typename Base::hasher hash_funct() { return this->hash_function(); } + void resize(typename Base::size_type hint) { this->rehash(hint); } +}; + +// ----------------------------------------------------------------------------- +// phmap::parallel_node_hash_map +// ----------------------------------------------------------------------------- +template +class parallel_node_hash_map + : public phmap::priv::parallel_hash_map< + N, phmap::priv::raw_hash_set, Mtx_, + phmap::priv::NodeHashMapPolicy, Hash, Eq, + Alloc> +{ + using Base = typename parallel_node_hash_map::parallel_hash_map; + +public: + parallel_node_hash_map() {} +#ifdef __INTEL_COMPILER + using Base::parallel_hash_map; +#else + using Base::Base; +#endif + using Base::hash; + using Base::subidx; + using Base::subcnt; + using Base::begin; + using Base::cbegin; + using Base::cend; + using Base::end; + using Base::capacity; + using Base::empty; + using Base::max_size; + using Base::size; + using Base::clear; + using Base::erase; + using Base::insert; + using Base::insert_or_assign; + using Base::emplace; + using Base::emplace_hint; + using Base::try_emplace; + using Base::emplace_with_hash; + using Base::emplace_hint_with_hash; + using Base::try_emplace_with_hash; + using Base::extract; + using Base::merge; + using Base::swap; + using Base::rehash; + using Base::reserve; + using Base::at; + using Base::contains; + using Base::count; + using Base::equal_range; + using Base::find; + using Base::operator[]; + using Base::bucket_count; + using Base::load_factor; + using Base::max_load_factor; + using Base::get_allocator; + using Base::hash_function; + using Base::key_eq; + typename Base::hasher hash_funct() { return this->hash_function(); } + void resize(typename Base::size_type hint) { this->rehash(hint); } +}; + +} // namespace phmap + + +namespace phmap { + namespace priv { + template + std::size_t erase_if(C &c, Pred pred) { + auto old_size = c.size(); + for (auto i = c.begin(), last = c.end(); i != last; ) { + if (pred(*i)) { + i = c.erase(i); + } else { + ++i; + } + } + return old_size - c.size(); + } + } // priv + + // ======== erase_if for phmap set containers ================================== + template + std::size_t erase_if(phmap::flat_hash_set& c, Pred pred) { + return phmap::priv::erase_if(c, std::move(pred)); + } + + template + std::size_t erase_if(phmap::node_hash_set& c, Pred pred) { + return phmap::priv::erase_if(c, std::move(pred)); + } + + template + std::size_t erase_if(phmap::parallel_flat_hash_set& c, Pred pred) { + return phmap::priv::erase_if(c, std::move(pred)); + } + + template + std::size_t erase_if(phmap::parallel_node_hash_set& c, Pred pred) { + return phmap::priv::erase_if(c, std::move(pred)); + } + + // ======== erase_if for phmap map containers ================================== + template + std::size_t erase_if(phmap::flat_hash_map& c, Pred pred) { + return phmap::priv::erase_if(c, std::move(pred)); + } + + template + std::size_t erase_if(phmap::node_hash_map& c, Pred pred) { + return phmap::priv::erase_if(c, std::move(pred)); + } + + template + std::size_t erase_if(phmap::parallel_flat_hash_map& c, Pred pred) { + return phmap::priv::erase_if(c, std::move(pred)); + } + + template + std::size_t erase_if(phmap::parallel_node_hash_map& c, Pred pred) { + return phmap::priv::erase_if(c, std::move(pred)); + } + +} // phmap + +#ifdef _MSC_VER + #pragma warning(pop) +#endif + + +#endif // phmap_h_guard_ diff --git a/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/parallel_hashmap/phmap_base.h b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/parallel_hashmap/phmap_base.h new file mode 100644 index 00000000..0cb89310 --- /dev/null +++ b/liberty/lib/SLAMP/SLAMPcustom/consumer/ProfilingModules/parallel_hashmap/phmap_base.h @@ -0,0 +1,5157 @@ +#if !defined(phmap_base_h_guard_) +#define phmap_base_h_guard_ + +// --------------------------------------------------------------------------- +// Copyright (c) 2019, Gregory Popovitch - greg7mdp@gmail.com +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Includes work from abseil-cpp (https://github.com/abseil/abseil-cpp) +// with modifications. +// +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// --------------------------------------------------------------------------- + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include // for std::lock + +#include "phmap_config.h" + +#ifdef PHMAP_HAVE_SHARED_MUTEX + #include // after "phmap_config.h" +#endif + +#ifdef _MSC_VER + #pragma warning(push) + #pragma warning(disable : 4514) // unreferenced inline function has been removed + #pragma warning(disable : 4582) // constructor is not implicitly called + #pragma warning(disable : 4625) // copy constructor was implicitly defined as deleted + #pragma warning(disable : 4626) // assignment operator was implicitly defined as deleted + #pragma warning(disable : 4710) // function not inlined + #pragma warning(disable : 4711) // selected for automatic inline expansion + #pragma warning(disable : 4820) // '6' bytes padding added after data member +#endif // _MSC_VER + +namespace phmap { + +template using Allocator = typename std::allocator; + +template using Pair = typename std::pair; + +template +struct EqualTo +{ + inline bool operator()(const T& a, const T& b) const + { + return std::equal_to()(a, b); + } +}; + +template +struct Less +{ + inline bool operator()(const T& a, const T& b) const + { + return std::less()(a, b); + } +}; + +namespace type_traits_internal { + +template +struct VoidTImpl { + using type = void; +}; + +// NOTE: The `is_detected` family of templates here differ from the library +// fundamentals specification in that for library fundamentals, `Op` is +// evaluated as soon as the type `is_detected` undergoes +// substitution, regardless of whether or not the `::value` is accessed. That +// is inconsistent with all other standard traits and prevents lazy evaluation +// in larger contexts (such as if the `is_detected` check is a trailing argument +// of a `conjunction`. This implementation opts to instead be lazy in the same +// way that the standard traits are (this "defect" of the detection idiom +// specifications has been reported). +// --------------------------------------------------------------------------- + +template class Op, class... Args> +struct is_detected_impl { + using type = std::false_type; +}; + +template