From a78ee8ca5e26dbfe77e5a994b8d8f4765bf1b151 Mon Sep 17 00:00:00 2001 From: demotomohiro Date: Mon, 10 Nov 2025 20:04:15 +0900 Subject: [PATCH 01/19] implements #25268; cache configuration evaluation --- compiler/commands.nim | 2 + compiler/nimconf.nim | 9 ++- compiler/nimconfcache.nim | 157 ++++++++++++++++++++++++++++++++++++++ compiler/options.nim | 1 + 4 files changed, 167 insertions(+), 2 deletions(-) create mode 100644 compiler/nimconfcache.nim diff --git a/compiler/commands.nim b/compiler/commands.nim index 415fe6b352f7..cda02a8a59fa 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -1009,6 +1009,8 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; processOnOffSwitchG(conf, {optSkipUserConfigFile}, arg, pass, info) of "skipparentcfg": processOnOffSwitchG(conf, {optSkipParentConfigFiles}, arg, pass, info) + of "cachecfg": + processOnOffSwitchG(conf, {optCacheConfig}, arg, pass, info) of "genscript", "gendeps": if switch.normalize == "gendeps": deprecatedAlias(switch, "genscript") processOnOffSwitchG(conf, {optGenScript}, arg, pass, info) diff --git a/compiler/nimconf.nim b/compiler/nimconf.nim index 5d31dea57f7e..16556f43585f 100644 --- a/compiler/nimconf.nim +++ b/compiler/nimconf.nim @@ -11,7 +11,7 @@ import llstream, commands, msgs, lexer, ast, - options, idents, wordrecg, lineinfos, pathutils, scriptconfig + options, idents, wordrecg, lineinfos, pathutils, scriptconfig, nimconfcache import std/[os, strutils, strtabs] @@ -244,7 +244,7 @@ proc getSystemConfigPath*(conf: ConfigRef; filename: RelativeFile): AbsoluteFile if not fileExists(result): result = p / RelativeDir"etc/nim" / filename if not fileExists(result): result = AbsoluteDir"/etc/nim" / filename -proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef; idgen: IdGenerator) = +proc loadConfigsImpl(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef; idgen: IdGenerator) = setDefaultLibpath(conf) template readConfigFile(path) = let configPath = path @@ -319,3 +319,8 @@ proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef; idgen: # `nim check foo.nims' means to check the syntax of the NimScript file discard showHintConf() + +proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef; idgen: IdGenerator) = + if optForceFullMake in conf.globalOptions or not loadConfigsFromCache(conf): + loadConfigsImpl(cfg, cache, conf, idgen) + storeConfigs(conf) diff --git a/compiler/nimconfcache.nim b/compiler/nimconfcache.nim new file mode 100644 index 000000000000..8929df7150b9 --- /dev/null +++ b/compiler/nimconfcache.nim @@ -0,0 +1,157 @@ +import options, pathutils, condsyms +import std/[assertions, os, times] +from std/sequtils import addUnique +from std/strutils import parseEnum +import "../dist/nimony/src/lib" / [bitabs, lineinfos, nifreader, nifstreams, nifcursors] + +proc toNifPath(p: AbsoluteDir|AbsoluteFile): string = + result = relativePath(p.string, getCurrentDir(), '/') + if p.string.len <= result.len: + result = p.string + +proc configToNif(conf: ConfigRef; dest: var TokenBuf) = + dest.addStrLit conf.commandLine + dest.buildTree pool.tags.getOrIncl("sources"), NoLineInfo: + for f in conf.m.fileInfos: + dest.addStrLit f.fullPath.toNifPath + + #echo "conf.options" + #echo conf.options + dest.buildTree pool.tags.getOrIncl("options"), NoLineInfo: + for opt in conf.options: + dest.addStrLit $opt + + #echo "conf.globalOptions" + #echo conf.globalOptions + dest.buildTree pool.tags.getOrIncl("globalOptions"), NoLineInfo: + for opt in conf.globalOptions: + dest.addStrLit $opt + + dest.buildTree pool.tags.getOrIncl("defines"), NoLineInfo: + for def in definedSymbolNames(conf.symbols): + dest.addStrLit def + + #echo "conf.nimblePaths" + #echo conf.nimblePaths + dest.buildTree pool.tags.getOrIncl("nimblepaths"), NoLineInfo: + for p in conf.nimblePaths: + dest.addStrLit p.toNifPath + + #echo "conf.searchPaths" + #echo conf.searchPaths + var paths: seq[string] = @[] + for p in conf.searchPaths: + paths.addUnique p.toNifPath + dest.buildTree pool.tags.getOrIncl("paths"), NoLineInfo: + for p in paths: + dest.addStrLit p + +proc cfgCachePath(conf: ConfigRef): (AbsoluteDir, RelativeFile) = + (conf.projectPath / RelativeDir"nimcache", RelativeFile"cache.cfg.nif") + +proc sourcesChanged(conf: ConfigRef; n: var Cursor; modTime: Time): bool = + assert pool.tags[n.tagId] == "config" + inc n + assert n.kind == StringLit + let cmdline = pool.strings[n.litId] + inc n + if cmdline != conf.commandLine: + #echo "commandLine is changed, dont use cache" + result = true + else: + assert pool.tags[n.tagId] == "sources" + inc n + while n.kind != ParRi: + assert n.kind == StringLit + let dep = pool.strings[n.litId] + inc n + if not fileExists(dep): + #echo dep, " is removed" + return true + elif getLastModificationTime(dep) >= modTime: + #echo dep, " is changed" + return true + inc n + #echo "commandLine is not changed" + result = false + +proc loadConfigsFromNif(config: ConfigRef; n: var Cursor) = + assert pool.tags[n.tagId] == "options" + inc n + while n.kind != ParRi: + assert n.kind == StringLit + let opt = pool.strings[n.litId] + inc n + config.options.incl parseEnum[TOption](opt) + inc n + #echo config.options + + assert pool.tags[n.tagId] == "globalOptions" + inc n + while n.kind != ParRi: + assert n.kind == StringLit + let opt = pool.strings[n.litId] + inc n + config.globalOptions.incl parseEnum[TGlobalOption](opt) + inc n + #echo config.globalOptions + + assert pool.tags[n.tagId] == "defines" + inc n + while n.kind != ParRi: + assert n.kind == StringLit + let def = pool.strings[n.litId] + inc n + config.symbols.defineSymbol(def) + inc n + + assert pool.tags[n.tagId] == "nimblepaths" + inc n + while n.kind != ParRi: + assert n.kind == StringLit + let p = pool.strings[n.litId] + inc n + config.nimblePaths.add p.toAbsoluteDir + inc n + #echo "config.nimblePaths" + #echo config.nimblePaths + + assert pool.tags[n.tagId] == "paths" + inc n + while n.kind != ParRi: + assert n.kind == StringLit + let p = pool.strings[n.litId] + inc n + config.searchPaths.add p.toAbsoluteDir + inc n + #echo "config.searchPaths" + #echo config.searchPaths + +proc loadConfigsFromCache*(conf: ConfigRef): bool = + if optCacheConfig in conf.globalOptions: + let (dir, file) = conf.cfgCachePath() + let path = dir / file + if fileExists(path): + let modTime = getLastModificationTime(path.string) + var stream = nifstreams.open(path.string) + discard processDirectives(stream.r) + var buf = fromStream(stream) + var n = beginRead(buf) + result = not sourcesChanged(conf, n, modTime) + if result: + loadConfigsFromNif(conf, n) + endRead(buf) + stream.close + else: + result = false + else: + result = false + +proc storeConfigs*(conf: ConfigRef) = + if optCacheConfig in conf.globalOptions: + let (dir, file) = conf.cfgCachePath() + createDir(dir) + var dest = createTokenBuf() + dest.buildTree pool.tags.getOrIncl("config"), NoLineInfo: + configToNif(conf, dest) + writeFile dir / file, "(.nif24)\n" & toString(dest) diff --git a/compiler/options.nim b/compiler/options.nim index 142080cc8f5c..4bceb25534ce 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -73,6 +73,7 @@ type # please make sure we have under 32 options optSkipProjConfigFile, # skip the project's cfg/nims config file optSkipUserConfigFile, # skip the users's cfg/nims config file optSkipParentConfigFiles, # skip parent dir's cfg/nims config files + optCacheConfig, # cache the results of configuration file evaluation optNoMain, # do not generate a "main" proc optUseColors, # use colors for hints, warnings, and errors optThreads, # support for multi-threading From 2a502d0eb0eb6bc9b81a0ad35c015dbfd7691c53 Mon Sep 17 00:00:00 2001 From: demotomohiro Date: Tue, 11 Nov 2025 20:07:58 +0900 Subject: [PATCH 02/19] don't use cached config when new config file is created --- compiler/nimconf.nim | 74 ++++++++++++++++++++++++++++++--------- compiler/nimconfcache.nim | 64 +++++++++++++++++++-------------- 2 files changed, 96 insertions(+), 42 deletions(-) diff --git a/compiler/nimconf.nim b/compiler/nimconf.nim index 16556f43585f..de79bbc934d0 100644 --- a/compiler/nimconf.nim +++ b/compiler/nimconf.nim @@ -13,7 +13,7 @@ import llstream, commands, msgs, lexer, ast, options, idents, wordrecg, lineinfos, pathutils, scriptconfig, nimconfcache -import std/[os, strutils, strtabs] +import std/[os, strutils, strtabs, sets, times] when defined(nimPreviewSlimSystem): import std/syncio @@ -244,25 +244,39 @@ proc getSystemConfigPath*(conf: ConfigRef; filename: RelativeFile): AbsoluteFile if not fileExists(result): result = p / RelativeDir"etc/nim" / filename if not fileExists(result): result = AbsoluteDir"/etc/nim" / filename -proc loadConfigsImpl(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef; idgen: IdGenerator) = +proc loadConfigsImpl(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef; idgen: IdGenerator; cachedCfgFiles: var HashSet[string]) = + # `cachedCfgFiles` is used to detect newly created config files. + # if cachedCfgFiles.len == 0, evaluate config files, + # if not, doesn't evaluate but find newly created config files. + # clears `cachedCfgFiles` if found. setDefaultLibpath(conf) template readConfigFile(path) = let configPath = path - conf.currentConfigDir = configPath.splitFile.dir.string - setConfigVar(conf, "selfDir", conf.currentConfigDir) - if readConfigFile(configPath, cache, conf): - conf.configFiles.add(configPath) + if cachedCfgFiles.len == 0: + conf.currentConfigDir = configPath.splitFile.dir.string + setConfigVar(conf, "selfDir", conf.currentConfigDir) + if readConfigFile(configPath, cache, conf): + conf.configFiles.add(configPath) + elif configPath.string notin cachedCfgFiles and configPath.fileExists: + #echo "config file was created ", configPath + cachedCfgFiles = HashSet[string]() + return template runNimScriptIfExists(path: AbsoluteFile, isMain = false) = let p = path # eval once - var s: PLLStream = nil - if isMain and optWasNimscript in conf.globalOptions: - if conf.projectIsStdin: s = stdin.llStreamOpen - elif conf.projectIsCmd: s = llStreamOpen(conf.cmdInput) - if s == nil and fileExists(p): s = llStreamOpen(p, fmRead) - if s != nil: - conf.configFiles.add(p) - runNimScript(cache, p, idgen, freshDefines = false, conf, s) + if cachedCfgFiles.len == 0: + var s: PLLStream = nil + if isMain and optWasNimscript in conf.globalOptions: + if conf.projectIsStdin: s = stdin.llStreamOpen + elif conf.projectIsCmd: s = llStreamOpen(conf.cmdInput) + if s == nil and fileExists(p): s = llStreamOpen(p, fmRead) + if s != nil: + conf.configFiles.add(p) + runNimScript(cache, p, idgen, freshDefines = false, conf, s) + elif p.string notin cachedCfgFiles and p.fileExists: + #echo "config file was created ", p + cachedCfgFiles = HashSet[string]() + return if optSkipSystemConfigFile notin conf.globalOptions: readConfigFile(getSystemConfigPath(conf, cfg)) @@ -320,7 +334,35 @@ proc loadConfigsImpl(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef; idge discard showHintConf() +proc canUseCache(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef; idgen: IdGenerator; cachedCfgFiles: var HashSet[string]): bool = + if optForceFullMake in conf.globalOptions or optCacheConfig notin conf.globalOptions: + result = false + elif conf.cmd == cmdNimscript: + # loadConfigsImpl runs Nim script + result = false + else: + # Don't use cached config when: + # - cache doesn't exists + # - command line parameter is changed (config file can depend it) + # - config file is changed, removed or created + cachedCfgFiles = sourceChanged(conf) + if cachedCfgFiles.len == 0: + result = false + else: + echo formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3), " checking added config" + loadConfigsImpl(cfg, cache, conf, idgen, cachedCfgFiles) + echo formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3), " done checking added config" + result = cachedCfgFiles.len != 0 + proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef; idgen: IdGenerator) = - if optForceFullMake in conf.globalOptions or not loadConfigsFromCache(conf): - loadConfigsImpl(cfg, cache, conf, idgen) + conf.lastCmdTime = epochTime() + var cachedCfgFiles = HashSet[string]() + if canUseCache(cfg, cache, conf, idgen, cachedCfgFiles): + echo formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3), " loading cached config" + loadConfigsFromCache(conf) + echo formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3), " done loading cached config" + else: + echo formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3), " evaluating config files" + loadConfigsImpl(cfg, cache, conf, idgen, cachedCfgFiles) + echo formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3), " done evaluating config files" storeConfigs(conf) diff --git a/compiler/nimconfcache.nim b/compiler/nimconfcache.nim index 8929df7150b9..4b5132401924 100644 --- a/compiler/nimconfcache.nim +++ b/compiler/nimconfcache.nim @@ -1,5 +1,5 @@ import options, pathutils, condsyms -import std/[assertions, os, times] +import std/[assertions, os, sets, times] from std/sequtils import addUnique from std/strutils import parseEnum import "../dist/nimony/src/lib" / [bitabs, lineinfos, nifreader, nifstreams, nifcursors] @@ -49,7 +49,8 @@ proc configToNif(conf: ConfigRef; dest: var TokenBuf) = proc cfgCachePath(conf: ConfigRef): (AbsoluteDir, RelativeFile) = (conf.projectPath / RelativeDir"nimcache", RelativeFile"cache.cfg.nif") -proc sourcesChanged(conf: ConfigRef; n: var Cursor; modTime: Time): bool = +proc sourcesChanged(conf: ConfigRef; n: var Cursor; modTime: Time): HashSet[string] = + result = initHashSet[string](8) assert pool.tags[n.tagId] == "config" inc n assert n.kind == StringLit @@ -57,7 +58,7 @@ proc sourcesChanged(conf: ConfigRef; n: var Cursor; modTime: Time): bool = inc n if cmdline != conf.commandLine: #echo "commandLine is changed, dont use cache" - result = true + discard else: assert pool.tags[n.tagId] == "sources" inc n @@ -67,13 +68,13 @@ proc sourcesChanged(conf: ConfigRef; n: var Cursor; modTime: Time): bool = inc n if not fileExists(dep): #echo dep, " is removed" - return true + return default(typeof(result)) elif getLastModificationTime(dep) >= modTime: #echo dep, " is changed" - return true + return default(typeof(result)) + else: + result.incl dep.toAbsolute(getCurrentDir().AbsoluteDir).string inc n - #echo "commandLine is not changed" - result = false proc loadConfigsFromNif(config: ConfigRef; n: var Cursor) = assert pool.tags[n.tagId] == "options" @@ -127,25 +128,36 @@ proc loadConfigsFromNif(config: ConfigRef; n: var Cursor) = #echo "config.searchPaths" #echo config.searchPaths -proc loadConfigsFromCache*(conf: ConfigRef): bool = - if optCacheConfig in conf.globalOptions: - let (dir, file) = conf.cfgCachePath() - let path = dir / file - if fileExists(path): - let modTime = getLastModificationTime(path.string) - var stream = nifstreams.open(path.string) - discard processDirectives(stream.r) - var buf = fromStream(stream) - var n = beginRead(buf) - result = not sourcesChanged(conf, n, modTime) - if result: - loadConfigsFromNif(conf, n) - endRead(buf) - stream.close - else: - result = false - else: - result = false +proc sourceChanged*(conf: ConfigRef): HashSet[string] = + result = HashSet[string]() + let (dir, file) = conf.cfgCachePath() + let path = dir / file + if fileExists(path): + let modTime = getLastModificationTime(path.string) + var stream = nifstreams.open(path.string) + discard processDirectives(stream.r) + var buf = fromStream(stream) + var n = beginRead(buf) + result = sourcesChanged(conf, n, modTime) + endRead(buf) + stream.close + +proc loadConfigsFromCache*(conf: ConfigRef) = + let (dir, file) = conf.cfgCachePath() + let path = dir / file + var stream = nifstreams.open(path.string) + discard processDirectives(stream.r) + var buf = fromStream(stream) + var n = beginRead(buf) + inc n # skip config + inc n # skip cmdline + inc n # skip sources + while n.kind != ParRi: + inc n + inc n + loadConfigsFromNif(conf, n) + endRead(buf) + stream.close proc storeConfigs*(conf: ConfigRef) = if optCacheConfig in conf.globalOptions: From e09844c65840affcf89659c08d6bcfe493eabb64 Mon Sep 17 00:00:00 2001 From: demotomohiro Date: Wed, 12 Nov 2025 04:39:21 +0900 Subject: [PATCH 03/19] store/load ConfigRef fields --- compiler/nimconfcache.nim | 70 ++++++++++++++++++++++++++++++++------- compiler/options.nim | 3 ++ 2 files changed, 61 insertions(+), 12 deletions(-) diff --git a/compiler/nimconfcache.nim b/compiler/nimconfcache.nim index 4b5132401924..e161d6340d18 100644 --- a/compiler/nimconfcache.nim +++ b/compiler/nimconfcache.nim @@ -10,11 +10,14 @@ proc toNifPath(p: AbsoluteDir|AbsoluteFile): string = result = p.string proc configToNif(conf: ConfigRef; dest: var TokenBuf) = + # store data used to decide whether to use cache or eval config files dest.addStrLit conf.commandLine dest.buildTree pool.tags.getOrIncl("sources"), NoLineInfo: for f in conf.m.fileInfos: dest.addStrLit f.fullPath.toNifPath + # store fields that can be changed on config files + # see processSwitch proc in commands.nim #echo "conf.options" #echo conf.options dest.buildTree pool.tags.getOrIncl("options"), NoLineInfo: @@ -27,6 +30,30 @@ proc configToNif(conf: ConfigRef; dest: var TokenBuf) = for opt in conf.globalOptions: dest.addStrLit $opt + #echo "conf.selectedGC" + #echo conf.selectedGC + dest.addStrLit $conf.selectedGC + + #echo "conf.exc" + #echo conf.exc + dest.addStrLit $conf.exc + + #echo "conf.verbosity" + #echo conf.verbosity + dest.addIntLit conf.verbosity + + #echo "conf.numberOfProcessors" + #echo conf.numberOfProcessors + dest.addIntLit conf.numberOfProcessors + + #echo "conf.spellSuggestMax" + #echo conf.spellSuggestMax + dest.addIntLit conf.spellSuggestMax + + #echo "conf.nimbasePattern" + #echo conf.nimbasePattern + dest.addStrLit conf.nimbasePattern + dest.buildTree pool.tags.getOrIncl("defines"), NoLineInfo: for def in definedSymbolNames(conf.symbols): dest.addStrLit def @@ -76,16 +103,16 @@ proc sourcesChanged(conf: ConfigRef; n: var Cursor; modTime: Time): HashSet[stri result.incl dep.toAbsolute(getCurrentDir().AbsoluteDir).string inc n -proc loadConfigsFromNif(config: ConfigRef; n: var Cursor) = +proc loadConfigsFromNif(conf: ConfigRef; n: var Cursor) = assert pool.tags[n.tagId] == "options" inc n while n.kind != ParRi: assert n.kind == StringLit let opt = pool.strings[n.litId] inc n - config.options.incl parseEnum[TOption](opt) + conf.options.incl parseEnum[TOption](opt) inc n - #echo config.options + #echo conf.options assert pool.tags[n.tagId] == "globalOptions" inc n @@ -93,9 +120,28 @@ proc loadConfigsFromNif(config: ConfigRef; n: var Cursor) = assert n.kind == StringLit let opt = pool.strings[n.litId] inc n - config.globalOptions.incl parseEnum[TGlobalOption](opt) + conf.globalOptions.incl parseEnum[TGlobalOption](opt) inc n - #echo config.globalOptions + #echo conf.globalOptions + + conf.selectedGC = parseEnum[TGCMode](pool.strings[n.litId]) + inc n + #echo conf.selectedGC + conf.exc = parseEnum[ExceptionSystem](pool.strings[n.litId]) + inc n + #echo conf.exc + conf.verbosity = pool.integers[n.intId] + inc n + #echo conf.verbosity + conf.numberOfProcessors = pool.integers[n.intId] + inc n + #echo conf.numberOfProcessors + conf.spellSuggestMax = pool.integers[n.intId] + inc n + #echo conf.spellSuggestMax + conf.nimbasePattern = pool.strings[n.litId] + inc n + #echo conf.nimbasePattern assert pool.tags[n.tagId] == "defines" inc n @@ -103,7 +149,7 @@ proc loadConfigsFromNif(config: ConfigRef; n: var Cursor) = assert n.kind == StringLit let def = pool.strings[n.litId] inc n - config.symbols.defineSymbol(def) + conf.symbols.defineSymbol(def) inc n assert pool.tags[n.tagId] == "nimblepaths" @@ -112,10 +158,10 @@ proc loadConfigsFromNif(config: ConfigRef; n: var Cursor) = assert n.kind == StringLit let p = pool.strings[n.litId] inc n - config.nimblePaths.add p.toAbsoluteDir + conf.nimblePaths.add p.toAbsoluteDir inc n - #echo "config.nimblePaths" - #echo config.nimblePaths + #echo "conf.nimblePaths" + #echo conf.nimblePaths assert pool.tags[n.tagId] == "paths" inc n @@ -123,10 +169,10 @@ proc loadConfigsFromNif(config: ConfigRef; n: var Cursor) = assert n.kind == StringLit let p = pool.strings[n.litId] inc n - config.searchPaths.add p.toAbsoluteDir + conf.searchPaths.add p.toAbsoluteDir inc n - #echo "config.searchPaths" - #echo config.searchPaths + #echo "conf.searchPaths" + #echo conf.searchPaths proc sourceChanged*(conf: ConfigRef): HashSet[string] = result = HashSet[string]() diff --git a/compiler/options.nim b/compiler/options.nim index 4bceb25534ce..a1163829221d 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -343,6 +343,9 @@ type ## fields marked with '*' are subject to ## the incremental compilation mechanisms ## (+) means "part of the dependency" + ## + ## when adds a field that can be changed on + ## config files, store/load it on nimconfcache.nim backend*: TBackend # set via `nim x` or `nim --backend:x` target*: Target # (+) linesCompiled*: int # all lines that have been compiled From 4e6e72c27d6c611226d49cb15eba3872d9e68f3f Mon Sep 17 00:00:00 2001 From: demotomohiro Date: Wed, 12 Nov 2025 19:55:13 +0900 Subject: [PATCH 04/19] adds test and fixes bug --- compiler/nimconfcache.nim | 76 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/compiler/nimconfcache.nim b/compiler/nimconfcache.nim index e161d6340d18..9abe0165f2b1 100644 --- a/compiler/nimconfcache.nim +++ b/compiler/nimconfcache.nim @@ -1,5 +1,5 @@ import options, pathutils, condsyms -import std/[assertions, os, sets, times] +import std/[assertions, os, sets, strtabs, times] from std/sequtils import addUnique from std/strutils import parseEnum import "../dist/nimony/src/lib" / [bitabs, lineinfos, nifreader, nifstreams, nifcursors] @@ -104,6 +104,9 @@ proc sourcesChanged(conf: ConfigRef; n: var Cursor; modTime: Time): HashSet[stri inc n proc loadConfigsFromNif(conf: ConfigRef; n: var Cursor) = + # clear fields so that it has the same value as it was stored to the cache. + # some switches turn off options that were turned on when conf was initialized. + conf.options = {} assert pool.tags[n.tagId] == "options" inc n while n.kind != ParRi: @@ -114,6 +117,7 @@ proc loadConfigsFromNif(conf: ConfigRef; n: var Cursor) = inc n #echo conf.options + conf.globalOptions = {} assert pool.tags[n.tagId] == "globalOptions" inc n while n.kind != ParRi: @@ -143,6 +147,7 @@ proc loadConfigsFromNif(conf: ConfigRef; n: var Cursor) = inc n #echo conf.nimbasePattern + conf.symbols.clear assert pool.tags[n.tagId] == "defines" inc n while n.kind != ParRi: @@ -152,6 +157,7 @@ proc loadConfigsFromNif(conf: ConfigRef; n: var Cursor) = conf.symbols.defineSymbol(def) inc n + conf.nimblePaths.setLen(0) assert pool.tags[n.tagId] == "nimblepaths" inc n while n.kind != ParRi: @@ -163,6 +169,7 @@ proc loadConfigsFromNif(conf: ConfigRef; n: var Cursor) = #echo "conf.nimblePaths" #echo conf.nimblePaths + conf.searchPaths.setLen(0) assert pool.tags[n.tagId] == "paths" inc n while n.kind != ParRi: @@ -213,3 +220,70 @@ proc storeConfigs*(conf: ConfigRef) = dest.buildTree pool.tags.getOrIncl("config"), NoLineInfo: configToNif(conf, dest) writeFile dir / file, "(.nif24)\n" & toString(dest) + +when isMainModule: + proc `==`(x, y: StringTableRef): bool = + for k, v in x: + if k notin y: + return false + + return true + + proc assertEq(x, y: ConfigRef) = + template assertImpl(f: untyped) = + assert x.f == y.f, $x.f & " / " & $y.f + + assertImpl options + assertImpl globalOptions + assertImpl selectedGC + assertImpl exc + assertImpl verbosity + assertImpl numberOfProcessors + assertImpl spellSuggestMax + assertImpl nimbasePattern + assertImpl symbols + assertImpl nimblePaths + assertImpl searchPaths + + proc testConfig(conf1: ConfigRef) = + var dest = createTokenBuf() + dest.buildTree pool.tags.getOrIncl("config"), NoLineInfo: + configToNif(conf1, dest) + let cacheContent = "(.nif24)\n" & toString(dest) + # writeFile "cachecfg.nif", cacheContent + + var stream = nifstreams.openFromBuffer(cacheContent) + discard processDirectives(stream.r) + var buf = fromStream(stream) + var n = beginRead(buf) + inc n # skip config + inc n # skip cmdline + inc n # skip sources + while n.kind != ParRi: + inc n + inc n + var conf2 = newConfigRef() + loadConfigsFromNif(conf2, n) + endRead(buf) + + assertEq(conf1, conf2) + + block: + var conf = newConfigRef() + testConfig conf + + block: + var conf = newConfigRef() + conf.options = {optObjCheck, optFieldCheck} + conf.globalOptions = {gloptNone, optRun} + conf.selectedGC = gcArc + conf.exc = excSetjmp + conf.verbosity = 3 + conf.numberOfProcessors = 123 + conf.spellSuggestMax = 456 + conf.nimbasePattern = "foo/nimbase.h" + conf.symbols.initDefines() + conf.symbols.defineSymbol("test") + conf.nimblePaths = @[AbsoluteDir"/foo", AbsoluteDir"/lib/nimble"] + conf.searchPaths = @[AbsoluteDir"/lib", AbsoluteDir"/user/lib"] + testConfig conf From a83ae2eb60651103c0f0ee51ce7a90671e31957b Mon Sep 17 00:00:00 2001 From: demotomohiro Date: Wed, 12 Nov 2025 23:57:14 +0900 Subject: [PATCH 05/19] store/load ConfigRef fields --- compiler/nimconfcache.nim | 52 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/compiler/nimconfcache.nim b/compiler/nimconfcache.nim index 9abe0165f2b1..5f05ec826fc2 100644 --- a/compiler/nimconfcache.nim +++ b/compiler/nimconfcache.nim @@ -30,6 +30,17 @@ proc configToNif(conf: ConfigRef; dest: var TokenBuf) = for opt in conf.globalOptions: dest.addStrLit $opt + dest.buildTree pool.tags.getOrIncl("macrosToExpand"), NoLineInfo: + for m in conf.macrosToExpand.keys: + dest.addStrLit m + + dest.buildTree pool.tags.getOrIncl("arcToExpand"), NoLineInfo: + for a in conf.arcToExpand.keys: + dest.addStrLit a + + dest.addStrLit $conf.filenameOption + dest.addStrLit conf.unitSep + #echo "conf.selectedGC" #echo conf.selectedGC dest.addStrLit $conf.selectedGC @@ -38,6 +49,8 @@ proc configToNif(conf: ConfigRef; dest: var TokenBuf) = #echo conf.exc dest.addStrLit $conf.exc + dest.addIntLit conf.hintProcessingDots.int + #echo "conf.verbosity" #echo conf.verbosity dest.addIntLit conf.verbosity @@ -128,12 +141,39 @@ proc loadConfigsFromNif(conf: ConfigRef; n: var Cursor) = inc n #echo conf.globalOptions + conf.macrosToExpand.clear + assert pool.tags[n.tagId] == "macrosToExpand" + inc n + while n.kind != ParRi: + assert n.kind == StringLit + let m = pool.strings[n.litId] + inc n + conf.macrosToExpand[m] = "T" + inc n + + conf.arcToExpand.clear + assert pool.tags[n.tagId] == "arcToExpand" + inc n + while n.kind != ParRi: + assert n.kind == StringLit + let m = pool.strings[n.litId] + inc n + conf.arcToExpand[m] = "T" + inc n + + conf.filenameOption = parseEnum[FilenameOption](pool.strings[n.litId]) + inc n + conf.unitSep = pool.strings[n.litId] + inc n + conf.selectedGC = parseEnum[TGCMode](pool.strings[n.litId]) inc n #echo conf.selectedGC conf.exc = parseEnum[ExceptionSystem](pool.strings[n.litId]) inc n #echo conf.exc + conf.hintProcessingDots = pool.integers[n.intId].bool + inc n conf.verbosity = pool.integers[n.intId] inc n #echo conf.verbosity @@ -235,8 +275,13 @@ when isMainModule: assertImpl options assertImpl globalOptions + assertImpl macrosToExpand + assertImpl arcToExpand + assertImpl filenameOption + assertImpl unitSep assertImpl selectedGC assertImpl exc + assertImpl hintProcessingDots assertImpl verbosity assertImpl numberOfProcessors assertImpl spellSuggestMax @@ -276,8 +321,15 @@ when isMainModule: var conf = newConfigRef() conf.options = {optObjCheck, optFieldCheck} conf.globalOptions = {gloptNone, optRun} + conf.macrosToExpand["foomacro"] = "T" + conf.macrosToExpand["barMacro"] = "T" + conf.arcToExpand["fooarc"] = "T" + conf.arcToExpand["barArc"] = "T" + conf.filenameOption = foCanonical + conf.unitSep = "\32" conf.selectedGC = gcArc conf.exc = excSetjmp + conf.hintProcessingDots = false conf.verbosity = 3 conf.numberOfProcessors = 123 conf.spellSuggestMax = 456 From 7137d84488d35819a553d8da0efdeb6a9ef83dfd Mon Sep 17 00:00:00 2001 From: demotomohiro Date: Thu, 13 Nov 2025 03:04:37 +0900 Subject: [PATCH 06/19] clean up code --- compiler/nimconfcache.nim | 60 +++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/compiler/nimconfcache.nim b/compiler/nimconfcache.nim index 5f05ec826fc2..0557d3fb7f2d 100644 --- a/compiler/nimconfcache.nim +++ b/compiler/nimconfcache.nim @@ -9,10 +9,27 @@ proc toNifPath(p: AbsoluteDir|AbsoluteFile): string = if p.string.len <= result.len: result = p.string +proc fromNif[T: enum](result: var T; n: var Cursor) = + result = parseEnum[T](pool.strings[n.litId]) + inc n + +proc fromNif[T: enum](result: var set[T]; n: var Cursor) = + # clear it so that it has the same value as it was stored to the cache. + # some switches turn off options that were turned on when conf was initialized. + result = {} + assert n.kind == StringLit + while n.kind != ParRi: + result.incl parseEnum[T](pool.strings[n.litId]) + inc n + inc n + +template buildTree(dest: var TokenBuf; tag: string; body: untyped): untyped = + buildTree(dest, pool.tags.getOrIncl(tag), NoLineInfo, body) + proc configToNif(conf: ConfigRef; dest: var TokenBuf) = # store data used to decide whether to use cache or eval config files dest.addStrLit conf.commandLine - dest.buildTree pool.tags.getOrIncl("sources"), NoLineInfo: + dest.buildTree "sources": for f in conf.m.fileInfos: dest.addStrLit f.fullPath.toNifPath @@ -20,21 +37,21 @@ proc configToNif(conf: ConfigRef; dest: var TokenBuf) = # see processSwitch proc in commands.nim #echo "conf.options" #echo conf.options - dest.buildTree pool.tags.getOrIncl("options"), NoLineInfo: + dest.buildTree "options": for opt in conf.options: dest.addStrLit $opt #echo "conf.globalOptions" #echo conf.globalOptions - dest.buildTree pool.tags.getOrIncl("globalOptions"), NoLineInfo: + dest.buildTree "globalOptions": for opt in conf.globalOptions: dest.addStrLit $opt - dest.buildTree pool.tags.getOrIncl("macrosToExpand"), NoLineInfo: + dest.buildTree "macrosToExpand": for m in conf.macrosToExpand.keys: dest.addStrLit m - dest.buildTree pool.tags.getOrIncl("arcToExpand"), NoLineInfo: + dest.buildTree "arcToExpand": for a in conf.arcToExpand.keys: dest.addStrLit a @@ -67,13 +84,13 @@ proc configToNif(conf: ConfigRef; dest: var TokenBuf) = #echo conf.nimbasePattern dest.addStrLit conf.nimbasePattern - dest.buildTree pool.tags.getOrIncl("defines"), NoLineInfo: + dest.buildTree "defines": for def in definedSymbolNames(conf.symbols): dest.addStrLit def #echo "conf.nimblePaths" #echo conf.nimblePaths - dest.buildTree pool.tags.getOrIncl("nimblepaths"), NoLineInfo: + dest.buildTree "nimblepaths": for p in conf.nimblePaths: dest.addStrLit p.toNifPath @@ -82,7 +99,7 @@ proc configToNif(conf: ConfigRef; dest: var TokenBuf) = var paths: seq[string] = @[] for p in conf.searchPaths: paths.addUnique p.toNifPath - dest.buildTree pool.tags.getOrIncl("paths"), NoLineInfo: + dest.buildTree "paths": for p in paths: dest.addStrLit p @@ -117,28 +134,14 @@ proc sourcesChanged(conf: ConfigRef; n: var Cursor; modTime: Time): HashSet[stri inc n proc loadConfigsFromNif(conf: ConfigRef; n: var Cursor) = - # clear fields so that it has the same value as it was stored to the cache. - # some switches turn off options that were turned on when conf was initialized. - conf.options = {} assert pool.tags[n.tagId] == "options" inc n - while n.kind != ParRi: - assert n.kind == StringLit - let opt = pool.strings[n.litId] - inc n - conf.options.incl parseEnum[TOption](opt) - inc n + fromNif(conf.options, n) #echo conf.options - conf.globalOptions = {} assert pool.tags[n.tagId] == "globalOptions" inc n - while n.kind != ParRi: - assert n.kind == StringLit - let opt = pool.strings[n.litId] - inc n - conf.globalOptions.incl parseEnum[TGlobalOption](opt) - inc n + fromNif(conf.globalOptions, n) #echo conf.globalOptions conf.macrosToExpand.clear @@ -161,16 +164,13 @@ proc loadConfigsFromNif(conf: ConfigRef; n: var Cursor) = conf.arcToExpand[m] = "T" inc n - conf.filenameOption = parseEnum[FilenameOption](pool.strings[n.litId]) - inc n + fromNif(conf.filenameOption, n) conf.unitSep = pool.strings[n.litId] inc n - conf.selectedGC = parseEnum[TGCMode](pool.strings[n.litId]) - inc n + fromNif(conf.selectedGC, n) #echo conf.selectedGC - conf.exc = parseEnum[ExceptionSystem](pool.strings[n.litId]) - inc n + fromNif(conf.exc, n) #echo conf.exc conf.hintProcessingDots = pool.integers[n.intId].bool inc n From dda137eaddeacd134ba993f477d3503e9c814c4e Mon Sep 17 00:00:00 2001 From: demotomohiro Date: Thu, 13 Nov 2025 04:47:58 +0900 Subject: [PATCH 07/19] clean up code --- compiler/nimconfcache.nim | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/compiler/nimconfcache.nim b/compiler/nimconfcache.nim index 0557d3fb7f2d..051b724cfb31 100644 --- a/compiler/nimconfcache.nim +++ b/compiler/nimconfcache.nim @@ -9,6 +9,9 @@ proc toNifPath(p: AbsoluteDir|AbsoluteFile): string = if p.string.len <= result.len: result = p.string +proc expectTag(n: Cursor; tag: string) = + assert pool.tags[n.tagId] == tag, "expected tag: " & tag & " but got: " & pool.tags[n.tagId] + proc fromNif[T: enum](result: var T; n: var Cursor) = result = parseEnum[T](pool.strings[n.litId]) inc n @@ -134,18 +137,18 @@ proc sourcesChanged(conf: ConfigRef; n: var Cursor; modTime: Time): HashSet[stri inc n proc loadConfigsFromNif(conf: ConfigRef; n: var Cursor) = - assert pool.tags[n.tagId] == "options" + expectTag n, "options" inc n fromNif(conf.options, n) #echo conf.options - assert pool.tags[n.tagId] == "globalOptions" + expectTag n, "globalOptions" inc n fromNif(conf.globalOptions, n) #echo conf.globalOptions conf.macrosToExpand.clear - assert pool.tags[n.tagId] == "macrosToExpand" + expectTag n, "macrosToExpand" inc n while n.kind != ParRi: assert n.kind == StringLit @@ -155,7 +158,7 @@ proc loadConfigsFromNif(conf: ConfigRef; n: var Cursor) = inc n conf.arcToExpand.clear - assert pool.tags[n.tagId] == "arcToExpand" + expectTag n, "arcToExpand" inc n while n.kind != ParRi: assert n.kind == StringLit @@ -188,7 +191,7 @@ proc loadConfigsFromNif(conf: ConfigRef; n: var Cursor) = #echo conf.nimbasePattern conf.symbols.clear - assert pool.tags[n.tagId] == "defines" + expectTag n, "defines" inc n while n.kind != ParRi: assert n.kind == StringLit @@ -198,7 +201,7 @@ proc loadConfigsFromNif(conf: ConfigRef; n: var Cursor) = inc n conf.nimblePaths.setLen(0) - assert pool.tags[n.tagId] == "nimblepaths" + expectTag n, "nimblepaths" inc n while n.kind != ParRi: assert n.kind == StringLit @@ -210,7 +213,7 @@ proc loadConfigsFromNif(conf: ConfigRef; n: var Cursor) = #echo conf.nimblePaths conf.searchPaths.setLen(0) - assert pool.tags[n.tagId] == "paths" + expectTag n, "paths" inc n while n.kind != ParRi: assert n.kind == StringLit From 9e703c256fdd36aee282a7467ab1b56870585ff6 Mon Sep 17 00:00:00 2001 From: demotomohiro Date: Fri, 14 Nov 2025 03:32:27 +0900 Subject: [PATCH 08/19] store/load ConfigRef fields --- compiler/nimconfcache.nim | 114 +++++++++++++++++++++++++++++++++++++- 1 file changed, 112 insertions(+), 2 deletions(-) diff --git a/compiler/nimconfcache.nim b/compiler/nimconfcache.nim index 051b724cfb31..8249401f5659 100644 --- a/compiler/nimconfcache.nim +++ b/compiler/nimconfcache.nim @@ -1,4 +1,4 @@ -import options, pathutils, condsyms +import options, pathutils, platform, condsyms import std/[assertions, os, sets, strtabs, times] from std/sequtils import addUnique from std/strutils import parseEnum @@ -20,12 +20,22 @@ proc fromNif[T: enum](result: var set[T]; n: var Cursor) = # clear it so that it has the same value as it was stored to the cache. # some switches turn off options that were turned on when conf was initialized. result = {} - assert n.kind == StringLit + assert n.kind in {StringLit, ParRi} while n.kind != ParRi: result.incl parseEnum[T](pool.strings[n.litId]) inc n inc n +proc fromNif2[T: enum](result: var set[T]; n: var Cursor) = + # same to fromNif but works with enums `parseEnum` doesn't support. + # e.g. TNoteKind + result = {} + assert n.kind in {IntLit, ParRi} + while n.kind != ParRi: + result.incl pool.integers[n.intId].T + inc n + inc n + template buildTree(dest: var TokenBuf; tag: string; body: untyped): untyped = buildTree(dest, pool.tags.getOrIncl(tag), NoLineInfo, body) @@ -38,6 +48,10 @@ proc configToNif(conf: ConfigRef; dest: var TokenBuf) = # store fields that can be changed on config files # see processSwitch proc in commands.nim + dest.addStrLit $conf.backend + dest.addStrLit $conf.target.targetOS + dest.addStrLit $conf.target.targetCPU + #echo "conf.options" #echo conf.options dest.buildTree "options": @@ -87,6 +101,37 @@ proc configToNif(conf: ConfigRef; dest: var TokenBuf) = #echo conf.nimbasePattern dest.addStrLit conf.nimbasePattern + dest.buildTree "features": + for f in conf.features: + dest.addStrLit $f + + dest.buildTree "legacyFeatures": + for f in conf.legacyFeatures: + dest.addStrLit $f + + dest.addStrLit $conf.cCompiler + + # stores TNoteKind as int as parseEnum doesn't work + dest.buildTree "modifiedyNotes": + for n in conf.modifiedyNotes: + dest.addIntLit n.ord + dest.buildTree "foreignPackageNotes": + for n in conf.foreignPackageNotes: + dest.addIntLit n.ord + dest.buildTree "notes": + for n in conf.notes: + dest.addIntLit n.ord + dest.buildTree "warningAsErrors": + for n in conf.warningAsErrors: + dest.addIntLit n.ord + dest.buildTree "mainPackageNotes": + for n in conf.mainPackageNotes: + dest.addIntLit n.ord + + dest.addIntLit conf.errorMax + dest.addIntLit conf.maxLoopIterationsVM + dest.addIntLit conf.maxCallDepthVM + dest.buildTree "defines": for def in definedSymbolNames(conf.symbols): dest.addStrLit def @@ -137,6 +182,13 @@ proc sourcesChanged(conf: ConfigRef; n: var Cursor; modTime: Time): HashSet[stri inc n proc loadConfigsFromNif(conf: ConfigRef; n: var Cursor) = + fromNif conf.backend, n + var targetOS = default(TSystemOS) + var targetCPU = default(TSystemCPU) + fromNif targetOS, n + fromNif targetCPU, n + conf.target.setTarget(targetOS, targetCPU) + expectTag n, "options" inc n fromNif(conf.options, n) @@ -190,6 +242,37 @@ proc loadConfigsFromNif(conf: ConfigRef; n: var Cursor) = inc n #echo conf.nimbasePattern + expectTag n, "features" + inc n + fromNif(conf.features, n) + expectTag n, "legacyFeatures" + inc n + fromNif(conf.legacyFeatures, n) + fromNif(conf.cCompiler, n) + + expectTag n, "modifiedyNotes" + inc n + fromNif2(conf.modifiedyNotes, n) + expectTag n, "foreignPackageNotes" + inc n + fromNif2(conf.foreignPackageNotes, n) + expectTag n, "notes" + inc n + fromNif2(conf.notes, n) + expectTag n, "warningAsErrors" + inc n + fromNif2(conf.warningAsErrors, n) + expectTag n, "mainPackageNotes" + inc n + fromNif2(conf.mainPackageNotes, n) + + conf.errorMax = pool.integers[n.intId] + inc n + conf.maxLoopIterationsVM = pool.integers[n.intId] + inc n + conf.maxCallDepthVM = pool.integers[n.intId] + inc n + conf.symbols.clear expectTag n, "defines" inc n @@ -276,6 +359,9 @@ when isMainModule: template assertImpl(f: untyped) = assert x.f == y.f, $x.f & " / " & $y.f + assertImpl backend + assert x.target.targetOS == y.target.targetOS + assert x.target.targetCPU == y.target.targetCPU assertImpl options assertImpl globalOptions assertImpl macrosToExpand @@ -289,6 +375,17 @@ when isMainModule: assertImpl numberOfProcessors assertImpl spellSuggestMax assertImpl nimbasePattern + assertImpl features + assertImpl legacyFeatures + assertImpl cCompiler + assertImpl modifiedyNotes + assertImpl foreignPackageNotes + assertImpl notes + assertImpl warningAsErrors + assertImpl mainPackageNotes + assertImpl errorMax + assertImpl maxLoopIterationsVM + assertImpl maxCallDepthVM assertImpl symbols assertImpl nimblePaths assertImpl searchPaths @@ -322,6 +419,8 @@ when isMainModule: block: var conf = newConfigRef() + conf.backend = backendCpp + conf.target.setTarget(osAny, cpuArm64) conf.options = {optObjCheck, optFieldCheck} conf.globalOptions = {gloptNone, optRun} conf.macrosToExpand["foomacro"] = "T" @@ -337,6 +436,17 @@ when isMainModule: conf.numberOfProcessors = 123 conf.spellSuggestMax = 456 conf.nimbasePattern = "foo/nimbase.h" + conf.features = {callOperator, dynamicBindSym} + conf.legacyFeatures = {laxEffects, emitGenerics} + conf.cCompiler = ccCLang + conf.modifiedyNotes = {} + conf.foreignPackageNotes = {} + conf.notes = {} + conf.warningAsErrors = {} + conf.mainPackageNotes = {} + conf.errorMax = 7 + conf.maxLoopIterationsVM = 1234 + conf.maxCallDepthVM = 111 conf.symbols.initDefines() conf.symbols.defineSymbol("test") conf.nimblePaths = @[AbsoluteDir"/foo", AbsoluteDir"/lib/nimble"] From 62ccf262eef9d3e235ca3e405df0c9aaba00e214 Mon Sep 17 00:00:00 2001 From: demotomohiro Date: Fri, 14 Nov 2025 22:34:02 +0900 Subject: [PATCH 09/19] store/load ConfigRef fields --- compiler/nimconfcache.nim | 78 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 74 insertions(+), 4 deletions(-) diff --git a/compiler/nimconfcache.nim b/compiler/nimconfcache.nim index 8249401f5659..2fa00198b6e2 100644 --- a/compiler/nimconfcache.nim +++ b/compiler/nimconfcache.nim @@ -36,9 +36,37 @@ proc fromNif2[T: enum](result: var set[T]; n: var Cursor) = inc n inc n +proc fromNif(result: StringTableRef; n: var Cursor) = + result.clear + assert n.kind in {ParLe, ParRi} + while n.kind != ParRi: + expectTag n, "kv" + inc n + assert n.kind == StringLit + let key = pool.strings[n.litId] + inc n + let val = pool.strings[n.litId] + inc n + result[key] = val + assert n.kind == ParRi + inc n + inc n + +proc fromNif(result: var AbsoluteDir; n: var Cursor) = + let d = pool.strings[n.litId] + inc n + result = if d.len > 0: d.toAbsoluteDir else: AbsoluteDir"" + template buildTree(dest: var TokenBuf; tag: string; body: untyped): untyped = buildTree(dest, pool.tags.getOrIncl(tag), NoLineInfo, body) +proc toNif(dest: var TokenBuf; tag: string; tab: StringTableRef) = + dest.buildTree tag: + for key, val in pairs(tab): + dest.buildTree "kv": + dest.addStrLit key + dest.addStrLit val + proc configToNif(conf: ConfigRef; dest: var TokenBuf) = # store data used to decide whether to use cache or eval config files dest.addStrLit conf.commandLine @@ -132,6 +160,8 @@ proc configToNif(conf: ConfigRef; dest: var TokenBuf) = dest.addIntLit conf.maxLoopIterationsVM dest.addIntLit conf.maxCallDepthVM + dest.toNif "configVars", conf.configVars + dest.buildTree "defines": for def in definedSymbolNames(conf.symbols): dest.addStrLit def @@ -151,6 +181,12 @@ proc configToNif(conf: ConfigRef; dest: var TokenBuf) = for p in paths: dest.addStrLit p + dest.addStrLit conf.outFile.string + dest.addStrLit conf.outDir.toNifPath + dest.addStrLit conf.prefixDir.toNifPath + dest.addStrLit conf.libpath.toNifPath + dest.addStrLit conf.nimcacheDir.toNifPath + proc cfgCachePath(conf: ConfigRef): (AbsoluteDir, RelativeFile) = (conf.projectPath / RelativeDir"nimcache", RelativeFile"cache.cfg.nif") @@ -273,6 +309,10 @@ proc loadConfigsFromNif(conf: ConfigRef; n: var Cursor) = conf.maxCallDepthVM = pool.integers[n.intId] inc n + expectTag n, "configVars" + inc n + fromNif(conf.configVars, n) + conf.symbols.clear expectTag n, "defines" inc n @@ -307,6 +347,13 @@ proc loadConfigsFromNif(conf: ConfigRef; n: var Cursor) = #echo "conf.searchPaths" #echo conf.searchPaths + conf.outFile = pool.strings[n.litId].RelativeFile + inc n + fromNif(conf.outDir, n) + fromNif(conf.prefixDir, n) + fromNif(conf.libpath, n) + fromNif(conf.nimcacheDir, n) + proc sourceChanged*(conf: ConfigRef): HashSet[string] = result = HashSet[string]() let (dir, file) = conf.cfgCachePath() @@ -348,13 +395,22 @@ proc storeConfigs*(conf: ConfigRef) = writeFile dir / file, "(.nif24)\n" & toString(dest) when isMainModule: - proc `==`(x, y: StringTableRef): bool = + proc eqlKeys(x, y: StringTableRef): bool = for k, v in x: if k notin y: return false return true + proc `==`(x, y: StringTableRef): bool = + for k, v in x.pairs: + if k notin y: + return false + elif y[k] != v: + return false + + return true + proc assertEq(x, y: ConfigRef) = template assertImpl(f: untyped) = assert x.f == y.f, $x.f & " / " & $y.f @@ -364,8 +420,8 @@ when isMainModule: assert x.target.targetCPU == y.target.targetCPU assertImpl options assertImpl globalOptions - assertImpl macrosToExpand - assertImpl arcToExpand + assert eqlKeys(x.macrosToExpand, y.macrosToExpand) + assert eqlKeys(x.arcToExpand, y.arcToExpand) assertImpl filenameOption assertImpl unitSep assertImpl selectedGC @@ -386,9 +442,15 @@ when isMainModule: assertImpl errorMax assertImpl maxLoopIterationsVM assertImpl maxCallDepthVM - assertImpl symbols + assertImpl configVars + assert eqlKeys(x.symbols, y.symbols) assertImpl nimblePaths assertImpl searchPaths + assertImpl outFile + assertImpl outDir + assertImpl prefixDir + assertImpl libpath + assertImpl nimcacheDir proc testConfig(conf1: ConfigRef) = var dest = createTokenBuf() @@ -447,8 +509,16 @@ when isMainModule: conf.errorMax = 7 conf.maxLoopIterationsVM = 1234 conf.maxCallDepthVM = 111 + conf.setConfigVar("foo.bar", "baz") + conf.setConfigVar("abc.def.ghi", "123") + conf.setConfigVar(".", "") conf.symbols.initDefines() conf.symbols.defineSymbol("test") conf.nimblePaths = @[AbsoluteDir"/foo", AbsoluteDir"/lib/nimble"] conf.searchPaths = @[AbsoluteDir"/lib", AbsoluteDir"/user/lib"] + conf.outFile = RelativeFile"foo" + conf.outDir = AbsoluteDir"/foo/var" + conf.prefixDir = AbsoluteDir"/home/foo/Nim" + conf.libpath = AbsoluteDir"/home/foo/Nim/lib" + conf.nimcacheDir = AbsoluteDir"/root/nimcache" testConfig conf From b4c08b1964f358d967396cd6150a391cd473b524 Mon Sep 17 00:00:00 2001 From: demotomohiro Date: Sat, 15 Nov 2025 06:37:25 +0900 Subject: [PATCH 10/19] store/load ConfigRef fields --- compiler/nimconfcache.nim | 151 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) diff --git a/compiler/nimconfcache.nim b/compiler/nimconfcache.nim index 2fa00198b6e2..b97ab7e557fd 100644 --- a/compiler/nimconfcache.nim +++ b/compiler/nimconfcache.nim @@ -187,6 +187,50 @@ proc configToNif(conf: ConfigRef; dest: var TokenBuf) = dest.addStrLit conf.libpath.toNifPath dest.addStrLit conf.nimcacheDir.toNifPath + dest.buildTree "dllOverrides": + for s in conf.dllOverrides.keys: + dest.addStrLit s + + dest.toNif "moduleOverrides", conf.moduleOverrides + + dest.buildTree "implicitImports": + for m in conf.implicitImports: + dest.addStrLit m + + dest.buildTree "implicitIncludes": + for m in conf.implicitIncludes: + dest.addStrLit m + + dest.addStrLit conf.docSeeSrcUrl + dest.addStrLit conf.docRoot + dest.addStrLit conf.docCmd + + dest.buildTree "configFiles": + for c in conf.configFiles: + dest.addStrLit c.toNifPath + + dest.buildTree "cIncludes": + for d in conf.cIncludes: + dest.addStrLit d.toNifPath + + dest.buildTree "cLibs": + for d in conf.cLibs: + dest.addStrLit d.toNifPath + + dest.buildTree "cLinkedLibs": + for l in conf.cLinkedLibs: + dest.addStrLit l + + dest.buildTree "externalToLink": + for f in conf.externalToLink: + dest.addStrLit f + + dest.addStrLit conf.linkOptionsCmd + + dest.buildTree "compileOptionsCmd": + for c in conf.compileOptionsCmd: + dest.addStrLit c + proc cfgCachePath(conf: ConfigRef): (AbsoluteDir, RelativeFile) = (conf.projectPath / RelativeDir"nimcache", RelativeFile"cache.cfg.nif") @@ -354,6 +398,83 @@ proc loadConfigsFromNif(conf: ConfigRef; n: var Cursor) = fromNif(conf.libpath, n) fromNif(conf.nimcacheDir, n) + expectTag n, "dllOverrides" + inc n + while n.kind != ParRi: + conf.inclDynlibOverride(pool.strings[n.litId]) + inc n + inc n + + expectTag n, "moduleOverrides" + inc n + fromNif(conf.moduleOverrides, n) + + expectTag n, "implicitImports" + inc n + while n.kind != ParRi: + conf.implicitImports.add pool.strings[n.litId] + inc n + inc n + + expectTag n, "implicitIncludes" + inc n + while n.kind != ParRi: + conf.implicitIncludes.add pool.strings[n.litId] + inc n + inc n + + conf.docSeeSrcUrl = pool.strings[n.litId] + inc n + conf.docRoot = pool.strings[n.litId] + inc n + conf.docCmd = pool.strings[n.litId] + inc n + + expectTag n, "configFiles" + inc n + while n.kind != ParRi: + conf.configFiles.add pool.strings[n.litId].toAbsolute(getCurrentDir().AbsoluteDir) + inc n + inc n + + expectTag n, "cIncludes" + inc n + while n.kind != ParRi: + conf.cIncludes.add pool.strings[n.litId].toAbsoluteDir + inc n + inc n + + expectTag n, "cLibs" + inc n + while n.kind != ParRi: + conf.cLibs.add pool.strings[n.litId].toAbsoluteDir + inc n + inc n + + expectTag n, "cLinkedLibs" + inc n + while n.kind != ParRi: + conf.cLinkedLibs.add pool.strings[n.litId] + inc n + inc n + + expectTag n, "externalToLink" + inc n + while n.kind != ParRi: + conf.externalToLink.add pool.strings[n.litId] + inc n + inc n + + conf.linkOptionsCmd = pool.strings[n.litId] + inc n + + expectTag n, "compileOptionsCmd" + inc n + while n.kind != ParRi: + conf.compileOptionsCmd.add pool.strings[n.litId] + inc n + inc n + proc sourceChanged*(conf: ConfigRef): HashSet[string] = result = HashSet[string]() let (dir, file) = conf.cfgCachePath() @@ -451,6 +572,20 @@ when isMainModule: assertImpl prefixDir assertImpl libpath assertImpl nimcacheDir + assert eqlKeys(x.dllOverrides, y.dllOverrides) + assertImpl moduleOverrides + assertImpl implicitImports + assertImpl implicitIncludes + assertImpl docSeeSrcUrl + assertImpl docRoot + assertImpl docCmd + assertImpl configFiles + assertImpl cIncludes + assertImpl cLibs + assertImpl cLinkedLibs + assertImpl externalToLink + assertImpl linkOptionsCmd + assertImpl compileOptionsCmd proc testConfig(conf1: ConfigRef) = var dest = createTokenBuf() @@ -521,4 +656,20 @@ when isMainModule: conf.prefixDir = AbsoluteDir"/home/foo/Nim" conf.libpath = AbsoluteDir"/home/foo/Nim/lib" conf.nimcacheDir = AbsoluteDir"/root/nimcache" + conf.inclDynlibOverride("foolib") + conf.inclDynlibOverride("libbar") + conf.moduleOverrides["foo"] = "overridefoo" + conf.moduleOverrides["bar"] = "overridebar" + conf.implicitImports = @["foo", "bar"] + conf.implicitIncludes = @["foo", "bar"] + conf.docSeeSrcUrl = "doc see src url" + conf.docRoot = "doc root" + conf.docCmd = "doc command" + conf.configFiles = @[AbsoluteFile"/foo/nim.cfg", AbsoluteFile"/bar/baz/config.nims"] + conf.cIncludes = @[AbsoluteDir"/include", AbsoluteDir"/foo/bar"] + conf.cLibs = @[AbsoluteDir"/lib", AbsoluteDir"/foo/bar"] + conf.cLinkedLibs = @["foolib", "libbar"] + conf.externalToLink = @["foo", "bar"] + conf.linkOptionsCmd = "--foo -lbar" + conf.compileOptionsCmd = @["-O3", "-g"] testConfig conf From dac44c38ab1f17e8be27a685139df23ec0e151bf Mon Sep 17 00:00:00 2001 From: demotomohiro Date: Sat, 15 Nov 2025 23:15:03 +0900 Subject: [PATCH 11/19] clean up code --- compiler/nimconfcache.nim | 94 ++++++++++++++------------------------- 1 file changed, 34 insertions(+), 60 deletions(-) diff --git a/compiler/nimconfcache.nim b/compiler/nimconfcache.nim index b97ab7e557fd..6e67407d70f2 100644 --- a/compiler/nimconfcache.nim +++ b/compiler/nimconfcache.nim @@ -9,6 +9,21 @@ proc toNifPath(p: AbsoluteDir|AbsoluteFile): string = if p.string.len <= result.len: result = p.string +template buildTree(dest: var TokenBuf; tag: string; body: untyped): untyped = + buildTree(dest, pool.tags.getOrIncl(tag), NoLineInfo, body) + +proc toNif(dest: var TokenBuf; tag: string; tab: StringTableRef) = + dest.buildTree tag: + for key, val in pairs(tab): + dest.buildTree "kv": + dest.addStrLit key + dest.addStrLit val + +proc toNif(dest: var TokenBuf; tag: string; strSeq: seq[string]) = + dest.buildTree tag: + for s in strSeq: + dest.addStrLit s + proc expectTag(n: Cursor; tag: string) = assert pool.tags[n.tagId] == tag, "expected tag: " & tag & " but got: " & pool.tags[n.tagId] @@ -57,15 +72,15 @@ proc fromNif(result: var AbsoluteDir; n: var Cursor) = inc n result = if d.len > 0: d.toAbsoluteDir else: AbsoluteDir"" -template buildTree(dest: var TokenBuf; tag: string; body: untyped): untyped = - buildTree(dest, pool.tags.getOrIncl(tag), NoLineInfo, body) - -proc toNif(dest: var TokenBuf; tag: string; tab: StringTableRef) = - dest.buildTree tag: - for key, val in pairs(tab): - dest.buildTree "kv": - dest.addStrLit key - dest.addStrLit val +proc fromNif(result: var seq[string]; tag: string; n: var Cursor) = + expectTag n, tag + inc n + result.setLen(0) + assert n.kind in {StringLit, ParRi} + while n.kind != ParRi: + result.add pool.strings[n.litId] + inc n + inc n proc configToNif(conf: ConfigRef; dest: var TokenBuf) = # store data used to decide whether to use cache or eval config files @@ -193,13 +208,8 @@ proc configToNif(conf: ConfigRef; dest: var TokenBuf) = dest.toNif "moduleOverrides", conf.moduleOverrides - dest.buildTree "implicitImports": - for m in conf.implicitImports: - dest.addStrLit m - - dest.buildTree "implicitIncludes": - for m in conf.implicitIncludes: - dest.addStrLit m + dest.toNif "implicitImports", conf.implicitImports + dest.toNif "implicitIncludes", conf.implicitIncludes dest.addStrLit conf.docSeeSrcUrl dest.addStrLit conf.docRoot @@ -217,19 +227,10 @@ proc configToNif(conf: ConfigRef; dest: var TokenBuf) = for d in conf.cLibs: dest.addStrLit d.toNifPath - dest.buildTree "cLinkedLibs": - for l in conf.cLinkedLibs: - dest.addStrLit l - - dest.buildTree "externalToLink": - for f in conf.externalToLink: - dest.addStrLit f - + dest.toNif "cLinkedLibs", conf.cLinkedLibs + dest.toNif "externalToLink", conf.externalToLink dest.addStrLit conf.linkOptionsCmd - - dest.buildTree "compileOptionsCmd": - for c in conf.compileOptionsCmd: - dest.addStrLit c + dest.toNif "compileOptionsCmd", conf.compileOptionsCmd proc cfgCachePath(conf: ConfigRef): (AbsoluteDir, RelativeFile) = (conf.projectPath / RelativeDir"nimcache", RelativeFile"cache.cfg.nif") @@ -409,19 +410,8 @@ proc loadConfigsFromNif(conf: ConfigRef; n: var Cursor) = inc n fromNif(conf.moduleOverrides, n) - expectTag n, "implicitImports" - inc n - while n.kind != ParRi: - conf.implicitImports.add pool.strings[n.litId] - inc n - inc n - - expectTag n, "implicitIncludes" - inc n - while n.kind != ParRi: - conf.implicitIncludes.add pool.strings[n.litId] - inc n - inc n + fromNif conf.implicitImports, "implicitImports", n + fromNif conf.implicitIncludes, "implicitIncludes", n conf.docSeeSrcUrl = pool.strings[n.litId] inc n @@ -451,29 +441,13 @@ proc loadConfigsFromNif(conf: ConfigRef; n: var Cursor) = inc n inc n - expectTag n, "cLinkedLibs" - inc n - while n.kind != ParRi: - conf.cLinkedLibs.add pool.strings[n.litId] - inc n - inc n - - expectTag n, "externalToLink" - inc n - while n.kind != ParRi: - conf.externalToLink.add pool.strings[n.litId] - inc n - inc n + fromNif conf.cLinkedLibs, "cLinkedLibs", n + fromNif conf.externalToLink, "externalToLink", n conf.linkOptionsCmd = pool.strings[n.litId] inc n - expectTag n, "compileOptionsCmd" - inc n - while n.kind != ParRi: - conf.compileOptionsCmd.add pool.strings[n.litId] - inc n - inc n + fromNif conf.compileOptionsCmd, "compileOptionsCmd", n proc sourceChanged*(conf: ConfigRef): HashSet[string] = result = HashSet[string]() From cb3cf558cc04cc08791ac8152486974898d5a324 Mon Sep 17 00:00:00 2001 From: demotomohiro Date: Sun, 16 Nov 2025 01:07:24 +0900 Subject: [PATCH 12/19] clean up code --- compiler/nimconfcache.nim | 91 +++++++++++++-------------------------- 1 file changed, 31 insertions(+), 60 deletions(-) diff --git a/compiler/nimconfcache.nim b/compiler/nimconfcache.nim index 6e67407d70f2..98d7a77df960 100644 --- a/compiler/nimconfcache.nim +++ b/compiler/nimconfcache.nim @@ -1,6 +1,5 @@ import options, pathutils, platform, condsyms import std/[assertions, os, sets, strtabs, times] -from std/sequtils import addUnique from std/strutils import parseEnum import "../dist/nimony/src/lib" / [bitabs, lineinfos, nifreader, nifstreams, nifcursors] @@ -24,6 +23,19 @@ proc toNif(dest: var TokenBuf; tag: string; strSeq: seq[string]) = for s in strSeq: dest.addStrLit s +proc toNif(dest: var TokenBuf; tag: string; absDirs: seq[AbsoluteDir]) = + # remove duplicated paths without changing the order of paths + var paths: seq[string] = @[] + # use HashSet[string] to avoid O(n^2) computation + var inPaths = initHashSet[string]() + for p in absDirs: + let p2 = toNifPath p + if not inPaths.containsOrIncl(p2): + paths.add p2 + dest.buildTree tag: + for p in paths: + dest.addStrLit p + proc expectTag(n: Cursor; tag: string) = assert pool.tags[n.tagId] == tag, "expected tag: " & tag & " but got: " & pool.tags[n.tagId] @@ -82,6 +94,16 @@ proc fromNif(result: var seq[string]; tag: string; n: var Cursor) = inc n inc n +proc fromNif(result: var seq[AbsoluteDir]; tag: string; n: var Cursor) = + expectTag n, tag + inc n + result.setLen(0) + assert n.kind in {StringLit, ParRi} + while n.kind != ParRi: + result.add pool.strings[n.litId].toAbsoluteDir + inc n + inc n + proc configToNif(conf: ConfigRef; dest: var TokenBuf) = # store data used to decide whether to use cache or eval config files dest.addStrLit conf.commandLine @@ -181,20 +203,8 @@ proc configToNif(conf: ConfigRef; dest: var TokenBuf) = for def in definedSymbolNames(conf.symbols): dest.addStrLit def - #echo "conf.nimblePaths" - #echo conf.nimblePaths - dest.buildTree "nimblepaths": - for p in conf.nimblePaths: - dest.addStrLit p.toNifPath - - #echo "conf.searchPaths" - #echo conf.searchPaths - var paths: seq[string] = @[] - for p in conf.searchPaths: - paths.addUnique p.toNifPath - dest.buildTree "paths": - for p in paths: - dest.addStrLit p + dest.toNif "nimblepaths", conf.nimblePaths + dest.toNif "searchPaths", conf.searchPaths dest.addStrLit conf.outFile.string dest.addStrLit conf.outDir.toNifPath @@ -219,14 +229,8 @@ proc configToNif(conf: ConfigRef; dest: var TokenBuf) = for c in conf.configFiles: dest.addStrLit c.toNifPath - dest.buildTree "cIncludes": - for d in conf.cIncludes: - dest.addStrLit d.toNifPath - - dest.buildTree "cLibs": - for d in conf.cLibs: - dest.addStrLit d.toNifPath - + dest.toNif "cIncludes", conf.cIncludes + dest.toNif "cLibs", conf.cLibs dest.toNif "cLinkedLibs", conf.cLinkedLibs dest.toNif "externalToLink", conf.externalToLink dest.addStrLit conf.linkOptionsCmd @@ -368,29 +372,8 @@ proc loadConfigsFromNif(conf: ConfigRef; n: var Cursor) = conf.symbols.defineSymbol(def) inc n - conf.nimblePaths.setLen(0) - expectTag n, "nimblepaths" - inc n - while n.kind != ParRi: - assert n.kind == StringLit - let p = pool.strings[n.litId] - inc n - conf.nimblePaths.add p.toAbsoluteDir - inc n - #echo "conf.nimblePaths" - #echo conf.nimblePaths - - conf.searchPaths.setLen(0) - expectTag n, "paths" - inc n - while n.kind != ParRi: - assert n.kind == StringLit - let p = pool.strings[n.litId] - inc n - conf.searchPaths.add p.toAbsoluteDir - inc n - #echo "conf.searchPaths" - #echo conf.searchPaths + fromNif conf.nimblePaths, "nimblepaths", n + fromNif conf.searchPaths, "searchPaths", n conf.outFile = pool.strings[n.litId].RelativeFile inc n @@ -427,20 +410,8 @@ proc loadConfigsFromNif(conf: ConfigRef; n: var Cursor) = inc n inc n - expectTag n, "cIncludes" - inc n - while n.kind != ParRi: - conf.cIncludes.add pool.strings[n.litId].toAbsoluteDir - inc n - inc n - - expectTag n, "cLibs" - inc n - while n.kind != ParRi: - conf.cLibs.add pool.strings[n.litId].toAbsoluteDir - inc n - inc n - + fromNif conf.cIncludes, "cIncludes", n + fromNif conf.cLibs, "cLibs", n fromNif conf.cLinkedLibs, "cLinkedLibs", n fromNif conf.externalToLink, "externalToLink", n From 75ff634b03158a275a860dadcf161a784770dbd9 Mon Sep 17 00:00:00 2001 From: demotomohiro Date: Sun, 16 Nov 2025 01:42:38 +0900 Subject: [PATCH 13/19] clean up code --- compiler/nimconfcache.nim | 78 ++++++++++++++------------------------- 1 file changed, 28 insertions(+), 50 deletions(-) diff --git a/compiler/nimconfcache.nim b/compiler/nimconfcache.nim index 98d7a77df960..abe1182e8972 100644 --- a/compiler/nimconfcache.nim +++ b/compiler/nimconfcache.nim @@ -43,7 +43,9 @@ proc fromNif[T: enum](result: var T; n: var Cursor) = result = parseEnum[T](pool.strings[n.litId]) inc n -proc fromNif[T: enum](result: var set[T]; n: var Cursor) = +proc fromNif[T: enum](result: var set[T]; tag: string; n: var Cursor) = + expectTag n, tag + inc n # clear it so that it has the same value as it was stored to the cache. # some switches turn off options that were turned on when conf was initialized. result = {} @@ -53,9 +55,11 @@ proc fromNif[T: enum](result: var set[T]; n: var Cursor) = inc n inc n -proc fromNif2[T: enum](result: var set[T]; n: var Cursor) = +proc fromNif2[T: enum](result: var set[T]; tag: string; n: var Cursor) = # same to fromNif but works with enums `parseEnum` doesn't support. # e.g. TNoteKind + expectTag n, tag + inc n result = {} assert n.kind in {IntLit, ParRi} while n.kind != ParRi: @@ -63,7 +67,9 @@ proc fromNif2[T: enum](result: var set[T]; n: var Cursor) = inc n inc n -proc fromNif(result: StringTableRef; n: var Cursor) = +proc fromNif(result: StringTableRef; tag: string; n: var Cursor) = + expectTag n, tag + inc n result.clear assert n.kind in {ParLe, ParRi} while n.kind != ParRi: @@ -274,15 +280,8 @@ proc loadConfigsFromNif(conf: ConfigRef; n: var Cursor) = fromNif targetCPU, n conf.target.setTarget(targetOS, targetCPU) - expectTag n, "options" - inc n - fromNif(conf.options, n) - #echo conf.options - - expectTag n, "globalOptions" - inc n - fromNif(conf.globalOptions, n) - #echo conf.globalOptions + fromNif conf.options, "options", n + fromNif conf.globalOptions, "globalOptions", n conf.macrosToExpand.clear expectTag n, "macrosToExpand" @@ -304,14 +303,12 @@ proc loadConfigsFromNif(conf: ConfigRef; n: var Cursor) = conf.arcToExpand[m] = "T" inc n - fromNif(conf.filenameOption, n) + fromNif conf.filenameOption, n conf.unitSep = pool.strings[n.litId] inc n - fromNif(conf.selectedGC, n) - #echo conf.selectedGC - fromNif(conf.exc, n) - #echo conf.exc + fromNif conf.selectedGC, n + fromNif conf.exc, n conf.hintProcessingDots = pool.integers[n.intId].bool inc n conf.verbosity = pool.integers[n.intId] @@ -327,29 +324,15 @@ proc loadConfigsFromNif(conf: ConfigRef; n: var Cursor) = inc n #echo conf.nimbasePattern - expectTag n, "features" - inc n - fromNif(conf.features, n) - expectTag n, "legacyFeatures" - inc n - fromNif(conf.legacyFeatures, n) - fromNif(conf.cCompiler, n) + fromNif conf.features, "features", n + fromNif conf.legacyFeatures, "legacyFeatures", n + fromNif conf.cCompiler, n - expectTag n, "modifiedyNotes" - inc n - fromNif2(conf.modifiedyNotes, n) - expectTag n, "foreignPackageNotes" - inc n - fromNif2(conf.foreignPackageNotes, n) - expectTag n, "notes" - inc n - fromNif2(conf.notes, n) - expectTag n, "warningAsErrors" - inc n - fromNif2(conf.warningAsErrors, n) - expectTag n, "mainPackageNotes" - inc n - fromNif2(conf.mainPackageNotes, n) + fromNif2 conf.modifiedyNotes, "modifiedyNotes", n + fromNif2 conf.foreignPackageNotes, "foreignPackageNotes", n + fromNif2 conf.notes, "notes", n + fromNif2 conf.warningAsErrors, "warningAsErrors", n + fromNif2 conf.mainPackageNotes, "mainPackageNotes", n conf.errorMax = pool.integers[n.intId] inc n @@ -358,9 +341,7 @@ proc loadConfigsFromNif(conf: ConfigRef; n: var Cursor) = conf.maxCallDepthVM = pool.integers[n.intId] inc n - expectTag n, "configVars" - inc n - fromNif(conf.configVars, n) + fromNif conf.configVars, "configVars", n conf.symbols.clear expectTag n, "defines" @@ -377,10 +358,10 @@ proc loadConfigsFromNif(conf: ConfigRef; n: var Cursor) = conf.outFile = pool.strings[n.litId].RelativeFile inc n - fromNif(conf.outDir, n) - fromNif(conf.prefixDir, n) - fromNif(conf.libpath, n) - fromNif(conf.nimcacheDir, n) + fromNif conf.outDir, n + fromNif conf.prefixDir, n + fromNif conf.libpath, n + fromNif conf.nimcacheDir, n expectTag n, "dllOverrides" inc n @@ -388,10 +369,7 @@ proc loadConfigsFromNif(conf: ConfigRef; n: var Cursor) = conf.inclDynlibOverride(pool.strings[n.litId]) inc n inc n - - expectTag n, "moduleOverrides" - inc n - fromNif(conf.moduleOverrides, n) + fromNif conf.moduleOverrides, "moduleOverrides", n fromNif conf.implicitImports, "implicitImports", n fromNif conf.implicitIncludes, "implicitIncludes", n From 3b867a16f152ab0e990bc7df0f3a04eed3148f89 Mon Sep 17 00:00:00 2001 From: demotomohiro Date: Sun, 16 Nov 2025 12:06:32 +0900 Subject: [PATCH 14/19] store/load ConfigRef fields --- compiler/nimconfcache.nim | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/compiler/nimconfcache.nim b/compiler/nimconfcache.nim index abe1182e8972..224169830286 100644 --- a/compiler/nimconfcache.nim +++ b/compiler/nimconfcache.nim @@ -1,4 +1,4 @@ -import options, pathutils, platform, condsyms +import options, pathutils, platform, condsyms, extccomp import std/[assertions, os, sets, strtabs, times] from std/strutils import parseEnum import "../dist/nimony/src/lib" / [bitabs, lineinfos, nifreader, nifstreams, nifcursors] @@ -241,6 +241,15 @@ proc configToNif(conf: ConfigRef; dest: var TokenBuf) = dest.toNif "externalToLink", conf.externalToLink dest.addStrLit conf.linkOptionsCmd dest.toNif "compileOptionsCmd", conf.compileOptionsCmd + dest.addStrLit conf.compileOptions + dest.addStrLit conf.cCompilerPath + + dest.buildTree "toCompile": + for c in conf.toCompile: + dest.addStrLit c.cname.string + + dest.addStrLit conf.cppCustomNamespace + dest.addStrLit conf.nimMainPrefix proc cfgCachePath(conf: ConfigRef): (AbsoluteDir, RelativeFile) = (conf.projectPath / RelativeDir"nimcache", RelativeFile"cache.cfg.nif") @@ -397,6 +406,23 @@ proc loadConfigsFromNif(conf: ConfigRef; n: var Cursor) = inc n fromNif conf.compileOptionsCmd, "compileOptionsCmd", n + conf.compileOptions = pool.strings[n.litId] + inc n + conf.cCompilerPath = pool.strings[n.litId] + inc n + + expectTag n, "toCompile" + inc n + while n.kind != ParRi: + let c = pool.strings[n.litId] + inc n + conf.addExternalFileToCompile(AbsoluteFile(c)) + inc n + + conf.cppCustomNamespace = pool.strings[n.litId] + inc n + conf.nimMainPrefix = pool.strings[n.litId] + inc n proc sourceChanged*(conf: ConfigRef): HashSet[string] = result = HashSet[string]() @@ -509,6 +535,10 @@ when isMainModule: assertImpl externalToLink assertImpl linkOptionsCmd assertImpl compileOptionsCmd + assertImpl compileOptions + assertImpl cCompilerPath + assertImpl cppCustomNamespace + assertImpl nimMainPrefix proc testConfig(conf1: ConfigRef) = var dest = createTokenBuf() @@ -595,4 +625,8 @@ when isMainModule: conf.externalToLink = @["foo", "bar"] conf.linkOptionsCmd = "--foo -lbar" conf.compileOptionsCmd = @["-O3", "-g"] + conf.compileOptions = "-foo --bar" + conf.cCompilerPath = "/foo/bar" + conf.cppCustomNamespace = "foobar" + conf.nimMainPrefix = "prefix" testConfig conf From a2ddad9abf0fa248c70102799110d3ef0b00f56e Mon Sep 17 00:00:00 2001 From: demotomohiro Date: Sun, 16 Nov 2025 13:33:48 +0900 Subject: [PATCH 15/19] makes compile-time define pragmas works when using cache --- compiler/nimconfcache.nim | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/compiler/nimconfcache.nim b/compiler/nimconfcache.nim index 224169830286..fe8926d6a87c 100644 --- a/compiler/nimconfcache.nim +++ b/compiler/nimconfcache.nim @@ -206,8 +206,13 @@ proc configToNif(conf: ConfigRef; dest: var TokenBuf) = dest.toNif "configVars", conf.configVars dest.buildTree "defines": - for def in definedSymbolNames(conf.symbols): - dest.addStrLit def + for def, val in conf.symbols: + if val == "true": + dest.addStrLit def + else: + dest.buildTree "kv": + dest.addStrLit def + dest.addStrLit val dest.toNif "nimblepaths", conf.nimblePaths dest.toNif "searchPaths", conf.searchPaths @@ -356,10 +361,20 @@ proc loadConfigsFromNif(conf: ConfigRef; n: var Cursor) = expectTag n, "defines" inc n while n.kind != ParRi: - assert n.kind == StringLit - let def = pool.strings[n.litId] - inc n - conf.symbols.defineSymbol(def) + if n.kind == ParLe: + expectTag n, "kv" + inc n + let def = pool.strings[n.litId] + inc n + let val = pool.strings[n.litId] + inc n + conf.symbols.defineSymbol(def, val) + assert n.kind == ParRi + inc n + else: + let def = pool.strings[n.litId] + inc n + conf.symbols.defineSymbol(def) inc n fromNif conf.nimblePaths, "nimblepaths", n @@ -513,7 +528,7 @@ when isMainModule: assertImpl maxLoopIterationsVM assertImpl maxCallDepthVM assertImpl configVars - assert eqlKeys(x.symbols, y.symbols) + assertImpl symbols assertImpl nimblePaths assertImpl searchPaths assertImpl outFile @@ -602,6 +617,7 @@ when isMainModule: conf.setConfigVar(".", "") conf.symbols.initDefines() conf.symbols.defineSymbol("test") + conf.symbols.defineSymbol("FooBar", "123") conf.nimblePaths = @[AbsoluteDir"/foo", AbsoluteDir"/lib/nimble"] conf.searchPaths = @[AbsoluteDir"/lib", AbsoluteDir"/user/lib"] conf.outFile = RelativeFile"foo" From 29b2c990182ecb6de67ca38b590e3fef468750af Mon Sep 17 00:00:00 2001 From: demotomohiro Date: Sun, 16 Nov 2025 13:44:06 +0900 Subject: [PATCH 16/19] clean up code --- compiler/nimconf.nim | 14 +++++++------- compiler/nimconfcache.nim | 20 -------------------- 2 files changed, 7 insertions(+), 27 deletions(-) diff --git a/compiler/nimconf.nim b/compiler/nimconf.nim index de79bbc934d0..9cf4e78eff26 100644 --- a/compiler/nimconf.nim +++ b/compiler/nimconf.nim @@ -349,20 +349,20 @@ proc canUseCache(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef; idgen: I if cachedCfgFiles.len == 0: result = false else: - echo formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3), " checking added config" + #echo formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3), " checking added config" loadConfigsImpl(cfg, cache, conf, idgen, cachedCfgFiles) - echo formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3), " done checking added config" + #echo formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3), " done checking added config" result = cachedCfgFiles.len != 0 proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef; idgen: IdGenerator) = - conf.lastCmdTime = epochTime() + #conf.lastCmdTime = epochTime() var cachedCfgFiles = HashSet[string]() if canUseCache(cfg, cache, conf, idgen, cachedCfgFiles): - echo formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3), " loading cached config" + #echo formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3), " loading cached config" loadConfigsFromCache(conf) - echo formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3), " done loading cached config" + #echo formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3), " done loading cached config" else: - echo formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3), " evaluating config files" + #echo formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3), " evaluating config files" loadConfigsImpl(cfg, cache, conf, idgen, cachedCfgFiles) - echo formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3), " done evaluating config files" + #echo formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3), " done evaluating config files" storeConfigs(conf) diff --git a/compiler/nimconfcache.nim b/compiler/nimconfcache.nim index fe8926d6a87c..b246cfcfea7b 100644 --- a/compiler/nimconfcache.nim +++ b/compiler/nimconfcache.nim @@ -123,14 +123,10 @@ proc configToNif(conf: ConfigRef; dest: var TokenBuf) = dest.addStrLit $conf.target.targetOS dest.addStrLit $conf.target.targetCPU - #echo "conf.options" - #echo conf.options dest.buildTree "options": for opt in conf.options: dest.addStrLit $opt - #echo "conf.globalOptions" - #echo conf.globalOptions dest.buildTree "globalOptions": for opt in conf.globalOptions: dest.addStrLit $opt @@ -146,30 +142,18 @@ proc configToNif(conf: ConfigRef; dest: var TokenBuf) = dest.addStrLit $conf.filenameOption dest.addStrLit conf.unitSep - #echo "conf.selectedGC" - #echo conf.selectedGC dest.addStrLit $conf.selectedGC - #echo "conf.exc" - #echo conf.exc dest.addStrLit $conf.exc dest.addIntLit conf.hintProcessingDots.int - #echo "conf.verbosity" - #echo conf.verbosity dest.addIntLit conf.verbosity - #echo "conf.numberOfProcessors" - #echo conf.numberOfProcessors dest.addIntLit conf.numberOfProcessors - #echo "conf.spellSuggestMax" - #echo conf.spellSuggestMax dest.addIntLit conf.spellSuggestMax - #echo "conf.nimbasePattern" - #echo conf.nimbasePattern dest.addStrLit conf.nimbasePattern dest.buildTree "features": @@ -327,16 +311,12 @@ proc loadConfigsFromNif(conf: ConfigRef; n: var Cursor) = inc n conf.verbosity = pool.integers[n.intId] inc n - #echo conf.verbosity conf.numberOfProcessors = pool.integers[n.intId] inc n - #echo conf.numberOfProcessors conf.spellSuggestMax = pool.integers[n.intId] inc n - #echo conf.spellSuggestMax conf.nimbasePattern = pool.strings[n.litId] inc n - #echo conf.nimbasePattern fromNif conf.features, "features", n fromNif conf.legacyFeatures, "legacyFeatures", n From a7fdf6e1508140b40b88d3d345aabc7fedc066ea Mon Sep 17 00:00:00 2001 From: demotomohiro Date: Mon, 17 Nov 2025 23:25:15 +0900 Subject: [PATCH 17/19] store/load ConfigRef fields --- compiler/nimconfcache.nim | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/compiler/nimconfcache.nim b/compiler/nimconfcache.nim index b246cfcfea7b..c20792873a66 100644 --- a/compiler/nimconfcache.nim +++ b/compiler/nimconfcache.nim @@ -143,17 +143,13 @@ proc configToNif(conf: ConfigRef; dest: var TokenBuf) = dest.addStrLit conf.unitSep dest.addStrLit $conf.selectedGC - dest.addStrLit $conf.exc - dest.addIntLit conf.hintProcessingDots.int - dest.addIntLit conf.verbosity - dest.addIntLit conf.numberOfProcessors - + dest.addStrLit $conf.symbolFiles dest.addIntLit conf.spellSuggestMax - + dest.addStrLit conf.headerFile dest.addStrLit conf.nimbasePattern dest.buildTree "features": @@ -313,8 +309,11 @@ proc loadConfigsFromNif(conf: ConfigRef; n: var Cursor) = inc n conf.numberOfProcessors = pool.integers[n.intId] inc n + fromNif conf.symbolFiles, n conf.spellSuggestMax = pool.integers[n.intId] inc n + conf.headerFile = pool.strings[n.litId] + inc n conf.nimbasePattern = pool.strings[n.litId] inc n @@ -494,7 +493,9 @@ when isMainModule: assertImpl hintProcessingDots assertImpl verbosity assertImpl numberOfProcessors + assertImpl symbolFiles assertImpl spellSuggestMax + assertImpl headerFile assertImpl nimbasePattern assertImpl features assertImpl legacyFeatures @@ -579,7 +580,9 @@ when isMainModule: conf.hintProcessingDots = false conf.verbosity = 3 conf.numberOfProcessors = 123 + conf.symbolFiles = stressTest conf.spellSuggestMax = 456 + conf.headerFile = "foo.h" conf.nimbasePattern = "foo/nimbase.h" conf.features = {callOperator, dynamicBindSym} conf.legacyFeatures = {laxEffects, emitGenerics} From 5213d6da8802f182ed77dd5d98a0d78cd1b5ec4d Mon Sep 17 00:00:00 2001 From: demotomohiro Date: Wed, 19 Nov 2025 04:04:24 +0900 Subject: [PATCH 18/19] updates documentations --- changelog.md | 1 + doc/advopt.txt | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/changelog.md b/changelog.md index bf7d343d1bfb..1b71846438be 100644 --- a/changelog.md +++ b/changelog.md @@ -101,6 +101,7 @@ errors. ## Compiler changes +- Added `--cachecfg` command line option that caches the result of evaluation of cfg/NimScript configuration files to reduces the compile time. ## Tool changes diff --git a/doc/advopt.txt b/doc/advopt.txt index 85da350a8011..0621d5075a8d 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -180,3 +180,11 @@ Advanced options: the JavaScript backend (default: on) --nimBasePattern:nimbase.h allows to specify a custom pattern for `nimbase.h` + --cachecfg:on|off cache the result of evaluation of cfg/NimScript configuration files to reduces the compile time. + The cache file is created in `$projectdir/nimcache`, not in `$nimcache`. + The cache file is not loaded and configuration files are evaluated when: + - `--cachecfg` is off or `--forceBuild` is on + - cache file doesn't exists + - command line parameters are changed + - any configuration file is changed or removed + - new configuration file is created From 7a314b7dd5e282c961dc2f693ea15b79ce0a89e9 Mon Sep 17 00:00:00 2001 From: demotomohiro Date: Wed, 19 Nov 2025 04:20:18 +0900 Subject: [PATCH 19/19] fixes doc/advopt.txt --- doc/advopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/advopt.txt b/doc/advopt.txt index 0621d5075a8d..2b9534951519 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -181,7 +181,7 @@ Advanced options: --nimBasePattern:nimbase.h allows to specify a custom pattern for `nimbase.h` --cachecfg:on|off cache the result of evaluation of cfg/NimScript configuration files to reduces the compile time. - The cache file is created in `$projectdir/nimcache`, not in `$nimcache`. + The cache file is created in `$$projectdir/nimcache`, not in `$$nimcache`. The cache file is not loaded and configuration files are evaluated when: - `--cachecfg` is off or `--forceBuild` is on - cache file doesn't exists