From e9127b2567d079f666f32f3a3e7eba63e0331dfe Mon Sep 17 00:00:00 2001 From: demotomohiro Date: Fri, 24 Oct 2025 06:09:33 +0900 Subject: [PATCH 01/38] adds Nim's AST to/from NIF --- compiler/icnif/enum2nif.nim | 1829 ++++++++++++++++++ compiler/icnif/icniftags.nim | 8 + compiler/icnif/nifdecoder.nim | 283 +++ compiler/icnif/nifencoder.nim | 177 ++ tests/icnif/tencode_node2node.nim | 271 +++ tests/icnif/testcode/modtest1.nim | 2 + tests/icnif/testcode/modtestliterals.nim | 29 + tests/icnif/testcode/modtestpragmas.nim | 3 + tests/icnif/testcode/modtesttypesections.nim | 36 + 9 files changed, 2638 insertions(+) create mode 100644 compiler/icnif/enum2nif.nim create mode 100644 compiler/icnif/icniftags.nim create mode 100644 compiler/icnif/nifdecoder.nim create mode 100644 compiler/icnif/nifencoder.nim create mode 100644 tests/icnif/tencode_node2node.nim create mode 100644 tests/icnif/testcode/modtest1.nim create mode 100644 tests/icnif/testcode/modtestliterals.nim create mode 100644 tests/icnif/testcode/modtestpragmas.nim create mode 100644 tests/icnif/testcode/modtesttypesections.nim diff --git a/compiler/icnif/enum2nif.nim b/compiler/icnif/enum2nif.nim new file mode 100644 index 000000000000..ecfdcfa0db2e --- /dev/null +++ b/compiler/icnif/enum2nif.nim @@ -0,0 +1,1829 @@ +# Generated by gear2/generator/enumgen.nim in Nimony repo. DO NOT EDIT! + +import ".." / [ast, options] + +proc toNifTag*(s: TNodeKind): string = + case s + of nkNone: "none" + of nkEmpty: "empty" + of nkIdent: "ident" + of nkSym: "sym" + of nkType: "onlytype" + of nkCharLit: "charlit" + of nkIntLit: "intlit" + of nkInt8Lit: "int8lit" + of nkInt16Lit: "int16lit" + of nkInt32Lit: "int32lit" + of nkInt64Lit: "int64lit" + of nkUIntLit: "uintlit" + of nkUInt8Lit: "uint8lit" + of nkUInt16Lit: "uint16lit" + of nkUInt32Lit: "uint32lit" + of nkUInt64Lit: "uint64lit" + of nkFloatLit: "floatlit" + of nkFloat32Lit: "float32lit" + of nkFloat64Lit: "float64lit" + of nkFloat128Lit: "float128lit" + of nkStrLit: "strlit" + of nkRStrLit: "rstrlit" + of nkTripleStrLit: "triplestrlit" + of nkNilLit: "nil" + of nkComesFrom: "comesfrom" + of nkDotCall: "dotcall" + of nkCommand: "cmd" + of nkCall: "call" + of nkCallStrLit: "callstrlit" + of nkInfix: "infix" + of nkPrefix: "prefix" + of nkPostfix: "postfix" + of nkHiddenCallConv: "hcallconv" + of nkExprEqExpr: "vv" + of nkExprColonExpr: "kv" + of nkIdentDefs: "identdefs" + of nkVarTuple: "vartuple" + of nkPar: "par" + of nkObjConstr: "objconstr" + of nkCurly: "curly" + of nkCurlyExpr: "curlyx" + of nkBracket: "bracket" + of nkBracketExpr: "at" + of nkPragmaExpr: "pragmax" + of nkRange: "range" + of nkDotExpr: "dot" + of nkCheckedFieldExpr: "checkedfieldx" + of nkDerefExpr: "deref" + of nkIfExpr: "ifx" + of nkElifExpr: "elifx" + of nkElseExpr: "elsex" + of nkLambda: "lambda" + of nkDo: "do" + of nkAccQuoted: "accquoted" + of nkTableConstr: "tableconstr" + of nkBind: "bind" + of nkClosedSymChoice: "closedsymchoice" + of nkOpenSymChoice: "opensymchoice" + of nkHiddenStdConv: "hstdconv" + of nkHiddenSubConv: "hsubconv" + of nkConv: "conv" + of nkCast: "cast" + of nkStaticExpr: "staticx" + of nkAddr: "addr" + of nkHiddenAddr: "haddr" + of nkHiddenDeref: "hderef" + of nkObjDownConv: "objdownconv" + of nkObjUpConv: "objupconv" + of nkChckRangeF: "chckrangef" + of nkChckRange64: "chckrange64" + of nkChckRange: "chckrange" + of nkStringToCString: "stringtocstring" + of nkCStringToString: "cstringtostring" + of nkAsgn: "asgn" + of nkFastAsgn: "fastasgn" + of nkGenericParams: "genericparams" + of nkFormalParams: "formalparams" + of nkOfInherit: "ofinherit" + of nkImportAs: "importas" + of nkProcDef: "proc" + of nkMethodDef: "method" + of nkConverterDef: "converter" + of nkMacroDef: "macro" + of nkTemplateDef: "template" + of nkIteratorDef: "iterator" + of nkOfBranch: "of" + of nkElifBranch: "elif" + of nkExceptBranch: "except" + of nkElse: "else" + of nkAsmStmt: "asm" + of nkPragma: "pragma" + of nkPragmaBlock: "pragmablock" + of nkIfStmt: "if" + of nkWhenStmt: "when" + of nkForStmt: "for" + of nkParForStmt: "parfor" + of nkWhileStmt: "while" + of nkCaseStmt: "case" + of nkTypeSection: "type" + of nkVarSection: "var" + of nkLetSection: "let" + of nkConstSection: "const" + of nkConstDef: "const0" + of nkTypeDef: "type0" + of nkYieldStmt: "yield" + of nkDefer: "defer" + of nkTryStmt: "try" + of nkFinally: "finally" + of nkRaiseStmt: "raise" + of nkReturnStmt: "ret" + of nkBreakStmt: "brk" + of nkContinueStmt: "continue" + of nkBlockStmt: "block" + of nkStaticStmt: "static" + of nkDiscardStmt: "discard" + of nkStmtList: "stmts" + of nkImportStmt: "import" + of nkImportExceptStmt: "importexcept" + of nkExportStmt: "export" + of nkExportExceptStmt: "exportexcept" + of nkFromStmt: "from" + of nkIncludeStmt: "include" + of nkBindStmt: "bind0" + of nkMixinStmt: "mixin" + of nkUsingStmt: "using" + of nkCommentStmt: "comment" + of nkStmtListExpr: "expr" + of nkBlockExpr: "blockx" + of nkStmtListType: "stmtlisttype" + of nkBlockType: "blocktype" + of nkWith: "with" + of nkWithout: "without" + of nkTypeOfExpr: "typeofx" + of nkObjectTy: "objectty" + of nkTupleTy: "tuplety" + of nkTupleClassTy: "tupleclassty" + of nkTypeClassTy: "typeclassty" + of nkStaticTy: "staticty" + of nkRecList: "reclist" + of nkRecCase: "reccase" + of nkRecWhen: "recwhen" + of nkRefTy: "refty" + of nkPtrTy: "ptrty" + of nkVarTy: "varty" + of nkConstTy: "constty" + of nkOutTy: "outty" + of nkDistinctTy: "distinctty" + of nkProcTy: "procty" + of nkIteratorTy: "iteratorty" + of nkSinkAsgn: "sinkasgn" + of nkEnumTy: "enumty" + of nkEnumFieldDef: "efld" + of nkArgList: "arglist" + of nkPattern: "pattern" + of nkHiddenTryStmt: "htrystmt" + of nkClosure: "closure" + of nkGotoState: "gotostate" + of nkState: "state" + of nkBreakState: "breakstate" + of nkFuncDef: "func" + of nkTupleConstr: "tupleconstr" + of nkError: "err" + of nkModuleRef: "moduleref" + of nkReplayAction: "replayaction" + of nkNilRodNode: "nilrodnode" + of nkOpenSym: "opensym" + + +proc parseNodeKind*(s: string): TNodeKind = + case s + of "none": nkNone + of "empty": nkEmpty + of "ident": nkIdent + of "sym": nkSym + of "onlytype": nkType + of "charlit": nkCharLit + of "intlit": nkIntLit + of "int8lit": nkInt8Lit + of "int16lit": nkInt16Lit + of "int32lit": nkInt32Lit + of "int64lit": nkInt64Lit + of "uintlit": nkUIntLit + of "uint8lit": nkUInt8Lit + of "uint16lit": nkUInt16Lit + of "uint32lit": nkUInt32Lit + of "uint64lit": nkUInt64Lit + of "floatlit": nkFloatLit + of "float32lit": nkFloat32Lit + of "float64lit": nkFloat64Lit + of "float128lit": nkFloat128Lit + of "strlit": nkStrLit + of "rstrlit": nkRStrLit + of "triplestrlit": nkTripleStrLit + of "nil": nkNilLit + of "comesfrom": nkComesFrom + of "dotcall": nkDotCall + of "cmd": nkCommand + of "call": nkCall + of "callstrlit": nkCallStrLit + of "infix": nkInfix + of "prefix": nkPrefix + of "postfix": nkPostfix + of "hcallconv": nkHiddenCallConv + of "vv": nkExprEqExpr + of "kv": nkExprColonExpr + of "identdefs": nkIdentDefs + of "vartuple": nkVarTuple + of "par": nkPar + of "objconstr": nkObjConstr + of "curly": nkCurly + of "curlyx": nkCurlyExpr + of "bracket": nkBracket + of "at": nkBracketExpr + of "pragmax": nkPragmaExpr + of "range": nkRange + of "dot": nkDotExpr + of "checkedfieldx": nkCheckedFieldExpr + of "deref": nkDerefExpr + of "ifx": nkIfExpr + of "elifx": nkElifExpr + of "elsex": nkElseExpr + of "lambda": nkLambda + of "do": nkDo + of "accquoted": nkAccQuoted + of "tableconstr": nkTableConstr + of "bind": nkBind + of "closedsymchoice": nkClosedSymChoice + of "opensymchoice": nkOpenSymChoice + of "hstdconv": nkHiddenStdConv + of "hsubconv": nkHiddenSubConv + of "conv": nkConv + of "cast": nkCast + of "staticx": nkStaticExpr + of "addr": nkAddr + of "haddr": nkHiddenAddr + of "hderef": nkHiddenDeref + of "objdownconv": nkObjDownConv + of "objupconv": nkObjUpConv + of "chckrangef": nkChckRangeF + of "chckrange64": nkChckRange64 + of "chckrange": nkChckRange + of "stringtocstring": nkStringToCString + of "cstringtostring": nkCStringToString + of "asgn": nkAsgn + of "fastasgn": nkFastAsgn + of "genericparams": nkGenericParams + of "formalparams": nkFormalParams + of "ofinherit": nkOfInherit + of "importas": nkImportAs + of "proc": nkProcDef + of "method": nkMethodDef + of "converter": nkConverterDef + of "macro": nkMacroDef + of "template": nkTemplateDef + of "iterator": nkIteratorDef + of "of": nkOfBranch + of "elif": nkElifBranch + of "except": nkExceptBranch + of "else": nkElse + of "asm": nkAsmStmt + of "pragma": nkPragma + of "pragmablock": nkPragmaBlock + of "if": nkIfStmt + of "when": nkWhenStmt + of "for": nkForStmt + of "parfor": nkParForStmt + of "while": nkWhileStmt + of "case": nkCaseStmt + of "type": nkTypeSection + of "var": nkVarSection + of "let": nkLetSection + of "const": nkConstSection + of "const0": nkConstDef + of "type0": nkTypeDef + of "yield": nkYieldStmt + of "defer": nkDefer + of "try": nkTryStmt + of "finally": nkFinally + of "raise": nkRaiseStmt + of "ret": nkReturnStmt + of "brk": nkBreakStmt + of "continue": nkContinueStmt + of "block": nkBlockStmt + of "static": nkStaticStmt + of "discard": nkDiscardStmt + of "stmts": nkStmtList + of "import": nkImportStmt + of "importexcept": nkImportExceptStmt + of "export": nkExportStmt + of "exportexcept": nkExportExceptStmt + of "from": nkFromStmt + of "include": nkIncludeStmt + of "bind0": nkBindStmt + of "mixin": nkMixinStmt + of "using": nkUsingStmt + of "comment": nkCommentStmt + of "expr": nkStmtListExpr + of "blockx": nkBlockExpr + of "stmtlisttype": nkStmtListType + of "blocktype": nkBlockType + of "with": nkWith + of "without": nkWithout + of "typeofx": nkTypeOfExpr + of "objectty": nkObjectTy + of "tuplety": nkTupleTy + of "tupleclassty": nkTupleClassTy + of "typeclassty": nkTypeClassTy + of "staticty": nkStaticTy + of "reclist": nkRecList + of "reccase": nkRecCase + of "recwhen": nkRecWhen + of "refty": nkRefTy + of "ptrty": nkPtrTy + of "varty": nkVarTy + of "constty": nkConstTy + of "outty": nkOutTy + of "distinctty": nkDistinctTy + of "procty": nkProcTy + of "iteratorty": nkIteratorTy + of "sinkasgn": nkSinkAsgn + of "enumty": nkEnumTy + of "efld": nkEnumFieldDef + of "arglist": nkArgList + of "pattern": nkPattern + of "htrystmt": nkHiddenTryStmt + of "closure": nkClosure + of "gotostate": nkGotoState + of "state": nkState + of "breakstate": nkBreakState + of "func": nkFuncDef + of "tupleconstr": nkTupleConstr + of "err": nkError + of "moduleref": nkModuleRef + of "replayaction": nkReplayAction + of "nilrodnode": nkNilRodNode + of "opensym": nkOpenSym + else: nkNone + + +proc toNifTag*(s: TSymKind): string = + case s + of skUnknown: "unknown" + of skConditional: "conditional" + of skDynLib: "dynlib" + of skParam: "param" + of skGenericParam: "genericparam" + of skTemp: "temp" + of skModule: "module" + of skType: "type" + of skVar: "var" + of skLet: "let" + of skConst: "const" + of skResult: "result" + of skProc: "proc" + of skFunc: "func" + of skMethod: "method" + of skIterator: "iterator" + of skConverter: "converter" + of skMacro: "macro" + of skTemplate: "template" + of skField: "field" + of skEnumField: "enumfield" + of skForVar: "forvar" + of skLabel: "label" + of skStub: "stub" + of skPackage: "package" + + +proc parseSymKind*(s: string): TSymKind = + case s + of "unknown": skUnknown + of "conditional": skConditional + of "dynlib": skDynLib + of "param": skParam + of "genericparam": skGenericParam + of "temp": skTemp + of "module": skModule + of "type": skType + of "var": skVar + of "let": skLet + of "const": skConst + of "result": skResult + of "proc": skProc + of "func": skFunc + of "method": skMethod + of "iterator": skIterator + of "converter": skConverter + of "macro": skMacro + of "template": skTemplate + of "field": skField + of "enumfield": skEnumField + of "forvar": skForVar + of "label": skLabel + of "stub": skStub + of "package": skPackage + else: skUnknown + + +proc toNifTag*(s: TTypeKind): string = + case s + of tyNone: "none" + of tyBool: "bool" + of tyChar: "char" + of tyEmpty: "empty" + of tyAlias: "alias" + of tyNil: "nil" + of tyUntyped: "untyped" + of tyTyped: "typed" + of tyTypeDesc: "typedesc" + of tyGenericInvocation: "ginvoke" + of tyGenericBody: "gbody" + of tyGenericInst: "ginst" + of tyGenericParam: "gparam" + of tyDistinct: "distinct" + of tyEnum: "enum" + of tyOrdinal: "ordinal" + of tyArray: "array" + of tyObject: "object" + of tyTuple: "tuple" + of tySet: "set" + of tyRange: "range" + of tyPtr: "ptr" + of tyRef: "ref" + of tyVar: "mut" + of tySequence: "seq" + of tyProc: "proctype" + of tyPointer: "pointer" + of tyOpenArray: "openarray" + of tyString: "string" + of tyCstring: "cstring" + of tyForward: "forward" + of tyInt: "int" + of tyInt8: "int8" + of tyInt16: "int16" + of tyInt32: "int32" + of tyInt64: "int64" + of tyFloat: "float" + of tyFloat32: "float32" + of tyFloat64: "float64" + of tyFloat128: "float128" + of tyUInt: "uint" + of tyUInt8: "uint8" + of tyUInt16: "uint16" + of tyUInt32: "uint32" + of tyUInt64: "uint64" + of tyOwned: "owned" + of tySink: "sink" + of tyLent: "lent" + of tyVarargs: "varargs" + of tyUncheckedArray: "uarray" + of tyError: "error" + of tyBuiltInTypeClass: "bconcept" + of tyUserTypeClass: "uconcept" + of tyUserTypeClassInst: "uconceptinst" + of tyCompositeTypeClass: "cconcept" + of tyInferred: "inferred" + of tyAnd: "and" + of tyOr: "or" + of tyNot: "not" + of tyAnything: "anything" + of tyStatic: "static" + of tyFromExpr: "fromx" + of tyConcept: "concept" + of tyVoid: "void" + of tyIterable: "iterable" + + +proc parseTypeKind*(s: string): TTypeKind = + case s + of "none": tyNone + of "bool": tyBool + of "char": tyChar + of "empty": tyEmpty + of "alias": tyAlias + of "nil": tyNil + of "untyped": tyUntyped + of "typed": tyTyped + of "typedesc": tyTypeDesc + of "ginvoke": tyGenericInvocation + of "gbody": tyGenericBody + of "ginst": tyGenericInst + of "gparam": tyGenericParam + of "distinct": tyDistinct + of "enum": tyEnum + of "ordinal": tyOrdinal + of "array": tyArray + of "object": tyObject + of "tuple": tyTuple + of "set": tySet + of "range": tyRange + of "ptr": tyPtr + of "ref": tyRef + of "mut": tyVar + of "seq": tySequence + of "proctype": tyProc + of "pointer": tyPointer + of "openarray": tyOpenArray + of "string": tyString + of "cstring": tyCstring + of "forward": tyForward + of "int": tyInt + of "int8": tyInt8 + of "int16": tyInt16 + of "int32": tyInt32 + of "int64": tyInt64 + of "float": tyFloat + of "float32": tyFloat32 + of "float64": tyFloat64 + of "float128": tyFloat128 + of "uint": tyUInt + of "uint8": tyUInt8 + of "uint16": tyUInt16 + of "uint32": tyUInt32 + of "uint64": tyUInt64 + of "owned": tyOwned + of "sink": tySink + of "lent": tyLent + of "varargs": tyVarargs + of "uarray": tyUncheckedArray + of "error": tyError + of "bconcept": tyBuiltInTypeClass + of "uconcept": tyUserTypeClass + of "uconceptinst": tyUserTypeClassInst + of "cconcept": tyCompositeTypeClass + of "inferred": tyInferred + of "and": tyAnd + of "or": tyOr + of "not": tyNot + of "anything": tyAnything + of "static": tyStatic + of "fromx": tyFromExpr + of "concept": tyConcept + of "void": tyVoid + of "iterable": tyIterable + else: tyNone + + +proc toNifTag*(s: TLocKind): string = + case s + of locNone: "none" + of locTemp: "temp" + of locLocalVar: "localvar" + of locGlobalVar: "globalvar" + of locParam: "param" + of locField: "field" + of locExpr: "expr" + of locProc: "proc" + of locData: "data" + of locCall: "call" + of locOther: "other" + + +proc parseLocKind*(s: string): TLocKind = + case s + of "none": locNone + of "temp": locTemp + of "localvar": locLocalVar + of "globalvar": locGlobalVar + of "param": locParam + of "field": locField + of "expr": locExpr + of "proc": locProc + of "data": locData + of "call": locCall + of "other": locOther + else: locNone + + +proc toNifTag*(s: TCallingConvention): string = + case s + of ccNimCall: "nimcall" + of ccStdCall: "stdcall" + of ccCDecl: "cdecl" + of ccSafeCall: "safecall" + of ccSysCall: "syscall" + of ccInline: "inline" + of ccNoInline: "noinline" + of ccFastCall: "fastcall" + of ccThisCall: "thiscall" + of ccClosure: "closure" + of ccNoConvention: "noconv" + of ccMember: "member" + + +proc parseCallingConvention*(s: string): TCallingConvention = + case s + of "nimcall": ccNimCall + of "stdcall": ccStdCall + of "cdecl": ccCDecl + of "safecall": ccSafeCall + of "syscall": ccSysCall + of "inline": ccInline + of "noinline": ccNoInline + of "fastcall": ccFastCall + of "thiscall": ccThisCall + of "closure": ccClosure + of "noconv": ccNoConvention + of "member": ccMember + else: ccNimCall + + +proc toNifTag*(s: TMagic): string = + case s + of mNone: "nonem" + of mDefined: "defined" + of mDeclared: "declared" + of mDeclaredInScope: "declaredinscope" + of mCompiles: "compiles" + of mArrGet: "arrget" + of mArrPut: "arrput" + of mAsgn: "asgnm" + of mLow: "low" + of mHigh: "high" + of mSizeOf: "sizeof" + of mAlignOf: "alignof" + of mOffsetOf: "offsetof" + of mTypeTrait: "typetrait" + of mIs: "is" + of mOf: "ofm" + of mAddr: "addrm" + of mType: "typem" + of mTypeOf: "typeof" + of mPlugin: "plugin" + of mEcho: "echo" + of mShallowCopy: "shallowcopy" + of mSlurp: "slurp" + of mStaticExec: "staticexec" + of mStatic: "staticm" + of mParseExprToAst: "parseexprtoast" + of mParseStmtToAst: "parsestmttoast" + of mExpandToAst: "expandtoast" + of mQuoteAst: "quoteast" + of mInc: "inc" + of mDec: "dec" + of mOrd: "ord" + of mNew: "new" + of mNewFinalize: "newfinalize" + of mNewSeq: "newseq" + of mNewSeqOfCap: "newseqofcap" + of mLengthOpenArray: "lenopenarray" + of mLengthStr: "lenstr" + of mLengthArray: "lenarray" + of mLengthSeq: "lenseq" + of mIncl: "incl" + of mExcl: "excl" + of mCard: "card" + of mChr: "chr" + of mGCref: "gcref" + of mGCunref: "gcunref" + of mAddI: "add" + of mSubI: "sub" + of mMulI: "mul" + of mDivI: "div" + of mModI: "mod" + of mSucc: "succ" + of mPred: "pred" + of mAddF64: "addf64" + of mSubF64: "subf64" + of mMulF64: "mulf64" + of mDivF64: "divf64" + of mShrI: "shr" + of mShlI: "shl" + of mAshrI: "ashr" + of mBitandI: "bitand" + of mBitorI: "bitor" + of mBitxorI: "bitxor" + of mMinI: "min" + of mMaxI: "max" + of mAddU: "addu" + of mSubU: "subu" + of mMulU: "mulu" + of mDivU: "divu" + of mModU: "modu" + of mEqI: "eq" + of mLeI: "le" + of mLtI: "lt" + of mEqF64: "eqf64" + of mLeF64: "lef64" + of mLtF64: "ltf64" + of mLeU: "leu" + of mLtU: "ltu" + of mEqEnum: "eqenum" + of mLeEnum: "leenum" + of mLtEnum: "ltenum" + of mEqCh: "eqch" + of mLeCh: "lech" + of mLtCh: "ltch" + of mEqB: "eqb" + of mLeB: "leb" + of mLtB: "ltb" + of mEqRef: "eqref" + of mLePtr: "leptr" + of mLtPtr: "ltptr" + of mXor: "xor" + of mEqCString: "eqcstring" + of mEqProc: "eqproc" + of mUnaryMinusI: "unaryminus" + of mUnaryMinusI64: "unaryminusi64" + of mAbsI: "abs" + of mNot: "not" + of mUnaryPlusI: "unaryplus" + of mBitnotI: "bitnot" + of mUnaryPlusF64: "unaryplusf64" + of mUnaryMinusF64: "unaryminusf64" + of mCharToStr: "chartostr" + of mBoolToStr: "booltostr" + of mCStrToStr: "cstrtostr" + of mStrToStr: "strtostr" + of mEnumToStr: "enumtostr" + of mAnd: "and" + of mOr: "or" + of mImplies: "implies" + of mIff: "iff" + of mExists: "exists" + of mForall: "forall" + of mOld: "old" + of mEqStr: "eqstr" + of mLeStr: "lestr" + of mLtStr: "ltstr" + of mEqSet: "eqset" + of mLeSet: "leset" + of mLtSet: "ltset" + of mMulSet: "mulset" + of mPlusSet: "plusset" + of mMinusSet: "minusset" + of mXorSet: "xorset" + of mConStrStr: "constrstr" + of mSlice: "slice" + of mDotDot: "dotdot" + of mFields: "fields" + of mFieldPairs: "fieldpairs" + of mOmpParFor: "ompparfor" + of mAppendStrCh: "addstrch" + of mAppendStrStr: "addstrstr" + of mAppendSeqElem: "addseqelem" + of mInSet: "contains" + of mRepr: "repr" + of mExit: "exit" + of mSetLengthStr: "setlenstr" + of mSetLengthSeq: "setlenseq" + of mSetLengthSeqUninit: "setlensequninit" + of mIsPartOf: "ispartof" + of mAstToStr: "asttostr" + of mParallel: "parallel" + of mSwap: "swap" + of mIsNil: "isnil" + of mArrToSeq: "arrtoseq" + of mOpenArrayToSeq: "openarraytoseq" + of mNewString: "newstring" + of mNewStringOfCap: "newstringofcap" + of mParseBiggestFloat: "parsebiggestfloat" + of mMove: "move" + of mEnsureMove: "ensuremove" + of mWasMoved: "wasmoved" + of mDup: "dup" + of mDestroy: "destroy" + of mTrace: "trace" + of mDefault: "default" + of mUnown: "unown" + of mFinished: "finished" + of mIsolate: "isolate" + of mAccessEnv: "accessenv" + of mAccessTypeField: "accesstypefield" + of mArray: "array" + of mOpenArray: "openarray" + of mRange: "rangem" + of mSet: "set" + of mSeq: "seq" + of mVarargs: "varargs" + of mRef: "ref" + of mPtr: "ptr" + of mVar: "varm" + of mDistinct: "distinct" + of mVoid: "void" + of mTuple: "tuple" + of mOrdinal: "ordinal" + of mIterableType: "iterabletype" + of mInt: "int" + of mInt8: "int8" + of mInt16: "int16" + of mInt32: "int32" + of mInt64: "int64" + of mUInt: "uint" + of mUInt8: "uint8" + of mUInt16: "uint16" + of mUInt32: "uint32" + of mUInt64: "uint64" + of mFloat: "float" + of mFloat32: "float32" + of mFloat64: "float64" + of mFloat128: "float128" + of mBool: "bool" + of mChar: "char" + of mString: "string" + of mCstring: "cstring" + of mPointer: "pointer" + of mNil: "nilm" + of mExpr: "exprm" + of mStmt: "stmtm" + of mTypeDesc: "typedesc" + of mVoidType: "voidtype" + of mPNimrodNode: "nimnode" + of mSpawn: "spawn" + of mDeepCopy: "deepcopy" + of mIsMainModule: "ismainmodule" + of mCompileDate: "compiledate" + of mCompileTime: "compiletime" + of mProcCall: "proccall" + of mCpuEndian: "cpuendian" + of mHostOS: "hostos" + of mHostCPU: "hostcpu" + of mBuildOS: "buildos" + of mBuildCPU: "buildcpu" + of mAppType: "apptype" + of mCompileOption: "compileoption" + of mCompileOptionArg: "compileoptionarg" + of mNLen: "nlen" + of mNChild: "nchild" + of mNSetChild: "nsetchild" + of mNAdd: "nadd" + of mNAddMultiple: "naddmultiple" + of mNDel: "ndel" + of mNKind: "nkind" + of mNSymKind: "nsymkind" + of mNccValue: "nccvalue" + of mNccInc: "nccinc" + of mNcsAdd: "ncsadd" + of mNcsIncl: "ncsincl" + of mNcsLen: "ncslen" + of mNcsAt: "ncsat" + of mNctPut: "nctput" + of mNctLen: "nctlen" + of mNctGet: "nctget" + of mNctHasNext: "ncthasnext" + of mNctNext: "nctnext" + of mNIntVal: "nintval" + of mNFloatVal: "nfloatval" + of mNSymbol: "nsymbol" + of mNIdent: "nident" + of mNGetType: "ngettype" + of mNStrVal: "nstrval" + of mNSetIntVal: "nsetintval" + of mNSetFloatVal: "nsetfloatval" + of mNSetSymbol: "nsetsymbol" + of mNSetIdent: "nsetident" + of mNSetStrVal: "nsetstrval" + of mNLineInfo: "nlineinfo" + of mNNewNimNode: "nnewnimnode" + of mNCopyNimNode: "ncopynimnode" + of mNCopyNimTree: "ncopynimtree" + of mStrToIdent: "strtoident" + of mNSigHash: "nsighash" + of mNSizeOf: "nsizeof" + of mNBindSym: "nbindsym" + of mNCallSite: "ncallsite" + of mEqIdent: "eqident" + of mEqNimrodNode: "eqnimnode" + of mSameNodeType: "samenodetype" + of mGetImpl: "getimpl" + of mNGenSym: "ngensym" + of mNHint: "nhint" + of mNWarning: "nwarning" + of mNError: "nerror" + of mInstantiationInfo: "instantiationinfo" + of mGetTypeInfo: "gettypeinfo" + of mGetTypeInfoV2: "gettypeinfov2" + of mNimvm: "nimvm" + of mIntDefine: "intdefine" + of mStrDefine: "strdefine" + of mBoolDefine: "booldefine" + of mGenericDefine: "genericdefine" + of mRunnableExamples: "runnableexamples" + of mException: "exception" + of mBuiltinType: "builtintype" + of mSymOwner: "symowner" + of mUncheckedArray: "uncheckedarray" + of mGetImplTransf: "getimpltransf" + of mSymIsInstantiationOf: "symisinstantiationof" + of mNodeId: "nodeid" + of mPrivateAccess: "privateaccess" + of mZeroDefault: "zerodefault" + + +proc parseMagic*(s: string): TMagic = + case s + of "nonem": mNone + of "defined": mDefined + of "declared": mDeclared + of "declaredinscope": mDeclaredInScope + of "compiles": mCompiles + of "arrget": mArrGet + of "arrput": mArrPut + of "asgnm": mAsgn + of "low": mLow + of "high": mHigh + of "sizeof": mSizeOf + of "alignof": mAlignOf + of "offsetof": mOffsetOf + of "typetrait": mTypeTrait + of "is": mIs + of "ofm": mOf + of "addrm": mAddr + of "typem": mType + of "typeof": mTypeOf + of "plugin": mPlugin + of "echo": mEcho + of "shallowcopy": mShallowCopy + of "slurp": mSlurp + of "staticexec": mStaticExec + of "staticm": mStatic + of "parseexprtoast": mParseExprToAst + of "parsestmttoast": mParseStmtToAst + of "expandtoast": mExpandToAst + of "quoteast": mQuoteAst + of "inc": mInc + of "dec": mDec + of "ord": mOrd + of "new": mNew + of "newfinalize": mNewFinalize + of "newseq": mNewSeq + of "newseqofcap": mNewSeqOfCap + of "lenopenarray": mLengthOpenArray + of "lenstr": mLengthStr + of "lenarray": mLengthArray + of "lenseq": mLengthSeq + of "incl": mIncl + of "excl": mExcl + of "card": mCard + of "chr": mChr + of "gcref": mGCref + of "gcunref": mGCunref + of "add": mAddI + of "sub": mSubI + of "mul": mMulI + of "div": mDivI + of "mod": mModI + of "succ": mSucc + of "pred": mPred + of "addf64": mAddF64 + of "subf64": mSubF64 + of "mulf64": mMulF64 + of "divf64": mDivF64 + of "shr": mShrI + of "shl": mShlI + of "ashr": mAshrI + of "bitand": mBitandI + of "bitor": mBitorI + of "bitxor": mBitxorI + of "min": mMinI + of "max": mMaxI + of "addu": mAddU + of "subu": mSubU + of "mulu": mMulU + of "divu": mDivU + of "modu": mModU + of "eq": mEqI + of "le": mLeI + of "lt": mLtI + of "eqf64": mEqF64 + of "lef64": mLeF64 + of "ltf64": mLtF64 + of "leu": mLeU + of "ltu": mLtU + of "eqenum": mEqEnum + of "leenum": mLeEnum + of "ltenum": mLtEnum + of "eqch": mEqCh + of "lech": mLeCh + of "ltch": mLtCh + of "eqb": mEqB + of "leb": mLeB + of "ltb": mLtB + of "eqref": mEqRef + of "leptr": mLePtr + of "ltptr": mLtPtr + of "xor": mXor + of "eqcstring": mEqCString + of "eqproc": mEqProc + of "unaryminus": mUnaryMinusI + of "unaryminusi64": mUnaryMinusI64 + of "abs": mAbsI + of "not": mNot + of "unaryplus": mUnaryPlusI + of "bitnot": mBitnotI + of "unaryplusf64": mUnaryPlusF64 + of "unaryminusf64": mUnaryMinusF64 + of "chartostr": mCharToStr + of "booltostr": mBoolToStr + of "cstrtostr": mCStrToStr + of "strtostr": mStrToStr + of "enumtostr": mEnumToStr + of "and": mAnd + of "or": mOr + of "implies": mImplies + of "iff": mIff + of "exists": mExists + of "forall": mForall + of "old": mOld + of "eqstr": mEqStr + of "lestr": mLeStr + of "ltstr": mLtStr + of "eqset": mEqSet + of "leset": mLeSet + of "ltset": mLtSet + of "mulset": mMulSet + of "plusset": mPlusSet + of "minusset": mMinusSet + of "xorset": mXorSet + of "constrstr": mConStrStr + of "slice": mSlice + of "dotdot": mDotDot + of "fields": mFields + of "fieldpairs": mFieldPairs + of "ompparfor": mOmpParFor + of "addstrch": mAppendStrCh + of "addstrstr": mAppendStrStr + of "addseqelem": mAppendSeqElem + of "contains": mInSet + of "repr": mRepr + of "exit": mExit + of "setlenstr": mSetLengthStr + of "setlenseq": mSetLengthSeq + of "setlensequninit": mSetLengthSeqUninit + of "ispartof": mIsPartOf + of "asttostr": mAstToStr + of "parallel": mParallel + of "swap": mSwap + of "isnil": mIsNil + of "arrtoseq": mArrToSeq + of "openarraytoseq": mOpenArrayToSeq + of "newstring": mNewString + of "newstringofcap": mNewStringOfCap + of "parsebiggestfloat": mParseBiggestFloat + of "move": mMove + of "ensuremove": mEnsureMove + of "wasmoved": mWasMoved + of "dup": mDup + of "destroy": mDestroy + of "trace": mTrace + of "default": mDefault + of "unown": mUnown + of "finished": mFinished + of "isolate": mIsolate + of "accessenv": mAccessEnv + of "accesstypefield": mAccessTypeField + of "array": mArray + of "openarray": mOpenArray + of "rangem": mRange + of "set": mSet + of "seq": mSeq + of "varargs": mVarargs + of "ref": mRef + of "ptr": mPtr + of "varm": mVar + of "distinct": mDistinct + of "void": mVoid + of "tuple": mTuple + of "ordinal": mOrdinal + of "iterabletype": mIterableType + of "int": mInt + of "int8": mInt8 + of "int16": mInt16 + of "int32": mInt32 + of "int64": mInt64 + of "uint": mUInt + of "uint8": mUInt8 + of "uint16": mUInt16 + of "uint32": mUInt32 + of "uint64": mUInt64 + of "float": mFloat + of "float32": mFloat32 + of "float64": mFloat64 + of "float128": mFloat128 + of "bool": mBool + of "char": mChar + of "string": mString + of "cstring": mCstring + of "pointer": mPointer + of "nilm": mNil + of "exprm": mExpr + of "stmtm": mStmt + of "typedesc": mTypeDesc + of "voidtype": mVoidType + of "nimnode": mPNimrodNode + of "spawn": mSpawn + of "deepcopy": mDeepCopy + of "ismainmodule": mIsMainModule + of "compiledate": mCompileDate + of "compiletime": mCompileTime + of "proccall": mProcCall + of "cpuendian": mCpuEndian + of "hostos": mHostOS + of "hostcpu": mHostCPU + of "buildos": mBuildOS + of "buildcpu": mBuildCPU + of "apptype": mAppType + of "compileoption": mCompileOption + of "compileoptionarg": mCompileOptionArg + of "nlen": mNLen + of "nchild": mNChild + of "nsetchild": mNSetChild + of "nadd": mNAdd + of "naddmultiple": mNAddMultiple + of "ndel": mNDel + of "nkind": mNKind + of "nsymkind": mNSymKind + of "nccvalue": mNccValue + of "nccinc": mNccInc + of "ncsadd": mNcsAdd + of "ncsincl": mNcsIncl + of "ncslen": mNcsLen + of "ncsat": mNcsAt + of "nctput": mNctPut + of "nctlen": mNctLen + of "nctget": mNctGet + of "ncthasnext": mNctHasNext + of "nctnext": mNctNext + of "nintval": mNIntVal + of "nfloatval": mNFloatVal + of "nsymbol": mNSymbol + of "nident": mNIdent + of "ngettype": mNGetType + of "nstrval": mNStrVal + of "nsetintval": mNSetIntVal + of "nsetfloatval": mNSetFloatVal + of "nsetsymbol": mNSetSymbol + of "nsetident": mNSetIdent + of "nsetstrval": mNSetStrVal + of "nlineinfo": mNLineInfo + of "nnewnimnode": mNNewNimNode + of "ncopynimnode": mNCopyNimNode + of "ncopynimtree": mNCopyNimTree + of "strtoident": mStrToIdent + of "nsighash": mNSigHash + of "nsizeof": mNSizeOf + of "nbindsym": mNBindSym + of "ncallsite": mNCallSite + of "eqident": mEqIdent + of "eqnimnode": mEqNimrodNode + of "samenodetype": mSameNodeType + of "getimpl": mGetImpl + of "ngensym": mNGenSym + of "nhint": mNHint + of "nwarning": mNWarning + of "nerror": mNError + of "instantiationinfo": mInstantiationInfo + of "gettypeinfo": mGetTypeInfo + of "gettypeinfov2": mGetTypeInfoV2 + of "nimvm": mNimvm + of "intdefine": mIntDefine + of "strdefine": mStrDefine + of "booldefine": mBoolDefine + of "genericdefine": mGenericDefine + of "runnableexamples": mRunnableExamples + of "exception": mException + of "builtintype": mBuiltinType + of "symowner": mSymOwner + of "uncheckedarray": mUncheckedArray + of "getimpltransf": mGetImplTransf + of "symisinstantiationof": mSymIsInstantiationOf + of "nodeid": mNodeId + of "privateaccess": mPrivateAccess + of "zerodefault": mZeroDefault + else: mNone + + +proc genFlags*(s: set[TSymFlag]; dest: var string) = + for e in s: + case e + of sfUsed: dest.add "u" + of sfExported: dest.add "e" + of sfFromGeneric: dest.add "f" + of sfGlobal: dest.add "g" + of sfForward: dest.add "f0" + of sfWasForwarded: dest.add "w" + of sfImportc: dest.add "i" + of sfExportc: dest.add "e0" + of sfMangleCpp: dest.add "m" + of sfVolatile: dest.add "v" + of sfRegister: dest.add "r" + of sfPure: dest.add "p" + of sfNoSideEffect: dest.add "n" + of sfSideEffect: dest.add "s" + of sfMainModule: dest.add "m0" + of sfSystemModule: dest.add "s0" + of sfNoReturn: dest.add "n0" + of sfAddrTaken: dest.add "a" + of sfCompilerProc: dest.add "c" + of sfEscapes: dest.add "e1" + of sfDiscriminant: dest.add "d" + of sfRequiresInit: dest.add "r0" + of sfDeprecated: dest.add "d0" + of sfExplain: dest.add "e2" + of sfError: dest.add "e3" + of sfShadowed: dest.add "s1" + of sfThread: dest.add "t" + of sfCppNonPod: dest.add "c0" + of sfCompileTime: dest.add "c1" + of sfConstructor: dest.add "c2" + of sfDispatcher: dest.add "d1" + of sfBorrow: dest.add "b" + of sfInfixCall: dest.add "i0" + of sfNamedParamCall: dest.add "n1" + of sfDiscardable: dest.add "d2" + of sfOverridden: dest.add "o" + of sfCallsite: dest.add "c3" + of sfGenSym: dest.add "g0" + of sfNonReloadable: dest.add "n2" + of sfGeneratedOp: dest.add "g1" + of sfTemplateParam: dest.add "t0" + of sfCursor: dest.add "c4" + of sfInjectDestructors: dest.add "i1" + of sfNeverRaises: dest.add "n3" + of sfSystemRaisesDefect: dest.add "s2" + of sfUsedInFinallyOrExcept: dest.add "u0" + of sfSingleUsedTemp: dest.add "s3" + of sfNoalias: dest.add "n4" + of sfEffectsDelayed: dest.add "e4" + of sfGeneratedType: dest.add "g2" + of sfVirtual: dest.add "v0" + of sfByCopy: dest.add "b0" + of sfMember: dest.add "m1" + of sfCodegenDecl: dest.add "c5" + of sfWasGenSym: dest.add "w0" + of sfForceLift: dest.add "l" + of sfDirty: dest.add "d3" + of sfCustomPragma: dest.add "c6" + of sfBase: dest.add "b1" + of sfGoto: dest.add "g3" + of sfAnon: dest.add "a0" + of sfAllUntyped: dest.add "a1" + of sfTemplateRedefinition: dest.add "t1" + + +proc parseSymFlags*(s: string): set[TSymFlag] = + result = {} + var i = 0 + while i < s.len: + case s[i] + of 'a': + if i+1 < s.len and s[i+1] == '0': + result.incl sfAnon + inc i + elif i+1 < s.len and s[i+1] == '1': + result.incl sfAllUntyped + inc i + else: result.incl sfAddrTaken + of 'b': + if i+1 < s.len and s[i+1] == '0': + result.incl sfByCopy + inc i + elif i+1 < s.len and s[i+1] == '1': + result.incl sfBase + inc i + else: result.incl sfBorrow + of 'c': + if i+1 < s.len and s[i+1] == '0': + result.incl sfCppNonPod + inc i + elif i+1 < s.len and s[i+1] == '1': + result.incl sfCompileTime + inc i + elif i+1 < s.len and s[i+1] == '2': + result.incl sfConstructor + inc i + elif i+1 < s.len and s[i+1] == '3': + result.incl sfCallsite + inc i + elif i+1 < s.len and s[i+1] == '4': + result.incl sfCursor + inc i + elif i+1 < s.len and s[i+1] == '5': + result.incl sfCodegenDecl + inc i + elif i+1 < s.len and s[i+1] == '6': + result.incl sfCustomPragma + inc i + else: result.incl sfCompilerProc + of 'd': + if i+1 < s.len and s[i+1] == '0': + result.incl sfDeprecated + inc i + elif i+1 < s.len and s[i+1] == '1': + result.incl sfDispatcher + inc i + elif i+1 < s.len and s[i+1] == '2': + result.incl sfDiscardable + inc i + elif i+1 < s.len and s[i+1] == '3': + result.incl sfDirty + inc i + else: result.incl sfDiscriminant + of 'e': + if i+1 < s.len and s[i+1] == '0': + result.incl sfExportc + inc i + elif i+1 < s.len and s[i+1] == '1': + result.incl sfEscapes + inc i + elif i+1 < s.len and s[i+1] == '2': + result.incl sfExplain + inc i + elif i+1 < s.len and s[i+1] == '3': + result.incl sfError + inc i + elif i+1 < s.len and s[i+1] == '4': + result.incl sfEffectsDelayed + inc i + else: result.incl sfExported + of 'f': + if i+1 < s.len and s[i+1] == '0': + result.incl sfForward + inc i + else: result.incl sfFromGeneric + of 'g': + if i+1 < s.len and s[i+1] == '0': + result.incl sfGenSym + inc i + elif i+1 < s.len and s[i+1] == '1': + result.incl sfGeneratedOp + inc i + elif i+1 < s.len and s[i+1] == '2': + result.incl sfGeneratedType + inc i + elif i+1 < s.len and s[i+1] == '3': + result.incl sfGoto + inc i + else: result.incl sfGlobal + of 'i': + if i+1 < s.len and s[i+1] == '0': + result.incl sfInfixCall + inc i + elif i+1 < s.len and s[i+1] == '1': + result.incl sfInjectDestructors + inc i + else: result.incl sfImportc + of 'l': result.incl sfForceLift + of 'm': + if i+1 < s.len and s[i+1] == '0': + result.incl sfMainModule + inc i + elif i+1 < s.len and s[i+1] == '1': + result.incl sfMember + inc i + else: result.incl sfMangleCpp + of 'n': + if i+1 < s.len and s[i+1] == '0': + result.incl sfNoReturn + inc i + elif i+1 < s.len and s[i+1] == '1': + result.incl sfNamedParamCall + inc i + elif i+1 < s.len and s[i+1] == '2': + result.incl sfNonReloadable + inc i + elif i+1 < s.len and s[i+1] == '3': + result.incl sfNeverRaises + inc i + elif i+1 < s.len and s[i+1] == '4': + result.incl sfNoalias + inc i + else: result.incl sfNoSideEffect + of 'o': result.incl sfOverridden + of 'p': result.incl sfPure + of 'r': + if i+1 < s.len and s[i+1] == '0': + result.incl sfRequiresInit + inc i + else: result.incl sfRegister + of 's': + if i+1 < s.len and s[i+1] == '0': + result.incl sfSystemModule + inc i + elif i+1 < s.len and s[i+1] == '1': + result.incl sfShadowed + inc i + elif i+1 < s.len and s[i+1] == '2': + result.incl sfSystemRaisesDefect + inc i + elif i+1 < s.len and s[i+1] == '3': + result.incl sfSingleUsedTemp + inc i + else: result.incl sfSideEffect + of 't': + if i+1 < s.len and s[i+1] == '0': + result.incl sfTemplateParam + inc i + elif i+1 < s.len and s[i+1] == '1': + result.incl sfTemplateRedefinition + inc i + else: result.incl sfThread + of 'u': + if i+1 < s.len and s[i+1] == '0': + result.incl sfUsedInFinallyOrExcept + inc i + else: result.incl sfUsed + of 'v': + if i+1 < s.len and s[i+1] == '0': + result.incl sfVirtual + inc i + else: result.incl sfVolatile + of 'w': + if i+1 < s.len and s[i+1] == '0': + result.incl sfWasGenSym + inc i + else: result.incl sfWasForwarded + else: discard + inc i + +proc genFlags*(s: set[TNodeFlag]; dest: var string) = + for e in s: + case e + of nfNone: dest.add "n" + of nfBase2: dest.add "b" + of nfBase8: dest.add "b0" + of nfBase16: dest.add "b1" + of nfAllConst: dest.add "a" + of nfTransf: dest.add "t" + of nfNoRewrite: dest.add "r" + of nfSem: dest.add "s" + of nfLL: dest.add "l" + of nfDotField: dest.add "d" + of nfDotSetter: dest.add "d0" + of nfExplicitCall: dest.add "e" + of nfExprCall: dest.add "c" + of nfIsRef: dest.add "i" + of nfIsPtr: dest.add "p" + of nfPreventCg: dest.add "p0" + of nfBlockArg: dest.add "b2" + of nfFromTemplate: dest.add "f" + of nfDefaultParam: dest.add "d1" + of nfDefaultRefsParam: dest.add "d2" + of nfExecuteOnReload: dest.add "o" + of nfLastRead: dest.add "l0" + of nfFirstWrite: dest.add "w" + of nfHasComment: dest.add "h" + of nfSkipFieldChecking: dest.add "s0" + of nfDisabledOpenSym: dest.add "d3" + + +proc parseNodeFlags*(s: string): set[TNodeFlag] = + result = {} + var i = 0 + while i < s.len: + case s[i] + of 'a': result.incl nfAllConst + of 'b': + if i+1 < s.len and s[i+1] == '0': + result.incl nfBase8 + inc i + elif i+1 < s.len and s[i+1] == '1': + result.incl nfBase16 + inc i + elif i+1 < s.len and s[i+1] == '2': + result.incl nfBlockArg + inc i + else: result.incl nfBase2 + of 'c': result.incl nfExprCall + of 'd': + if i+1 < s.len and s[i+1] == '0': + result.incl nfDotSetter + inc i + elif i+1 < s.len and s[i+1] == '1': + result.incl nfDefaultParam + inc i + elif i+1 < s.len and s[i+1] == '2': + result.incl nfDefaultRefsParam + inc i + elif i+1 < s.len and s[i+1] == '3': + result.incl nfDisabledOpenSym + inc i + else: result.incl nfDotField + of 'e': result.incl nfExplicitCall + of 'f': result.incl nfFromTemplate + of 'h': result.incl nfHasComment + of 'i': result.incl nfIsRef + of 'l': + if i+1 < s.len and s[i+1] == '0': + result.incl nfLastRead + inc i + else: result.incl nfLL + of 'n': result.incl nfNone + of 'o': result.incl nfExecuteOnReload + of 'p': + if i+1 < s.len and s[i+1] == '0': + result.incl nfPreventCg + inc i + else: result.incl nfIsPtr + of 'r': result.incl nfNoRewrite + of 's': + if i+1 < s.len and s[i+1] == '0': + result.incl nfSkipFieldChecking + inc i + else: result.incl nfSem + of 't': result.incl nfTransf + of 'w': result.incl nfFirstWrite + else: discard + inc i + +proc genFlags*(s: set[TTypeFlag]; dest: var string) = + for e in s: + case e + of tfVarargs: dest.add "v" + of tfNoSideEffect: dest.add "n" + of tfFinal: dest.add "f" + of tfInheritable: dest.add "i" + of tfHasOwned: dest.add "h" + of tfEnumHasHoles: dest.add "e" + of tfShallow: dest.add "s" + of tfThread: dest.add "t" + of tfFromGeneric: dest.add "g" + of tfUnresolved: dest.add "u" + of tfResolved: dest.add "r" + of tfRetType: dest.add "r0" + of tfCapturesEnv: dest.add "c" + of tfByCopy: dest.add "b" + of tfByRef: dest.add "b0" + of tfIterator: dest.add "i0" + of tfPartial: dest.add "p" + of tfNotNil: dest.add "n0" + of tfRequiresInit: dest.add "r1" + of tfNeedsFullInit: dest.add "n1" + of tfVarIsPtr: dest.add "v0" + of tfHasMeta: dest.add "m" + of tfHasGCedMem: dest.add "h0" + of tfPacked: dest.add "p0" + of tfHasStatic: dest.add "h1" + of tfGenericTypeParam: dest.add "g0" + of tfImplicitTypeParam: dest.add "i1" + of tfInferrableStatic: dest.add "i2" + of tfConceptMatchedTypeSym: dest.add "c0" + of tfExplicit: dest.add "e0" + of tfWildcard: dest.add "w" + of tfHasAsgn: dest.add "a" + of tfBorrowDot: dest.add "d" + of tfTriggersCompileTime: dest.add "t0" + of tfRefsAnonObj: dest.add "o" + of tfCovariant: dest.add "c1" + of tfWeakCovariant: dest.add "w0" + of tfContravariant: dest.add "c2" + of tfCheckedForDestructor: dest.add "c3" + of tfAcyclic: dest.add "a0" + of tfIncompleteStruct: dest.add "i3" + of tfCompleteStruct: dest.add "c4" + of tfExplicitCallConv: dest.add "e1" + of tfIsConstructor: dest.add "i4" + of tfEffectSystemWorkaround: dest.add "e2" + of tfIsOutParam: dest.add "i5" + of tfSendable: dest.add "s0" + of tfImplicitStatic: dest.add "i6" + + +proc parseTypeFlags*(s: string): set[TTypeFlag] = + result = {} + var i = 0 + while i < s.len: + case s[i] + of 'a': + if i+1 < s.len and s[i+1] == '0': + result.incl tfAcyclic + inc i + else: result.incl tfHasAsgn + of 'b': + if i+1 < s.len and s[i+1] == '0': + result.incl tfByRef + inc i + else: result.incl tfByCopy + of 'c': + if i+1 < s.len and s[i+1] == '0': + result.incl tfConceptMatchedTypeSym + inc i + elif i+1 < s.len and s[i+1] == '1': + result.incl tfCovariant + inc i + elif i+1 < s.len and s[i+1] == '2': + result.incl tfContravariant + inc i + elif i+1 < s.len and s[i+1] == '3': + result.incl tfCheckedForDestructor + inc i + elif i+1 < s.len and s[i+1] == '4': + result.incl tfCompleteStruct + inc i + else: result.incl tfCapturesEnv + of 'd': result.incl tfBorrowDot + of 'e': + if i+1 < s.len and s[i+1] == '0': + result.incl tfExplicit + inc i + elif i+1 < s.len and s[i+1] == '1': + result.incl tfExplicitCallConv + inc i + elif i+1 < s.len and s[i+1] == '2': + result.incl tfEffectSystemWorkaround + inc i + else: result.incl tfEnumHasHoles + of 'f': result.incl tfFinal + of 'g': + if i+1 < s.len and s[i+1] == '0': + result.incl tfGenericTypeParam + inc i + else: result.incl tfFromGeneric + of 'h': + if i+1 < s.len and s[i+1] == '0': + result.incl tfHasGCedMem + inc i + elif i+1 < s.len and s[i+1] == '1': + result.incl tfHasStatic + inc i + else: result.incl tfHasOwned + of 'i': + if i+1 < s.len and s[i+1] == '0': + result.incl tfIterator + inc i + elif i+1 < s.len and s[i+1] == '1': + result.incl tfImplicitTypeParam + inc i + elif i+1 < s.len and s[i+1] == '2': + result.incl tfInferrableStatic + inc i + elif i+1 < s.len and s[i+1] == '3': + result.incl tfIncompleteStruct + inc i + elif i+1 < s.len and s[i+1] == '4': + result.incl tfIsConstructor + inc i + elif i+1 < s.len and s[i+1] == '5': + result.incl tfIsOutParam + inc i + elif i+1 < s.len and s[i+1] == '6': + result.incl tfImplicitStatic + inc i + else: result.incl tfInheritable + of 'm': result.incl tfHasMeta + of 'n': + if i+1 < s.len and s[i+1] == '0': + result.incl tfNotNil + inc i + elif i+1 < s.len and s[i+1] == '1': + result.incl tfNeedsFullInit + inc i + else: result.incl tfNoSideEffect + of 'o': result.incl tfRefsAnonObj + of 'p': + if i+1 < s.len and s[i+1] == '0': + result.incl tfPacked + inc i + else: result.incl tfPartial + of 'r': + if i+1 < s.len and s[i+1] == '0': + result.incl tfRetType + inc i + elif i+1 < s.len and s[i+1] == '1': + result.incl tfRequiresInit + inc i + else: result.incl tfResolved + of 's': + if i+1 < s.len and s[i+1] == '0': + result.incl tfSendable + inc i + else: result.incl tfShallow + of 't': + if i+1 < s.len and s[i+1] == '0': + result.incl tfTriggersCompileTime + inc i + else: result.incl tfThread + of 'u': result.incl tfUnresolved + of 'v': + if i+1 < s.len and s[i+1] == '0': + result.incl tfVarIsPtr + inc i + else: result.incl tfVarargs + of 'w': + if i+1 < s.len and s[i+1] == '0': + result.incl tfWeakCovariant + inc i + else: result.incl tfWildcard + else: discard + inc i + +proc genFlags*(s: set[TLocFlag]; dest: var string) = + for e in s: + case e + of lfIndirect: dest.add "i" + of lfNoDeepCopy: dest.add "n" + of lfNoDecl: dest.add "d" + of lfDynamicLib: dest.add "l" + of lfExportLib: dest.add "e" + of lfHeader: dest.add "h" + of lfImportCompilerProc: dest.add "c" + of lfSingleUse: dest.add "s" + of lfEnforceDeref: dest.add "e0" + of lfPrepareForMutation: dest.add "p" + + +proc parseLocFlags*(s: string): set[TLocFlag] = + result = {} + var i = 0 + while i < s.len: + case s[i] + of 'c': result.incl lfImportCompilerProc + of 'd': result.incl lfNoDecl + of 'e': + if i+1 < s.len and s[i+1] == '0': + result.incl lfEnforceDeref + inc i + else: result.incl lfExportLib + of 'h': result.incl lfHeader + of 'i': result.incl lfIndirect + of 'l': result.incl lfDynamicLib + of 'n': result.incl lfNoDeepCopy + of 'p': result.incl lfPrepareForMutation + of 's': result.incl lfSingleUse + else: discard + inc i + +proc genFlags*(s: set[TOption]; dest: var string) = + for e in s: + case e + of optNone: dest.add "n" + of optObjCheck: dest.add "o" + of optFieldCheck: dest.add "f" + of optRangeCheck: dest.add "r" + of optBoundsCheck: dest.add "b" + of optOverflowCheck: dest.add "c" + of optRefCheck: dest.add "r0" + of optNaNCheck: dest.add "n0" + of optInfCheck: dest.add "i" + of optStaticBoundsCheck: dest.add "s" + of optStyleCheck: dest.add "s0" + of optAssert: dest.add "a" + of optLineDir: dest.add "l" + of optWarns: dest.add "w" + of optHints: dest.add "h" + of optOptimizeSpeed: dest.add "o0" + of optOptimizeSize: dest.add "o1" + of optStackTrace: dest.add "t" + of optStackTraceMsgs: dest.add "m" + of optLineTrace: dest.add "l0" + of optByRef: dest.add "b0" + of optProfiler: dest.add "p" + of optImplicitStatic: dest.add "i0" + of optTrMacros: dest.add "t0" + of optMemTracker: dest.add "m0" + of optSinkInference: dest.add "s1" + of optCursorInference: dest.add "c0" + of optImportHidden: dest.add "i1" + of optQuirky: dest.add "q" + + +proc parseOptions*(s: string): set[TOption] = + result = {} + var i = 0 + while i < s.len: + case s[i] + of 'a': result.incl optAssert + of 'b': + if i+1 < s.len and s[i+1] == '0': + result.incl optByRef + inc i + else: result.incl optBoundsCheck + of 'c': + if i+1 < s.len and s[i+1] == '0': + result.incl optCursorInference + inc i + else: result.incl optOverflowCheck + of 'f': result.incl optFieldCheck + of 'h': result.incl optHints + of 'i': + if i+1 < s.len and s[i+1] == '0': + result.incl optImplicitStatic + inc i + elif i+1 < s.len and s[i+1] == '1': + result.incl optImportHidden + inc i + else: result.incl optInfCheck + of 'l': + if i+1 < s.len and s[i+1] == '0': + result.incl optLineTrace + inc i + else: result.incl optLineDir + of 'm': + if i+1 < s.len and s[i+1] == '0': + result.incl optMemTracker + inc i + else: result.incl optStackTraceMsgs + of 'n': + if i+1 < s.len and s[i+1] == '0': + result.incl optNaNCheck + inc i + else: result.incl optNone + of 'o': + if i+1 < s.len and s[i+1] == '0': + result.incl optOptimizeSpeed + inc i + elif i+1 < s.len and s[i+1] == '1': + result.incl optOptimizeSize + inc i + else: result.incl optObjCheck + of 'p': result.incl optProfiler + of 'q': result.incl optQuirky + of 'r': + if i+1 < s.len and s[i+1] == '0': + result.incl optRefCheck + inc i + else: result.incl optRangeCheck + of 's': + if i+1 < s.len and s[i+1] == '0': + result.incl optStyleCheck + inc i + elif i+1 < s.len and s[i+1] == '1': + result.incl optSinkInference + inc i + else: result.incl optStaticBoundsCheck + of 't': + if i+1 < s.len and s[i+1] == '0': + result.incl optTrMacros + inc i + else: result.incl optStackTrace + of 'w': result.incl optWarns + else: discard + inc i + diff --git a/compiler/icnif/icniftags.nim b/compiler/icnif/icniftags.nim new file mode 100644 index 000000000000..2dc7d3fa3481 --- /dev/null +++ b/compiler/icnif/icniftags.nim @@ -0,0 +1,8 @@ +import "../../dist/nimony/src/lib" / [nifstreams] + +let + symIdTag* = registerTag("symId") + symTag* = registerTag("s") + typeIdTag* = registerTag("typeId") + typeTag* = registerTag("t") + sonsTag* = registerTag("sons") diff --git a/compiler/icnif/nifdecoder.nim b/compiler/icnif/nifdecoder.nim new file mode 100644 index 000000000000..f658f4ba6a43 --- /dev/null +++ b/compiler/icnif/nifdecoder.nim @@ -0,0 +1,283 @@ +import std / [assertions, tables] +import "../../dist/nimony/src/lib" / [bitabs, nifreader, nifstreams, nifcursors] +import ".." / [ast, idents, lineinfos, options, modules, modulegraphs, msgs, pathutils] +import enum2nif, icniftags + +type + DecodeContext = object + graph: ModuleGraph + symbols: Table[int, PSym] + types: Table[int, PType] + +proc nodeKind(n: Cursor): TNodeKind {.inline.} = + assert n.kind == ParLe + pool.tags[n.tagId].parseNodeKind() + +proc expect(n: Cursor; k: set[NifKind]) = + if n.kind notin k: + when defined(debug): + writeStackTrace() + quit "[NIF decoder] expected: " & $k & " but got: " & $n.kind & toString n + +proc expect(n: Cursor; k: NifKind) {.inline.} = + expect n, {k} + +proc incExpect(n: var Cursor; k: set[NifKind]) = + inc n + expect n, k + +proc incExpect(n: var Cursor; k: NifKind) {.inline.} = + incExpect n, {k} + +proc skipParRi(n: var Cursor) = + expect n, {ParRi} + inc n + +proc expectTag(n: Cursor; tagId: TagId) = + if n.kind == ParLe and n.tagId == tagId: + discard + else: + when defined(debug): + writeStackTrace() + if n.kind != ParLe: + quit "[NIF decoder] expected: ParLe but got: " & $n.kind & toString n + else: + quit "[NIF decoder] expected: " & pool.tags[tagId] & " but got: " & pool.tags[n.tagId] & toString n + +proc incExpectTag(n: var Cursor; tagId: TagId) = + inc n + expectTag(n, tagId) + +when false: + proc expectTag(n: Cursor; tag: string) = + let id = pool.tags.getKeyId(tag) + if id == TagId(0): + when defined(debug): + writeStackTrace() + quit "[NIF decoder] expected: " & tag & " but doesn't exist" & toString n + else: + expectTag(n, id) + +proc fromNifSymbol(c: var DecodeContext; n: var Cursor): PSym +proc fromNifType(c: var DecodeContext; n: var Cursor): PType +proc fromNif(c: var DecodeContext; n: var Cursor): PNode + +proc fromNifSymDef(c: var DecodeContext; n: var Cursor): PSym = + expectTag n, symIdTag + incExpect n, IntLit + let id = pool.integers[n.intId] + incExpect n, Ident + let ident = c.graph.cache.getIdent(pool.strings[n.litId]) + incExpect n, IntLit + let itemId = pool.integers[n.intId].int32 + incExpect n, ParLe + let kind = parseSymKind(pool.tags[n.tagId]) + # TODO: add kind specific data + inc n + skipParRi n + expect n, {Ident, DotToken} + let flags = if n.kind == Ident: pool.strings[n.litId].parseSymFlags else: {} + inc n + var position = if kind == skModule: + expect n, StringLit + let path = pool.strings[n.litId].AbsoluteFile + fileInfoIdx(c.graph.config, path).int + else: + expect n, IntLit + pool.integers[n.intId] + incExpect n, IntLit + let disamb = pool.integers[n.intId].int32 + inc n + + result = PSym(itemId: ItemId(module: 0, item: itemId), + kind: kind, + name: ident, + flags: flags, + position: position, + disamb: disamb) + + # PNode, PSym or PType type fields in PSym can have cycles. + # Add PSym to `c.symbols` before parsing these fields so that + # they can refer this PSym. + assert id notin c.symbols + c.symbols[id] = result + + result.typ = c.fromNifType n + result.setOwner(c.fromNifSymbol n) + + expect n, Ident + result.loc.k = pool.strings[n.litId].parseLocKind() + incExpect n, StringLit + result.loc.snippet.add pool.strings[n.litId] + inc n + skipParRi n + +proc fromNifTypeDef(c: var DecodeContext; n: var Cursor): PType = + expectTag n, typeIdTag + incExpect n, IntLit + let id = pool.integers[n.intId] + incExpect n, IntLit + let itemId = pool.integers[n.intId].int32 + incExpect n, Ident + let kind = parseTypeKind(pool.strings[n.litId]) + incExpect n, {Ident, DotToken} + let flags = if n.kind == Ident: pool.strings[n.litId].parseTypeFlags else: {} + inc n + + result = PType(itemId: ItemId(module: 0, item: itemId), + kind: kind, + flags: flags) + assert id notin c.types + c.types[id] = result + + expect n, {DotToken, ParLe} + if n.kind == DotToken: + inc n + else: + expectTag n, sonsTag + inc n + while n.kind != ParRi: + result.addAllowNil c.fromNifType n + inc n + result.n = c.fromNif n + result.setOwner c.fromNifSymbol n + result.sym = c.fromNifSymbol n + skipParRi n + +proc fromNifNodeFlags(n: var Cursor): set[TNodeFlag] = + if n.kind == DotToken: + result = {} + inc n + elif n.kind == Ident: + result = parseNodeFlags(pool.strings[n.litId]) + inc n + else: + assert false, "expected Node flag but got " & $n.kind + +proc fromNifSymbol(c: var DecodeContext; n: var Cursor): PSym = + if n.kind == DotToken: + result = nil + inc n + else: + expect n, ParLe + if n.tagId == symIdTag: + result = c.fromNifSymDef n + elif n.tagId == symTag: + incExpect n, IntLit + result = c.symbols[pool.integers[n.intId]] + inc n + skipParRi n + else: + assert false, "expected symbol tag but got " & pool.tags[n.tagId] + +proc fromNifType(c: var DecodeContext; n: var Cursor): PType = + if n.kind == DotToken: + result = nil + inc n + else: + expect n, ParLe + if n.tagId == typeIdTag: + result = c.fromNifTypeDef n + elif n.tagId == typeTag: + incExpect n, IntLit + result = c.types[pool.integers[n.intId]] + inc n + skipParRi n + else: + assert false, "expected type tag but got " & pool.tags[n.tagId] + +template withNode(c: var DecodeContext; n: var Cursor; result: PNode; kind: TNodeKind; body: untyped) = + incExpect n, {DotToken, Ident} + let flags = fromNifNodeFlags n + result = newNode(kind) + result.flags = flags + result.typ = c.fromNifType n + body + skipParRi n + +proc fromNif(c: var DecodeContext; n: var Cursor): PNode = + result = nil + case n.kind: + of DotToken: + result = nil + inc n + of ParLe: + let kind = n.nodeKind + case kind: + of nkEmpty: + result = newNode(nkEmpty) + inc n + skipParRi n + of nkIdent: + incExpect n, Ident + result = newIdentNode(c.graph.cache.getIdent(pool.strings[n.litId]), unknownLineInfo) + inc n + skipParRi n + of nkSym: + c.withNode n, result, kind: + result.sym = c.fromNifSymbol n + of nkCharLit: + c.withNode n, result, kind: + expect n, CharLit + result.intVal = n.charLit.int + inc n + of nkIntLit .. nkInt64Lit: + c.withNode n, result, kind: + expect n, IntLit + result.intVal = pool.integers[n.intId] + inc n + of nkUIntLit .. nkUInt64Lit: + c.withNode n, result, kind: + expect n, UIntLit + result.intVal = cast[BiggestInt](pool.uintegers[n.uintId]) + inc n + of nkFloatLit .. nkFloat128Lit: + c.withNode n, result, kind: + expect n, FloatLit + result.floatVal = pool.floats[n.floatId] + inc n + of nkStrLit .. nkTripleStrLit: + c.withNode n, result, kind: + expect n, StringLit + result.strVal = pool.strings[n.litId] + inc n + of nkNilLit: + c.withNode n, result, kind: + discard + of nkNone: + assert false, "Unknown tag " & pool.tags[n.tagId] + else: + c.withNode n, result, kind: + while n.kind != ParRi: + result.add c.fromNif n + else: + assert false, "Not yet implemented " & $n.kind + +proc loadNif(stream: var Stream; graph: ModuleGraph): PNode = + discard processDirectives(stream.r) + + var buf = fromStream(stream) + var n = beginRead(buf) + + var c = DecodeContext(graph: graph) + + result = fromNif(c, n) + + endRead(buf) + +proc loadNifFile*(infile: AbsoluteFile; graph: ModuleGraph): PNode = + var stream = nifstreams.open(infile.string) + result = loadNif(stream, graph) + stream.close + +proc loadNifFromBuffer*(strbuf: sink string; graph: ModuleGraph): PNode = + var stream = nifstreams.openFromBuffer(strbuf) + result = loadNif(stream, graph) + +when isMainModule: + import std/cmdline + + if paramCount() > 0: + var graph = newModuleGraph(newIdentCache(), newConfigRef()) + var node = loadNifFile(paramStr(1).toAbsolute(toAbsoluteDir(".")), graph) + debug(node) diff --git a/compiler/icnif/nifencoder.nim b/compiler/icnif/nifencoder.nim new file mode 100644 index 000000000000..c323e9654173 --- /dev/null +++ b/compiler/icnif/nifencoder.nim @@ -0,0 +1,177 @@ +import std / [assertions, sets] +import ".." / [ast, idents, lineinfos, msgs, options] +import "../../dist/nimony/src/lib" / [bitabs, nifstreams, nifcursors, lineinfos] +import enum2nif, icniftags + +type + EncodeContext = object + conf: ConfigRef + decodedSyms: HashSet[PSym] + decodedTypes: HashSet[PType] + dest: TokenBuf + +proc initEncodeContext(conf: ConfigRef): EncodeContext = + result = EncodeContext(conf: conf, + dest: createTokenBuf()) + +template buildTree(dest: var TokenBuf; tag: TagId; body: untyped) = + dest.addParLe tag + body + dest.addParRi + +template buildTree(dest: var TokenBuf; tag: string; body: untyped) = + buildTree dest, pool.tags.getOrIncl(tag), body + +proc writeFlags[E](dest: var TokenBuf; flags: set[E]) = + var flagsAsIdent = "" + genFlags(flags, flagsAsIdent) + if flagsAsIdent.len > 0: + dest.addIdent flagsAsIdent + else: + dest.addDotToken + +proc toNif(c: var EncodeContext; sym: PSym) +proc toNif(c: var EncodeContext; typ: PType) +proc toNif(c: var EncodeContext; n: PNode) + +proc toNifDef(c: var EncodeContext; sym: PSym) = + c.dest.buildTree symIdTag: + c.dest.addIntLit sym.id + c.dest.addIdent sym.name.s + c.dest.addIntLit sym.itemId.item + c.dest.buildTree sym.kind.toNifTag: + # TODO: add kind specific data + discard + c.dest.writeFlags sym.flags + if sym.kind == skModule: + # position is module's FileIndex but it cannot be directly encoded + # as the uniqueness of it can broke + # if any import/include statements are changed. + let path = toFullPath(c.conf, sym.position.FileIndex) + c.dest.addStrLit path + else: + c.dest.addIntLit sym.position + c.dest.addIntLit sym.disamb + c.toNif sym.typ + c.toNif sym.owner + c.dest.addIdent toNifTag(sym.loc.k) + c.dest.addStrLit sym.loc.snippet + +proc toNifDef(c: var EncodeContext; typ: PType) = + c.dest.buildTree typeIdTag: + c.dest.addIntLit typ.id + c.dest.addIntLit typ.itemId.item + c.dest.addIdent toNifTag(typ.kind) + c.dest.writeFlags typ.flags + # following PType or PSym type field can have cycles but this proc should not called recursively + # as c.decodedTypes prevents it. + if typ.len == 0: + c.dest.addDotToken + else: + c.dest.buildTree sonsTag: + for ch in typ.kids: + c.toNif ch + c.toNif typ.n + c.toNif typ.owner + c.toNif typ.sym + +#include nifencodertypes + +proc toNif(c: var EncodeContext; sym: PSym) = + if sym == nil: + c.dest.addDotToken() + else: + if not c.decodedSyms.containsOrIncl(sym): + c.toNifDef sym + else: + c.dest.buildTree symTag: + c.dest.addIntLit sym.id + +proc toNif(c: var EncodeContext; typ: PType) = + if typ == nil: + c.dest.addDotToken() + else: + if not c.decodedTypes.containsOrIncl(typ): + c.toNifDef typ + else: + c.dest.buildTree typeTag: + c.dest.addIntLit typ.id + +proc writeNodeFlags(dest: var TokenBuf; flags: set[TNodeFlag]) {.inline.} = + writeFlags dest, flags + +template withNode(c: var EncodeContext; n: PNode; body: untyped) = + c.dest.addParLe pool.tags.getOrIncl(toNifTag(n.kind)) + writeNodeFlags(c.dest, n.flags) + c.toNif n.typ + body + c.dest.addParRi + +proc toNif(c: var EncodeContext; n: PNode) = + if n == nil: + c.dest.addDotToken + else: + case n.kind: + of nkEmpty: + c.dest.addParLe pool.tags.getOrIncl(toNifTag(nkEmpty)) + c.dest.addParRi + of nkIdent: + c.dest.addParLe pool.tags.getOrIncl(toNifTag(nkIdent)) + c.dest.addIdent n.ident.s + c.dest.addParRi + of nkSym: + when false: + echo "nkSym: ", n.sym.name.s + if n.sym.kind == skModule: + echo "position = ", n.sym.position + debug(n.sym) + var o = n.sym.owner + for i in 0 .. 20: + if o == nil: + break + echo "owner ", i, ":" + if o.kind == skModule: + echo "position = ", o.position + debug(o) + o = o.owner + # PNode.typ and PNode.sym.typ are different in `int` nkSym Node in following statement: + # type TestInt = int + c.withNode n: + c.toNif n.sym + of nkCharLit: + c.withNode n: + c.dest.add charToken(n.intVal.char, NoLineInfo) + of nkIntLit .. nkInt64Lit: + c.withNode n: + c.dest.addIntLit n.intVal + of nkUIntLit .. nkUInt64Lit: + c.withNode n: + c.dest.addUIntLit cast[BiggestUInt](n.intVal) + of nkFloatLit .. nkFloat128Lit: + c.withNode n: + c.dest.add floatToken(pool.floats.getOrIncl(n.floatVal), NoLineInfo) + of nkStrLit .. nkTripleStrLit: + c.withNode n: + c.dest.addStrLit n.strVal + of nkNilLit: + c.withNode n: + discard + else: + assert n.len > 0, $n.kind + c.withNode(n): + for i in 0 ..< n.len: + c.toNif n[i] + +proc saveNif(c: var EncodeContext; n: PNode): string = + toNif c, n + + result = "(.nif24)\n" & toString(c.dest) + +proc saveNifFile*(module: PSym; n: PNode; conf: ConfigRef) = + let outfile = module.name.s & ".nif" + var c = initEncodeContext(conf) + writeFile outfile, saveNif(c, n) + +proc saveNifToBuffer*(n: PNode; conf: ConfigRef): string = + var c = initEncodeContext(conf) + result = saveNif(c, n) diff --git a/tests/icnif/tencode_node2node.nim b/tests/icnif/tencode_node2node.nim new file mode 100644 index 000000000000..a920744404bc --- /dev/null +++ b/tests/icnif/tencode_node2node.nim @@ -0,0 +1,271 @@ +import std/assertions +import "../../compiler/icnif" / [nifencoder, nifdecoder] +import "../../compiler" / [idents, ast, astalgo, options, pathutils, modulegraphs, modules, msgs, pipelines, syntaxes, sem, llstream, lineinfos] + +# This test generates PNode by semchecks test code. +# Then it is used to test icnif/nifencoder and nifdecoder. + +const TestCodeDir = currentSourcePath().AbsoluteFile.splitFile().dir / RelativeDir"testcode" + +proc newConfigRefForTest(): ConfigRef = + var conf = newConfigRef() + conf.setDefaultLibpath() + conf.searchPaths.add(conf.libpath) + excl(conf.notes, hintProcessing) + excl(conf.mainPackageNotes, hintProcessing) + result = conf + +proc newModuleGraphForSem(cache: IdentCache; conf: ConfigRef): ModuleGraph = + var graph = newModuleGraph(cache, conf) + graph.setPipeLinePass(SemPass) + graph.compilePipelineSystemModule() + result = graph + +proc sem(graph: ModuleGraph; path: AbsoluteFile): PNode = + result = nil + + let fileIdx = fileInfoIdx(graph.config, path) + var module = newModule(graph, fileIdx) + registerModule(graph, module) + + var idgen = idGeneratorFromModule(module) + let ctx = preparePContext(graph, module, idgen) + + var stream = llStreamOpen(path, fmRead) + if stream == nil: + rawMessage(graph.config, errCannotOpenFile, path.string) + return nil + + var p: Parser = default(Parser) + syntaxes.openParser(p, fileIdx, stream, graph.cache, graph.config) + + checkFirstLineIndentation(p) + block processCode: + if graph.stopCompile(): break processCode + var n = parseTopLevelStmt(p) + if n.kind == nkEmpty: break processCode + # read everything, no streaming possible + var sl = newNodeI(nkStmtList, n.info) + sl.add n + while true: + var n = parseTopLevelStmt(p) + if n.kind == nkEmpty: break + sl.add n + + var semNode = semWithPContext(ctx, sl) + + return semNode + +type + # Nim's AST has cycles that causes infinite recursive loop in eql procs. + # this is used to prevent that happen. + EqlContext = object + nodeStack: seq[PNode] + symStack: seq[PSym] + typStack: seq[PType] + confX, confY: ConfigRef # used to print the line info when there is a mismatch + # and get path from FileIndex + +# Compare PType, PSym and PNode but ignores fields nifencoder and nifdecoder doesn't support +# `x` is generated by sem.nim and `y` is decoded by icnif/nifdecoder. +proc eql(x, y: PNode; c: var EqlContext): bool +proc eql(x, y: PType; c: var EqlContext): bool + +proc eql(x, y: TLoc): bool = + if x.k != y.k: + echo "loc kind mismatch: ", x.k, "/", y.k + result = false + elif x.snippet != y.snippet: + echo "loc snippet mismatch: ", x.snippet, "/", y.snippet + result = false + else: + result = true + +proc eqlSymPos(x, y: PSym; c: EqlContext): bool = + if x.kind == skModule: + let xpath = c.confX.toFullPath(x.position.FileIndex) + let ypath = c.confY.toFullPath(y.position.FileIndex) + if xpath != ypath: + echo "symbol position mismatch: ", xpath, "/", ypath + result = false + else: + result = true + elif x.position != y.position: + echo "symbol position mismatch: ", x.position, "/", y.position + result = false + else: + result = true + +proc eql(x, y: PSym; c: var EqlContext): bool = + if x == nil and y == nil: + result = true + elif x == nil or y == nil: + echo "symbol is missing" + result = false + elif x.name.s != y.name.s: + echo "symbol name mismatch: ", x.name.s, "/", y.name.s + result = false + elif x.itemId.item != y.itemId.item: + echo "symbol itemId.item mismatch: ", x.itemId.item, "/", y.itemId.item + result = false + elif x.kind != y.kind: + echo "symbol kind mismatch: ", x.kind, "/", y.kind + result = false + elif x.flags != y.flags: + echo "symbol flag mismatch: ", x.flags, "/", y.flags + result = false + elif not eqlSymPos(x, y, c): + result = false + elif x.disamb != y.disamb: + echo "symbol disamb mismatch: ", x.disamb, "/", y.disamb + result = false + elif not eql(x.loc, y.loc): + echo "symbol.loc mismatch" + result = false + else: + if c.symStack.len != 0: + for i in countDown(c.symStack.len - 1, 0): + if x == c.symStack[i]: + return true + c.symStack.add x + if not eql(x.typ, y.typ, c): + echo "symbol type mismatch:" + result = false + elif not eql(x.owner, y.owner, c): + echo "Symbol owner mismatch:" + debug(x.owner) + debug(y.owner) + result = false + else: + result = true + discard c.symStack.pop + +proc eql(x, y: PType; c: var EqlContext): bool = + if x == nil and y == nil: + result = true + elif x == nil or y == nil: + echo "type is missing" + result = false + elif x.itemId.item != y.itemId.item: + echo "type itemId.item mismatch: ", x.itemId.item, "/", y.itemId.item + result = false + elif x.kind != y.kind: + echo "type kind mismatch: ", x.kind, "/", y.kind + result = false + elif x.flags != y.flags: + echo "type flag mismatch: ", x.flags, "/", y.flags + result = false + else: + if c.typStack.len != 0: + for i in countDown(c.typStack.len - 1, 0): + if x == c.typStack[i]: + # echo "cycle is detected in PType" + return true + c.typStack.add x + if not eql(x.n, y.n, c): + echo "type.n mismatch" + debug(x.n) + debug(y.n) + result = false + elif not eql(x.owner, y.owner, c): + echo "type owner mismatch: " + debug(x.owner) + debug(y.owner) + result = false + elif not eql(x.sym, y.sym, c): + echo "type sym mismatch:" + debug(x.sym) + debug(y.sym) + result = false + elif x.kidsLen != y.kidsLen: + echo "type kidsLen mismatch" + result = false + else: + result = true + for i in 0 ..< x.kidsLen: + if not eql(x[i], y[i], c): + echo "type kids mismatch: " + debug(x[i]) + debug(y[i]) + result = false + break + discard c.typStack.pop + +proc eql(x, y: PNode; c: var EqlContext): bool = + if x == nil and y == nil: + result = true + elif x == nil or y == nil: + result = false + elif x.kind != y.kind: + echo "node kind mismatch: ", x.kind, "/", y.kind + result = false + elif x.flags != y.flags: + echo "node flag mismatch: ", x.flags, "/", y.flags + debug(x) + debug(y) + result = false + elif x.safeLen == y.safeLen: + if c.nodeStack.len != 0: + for i in countDown(c.nodeStack.len - 1, 0): + if x == c.nodeStack[i]: + # echo "cycle is detected in PNode" + return true + c.nodeStack.add x + if not eql(x.typ, y.typ, c): + echo "PNode type mismatch at ", `$`(c.confX, x.info), ":" + debug(x) + debug(y) + debug(x.typ) + debug(y.typ) + result = false + else: + case x.kind: + of nkIdent: + # these idents are generated from different IdentCache + result = x.ident.s == y.ident.s + if not result: + echo "PNode identifier mismatch: ", `$`(c.confX, x.info), x.ident.s, "/", y.ident.s + of nkSym: + result = eql(x.sym, y.sym, c) + if not result: + echo "Symbol mismatch:" + debug(x.sym) + debug(y.sym) + debug(x.sym.typ) + debug(y.sym.typ) + of nkCharLit .. nkTripleStrLit: + result = sameValue(x, y) + else: + result = true + for i in 0 ..< x.safeLen: + if not eql(x[i], y[i], c): + result = false + break + discard c.nodeStack.pop + else: + echo "node length mismatch" + debug(x) + debug(y) + result = false + +proc testNifEncDec(graph: ModuleGraph; src: string) = + let fullPath = TestCodeDir / RelativeFile(src) + let n = sem(graph, fullPath) + #debug(n) + let nif = saveNifToBuffer(n, graph.config) + #echo nif + + # Don't reuse the ModuleGraph used for semcheck when load NIF. + var graphForLoad = newModuleGraph(newIdentCache(), newConfigRefForTest()) + let n2 = loadNifFromBuffer(nif, graphForLoad) + #debug(n2) + var c = EqlContext(confX: graph.config, confY: graphForLoad.config) + assert eql(n, n2, c) + +var conf = newConfigRefForTest() +var cache = newIdentCache() +var graph = newModuleGraphForSem(cache, conf) +testNifEncDec(graph, "modtest1.nim") +testNifEncDec(graph, "modtestliterals.nim") +testNifEncDec(graph, "modtesttypesections.nim") +testNifEncDec(graph, "modtestpragmas.nim") diff --git a/tests/icnif/testcode/modtest1.nim b/tests/icnif/testcode/modtest1.nim new file mode 100644 index 000000000000..f36fa1afd239 --- /dev/null +++ b/tests/icnif/testcode/modtest1.nim @@ -0,0 +1,2 @@ +var x* = 123 +var y = x diff --git a/tests/icnif/testcode/modtestliterals.nim b/tests/icnif/testcode/modtestliterals.nim new file mode 100644 index 000000000000..2d11cf8fa91f --- /dev/null +++ b/tests/icnif/testcode/modtestliterals.nim @@ -0,0 +1,29 @@ +var strlit = "test string" +var rstrlit = r"test\t raw" +var triplestrlit = """Triple +string +literal +""" +var charlit = 'a' +var intlit1 = 123456 +var intlit2 = -123456 +var int8litH = 127'i8 +var int8litL = -128'i8 +var int16litH = 32767'i16 +var int16litL = -32768'i16 +var int32litH = 2147483647'i32 +var int32litL = -2147483648'i32 +var int64litH = 9223372036854775807'i64 +var int64litL = -9223372036854775808'i64 + +var uintlitH = 18446744073709551615'u +var uint8litH = 255'u8 +var uint16litH = 65535'u16 +var uint32litH = 4294967295'u32 +var uint64litH = 18446744073709551615'u64 + +var floatlit = 1.25 +var float32lit = 1.25'f32 +var float64lit = 1.25'f64 + +var nillit: ptr int = nil diff --git a/tests/icnif/testcode/modtestpragmas.nim b/tests/icnif/testcode/modtestpragmas.nim new file mode 100644 index 000000000000..7b8e22ccdfe0 --- /dev/null +++ b/tests/icnif/testcode/modtestpragmas.nim @@ -0,0 +1,3 @@ +#var exportcTest {.exportc.}: int +var importcTest {.importc.}: int +#var y* {.importc, header: "test.h".}: int diff --git a/tests/icnif/testcode/modtesttypesections.nim b/tests/icnif/testcode/modtesttypesections.nim new file mode 100644 index 000000000000..7ff5b0e5be67 --- /dev/null +++ b/tests/icnif/testcode/modtesttypesections.nim @@ -0,0 +1,36 @@ +type + TestInt = int + TestEnum = enum + X + Y + TestDistinct = distinct int + + TestObject = object + x*: int + y: int + + TestObject2* = object + x: TestObject + + TestRefInt = ref int + TestPtrInt = ptr int + + TestRefObj = ref object + x: int + + TestPtrObj = ptr object + x: int + +var x: TestInt +var testEnum: TestEnum +var testEnum1 = X +var testDistinct: TestDistinct +var testObject: TestObject +var testObject2*: TestObject2 + +var testRefInt: TestRefInt = nil +var testRefInt2: ref int = nil +var testPtrInt: TestPtrInt = nil +var testPtrInt2: ptr int = nil +var testRefObj: TestRefObj = nil +var testPtrObj: TestPtrObj = nil From 25ca7ccc79e9203ae6aab1684eb89f52b23e4a05 Mon Sep 17 00:00:00 2001 From: demotomohiro Date: Mon, 27 Oct 2025 15:40:11 +0900 Subject: [PATCH 02/38] encodes/decodes itemId.module --- compiler/icnif/icniftags.nim | 2 ++ compiler/icnif/nifdecoder.nim | 40 ++++++++++++++++++++++++------- compiler/icnif/nifencoder.nim | 23 ++++++++++++------ tests/icnif/tencode_node2node.nim | 34 +++++++++++++++++--------- 4 files changed, 72 insertions(+), 27 deletions(-) diff --git a/compiler/icnif/icniftags.nim b/compiler/icnif/icniftags.nim index 2dc7d3fa3481..f3c1fe3785f2 100644 --- a/compiler/icnif/icniftags.nim +++ b/compiler/icnif/icniftags.nim @@ -6,3 +6,5 @@ let typeIdTag* = registerTag("typeId") typeTag* = registerTag("t") sonsTag* = registerTag("sons") + + modIdTag* = registerTag("modId") diff --git a/compiler/icnif/nifdecoder.nim b/compiler/icnif/nifdecoder.nim index f658f4ba6a43..f0a36f544194 100644 --- a/compiler/icnif/nifdecoder.nim +++ b/compiler/icnif/nifdecoder.nim @@ -8,6 +8,7 @@ type graph: ModuleGraph symbols: Table[int, PSym] types: Table[int, PType] + modules: Table[int, FileIndex] # maps module id in NIF to FileIndex of the module proc nodeKind(n: Cursor): TNodeKind {.inline.} = assert n.kind == ParLe @@ -58,6 +59,23 @@ when false: else: expectTag(n, id) +proc fromNifModuleId(c: var DecodeContext; n: var Cursor): FileIndex = + if n.kind == ParLe: + expectTag n, modIdTag + incExpect n, IntLit + let id = pool.integers[n.intId] + incExpect n, StringLit + let path = pool.strings[n.litId].AbsoluteFile + result = fileInfoIdx(c.graph.config, path) + assert id notin c.modules + c.modules[id] = result + inc n + skipParRi n + elif n.kind == IntLit: + let id = pool.integers[n.intId] + result = c.modules[id] + inc n + proc fromNifSymbol(c: var DecodeContext; n: var Cursor): PSym proc fromNifType(c: var DecodeContext; n: var Cursor): PType proc fromNif(c: var DecodeContext; n: var Cursor): PNode @@ -68,7 +86,9 @@ proc fromNifSymDef(c: var DecodeContext; n: var Cursor): PSym = let id = pool.integers[n.intId] incExpect n, Ident let ident = c.graph.cache.getIdent(pool.strings[n.litId]) - incExpect n, IntLit + inc n + let itemIdModule = c.fromNifModuleId(n).int32 + expect n, IntLit let itemId = pool.integers[n.intId].int32 incExpect n, ParLe let kind = parseSymKind(pool.tags[n.tagId]) @@ -79,17 +99,17 @@ proc fromNifSymDef(c: var DecodeContext; n: var Cursor): PSym = let flags = if n.kind == Ident: pool.strings[n.litId].parseSymFlags else: {} inc n var position = if kind == skModule: - expect n, StringLit - let path = pool.strings[n.litId].AbsoluteFile - fileInfoIdx(c.graph.config, path).int + c.fromNifModuleId(n).int else: expect n, IntLit - pool.integers[n.intId] - incExpect n, IntLit + let p = pool.integers[n.intId] + inc n + p + expect n, IntLit let disamb = pool.integers[n.intId].int32 inc n - result = PSym(itemId: ItemId(module: 0, item: itemId), + result = PSym(itemId: ItemId(module: itemIdModule, item: itemId), kind: kind, name: ident, flags: flags, @@ -116,7 +136,9 @@ proc fromNifTypeDef(c: var DecodeContext; n: var Cursor): PType = expectTag n, typeIdTag incExpect n, IntLit let id = pool.integers[n.intId] - incExpect n, IntLit + inc n + let itemIdModule = c.fromNifModuleId(n).int32 + expect n, IntLit let itemId = pool.integers[n.intId].int32 incExpect n, Ident let kind = parseTypeKind(pool.strings[n.litId]) @@ -124,7 +146,7 @@ proc fromNifTypeDef(c: var DecodeContext; n: var Cursor): PType = let flags = if n.kind == Ident: pool.strings[n.litId].parseTypeFlags else: {} inc n - result = PType(itemId: ItemId(module: 0, item: itemId), + result = PType(itemId: ItemId(module: itemIdModule, item: itemId), kind: kind, flags: flags) assert id notin c.types diff --git a/compiler/icnif/nifencoder.nim b/compiler/icnif/nifencoder.nim index c323e9654173..69c16af5c70b 100644 --- a/compiler/icnif/nifencoder.nim +++ b/compiler/icnif/nifencoder.nim @@ -8,6 +8,7 @@ type conf: ConfigRef decodedSyms: HashSet[PSym] decodedTypes: HashSet[PType] + decodedFileIndices: HashSet[FileIndex] dest: TokenBuf proc initEncodeContext(conf: ConfigRef): EncodeContext = @@ -30,6 +31,18 @@ proc writeFlags[E](dest: var TokenBuf; flags: set[E]) = else: dest.addDotToken +proc toNifModuleId(c: var EncodeContext; moduleId: int) = + # `ItemId.module` in PType and PSym (and `PSym.position` when it is skModule) are module's FileIndex + # but it cannot be directly encoded as the uniqueness of it can broke + # if any import/include statements are changed. + if not c.decodedFileIndices.containsOrIncl(moduleId.FileIndex): + c.dest.buildTree modIdTag: + c.dest.addIntLit moduleId + let path = toFullPath(c.conf, moduleId.FileIndex) + c.dest.addStrLit path + else: + c.dest.addIntLit moduleId + proc toNif(c: var EncodeContext; sym: PSym) proc toNif(c: var EncodeContext; typ: PType) proc toNif(c: var EncodeContext; n: PNode) @@ -38,17 +51,14 @@ proc toNifDef(c: var EncodeContext; sym: PSym) = c.dest.buildTree symIdTag: c.dest.addIntLit sym.id c.dest.addIdent sym.name.s + c.toNifModuleId sym.itemId.module c.dest.addIntLit sym.itemId.item c.dest.buildTree sym.kind.toNifTag: # TODO: add kind specific data discard c.dest.writeFlags sym.flags if sym.kind == skModule: - # position is module's FileIndex but it cannot be directly encoded - # as the uniqueness of it can broke - # if any import/include statements are changed. - let path = toFullPath(c.conf, sym.position.FileIndex) - c.dest.addStrLit path + c.toNifModuleId sym.position else: c.dest.addIntLit sym.position c.dest.addIntLit sym.disamb @@ -60,6 +70,7 @@ proc toNifDef(c: var EncodeContext; sym: PSym) = proc toNifDef(c: var EncodeContext; typ: PType) = c.dest.buildTree typeIdTag: c.dest.addIntLit typ.id + c.toNifModuleId typ.itemId.module c.dest.addIntLit typ.itemId.item c.dest.addIdent toNifTag(typ.kind) c.dest.writeFlags typ.flags @@ -75,8 +86,6 @@ proc toNifDef(c: var EncodeContext; typ: PType) = c.toNif typ.owner c.toNif typ.sym -#include nifencodertypes - proc toNif(c: var EncodeContext; sym: PSym) = if sym == nil: c.dest.addDotToken() diff --git a/tests/icnif/tencode_node2node.nim b/tests/icnif/tencode_node2node.nim index a920744404bc..21875c1623a8 100644 --- a/tests/icnif/tencode_node2node.nim +++ b/tests/icnif/tencode_node2node.nim @@ -81,21 +81,33 @@ proc eql(x, y: TLoc): bool = else: result = true +proc eqlFileIndex(x, y: int; c: EqlContext): bool = + let xpath = c.confX.toFullPath(x.FileIndex) + let ypath = c.confY.toFullPath(y.FileIndex) + if xpath != ypath: + echo "file index mismatch: ", xpath, "/", ypath + result = false + else: + result = true + proc eqlSymPos(x, y: PSym; c: EqlContext): bool = if x.kind == skModule: - let xpath = c.confX.toFullPath(x.position.FileIndex) - let ypath = c.confY.toFullPath(y.position.FileIndex) - if xpath != ypath: - echo "symbol position mismatch: ", xpath, "/", ypath - result = false - else: - result = true + result = eqlFileIndex(x.position, y.position, c) elif x.position != y.position: echo "symbol position mismatch: ", x.position, "/", y.position result = false else: result = true +proc eqlItemId(x, y: ItemId; c: EqlContext): bool = + if x.item != y.item: + echo "itemId.item mismatch: ", x.item, "/", y.item + result = false + elif not eqlFileIndex(x.module, y.module, c): + result = false + else: + result = true + proc eql(x, y: PSym; c: var EqlContext): bool = if x == nil and y == nil: result = true @@ -105,8 +117,8 @@ proc eql(x, y: PSym; c: var EqlContext): bool = elif x.name.s != y.name.s: echo "symbol name mismatch: ", x.name.s, "/", y.name.s result = false - elif x.itemId.item != y.itemId.item: - echo "symbol itemId.item mismatch: ", x.itemId.item, "/", y.itemId.item + elif not eqlItemId(x.itemId, y.itemId, c): + echo "symbol itemId mismatch" result = false elif x.kind != y.kind: echo "symbol kind mismatch: ", x.kind, "/", y.kind @@ -146,8 +158,8 @@ proc eql(x, y: PType; c: var EqlContext): bool = elif x == nil or y == nil: echo "type is missing" result = false - elif x.itemId.item != y.itemId.item: - echo "type itemId.item mismatch: ", x.itemId.item, "/", y.itemId.item + elif not eqlItemId(x.itemId, y.itemId, c): + echo "type itemId mismatch" result = false elif x.kind != y.kind: echo "type kind mismatch: ", x.kind, "/", y.kind From 5eeede3a432f830d683451df0be358c7ae56b9ef Mon Sep 17 00:00:00 2001 From: demotomohiro Date: Tue, 28 Oct 2025 00:40:19 +0900 Subject: [PATCH 03/38] uses PSym.item and PType.item as id in NIF --- compiler/icnif/nifdecoder.nim | 53 ++++++++++++++++++++--------------- compiler/icnif/nifencoder.nim | 10 +++---- 2 files changed, 35 insertions(+), 28 deletions(-) diff --git a/compiler/icnif/nifdecoder.nim b/compiler/icnif/nifdecoder.nim index f0a36f544194..567277302a5f 100644 --- a/compiler/icnif/nifdecoder.nim +++ b/compiler/icnif/nifdecoder.nim @@ -6,8 +6,8 @@ import enum2nif, icniftags type DecodeContext = object graph: ModuleGraph - symbols: Table[int, PSym] - types: Table[int, PType] + symbols: Table[ItemId, PSym] + types: Table[ItemId, PType] modules: Table[int, FileIndex] # maps module id in NIF to FileIndex of the module proc nodeKind(n: Cursor): TNodeKind {.inline.} = @@ -59,21 +59,22 @@ when false: else: expectTag(n, id) -proc fromNifModuleId(c: var DecodeContext; n: var Cursor): FileIndex = +proc fromNifModuleId(c: var DecodeContext; n: var Cursor): (FileIndex, int32) = + expect n, {ParLe, IntLit} if n.kind == ParLe: expectTag n, modIdTag incExpect n, IntLit let id = pool.integers[n.intId] incExpect n, StringLit let path = pool.strings[n.litId].AbsoluteFile - result = fileInfoIdx(c.graph.config, path) + result = (fileInfoIdx(c.graph.config, path), id.int32) assert id notin c.modules - c.modules[id] = result + c.modules[id] = result[0] inc n skipParRi n elif n.kind == IntLit: let id = pool.integers[n.intId] - result = c.modules[id] + result = (c.modules[id], id.int32) inc n proc fromNifSymbol(c: var DecodeContext; n: var Cursor): PSym @@ -82,14 +83,12 @@ proc fromNif(c: var DecodeContext; n: var Cursor): PNode proc fromNifSymDef(c: var DecodeContext; n: var Cursor): PSym = expectTag n, symIdTag - incExpect n, IntLit - let id = pool.integers[n.intId] - incExpect n, Ident - let ident = c.graph.cache.getIdent(pool.strings[n.litId]) inc n - let itemIdModule = c.fromNifModuleId(n).int32 + let (itemIdModule, nifModId) = c.fromNifModuleId(n) expect n, IntLit let itemId = pool.integers[n.intId].int32 + incExpect n, Ident + let ident = c.graph.cache.getIdent(pool.strings[n.litId]) incExpect n, ParLe let kind = parseSymKind(pool.tags[n.tagId]) # TODO: add kind specific data @@ -99,7 +98,7 @@ proc fromNifSymDef(c: var DecodeContext; n: var Cursor): PSym = let flags = if n.kind == Ident: pool.strings[n.litId].parseSymFlags else: {} inc n var position = if kind == skModule: - c.fromNifModuleId(n).int + c.fromNifModuleId(n)[0].int else: expect n, IntLit let p = pool.integers[n.intId] @@ -109,7 +108,7 @@ proc fromNifSymDef(c: var DecodeContext; n: var Cursor): PSym = let disamb = pool.integers[n.intId].int32 inc n - result = PSym(itemId: ItemId(module: itemIdModule, item: itemId), + result = PSym(itemId: ItemId(module: itemIdModule.int32, item: itemId), kind: kind, name: ident, flags: flags, @@ -119,8 +118,9 @@ proc fromNifSymDef(c: var DecodeContext; n: var Cursor): PSym = # PNode, PSym or PType type fields in PSym can have cycles. # Add PSym to `c.symbols` before parsing these fields so that # they can refer this PSym. - assert id notin c.symbols - c.symbols[id] = result + let nifItemId = ItemId(module: nifModId, item: itemId) + assert nifItemId notin c.symbols + c.symbols[nifItemId] = result result.typ = c.fromNifType n result.setOwner(c.fromNifSymbol n) @@ -134,10 +134,8 @@ proc fromNifSymDef(c: var DecodeContext; n: var Cursor): PSym = proc fromNifTypeDef(c: var DecodeContext; n: var Cursor): PType = expectTag n, typeIdTag - incExpect n, IntLit - let id = pool.integers[n.intId] inc n - let itemIdModule = c.fromNifModuleId(n).int32 + let (itemIdModule, nifModId) = c.fromNifModuleId(n) expect n, IntLit let itemId = pool.integers[n.intId].int32 incExpect n, Ident @@ -146,11 +144,12 @@ proc fromNifTypeDef(c: var DecodeContext; n: var Cursor): PType = let flags = if n.kind == Ident: pool.strings[n.litId].parseTypeFlags else: {} inc n - result = PType(itemId: ItemId(module: itemIdModule, item: itemId), + result = PType(itemId: ItemId(module: itemIdModule.int32, item: itemId), kind: kind, flags: flags) - assert id notin c.types - c.types[id] = result + let nifItemId = ItemId(module: nifModId, item: itemId) + assert nifItemId notin c.types + c.types[nifItemId] = result expect n, {DotToken, ParLe} if n.kind == DotToken: @@ -186,7 +185,11 @@ proc fromNifSymbol(c: var DecodeContext; n: var Cursor): PSym = result = c.fromNifSymDef n elif n.tagId == symTag: incExpect n, IntLit - result = c.symbols[pool.integers[n.intId]] + let nifModId = pool.integers[n.intId].int32 + incExpect n, IntLit + let item = pool.integers[n.intId].int32 + let nifItemId = ItemId(module: nifModId, item: item) + result = c.symbols[nifItemId] inc n skipParRi n else: @@ -202,7 +205,11 @@ proc fromNifType(c: var DecodeContext; n: var Cursor): PType = result = c.fromNifTypeDef n elif n.tagId == typeTag: incExpect n, IntLit - result = c.types[pool.integers[n.intId]] + let nifModId = pool.integers[n.intId].int32 + incExpect n, IntLit + let item = pool.integers[n.intId].int32 + let nifItemId = ItemId(module: nifModId, item: item) + result = c.types[nifItemId] inc n skipParRi n else: diff --git a/compiler/icnif/nifencoder.nim b/compiler/icnif/nifencoder.nim index 69c16af5c70b..1a6d9cb0b457 100644 --- a/compiler/icnif/nifencoder.nim +++ b/compiler/icnif/nifencoder.nim @@ -49,10 +49,9 @@ proc toNif(c: var EncodeContext; n: PNode) proc toNifDef(c: var EncodeContext; sym: PSym) = c.dest.buildTree symIdTag: - c.dest.addIntLit sym.id - c.dest.addIdent sym.name.s c.toNifModuleId sym.itemId.module c.dest.addIntLit sym.itemId.item + c.dest.addIdent sym.name.s c.dest.buildTree sym.kind.toNifTag: # TODO: add kind specific data discard @@ -69,7 +68,6 @@ proc toNifDef(c: var EncodeContext; sym: PSym) = proc toNifDef(c: var EncodeContext; typ: PType) = c.dest.buildTree typeIdTag: - c.dest.addIntLit typ.id c.toNifModuleId typ.itemId.module c.dest.addIntLit typ.itemId.item c.dest.addIdent toNifTag(typ.kind) @@ -94,7 +92,8 @@ proc toNif(c: var EncodeContext; sym: PSym) = c.toNifDef sym else: c.dest.buildTree symTag: - c.dest.addIntLit sym.id + c.dest.addIntLit sym.itemId.module + c.dest.addIntLit sym.itemId.item proc toNif(c: var EncodeContext; typ: PType) = if typ == nil: @@ -104,7 +103,8 @@ proc toNif(c: var EncodeContext; typ: PType) = c.toNifDef typ else: c.dest.buildTree typeTag: - c.dest.addIntLit typ.id + c.dest.addIntLit typ.itemId.module + c.dest.addIntLit typ.itemId.item proc writeNodeFlags(dest: var TokenBuf; flags: set[TNodeFlag]) {.inline.} = writeFlags dest, flags From 51eccea9106a9e68ce92b8f657dd7c0f383c7ded Mon Sep 17 00:00:00 2001 From: demotomohiro Date: Tue, 28 Oct 2025 14:40:05 +0900 Subject: [PATCH 04/38] saves/loads PSym guard, bitsize and alignment --- compiler/icnif/nifdecoder.nim | 39 +++++++++++++++---------- compiler/icnif/nifencoder.nim | 13 ++++++--- tests/icnif/tencode_node2node.nim | 15 +++++++++- tests/icnif/testcode/modtestpragmas.nim | 4 +-- 4 files changed, 48 insertions(+), 23 deletions(-) diff --git a/compiler/icnif/nifdecoder.nim b/compiler/icnif/nifdecoder.nim index 567277302a5f..8ec3d19a0d0c 100644 --- a/compiler/icnif/nifdecoder.nim +++ b/compiler/icnif/nifdecoder.nim @@ -89,30 +89,18 @@ proc fromNifSymDef(c: var DecodeContext; n: var Cursor): PSym = let itemId = pool.integers[n.intId].int32 incExpect n, Ident let ident = c.graph.cache.getIdent(pool.strings[n.litId]) - incExpect n, ParLe - let kind = parseSymKind(pool.tags[n.tagId]) - # TODO: add kind specific data - inc n - skipParRi n - expect n, {Ident, DotToken} + incExpect n, {Ident, DotToken} let flags = if n.kind == Ident: pool.strings[n.litId].parseSymFlags else: {} - inc n - var position = if kind == skModule: - c.fromNifModuleId(n)[0].int - else: - expect n, IntLit - let p = pool.integers[n.intId] - inc n - p - expect n, IntLit + incExpect n, IntLit let disamb = pool.integers[n.intId].int32 + incExpect n, ParLe + let kind = parseSymKind(pool.tags[n.tagId]) inc n result = PSym(itemId: ItemId(module: itemIdModule.int32, item: itemId), kind: kind, name: ident, flags: flags, - position: position, disamb: disamb) # PNode, PSym or PType type fields in PSym can have cycles. @@ -122,6 +110,25 @@ proc fromNifSymDef(c: var DecodeContext; n: var Cursor): PSym = assert nifItemId notin c.symbols c.symbols[nifItemId] = result + case kind + of skLet, skVar, skField, skForVar: + result.guard = c.fromNifSymbol n + expect n, IntLit + result.bitsize = pool.integers[n.intId] + incExpect n, IntLit + result.alignment = pool.integers[n.intId] + inc n + else: + discard + skipParRi n + result.position = if kind == skModule: + c.fromNifModuleId(n)[0].int + else: + expect n, IntLit + let p = pool.integers[n.intId] + inc n + p + result.typ = c.fromNifType n result.setOwner(c.fromNifSymbol n) diff --git a/compiler/icnif/nifencoder.nim b/compiler/icnif/nifencoder.nim index 1a6d9cb0b457..bd35c988e2a8 100644 --- a/compiler/icnif/nifencoder.nim +++ b/compiler/icnif/nifencoder.nim @@ -52,15 +52,20 @@ proc toNifDef(c: var EncodeContext; sym: PSym) = c.toNifModuleId sym.itemId.module c.dest.addIntLit sym.itemId.item c.dest.addIdent sym.name.s - c.dest.buildTree sym.kind.toNifTag: - # TODO: add kind specific data - discard c.dest.writeFlags sym.flags + c.dest.addIntLit sym.disamb + c.dest.buildTree sym.kind.toNifTag: + case sym.kind + of skLet, skVar, skField, skForVar: + c.toNif sym.guard + c.dest.addIntLit sym.bitsize + c.dest.addIntLit sym.alignment + else: + discard if sym.kind == skModule: c.toNifModuleId sym.position else: c.dest.addIntLit sym.position - c.dest.addIntLit sym.disamb c.toNif sym.typ c.toNif sym.owner c.dest.addIdent toNifTag(sym.loc.k) diff --git a/tests/icnif/tencode_node2node.nim b/tests/icnif/tencode_node2node.nim index 21875c1623a8..2fb35785ef75 100644 --- a/tests/icnif/tencode_node2node.nim +++ b/tests/icnif/tencode_node2node.nim @@ -149,7 +149,20 @@ proc eql(x, y: PSym; c: var EqlContext): bool = debug(y.owner) result = false else: - result = true + if x.kind in {skLet, skVar, skField, skForVar}: + if not eql(x.guard, y.guard, c): + echo "symbol guard mismatch" + result = false + elif x.bitsize != y.bitsize: + echo "symbol bitsize mismatch: ", x.bitsize, "/", y.bitsize + result = false + elif x.alignment != y.alignment: + echo "symbol alignment mismatch: ", x.alignment, "/", y.alignment + result = false + else: + result = true + else: + result = true discard c.symStack.pop proc eql(x, y: PType; c: var EqlContext): bool = diff --git a/tests/icnif/testcode/modtestpragmas.nim b/tests/icnif/testcode/modtestpragmas.nim index 7b8e22ccdfe0..c20c87fefce5 100644 --- a/tests/icnif/testcode/modtestpragmas.nim +++ b/tests/icnif/testcode/modtestpragmas.nim @@ -1,3 +1,3 @@ -#var exportcTest {.exportc.}: int +var exportcTest {.exportc.}: int var importcTest {.importc.}: int -#var y* {.importc, header: "test.h".}: int +var y* {.importc, header: "test.h".}: int From 8a4d921d6bb018fed0db3c5f67fa828fd5eb473e Mon Sep 17 00:00:00 2001 From: demotomohiro Date: Wed, 29 Oct 2025 10:42:19 +0900 Subject: [PATCH 05/38] saves/loads PNode and PSym line info --- compiler/icnif/nifdecoder.nim | 18 +++++++--- compiler/icnif/nifencoder.nim | 58 ++++++++++++++++++------------- tests/icnif/tencode_node2node.nim | 27 ++++++++++++++ 3 files changed, 75 insertions(+), 28 deletions(-) diff --git a/compiler/icnif/nifdecoder.nim b/compiler/icnif/nifdecoder.nim index 8ec3d19a0d0c..aca9ac47dd5b 100644 --- a/compiler/icnif/nifdecoder.nim +++ b/compiler/icnif/nifdecoder.nim @@ -1,5 +1,5 @@ import std / [assertions, tables] -import "../../dist/nimony/src/lib" / [bitabs, nifreader, nifstreams, nifcursors] +import "../../dist/nimony/src/lib" / [bitabs, nifreader, nifstreams, nifcursors, lineinfos] import ".." / [ast, idents, lineinfos, options, modules, modulegraphs, msgs, pathutils] import enum2nif, icniftags @@ -59,6 +59,13 @@ when false: else: expectTag(n, id) +proc fromNifLineInfo(c: var DecodeContext; n: Cursor): TLineInfo = + if n.info == NoLineInfo: + unknownLineInfo + else: + let info = pool.man.unpack(n.info) + c.graph.config.newLineInfo(pool.files[info.file].AbsoluteFile, info.line, info.col) + proc fromNifModuleId(c: var DecodeContext; n: var Cursor): (FileIndex, int32) = expect n, {ParLe, IntLit} if n.kind == ParLe: @@ -83,6 +90,7 @@ proc fromNif(c: var DecodeContext; n: var Cursor): PNode proc fromNifSymDef(c: var DecodeContext; n: var Cursor): PSym = expectTag n, symIdTag + let info = c.fromNifLineInfo n inc n let (itemIdModule, nifModId) = c.fromNifModuleId(n) expect n, IntLit @@ -100,6 +108,7 @@ proc fromNifSymDef(c: var DecodeContext; n: var Cursor): PSym = result = PSym(itemId: ItemId(module: itemIdModule.int32, item: itemId), kind: kind, name: ident, + info: info, flags: flags, disamb: disamb) @@ -223,9 +232,10 @@ proc fromNifType(c: var DecodeContext; n: var Cursor): PType = assert false, "expected type tag but got " & pool.tags[n.tagId] template withNode(c: var DecodeContext; n: var Cursor; result: PNode; kind: TNodeKind; body: untyped) = + let info = c.fromNifLineInfo(n) incExpect n, {DotToken, Ident} let flags = fromNifNodeFlags n - result = newNode(kind) + result = newNodeI(kind, info) result.flags = flags result.typ = c.fromNifType n body @@ -241,12 +251,12 @@ proc fromNif(c: var DecodeContext; n: var Cursor): PNode = let kind = n.nodeKind case kind: of nkEmpty: - result = newNode(nkEmpty) + result = newNodeI(nkEmpty, c.fromNifLineInfo(n)) inc n skipParRi n of nkIdent: incExpect n, Ident - result = newIdentNode(c.graph.cache.getIdent(pool.strings[n.litId]), unknownLineInfo) + result = newIdentNode(c.graph.cache.getIdent(pool.strings[n.litId]), c.fromNifLineInfo(n)) inc n skipParRi n of nkSym: diff --git a/compiler/icnif/nifencoder.nim b/compiler/icnif/nifencoder.nim index bd35c988e2a8..6a91064ac857 100644 --- a/compiler/icnif/nifencoder.nim +++ b/compiler/icnif/nifencoder.nim @@ -31,6 +31,13 @@ proc writeFlags[E](dest: var TokenBuf; flags: set[E]) = else: dest.addDotToken +proc toNif(c: var EncodeContext; info: TLineInfo): PackedLineInfo = + if info == unknownLineInfo: + NoLineInfo + else: + let fileId = pool.files.getOrIncl(c.conf.toFullPath(info.fileIndex)) + pack(pool.man, fileId, info.line.int32, info.col) + proc toNifModuleId(c: var EncodeContext; moduleId: int) = # `ItemId.module` in PType and PSym (and `PSym.position` when it is skModule) are module's FileIndex # but it cannot be directly encoded as the uniqueness of it can broke @@ -48,28 +55,29 @@ proc toNif(c: var EncodeContext; typ: PType) proc toNif(c: var EncodeContext; n: PNode) proc toNifDef(c: var EncodeContext; sym: PSym) = - c.dest.buildTree symIdTag: - c.toNifModuleId sym.itemId.module - c.dest.addIntLit sym.itemId.item - c.dest.addIdent sym.name.s - c.dest.writeFlags sym.flags - c.dest.addIntLit sym.disamb - c.dest.buildTree sym.kind.toNifTag: - case sym.kind - of skLet, skVar, skField, skForVar: - c.toNif sym.guard - c.dest.addIntLit sym.bitsize - c.dest.addIntLit sym.alignment - else: - discard - if sym.kind == skModule: - c.toNifModuleId sym.position + c.dest.addParLe symIdTag, c.toNif sym.info + c.toNifModuleId sym.itemId.module + c.dest.addIntLit sym.itemId.item + c.dest.addIdent sym.name.s + c.dest.writeFlags sym.flags + c.dest.addIntLit sym.disamb + c.dest.buildTree sym.kind.toNifTag: + case sym.kind + of skLet, skVar, skField, skForVar: + c.toNif sym.guard + c.dest.addIntLit sym.bitsize + c.dest.addIntLit sym.alignment else: - c.dest.addIntLit sym.position - c.toNif sym.typ - c.toNif sym.owner - c.dest.addIdent toNifTag(sym.loc.k) - c.dest.addStrLit sym.loc.snippet + discard + if sym.kind == skModule: + c.toNifModuleId sym.position + else: + c.dest.addIntLit sym.position + c.toNif sym.typ + c.toNif sym.owner + c.dest.addIdent toNifTag(sym.loc.k) + c.dest.addStrLit sym.loc.snippet + c.dest.addParRi proc toNifDef(c: var EncodeContext; typ: PType) = c.dest.buildTree typeIdTag: @@ -115,7 +123,7 @@ proc writeNodeFlags(dest: var TokenBuf; flags: set[TNodeFlag]) {.inline.} = writeFlags dest, flags template withNode(c: var EncodeContext; n: PNode; body: untyped) = - c.dest.addParLe pool.tags.getOrIncl(toNifTag(n.kind)) + c.dest.addParLe pool.tags.getOrIncl(toNifTag(n.kind)), c.toNif n.info writeNodeFlags(c.dest, n.flags) c.toNif n.typ body @@ -127,10 +135,12 @@ proc toNif(c: var EncodeContext; n: PNode) = else: case n.kind: of nkEmpty: - c.dest.addParLe pool.tags.getOrIncl(toNifTag(nkEmpty)) + let info = c.toNif n.info + c.dest.addParLe pool.tags.getOrIncl(toNifTag(nkEmpty)), info c.dest.addParRi of nkIdent: - c.dest.addParLe pool.tags.getOrIncl(toNifTag(nkIdent)) + let info = c.toNif n.info + c.dest.addParLe pool.tags.getOrIncl(toNifTag(nkIdent)), info c.dest.addIdent n.ident.s c.dest.addParRi of nkSym: diff --git a/tests/icnif/tencode_node2node.nim b/tests/icnif/tencode_node2node.nim index 2fb35785ef75..b68854271744 100644 --- a/tests/icnif/tencode_node2node.nim +++ b/tests/icnif/tencode_node2node.nim @@ -90,6 +90,26 @@ proc eqlFileIndex(x, y: int; c: EqlContext): bool = else: result = true +proc eql(x, y: TLineInfo; c: EqlContext): bool = + # If parent PNode has a valid line info but it's child doesn't have one, + # cannot translate such a tree to NIF. + # Because in NIF, if a child node doesn't have line info, + # nifstream assign the parent's line info to it. + # So cannot have child node without line info if parent has a valid line info. + if x == unknownLineInfo: + result = true + elif x.line != y.line: + echo "line number mismatch: ", x.line, "/", y.line + result = false + elif x.col != y.col: + echo "column number mismatch: ", x.col, "/", y.col + result = false + elif not eqlFileIndex(x.fileIndex.int, y.fileIndex.int, c): + echo "file in line info mismatch" + result = false + else: + result = true + proc eqlSymPos(x, y: PSym; c: EqlContext): bool = if x.kind == skModule: result = eqlFileIndex(x.position, y.position, c) @@ -123,6 +143,9 @@ proc eql(x, y: PSym; c: var EqlContext): bool = elif x.kind != y.kind: echo "symbol kind mismatch: ", x.kind, "/", y.kind result = false + elif not eql(x.info, y.info, c): + echo "symbol line info mismatch" + result = false elif x.flags != y.flags: echo "symbol flag mismatch: ", x.flags, "/", y.flags result = false @@ -229,6 +252,10 @@ proc eql(x, y: PNode; c: var EqlContext): bool = debug(x) debug(y) result = false + elif not eql(x.info, y. info, c): + echo "node lineinfo mismatch at ", `$`(c.confX, x.info) + debug(x) + result = false elif x.safeLen == y.safeLen: if c.nodeStack.len != 0: for i in countDown(c.nodeStack.len - 1, 0): From 6f237be68665da8d201dcf8162fe25772989c769 Mon Sep 17 00:00:00 2001 From: demotomohiro Date: Wed, 29 Oct 2025 14:03:14 +0900 Subject: [PATCH 06/38] saves/loads symbol magic, options and offset --- compiler/icnif/nifdecoder.nim | 9 +++++++++ compiler/icnif/nifencoder.nim | 6 ++++++ tests/icnif/tencode_node2node.nim | 9 +++++++++ 3 files changed, 24 insertions(+) diff --git a/compiler/icnif/nifdecoder.nim b/compiler/icnif/nifdecoder.nim index aca9ac47dd5b..77cbff5517a8 100644 --- a/compiler/icnif/nifdecoder.nim +++ b/compiler/icnif/nifdecoder.nim @@ -98,7 +98,13 @@ proc fromNifSymDef(c: var DecodeContext; n: var Cursor): PSym = incExpect n, Ident let ident = c.graph.cache.getIdent(pool.strings[n.litId]) incExpect n, {Ident, DotToken} + let magic = if n.kind == Ident: pool.strings[n.litId].parseMagic else: mNone + incExpect n, {Ident, DotToken} let flags = if n.kind == Ident: pool.strings[n.litId].parseSymFlags else: {} + incExpect n, {Ident, DotToken} + let options = if n.kind == Ident: pool.strings[n.litId].parseOptions else: {} + incExpect n, IntLit + let offset = pool.integers[n.intId].int32 incExpect n, IntLit let disamb = pool.integers[n.intId].int32 incExpect n, ParLe @@ -107,9 +113,12 @@ proc fromNifSymDef(c: var DecodeContext; n: var Cursor): PSym = result = PSym(itemId: ItemId(module: itemIdModule.int32, item: itemId), kind: kind, + magic: magic, name: ident, info: info, flags: flags, + options: options, + offset: offset, disamb: disamb) # PNode, PSym or PType type fields in PSym can have cycles. diff --git a/compiler/icnif/nifencoder.nim b/compiler/icnif/nifencoder.nim index 6a91064ac857..0f7b6720f4a6 100644 --- a/compiler/icnif/nifencoder.nim +++ b/compiler/icnif/nifencoder.nim @@ -59,7 +59,13 @@ proc toNifDef(c: var EncodeContext; sym: PSym) = c.toNifModuleId sym.itemId.module c.dest.addIntLit sym.itemId.item c.dest.addIdent sym.name.s + if sym.magic == mNone: + c.dest.addDotToken + else: + c.dest.addIdent toNifTag(sym.magic) c.dest.writeFlags sym.flags + c.dest.writeFlags sym.options + c.dest.addIntLit sym.offset c.dest.addIntLit sym.disamb c.dest.buildTree sym.kind.toNifTag: case sym.kind diff --git a/tests/icnif/tencode_node2node.nim b/tests/icnif/tencode_node2node.nim index b68854271744..c71f53c63ce2 100644 --- a/tests/icnif/tencode_node2node.nim +++ b/tests/icnif/tencode_node2node.nim @@ -143,14 +143,23 @@ proc eql(x, y: PSym; c: var EqlContext): bool = elif x.kind != y.kind: echo "symbol kind mismatch: ", x.kind, "/", y.kind result = false + elif x.magic != y.magic: + echo "symbol magic mismatch: ", x.magic, "/", y.magic + result = false elif not eql(x.info, y.info, c): echo "symbol line info mismatch" result = false elif x.flags != y.flags: echo "symbol flag mismatch: ", x.flags, "/", y.flags result = false + elif x.options != y.options: + echo "symbol options mismatch: ", x.options, "/", y.options + result = false elif not eqlSymPos(x, y, c): result = false + elif x.offset != y.offset: + echo "symbol offset mismatch: ", x.offset, "/", y.offset + result = false elif x.disamb != y.disamb: echo "symbol disamb mismatch: ", x.disamb, "/", y.disamb result = false From b50c453b140b29b24d340d7b0aa61791bfca405d Mon Sep 17 00:00:00 2001 From: demotomohiro Date: Wed, 29 Oct 2025 18:26:45 +0900 Subject: [PATCH 07/38] adds procedure test code and fix bugs --- compiler/icnif/nifdecoder.nim | 6 ++++-- compiler/icnif/nifencoder.nim | 3 ++- tests/icnif/tencode_node2node.nim | 1 + tests/icnif/testcode/modtestprocs.nim | 9 +++++++++ 4 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 tests/icnif/testcode/modtestprocs.nim diff --git a/compiler/icnif/nifdecoder.nim b/compiler/icnif/nifdecoder.nim index 77cbff5517a8..08ac2040648f 100644 --- a/compiler/icnif/nifdecoder.nim +++ b/compiler/icnif/nifdecoder.nim @@ -261,7 +261,9 @@ proc fromNif(c: var DecodeContext; n: var Cursor): PNode = case kind: of nkEmpty: result = newNodeI(nkEmpty, c.fromNifLineInfo(n)) - inc n + incExpect n, {Ident, DotToken} + let flags = fromNifNodeFlags n + result.flags = flags skipParRi n of nkIdent: incExpect n, Ident @@ -304,7 +306,7 @@ proc fromNif(c: var DecodeContext; n: var Cursor): PNode = else: c.withNode n, result, kind: while n.kind != ParRi: - result.add c.fromNif n + result.addAllowNil c.fromNif n else: assert false, "Not yet implemented " & $n.kind diff --git a/compiler/icnif/nifencoder.nim b/compiler/icnif/nifencoder.nim index 0f7b6720f4a6..0903144c8035 100644 --- a/compiler/icnif/nifencoder.nim +++ b/compiler/icnif/nifencoder.nim @@ -143,6 +143,7 @@ proc toNif(c: var EncodeContext; n: PNode) = of nkEmpty: let info = c.toNif n.info c.dest.addParLe pool.tags.getOrIncl(toNifTag(nkEmpty)), info + c.dest.writeNodeFlags(n.flags) c.dest.addParRi of nkIdent: let info = c.toNif n.info @@ -187,7 +188,7 @@ proc toNif(c: var EncodeContext; n: PNode) = c.withNode n: discard else: - assert n.len > 0, $n.kind + assert n.kind in {nkArgList, nkBracket} or n.len > 0, $n.kind c.withNode(n): for i in 0 ..< n.len: c.toNif n[i] diff --git a/tests/icnif/tencode_node2node.nim b/tests/icnif/tencode_node2node.nim index c71f53c63ce2..923057c4260f 100644 --- a/tests/icnif/tencode_node2node.nim +++ b/tests/icnif/tencode_node2node.nim @@ -330,3 +330,4 @@ testNifEncDec(graph, "modtest1.nim") testNifEncDec(graph, "modtestliterals.nim") testNifEncDec(graph, "modtesttypesections.nim") testNifEncDec(graph, "modtestpragmas.nim") +testNifEncDec(graph, "modtestprocs.nim") diff --git a/tests/icnif/testcode/modtestprocs.nim b/tests/icnif/testcode/modtestprocs.nim new file mode 100644 index 000000000000..6f2f0eab00df --- /dev/null +++ b/tests/icnif/testcode/modtestprocs.nim @@ -0,0 +1,9 @@ +proc foo() = discard +proc bar(x: int): int = x +proc baz(x, y: int): string = $(x + y) + +proc baz(a: bool; b: string; c: int): float = + if a and b == "" and c == 0: + result = 0.0 + else: + result = 1.0 From 3ceb223089632d8cafd296d8f9a70ead1fa3906f Mon Sep 17 00:00:00 2001 From: demotomohiro Date: Wed, 29 Oct 2025 19:08:08 +0900 Subject: [PATCH 08/38] adds more procedure test code --- tests/icnif/testcode/modtestprocs.nim | 38 +++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tests/icnif/testcode/modtestprocs.nim b/tests/icnif/testcode/modtestprocs.nim index 6f2f0eab00df..f17966586b31 100644 --- a/tests/icnif/testcode/modtestprocs.nim +++ b/tests/icnif/testcode/modtestprocs.nim @@ -7,3 +7,41 @@ proc baz(a: bool; b: string; c: int): float = result = 0.0 else: result = 1.0 + +proc forwardDecl() +proc forwardDecl() = + foo() + +forwardDecl() + +proc forwardDecl2*(): int +proc forwardDecl2*(): int = bar(1) + +discard forwardDecl2() + +proc forwardDecl3(x, y: int): int +proc forwardDecl3(x, y: int): int = x - y + +discard forwardDecl3(3, 2) + +func func1(): int = 123 +discard func1() +func func2(x: int): int = x +func func3*(x, y: bool): bool = x and y + +proc withDefaultValue(x = 1) = echo x +withDefaultValue() +withDefaultValue(2) +withDefaultValue(x = 3) + +proc withDefaultValue2(x = "foo"; y = true) = echo x, y +withDefaultValue2() +withDefaultValue2("bar") +withDefaultValue2(x = "baz", y = false) + +proc varParam(x: var int) = x = 10 +var x = 0 +varParam(x) + +proc varRet(x: var int): var int = x +varRet(x) = 100 From 10885d4dce5d0ebde80b2c0948e486575325857d Mon Sep 17 00:00:00 2001 From: demotomohiro Date: Wed, 29 Oct 2025 21:36:28 +0900 Subject: [PATCH 09/38] adds statements test code --- tests/icnif/tencode_node2node.nim | 1 + tests/icnif/testcode/modteststatements.nim | 44 ++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 tests/icnif/testcode/modteststatements.nim diff --git a/tests/icnif/tencode_node2node.nim b/tests/icnif/tencode_node2node.nim index 923057c4260f..0b3310fba01a 100644 --- a/tests/icnif/tencode_node2node.nim +++ b/tests/icnif/tencode_node2node.nim @@ -331,3 +331,4 @@ testNifEncDec(graph, "modtestliterals.nim") testNifEncDec(graph, "modtesttypesections.nim") testNifEncDec(graph, "modtestpragmas.nim") testNifEncDec(graph, "modtestprocs.nim") +testNifEncDec(graph, "modteststatements.nim") diff --git a/tests/icnif/testcode/modteststatements.nim b/tests/icnif/testcode/modteststatements.nim new file mode 100644 index 000000000000..f82c258c0206 --- /dev/null +++ b/tests/icnif/testcode/modteststatements.nim @@ -0,0 +1,44 @@ +var stmtListExpr = (echo "foo"; 111) + +if false: + discard +elif false: + discard +else: + discard + +var caseExpr = true +case caseExpr +of true: + discard +else: + discard + +when sizeof(int) == 2: + echo "running on a 16 bit system!" +elif sizeof(int) == 4: + echo "running on a 32 bit system!" +elif sizeof(int) == 8: + echo "running on a 64 bit system!" +else: + echo "cannot happen!" + +while true: + break + +var x = 2 + +while x != 0: + if x == 2: + x = 0 + continue + else: + break + +block testblock: + while true: + if x > -1: + break testblock + +for i in 0 .. 3: + discard i From 6a74316c0e0fd22dcec4a7e1effe4c9306365733 Mon Sep 17 00:00:00 2001 From: demotomohiro Date: Wed, 29 Oct 2025 22:50:06 +0900 Subject: [PATCH 10/38] fixes assertion defect when encoding empty object type definition --- compiler/icnif/nifencoder.nim | 3 ++- tests/icnif/testcode/modtesttypesections.nim | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/compiler/icnif/nifencoder.nim b/compiler/icnif/nifencoder.nim index 0903144c8035..dccdcbd08dd9 100644 --- a/compiler/icnif/nifencoder.nim +++ b/compiler/icnif/nifencoder.nim @@ -99,6 +99,7 @@ proc toNifDef(c: var EncodeContext; typ: PType) = c.dest.buildTree sonsTag: for ch in typ.kids: c.toNif ch + c.toNif typ.n c.toNif typ.owner c.toNif typ.sym @@ -188,7 +189,7 @@ proc toNif(c: var EncodeContext; n: PNode) = c.withNode n: discard else: - assert n.kind in {nkArgList, nkBracket} or n.len > 0, $n.kind + assert n.kind in {nkArgList, nkBracket, nkRecList} or n.len > 0, $n.kind c.withNode(n): for i in 0 ..< n.len: c.toNif n[i] diff --git a/tests/icnif/testcode/modtesttypesections.nim b/tests/icnif/testcode/modtesttypesections.nim index 7ff5b0e5be67..d858b5af9a05 100644 --- a/tests/icnif/testcode/modtesttypesections.nim +++ b/tests/icnif/testcode/modtesttypesections.nim @@ -21,6 +21,8 @@ type TestPtrObj = ptr object x: int + TestEmptyObj = object + var x: TestInt var testEnum: TestEnum var testEnum1 = X @@ -34,3 +36,4 @@ var testPtrInt: TestPtrInt = nil var testPtrInt2: ptr int = nil var testRefObj: TestRefObj = nil var testPtrObj: TestPtrObj = nil +var testEmptyObj: TestEmptyObj From 467a27948cba7a179ab3368d55a88a8771362db3 Mon Sep 17 00:00:00 2001 From: demotomohiro Date: Thu, 30 Oct 2025 11:23:21 +0900 Subject: [PATCH 11/38] adds pragma test code --- tests/icnif/testcode/modtestpragmas.nim | 45 +++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/tests/icnif/testcode/modtestpragmas.nim b/tests/icnif/testcode/modtestpragmas.nim index c20c87fefce5..a9a64b6f398e 100644 --- a/tests/icnif/testcode/modtestpragmas.nim +++ b/tests/icnif/testcode/modtestpragmas.nim @@ -1,3 +1,48 @@ var exportcTest {.exportc.}: int var importcTest {.importc.}: int var y* {.importc, header: "test.h".}: int +var EACCES {.importc, nodecl.}: cint +var volatileTest {.volatile.}: int + +const FooBar {.intdefine.}: int = 5 +echo FooBar + +{.passc: "-Wall -Werror".} +{.link: "myfile.o".} + +type + TestImportC {.importc.} = object + x: int + + TestImportC2 {.importc: "TestImportC2Name".} = object + x {.importc.}: int + y {.importc: "yyy".}: int + + TestBitfield = object + flag {.bitsize:1.}: cuint + + TestEnumWithSize* {.size: sizeof(uint32).} = enum + X, + Y, + Z + + sseType = object + sseData {.align(16).}: array[4, float32] + + TestUnionObj {.union.} = object + x: cint + y: cfloat + +const irr = "" +type + IrrlichtDeviceObj {.header: irr, + importcpp: "irr::IrrlichtDevice".} = object + +proc importCProc() {.importc.} +proc importCProc2(x: cint) {.importc: "import_c_proc_2".} +proc headerProc(): cint {.importc, header: "foo.h".} + +proc exportCProc() {.exportc.} + +{.pragma: pragmaPragmaTest, importc, header: "foo.h".} +proc pragmaPragmaTestProc() {.pragmaPragmaTest.} From 66d7a8e0b4b09daf66952f5b014965d6a4f395c8 Mon Sep 17 00:00:00 2001 From: demotomohiro Date: Thu, 30 Oct 2025 12:53:49 +0900 Subject: [PATCH 12/38] adds generics test code and fixes bug --- compiler/icnif/nifdecoder.nim | 5 ++- compiler/icnif/nifencoder.nim | 3 +- tests/icnif/tencode_node2node.nim | 1 + tests/icnif/testcode/modtestgenerics.nim | 43 ++++++++++++++++++++++++ 4 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 tests/icnif/testcode/modtestgenerics.nim diff --git a/compiler/icnif/nifdecoder.nim b/compiler/icnif/nifdecoder.nim index 08ac2040648f..f10e49c16865 100644 --- a/compiler/icnif/nifdecoder.nim +++ b/compiler/icnif/nifdecoder.nim @@ -266,9 +266,12 @@ proc fromNif(c: var DecodeContext; n: var Cursor): PNode = result.flags = flags skipParRi n of nkIdent: - incExpect n, Ident + inc n + let typ = c.fromNifType n + expect n, Ident result = newIdentNode(c.graph.cache.getIdent(pool.strings[n.litId]), c.fromNifLineInfo(n)) inc n + result.typ = typ skipParRi n of nkSym: c.withNode n, result, kind: diff --git a/compiler/icnif/nifencoder.nim b/compiler/icnif/nifencoder.nim index dccdcbd08dd9..fad4e7de4d0a 100644 --- a/compiler/icnif/nifencoder.nim +++ b/compiler/icnif/nifencoder.nim @@ -149,6 +149,7 @@ proc toNif(c: var EncodeContext; n: PNode) = of nkIdent: let info = c.toNif n.info c.dest.addParLe pool.tags.getOrIncl(toNifTag(nkIdent)), info + c.toNif n.typ c.dest.addIdent n.ident.s c.dest.addParRi of nkSym: @@ -189,7 +190,7 @@ proc toNif(c: var EncodeContext; n: PNode) = c.withNode n: discard else: - assert n.kind in {nkArgList, nkBracket, nkRecList} or n.len > 0, $n.kind + assert n.kind in {nkArgList, nkBracket, nkRecList, nkPragma} or n.len > 0, $n.kind c.withNode(n): for i in 0 ..< n.len: c.toNif n[i] diff --git a/tests/icnif/tencode_node2node.nim b/tests/icnif/tencode_node2node.nim index 0b3310fba01a..6054368d0235 100644 --- a/tests/icnif/tencode_node2node.nim +++ b/tests/icnif/tencode_node2node.nim @@ -332,3 +332,4 @@ testNifEncDec(graph, "modtesttypesections.nim") testNifEncDec(graph, "modtestpragmas.nim") testNifEncDec(graph, "modtestprocs.nim") testNifEncDec(graph, "modteststatements.nim") +testNifEncDec(graph, "modtestgenerics.nim") diff --git a/tests/icnif/testcode/modtestgenerics.nim b/tests/icnif/testcode/modtestgenerics.nim new file mode 100644 index 000000000000..dbf023e88c53 --- /dev/null +++ b/tests/icnif/testcode/modtestgenerics.nim @@ -0,0 +1,43 @@ +type + GenericsType[T] = object + GenericsType2[T] = object + x: T + GenericsType3[T, U] = object + x: T + y: U + GenericsType4[T: int or bool] = distinct int + GenericsType5[N: static[int]; T] = object + x: array[N, T] + GenericsType6[T: SomeNumber] = object + x: T + +var genericsType: GenericsType[int] +var genericsType2: GenericsType2[int] +var genericsType3: GenericsType3[float, string] +var genericsType4: GenericsType4[bool] +var genericsType5: GenericsType5[3, float] +var genericsType6: GenericsType6[int8] + +proc genericsProc[T]() = discard +genericsProc[int]() + +proc genericsProc2[T](x: T) = discard x +genericsProc2(123) + +proc genericsProc3[T, U](x: T; y: U) = discard x +genericsProc3("foo", true) + +proc genericsProc4[T: int or float](x: T): T = x +discard genericsProc4(1.23) + +proc genericsProc5(x: static[int]) = discard x +genericsProc5(123) + +proc genericsProc6[T: SomeNumber](x: T) = discard x +genericsProc6(321) + +proc genericsProc7[T: SomeNumber](x: T): T = x +discard genericsProc7(321) + +proc genericsProc8[N: static[int]; T](x: array[N, T]): T = x[0] +discard genericsProc8([1, 2, 3]) From 3c599ed7db20aeaa18c77a6beaecec4681b8ef0e Mon Sep 17 00:00:00 2001 From: demotomohiro Date: Thu, 30 Oct 2025 21:26:39 +0900 Subject: [PATCH 13/38] adds expression test code --- compiler/icnif/nifencoder.nim | 2 +- tests/icnif/tencode_node2node.nim | 1 + tests/icnif/testcode/modtestexprs.nim | 79 +++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 tests/icnif/testcode/modtestexprs.nim diff --git a/compiler/icnif/nifencoder.nim b/compiler/icnif/nifencoder.nim index fad4e7de4d0a..bae7b039ad3a 100644 --- a/compiler/icnif/nifencoder.nim +++ b/compiler/icnif/nifencoder.nim @@ -190,7 +190,7 @@ proc toNif(c: var EncodeContext; n: PNode) = c.withNode n: discard else: - assert n.kind in {nkArgList, nkBracket, nkRecList, nkPragma} or n.len > 0, $n.kind + assert n.kind in {nkArgList, nkBracket, nkRecList, nkPragma, nkType} or n.len > 0, $n.kind c.withNode(n): for i in 0 ..< n.len: c.toNif n[i] diff --git a/tests/icnif/tencode_node2node.nim b/tests/icnif/tencode_node2node.nim index 6054368d0235..564ea8606a18 100644 --- a/tests/icnif/tencode_node2node.nim +++ b/tests/icnif/tencode_node2node.nim @@ -333,3 +333,4 @@ testNifEncDec(graph, "modtestpragmas.nim") testNifEncDec(graph, "modtestprocs.nim") testNifEncDec(graph, "modteststatements.nim") testNifEncDec(graph, "modtestgenerics.nim") +testNifEncDec(graph, "modtestexprs.nim") diff --git a/tests/icnif/testcode/modtestexprs.nim b/tests/icnif/testcode/modtestexprs.nim new file mode 100644 index 000000000000..1ffee629055c --- /dev/null +++ b/tests/icnif/testcode/modtestexprs.nim @@ -0,0 +1,79 @@ +type + FooEnum = enum + X, + Y, + Z + +var enumTest = X +enumTest = Y +assert enumTest == Y + +var enumSet = {X, Y} +enumSet.incl Z + +var intArray = [1, 1 + 1, 1 * 2, 0] +intArray[3] = intArray[1] + intArray[0] * intArray[2] +var strArray = ["foo", "ba" & "r", ""] +strArray[2] = $(intArray[2]) +var floatArray = [intArray[0].float, 1.0, 0.0] +floatArray[2] = floatArray[0] + floatArray[1] +var intSeq = @[3, 2] +intSeq.add 1 + +var tup1 = (foo: "Foo", bar: 123) +tup1.foo = "Bar" +tup1.bar = 321 +tup1[0] = "Baz" +assert tup1 is (string, int) +let (tup1foo, tup1bar) = tup1 +let (_, tup1bar2) = tup1 +let (tup1foo2, _) = tup1 + +var testAddr: int +var testPtr = addr testAddr +testPtr[] = 123 + +var testAddr2: array[2, int] +var testPtr2: ptr int +testPtr2 = addr testAddr2[0] +var testPointer: pointer = testPtr2 +testPtr2 = cast[ptr int](testPointer) + +var stmtListExpr = (echo "foo"; "stmtListExpr") +var cond1 = true +var testIfExpr = if cond1: 1 else: -1 +const TestWhenExpr = when sizeof(int) == 8: 1 else: -1 +var cond2: FooEnum = X +var testCaseExpr = case cond2 + of X: + 1 + of Y: + 2 + else: + 3 + +var testBlockExpr = block: + var a = "test" + a + +var testTryExpr = try: + if cond1: + 222 + else: + raise newException(CatchableError, "test") + except CatchableError: + -123 + +var testTryExpr2 = try: + if cond1: + 333 + else: + raise newException(CatchableError, "test") + except CatchableError as e: + echo e.msg + -1234 + finally: + echo "finally" + +proc getNum(a: int): int = a +echo static(getNum(123)) From 91db9ac8f5d1a1632fa65f939a9f219f53060894 Mon Sep 17 00:00:00 2001 From: demotomohiro Date: Thu, 30 Oct 2025 23:29:19 +0900 Subject: [PATCH 14/38] uses ItemId instead of PSym or PType for the set of decoded syms and types --- compiler/icnif/nifencoder.nim | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/compiler/icnif/nifencoder.nim b/compiler/icnif/nifencoder.nim index bae7b039ad3a..6c5b0686e46a 100644 --- a/compiler/icnif/nifencoder.nim +++ b/compiler/icnif/nifencoder.nim @@ -6,8 +6,8 @@ import enum2nif, icniftags type EncodeContext = object conf: ConfigRef - decodedSyms: HashSet[PSym] - decodedTypes: HashSet[PType] + decodedSyms: HashSet[ItemId] + decodedTypes: HashSet[ItemId] decodedFileIndices: HashSet[FileIndex] dest: TokenBuf @@ -108,7 +108,7 @@ proc toNif(c: var EncodeContext; sym: PSym) = if sym == nil: c.dest.addDotToken() else: - if not c.decodedSyms.containsOrIncl(sym): + if not c.decodedSyms.containsOrIncl(sym.itemId): c.toNifDef sym else: c.dest.buildTree symTag: @@ -119,7 +119,7 @@ proc toNif(c: var EncodeContext; typ: PType) = if typ == nil: c.dest.addDotToken() else: - if not c.decodedTypes.containsOrIncl(typ): + if not c.decodedTypes.containsOrIncl(typ.itemId): c.toNifDef typ else: c.dest.buildTree typeTag: @@ -190,7 +190,7 @@ proc toNif(c: var EncodeContext; n: PNode) = c.withNode n: discard else: - assert n.kind in {nkArgList, nkBracket, nkRecList, nkPragma, nkType} or n.len > 0, $n.kind + #assert n.kind in {nkArgList, nkBracket, nkRecList, nkPragma, nkType} or n.len > 0, $n.kind c.withNode(n): for i in 0 ..< n.len: c.toNif n[i] From 9f73412e932833879111aa93caa70285b1bbe467 Mon Sep 17 00:00:00 2001 From: demotomohiro Date: Fri, 31 Oct 2025 18:39:51 +0900 Subject: [PATCH 15/38] saves/loads float Inf, NaN and NegInf literals --- compiler/icnif/nifdecoder.nim | 20 +++++++++++++++++--- tests/icnif/tencode_node2node.nim | 21 +++++++++++++++++++-- tests/icnif/testcode/modtestliterals.nim | 7 +++++++ 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/compiler/icnif/nifdecoder.nim b/compiler/icnif/nifdecoder.nim index f10e49c16865..03b008690c13 100644 --- a/compiler/icnif/nifdecoder.nim +++ b/compiler/icnif/nifdecoder.nim @@ -293,9 +293,23 @@ proc fromNif(c: var DecodeContext; n: var Cursor): PNode = inc n of nkFloatLit .. nkFloat128Lit: c.withNode n, result, kind: - expect n, FloatLit - result.floatVal = pool.floats[n.floatId] - inc n + if n.kind == FloatLit: + result.floatVal = pool.floats[n.floatId] + inc n + elif n.kind == ParLe: + case pool.tags[n.tagId] + of "inf": + result.floatVal = Inf + of "nan": + result.floatVal = NaN + of "neginf": + result.floatVal = NegInf + else: + assert false, "expected float literal but got " & pool.tags[n.tagId] + inc n + skipParRi n + else: + assert false, "expected float literal but got " & $n.kind of nkStrLit .. nkTripleStrLit: c.withNode n, result, kind: expect n, StringLit diff --git a/tests/icnif/tencode_node2node.nim b/tests/icnif/tencode_node2node.nim index 564ea8606a18..a6c183a9287d 100644 --- a/tests/icnif/tencode_node2node.nim +++ b/tests/icnif/tencode_node2node.nim @@ -1,4 +1,4 @@ -import std/assertions +import std/[assertions, math] import "../../compiler/icnif" / [nifencoder, nifdecoder] import "../../compiler" / [idents, ast, astalgo, options, pathutils, modulegraphs, modules, msgs, pipelines, syntaxes, sem, llstream, lineinfos] @@ -294,8 +294,25 @@ proc eql(x, y: PNode; c: var EqlContext): bool = debug(y.sym) debug(x.sym.typ) debug(y.sym.typ) - of nkCharLit .. nkTripleStrLit: + of nkCharLit .. nkUInt64Lit, nkStrLit .. nkTripleStrLit: result = sameValue(x, y) + of nkFloatLit .. nkFloat128Lit: + # want to know if x and y are identical float value. + # so x == y doesn't work if both x and y are NaN or x == 0 and y == -0. + let xc = classify(x.floatVal) + let yc = classify(y.floatVal) + if xc == yc: + if xc in {fcNormal, fcSubnormal}: + if x.floatVal != y.floatVal: + echo "float literal mismatch: ", x.floatVal, "/", y.floatVal + result = false + else: + result = true + else: + result = true + else: + echo "float literal mismatch: ", xc, "/", yc + result = false else: result = true for i in 0 ..< x.safeLen: diff --git a/tests/icnif/testcode/modtestliterals.nim b/tests/icnif/testcode/modtestliterals.nim index 2d11cf8fa91f..bb63ceb26595 100644 --- a/tests/icnif/testcode/modtestliterals.nim +++ b/tests/icnif/testcode/modtestliterals.nim @@ -23,7 +23,14 @@ var uint32litH = 4294967295'u32 var uint64litH = 18446744073709551615'u64 var floatlit = 1.25 +var floatlit2 = -1.25 var float32lit = 1.25'f32 var float64lit = 1.25'f64 +var floatZero = 0.0 +# Needs newer Nimony to save `-0.0` to NIF correclty +#var floatNegZero = -0.0 +var floatInf = Inf +var floatNaN = NaN +var floatNegInf = NegInf var nillit: ptr int = nil From f3178e4515e1a9f38a7339dcf02c9e64630e2b27 Mon Sep 17 00:00:00 2001 From: demotomohiro Date: Sat, 1 Nov 2025 10:14:38 +0900 Subject: [PATCH 16/38] saves/loads flags of nkIdent node --- compiler/icnif/nifdecoder.nim | 7 +++++-- compiler/icnif/nifencoder.nim | 8 +++----- tests/icnif/tencode_node2node.nim | 4 ++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/compiler/icnif/nifdecoder.nim b/compiler/icnif/nifdecoder.nim index 03b008690c13..0203b6e37dd3 100644 --- a/compiler/icnif/nifdecoder.nim +++ b/compiler/icnif/nifdecoder.nim @@ -266,11 +266,14 @@ proc fromNif(c: var DecodeContext; n: var Cursor): PNode = result.flags = flags skipParRi n of nkIdent: - inc n + let info = c.fromNifLineInfo(n) + incExpect n, {DotToken, Ident} + let flags = fromNifNodeFlags n let typ = c.fromNifType n expect n, Ident - result = newIdentNode(c.graph.cache.getIdent(pool.strings[n.litId]), c.fromNifLineInfo(n)) + result = newIdentNode(c.graph.cache.getIdent(pool.strings[n.litId]), info) inc n + result.flags = flags result.typ = typ skipParRi n of nkSym: diff --git a/compiler/icnif/nifencoder.nim b/compiler/icnif/nifencoder.nim index 6c5b0686e46a..78e9951f7097 100644 --- a/compiler/icnif/nifencoder.nim +++ b/compiler/icnif/nifencoder.nim @@ -147,11 +147,9 @@ proc toNif(c: var EncodeContext; n: PNode) = c.dest.writeNodeFlags(n.flags) c.dest.addParRi of nkIdent: - let info = c.toNif n.info - c.dest.addParLe pool.tags.getOrIncl(toNifTag(nkIdent)), info - c.toNif n.typ - c.dest.addIdent n.ident.s - c.dest.addParRi + # nkIdent uses flags and typ when it is a generic parameter + c.withNode n: + c.dest.addIdent n.ident.s of nkSym: when false: echo "nkSym: ", n.sym.name.s diff --git a/tests/icnif/tencode_node2node.nim b/tests/icnif/tencode_node2node.nim index a6c183a9287d..6c6797352db2 100644 --- a/tests/icnif/tencode_node2node.nim +++ b/tests/icnif/tencode_node2node.nim @@ -257,7 +257,7 @@ proc eql(x, y: PNode; c: var EqlContext): bool = echo "node kind mismatch: ", x.kind, "/", y.kind result = false elif x.flags != y.flags: - echo "node flag mismatch: ", x.flags, "/", y.flags + echo "node flag mismatch: ", x.flags, "/", y.flags, " at ", `$`(c.confX, x.info) debug(x) debug(y) result = false @@ -338,7 +338,7 @@ proc testNifEncDec(graph: ModuleGraph; src: string) = let n2 = loadNifFromBuffer(nif, graphForLoad) #debug(n2) var c = EqlContext(confX: graph.config, confY: graphForLoad.config) - assert eql(n, n2, c) + assert eql(n, n2, c), "test failed: " & $fullPath var conf = newConfigRefForTest() var cache = newIdentCache() From 5f3be1e991229891b8a6e74766285f4c86c6198a Mon Sep 17 00:00:00 2001 From: demotomohiro Date: Sat, 1 Nov 2025 22:02:31 +0900 Subject: [PATCH 17/38] makes tencode_node2node.nim faster --- tests/icnif/tencode_node2node.nim | 44 ++++++++++++++++++------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/tests/icnif/tencode_node2node.nim b/tests/icnif/tencode_node2node.nim index 6c6797352db2..2ff7db25989c 100644 --- a/tests/icnif/tencode_node2node.nim +++ b/tests/icnif/tencode_node2node.nim @@ -1,4 +1,4 @@ -import std/[assertions, math] +import std/[assertions, math, tables] import "../../compiler/icnif" / [nifencoder, nifdecoder] import "../../compiler" / [idents, ast, astalgo, options, pathutils, modulegraphs, modules, msgs, pipelines, syntaxes, sem, llstream, lineinfos] @@ -61,8 +61,9 @@ type # this is used to prevent that happen. EqlContext = object nodeStack: seq[PNode] - symStack: seq[PSym] - typStack: seq[PType] + checkedSyms: Table[ItemId, PSym] # used to check if each PSym has unique ItemId + # and also prevents inifinite loop + checkedTypes: Table[ItemId, PType]# used like checkedSyms confX, confY: ConfigRef # used to print the line info when there is a mismatch # and get path from FileIndex @@ -134,12 +135,21 @@ proc eql(x, y: PSym; c: var EqlContext): bool = elif x == nil or y == nil: echo "symbol is missing" result = false - elif x.name.s != y.name.s: - echo "symbol name mismatch: ", x.name.s, "/", y.name.s - result = false elif not eqlItemId(x.itemId, y.itemId, c): echo "symbol itemId mismatch" result = false + elif c.checkedSyms.hasKeyOrPut(y.itemId, y): + if c.checkedSyms[y.itemId] == y: + result = true + else: + echo "detected duplicated symbol ItemId:" + debug(x) + debug(c.checkedSyms[y.itemId]) + debug(y) + result = false + elif x.name.s != y.name.s: + echo "symbol name mismatch: ", x.name.s, "/", y.name.s + result = false elif x.kind != y.kind: echo "symbol kind mismatch: ", x.kind, "/", y.kind result = false @@ -167,11 +177,6 @@ proc eql(x, y: PSym; c: var EqlContext): bool = echo "symbol.loc mismatch" result = false else: - if c.symStack.len != 0: - for i in countDown(c.symStack.len - 1, 0): - if x == c.symStack[i]: - return true - c.symStack.add x if not eql(x.typ, y.typ, c): echo "symbol type mismatch:" result = false @@ -195,7 +200,6 @@ proc eql(x, y: PSym; c: var EqlContext): bool = result = true else: result = true - discard c.symStack.pop proc eql(x, y: PType; c: var EqlContext): bool = if x == nil and y == nil: @@ -206,6 +210,15 @@ proc eql(x, y: PType; c: var EqlContext): bool = elif not eqlItemId(x.itemId, y.itemId, c): echo "type itemId mismatch" result = false + elif c.checkedTypes.hasKeyOrPut(y.itemId, y): + if c.checkedTypes[y.itemId] == y: + result = true + else: + echo "detected duplicated type ItemId:" + debug(x) + debug(c.checkedTypes[y.itemId]) + debug(y) + result = false elif x.kind != y.kind: echo "type kind mismatch: ", x.kind, "/", y.kind result = false @@ -213,12 +226,6 @@ proc eql(x, y: PType; c: var EqlContext): bool = echo "type flag mismatch: ", x.flags, "/", y.flags result = false else: - if c.typStack.len != 0: - for i in countDown(c.typStack.len - 1, 0): - if x == c.typStack[i]: - # echo "cycle is detected in PType" - return true - c.typStack.add x if not eql(x.n, y.n, c): echo "type.n mismatch" debug(x.n) @@ -246,7 +253,6 @@ proc eql(x, y: PType; c: var EqlContext): bool = debug(y[i]) result = false break - discard c.typStack.pop proc eql(x, y: PNode; c: var EqlContext): bool = if x == nil and y == nil: From ab13900fadb7e77e133708a1e8d5c3761906f255 Mon Sep 17 00:00:00 2001 From: demotomohiro Date: Sat, 1 Nov 2025 23:05:41 +0900 Subject: [PATCH 18/38] saves/loads PSym.ast, constraint and instantiatedFrom --- compiler/icnif/nifdecoder.nim | 3 +++ compiler/icnif/nifencoder.nim | 3 +++ tests/icnif/tencode_node2node.nim | 11 +++++++++++ 3 files changed, 17 insertions(+) diff --git a/compiler/icnif/nifdecoder.nim b/compiler/icnif/nifdecoder.nim index 0203b6e37dd3..6c5e8b7834f9 100644 --- a/compiler/icnif/nifdecoder.nim +++ b/compiler/icnif/nifdecoder.nim @@ -149,12 +149,15 @@ proc fromNifSymDef(c: var DecodeContext; n: var Cursor): PSym = result.typ = c.fromNifType n result.setOwner(c.fromNifSymbol n) + result.ast = c.fromNif n expect n, Ident result.loc.k = pool.strings[n.litId].parseLocKind() incExpect n, StringLit result.loc.snippet.add pool.strings[n.litId] inc n + result.constraint = c.fromNif n + result.instantiatedFrom = c.fromNifSymbol n skipParRi n proc fromNifTypeDef(c: var DecodeContext; n: var Cursor): PType = diff --git a/compiler/icnif/nifencoder.nim b/compiler/icnif/nifencoder.nim index 78e9951f7097..eec6cab68918 100644 --- a/compiler/icnif/nifencoder.nim +++ b/compiler/icnif/nifencoder.nim @@ -81,8 +81,11 @@ proc toNifDef(c: var EncodeContext; sym: PSym) = c.dest.addIntLit sym.position c.toNif sym.typ c.toNif sym.owner + c.toNif sym.ast # drastically increase output NIF size! c.dest.addIdent toNifTag(sym.loc.k) c.dest.addStrLit sym.loc.snippet + c.toNif sym.constraint + c.toNif sym.instantiatedFrom c.dest.addParRi proc toNifDef(c: var EncodeContext; typ: PType) = diff --git a/tests/icnif/tencode_node2node.nim b/tests/icnif/tencode_node2node.nim index 2ff7db25989c..95923a20c029 100644 --- a/tests/icnif/tencode_node2node.nim +++ b/tests/icnif/tencode_node2node.nim @@ -185,6 +185,15 @@ proc eql(x, y: PSym; c: var EqlContext): bool = debug(x.owner) debug(y.owner) result = false + elif not eql(x.ast, y.ast, c): + echo "symbol ast mismatch" + result = false + elif not eql(x.constraint, y.constraint, c): + echo "symbol constraint mismatch" + result = false + elif not eql(x.instantiatedFrom, y.instantiatedFrom, c): + echo "symbol instantiatedFrom mismatch" + result = false else: if x.kind in {skLet, skVar, skField, skForVar}: if not eql(x.guard, y.guard, c): @@ -338,6 +347,8 @@ proc testNifEncDec(graph: ModuleGraph; src: string) = #debug(n) let nif = saveNifToBuffer(n, graph.config) #echo nif + #echo "NIF size of ", src, ": ", nif.len + #writeFile(src & ".nif", nif) # Don't reuse the ModuleGraph used for semcheck when load NIF. var graphForLoad = newModuleGraph(newIdentCache(), newConfigRefForTest()) From 54aa552fb6519bb3a9e772d36e2aed4be40f59a2 Mon Sep 17 00:00:00 2001 From: demotomohiro Date: Tue, 4 Nov 2025 17:16:10 +0900 Subject: [PATCH 19/38] refers imported symbols like Nimony --- compiler/icnif/nifbasics.nim | 19 ++++++++ compiler/icnif/nifdecoder.nim | 32 ++++++++++--- compiler/icnif/nifencoder.nim | 20 +++++--- tests/icnif/tencode_node2node.nim | 66 ++++++++++++++++++--------- tests/icnif/testcode/modtestprocs.nim | 21 +++++---- 5 files changed, 113 insertions(+), 45 deletions(-) create mode 100644 compiler/icnif/nifbasics.nim diff --git a/compiler/icnif/nifbasics.nim b/compiler/icnif/nifbasics.nim new file mode 100644 index 000000000000..72166c93a990 --- /dev/null +++ b/compiler/icnif/nifbasics.nim @@ -0,0 +1,19 @@ +import std/[tables] +import ".." / [ast, lineinfos, msgs, options] +import "../../dist/nimony/src/gear2" / modnames + +proc modname(moduleToNifSuffix: var Table[FileIndex, string]; module: PSym; conf: ConfigRef): string = + assert module.kind == skModule + let idx: FileIndex = module.position.FileIndex + # copied from ../nifgen.nim + result = moduleToNifSuffix.getOrDefault(idx) + if result.len == 0: + let fp = toFullPath(conf, idx) + result = moduleSuffix(fp, cast[seq[string]](conf.searchPaths)) + moduleToNifSuffix[idx] = result + #echo result, " -> ", fp + +proc toNifSym*(sym: PSym; moduleToNifSuffix: var Table[FileIndex, string]; conf: ConfigRef): string = + let module = sym.originatingModule + + result = sym.name.s & '.' & $sym.disamb & '.' & modname(moduleToNifSuffix, module, conf) diff --git a/compiler/icnif/nifdecoder.nim b/compiler/icnif/nifdecoder.nim index 6c5e8b7834f9..6935e295242c 100644 --- a/compiler/icnif/nifdecoder.nim +++ b/compiler/icnif/nifdecoder.nim @@ -1,14 +1,19 @@ import std / [assertions, tables] import "../../dist/nimony/src/lib" / [bitabs, nifreader, nifstreams, nifcursors, lineinfos] import ".." / [ast, idents, lineinfos, options, modules, modulegraphs, msgs, pathutils] -import enum2nif, icniftags +import enum2nif, icniftags, nifbasics type + NifProgram* = ref object + nifSymIdToPSym: Table[SymId, PSym] + moduleToNifSuffix: Table[FileIndex, string] # FileIndex (PSym.position) -> module suffix + DecodeContext = object graph: ModuleGraph symbols: Table[ItemId, PSym] types: Table[ItemId, PType] modules: Table[int, FileIndex] # maps module id in NIF to FileIndex of the module + prog: NifProgram proc nodeKind(n: Cursor): TNodeKind {.inline.} = assert n.kind == ParLe @@ -160,6 +165,11 @@ proc fromNifSymDef(c: var DecodeContext; n: var Cursor): PSym = result.instantiatedFrom = c.fromNifSymbol n skipParRi n + if sfExported in flags or kind == skModule: + let nifSym = toNifSym(result, c.prog.moduleToNifSuffix, c.graph.config) + let symId = pool.syms.getOrIncl(nifSym) + c.prog.nifSymIdToPSym[symId] = result + proc fromNifTypeDef(c: var DecodeContext; n: var Cursor): PType = expectTag n, typeIdTag inc n @@ -207,6 +217,14 @@ proc fromNifSymbol(c: var DecodeContext; n: var Cursor): PSym = if n.kind == DotToken: result = nil inc n + elif n.kind == Symbol: + if n.symId notin c.prog.nifSymIdToPSym: + # TODO: Support import statement and remove this branch + #echo pool.syms[n.symId], " is not found" + result = nil + else: + result = c.prog.nifSymIdToPSym[n.symId] + inc n else: expect n, ParLe if n.tagId == symIdTag: @@ -333,26 +351,26 @@ proc fromNif(c: var DecodeContext; n: var Cursor): PNode = else: assert false, "Not yet implemented " & $n.kind -proc loadNif(stream: var Stream; graph: ModuleGraph): PNode = +proc loadNif(stream: var Stream; graph: ModuleGraph; prog: NifProgram): PNode = discard processDirectives(stream.r) var buf = fromStream(stream) var n = beginRead(buf) - var c = DecodeContext(graph: graph) + var c = DecodeContext(graph: graph, prog: prog) result = fromNif(c, n) endRead(buf) -proc loadNifFile*(infile: AbsoluteFile; graph: ModuleGraph): PNode = +proc loadNifFile*(infile: AbsoluteFile; graph: ModuleGraph; prog: NifProgram): PNode = var stream = nifstreams.open(infile.string) - result = loadNif(stream, graph) + result = loadNif(stream, graph, prog) stream.close -proc loadNifFromBuffer*(strbuf: sink string; graph: ModuleGraph): PNode = +proc loadNifFromBuffer*(strbuf: sink string; graph: ModuleGraph; prog: NifProgram): PNode = var stream = nifstreams.openFromBuffer(strbuf) - result = loadNif(stream, graph) + result = loadNif(stream, graph, prog) when isMainModule: import std/cmdline diff --git a/compiler/icnif/nifencoder.nim b/compiler/icnif/nifencoder.nim index eec6cab68918..2e562aca7faf 100644 --- a/compiler/icnif/nifencoder.nim +++ b/compiler/icnif/nifencoder.nim @@ -1,18 +1,21 @@ -import std / [assertions, sets] +import std / [assertions, sets, tables] import ".." / [ast, idents, lineinfos, msgs, options] import "../../dist/nimony/src/lib" / [bitabs, nifstreams, nifcursors, lineinfos] -import enum2nif, icniftags +import enum2nif, icniftags, nifbasics type EncodeContext = object conf: ConfigRef + currentModule: PSym decodedSyms: HashSet[ItemId] decodedTypes: HashSet[ItemId] decodedFileIndices: HashSet[FileIndex] dest: TokenBuf + moduleToNifSuffix: Table[FileIndex, string] # FileIndex (PSym.position) -> module suffix -proc initEncodeContext(conf: ConfigRef): EncodeContext = +proc initEncodeContext(conf: ConfigRef; currentModule: PSym): EncodeContext = result = EncodeContext(conf: conf, + currentModule: currentModule, dest: createTokenBuf()) template buildTree(dest: var TokenBuf; tag: TagId; body: untyped) = @@ -110,6 +113,9 @@ proc toNifDef(c: var EncodeContext; typ: PType) = proc toNif(c: var EncodeContext; sym: PSym) = if sym == nil: c.dest.addDotToken() + elif sym.owner != nil and sym.originatingModule != c.currentModule: + let nifSym = toNifSym(sym, c.moduleToNifSuffix, c.conf) + c.dest.addSymUse(pool.syms.getOrIncl(nifSym), NoLineInfo) else: if not c.decodedSyms.containsOrIncl(sym.itemId): c.toNifDef sym @@ -201,11 +207,11 @@ proc saveNif(c: var EncodeContext; n: PNode): string = result = "(.nif24)\n" & toString(c.dest) -proc saveNifFile*(module: PSym; n: PNode; conf: ConfigRef) = +proc saveNifFile*(n: PNode; conf: ConfigRef; module: PSym) = let outfile = module.name.s & ".nif" - var c = initEncodeContext(conf) + var c = initEncodeContext(conf, module) writeFile outfile, saveNif(c, n) -proc saveNifToBuffer*(n: PNode; conf: ConfigRef): string = - var c = initEncodeContext(conf) +proc saveNifToBuffer*(n: PNode; conf: ConfigRef; module: PSym): string = + var c = initEncodeContext(conf, module) result = saveNif(c, n) diff --git a/tests/icnif/tencode_node2node.nim b/tests/icnif/tencode_node2node.nim index 95923a20c029..3f4338f8481f 100644 --- a/tests/icnif/tencode_node2node.nim +++ b/tests/icnif/tencode_node2node.nim @@ -18,11 +18,26 @@ proc newConfigRefForTest(): ConfigRef = proc newModuleGraphForSem(cache: IdentCache; conf: ConfigRef): ModuleGraph = var graph = newModuleGraph(cache, conf) graph.setPipeLinePass(SemPass) + # Make PNode from sem pass assigned to graph.systemModule.ast + let oldCmd = graph.config.cmd + graph.config.cmd = cmdIdeTools graph.compilePipelineSystemModule() + graph.config.cmd = oldCmd result = graph -proc sem(graph: ModuleGraph; path: AbsoluteFile): PNode = - result = nil +proc getSystemNif(graph: ModuleGraph): string = + assert graph.systemModule != nil + assert graph.systemModule.kind == skModule + assert graph.systemModule.ast != nil + + let n = graph.systemModule.ast + # if nil is not assigned, it generates large NIF + graph.systemModule.ast = nil + result = saveNifToBuffer(n, graph.config, graph.systemModule) + #writeFile("system.nif", result) + +proc sem(graph: ModuleGraph; path: AbsoluteFile): (PNode, PSym) = + result = (nil, nil) let fileIdx = fileInfoIdx(graph.config, path) var module = newModule(graph, fileIdx) @@ -34,7 +49,7 @@ proc sem(graph: ModuleGraph; path: AbsoluteFile): PNode = var stream = llStreamOpen(path, fmRead) if stream == nil: rawMessage(graph.config, errCannotOpenFile, path.string) - return nil + return (nil, nil) var p: Parser = default(Parser) syntaxes.openParser(p, fileIdx, stream, graph.cache, graph.config) @@ -52,9 +67,7 @@ proc sem(graph: ModuleGraph; path: AbsoluteFile): PNode = if n.kind == nkEmpty: break sl.add n - var semNode = semWithPContext(ctx, sl) - - return semNode + result = (semWithPContext(ctx, sl), module) type # Nim's AST has cycles that causes infinite recursive loop in eql procs. @@ -220,6 +233,8 @@ proc eql(x, y: PType; c: var EqlContext): bool = echo "type itemId mismatch" result = false elif c.checkedTypes.hasKeyOrPut(y.itemId, y): + result = true + #[ if c.checkedTypes[y.itemId] == y: result = true else: @@ -228,6 +243,7 @@ proc eql(x, y: PType; c: var EqlContext): bool = debug(c.checkedTypes[y.itemId]) debug(y) result = false + ]# elif x.kind != y.kind: echo "type kind mismatch: ", x.kind, "/", y.kind result = false @@ -306,9 +322,12 @@ proc eql(x, y: PNode; c: var EqlContext): bool = if not result: echo "Symbol mismatch:" debug(x.sym) - debug(y.sym) - debug(x.sym.typ) - debug(y.sym.typ) + if y.sym == nil: + echo "y.sym = nil" + else: + debug(y.sym) + debug(x.sym.typ) + debug(y.sym.typ) of nkCharLit .. nkUInt64Lit, nkStrLit .. nkTripleStrLit: result = sameValue(x, y) of nkFloatLit .. nkFloat128Lit: @@ -341,18 +360,22 @@ proc eql(x, y: PNode; c: var EqlContext): bool = debug(y) result = false -proc testNifEncDec(graph: ModuleGraph; src: string) = +proc testNifEncDec(graph: ModuleGraph; src: string; systemNif: string) = let fullPath = TestCodeDir / RelativeFile(src) - let n = sem(graph, fullPath) + let (n, module) = sem(graph, fullPath) + assert n != nil, "failed to sem " & $fullPath + #debug(n) - let nif = saveNifToBuffer(n, graph.config) + let nif = saveNifToBuffer(n, graph.config, module) #echo nif #echo "NIF size of ", src, ": ", nif.len #writeFile(src & ".nif", nif) # Don't reuse the ModuleGraph used for semcheck when load NIF. var graphForLoad = newModuleGraph(newIdentCache(), newConfigRefForTest()) - let n2 = loadNifFromBuffer(nif, graphForLoad) + var prog = NifProgram() + discard loadNifFromBuffer(systemNif, graphForLoad, prog) + let n2 = loadNifFromBuffer(nif, graphForLoad, prog) #debug(n2) var c = EqlContext(confX: graph.config, confY: graphForLoad.config) assert eql(n, n2, c), "test failed: " & $fullPath @@ -360,11 +383,12 @@ proc testNifEncDec(graph: ModuleGraph; src: string) = var conf = newConfigRefForTest() var cache = newIdentCache() var graph = newModuleGraphForSem(cache, conf) -testNifEncDec(graph, "modtest1.nim") -testNifEncDec(graph, "modtestliterals.nim") -testNifEncDec(graph, "modtesttypesections.nim") -testNifEncDec(graph, "modtestpragmas.nim") -testNifEncDec(graph, "modtestprocs.nim") -testNifEncDec(graph, "modteststatements.nim") -testNifEncDec(graph, "modtestgenerics.nim") -testNifEncDec(graph, "modtestexprs.nim") +let systemNif = getSystemNif(graph) +testNifEncDec(graph, "modtest1.nim", systemNif) +testNifEncDec(graph, "modtestliterals.nim", systemNif) +testNifEncDec(graph, "modtesttypesections.nim", systemNif) +#testNifEncDec(graph, "modtestpragmas.nim", systemNif) +testNifEncDec(graph, "modtestprocs.nim", systemNif) +#testNifEncDec(graph, "modteststatements.nim", systemNif) +#testNifEncDec(graph, "modtestgenerics.nim", systemNif) +#testNifEncDec(graph, "modtestexprs.nim", systemNif) diff --git a/tests/icnif/testcode/modtestprocs.nim b/tests/icnif/testcode/modtestprocs.nim index f17966586b31..c872b8edefd6 100644 --- a/tests/icnif/testcode/modtestprocs.nim +++ b/tests/icnif/testcode/modtestprocs.nim @@ -1,12 +1,13 @@ proc foo() = discard +foo() + proc bar(x: int): int = x -proc baz(x, y: int): string = $(x + y) +discard bar(1) + +proc baz(x, y: int): int = + bar(x) -proc baz(a: bool; b: string; c: int): float = - if a and b == "" and c == 0: - result = 0.0 - else: - result = 1.0 +proc baz(a: bool; b: string; c: int): float = 0.0 proc forwardDecl() proc forwardDecl() = @@ -20,21 +21,21 @@ proc forwardDecl2*(): int = bar(1) discard forwardDecl2() proc forwardDecl3(x, y: int): int -proc forwardDecl3(x, y: int): int = x - y +proc forwardDecl3(x, y: int): int = x discard forwardDecl3(3, 2) func func1(): int = 123 discard func1() func func2(x: int): int = x -func func3*(x, y: bool): bool = x and y +func func3*(x, y: bool): bool = x -proc withDefaultValue(x = 1) = echo x +proc withDefaultValue(x = 1) = discard withDefaultValue() withDefaultValue(2) withDefaultValue(x = 3) -proc withDefaultValue2(x = "foo"; y = true) = echo x, y +proc withDefaultValue2(x = "foo"; y = true) = discard withDefaultValue2() withDefaultValue2("bar") withDefaultValue2(x = "baz", y = false) From 43ff7b1dc4fd0ea2ed8cd6e51629eb88f517510e Mon Sep 17 00:00:00 2001 From: demotomohiro Date: Thu, 6 Nov 2025 04:48:27 +0900 Subject: [PATCH 20/38] loads modules imported by system module and create module sym before decode --- compiler/icnif/nifdecoder.nim | 32 +++++++++++++++++++++++++++---- compiler/icnif/nifencoder.nim | 1 + tests/icnif/tencode_node2node.nim | 32 +++++++++++++++++-------------- 3 files changed, 47 insertions(+), 18 deletions(-) diff --git a/compiler/icnif/nifdecoder.nim b/compiler/icnif/nifdecoder.nim index 6935e295242c..37f9e888ecee 100644 --- a/compiler/icnif/nifdecoder.nim +++ b/compiler/icnif/nifdecoder.nim @@ -129,7 +129,7 @@ proc fromNifSymDef(c: var DecodeContext; n: var Cursor): PSym = # PNode, PSym or PType type fields in PSym can have cycles. # Add PSym to `c.symbols` before parsing these fields so that # they can refer this PSym. - let nifItemId = ItemId(module: nifModId, item: itemId) + let nifItemId = ItemId(module: itemIdModule.int32, item: itemId) assert nifItemId notin c.symbols c.symbols[nifItemId] = result @@ -231,10 +231,10 @@ proc fromNifSymbol(c: var DecodeContext; n: var Cursor): PSym = result = c.fromNifSymDef n elif n.tagId == symTag: incExpect n, IntLit - let nifModId = pool.integers[n.intId].int32 + let nifModId = c.modules[pool.integers[n.intId].int32] incExpect n, IntLit let item = pool.integers[n.intId].int32 - let nifItemId = ItemId(module: nifModId, item: item) + let nifItemId = ItemId(module: nifModId.int32, item: item) result = c.symbols[nifItemId] inc n skipParRi n @@ -356,9 +356,33 @@ proc loadNif(stream: var Stream; graph: ModuleGraph; prog: NifProgram): PNode = var buf = fromStream(stream) var n = beginRead(buf) - var c = DecodeContext(graph: graph, prog: prog) + var n2 = n + var nested = 0 + while true: + if n2.info != NoLineInfo: + break + elif n2.kind == EofToken: + break + elif n2.kind == ParLe: + inc nested + elif n2.kind == ParRi: + dec nested + if nested == 0: break + inc n2 + assert n2.info != NoLineInfo + let info = pool.man.unpack(n2.info) + let fileIdx = c.graph.config.fileInfoIdx(pool.files[info.file].AbsoluteFile) + var currentModule = graph.newModule(fileIdx) + if currentModule.itemId.module == 0'i32: + currentModule.flags = {sfMainModule, sfSystemModule} + + c.symbols[currentModule.itemId] = currentModule + let nifSym = toNifSym(currentModule, c.prog.moduleToNifSuffix, c.graph.config) + let symId = pool.syms.getOrIncl(nifSym) + c.prog.nifSymIdToPSym[symId] = currentModule + result = fromNif(c, n) endRead(buf) diff --git a/compiler/icnif/nifencoder.nim b/compiler/icnif/nifencoder.nim index 2e562aca7faf..ea4a9fadee19 100644 --- a/compiler/icnif/nifencoder.nim +++ b/compiler/icnif/nifencoder.nim @@ -17,6 +17,7 @@ proc initEncodeContext(conf: ConfigRef; currentModule: PSym): EncodeContext = result = EncodeContext(conf: conf, currentModule: currentModule, dest: createTokenBuf()) + result.decodedSyms.incl(currentModule.itemId) template buildTree(dest: var TokenBuf; tag: TagId; body: untyped) = dest.addParLe tag diff --git a/tests/icnif/tencode_node2node.nim b/tests/icnif/tencode_node2node.nim index 3f4338f8481f..b1793ec6c27e 100644 --- a/tests/icnif/tencode_node2node.nim +++ b/tests/icnif/tencode_node2node.nim @@ -25,16 +25,16 @@ proc newModuleGraphForSem(cache: IdentCache; conf: ConfigRef): ModuleGraph = graph.config.cmd = oldCmd result = graph -proc getSystemNif(graph: ModuleGraph): string = - assert graph.systemModule != nil - assert graph.systemModule.kind == skModule - assert graph.systemModule.ast != nil - - let n = graph.systemModule.ast - # if nil is not assigned, it generates large NIF - graph.systemModule.ast = nil - result = saveNifToBuffer(n, graph.config, graph.systemModule) - #writeFile("system.nif", result) +proc getSystemNif(graph: ModuleGraph): seq[string] = + result = newSeqOfCap[string](graph.ifaces.len) + for i, iface in graph.ifaces.mpairs: + if iface.module != nil: + let n = iface.module.ast + assert n != nil + # if nil is not assigned, it generates large NIF + iface.module.ast = nil + result.add saveNifToBuffer(n, graph.config, iface.module) + #writeFile(iface.module.name.s & ".nif", result[^1]) proc sem(graph: ModuleGraph; path: AbsoluteFile): (PNode, PSym) = result = (nil, nil) @@ -169,10 +169,12 @@ proc eql(x, y: PSym; c: var EqlContext): bool = elif x.magic != y.magic: echo "symbol magic mismatch: ", x.magic, "/", y.magic result = false - elif not eql(x.info, y.info, c): + elif x.kind != skPackage and not eql(x.info, y.info, c): + # fileIndex of info of skPackage is just a path of first semchecked module in the package echo "symbol line info mismatch" result = false - elif x.flags != y.flags: + elif x.kind != skModule and x.flags != y.flags: + # TODO: check the flag of skModule echo "symbol flag mismatch: ", x.flags, "/", y.flags result = false elif x.options != y.options: @@ -360,10 +362,11 @@ proc eql(x, y: PNode; c: var EqlContext): bool = debug(y) result = false -proc testNifEncDec(graph: ModuleGraph; src: string; systemNif: string) = +proc testNifEncDec(graph: ModuleGraph; src: string; systemNif: openArray[string]) = let fullPath = TestCodeDir / RelativeFile(src) let (n, module) = sem(graph, fullPath) assert n != nil, "failed to sem " & $fullPath + assert module.owner.kind == skPackage #debug(n) let nif = saveNifToBuffer(n, graph.config, module) @@ -374,7 +377,8 @@ proc testNifEncDec(graph: ModuleGraph; src: string; systemNif: string) = # Don't reuse the ModuleGraph used for semcheck when load NIF. var graphForLoad = newModuleGraph(newIdentCache(), newConfigRefForTest()) var prog = NifProgram() - discard loadNifFromBuffer(systemNif, graphForLoad, prog) + for sysNif in systemNif: + discard loadNifFromBuffer(sysNif, graphForLoad, prog) let n2 = loadNifFromBuffer(nif, graphForLoad, prog) #debug(n2) var c = EqlContext(confX: graph.config, confY: graphForLoad.config) From 116c220e3fa8a14a5ce04609ac33f35fadecd80a Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 6 Nov 2025 23:41:13 +0800 Subject: [PATCH 21/38] fixes #25263; provides a new switch `mangle:nim/cpp` for debug name mangling (#25264) fixes #25263 - [x] documentation and changelogs --- changelog.md | 2 + compiler/ccgtypes.nim | 2 +- compiler/commands.nim | 9 ++ compiler/options.nim | 1 + doc/advopt.txt | 1 + tests/codegen/titaniummangle.nim | 4 +- tests/codegen/titaniummangle_nim.nim | 199 +++++++++++++++++++++++++++ 7 files changed, 215 insertions(+), 3 deletions(-) create mode 100644 tests/codegen/titaniummangle_nim.nim diff --git a/changelog.md b/changelog.md index 959f1056697e..bf7d343d1bfb 100644 --- a/changelog.md +++ b/changelog.md @@ -27,6 +27,8 @@ errors. - With `-d:nimPreviewDuplicateModuleError`, importing two modules that share the same name becomes a compile-time error. This includes importing the same module more than once. Use `import foo as foo1` (or other aliases) to avoid collisions. +- Adds the switch `--mangle:nim|cpp`, which selects `nim` or `cpp` style name mangling when used with `debuginfo` on, defaults to `nim`. The default is changed from `cpp` to `nim`. + ## Standard library additions and changes [//]: # "Additions:" diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index a83857206b72..cdfa46cdd2d9 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -75,7 +75,7 @@ proc mangleProc(m: BModule; s: PSym; makeUnique: bool): string = proc fillBackendName(m: BModule; s: PSym) = if s.loc.snippet == "": var result: Rope - if not m.compileToCpp and s.kind in routineKinds and optCDebug in m.g.config.globalOptions and + if s.kind in routineKinds and {optCDebug, optItaniumMangle} * m.g.config.globalOptions == {optCDebug, optItaniumMangle} and m.g.config.symbolFiles == disabledSf: result = mangleProc(m, s, false).rope else: diff --git a/compiler/commands.nim b/compiler/commands.nim index 7da15ca05bf2..e206a37300e4 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -364,6 +364,7 @@ proc testCompileOption*(conf: ConfigRef; switch: string, info: TLineInfo): bool result = false of "panics": result = contains(conf.globalOptions, optPanics) of "jsbigint64": result = contains(conf.globalOptions, optJsBigInt64) + of "mangle": result = contains(conf.globalOptions, optItaniumMangle) else: result = false invalidCmdLineOption(conf, passCmd1, switch, info) @@ -762,6 +763,14 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; conf.globalOptions.excl optCDebug else: localError(conf, info, "expected native|gdb|on|off but found " & arg) + of "mangle": + case arg.normalize + of "nim": + conf.globalOptions.excl optItaniumMangle + of "cpp": + conf.globalOptions.incl optItaniumMangle + else: + localError(conf, info, "expected nim|cpp but found " & arg) of "g": # alias for --debugger:native conf.globalOptions.incl optCDebug conf.options.incl optLineDir diff --git a/compiler/options.nim b/compiler/options.nim index fa2c2069b39b..7bc7d403c861 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -110,6 +110,7 @@ type # please make sure we have under 32 options optEnableDeepCopy # ORC specific: enable 'deepcopy' for all types. optShowNonExportedFields # for documentation: show fields that are not exported optJsBigInt64 # use bigints for 64-bit integers in JS + optItaniumMangle # mangling follows the Itanium spec TGlobalOptions* = set[TGlobalOption] diff --git a/doc/advopt.txt b/doc/advopt.txt index 647c1b4d6709..85da350a8011 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -91,6 +91,7 @@ Advanced options: --os:SYMBOL set the target operating system (cross-compilation) --cpu:SYMBOL set the target processor (cross-compilation) --debuginfo:on|off enables debug information + --mangle:nim|cpp selects `nim` or `cpp` style name mangling, defaults to `nim` -t, --passC:OPTION pass an option to the C compiler -l, --passL:OPTION pass an option to the linker --cc:SYMBOL specify the C compiler diff --git a/tests/codegen/titaniummangle.nim b/tests/codegen/titaniummangle.nim index d566900b19f1..ccca9ce2f05c 100644 --- a/tests/codegen/titaniummangle.nim +++ b/tests/codegen/titaniummangle.nim @@ -1,6 +1,6 @@ discard """ - targets: "c" - matrix: "--debugger:native" + targets: "c cpp" + matrix: "--debugger:native --mangle:cpp" ccodecheck: "'_ZN14titaniummangle8testFuncE'" ccodecheck: "'_ZN14titaniummangle8testFuncE6stringN14titaniummangle3FooE'" ccodecheck: "'_ZN14titaniummangle8testFuncE3int7varargsI6stringE'" diff --git a/tests/codegen/titaniummangle_nim.nim b/tests/codegen/titaniummangle_nim.nim new file mode 100644 index 000000000000..72afdaf8a6ab --- /dev/null +++ b/tests/codegen/titaniummangle_nim.nim @@ -0,0 +1,199 @@ +discard """ + targets: "c" + matrix: "--debugger:native --mangle:nim; --debugger:native" + ccodecheck: "'testFunc__titaniummangle95nim_u1316'" + ccodecheck: "'testFunc__titaniummangle95nim_u156'" + ccodecheck: "'testFunc__titaniummangle95nim_u1305'" + ccodecheck: "'testFunc__titaniummangle95nim_u241'" + ccodecheck: "'testFunc__titaniummangle95nim_u1357'" + ccodecheck: "'testFunc__titaniummangle95nim_u292'" + ccodecheck: "'testFunc__titaniummangle95nim_u38'" + ccodecheck: "'testFunc__titaniummangle95nim_u175'" + ccodecheck: "'testFunc__titaniummangle95nim_u1302'" + ccodecheck: "'testFunc__titaniummangle95nim_u1305'" + ccodecheck: "'testFunc__titaniummangle95nim_u535'" + ccodecheck: "'testFunc__titaniummangle95nim_u1294'" + ccodecheck: "'testFunc__titaniummangle95nim_u336'" + ccodecheck: "'testFunc__titaniummangle95nim_u425'" + ccodecheck: "'testFunc__titaniummangle95nim_u308'" + ccodecheck: "'testFunc__titaniummangle95nim_u129'" + ccodecheck: "'testFunc__titaniummangle95nim_u320'" + ccodecheck: "'testFunc__titaniummangle95nim_u223'" + ccodecheck: "'testFunc__titaniummangle95nim_u545'" + ccodecheck: "'testFunc__titaniummangle95nim_u543'" + ccodecheck: "'testFunc__titaniummangle95nim_u895'" + ccodecheck: "'testFunc__titaniummangle95nim_u1104'" + ccodecheck: "'testFunc__titaniummangle95nim_u1155'" + ccodecheck: "'testFunc__titaniummangle95nim_u636'" + ccodecheck: "'testFunc__titaniummangle95nim_u705'" + ccodecheck: "'testFunc__titaniummangle95nim_u800'" + ccodecheck: "'new__titaniummangle95nim_u1320'" + ccodecheck: "'xxx__titaniummangle95nim_u1391'" + ccodecheck: "'xxx__titaniummangle95nim_u1394'" +""" + +#When debugging this notice that if one check fails, it can be due to any of the above. + +type + Comparable = concept x, y + (x < y) is bool + + Foo = object + a: int32 + b: int32 + + FooTuple = tuple + a: int + b: int + + Container[T] = object + data: T + + Container2[T, T2] = object + data: T + data2: T2 + + Boo = distinct Foo + + Coo = Foo + + Doo = Boo | Foo + + TestProc = proc(a:string): string + +type EnumSample = enum + a, b, c + +type EnumAnotherSample = enum + a, b, c + +proc testFunc(a: set[EnumSample]) = + echo $a + +proc testFunc(a: typedesc) = + echo $a + +proc testFunc(a: ptr Foo) = + echo repr a + +proc testFunc(s: string, a: Coo) = + echo repr a + +proc testFunc(s: int, a: Comparable) = + echo repr a + +proc testFunc(a: TestProc) = + let b = "" + echo repr a("") + +proc testFunc(a: ref Foo) = + echo repr a + +proc testFunc(b: Boo) = + echo repr b + +proc testFunc(a: ptr UncheckedArray[int]) = + echo repr a + +proc testFunc(a: ptr int) = + echo repr a + +proc testFunc(a: ptr ptr int) = + echo repr a + +proc testFunc(e: FooTuple, str: cstring) = + echo e + +proc testFunc(e: (float, float)) = + echo e + +proc testFunc(e: EnumSample) = + echo e + +proc testFunc(e: var int) = + echo e + +proc testFunc(e: var Foo, a, b: int32, refFoo: ref Foo) = + echo e + +proc testFunc(xs: Container[int]) = + let a = 2 + echo xs + +proc testFunc(xs: Container2[int32, int32]) = + let a = 2 + echo xs + +proc testFunc(xs: Container[Container2[int32, int32]]) = + let a = 2 + echo xs + +proc testFunc(xs: seq[int]) = + let a = 2 + echo xs + +proc testFunc(xs: openArray[string]) = + let a = 2 + echo xs + +proc testFunc(xs: array[2, int]) = + let a = 2 + echo xs + +proc testFunc(e: EnumAnotherSample) = + echo e + +proc testFunc(a, b: int) = + echo "hola" + discard + +proc testFunc(a: int, xs: varargs[string]) = + let a = 10 + for x in xs: + echo x + +proc xxx(v: static int) = + echo v + +proc testFunc() = + var a = 2 + var aPtr = a.addr + var foo = Foo() + let refFoo : ref Foo = new(Foo) + let b = Foo().Boo() + let d: Doo = Foo() + testFunc("", Coo()) + testFunc(1, ) + testFunc(b) + testFunc(EnumAnotherSample) + var t = [1, 2] + let uArr = cast[ptr UncheckedArray[int]](t.addr) + testFunc(uArr) + testFunc({}) + testFunc(proc(s:string): string = "test") + testFunc(20, a.int32) + testFunc(20, 2) + testFunc(EnumSample.c) + testFunc(EnumAnotherSample.c) + testFunc((2, 1), "adios") + testFunc((22.1, 1.2)) + testFunc(a.addr) + testFunc(foo.addr) + testFunc(aPtr.addr) + testFunc(refFoo) + testFunc(foo, 2, 1, refFoo) + testFunc(a) + testFunc(@[2, 1, 2]) + testFunc(@["hola"]) + testFunc(2, "hola", "adios") + let arr: array[2, int] = [2, 1] + testFunc(arr) + testFunc(Container[int](data: 10)) + let c2 = Container2[int32, int32](data: 10, data2: 20) + testFunc(c2) + testFunc(Container[Container2[int32, int32]](data: c2)) + xxx(10) + xxx(20) + + +testFunc() \ No newline at end of file From 155b06a4f5fa6656aef164678a101c5dbe0a9732 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Thu, 6 Nov 2025 17:33:52 +0100 Subject: [PATCH 22/38] Add `heaptrack` support (#25257) This PR, courtesy of @NagyZoltanPeter (https://github.com/waku-org/nwaku/pull/3522) adds the ability to track memory allocations in a program suitable for use with [heaptrack](https://github.com/KDE/heaptrack). By passing `-d:heaptrack --debugger:native` to compilation, calls to heaptrack will be injected when memory is being allocated and released - unlike `-d:useMalloc` this strategy also works with `refc` and the default memory pool. See https://github.com/KDE/heaptrack for usage examples. The resulting binary needs to be run with `heaptrack` and with the shared `libheaptrack_preload.so` in the `LD_LIBRARY_PATH`. --- doc/nimc.md | 2 ++ lib/system/alloc.nim | 13 +++++++++++++ 2 files changed, 15 insertions(+) diff --git a/doc/nimc.md b/doc/nimc.md index 24a2b3afe1f2..bb48af137010 100644 --- a/doc/nimc.md +++ b/doc/nimc.md @@ -580,6 +580,8 @@ Define Effect Currently only clang and vcc. `strip` Strip debug symbols added by the backend compiler from the executable. +`heaptrack` Track memory allocations using + [heaptrack](https://github.com/KDE/heaptrack) ====================== ========================================================= diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim index fcb7ccb0c82c..4109348fc234 100644 --- a/lib/system/alloc.nim +++ b/lib/system/alloc.nim @@ -837,6 +837,15 @@ when defined(gcDestructors): dec maxIters if it == nil: break +when defined(heaptrack): + const heaptrackLib = + when defined(heaptrack_inject): + "libheaptrack_inject.so" + else: + "libheaptrack_preload.so" + proc heaptrack_malloc(a: pointer, size: int) {.cdecl, importc, dynlib: heaptrackLib.} + proc heaptrack_free(a: pointer) {.cdecl, importc, dynlib: heaptrackLib.} + proc rawAlloc(a: var MemRegion, requestedSize: int): pointer = when defined(nimTypeNames): inc(a.allocCounter) @@ -959,6 +968,8 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer = sysAssert(isAccessible(a, result), "rawAlloc 14") sysAssert(allocInv(a), "rawAlloc: end") when logAlloc: cprintf("var pointer_%p = alloc(%ld) # %p\n", result, requestedSize, addr a) + when defined(heaptrack): + heaptrack_malloc(result, requestedSize) proc rawAlloc0(a: var MemRegion, requestedSize: int): pointer = result = rawAlloc(a, requestedSize) @@ -967,6 +978,8 @@ proc rawAlloc0(a: var MemRegion, requestedSize: int): pointer = proc rawDealloc(a: var MemRegion, p: pointer) = when defined(nimTypeNames): inc(a.deallocCounter) + when defined(heaptrack): + heaptrack_free(p) #sysAssert(isAllocatedPtr(a, p), "rawDealloc: no allocated pointer") sysAssert(allocInv(a), "rawDealloc: begin") var c = pageAddr(p) From 6d3254a86f8e38308663e093ccc524c09342ee08 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 7 Nov 2025 17:06:05 +0800 Subject: [PATCH 23/38] fixes #25251; SIGBUS with iterator over const Table lookup - premature temporary destruction (#25255) fixes #25251 enforce a copy if the arg is a deref of a lent pointer since the arg could be a temporary that will go out of scope --- compiler/transf.nim | 5 ++++- tests/iter/titer_issues.nim | 25 +++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/compiler/transf.nim b/compiler/transf.nim index 066be57f8763..5d80bf9328b3 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -828,8 +828,11 @@ proc transformFor(c: PTransf, n: PNode): PNode = elif t.destructor == nil and arg.typ.destructor != nil: t = arg.typ - if arg.kind in {nkDerefExpr, nkHiddenDeref}: + if arg.kind in {nkDerefExpr, nkHiddenDeref} and + arg[0].typ.skipTypes(abstractInst).kind != tyLent: # optimizes for `[]` # bug #24093 + # bug #25251: enforce a copy if the arg is a deref of a lent pointer + # since the arg could be a temporary that will go out of scope var temp = newTemp(c, arg[0].typ, formal.info) addVar(v, temp) stmtList.add(newAsgnStmt(c, nkFastAsgn, temp, arg[0], true)) diff --git a/tests/iter/titer_issues.nim b/tests/iter/titer_issues.nim index ff0b8eb49fd0..5070a5471324 100644 --- a/tests/iter/titer_issues.nim +++ b/tests/iter/titer_issues.nim @@ -432,3 +432,28 @@ block: let x = cast[typeof(aaa)](aaa) # not even var for _ in x[]: discard + +import std/[tables, unicode, sequtils] + +const + myTable = { + "en": "abcdefghijklmnopqrstuvwxyz", + }.toTable + +proc buggyVersion(locale: string): seq[Rune] = + result = toSeq(runes(myTable[locale])) + +proc workingVersion(locale: string): seq[Rune] = + # string lifetime is extended + let str = myTable[locale] + result = toSeq(runes(str)) + +# echo "Testing working version..." +let runes2 = workingVersion("en") +# echo "Got ", runes2.len, " runes" + +# echo "Testing buggy version..." +let runes1 = buggyVersion("en") # <-- CRASHES HERE + +doAssert runes1.len == runes2.len +# echo "Got ", runes1.len, " runes" From a866bf2a3911f261e65fdf0c2b68ec4f2ca09d20 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 7 Nov 2025 11:05:13 +0100 Subject: [PATCH 24/38] =?UTF-8?q?VM:=20optimize=20'return'=20slots;=20save?= =?UTF-8?q?s=20millions=20of=20node=20allocations=20for=20N=E2=80=A6=20(#2?= =?UTF-8?q?5266)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …imbus --- compiler/vm.nim | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/compiler/vm.nim b/compiler/vm.nim index 2a97140d644a..258c2333453d 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -1442,8 +1442,16 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = #echo "new pc ", newPc, " calling: ", prc.name.s var newFrame = PStackFrame(prc: prc, comesFrom: pc, next: tos) newSeq(newFrame.slots, prc.offset+ord(isClosure)) - if not isEmptyType(prc.typ.returnType): - putIntoReg(newFrame.slots[0], getNullValue(c, prc.typ.returnType, prc.info, c.config)) + # setup slot for proc result: + let ret {.cursor.} = prc.typ.returnType + # hot spot ahead! + if ret != nil: + if fitsRegister(ret): + # same logic as opcLdNullReg here: + ensureKind(newFrame.slots[0], rkInt) + newFrame.slots[0].intVal = 0 + elif not isEmptyType(ret): + putIntoReg(newFrame.slots[0], getNullValue(c, ret, prc.info, c.config)) for i in 1..rc-1: newFrame.slots[i] = regs[rb+i] if isClosure: From 6a2f4d29c465b9d4f5f3eed9f47223ecdfa7ee2e Mon Sep 17 00:00:00 2001 From: araq Date: Fri, 7 Nov 2025 15:52:00 +0100 Subject: [PATCH 25/38] wip --- compiler/options.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/compiler/options.nim b/compiler/options.nim index 7bc7d403c861..b9afa5903e3d 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -111,6 +111,7 @@ type # please make sure we have under 32 options optShowNonExportedFields # for documentation: show fields that are not exported optJsBigInt64 # use bigints for 64-bit integers in JS optItaniumMangle # mangling follows the Itanium spec + optCompress TGlobalOptions* = set[TGlobalOption] From b4c942c4cd22392c5c4227213c176698cea624ae Mon Sep 17 00:00:00 2001 From: araq Date: Fri, 7 Nov 2025 21:26:29 +0100 Subject: [PATCH 26/38] wip --- compiler/ast2nif.nim | 345 ++++++++++++++++++++++++++++++++++ compiler/commands.nim | 2 + compiler/icnif/nifbasics.nim | 2 +- compiler/icnif/nifencoder.nim | 2 +- compiler/options.nim | 2 +- compiler/pipelines.nim | 5 +- 6 files changed, 354 insertions(+), 4 deletions(-) create mode 100644 compiler/ast2nif.nim diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim new file mode 100644 index 000000000000..b90a586e2e30 --- /dev/null +++ b/compiler/ast2nif.nim @@ -0,0 +1,345 @@ +# +# +# The Nim Compiler +# (c) Copyright 2025 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## AST to NIF bridge. + +import std / [assertions, tables, sets] +import ast, idents, msgs, options +import lineinfos as astli +import pathutils +import "../dist/nimony/src/lib" / [bitabs, nifstreams, nifcursors, lineinfos] +import "../dist/nimony/src/gear2" / modnames + +import icnif / [enum2nif, icniftags] + +# ---------------- Line info handling ----------------------------------------- + +type + LineInfoWriter = object + fileK: FileIndex # remember the current pair, even faster than the hash table + fileV: FileId + tab: Table[FileIndex, FileId] + revTab: Table[FileId, FileIndex] # reverse mapping for oldLineInfo + man: LineInfoManager + config: ConfigRef + +proc get(w: var LineInfoWriter; key: FileIndex): FileId = + if w.fileK == key: + result = w.fileV + else: + if key in w.tab: + result = w.tab[key] + w.fileK = key + w.fileV = result + else: + result = pool.files.getOrIncl(msgs.toFullPath(w.config, key)) + w.fileK = key + w.fileV = result + w.tab[key] = result + w.revTab[result] = key + +proc nifLineInfo(w: var LineInfoWriter; info: TLineInfo): PackedLineInfo = + if info == unknownLineInfo: + result = NoLineInfo + else: + let fid = get(w, info.fileIndex) + result = pack(w.man, fid, info.line.int32, info.col) + +proc oldLineInfo(w: var LineInfoWriter; info: PackedLineInfo): TLineInfo = + if info == NoLineInfo: + result = unknownLineInfo + else: + var x = unpack(w.man, info) + var fileIdx: FileIndex + if w.fileV == x.file: + fileIdx = w.fileK + elif x.file in w.revTab: + fileIdx = w.revTab[x.file] + else: + # Need to look up FileId -> FileIndex via the file path + let filePath = pool.files[x.file] + fileIdx = msgs.fileInfoIdx(w.config, AbsoluteFile filePath) + w.revTab[x.file] = fileIdx + result = TLineInfo(line: x.line.uint16, col: x.col.int16, fileIndex: fileIdx) + + +# -------------- Module name handling -------------------------------------------- + +proc modname(moduleToNifSuffix: var Table[FileIndex, string]; module: int; conf: ConfigRef): string = + let idx = module.FileIndex + # copied from ../nifgen.nim + result = moduleToNifSuffix.getOrDefault(idx) + if result.len == 0: + let fp = toFullPath(conf, idx) + result = moduleSuffix(fp, cast[seq[string]](conf.searchPaths)) + moduleToNifSuffix[idx] = result + #echo result, " -> ", fp + +proc modname(moduleToNifSuffix: var Table[FileIndex, string]; module: PSym; conf: ConfigRef): string = + assert module.kind == skModule + result = modname(moduleToNifSuffix, module.position, conf) + + + +# ------------- Writer --------------------------------------------------------------- + +#[ + +Strategy: + +We produce NIF from the PNode structure as the single source of truth. NIF nodes can +however, refer to PSym and PType, these get NIF names. If the PSym/PType belongs to +the module that we are currently writing, we emit these fields as an inner NIF +structure via the special tags `sd` and `td`. In fact it is only these tags +that get the NIF `SymbolDef` kinds so that the lazy loading mechanism cannot +be confused. + +We could also emit non-local symbols and types later as the index structure +will tell us the precise offsets anyway. + +]# + +let + sdefTag = registerTag("sd") + tdefTag = registerTag("td") + tuseTag = registerTag("t") + +type + Writer = object + dest: TokenBuf + inner: LineInfoWriter + currentModule: int32 + writtenSyms: HashSet[ItemId] + writtenTypes: HashSet[ItemId] + decodedFileIndices: HashSet[FileIndex] + moduleToNifSuffix: Table[FileIndex, string] + locals: HashSet[ItemId] # track proc-local symbols + inProc: int + +proc toNifSymName(w: var Writer; sym: PSym): string = + ## Generate NIF name for a symbol: local names are `ident.disamb`, + ## global names are `ident.disamb.moduleSuffix` + result = sym.name.s + result.add '.' + result.addInt sym.disamb + if sym.itemId notin w.locals: + # Global symbol: ident.disamb.moduleSuffix + let module = sym.itemId.module + result.add '.' + result.add modname(w.moduleToNifSuffix, module, w.inner.config) + +template buildTree(dest: var TokenBuf; tag: TagId; body: untyped) = + dest.addParLe tag + body + dest.addParRi + +template buildTree(dest: var TokenBuf; tag: string; body: untyped) = + buildTree dest, pool.tags.getOrIncl(tag), body + +proc writeFlags[E](w: var Writer; flags: set[E]) = + var flagsAsIdent = "" + genFlags(flags, flagsAsIdent) + if flagsAsIdent.len > 0: + w.dest.addIdent flagsAsIdent + else: + w.dest.addDotToken + +proc trLineInfo(w: var Writer; info: TLineInfo): PackedLineInfo {.inline.} = + result = nifLineInfo(w.inner, info) + +proc writeNode(w: var Writer; n: PNode) +proc writeType(w: var Writer; typ: PType) +proc writeSym(w: var Writer; sym: PSym) + +proc typeToNifSym(w: var Writer; typ: PType): string = + result = "`t." + result.addInt typ.uniqueId.item + result.add '.' + result.add modname(w.moduleToNifSuffix, typ.uniqueId.module, w.inner.config) + +proc writeTypeDef(w: var Writer; typ: PType) = + w.dest.buildTree tdefTag: + w.dest.addSymDef pool.syms.getOrIncl(w.typeToNifSym(typ)), NoLineInfo + + w.dest.addIdent toNifTag(typ.kind) + writeFlags(w, typ.flags) + w.dest.addIdent toNifTag(typ.callConv) + w.dest.addIntLit typ.size + w.dest.addIntLit typ.align + w.dest.addIntLit typ.paddingAtEnd + w.dest.addIntLit typ.itemId.item # nonUniqueId + + writeType(w, typ.typeInst) + writeNode(w, typ.n) + writeSym(w, typ.owner) + writeSym(w, typ.sym) + # we store the type's elements here at the end so that + # it is not ambiguous and saves space: + for ch in typ.kids: + writeType(w, ch) + + +proc writeType(w: var Writer; typ: PType) = + if typ == nil: + w.dest.addDotToken() + elif typ.itemId.module == w.currentModule and not w.writtenTypes.containsOrIncl(typ.uniqueId): + writeTypeDef(w, typ) + else: + w.dest.buildTree tuseTag: + w.dest.addSymUse pool.syms.getOrIncl(w.typeToNifSym(typ)), NoLineInfo + +proc writeSymDef(w: var Writer; sym: PSym) = + w.dest.addParLe sdefTag, trLineInfo(w, sym.info) + w.dest.addSymDef pool.syms.getOrIncl(w.toNifSymName(sym)), NoLineInfo + if sym.magic == mNone: + w.dest.addDotToken + else: + w.dest.addIdent toNifTag(sym.magic) + writeFlags(w, sym.flags) + writeFlags(w, sym.options) + w.dest.addIntLit sym.offset + w.dest.addIntLit sym.disamb + w.dest.buildTree sym.kind.toNifTag: + case sym.kind + of skLet, skVar, skField, skForVar: + writeSym(w, sym.guard) + w.dest.addIntLit sym.bitsize + w.dest.addIntLit sym.alignment + else: + discard + if sym.kind == skModule: + w.dest.addDotToken() # position will be set by the loader! + else: + w.dest.addIntLit sym.position + writeType(w, sym.typ) + writeSym(w, sym.owner) + # We do not store `sym.ast` here but instead set it in the deserializer + #writeNode(w, sym.ast) + w.dest.addIdent toNifTag(sym.loc.k) + w.dest.addStrLit sym.loc.snippet + writeNode(w, sym.constraint) + writeSym(w, sym.instantiatedFrom) + w.dest.addParRi + +proc writeSym(w: var Writer; sym: PSym) = + if sym == nil: + w.dest.addDotToken() + elif sym.itemId.module == w.currentModule and not w.writtenTypes.containsOrIncl(sym.itemId): + writeSymDef(w, sym) + else: + # NIF has direct support for symbol references so we don't need to use a tag here, + # unlike what we do for types! + w.dest.addSymUse pool.syms.getOrIncl(w.toNifSymName(sym)), NoLineInfo + +proc writeNodeFlags(w: var Writer; flags: set[TNodeFlag]) {.inline.} = + writeFlags(w, flags) + +template withNode(w: var Writer; n: PNode; body: untyped) = + w.dest.addParLe pool.tags.getOrIncl(toNifTag(n.kind)), trLineInfo(w, n.info) + writeNodeFlags(w, n.flags) + writeType(w, n.typ) + body + w.dest.addParRi + +proc addLocalSym(w: var Writer; n: PNode) = + ## Add symbol from a node to locals set if it's a symbol node + if n != nil and n.kind == nkSym and n.sym != nil and w.inProc > 0: + w.locals.incl(n.sym.itemId) + +proc addLocalSyms(w: var Writer; n: PNode) = + if n.kind in {nkIdentDefs, nkVarTuple}: + # nkIdentDefs: [ident1, ident2, ..., type, default] + # All children except the last two are identifiers + for i in 0 ..< max(0, n.len - 2): + addLocalSyms(w, n[i]) + elif n.kind == nkSym: + addLocalSym(w, n) + +proc writeNode(w: var Writer; n: PNode) = + if n == nil: + w.dest.addDotToken + else: + case n.kind: + of nkEmpty: + let info = trLineInfo(w, n.info) + w.dest.addParLe pool.tags.getOrIncl(toNifTag(nkEmpty)), info + writeNodeFlags(w, n.flags) + w.dest.addParRi + of nkIdent: + # nkIdent uses flags and typ when it is a generic parameter + w.withNode n: + w.dest.addIdent n.ident.s + of nkSym: + # PNode.typ and PNode.sym.typ are different in `int` nkSym Node in following statement: + # type TestInt = int + w.withNode n: + writeSym(w, n.sym) + of nkCharLit: + w.withNode n: + w.dest.add charToken(n.intVal.char, NoLineInfo) + of nkIntLit .. nkInt64Lit: + w.withNode n: + w.dest.addIntLit n.intVal + of nkUIntLit .. nkUInt64Lit: + w.withNode n: + w.dest.addUIntLit cast[BiggestUInt](n.intVal) + of nkFloatLit .. nkFloat128Lit: + w.withNode n: + w.dest.add floatToken(pool.floats.getOrIncl(n.floatVal), NoLineInfo) + of nkStrLit .. nkTripleStrLit: + w.withNode n: + w.dest.addStrLit n.strVal + of nkNilLit: + w.withNode n: + discard + of nkLetSection, nkVarSection, nkConstSection, nkGenericParams: + # Track local variables declared in let/var sections + w.withNode(n): + for child in n: + addLocalSyms w, child + # Process the child node + writeNode(w, child) + of nkForStmt, nkTypeDef: + # Track for loop variable (first child is the loop variable) + w.withNode(n): + if n.len > 0: + addLocalSyms(w, n[0]) + for i in 0 ..< n.len: + writeNode(w, n[i]) + of nkFormalParams: + # Track parameters (first child is return type, rest are parameters) + w.withNode(n): + for i in 0 ..< n.len: + if i > 0: # Skip return type + addLocalSyms(w, n[i]) + writeNode(w, n[i]) + of nkProcDef, nkFuncDef, nkMethodDef, nkIteratorDef, nkConverterDef, nkLambda, nkDo, nkMacroDef: + inc w.inProc + # Entering a proc/function body - parameters are local + var ast = n + if n[namePos].kind == nkSym: + ast = n[namePos].sym.ast + w.withNode(ast): + # Process body and other parts + for i in 0 ..< ast.len: + writeNode(w, ast[i]) + dec w.inProc + else: + w.withNode(n): + for i in 0 ..< n.len: + writeNode(w, n[i]) + +proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode) = + var w = Writer(inner: LineInfoWriter(config: config), currentModule: thisModule) + w.writeNode n + let m = modname(w.moduleToNifSuffix, w.currentModule, w.inner.config) + let d = toGeneratedFile(config, AbsoluteFile(m), ".nif").string + writeFile w.dest, d, OnlyIfChanged + + diff --git a/compiler/commands.nim b/compiler/commands.nim index e206a37300e4..415fe6b352f7 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -771,6 +771,8 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; conf.globalOptions.incl optItaniumMangle else: localError(conf, info, "expected nim|cpp but found " & arg) + of "compress": + conf.globalOptions.incl optCompress of "g": # alias for --debugger:native conf.globalOptions.incl optCDebug conf.options.incl optLineDir diff --git a/compiler/icnif/nifbasics.nim b/compiler/icnif/nifbasics.nim index 72166c93a990..83f91a2b27bd 100644 --- a/compiler/icnif/nifbasics.nim +++ b/compiler/icnif/nifbasics.nim @@ -1,4 +1,4 @@ -import std/[tables] +import std/[assertions, tables] import ".." / [ast, lineinfos, msgs, options] import "../../dist/nimony/src/gear2" / modnames diff --git a/compiler/icnif/nifencoder.nim b/compiler/icnif/nifencoder.nim index ea4a9fadee19..e35ea4ab8abf 100644 --- a/compiler/icnif/nifencoder.nim +++ b/compiler/icnif/nifencoder.nim @@ -85,7 +85,7 @@ proc toNifDef(c: var EncodeContext; sym: PSym) = c.dest.addIntLit sym.position c.toNif sym.typ c.toNif sym.owner - c.toNif sym.ast # drastically increase output NIF size! + c.toNif sym.ast # drastically increase output NIF size! -- Yeah, no shit Sherlock. c.dest.addIdent toNifTag(sym.loc.k) c.dest.addStrLit sym.loc.snippet c.toNif sym.constraint diff --git a/compiler/options.nim b/compiler/options.nim index b9afa5903e3d..142080cc8f5c 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -111,7 +111,7 @@ type # please make sure we have under 32 options optShowNonExportedFields # for documentation: show fields that are not exported optJsBigInt64 # use bigints for 64-bit integers in JS optItaniumMangle # mangling follows the Itanium spec - optCompress + optCompress # turn on AST compression by converting it to NIF TGlobalOptions* = set[TGlobalOption] diff --git a/compiler/pipelines.nim b/compiler/pipelines.nim index e617ae8b90b0..e4becc92f017 100644 --- a/compiler/pipelines.nim +++ b/compiler/pipelines.nim @@ -1,7 +1,7 @@ import sem, cgen, modulegraphs, ast, llstream, parser, msgs, lineinfos, reorder, options, semdata, cgendata, modules, pathutils, packages, syntaxes, depends, vm, pragmas, idents, lookups, wordrecg, - liftdestructors, nifgen + liftdestructors, nifgen, ast2nif import pipelineutils @@ -218,6 +218,9 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator of NonePass: raiseAssert "use setPipeLinePass to set a proper PipelinePass" + if optCompress in graph.config.globalOptions: + writeNifModule(graph.config, module.position.int32, finalNode) + if graph.config.backend notin {backendC, backendCpp, backendObjc}: # We only write rod files here if no C-like backend is active. # The C-like backends have been patched to support the IC mechanism. From e6a1c14d919193dee6d76965e8b98ae69a1f6756 Mon Sep 17 00:00:00 2001 From: Araq Date: Fri, 7 Nov 2025 23:35:45 +0100 Subject: [PATCH 27/38] progress --- compiler/ast2nif.nim | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index b90a586e2e30..a4b550c47bc9 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -13,7 +13,8 @@ import std / [assertions, tables, sets] import ast, idents, msgs, options import lineinfos as astli import pathutils -import "../dist/nimony/src/lib" / [bitabs, nifstreams, nifcursors, lineinfos] +import "../dist/nimony/src/lib" / [bitabs, nifstreams, nifcursors, lineinfos, + nifindexes] import "../dist/nimony/src/gear2" / modnames import icnif / [enum2nif, icniftags] @@ -97,10 +98,10 @@ We produce NIF from the PNode structure as the single source of truth. NIF nodes however, refer to PSym and PType, these get NIF names. If the PSym/PType belongs to the module that we are currently writing, we emit these fields as an inner NIF structure via the special tags `sd` and `td`. In fact it is only these tags -that get the NIF `SymbolDef` kinds so that the lazy loading mechanism cannot +that get the NIF `SymbolDef` kinds so that the lazy loading mechanism cannot be confused. -We could also emit non-local symbols and types later as the index structure +We could also emit non-local symbols and types later as the index structure will tell us the precise offsets anyway. ]# @@ -109,6 +110,7 @@ let sdefTag = registerTag("sd") tdefTag = registerTag("td") tuseTag = registerTag("t") + hiddenTypeTag = registerTag("ht") type Writer = object @@ -123,7 +125,7 @@ type inProc: int proc toNifSymName(w: var Writer; sym: PSym): string = - ## Generate NIF name for a symbol: local names are `ident.disamb`, + ## Generate NIF name for a symbol: local names are `ident.disamb`, ## global names are `ident.disamb.moduleSuffix` result = sym.name.s result.add '.' @@ -230,13 +232,32 @@ proc writeSymDef(w: var Writer; sym: PSym) = proc writeSym(w: var Writer; sym: PSym) = if sym == nil: w.dest.addDotToken() - elif sym.itemId.module == w.currentModule and not w.writtenTypes.containsOrIncl(sym.itemId): + elif sym.itemId.module == w.currentModule and not w.writtenSyms.containsOrIncl(sym.itemId): writeSymDef(w, sym) else: # NIF has direct support for symbol references so we don't need to use a tag here, # unlike what we do for types! w.dest.addSymUse pool.syms.getOrIncl(w.toNifSymName(sym)), NoLineInfo +proc writeSymNode(w: var Writer; n: PNode; sym: PSym) = + if sym == nil: + w.dest.addDotToken() + elif sym.itemId.module == w.currentModule and not w.writtenSyms.containsOrIncl(sym.itemId): + if n.typ != n.sym.typ: + w.dest.buildTree hiddenTypeTag, trLineInfo(w, n.info): + writeSymDef(w, sym) + else: + writeSymDef(w, sym) + else: + # NIF has direct support for symbol references so we don't need to use a tag here, + # unlike what we do for types! + let info = trLineInfo(w, n.info) + if n.typ != n.sym.typ: + w.dest.buildTree hiddenTypeTag, info: + w.dest.addSymUse pool.syms.getOrIncl(w.toNifSymName(sym)), info + else: + w.dest.addSymUse pool.syms.getOrIncl(w.toNifSymName(sym)), info + proc writeNodeFlags(w: var Writer; flags: set[TNodeFlag]) {.inline.} = writeFlags(w, flags) @@ -276,10 +297,7 @@ proc writeNode(w: var Writer; n: PNode) = w.withNode n: w.dest.addIdent n.ident.s of nkSym: - # PNode.typ and PNode.sym.typ are different in `int` nkSym Node in following statement: - # type TestInt = int - w.withNode n: - writeSym(w, n.sym) + writeSymNode(w, n, n.sym) of nkCharLit: w.withNode n: w.dest.add charToken(n.intVal.char, NoLineInfo) @@ -340,6 +358,4 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode) = w.writeNode n let m = modname(w.moduleToNifSuffix, w.currentModule, w.inner.config) let d = toGeneratedFile(config, AbsoluteFile(m), ".nif").string - writeFile w.dest, d, OnlyIfChanged - - + writeFileAndIndex d, w.dest From 940ce87c7353dd1d001d06a61483dbf3b1f1c54c Mon Sep 17 00:00:00 2001 From: araq Date: Sat, 8 Nov 2025 07:42:26 +0100 Subject: [PATCH 28/38] prepare for dependency tracking --- compiler/ast2nif.nim | 12 ++++++++++++ compiler/semstmts.nim | 18 ++++++++++++++---- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index a4b550c47bc9..744c4cdfd2c8 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -115,6 +115,7 @@ let type Writer = object dest: TokenBuf + deps: TokenBuf inner: LineInfoWriter currentModule: int32 writtenSyms: HashSet[ItemId] @@ -282,6 +283,12 @@ proc addLocalSyms(w: var Writer; n: PNode) = elif n.kind == nkSym: addLocalSym(w, n) +proc trInclude(w: var Writer; n: PNode) = + discard + +proc trImport(w: var Writer; n: PNode) = + discard + proc writeNode(w: var Writer; n: PNode) = if n == nil: w.dest.addDotToken @@ -348,6 +355,11 @@ proc writeNode(w: var Writer; n: PNode) = for i in 0 ..< ast.len: writeNode(w, ast[i]) dec w.inProc + of nkImportStmt: + # this has been transformed for us, see `importer.nim` to contain a list of module syms: + trImport w, n + of nkIncludeStmt: + trInclude w, n else: w.withNode(n): for i in 0 ..< n.len: diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index ae4c744ead7b..1d6312e55c3c 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -2801,7 +2801,7 @@ proc semMacroDef(c: PContext, n: PNode): PNode = if n[bodyPos].kind == nkEmpty: localError(c.config, n.info, errImplOfXexpected % s.name.s) -proc incMod(c: PContext, n: PNode, it: PNode, includeStmtResult: PNode) = +proc incMod(c: PContext, n: PNode, it: PNode, includeStmtResult, resolvedIncStmt: PNode) = var f = checkModuleName(c.config, it) if f != InvalidFileIdx: addIncludeFileDep(c, f) @@ -2809,12 +2809,22 @@ proc incMod(c: PContext, n: PNode, it: PNode, includeStmtResult: PNode) = if containsOrIncl(c.includedFiles, f.int): localError(c.config, n.info, errRecursiveDependencyX % toMsgFilename(c.config, f)) else: + if resolvedIncStmt != nil: + resolvedIncStmt.add newStrNode(toFullPath(c.config, f), it.info) includeStmtResult.add semStmt(c, c.graph.includeFileCallback(c.graph, c.module, f), {}) excl(c.includedFiles, f.int) proc evalInclude(c: PContext, n: PNode): PNode = result = newNodeI(nkStmtList, n.info) - result.add n + var resolvedIncStmt: PNode = nil + if optCompress in c.config.globalOptions: + # New resolve the include filenames to string literals that contain absolute paths, + # nicer for IC: + resolvedIncStmt = newNodeI(nkIncludeStmt, n.info) + result.add resolvedIncStmt + else: + # Legacy: Keep `include` statement as is: + result.add n template checkAs(it: PNode) = if it.kind == nkInfix and it.len == 3: let op = it[0].getPIdent @@ -2832,9 +2842,9 @@ proc evalInclude(c: PContext, n: PNode): PNode = for x in it[lastPos]: checkAs(x) imp[lastPos] = x - incMod(c, n, imp, result) + incMod(c, n, imp, result, resolvedIncStmt) else: - incMod(c, n, it, result) + incMod(c, n, it, result, resolvedIncStmt) proc recursiveSetFlag(n: PNode, flag: TNodeFlag) = if n != nil: From 0bf8bbfd9f47bcd12370ef7d07390f09cc0e0f9a Mon Sep 17 00:00:00 2001 From: araq Date: Sat, 8 Nov 2025 08:52:19 +0100 Subject: [PATCH 29/38] progress --- compiler/ast2nif.nim | 271 +++++++++++++++++++++++++------------------ 1 file changed, 158 insertions(+), 113 deletions(-) diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index 744c4cdfd2c8..5ab44e4ca1bc 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -17,7 +17,7 @@ import "../dist/nimony/src/lib" / [bitabs, nifstreams, nifcursors, lineinfos, nifindexes] import "../dist/nimony/src/gear2" / modnames -import icnif / [enum2nif, icniftags] +import icnif / [enum2nif] # ---------------- Line info handling ----------------------------------------- @@ -114,9 +114,8 @@ let type Writer = object - dest: TokenBuf - deps: TokenBuf - inner: LineInfoWriter + deps: TokenBuf # include&import deps + infos: LineInfoWriter currentModule: int32 writtenSyms: HashSet[ItemId] writtenTypes: HashSet[ItemId] @@ -135,7 +134,7 @@ proc toNifSymName(w: var Writer; sym: PSym): string = # Global symbol: ident.disamb.moduleSuffix let module = sym.itemId.module result.add '.' - result.add modname(w.moduleToNifSuffix, module, w.inner.config) + result.add modname(w.moduleToNifSuffix, module, w.infos.config) template buildTree(dest: var TokenBuf; tag: TagId; body: untyped) = dest.addParLe tag @@ -145,129 +144,129 @@ template buildTree(dest: var TokenBuf; tag: TagId; body: untyped) = template buildTree(dest: var TokenBuf; tag: string; body: untyped) = buildTree dest, pool.tags.getOrIncl(tag), body -proc writeFlags[E](w: var Writer; flags: set[E]) = +proc writeFlags[E](dest: var TokenBuf; flags: set[E]) = var flagsAsIdent = "" genFlags(flags, flagsAsIdent) if flagsAsIdent.len > 0: - w.dest.addIdent flagsAsIdent + dest.addIdent flagsAsIdent else: - w.dest.addDotToken + dest.addDotToken proc trLineInfo(w: var Writer; info: TLineInfo): PackedLineInfo {.inline.} = - result = nifLineInfo(w.inner, info) + result = nifLineInfo(w.infos, info) -proc writeNode(w: var Writer; n: PNode) -proc writeType(w: var Writer; typ: PType) -proc writeSym(w: var Writer; sym: PSym) +proc writeNode(w: var Writer; dest: var TokenBuf; n: PNode) +proc writeType(w: var Writer; dest: var TokenBuf; typ: PType) +proc writeSym(w: var Writer; dest: var TokenBuf; sym: PSym) proc typeToNifSym(w: var Writer; typ: PType): string = result = "`t." result.addInt typ.uniqueId.item result.add '.' - result.add modname(w.moduleToNifSuffix, typ.uniqueId.module, w.inner.config) - -proc writeTypeDef(w: var Writer; typ: PType) = - w.dest.buildTree tdefTag: - w.dest.addSymDef pool.syms.getOrIncl(w.typeToNifSym(typ)), NoLineInfo - - w.dest.addIdent toNifTag(typ.kind) - writeFlags(w, typ.flags) - w.dest.addIdent toNifTag(typ.callConv) - w.dest.addIntLit typ.size - w.dest.addIntLit typ.align - w.dest.addIntLit typ.paddingAtEnd - w.dest.addIntLit typ.itemId.item # nonUniqueId - - writeType(w, typ.typeInst) - writeNode(w, typ.n) - writeSym(w, typ.owner) - writeSym(w, typ.sym) + result.add modname(w.moduleToNifSuffix, typ.uniqueId.module, w.infos.config) + +proc writeTypeDef(w: var Writer; dest: var TokenBuf; typ: PType) = + dest.buildTree tdefTag: + dest.addSymDef pool.syms.getOrIncl(w.typeToNifSym(typ)), NoLineInfo + + dest.addIdent toNifTag(typ.kind) + writeFlags(dest, typ.flags) + dest.addIdent toNifTag(typ.callConv) + dest.addIntLit typ.size + dest.addIntLit typ.align + dest.addIntLit typ.paddingAtEnd + dest.addIntLit typ.itemId.item # nonUniqueId + + writeType(w, dest, typ.typeInst) + writeNode(w, dest, typ.n) + writeSym(w, dest, typ.owner) + writeSym(w, dest, typ.sym) # we store the type's elements here at the end so that # it is not ambiguous and saves space: for ch in typ.kids: - writeType(w, ch) + writeType(w, dest, ch) -proc writeType(w: var Writer; typ: PType) = +proc writeType(w: var Writer; dest: var TokenBuf; typ: PType) = if typ == nil: - w.dest.addDotToken() + dest.addDotToken() elif typ.itemId.module == w.currentModule and not w.writtenTypes.containsOrIncl(typ.uniqueId): - writeTypeDef(w, typ) + writeTypeDef(w, dest, typ) else: - w.dest.buildTree tuseTag: - w.dest.addSymUse pool.syms.getOrIncl(w.typeToNifSym(typ)), NoLineInfo + dest.buildTree tuseTag: + dest.addSymUse pool.syms.getOrIncl(w.typeToNifSym(typ)), NoLineInfo -proc writeSymDef(w: var Writer; sym: PSym) = - w.dest.addParLe sdefTag, trLineInfo(w, sym.info) - w.dest.addSymDef pool.syms.getOrIncl(w.toNifSymName(sym)), NoLineInfo +proc writeSymDef(w: var Writer; dest: var TokenBuf; sym: PSym) = + dest.addParLe sdefTag, trLineInfo(w, sym.info) + dest.addSymDef pool.syms.getOrIncl(w.toNifSymName(sym)), NoLineInfo if sym.magic == mNone: - w.dest.addDotToken + dest.addDotToken else: - w.dest.addIdent toNifTag(sym.magic) - writeFlags(w, sym.flags) - writeFlags(w, sym.options) - w.dest.addIntLit sym.offset - w.dest.addIntLit sym.disamb - w.dest.buildTree sym.kind.toNifTag: + dest.addIdent toNifTag(sym.magic) + writeFlags(dest, sym.flags) + writeFlags(dest, sym.options) + dest.addIntLit sym.offset + dest.addIntLit sym.disamb + dest.buildTree sym.kind.toNifTag: case sym.kind of skLet, skVar, skField, skForVar: - writeSym(w, sym.guard) - w.dest.addIntLit sym.bitsize - w.dest.addIntLit sym.alignment + writeSym(w, dest, sym.guard) + dest.addIntLit sym.bitsize + dest.addIntLit sym.alignment else: discard if sym.kind == skModule: - w.dest.addDotToken() # position will be set by the loader! + dest.addDotToken() # position will be set by the loader! else: - w.dest.addIntLit sym.position - writeType(w, sym.typ) - writeSym(w, sym.owner) + dest.addIntLit sym.position + writeType(w, dest, sym.typ) + writeSym(w, dest, sym.owner) # We do not store `sym.ast` here but instead set it in the deserializer #writeNode(w, sym.ast) - w.dest.addIdent toNifTag(sym.loc.k) - w.dest.addStrLit sym.loc.snippet - writeNode(w, sym.constraint) - writeSym(w, sym.instantiatedFrom) - w.dest.addParRi + dest.addIdent toNifTag(sym.loc.k) + dest.addStrLit sym.loc.snippet + writeNode(w, dest, sym.constraint) + writeSym(w, dest, sym.instantiatedFrom) + dest.addParRi -proc writeSym(w: var Writer; sym: PSym) = +proc writeSym(w: var Writer; dest: var TokenBuf; sym: PSym) = if sym == nil: - w.dest.addDotToken() + dest.addDotToken() elif sym.itemId.module == w.currentModule and not w.writtenSyms.containsOrIncl(sym.itemId): - writeSymDef(w, sym) + writeSymDef(w, dest, sym) else: # NIF has direct support for symbol references so we don't need to use a tag here, # unlike what we do for types! - w.dest.addSymUse pool.syms.getOrIncl(w.toNifSymName(sym)), NoLineInfo + dest.addSymUse pool.syms.getOrIncl(w.toNifSymName(sym)), NoLineInfo -proc writeSymNode(w: var Writer; n: PNode; sym: PSym) = +proc writeSymNode(w: var Writer; dest: var TokenBuf; n: PNode; sym: PSym) = if sym == nil: - w.dest.addDotToken() + dest.addDotToken() elif sym.itemId.module == w.currentModule and not w.writtenSyms.containsOrIncl(sym.itemId): if n.typ != n.sym.typ: - w.dest.buildTree hiddenTypeTag, trLineInfo(w, n.info): - writeSymDef(w, sym) + dest.buildTree hiddenTypeTag, trLineInfo(w, n.info): + writeSymDef(w, dest, sym) else: - writeSymDef(w, sym) + writeSymDef(w, dest, sym) else: # NIF has direct support for symbol references so we don't need to use a tag here, # unlike what we do for types! let info = trLineInfo(w, n.info) if n.typ != n.sym.typ: - w.dest.buildTree hiddenTypeTag, info: - w.dest.addSymUse pool.syms.getOrIncl(w.toNifSymName(sym)), info + dest.buildTree hiddenTypeTag, info: + dest.addSymUse pool.syms.getOrIncl(w.toNifSymName(sym)), info else: - w.dest.addSymUse pool.syms.getOrIncl(w.toNifSymName(sym)), info + dest.addSymUse pool.syms.getOrIncl(w.toNifSymName(sym)), info -proc writeNodeFlags(w: var Writer; flags: set[TNodeFlag]) {.inline.} = - writeFlags(w, flags) +proc writeNodeFlags(dest: var TokenBuf; flags: set[TNodeFlag]) {.inline.} = + writeFlags(dest, flags) -template withNode(w: var Writer; n: PNode; body: untyped) = - w.dest.addParLe pool.tags.getOrIncl(toNifTag(n.kind)), trLineInfo(w, n.info) - writeNodeFlags(w, n.flags) - writeType(w, n.typ) +template withNode(w: var Writer; dest: var TokenBuf; n: PNode; body: untyped) = + dest.addParLe pool.tags.getOrIncl(toNifTag(n.kind)), trLineInfo(w, n.info) + writeNodeFlags(dest, n.flags) + writeType(w, dest, n.typ) body - w.dest.addParRi + dest.addParRi proc addLocalSym(w: var Writer; n: PNode) = ## Add symbol from a node to locals set if it's a symbol node @@ -284,76 +283,87 @@ proc addLocalSyms(w: var Writer; n: PNode) = addLocalSym(w, n) proc trInclude(w: var Writer; n: PNode) = - discard + w.deps.addParLe pool.tags.getOrIncl(toNifTag(n.kind)), trLineInfo(w, n.info) + for child in n: + assert child.kind == nkStrLit + w.deps.addStrLit child.strVal + w.deps.addParRi proc trImport(w: var Writer; n: PNode) = - discard - -proc writeNode(w: var Writer; n: PNode) = + w.deps.addParLe pool.tags.getOrIncl(toNifTag(n.kind)), trLineInfo(w, n.info) + for child in n: + assert child.kind == nkSym + let s = child.sym + assert s.kind == skModule + let fp = toFullPath(w.infos.config, s.position.FileIndex) + w.deps.addStrLit fp + w.deps.addParRi + +proc writeNode(w: var Writer; dest: var TokenBuf; n: PNode) = if n == nil: - w.dest.addDotToken + dest.addDotToken else: case n.kind: of nkEmpty: let info = trLineInfo(w, n.info) - w.dest.addParLe pool.tags.getOrIncl(toNifTag(nkEmpty)), info - writeNodeFlags(w, n.flags) - w.dest.addParRi + dest.addParLe pool.tags.getOrIncl(toNifTag(nkEmpty)), info + writeNodeFlags(dest, n.flags) + dest.addParRi of nkIdent: # nkIdent uses flags and typ when it is a generic parameter - w.withNode n: - w.dest.addIdent n.ident.s + w.withNode dest, n: + dest.addIdent n.ident.s of nkSym: - writeSymNode(w, n, n.sym) + writeSymNode(w, dest, n, n.sym) of nkCharLit: - w.withNode n: - w.dest.add charToken(n.intVal.char, NoLineInfo) + w.withNode dest, n: + dest.add charToken(n.intVal.char, NoLineInfo) of nkIntLit .. nkInt64Lit: - w.withNode n: - w.dest.addIntLit n.intVal + w.withNode dest, n: + dest.addIntLit n.intVal of nkUIntLit .. nkUInt64Lit: - w.withNode n: - w.dest.addUIntLit cast[BiggestUInt](n.intVal) + w.withNode dest, n: + dest.addUIntLit cast[BiggestUInt](n.intVal) of nkFloatLit .. nkFloat128Lit: - w.withNode n: - w.dest.add floatToken(pool.floats.getOrIncl(n.floatVal), NoLineInfo) + w.withNode dest, n: + dest.add floatToken(pool.floats.getOrIncl(n.floatVal), NoLineInfo) of nkStrLit .. nkTripleStrLit: - w.withNode n: - w.dest.addStrLit n.strVal + w.withNode dest, n: + dest.addStrLit n.strVal of nkNilLit: - w.withNode n: + w.withNode dest, n: discard of nkLetSection, nkVarSection, nkConstSection, nkGenericParams: # Track local variables declared in let/var sections - w.withNode(n): + w.withNode dest, n: for child in n: addLocalSyms w, child # Process the child node - writeNode(w, child) + writeNode(w, dest, child) of nkForStmt, nkTypeDef: # Track for loop variable (first child is the loop variable) - w.withNode(n): + w.withNode dest, n: if n.len > 0: addLocalSyms(w, n[0]) for i in 0 ..< n.len: - writeNode(w, n[i]) + writeNode(w, dest, n[i]) of nkFormalParams: # Track parameters (first child is return type, rest are parameters) - w.withNode(n): + w.withNode dest, n: for i in 0 ..< n.len: if i > 0: # Skip return type addLocalSyms(w, n[i]) - writeNode(w, n[i]) + writeNode(w, dest, n[i]) of nkProcDef, nkFuncDef, nkMethodDef, nkIteratorDef, nkConverterDef, nkLambda, nkDo, nkMacroDef: inc w.inProc # Entering a proc/function body - parameters are local var ast = n if n[namePos].kind == nkSym: ast = n[namePos].sym.ast - w.withNode(ast): + w.withNode dest, ast: # Process body and other parts for i in 0 ..< ast.len: - writeNode(w, ast[i]) + writeNode(w, dest, ast[i]) dec w.inProc of nkImportStmt: # this has been transformed for us, see `importer.nim` to contain a list of module syms: @@ -361,13 +371,48 @@ proc writeNode(w: var Writer; n: PNode) = of nkIncludeStmt: trInclude w, n else: - w.withNode(n): + w.withNode dest, n: for i in 0 ..< n.len: - writeNode(w, n[i]) + writeNode(w, dest, n[i]) + +proc writeToplevelNode(w: var Writer; outer, inner: var TokenBuf; n: PNode) = + case n.kind + of nkStmtList, nkStmtListExpr: + for son in n: writeToplevelNode(w, outer, inner, son) + of nkProcDef, nkFuncDef, nkMethodDef, nkIteratorDef, nkConverterDef, nkLambda, nkDo, nkMacroDef: + # Delegate to `w.topLevel`! + writeNode w, inner, n + of nkConstSection, nkTypeSection, nkTypeDef: + writeNode w, inner, n + else: + writeNode w, outer, n proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode) = - var w = Writer(inner: LineInfoWriter(config: config), currentModule: thisModule) - w.writeNode n - let m = modname(w.moduleToNifSuffix, w.currentModule, w.inner.config) + var w = Writer(infos: LineInfoWriter(config: config), currentModule: thisModule) + var outer = createTokenBuf(300) + var inner = createTokenBuf(300) + + w.writeToplevelNode outer, inner, n + let m = modname(w.moduleToNifSuffix, w.currentModule, w.infos.config) let d = toGeneratedFile(config, AbsoluteFile(m), ".nif").string - writeFileAndIndex d, w.dest + + var dest = createTokenBuf(600) + let rootInfo = if outer.len > 0: outer[0].info else: NoLineInfo + dest.addParLe pool.tags.getOrIncl(toNifTag(nkStmtList)), rootInfo + dest.add w.deps + dest.add outer + dest.add inner + dest.addParRi() + + writeFileAndIndex d, dest + + +# --------------------------- Loader (lazy!) ----------------------------------------------- + +proc loadNifModule*(config: ConfigRef; f: FileIndex): PNode = + var moduleToNifSuffix = initTable[FileIndex, string]() + + let m = modname(moduleToNifSuffix, f.int, config) + let d = toGeneratedFile(config, AbsoluteFile(m), ".nif").string + + result = nil From e173a6a7b5ca9b6decf6a1e5fabeee9cc38dd6b6 Mon Sep 17 00:00:00 2001 From: araq Date: Sat, 8 Nov 2025 09:04:37 +0100 Subject: [PATCH 30/38] added TLoc writing --- compiler/ast2nif.nim | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index 5ab44e4ca1bc..3eeb1c0fb47d 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -181,6 +181,12 @@ proc writeTypeDef(w: var Writer; dest: var TokenBuf; typ: PType) = writeNode(w, dest, typ.n) writeSym(w, dest, typ.owner) writeSym(w, dest, typ.sym) + + # Write TLoc structure + dest.addIdent toNifTag(typ.loc.k) + dest.addIntLit ord(typ.loc.storage) # TStorageLoc: OnUnknown=0, OnStatic=1, OnStack=2, OnHeap=3 + writeFlags(dest, typ.loc.flags) # TLocFlags + # we store the type's elements here at the end so that # it is not ambiguous and saves space: for ch in typ.kids: @@ -223,7 +229,10 @@ proc writeSymDef(w: var Writer; dest: var TokenBuf; sym: PSym) = writeSym(w, dest, sym.owner) # We do not store `sym.ast` here but instead set it in the deserializer #writeNode(w, sym.ast) + # Write TLoc structure dest.addIdent toNifTag(sym.loc.k) + dest.addIntLit ord(sym.loc.storage) # TStorageLoc: OnUnknown=0, OnStatic=1, OnStack=2, OnHeap=3 + writeFlags(dest, sym.loc.flags) # TLocFlags dest.addStrLit sym.loc.snippet writeNode(w, dest, sym.constraint) writeSym(w, dest, sym.instantiatedFrom) From 2cb1c539e061b2b5c23d7157707e6e8b6fe5b712 Mon Sep 17 00:00:00 2001 From: araq Date: Sat, 8 Nov 2025 15:51:55 +0100 Subject: [PATCH 31/38] wip --- compiler/ast.nim | 3 +- compiler/ast2nif.nim | 257 +++++++++++++++++++++++++++++++++-- compiler/ccgreset.nim | 2 +- compiler/expanddefaults.nim | 2 +- compiler/icnif/enum2nif.nim | 2 + compiler/jsgen.nim | 2 +- compiler/liftdestructors.nim | 2 +- compiler/nifgen.nim | 3 +- compiler/typeallowed.nim | 2 +- compiler/types.nim | 4 +- compiler/vmdeps.nim | 2 +- 11 files changed, 262 insertions(+), 19 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index d80589c087d3..1a686b2f6f72 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -268,6 +268,7 @@ type tyVoid # now different from tyEmpty, hurray! tyIterable + tyStub static: # remind us when TTypeKind stops to fit in a single 64-bit word @@ -786,7 +787,7 @@ type sym*: PSym # types have the sym associated with them # it is used for converting types to strings size*: BiggestInt # the size of the type in bytes - # -1 means that the size is unkwown + # -1 means that the size is unknown align*: int16 # the type's alignment requirements paddingAtEnd*: int16 # loc*: TLoc diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index 3eeb1c0fb47d..2956dbbc641a 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -10,6 +10,7 @@ ## AST to NIF bridge. import std / [assertions, tables, sets] +from std / strutils import startsWith import ast, idents, msgs, options import lineinfos as astli import pathutils @@ -165,6 +166,12 @@ proc typeToNifSym(w: var Writer; typ: PType): string = result.add '.' result.add modname(w.moduleToNifSuffix, typ.uniqueId.module, w.infos.config) +proc writeLoc(w: var Writer; dest: var TokenBuf; loc: TLoc) = + dest.addIdent toNifTag(loc.k) + dest.addIntLit ord(loc.storage) # TStorageLoc: OnUnknown=0, OnStatic=1, OnStack=2, OnHeap=3 + writeFlags(dest, loc.flags) # TLocFlags + dest.addStrLit loc.snippet + proc writeTypeDef(w: var Writer; dest: var TokenBuf; typ: PType) = dest.buildTree tdefTag: dest.addSymDef pool.syms.getOrIncl(w.typeToNifSym(typ)), NoLineInfo @@ -183,10 +190,7 @@ proc writeTypeDef(w: var Writer; dest: var TokenBuf; typ: PType) = writeSym(w, dest, typ.sym) # Write TLoc structure - dest.addIdent toNifTag(typ.loc.k) - dest.addIntLit ord(typ.loc.storage) # TStorageLoc: OnUnknown=0, OnStatic=1, OnStack=2, OnHeap=3 - writeFlags(dest, typ.loc.flags) # TLocFlags - + writeLoc w, dest, typ.loc # we store the type's elements here at the end so that # it is not ambiguous and saves space: for ch in typ.kids: @@ -202,6 +206,20 @@ proc writeType(w: var Writer; dest: var TokenBuf; typ: PType) = dest.buildTree tuseTag: dest.addSymUse pool.syms.getOrIncl(w.typeToNifSym(typ)), NoLineInfo +proc writeBool(dest: var TokenBuf; b: bool) = + dest.buildTree (if b: "true" else: "false"): + discard + +proc writeLib(w: var Writer; dest: var TokenBuf; lib: PLib) = + if lib == nil: + dest.addDotToken() + else: + dest.buildTree $lib.kind: + dest.writeBool lib.generated + dest.writeBool lib.isOverridden + dest.addStrLit lib.name + writeNode w, dest, lib.path + proc writeSymDef(w: var Writer; dest: var TokenBuf; sym: PSym) = dest.addParLe sdefTag, trLineInfo(w, sym.info) dest.addSymDef pool.syms.getOrIncl(w.toNifSymName(sym)), NoLineInfo @@ -229,11 +247,7 @@ proc writeSymDef(w: var Writer; dest: var TokenBuf; sym: PSym) = writeSym(w, dest, sym.owner) # We do not store `sym.ast` here but instead set it in the deserializer #writeNode(w, sym.ast) - # Write TLoc structure - dest.addIdent toNifTag(sym.loc.k) - dest.addIntLit ord(sym.loc.storage) # TStorageLoc: OnUnknown=0, OnStatic=1, OnStack=2, OnHeap=3 - writeFlags(dest, sym.loc.flags) # TLocFlags - dest.addStrLit sym.loc.snippet + writeLoc w, dest, sym.loc writeNode(w, dest, sym.constraint) writeSym(w, dest, sym.instantiatedFrom) dest.addParRi @@ -401,12 +415,19 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode) = var outer = createTokenBuf(300) var inner = createTokenBuf(300) + let rootInfo = trLineInfo(w, n.info) + outer.addParLe pool.tags.getOrIncl(toNifTag(nkStmtList)), rootInfo + inner.addParLe pool.tags.getOrIncl(toNifTag(nkStmtList)), rootInfo + w.writeToplevelNode outer, inner, n + + outer.addParRi() + inner.addParRi() + let m = modname(w.moduleToNifSuffix, w.currentModule, w.infos.config) let d = toGeneratedFile(config, AbsoluteFile(m), ".nif").string var dest = createTokenBuf(600) - let rootInfo = if outer.len > 0: outer[0].info else: NoLineInfo dest.addParLe pool.tags.getOrIncl(toNifTag(nkStmtList)), rootInfo dest.add w.deps dest.add outer @@ -418,6 +439,222 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode) = # --------------------------- Loader (lazy!) ----------------------------------------------- +proc nodeKind(n: Cursor): TNodeKind {.inline.} = + assert n.kind == ParLe + pool.tags[n.tagId].parseNodeKind() + +proc expect(n: Cursor; k: set[NifKind]) = + if n.kind notin k: + when defined(debug): + writeStackTrace() + quit "[NIF decoder] expected: " & $k & " but got: " & $n.kind & toString n + +proc expect(n: Cursor; k: NifKind) {.inline.} = + expect n, {k} + +proc incExpect(n: var Cursor; k: set[NifKind]) = + inc n + expect n, k + +proc incExpect(n: var Cursor; k: NifKind) {.inline.} = + incExpect n, {k} + +proc skipParRi(n: var Cursor) = + expect n, {ParRi} + inc n + +proc firstSon*(n: Cursor): Cursor {.inline.} = + result = n + inc result + +proc expectTag(n: Cursor; tagId: TagId) = + if n.kind == ParLe and n.tagId == tagId: + discard + else: + when defined(debug): + writeStackTrace() + if n.kind != ParLe: + quit "[NIF decoder] expected: ParLe but got: " & $n.kind & toString n + else: + quit "[NIF decoder] expected: " & pool.tags[tagId] & " but got: " & pool.tags[n.tagId] & toString n + +proc incExpectTag(n: var Cursor; tagId: TagId) = + inc n + expectTag(n, tagId) + +type + DecodeContext* = object + infos: LineInfoWriter + moduleIds: Table[string, int32] + types: Table[ItemId, (PType, TLineInfo)] + indexes: seq[NifIndex] + cache: IdentCache + +proc createDecodeContext*(config: ConfigRef; cache: IdentCache): DecodeContext = + ## Supposed to be a global variable + result = DecodeContext(infos: LineInfoWriter(config: config), cache: cache) + +proc moduleId(c: var DecodeContext; suffix: string): int32 = + # We don't know the "real" FileIndex due to our mapping to a short "Module suffix" + # This is not a problem, we use negative `ItemId.module` values here and then + # there is no interference with in-memory-modules. Modulegraphs.nim already uses -1 + # so we start at -2 here. + result = c.moduleIds.getOrDefault(suffix) + if result == 0: + result = -int32(c.moduleIds.len + 2) # negative index! + c.moduleIds[suffix] = result + c.indexes.add readIndex((getNimcacheDir(c.infos.config) / RelativeFile(suffix & ".idx.nif")).string) + +proc getOffset(c: var DecodeContext; module: int32; nifName: string): NifIndexEntry = + assert module < 0'i32 + let index = (-module) - 2'i32 + let ii = addr c.indexes[index] + result = ii.public.getOrDefault(nifName) + if result.offset == 0: + result = ii.private.getOrDefault(nifName) + if result.offset == 0: + raiseAssert "symbol has no offset: " & nifName + +proc fromNifNodeFlags(n: var Cursor): set[TNodeFlag] = + if n.kind == DotToken: + result = {} + inc n + elif n.kind == Ident: + result = parseNodeFlags(pool.strings[n.litId]) + inc n + else: + raiseAssert "expected Node flag (`ident`) but got " & $n.kind + +proc loadTypeStub(c: var DecodeContext; t: SymId): PType = + let name = pool.syms[t] + assert name.startsWith("`t.") + var i = len("`t.") + var itemId = 0'i32 + while i < name.len and name[i] in {'0'..'9'}: + itemId = itemId * 10'i32 + int32(name[i].ord - ord('0')) + inc i + if i < name.len and name[i] == '.': inc i + let suffix = name.substr(i) + let id = ItemId(module: moduleId(c, suffix), item: itemId) + result = c.types.getOrDefault(id)[0] + if result == nil: + let offs = c.getOffset(id.module, name) + result = PType(itemId: id, uniqueId: id, kind: tyStub, size: -offs.offset) + c.types[id] = (result, c.infos.oldLineInfo(offs.info)) + +proc loadTypeStub(c: var DecodeContext; n: var Cursor): PType = + if n.kind == DotToken: + result = nil + inc n + elif n.kind == Symbol: + let s = n.symId + result = loadTypeStub(c, s) + inc n + elif n.kind == ParLe and n.tagId == tdefTag: + let s = n.firstSon.symId + skip n + result = loadTypeStub(c, s) + else: + raiseAssert "type expected but got " & $n.kind + +proc isStub*(t: PType): bool = t.kind == tyStub + +proc loadTypeBody(c: var DecodeContext; t: PType) = + if t.kind != tyStub: return + assert t.size < 0, "type has no offset" + + +template withNode(c: var DecodeContext; n: var Cursor; result: PNode; kind: TNodeKind; body: untyped) = + let info = c.infos.oldLineInfo(n.info) + let flags = fromNifNodeFlags n + result = newNodeI(kind, info) + result.flags = flags + result.typ = c.loadTypeStub n + body + skipParRi n + +proc fromNif(c: var DecodeContext; n: var Cursor): PNode = + result = nil + case n.kind: + of DotToken: + result = nil + inc n + of ParLe: + let kind = n.nodeKind + case kind: + of nkEmpty: + result = newNodeI(nkEmpty, c.infos.oldLineInfo(n.info)) + incExpect n, {Ident, DotToken} + let flags = fromNifNodeFlags n + result.flags = flags + skipParRi n + of nkIdent: + let info = c.infos.oldLineInfo(n.info) + incExpect n, {DotToken, Ident} + let flags = fromNifNodeFlags n + let typ = c.loadTypeStub n + expect n, Ident + result = newIdentNode(c.cache.getIdent(pool.strings[n.litId]), info) + inc n + result.flags = flags + result.typ = typ + skipParRi n + of nkSym: + c.withNode n, result, kind: + #result.sym = c.fromNifSymbol n + discard + of nkCharLit: + c.withNode n, result, kind: + expect n, CharLit + result.intVal = n.charLit.int + inc n + of nkIntLit .. nkInt64Lit: + c.withNode n, result, kind: + expect n, IntLit + result.intVal = pool.integers[n.intId] + inc n + of nkUIntLit .. nkUInt64Lit: + c.withNode n, result, kind: + expect n, UIntLit + result.intVal = cast[BiggestInt](pool.uintegers[n.uintId]) + inc n + of nkFloatLit .. nkFloat128Lit: + c.withNode n, result, kind: + if n.kind == FloatLit: + result.floatVal = pool.floats[n.floatId] + inc n + elif n.kind == ParLe: + case pool.tags[n.tagId] + of "inf": + result.floatVal = Inf + of "nan": + result.floatVal = NaN + of "neginf": + result.floatVal = NegInf + else: + raiseAssert "expected float literal but got " & pool.tags[n.tagId] + inc n + skipParRi n + else: + raiseAssert "expected float literal but got " & $n.kind + of nkStrLit .. nkTripleStrLit: + c.withNode n, result, kind: + expect n, StringLit + result.strVal = pool.strings[n.litId] + inc n + of nkNilLit: + c.withNode n, result, kind: + discard + of nkNone: + raiseAssert "Unknown tag " & pool.tags[n.tagId] + else: + c.withNode n, result, kind: + while n.kind != ParRi: + result.addAllowNil c.fromNif n + else: + raiseAssert "Not yet implemented " & $n.kind + + proc loadNifModule*(config: ConfigRef; f: FileIndex): PNode = var moduleToNifSuffix = initTable[FileIndex, string]() diff --git a/compiler/ccgreset.nim b/compiler/ccgreset.nim index 84478dd07e15..0e4c73cd17ed 100644 --- a/compiler/ccgreset.nim +++ b/compiler/ccgreset.nim @@ -112,7 +112,7 @@ proc specializeResetT(p: BProc, accessor: Rope, typ: PType) = tyGenericParam, tyOrdinal, tyOpenArray, tyForward, tyVarargs, tyUncheckedArray, tyError, tyBuiltInTypeClass, tyUserTypeClass, tyUserTypeClassInst, tyCompositeTypeClass, tyAnd, tyOr, tyNot, - tyAnything, tyStatic, tyFromExpr, tyConcept, tyVoid, tyIterable: + tyAnything, tyStatic, tyFromExpr, tyConcept, tyVoid, tyIterable, tyStub: discard proc specializeReset(p: BProc, a: TLoc) = diff --git a/compiler/expanddefaults.nim b/compiler/expanddefaults.nim index c520d8849f2d..e343317e5414 100644 --- a/compiler/expanddefaults.nim +++ b/compiler/expanddefaults.nim @@ -124,7 +124,7 @@ proc expandDefault(t: PType; info: TLineInfo): PNode = result = newZero(t, info, nkBracket) of tyString: result = newZero(t, info, nkStrLit) - of tyNone, tyEmpty, tyUntyped, tyTyped, tyTypeDesc, + of tyNone, tyEmpty, tyUntyped, tyTyped, tyTypeDesc, tyStub, tyNil, tyGenericInvocation, tyError, tyBuiltInTypeClass, tyUserTypeClass, tyUserTypeClassInst, tyCompositeTypeClass, tyAnd, tyOr, tyNot, tyAnything, tyConcept, tyIterable, tyForward: diff --git a/compiler/icnif/enum2nif.nim b/compiler/icnif/enum2nif.nim index ecfdcfa0db2e..98b614382786 100644 --- a/compiler/icnif/enum2nif.nim +++ b/compiler/icnif/enum2nif.nim @@ -469,6 +469,7 @@ proc toNifTag*(s: TTypeKind): string = of tyConcept: "concept" of tyVoid: "void" of tyIterable: "iterable" + of tyStub: "stub" proc parseTypeKind*(s: string): TTypeKind = @@ -538,6 +539,7 @@ proc parseTypeKind*(s: string): TTypeKind = of "concept": tyConcept of "void": tyVoid of "iterable": tyIterable + of "stub": tyStub else: tyNone diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index fd8ef583d00a..4d63d44acb64 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -219,7 +219,7 @@ proc mapType(typ: PType): TJSTypeKind = else: result = etyNone of tyProc: result = etyProc of tyCstring: result = etyString - of tyConcept, tyIterable: + of tyConcept, tyIterable, tyStub: raiseAssert "unreachable" proc mapType(p: PProc; typ: PType): TJSTypeKind = diff --git a/compiler/liftdestructors.nim b/compiler/liftdestructors.nim index 5d8fbc179dd8..42b96aa1cab9 100644 --- a/compiler/liftdestructors.nim +++ b/compiler/liftdestructors.nim @@ -972,7 +972,7 @@ proc ownedClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode) = case t.kind - of tyNone, tyEmpty, tyVoid: discard + of tyNone, tyEmpty, tyVoid, tyStub: discard of tyPointer, tySet, tyBool, tyChar, tyEnum, tyInt..tyUInt64, tyCstring, tyPtr, tyUncheckedArray, tyVar, tyLent: defaultOp(c, t, body, x, y) diff --git a/compiler/nifgen.nim b/compiler/nifgen.nim index cf267ef14bc5..c9d86b50f00b 100644 --- a/compiler/nifgen.nim +++ b/compiler/nifgen.nim @@ -666,6 +666,7 @@ proc toNifTag(s: TTypeKind): string = of tyConcept: "concept" of tyVoid: "void" of tyIterable: "iterable" + of tyStub: "stub" proc atom(t: PType; c: var TranslationContext) = c.b.withTree toNifTag(t.kind): @@ -924,7 +925,7 @@ proc toNifType(t: PType; parent: PNode; c: var TranslationContext) = atom t, c, "err" of tyCompositeTypeClass: toNifType t.last, parent, c of tyInferred: toNifType t.skipModifier, parent, c - of tyAnything: atom t, c + of tyAnything, tyStub: atom t, c of tyStatic: c.typeHead t: if t.hasElementType: diff --git a/compiler/typeallowed.nim b/compiler/typeallowed.nim index 80b532371cb0..0529a6671f6d 100644 --- a/compiler/typeallowed.nim +++ b/compiler/typeallowed.nim @@ -201,7 +201,7 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, result = typeAllowedNode(marker, t.n, kind, c, flags) of tyEmpty: if kind in {skVar, skLet}: result = t - of tyError: + of tyError, tyStub: # for now same as error node; we say it's a valid type as it should # prevent cascading errors: result = nil diff --git a/compiler/types.nim b/compiler/types.nim index bd65c3f33137..b26a949407de 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -491,7 +491,7 @@ const "BuiltInTypeClass", "UserTypeClass", "UserTypeClassInst", "CompositeTypeClass", "inferred", "and", "or", "not", "any", "static", "TypeFromExpr", "concept", # xxx bugfix - "void", "iterable"] + "void", "iterable", "stub"] const preferToResolveSymbols = {preferName, preferTypeName, preferModuleInfo, preferGenericArg, preferResolved, preferMixed, preferInlayHint, preferInferredEffects} @@ -1344,6 +1344,8 @@ proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool = result = a.id == b.id and sameFlags(a, b) of tyError: result = b.kind == tyError + of tyStub: + result = false of tyTuple: withoutShallowFlags: cycleCheck() diff --git a/compiler/vmdeps.nim b/compiler/vmdeps.nim index 72eec34eadb6..882f030ee764 100644 --- a/compiler/vmdeps.nim +++ b/compiler/vmdeps.nim @@ -97,7 +97,7 @@ proc mapTypeToAstX(cache: IdentCache; t: PType; info: TLineInfo; return atomicType(t.sym) case t.kind - of tyNone: result = atomicType("none", mNone) + of tyNone, tyStub: result = atomicType("none", mNone) of tyBool: result = atomicType("bool", mBool) of tyChar: result = atomicType("char", mChar) of tyNil: result = atomicType("nil", mNil) From 44f3dcc9a344c9513db90821db8a823d892f0333 Mon Sep 17 00:00:00 2001 From: araq Date: Sat, 8 Nov 2025 23:42:04 +0100 Subject: [PATCH 32/38] wip --- compiler/ast2nif.nim | 119 ++++++++++++++++++++++++++++++++++-- compiler/icnif/enum2nif.nim | 17 ++++++ 2 files changed, 131 insertions(+), 5 deletions(-) diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index 2956dbbc641a..1fe4863742c8 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -137,6 +137,25 @@ proc toNifSymName(w: var Writer; sym: PSym): string = result.add '.' result.add modname(w.moduleToNifSuffix, module, w.infos.config) +type + ParsedSymName* = object + name*: string + module*: string + +proc parseSymName*(s: string): ParsedSymName = + var i = s.len - 2 + while i > 0: + if s[i] == '.': + if s[i+1] in {'0'..'9'}: + return ParsedSymName(name: substr(s, 0, i-1), module: "") + else: + let mend = s.high + var b = i-1 + while b > 0 and s[b] != '.': dec b + return ParsedSymName(name: substr(s, 0, b-1), module: substr(s, i+1, mend)) + dec i + return ParsedSymName(name: s, module: "") + template buildTree(dest: var TokenBuf; tag: TagId; body: untyped) = dest.addParLe tag body @@ -168,7 +187,7 @@ proc typeToNifSym(w: var Writer; typ: PType): string = proc writeLoc(w: var Writer; dest: var TokenBuf; loc: TLoc) = dest.addIdent toNifTag(loc.k) - dest.addIntLit ord(loc.storage) # TStorageLoc: OnUnknown=0, OnStatic=1, OnStack=2, OnHeap=3 + dest.addIdent toNifTag(loc.storage) writeFlags(dest, loc.flags) # TLocFlags dest.addStrLit loc.snippet @@ -486,7 +505,8 @@ type DecodeContext* = object infos: LineInfoWriter moduleIds: Table[string, int32] - types: Table[ItemId, (PType, TLineInfo)] + types: Table[ItemId, (PType, NifIndexEntry)] + syms: Table[ItemId, (PSym, NifIndexEntry)] indexes: seq[NifIndex] cache: IdentCache @@ -539,8 +559,8 @@ proc loadTypeStub(c: var DecodeContext; t: SymId): PType = result = c.types.getOrDefault(id)[0] if result == nil: let offs = c.getOffset(id.module, name) - result = PType(itemId: id, uniqueId: id, kind: tyStub, size: -offs.offset) - c.types[id] = (result, c.infos.oldLineInfo(offs.info)) + result = PType(itemId: id, uniqueId: id, kind: tyStub) + c.types[id] = (result, offs) proc loadTypeStub(c: var DecodeContext; n: var Cursor): PType = if n.kind == DotToken: @@ -557,12 +577,94 @@ proc loadTypeStub(c: var DecodeContext; n: var Cursor): PType = else: raiseAssert "type expected but got " & $n.kind +proc loadSymStub(c: var DecodeContext; t: SymId): PSym = + let name = parseSymName(pool.syms[t]) + let id = ItemId(module: moduleId(c, name.module), item: itemId) + result = c.types.getOrDefault(id)[0] + if result == nil: + let offs = c.getOffset(id.module, name) + result = PType(itemId: id, uniqueId: id, kind: tyStub) + c.types[id] = (result, offs) + +proc loadSymStub(c: var DecodeContext; n: var Cursor): PSym = + if n.kind == DotToken: + result = nil + inc n + elif n.kind == Symbol: + let s = n.symId + result = loadSymStub(c, s) + inc n + elif n.kind == ParLe and n.tagId == sdefTag: + let s = n.firstSon.symId + skip n + result = loadSymStub(c, s) + else: + raiseAssert "sym expected but got " & $n.kind + proc isStub*(t: PType): bool = t.kind == tyStub +proc loadLoc(c: var DecodeContext; n: var Cursor; loc: var TLoc) = + expect n, Ident + loc.k = pool.strings[n.litId].parseLocKind() + inc n + expect n, Ident + loc.storage = pool.strings[n.litId].parseStorageLoc() + inc n + expect n, Ident + loc.flags = pool.strings[n.litId].parseLocFlags() + inc n + expect n, StringLit + loc.snippet = pool.strings[n.litId] + inc n + proc loadTypeBody(c: var DecodeContext; t: PType) = if t.kind != tyStub: return assert t.size < 0, "type has no offset" - + var n = getCursorAt() + + expect n, ParLe + if n.tagId != tdefTag: + raiseAssert "(td) expected" + inc n + expect n, SymbolDef + # ignore the type's name, we have already used it to create this PType's itemId! + inc n + expect n, Ident + t.kind = parseTypeKind(pool.strings[n.litId]) + inc n + expect n, Ident + t.flags = parseTypeFlags(pool.strings[n.litId]) + inc n + expect n, Ident + t.callConv = parseCallingConvention(pool.strings[n.litId]) + inc n + expect n, IntLit + typ.size = pool.integers[n.intId] + inc n + + expect n, IntLit + typ.align = pool.integers[n.intId] + inc n + + expect n, IntLit + typ.paddingAtEnd = pool.integers[n.intId] + inc n + + expect n, IntLit + typ.itemId.item = pool.integers[n.intId] + inc n + + loadTypeStub c, n, typ.typeInst + loadNode c, n, typ.n + loadSymStub c, n, typ.owner + loadSymStub c, n, typ.sym + loadLoc c, n, typ.loc + + while n.kind != ParRi: + t.typ.kids.add loadTypeStub(c, n) + + skipParRi n + template withNode(c: var DecodeContext; n: var Cursor; result: PNode; kind: TNodeKind; body: untyped) = let info = c.infos.oldLineInfo(n.info) @@ -662,3 +764,10 @@ proc loadNifModule*(config: ConfigRef; f: FileIndex): PNode = let d = toGeneratedFile(config, AbsoluteFile(m), ".nif").string result = nil + +when isMainModule: + import std / syncio + let obj = parseSymName("a.123.sys") + echo obj.name, " ", obj.module + let objb = parseSymName("abcdef.0121") + echo objb.name, " ", objb.module diff --git a/compiler/icnif/enum2nif.nim b/compiler/icnif/enum2nif.nim index 98b614382786..20e62178551a 100644 --- a/compiler/icnif/enum2nif.nim +++ b/compiler/icnif/enum2nif.nim @@ -1172,6 +1172,23 @@ proc parseMagic*(s: string): TMagic = else: mNone +proc toNifTag*(s: TStorageLoc): string = + case s + of OnUnknown: "unknown" + of OnStatic: "static" + of OnStack: "stack" + of OnHeap: "heap" + + +proc parseStorageLoc*(s: string): TStorageLoc = + case s + of "unknown": OnUnknown + of "static": OnStatic + of "stack": OnStack + of "heap": OnHeap + else: OnUnknown + + proc genFlags*(s: set[TSymFlag]; dest: var string) = for e in s: case e From cf4ac376439c5abc117e1fa1d735e1ba8024ad85 Mon Sep 17 00:00:00 2001 From: araq Date: Sun, 9 Nov 2025 09:50:11 +0100 Subject: [PATCH 33/38] progress --- compiler/ast2nif.nim | 107 +++++++++++----- compiler/icnif/enum2nif.nim | 16 +-- tools/enumgen.nim | 247 ++++++++++++++++++++++++++++++++++++ 3 files changed, 331 insertions(+), 39 deletions(-) create mode 100644 tools/enumgen.nim diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index 1fe4863742c8..8e16f5da087d 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -141,18 +141,30 @@ type ParsedSymName* = object name*: string module*: string + count*: int proc parseSymName*(s: string): ParsedSymName = var i = s.len - 2 while i > 0: if s[i] == '.': if s[i+1] in {'0'..'9'}: - return ParsedSymName(name: substr(s, 0, i-1), module: "") + var count = ord(s[i+1]) - ord('0') + var j = i+2 + while j < s.len and s[j] in {'0'..'9'}: + count = count * 10 + ord(s[j]) - ord('0') + inc j + return ParsedSymName(name: substr(s, 0, i-1), module: "", count: count) else: let mend = s.high var b = i-1 while b > 0 and s[b] != '.': dec b - return ParsedSymName(name: substr(s, 0, b-1), module: substr(s, i+1, mend)) + var j = b+1 + var count = 0 + while j < s.len and s[j] in {'0'..'9'}: + count = count * 10 + ord(s[j]) - ord('0') + inc j + + return ParsedSymName(name: substr(s, 0, b-1), module: substr(s, i+1, mend), count: count) dec i return ParsedSymName(name: s, module: "") @@ -460,7 +472,7 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode) = proc nodeKind(n: Cursor): TNodeKind {.inline.} = assert n.kind == ParLe - pool.tags[n.tagId].parseNodeKind() + parse(TNodeKind, pool.tags[n.tagId]) proc expect(n: Cursor; k: set[NifKind]) = if n.kind notin k: @@ -501,6 +513,14 @@ proc incExpectTag(n: var Cursor; tagId: TagId) = inc n expectTag(n, tagId) +proc parseBool(n: var Cursor): bool = + if n.kind == ParLe: + result = pool.tags[n.tagId] == "true" + inc n + skipParRi n + else: + raiseAssert "(true)/(false) expected" + type DecodeContext* = object infos: LineInfoWriter @@ -508,6 +528,7 @@ type types: Table[ItemId, (PType, NifIndexEntry)] syms: Table[ItemId, (PSym, NifIndexEntry)] indexes: seq[NifIndex] + symCounters: Table[int32, int32] # Module ID -> counter cache: IdentCache proc createDecodeContext*(config: ConfigRef; cache: IdentCache): DecodeContext = @@ -545,6 +566,8 @@ proc fromNifNodeFlags(n: var Cursor): set[TNodeFlag] = else: raiseAssert "expected Node flag (`ident`) but got " & $n.kind +proc loadNode(c: var DecodeContext; n: var Cursor): PNode + proc loadTypeStub(c: var DecodeContext; t: SymId): PType = let name = pool.syms[t] assert name.startsWith("`t.") @@ -578,13 +601,18 @@ proc loadTypeStub(c: var DecodeContext; n: var Cursor): PType = raiseAssert "type expected but got " & $n.kind proc loadSymStub(c: var DecodeContext; t: SymId): PSym = - let name = parseSymName(pool.syms[t]) - let id = ItemId(module: moduleId(c, name.module), item: itemId) - result = c.types.getOrDefault(id)[0] + let symAsStr = pool.syms[t] + let sn = parseSymName(symAsStr) + let module = moduleId(c, sn.module) + let val = addr c.symCounters.mgetOrPut(module, 0) + inc val[] + + let id = ItemId(module: module, item: val[]) + result = c.syms.getOrDefault(id)[0] if result == nil: - let offs = c.getOffset(id.module, name) - result = PType(itemId: id, uniqueId: id, kind: tyStub) - c.types[id] = (result, offs) + let offs = c.getOffset(module, symAsStr) + result = PSym(itemId: id, kind: skStub, name: c.cache.getIdent(sn.name), disamb: sn.count.int32) + c.syms[id] = (result, offs) proc loadSymStub(c: var DecodeContext; n: var Cursor): PSym = if n.kind == DotToken: @@ -601,14 +629,15 @@ proc loadSymStub(c: var DecodeContext; n: var Cursor): PSym = else: raiseAssert "sym expected but got " & $n.kind -proc isStub*(t: PType): bool = t.kind == tyStub +proc isStub*(t: PType): bool {.inline.} = t.kind == tyStub +proc isStub*(s: PSym): bool {.inline.} = s.kind == skStub proc loadLoc(c: var DecodeContext; n: var Cursor; loc: var TLoc) = expect n, Ident - loc.k = pool.strings[n.litId].parseLocKind() + loc.k = parse(TLocKind, pool.strings[n.litId]) inc n expect n, Ident - loc.storage = pool.strings[n.litId].parseStorageLoc() + loc.storage = parse(TStorageLoc, pool.strings[n.litId]) inc n expect n, Ident loc.flags = pool.strings[n.litId].parseLocFlags() @@ -617,10 +646,9 @@ proc loadLoc(c: var DecodeContext; n: var Cursor; loc: var TLoc) = loc.snippet = pool.strings[n.litId] inc n -proc loadTypeBody(c: var DecodeContext; t: PType) = +proc loadType*(c: var DecodeContext; t: PType) = if t.kind != tyStub: return - assert t.size < 0, "type has no offset" - var n = getCursorAt() + var n = default(Cursor) # getCursorAt() expect n, ParLe if n.tagId != tdefTag: @@ -630,42 +658,59 @@ proc loadTypeBody(c: var DecodeContext; t: PType) = # ignore the type's name, we have already used it to create this PType's itemId! inc n expect n, Ident - t.kind = parseTypeKind(pool.strings[n.litId]) + t.kind = parse(TTypeKind, pool.strings[n.litId]) inc n expect n, Ident t.flags = parseTypeFlags(pool.strings[n.litId]) inc n expect n, Ident - t.callConv = parseCallingConvention(pool.strings[n.litId]) + t.callConv = parse(TCallingConvention, pool.strings[n.litId]) inc n expect n, IntLit - typ.size = pool.integers[n.intId] + t.size = pool.integers[n.intId] inc n expect n, IntLit - typ.align = pool.integers[n.intId] + t.align = pool.integers[n.intId].int16 inc n expect n, IntLit - typ.paddingAtEnd = pool.integers[n.intId] + t.paddingAtEnd = pool.integers[n.intId].int16 inc n expect n, IntLit - typ.itemId.item = pool.integers[n.intId] + t.itemId.item = pool.integers[n.intId].int32 inc n - loadTypeStub c, n, typ.typeInst - loadNode c, n, typ.n - loadSymStub c, n, typ.owner - loadSymStub c, n, typ.sym - loadLoc c, n, typ.loc + t.typeInst = loadTypeStub(c, n) + t.n = loadNode(c, n) + t.setOwner loadSymStub(c, n) + t.sym = loadSymStub(c, n) + loadLoc c, n, t.loc + var kids: seq[PType] = @[] while n.kind != ParRi: - t.typ.kids.add loadTypeStub(c, n) + kids.add loadTypeStub(c, n) + + t.setSons kids skipParRi n +proc loadSym*(c: var DecodeContext; s: PSym) = + if s.kind != skStub: return + var n = default(Cursor) # getCursorAt() + + expect n, ParLe + if n.tagId != sdefTag: + raiseAssert "(sd) expected" + inc n + expect n, SymbolDef + # ignore the symbol's name, we have already used it to create this PSym instance! + inc n + + + template withNode(c: var DecodeContext; n: var Cursor; result: PNode; kind: TNodeKind; body: untyped) = let info = c.infos.oldLineInfo(n.info) let flags = fromNifNodeFlags n @@ -675,7 +720,7 @@ template withNode(c: var DecodeContext; n: var Cursor; result: PNode; kind: TNod body skipParRi n -proc fromNif(c: var DecodeContext; n: var Cursor): PNode = +proc loadNode(c: var DecodeContext; n: var Cursor): PNode = result = nil case n.kind: of DotToken: @@ -752,7 +797,7 @@ proc fromNif(c: var DecodeContext; n: var Cursor): PNode = else: c.withNode n, result, kind: while n.kind != ParRi: - result.addAllowNil c.fromNif n + result.addAllowNil c.loadNode n else: raiseAssert "Not yet implemented " & $n.kind @@ -768,6 +813,6 @@ proc loadNifModule*(config: ConfigRef; f: FileIndex): PNode = when isMainModule: import std / syncio let obj = parseSymName("a.123.sys") - echo obj.name, " ", obj.module + echo obj.name, " ", obj.module, " ", obj.count let objb = parseSymName("abcdef.0121") - echo objb.name, " ", objb.module + echo objb.name, " ", objb.module, " ", objb.count diff --git a/compiler/icnif/enum2nif.nim b/compiler/icnif/enum2nif.nim index 20e62178551a..d11dd04b6af8 100644 --- a/compiler/icnif/enum2nif.nim +++ b/compiler/icnif/enum2nif.nim @@ -1,4 +1,4 @@ -# Generated by gear2/generator/enumgen.nim in Nimony repo. DO NOT EDIT! +# Generated by tools/enumgen.nim. DO NOT EDIT! import ".." / [ast, options] @@ -172,7 +172,7 @@ proc toNifTag*(s: TNodeKind): string = of nkOpenSym: "opensym" -proc parseNodeKind*(s: string): TNodeKind = +proc parse*(t: typedesc[TNodeKind]; s: string): TNodeKind = case s of "none": nkNone of "empty": nkEmpty @@ -372,7 +372,7 @@ proc toNifTag*(s: TSymKind): string = of skPackage: "package" -proc parseSymKind*(s: string): TSymKind = +proc parse*(t: typedesc[TSymKind]; s: string): TSymKind = case s of "unknown": skUnknown of "conditional": skConditional @@ -472,7 +472,7 @@ proc toNifTag*(s: TTypeKind): string = of tyStub: "stub" -proc parseTypeKind*(s: string): TTypeKind = +proc parse*(t: typedesc[TTypeKind]; s: string): TTypeKind = case s of "none": tyNone of "bool": tyBool @@ -558,7 +558,7 @@ proc toNifTag*(s: TLocKind): string = of locOther: "other" -proc parseLocKind*(s: string): TLocKind = +proc parse*(t: typedesc[TLocKind]; s: string): TLocKind = case s of "none": locNone of "temp": locTemp @@ -590,7 +590,7 @@ proc toNifTag*(s: TCallingConvention): string = of ccMember: "member" -proc parseCallingConvention*(s: string): TCallingConvention = +proc parse*(t: typedesc[TCallingConvention]; s: string): TCallingConvention = case s of "nimcall": ccNimCall of "stdcall": ccStdCall @@ -889,7 +889,7 @@ proc toNifTag*(s: TMagic): string = of mZeroDefault: "zerodefault" -proc parseMagic*(s: string): TMagic = +proc parse*(t: typedesc[TMagic]; s: string): TMagic = case s of "nonem": mNone of "defined": mDefined @@ -1180,7 +1180,7 @@ proc toNifTag*(s: TStorageLoc): string = of OnHeap: "heap" -proc parseStorageLoc*(s: string): TStorageLoc = +proc parse*(t: typedesc[TStorageLoc]; s: string): TStorageLoc = case s of "unknown": OnUnknown of "static": OnStatic diff --git a/tools/enumgen.nim b/tools/enumgen.nim new file mode 100644 index 000000000000..37493a99cca6 --- /dev/null +++ b/tools/enumgen.nim @@ -0,0 +1,247 @@ +## Generate effective NIF representation for `Enum` + +import ".." / compiler / [ast, options] + +import std / [syncio, assertions, strutils, tables] + +# We need to duplicate this type here as ast.nim's version of it does not work +# as it sets the string values explicitly breaking our logic... +type + TCallingConventionMirror = enum + ccNimCall + ccStdCall + ccCDecl + ccSafeCall + ccSysCall + ccInline + ccNoInline + ccFastCall + ccThisCall + ccClosure + ccNoConvention + ccMember + +const + SpecialCases = [ + ("nkCommand", "cmd"), + ("nkIfStmt", "if"), + ("nkError", "err"), + ("nkType", "onlytype"), + ("nkTypeSection", "type"), + ("tySequence", "seq"), + ("tyVar", "mut"), + ("tyProc", "proctype"), + ("tyUncheckedArray", "uarray"), + ("nkExprEqExpr", "vv"), + ("nkExprColonExpr", "kv"), + ("nkDerefExpr", "deref"), + ("nkReturnStmt", "ret"), + ("nkBreakStmt", "brk"), + ("nkStmtListExpr", "expr"), + ("nkEnumFieldDef", "efld"), + ("nkNilLit", "nil"), + ("ccNoConvention", "noconv"), + ("mExpr", "exprm"), + ("mStmt", "stmtm"), + ("mEqNimrodNode", "eqnimnode"), + ("mPNimrodNode", "nimnode"), + ("mNone", "nonem"), + ("mAsgn", "asgnm"), + ("mOf", "ofm"), + ("mAddr", "addrm"), + ("mType", "typem"), + ("mStatic", "staticm"), + ("mRange", "rangem"), + ("mVar", "varm"), + ("mInSet", "contains"), + ("mNil", "nilm"), + ("tyBuiltInTypeClass", "bconcept"), + ("tyUserTypeClass", "uconcept"), + ("tyUserTypeClassInst", "uconceptinst"), + ("tyCompositeTypeClass", "cconcept"), + ("tyGenericInvocation", "ginvoke"), + ("tyGenericBody", "gbody"), + ("tyGenericInst", "ginst"), + ("tyGenericParam", "gparam"), + ("nkStmtList", "stmts"), + ("nkDotExpr", "dot"), + ("nkBracketExpr", "at") + ] + SuffixesToReplace = [ + ("Section", ""), ("Branch", ""), ("Stmt", ""), ("I", ""), + ("Expr", "x"), ("Def", "") + ] + PrefixesToReplace = [ + ("Length", "len"), + ("SetLength", "setlen"), + ("Append", "add") + ] + AdditionalNodes = [ + "nf", # "node flag" + "tf", # "type flag" + "sf", # "sym flag" + "htype", # annotated with a hidden type + "missing" + ] + +proc genEnum[E](f: var File; enumName: string; known: var OrderedTable[string, bool]; prefixLen = 2) = + var mappingA = initOrderedTable[string, E]() + var cases = "" + for e in low(E)..high(E): + var es = $e + if es.startsWith("nkHidden"): + es = es.replace("nkHidden", "nkh") # prefix will be removed + else: + for (suffix, repl) in items SuffixesToReplace: + if es.len - prefixLen > suffix.len and es.endsWith(suffix): + es.setLen es.len - len(suffix) + es.add repl + break + for (suffix, repl) in items PrefixesToReplace: + if es.len - prefixLen > suffix.len and es.substr(prefixLen).startsWith(suffix): + es = es.substr(0, prefixLen-1) & repl & es.substr(prefixLen+suffix.len) + break + + let s = es.substr(prefixLen) + var done = false + for enu, key in items SpecialCases: + if $e == enu: + assert(not mappingA.hasKey(key)) + if known.hasKey(key): echo "conflict: ", key + known[key] = true + assert key.len > 0 + mappingA[key] = e + cases.add " of " & $e & ": " & escape(key) & "\n" + done = true + break + if not done: + let key = s.toLowerAscii + if not mappingA.hasKey(key): + assert key.len > 0, $e + if known.hasKey(key): echo "conflict: ", key + known[key] = true + mappingA[key] = e + cases.add " of " & $e & ": " & escape(key) & "\n" + done = true + if not done: + var d = 0 + while d < 10: + let key = s.toLowerAscii & $d + if not mappingA.hasKey(key): + assert key.len > 0 + mappingA[key] = e + cases.add " of " & $e & ": " & escape(key) & "\n" + done = true + break + inc d + if not done: + echo "Could not map: " & s + #echo mapping + var code = "" + code.add "proc toNifTag*(s: " & enumName & "): string =\n" + code.add " case s\n" + code.add cases + code.add "\n\n" + let procname = "parse" # & enumName.substr(1) + code.add "proc " & procname & "*(t: typedesc[" & enumName & "]; s: string): " & enumName & " =\n" + code.add " case s\n" + for (k, v) in pairs mappingA: + code.add " of " & escape(k) & ": " & $v & "\n" + code.add " else: " & $low(E) & "\n\n\n" + f.write code + +proc genEnum[E](f: var File; enumName: string; prefixLen = 2) = + var known = initOrderedTable[string, bool]() + genEnum[E](f, enumName, known, prefixLen) + + +proc genFlags[E](f: var File; enumName: string; prefixLen = 2) = + var mappingA = initOrderedTable[string, E]() + var mappingB = initOrderedTable[string, E]() + var cases = "" + for e in low(E)..high(E): + let s = ($e).substr(prefixLen) + var done = false + for c in s: + if c in {'A'..'Z'}: + let key = $c.toLowerAscii + if not mappingA.hasKey(key): + mappingA[key] = e + cases.add " of " & $e & ": dest.add " & escape(key) & "\n" + done = true + break + if not done: + var d = 0 + while d < 10: + let key = $s[0].toLowerAscii & $d + if not mappingB.hasKey(key): + mappingB[key] = e + cases.add " of " & $e & ": dest.add " & escape(key) & "\n" + done = true + break + inc d + if not done: + quit "Could not map: " & s + #echo mapping + var code = "" + code.add "proc genFlags*(s: set[" & enumName & "]; dest: var string) =\n" + code.add " for e in s:\n" + code.add " case e\n" + code.add cases + code.add "\n\n" + let procname = "parse" & enumName.substr(1) & "s" + code.add "proc " & procname & "*(s: string): set[" & enumName & "] =\n" + code.add " result = {}\n" + code.add " var i = 0\n" + code.add " while i < s.len:\n" + code.add " case s[i]\n" + for c in 'a'..'z': + var letterFound = false + var digitsFound = 0 + for d in '0'..'9': + if mappingB.hasKey($c & $d): + if not letterFound: + letterFound = true + code.add " of '" & c & "':\n" + if digitsFound == 0: + code.add " if" + else: + code.add " elif" + inc digitsFound + code.add " i+1 < s.len and s[i+1] == '" & d & "':\n" + code.add " result.incl " & $mappingB[$c & $d] & "\n" + code.add " inc i\n" + + if mappingA.hasKey($c): + if digitsFound == 0: + code.add " of '" & c & "': " + else: + code.add " else: " + code.add "result.incl " & $mappingA[$c] & "\n" + + code.add " else: discard\n" + code.add " inc i\n\n" + f.write code + +var f = open("compiler/icnif/enum2nif.nim", fmWrite) +f.write "# Generated by tools/enumgen.nim. DO NOT EDIT!\n\n" +f.write "import \"..\" / [ast, options]\n\n" +# use the same mapping for TNodeKind and TMagic so that we can detect conflicts! +var nodeTags = initOrderedTable[string, bool]() +for a in AdditionalNodes: + nodeTags[a] = true + +genEnum[TNodeKind](f, "TNodeKind", nodeTags) +genEnum[TSymKind](f, "TSymKind") +genEnum[TTypeKind](f, "TTypeKind") +genEnum[TLocKind](f, "TLocKind", 3) +genEnum[TCallingConventionMirror](f, "TCallingConvention", 2) +genEnum[TMagic](f, "TMagic", nodeTags, 1) +genEnum[TStorageLoc](f, "TStorageLoc") +genFlags[TSymFlag](f, "TSymFlag") +genFlags[TNodeFlag](f, "TNodeFlag") +genFlags[TTypeFlag](f, "TTypeFlag") +genFlags[TLocFlag](f, "TLocFlag") +genFlags[TOption](f, "TOption", 3) + +f.close() From 7cb0676bf0bb2fccc737f51498308cbf1486a845 Mon Sep 17 00:00:00 2001 From: araq Date: Sun, 9 Nov 2025 11:40:26 +0100 Subject: [PATCH 34/38] progress --- compiler/ast2nif.nim | 149 +++++++++++++++++++++++------------- compiler/icnif/enum2nif.nim | 23 ++++-- tools/enumgen.nim | 4 +- 3 files changed, 114 insertions(+), 62 deletions(-) diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index 8e16f5da087d..12acc7d16eac 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -245,7 +245,7 @@ proc writeLib(w: var Writer; dest: var TokenBuf; lib: PLib) = if lib == nil: dest.addDotToken() else: - dest.buildTree $lib.kind: + dest.buildTree toNifTag(lib.kind): dest.writeBool lib.generated dest.writeBool lib.isOverridden dest.addStrLit lib.name @@ -261,7 +261,7 @@ proc writeSymDef(w: var Writer; dest: var TokenBuf; sym: PSym) = writeFlags(dest, sym.flags) writeFlags(dest, sym.options) dest.addIntLit sym.offset - dest.addIntLit sym.disamb + # field `disamb` made part of the name, so do not store it here dest.buildTree sym.kind.toNifTag: case sym.kind of skLet, skVar, skField, skForVar: @@ -513,7 +513,7 @@ proc incExpectTag(n: var Cursor; tagId: TagId) = inc n expectTag(n, tagId) -proc parseBool(n: var Cursor): bool = +proc loadBool(n: var Cursor): bool = if n.kind == ParLe: result = pool.tags[n.tagId] == "true" inc n @@ -556,16 +556,6 @@ proc getOffset(c: var DecodeContext; module: int32; nifName: string): NifIndexEn if result.offset == 0: raiseAssert "symbol has no offset: " & nifName -proc fromNifNodeFlags(n: var Cursor): set[TNodeFlag] = - if n.kind == DotToken: - result = {} - inc n - elif n.kind == Ident: - result = parseNodeFlags(pool.strings[n.litId]) - inc n - else: - raiseAssert "expected Node flag (`ident`) but got " & $n.kind - proc loadNode(c: var DecodeContext; n: var Cursor): PNode proc loadTypeStub(c: var DecodeContext; t: SymId): PType = @@ -632,20 +622,43 @@ proc loadSymStub(c: var DecodeContext; n: var Cursor): PSym = proc isStub*(t: PType): bool {.inline.} = t.kind == tyStub proc isStub*(s: PSym): bool {.inline.} = s.kind == skStub -proc loadLoc(c: var DecodeContext; n: var Cursor; loc: var TLoc) = - expect n, Ident - loc.k = parse(TLocKind, pool.strings[n.litId]) - inc n - expect n, Ident - loc.storage = parse(TStorageLoc, pool.strings[n.litId]) - inc n - expect n, Ident - loc.flags = pool.strings[n.litId].parseLocFlags() - inc n +proc loadAtom[T](t: typedesc[set[T]]; n: var Cursor): set[T] = + if n.kind == DotToken: + result = {} + inc n + else: + expect n, Ident + result = parse(T, pool.strings[n.litId]) + inc n + +proc loadAtom[T: enum](t: typedesc[T]; n: var Cursor): T = + if n.kind == DotToken: + result = default(T) + inc n + else: + expect n, Ident + result = parse(T, pool.strings[n.litId]) + inc n + +proc loadAtom(t: typedesc[string]; n: var Cursor): string = expect n, StringLit - loc.snippet = pool.strings[n.litId] + result = pool.strings[n.litId] + inc n + +proc loadAtom[T: int16|int32|int64](t: typedesc[T]; n: var Cursor): T = + expect n, IntLit + result = pool.integers[n.intId].T inc n +template loadField(field) = + field = loadAtom(typeof(field), n) + +proc loadLoc(c: var DecodeContext; n: var Cursor; loc: var TLoc) = + loadField loc.k + loadField loc.storage + loadField loc.flags + loadField loc.snippet + proc loadType*(c: var DecodeContext; t: PType) = if t.kind != tyStub: return var n = default(Cursor) # getCursorAt() @@ -657,30 +670,13 @@ proc loadType*(c: var DecodeContext; t: PType) = expect n, SymbolDef # ignore the type's name, we have already used it to create this PType's itemId! inc n - expect n, Ident - t.kind = parse(TTypeKind, pool.strings[n.litId]) - inc n - expect n, Ident - t.flags = parseTypeFlags(pool.strings[n.litId]) - inc n - expect n, Ident - t.callConv = parse(TCallingConvention, pool.strings[n.litId]) - inc n - expect n, IntLit - t.size = pool.integers[n.intId] - inc n - - expect n, IntLit - t.align = pool.integers[n.intId].int16 - inc n - - expect n, IntLit - t.paddingAtEnd = pool.integers[n.intId].int16 - inc n - - expect n, IntLit - t.itemId.item = pool.integers[n.intId].int32 - inc n + loadField t.kind + loadField t.flags + loadField t.callConv + loadField t.size + loadField t.align + loadField t.paddingAtEnd + loadField t.itemId.item t.typeInst = loadTypeStub(c, n) t.n = loadNode(c, n) @@ -696,6 +692,22 @@ proc loadType*(c: var DecodeContext; t: PType) = skipParRi n +proc loadAnnex(c: var DecodeContext; n: var Cursor): PLib = + if n.kind == DotToken: + result = nil + inc n + elif n.kind == ParLe: + result = PLib(kind: parse(TLibKind, pool.tags[n.tagId])) + inc n + result.generated = loadBool(n) + result.isOverridden = loadBool(n) + expect n, StringLit + result.name = pool.strings[n.litId] + inc n + result.path = loadNode(c, n) + skipParRi n + else: + raiseAssert "`lib/annex` information expected" proc loadSym*(c: var DecodeContext; s: PSym) = if s.kind != skStub: return @@ -708,12 +720,42 @@ proc loadSym*(c: var DecodeContext; s: PSym) = expect n, SymbolDef # ignore the symbol's name, we have already used it to create this PSym instance! inc n - + loadField s.magic + loadField s.flags + loadField s.options + loadField s.offset + + expect n, ParLe + s.kind = parse(TSymKind, pool.tags[n.tagId]) + inc n + + case s.kind + of skLet, skVar, skField, skForVar: + s.guard = loadSymStub(c, n) + loadField s.bitsize + loadField s.alignment + else: + discard + skipParRi n + + if s.kind == skModule: + expect n, DotToken + inc n + else: + loadField s.position + s.typ = loadTypeStub(c, n) + s.setOwner loadSymStub(c, n) + # We do not store `sym.ast` here but instead set it in the deserializer + #writeNode(w, sym.ast) + loadLoc c, n, s.loc + s.constraint = loadNode(c, n) + s.instantiatedFrom = loadSymStub(c, n) + skipParRi n template withNode(c: var DecodeContext; n: var Cursor; result: PNode; kind: TNodeKind; body: untyped) = let info = c.infos.oldLineInfo(n.info) - let flags = fromNifNodeFlags n + let flags = loadAtom(TNodeFlags, n) result = newNodeI(kind, info) result.flags = flags result.typ = c.loadTypeStub n @@ -731,14 +773,11 @@ proc loadNode(c: var DecodeContext; n: var Cursor): PNode = case kind: of nkEmpty: result = newNodeI(nkEmpty, c.infos.oldLineInfo(n.info)) - incExpect n, {Ident, DotToken} - let flags = fromNifNodeFlags n - result.flags = flags + result.flags = loadAtom(TNodeFlags, n) skipParRi n of nkIdent: let info = c.infos.oldLineInfo(n.info) - incExpect n, {DotToken, Ident} - let flags = fromNifNodeFlags n + let flags = loadAtom(TNodeFlags, n) let typ = c.loadTypeStub n expect n, Ident result = newIdentNode(c.cache.getIdent(pool.strings[n.litId]), info) diff --git a/compiler/icnif/enum2nif.nim b/compiler/icnif/enum2nif.nim index d11dd04b6af8..1f77393c6aa4 100644 --- a/compiler/icnif/enum2nif.nim +++ b/compiler/icnif/enum2nif.nim @@ -1189,6 +1189,19 @@ proc parse*(t: typedesc[TStorageLoc]; s: string): TStorageLoc = else: OnUnknown +proc toNifTag*(s: TLibKind): string = + case s + of libHeader: "bheader" + of libDynamic: "bdynamic" + + +proc parse*(t: typedesc[TLibKind]; s: string): TLibKind = + case s + of "bheader": libHeader + of "bdynamic": libDynamic + else: libHeader + + proc genFlags*(s: set[TSymFlag]; dest: var string) = for e in s: case e @@ -1257,7 +1270,7 @@ proc genFlags*(s: set[TSymFlag]; dest: var string) = of sfTemplateRedefinition: dest.add "t1" -proc parseSymFlags*(s: string): set[TSymFlag] = +proc parse*(t: typedesc[TSymFlag]; s: string): set[TSymFlag] = result = {} var i = 0 while i < s.len: @@ -1463,7 +1476,7 @@ proc genFlags*(s: set[TNodeFlag]; dest: var string) = of nfDisabledOpenSym: dest.add "d3" -proc parseNodeFlags*(s: string): set[TNodeFlag] = +proc parse*(t: typedesc[TNodeFlag]; s: string): set[TNodeFlag] = result = {} var i = 0 while i < s.len: @@ -1575,7 +1588,7 @@ proc genFlags*(s: set[TTypeFlag]; dest: var string) = of tfImplicitStatic: dest.add "i6" -proc parseTypeFlags*(s: string): set[TTypeFlag] = +proc parse*(t: typedesc[TTypeFlag]; s: string): set[TTypeFlag] = result = {} var i = 0 while i < s.len: @@ -1718,7 +1731,7 @@ proc genFlags*(s: set[TLocFlag]; dest: var string) = of lfPrepareForMutation: dest.add "p" -proc parseLocFlags*(s: string): set[TLocFlag] = +proc parse*(t: typedesc[TLocFlag]; s: string): set[TLocFlag] = result = {} var i = 0 while i < s.len: @@ -1773,7 +1786,7 @@ proc genFlags*(s: set[TOption]; dest: var string) = of optQuirky: dest.add "q" -proc parseOptions*(s: string): set[TOption] = +proc parse*(t: typedesc[TOption]; s: string): set[TOption] = result = {} var i = 0 while i < s.len: diff --git a/tools/enumgen.nim b/tools/enumgen.nim index 37493a99cca6..fdcd132f92da 100644 --- a/tools/enumgen.nim +++ b/tools/enumgen.nim @@ -189,8 +189,7 @@ proc genFlags[E](f: var File; enumName: string; prefixLen = 2) = code.add " case e\n" code.add cases code.add "\n\n" - let procname = "parse" & enumName.substr(1) & "s" - code.add "proc " & procname & "*(s: string): set[" & enumName & "] =\n" + code.add "proc parse*(t: typedesc[" & enumName & "]; s: string): set[" & enumName & "] =\n" code.add " result = {}\n" code.add " var i = 0\n" code.add " while i < s.len:\n" @@ -238,6 +237,7 @@ genEnum[TLocKind](f, "TLocKind", 3) genEnum[TCallingConventionMirror](f, "TCallingConvention", 2) genEnum[TMagic](f, "TMagic", nodeTags, 1) genEnum[TStorageLoc](f, "TStorageLoc") +genEnum[TLibKind](f, "TLibKind") genFlags[TSymFlag](f, "TSymFlag") genFlags[TNodeFlag](f, "TNodeFlag") genFlags[TTypeFlag](f, "TTypeFlag") From a9bc2133fd5a01b818c6a15cbbe68f087cad4ad0 Mon Sep 17 00:00:00 2001 From: araq Date: Sun, 9 Nov 2025 15:35:32 +0100 Subject: [PATCH 35/38] progress --- compiler/ast2nif.nim | 59 +++++++++++++++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 15 deletions(-) diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index 12acc7d16eac..9bbc80ec345a 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -15,7 +15,7 @@ import ast, idents, msgs, options import lineinfos as astli import pathutils import "../dist/nimony/src/lib" / [bitabs, nifstreams, nifcursors, lineinfos, - nifindexes] + nifindexes, nifreader] import "../dist/nimony/src/gear2" / modnames import icnif / [enum2nif] @@ -522,19 +522,37 @@ proc loadBool(n: var Cursor): bool = raiseAssert "(true)/(false) expected" type + NifModule = object + stream: nifstreams.Stream + symCounter: int32 + index: NifIndex + DecodeContext* = object infos: LineInfoWriter moduleIds: Table[string, int32] types: Table[ItemId, (PType, NifIndexEntry)] syms: Table[ItemId, (PSym, NifIndexEntry)] - indexes: seq[NifIndex] - symCounters: Table[int32, int32] # Module ID -> counter + mods: seq[NifModule] cache: IdentCache + moduleToNifSuffix: Table[FileIndex, string] proc createDecodeContext*(config: ConfigRef; cache: IdentCache): DecodeContext = ## Supposed to be a global variable result = DecodeContext(infos: LineInfoWriter(config: config), cache: cache) +proc idToIdx(x: int32): int {.inline.} = + assert x <= -2'i32 + result = -(x+2) + +proc cursorFromIndexEntry(c: var DecodeContext; module: int32; entry: NifIndexEntry; + buf: var TokenBuf): Cursor = + let m = idToIdx(module) + let s = addr c.mods[m].stream + s.r.jumpTo entry.offset + var buf = createTokenBuf(30) + nifcursors.parse(s[], buf, entry.info) + result = cursorAt(buf, 0) + proc moduleId(c: var DecodeContext; suffix: string): int32 = # We don't know the "real" FileIndex due to our mapping to a short "Module suffix" # This is not a problem, we use negative `ItemId.module` values here and then @@ -543,13 +561,16 @@ proc moduleId(c: var DecodeContext; suffix: string): int32 = result = c.moduleIds.getOrDefault(suffix) if result == 0: result = -int32(c.moduleIds.len + 2) # negative index! + let modFile = (getNimcacheDir(c.infos.config) / RelativeFile(suffix & ".nif")).string + let idxFile = (getNimcacheDir(c.infos.config) / RelativeFile(suffix & ".idx.nif")).string c.moduleIds[suffix] = result - c.indexes.add readIndex((getNimcacheDir(c.infos.config) / RelativeFile(suffix & ".idx.nif")).string) + c.mods.add NifModule(stream: nifstreams.open(modFile), index: readIndex(idxFile)) + assert c.mods.len-1 == idToIdx(result) proc getOffset(c: var DecodeContext; module: int32; nifName: string): NifIndexEntry = assert module < 0'i32 - let index = (-module) - 2'i32 - let ii = addr c.indexes[index] + let index = idToIdx(module) + let ii = addr c.mods[index].index result = ii.public.getOrDefault(nifName) if result.offset == 0: result = ii.private.getOrDefault(nifName) @@ -594,7 +615,7 @@ proc loadSymStub(c: var DecodeContext; t: SymId): PSym = let symAsStr = pool.syms[t] let sn = parseSymName(symAsStr) let module = moduleId(c, sn.module) - let val = addr c.symCounters.mgetOrPut(module, 0) + let val = addr c.mods[idToIdx(module)].symCounter inc val[] let id = ItemId(module: module, item: val[]) @@ -661,7 +682,8 @@ proc loadLoc(c: var DecodeContext; n: var Cursor; loc: var TLoc) = proc loadType*(c: var DecodeContext; t: PType) = if t.kind != tyStub: return - var n = default(Cursor) # getCursorAt() + var buf = createTokenBuf(30) + var n = cursorFromIndexEntry(c, t.itemId.module, c.types[t.itemId][1], buf) expect n, ParLe if n.tagId != tdefTag: @@ -711,7 +733,8 @@ proc loadAnnex(c: var DecodeContext; n: var Cursor): PLib = proc loadSym*(c: var DecodeContext; s: PSym) = if s.kind != skStub: return - var n = default(Cursor) # getCursorAt() + var buf = createTokenBuf(30) + var n = cursorFromIndexEntry(c, s.itemId.module, c.syms[s.itemId][1], buf) expect n, ParLe if n.tagId != sdefTag: @@ -841,13 +864,19 @@ proc loadNode(c: var DecodeContext; n: var Cursor): PNode = raiseAssert "Not yet implemented " & $n.kind -proc loadNifModule*(config: ConfigRef; f: FileIndex): PNode = - var moduleToNifSuffix = initTable[FileIndex, string]() +proc loadNifModule*(c: var DecodeContext; f: FileIndex): PNode = + let moduleSuffix = modname(c.moduleToNifSuffix, f.int, c.infos.config) + let modFile = toGeneratedFile(c.infos.config, AbsoluteFile(moduleSuffix), ".nif").string - let m = modname(moduleToNifSuffix, f.int, config) - let d = toGeneratedFile(config, AbsoluteFile(m), ".nif").string - - result = nil + var buf = createTokenBuf(300) + var s = nifstreams.open(modFile) + # XXX We can optimize this here and only load the top level entries! + try: + nifcursors.parse(s, buf, NoLineInfo) + finally: + nifstreams.close(s) + var n = cursorAt(buf, 0) + result = loadNode(c, n) when isMainModule: import std / syncio From eb53434429f2eb609ce726d900ae64c26f642c0c Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 9 Nov 2025 17:28:18 +0100 Subject: [PATCH 36/38] make CI happy --- compiler/pipelines.nim | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/compiler/pipelines.nim b/compiler/pipelines.nim index e4becc92f017..2116428b2803 100644 --- a/compiler/pipelines.nim +++ b/compiler/pipelines.nim @@ -1,7 +1,10 @@ import sem, cgen, modulegraphs, ast, llstream, parser, msgs, lineinfos, reorder, options, semdata, cgendata, modules, pathutils, packages, syntaxes, depends, vm, pragmas, idents, lookups, wordrecg, - liftdestructors, nifgen, ast2nif + liftdestructors, nifgen + +when not defined(nimKochBootstrap): + import ast2nif import pipelineutils @@ -218,8 +221,9 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator of NonePass: raiseAssert "use setPipeLinePass to set a proper PipelinePass" - if optCompress in graph.config.globalOptions: - writeNifModule(graph.config, module.position.int32, finalNode) + when not defined(nimKochBootstrap): + if optCompress in graph.config.globalOptions: + writeNifModule(graph.config, module.position.int32, finalNode) if graph.config.backend notin {backendC, backendCpp, backendObjc}: # We only write rod files here if no C-like backend is active. From 908c4e41f4ac12aef130e59e88595164e03cde9a Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 9 Nov 2025 19:21:09 +0100 Subject: [PATCH 37/38] progress --- compiler/ast2nif.nim | 9 +- tests/icnif/tencode_node2node.nim | 398 ------------------------------ 2 files changed, 4 insertions(+), 403 deletions(-) delete mode 100644 tests/icnif/tencode_node2node.nim diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index 9bbc80ec345a..a57bbd65af7d 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -555,7 +555,7 @@ proc cursorFromIndexEntry(c: var DecodeContext; module: int32; entry: NifIndexEn proc moduleId(c: var DecodeContext; suffix: string): int32 = # We don't know the "real" FileIndex due to our mapping to a short "Module suffix" - # This is not a problem, we use negative `ItemId.module` values here and then + # This is not a problem, we use negative `ItemId.module` values here and then # there is no interference with in-memory-modules. Modulegraphs.nim already uses -1 # so we start at -2 here. result = c.moduleIds.getOrDefault(suffix) @@ -573,7 +573,7 @@ proc getOffset(c: var DecodeContext; module: int32; nifName: string): NifIndexEn let ii = addr c.mods[index].index result = ii.public.getOrDefault(nifName) if result.offset == 0: - result = ii.private.getOrDefault(nifName) + result = ii.private.getOrDefault(nifName) if result.offset == 0: raiseAssert "symbol has no offset: " & nifName @@ -702,7 +702,7 @@ proc loadType*(c: var DecodeContext; t: PType) = t.typeInst = loadTypeStub(c, n) t.n = loadNode(c, n) - t.setOwner loadSymStub(c, n) + t.setOwner loadSymStub(c, n) t.sym = loadSymStub(c, n) loadLoc c, n, t.loc @@ -810,8 +810,7 @@ proc loadNode(c: var DecodeContext; n: var Cursor): PNode = skipParRi n of nkSym: c.withNode n, result, kind: - #result.sym = c.fromNifSymbol n - discard + result.sym = c.loadSymStub n of nkCharLit: c.withNode n, result, kind: expect n, CharLit diff --git a/tests/icnif/tencode_node2node.nim b/tests/icnif/tencode_node2node.nim deleted file mode 100644 index b1793ec6c27e..000000000000 --- a/tests/icnif/tencode_node2node.nim +++ /dev/null @@ -1,398 +0,0 @@ -import std/[assertions, math, tables] -import "../../compiler/icnif" / [nifencoder, nifdecoder] -import "../../compiler" / [idents, ast, astalgo, options, pathutils, modulegraphs, modules, msgs, pipelines, syntaxes, sem, llstream, lineinfos] - -# This test generates PNode by semchecks test code. -# Then it is used to test icnif/nifencoder and nifdecoder. - -const TestCodeDir = currentSourcePath().AbsoluteFile.splitFile().dir / RelativeDir"testcode" - -proc newConfigRefForTest(): ConfigRef = - var conf = newConfigRef() - conf.setDefaultLibpath() - conf.searchPaths.add(conf.libpath) - excl(conf.notes, hintProcessing) - excl(conf.mainPackageNotes, hintProcessing) - result = conf - -proc newModuleGraphForSem(cache: IdentCache; conf: ConfigRef): ModuleGraph = - var graph = newModuleGraph(cache, conf) - graph.setPipeLinePass(SemPass) - # Make PNode from sem pass assigned to graph.systemModule.ast - let oldCmd = graph.config.cmd - graph.config.cmd = cmdIdeTools - graph.compilePipelineSystemModule() - graph.config.cmd = oldCmd - result = graph - -proc getSystemNif(graph: ModuleGraph): seq[string] = - result = newSeqOfCap[string](graph.ifaces.len) - for i, iface in graph.ifaces.mpairs: - if iface.module != nil: - let n = iface.module.ast - assert n != nil - # if nil is not assigned, it generates large NIF - iface.module.ast = nil - result.add saveNifToBuffer(n, graph.config, iface.module) - #writeFile(iface.module.name.s & ".nif", result[^1]) - -proc sem(graph: ModuleGraph; path: AbsoluteFile): (PNode, PSym) = - result = (nil, nil) - - let fileIdx = fileInfoIdx(graph.config, path) - var module = newModule(graph, fileIdx) - registerModule(graph, module) - - var idgen = idGeneratorFromModule(module) - let ctx = preparePContext(graph, module, idgen) - - var stream = llStreamOpen(path, fmRead) - if stream == nil: - rawMessage(graph.config, errCannotOpenFile, path.string) - return (nil, nil) - - var p: Parser = default(Parser) - syntaxes.openParser(p, fileIdx, stream, graph.cache, graph.config) - - checkFirstLineIndentation(p) - block processCode: - if graph.stopCompile(): break processCode - var n = parseTopLevelStmt(p) - if n.kind == nkEmpty: break processCode - # read everything, no streaming possible - var sl = newNodeI(nkStmtList, n.info) - sl.add n - while true: - var n = parseTopLevelStmt(p) - if n.kind == nkEmpty: break - sl.add n - - result = (semWithPContext(ctx, sl), module) - -type - # Nim's AST has cycles that causes infinite recursive loop in eql procs. - # this is used to prevent that happen. - EqlContext = object - nodeStack: seq[PNode] - checkedSyms: Table[ItemId, PSym] # used to check if each PSym has unique ItemId - # and also prevents inifinite loop - checkedTypes: Table[ItemId, PType]# used like checkedSyms - confX, confY: ConfigRef # used to print the line info when there is a mismatch - # and get path from FileIndex - -# Compare PType, PSym and PNode but ignores fields nifencoder and nifdecoder doesn't support -# `x` is generated by sem.nim and `y` is decoded by icnif/nifdecoder. -proc eql(x, y: PNode; c: var EqlContext): bool -proc eql(x, y: PType; c: var EqlContext): bool - -proc eql(x, y: TLoc): bool = - if x.k != y.k: - echo "loc kind mismatch: ", x.k, "/", y.k - result = false - elif x.snippet != y.snippet: - echo "loc snippet mismatch: ", x.snippet, "/", y.snippet - result = false - else: - result = true - -proc eqlFileIndex(x, y: int; c: EqlContext): bool = - let xpath = c.confX.toFullPath(x.FileIndex) - let ypath = c.confY.toFullPath(y.FileIndex) - if xpath != ypath: - echo "file index mismatch: ", xpath, "/", ypath - result = false - else: - result = true - -proc eql(x, y: TLineInfo; c: EqlContext): bool = - # If parent PNode has a valid line info but it's child doesn't have one, - # cannot translate such a tree to NIF. - # Because in NIF, if a child node doesn't have line info, - # nifstream assign the parent's line info to it. - # So cannot have child node without line info if parent has a valid line info. - if x == unknownLineInfo: - result = true - elif x.line != y.line: - echo "line number mismatch: ", x.line, "/", y.line - result = false - elif x.col != y.col: - echo "column number mismatch: ", x.col, "/", y.col - result = false - elif not eqlFileIndex(x.fileIndex.int, y.fileIndex.int, c): - echo "file in line info mismatch" - result = false - else: - result = true - -proc eqlSymPos(x, y: PSym; c: EqlContext): bool = - if x.kind == skModule: - result = eqlFileIndex(x.position, y.position, c) - elif x.position != y.position: - echo "symbol position mismatch: ", x.position, "/", y.position - result = false - else: - result = true - -proc eqlItemId(x, y: ItemId; c: EqlContext): bool = - if x.item != y.item: - echo "itemId.item mismatch: ", x.item, "/", y.item - result = false - elif not eqlFileIndex(x.module, y.module, c): - result = false - else: - result = true - -proc eql(x, y: PSym; c: var EqlContext): bool = - if x == nil and y == nil: - result = true - elif x == nil or y == nil: - echo "symbol is missing" - result = false - elif not eqlItemId(x.itemId, y.itemId, c): - echo "symbol itemId mismatch" - result = false - elif c.checkedSyms.hasKeyOrPut(y.itemId, y): - if c.checkedSyms[y.itemId] == y: - result = true - else: - echo "detected duplicated symbol ItemId:" - debug(x) - debug(c.checkedSyms[y.itemId]) - debug(y) - result = false - elif x.name.s != y.name.s: - echo "symbol name mismatch: ", x.name.s, "/", y.name.s - result = false - elif x.kind != y.kind: - echo "symbol kind mismatch: ", x.kind, "/", y.kind - result = false - elif x.magic != y.magic: - echo "symbol magic mismatch: ", x.magic, "/", y.magic - result = false - elif x.kind != skPackage and not eql(x.info, y.info, c): - # fileIndex of info of skPackage is just a path of first semchecked module in the package - echo "symbol line info mismatch" - result = false - elif x.kind != skModule and x.flags != y.flags: - # TODO: check the flag of skModule - echo "symbol flag mismatch: ", x.flags, "/", y.flags - result = false - elif x.options != y.options: - echo "symbol options mismatch: ", x.options, "/", y.options - result = false - elif not eqlSymPos(x, y, c): - result = false - elif x.offset != y.offset: - echo "symbol offset mismatch: ", x.offset, "/", y.offset - result = false - elif x.disamb != y.disamb: - echo "symbol disamb mismatch: ", x.disamb, "/", y.disamb - result = false - elif not eql(x.loc, y.loc): - echo "symbol.loc mismatch" - result = false - else: - if not eql(x.typ, y.typ, c): - echo "symbol type mismatch:" - result = false - elif not eql(x.owner, y.owner, c): - echo "Symbol owner mismatch:" - debug(x.owner) - debug(y.owner) - result = false - elif not eql(x.ast, y.ast, c): - echo "symbol ast mismatch" - result = false - elif not eql(x.constraint, y.constraint, c): - echo "symbol constraint mismatch" - result = false - elif not eql(x.instantiatedFrom, y.instantiatedFrom, c): - echo "symbol instantiatedFrom mismatch" - result = false - else: - if x.kind in {skLet, skVar, skField, skForVar}: - if not eql(x.guard, y.guard, c): - echo "symbol guard mismatch" - result = false - elif x.bitsize != y.bitsize: - echo "symbol bitsize mismatch: ", x.bitsize, "/", y.bitsize - result = false - elif x.alignment != y.alignment: - echo "symbol alignment mismatch: ", x.alignment, "/", y.alignment - result = false - else: - result = true - else: - result = true - -proc eql(x, y: PType; c: var EqlContext): bool = - if x == nil and y == nil: - result = true - elif x == nil or y == nil: - echo "type is missing" - result = false - elif not eqlItemId(x.itemId, y.itemId, c): - echo "type itemId mismatch" - result = false - elif c.checkedTypes.hasKeyOrPut(y.itemId, y): - result = true - #[ - if c.checkedTypes[y.itemId] == y: - result = true - else: - echo "detected duplicated type ItemId:" - debug(x) - debug(c.checkedTypes[y.itemId]) - debug(y) - result = false - ]# - elif x.kind != y.kind: - echo "type kind mismatch: ", x.kind, "/", y.kind - result = false - elif x.flags != y.flags: - echo "type flag mismatch: ", x.flags, "/", y.flags - result = false - else: - if not eql(x.n, y.n, c): - echo "type.n mismatch" - debug(x.n) - debug(y.n) - result = false - elif not eql(x.owner, y.owner, c): - echo "type owner mismatch: " - debug(x.owner) - debug(y.owner) - result = false - elif not eql(x.sym, y.sym, c): - echo "type sym mismatch:" - debug(x.sym) - debug(y.sym) - result = false - elif x.kidsLen != y.kidsLen: - echo "type kidsLen mismatch" - result = false - else: - result = true - for i in 0 ..< x.kidsLen: - if not eql(x[i], y[i], c): - echo "type kids mismatch: " - debug(x[i]) - debug(y[i]) - result = false - break - -proc eql(x, y: PNode; c: var EqlContext): bool = - if x == nil and y == nil: - result = true - elif x == nil or y == nil: - result = false - elif x.kind != y.kind: - echo "node kind mismatch: ", x.kind, "/", y.kind - result = false - elif x.flags != y.flags: - echo "node flag mismatch: ", x.flags, "/", y.flags, " at ", `$`(c.confX, x.info) - debug(x) - debug(y) - result = false - elif not eql(x.info, y. info, c): - echo "node lineinfo mismatch at ", `$`(c.confX, x.info) - debug(x) - result = false - elif x.safeLen == y.safeLen: - if c.nodeStack.len != 0: - for i in countDown(c.nodeStack.len - 1, 0): - if x == c.nodeStack[i]: - # echo "cycle is detected in PNode" - return true - c.nodeStack.add x - if not eql(x.typ, y.typ, c): - echo "PNode type mismatch at ", `$`(c.confX, x.info), ":" - debug(x) - debug(y) - debug(x.typ) - debug(y.typ) - result = false - else: - case x.kind: - of nkIdent: - # these idents are generated from different IdentCache - result = x.ident.s == y.ident.s - if not result: - echo "PNode identifier mismatch: ", `$`(c.confX, x.info), x.ident.s, "/", y.ident.s - of nkSym: - result = eql(x.sym, y.sym, c) - if not result: - echo "Symbol mismatch:" - debug(x.sym) - if y.sym == nil: - echo "y.sym = nil" - else: - debug(y.sym) - debug(x.sym.typ) - debug(y.sym.typ) - of nkCharLit .. nkUInt64Lit, nkStrLit .. nkTripleStrLit: - result = sameValue(x, y) - of nkFloatLit .. nkFloat128Lit: - # want to know if x and y are identical float value. - # so x == y doesn't work if both x and y are NaN or x == 0 and y == -0. - let xc = classify(x.floatVal) - let yc = classify(y.floatVal) - if xc == yc: - if xc in {fcNormal, fcSubnormal}: - if x.floatVal != y.floatVal: - echo "float literal mismatch: ", x.floatVal, "/", y.floatVal - result = false - else: - result = true - else: - result = true - else: - echo "float literal mismatch: ", xc, "/", yc - result = false - else: - result = true - for i in 0 ..< x.safeLen: - if not eql(x[i], y[i], c): - result = false - break - discard c.nodeStack.pop - else: - echo "node length mismatch" - debug(x) - debug(y) - result = false - -proc testNifEncDec(graph: ModuleGraph; src: string; systemNif: openArray[string]) = - let fullPath = TestCodeDir / RelativeFile(src) - let (n, module) = sem(graph, fullPath) - assert n != nil, "failed to sem " & $fullPath - assert module.owner.kind == skPackage - - #debug(n) - let nif = saveNifToBuffer(n, graph.config, module) - #echo nif - #echo "NIF size of ", src, ": ", nif.len - #writeFile(src & ".nif", nif) - - # Don't reuse the ModuleGraph used for semcheck when load NIF. - var graphForLoad = newModuleGraph(newIdentCache(), newConfigRefForTest()) - var prog = NifProgram() - for sysNif in systemNif: - discard loadNifFromBuffer(sysNif, graphForLoad, prog) - let n2 = loadNifFromBuffer(nif, graphForLoad, prog) - #debug(n2) - var c = EqlContext(confX: graph.config, confY: graphForLoad.config) - assert eql(n, n2, c), "test failed: " & $fullPath - -var conf = newConfigRefForTest() -var cache = newIdentCache() -var graph = newModuleGraphForSem(cache, conf) -let systemNif = getSystemNif(graph) -testNifEncDec(graph, "modtest1.nim", systemNif) -testNifEncDec(graph, "modtestliterals.nim", systemNif) -testNifEncDec(graph, "modtesttypesections.nim", systemNif) -#testNifEncDec(graph, "modtestpragmas.nim", systemNif) -testNifEncDec(graph, "modtestprocs.nim", systemNif) -#testNifEncDec(graph, "modteststatements.nim", systemNif) -#testNifEncDec(graph, "modtestgenerics.nim", systemNif) -#testNifEncDec(graph, "modtestexprs.nim", systemNif) From 1ab4f1fe197181532ad615f941a9cb2215acc1e2 Mon Sep 17 00:00:00 2001 From: araq Date: Mon, 10 Nov 2025 13:53:26 +0100 Subject: [PATCH 38/38] make explicit --- compiler/ast.nim | 7 ++++++- compiler/ast2nif.nim | 27 +++++++++++++++++---------- compiler/ccgreset.nim | 2 +- compiler/expanddefaults.nim | 2 +- compiler/icnif/enum2nif.nim | 2 -- compiler/jsgen.nim | 2 +- compiler/liftdestructors.nim | 2 +- compiler/nifgen.nim | 3 +-- compiler/typeallowed.nim | 2 +- compiler/types.nim | 4 +--- compiler/vmdeps.nim | 2 +- 11 files changed, 31 insertions(+), 24 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index 1a686b2f6f72..5f55c7ffa9f3 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -268,7 +268,6 @@ type tyVoid # now different from tyEmpty, hurray! tyIterable - tyStub static: # remind us when TTypeKind stops to fit in a single 64-bit word @@ -693,10 +692,15 @@ type PScope* = ref TScope + ItemState* = enum + Complete # completely in memory + Partial # partially in memory + PLib* = ref TLib TSym* {.acyclic.} = object # Keep in sync with PackedSym itemId*: ItemId # proc and type instantiations are cached in the generic symbol + state*: ItemState case kind*: TSymKind of routineKinds: #procInstCache*: seq[PInstantiation] @@ -771,6 +775,7 @@ type # Keep in sync with PackedType itemId*: ItemId kind*: TTypeKind # kind of type + state*: ItemState callConv*: TCallingConvention # for procs flags*: TTypeFlags # flags of the type sons: TTypeSeq # base types, etc. diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index a57bbd65af7d..aa552d61c422 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -192,7 +192,9 @@ proc writeType(w: var Writer; dest: var TokenBuf; typ: PType) proc writeSym(w: var Writer; dest: var TokenBuf; sym: PSym) proc typeToNifSym(w: var Writer; typ: PType): string = - result = "`t." + result = "`t" + result.addInt ord(typ.kind) + result.add '.' result.addInt typ.uniqueId.item result.add '.' result.add modname(w.moduleToNifSuffix, typ.uniqueId.module, w.infos.config) @@ -207,7 +209,7 @@ proc writeTypeDef(w: var Writer; dest: var TokenBuf; typ: PType) = dest.buildTree tdefTag: dest.addSymDef pool.syms.getOrIncl(w.typeToNifSym(typ)), NoLineInfo - dest.addIdent toNifTag(typ.kind) + #dest.addIdent toNifTag(typ.kind) writeFlags(dest, typ.flags) dest.addIdent toNifTag(typ.callConv) dest.addIntLit typ.size @@ -581,8 +583,13 @@ proc loadNode(c: var DecodeContext; n: var Cursor): PNode proc loadTypeStub(c: var DecodeContext; t: SymId): PType = let name = pool.syms[t] - assert name.startsWith("`t.") - var i = len("`t.") + assert name.startsWith("`t") + var i = len("`t") + var k = 0 + while i < name.len and name[i] in {'0'..'9'}: + k = k * 10 + name[i].ord - ord('0') + inc i + if i < name.len and name[i] == '.': inc i var itemId = 0'i32 while i < name.len and name[i] in {'0'..'9'}: itemId = itemId * 10'i32 + int32(name[i].ord - ord('0')) @@ -593,7 +600,7 @@ proc loadTypeStub(c: var DecodeContext; t: SymId): PType = result = c.types.getOrDefault(id)[0] if result == nil: let offs = c.getOffset(id.module, name) - result = PType(itemId: id, uniqueId: id, kind: tyStub) + result = PType(itemId: id, uniqueId: id, kind: TTypeKind(k), state: Partial) c.types[id] = (result, offs) proc loadTypeStub(c: var DecodeContext; n: var Cursor): PType = @@ -622,7 +629,7 @@ proc loadSymStub(c: var DecodeContext; t: SymId): PSym = result = c.syms.getOrDefault(id)[0] if result == nil: let offs = c.getOffset(module, symAsStr) - result = PSym(itemId: id, kind: skStub, name: c.cache.getIdent(sn.name), disamb: sn.count.int32) + result = PSym(itemId: id, kind: skStub, name: c.cache.getIdent(sn.name), disamb: sn.count.int32, state: Partial) c.syms[id] = (result, offs) proc loadSymStub(c: var DecodeContext; n: var Cursor): PSym = @@ -640,8 +647,8 @@ proc loadSymStub(c: var DecodeContext; n: var Cursor): PSym = else: raiseAssert "sym expected but got " & $n.kind -proc isStub*(t: PType): bool {.inline.} = t.kind == tyStub -proc isStub*(s: PSym): bool {.inline.} = s.kind == skStub +proc isStub*(t: PType): bool {.inline.} = t.state == Partial +proc isStub*(s: PSym): bool {.inline.} = s.state == Partial proc loadAtom[T](t: typedesc[set[T]]; n: var Cursor): set[T] = if n.kind == DotToken: @@ -681,7 +688,7 @@ proc loadLoc(c: var DecodeContext; n: var Cursor; loc: var TLoc) = loadField loc.snippet proc loadType*(c: var DecodeContext; t: PType) = - if t.kind != tyStub: return + if t.state != Partial: return var buf = createTokenBuf(30) var n = cursorFromIndexEntry(c, t.itemId.module, c.types[t.itemId][1], buf) @@ -692,7 +699,7 @@ proc loadType*(c: var DecodeContext; t: PType) = expect n, SymbolDef # ignore the type's name, we have already used it to create this PType's itemId! inc n - loadField t.kind + #loadField t.kind loadField t.flags loadField t.callConv loadField t.size diff --git a/compiler/ccgreset.nim b/compiler/ccgreset.nim index 0e4c73cd17ed..84478dd07e15 100644 --- a/compiler/ccgreset.nim +++ b/compiler/ccgreset.nim @@ -112,7 +112,7 @@ proc specializeResetT(p: BProc, accessor: Rope, typ: PType) = tyGenericParam, tyOrdinal, tyOpenArray, tyForward, tyVarargs, tyUncheckedArray, tyError, tyBuiltInTypeClass, tyUserTypeClass, tyUserTypeClassInst, tyCompositeTypeClass, tyAnd, tyOr, tyNot, - tyAnything, tyStatic, tyFromExpr, tyConcept, tyVoid, tyIterable, tyStub: + tyAnything, tyStatic, tyFromExpr, tyConcept, tyVoid, tyIterable: discard proc specializeReset(p: BProc, a: TLoc) = diff --git a/compiler/expanddefaults.nim b/compiler/expanddefaults.nim index e343317e5414..c520d8849f2d 100644 --- a/compiler/expanddefaults.nim +++ b/compiler/expanddefaults.nim @@ -124,7 +124,7 @@ proc expandDefault(t: PType; info: TLineInfo): PNode = result = newZero(t, info, nkBracket) of tyString: result = newZero(t, info, nkStrLit) - of tyNone, tyEmpty, tyUntyped, tyTyped, tyTypeDesc, tyStub, + of tyNone, tyEmpty, tyUntyped, tyTyped, tyTypeDesc, tyNil, tyGenericInvocation, tyError, tyBuiltInTypeClass, tyUserTypeClass, tyUserTypeClassInst, tyCompositeTypeClass, tyAnd, tyOr, tyNot, tyAnything, tyConcept, tyIterable, tyForward: diff --git a/compiler/icnif/enum2nif.nim b/compiler/icnif/enum2nif.nim index 1f77393c6aa4..b8626fe56d4f 100644 --- a/compiler/icnif/enum2nif.nim +++ b/compiler/icnif/enum2nif.nim @@ -469,7 +469,6 @@ proc toNifTag*(s: TTypeKind): string = of tyConcept: "concept" of tyVoid: "void" of tyIterable: "iterable" - of tyStub: "stub" proc parse*(t: typedesc[TTypeKind]; s: string): TTypeKind = @@ -539,7 +538,6 @@ proc parse*(t: typedesc[TTypeKind]; s: string): TTypeKind = of "concept": tyConcept of "void": tyVoid of "iterable": tyIterable - of "stub": tyStub else: tyNone diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 4d63d44acb64..fd8ef583d00a 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -219,7 +219,7 @@ proc mapType(typ: PType): TJSTypeKind = else: result = etyNone of tyProc: result = etyProc of tyCstring: result = etyString - of tyConcept, tyIterable, tyStub: + of tyConcept, tyIterable: raiseAssert "unreachable" proc mapType(p: PProc; typ: PType): TJSTypeKind = diff --git a/compiler/liftdestructors.nim b/compiler/liftdestructors.nim index 42b96aa1cab9..5d8fbc179dd8 100644 --- a/compiler/liftdestructors.nim +++ b/compiler/liftdestructors.nim @@ -972,7 +972,7 @@ proc ownedClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode) = case t.kind - of tyNone, tyEmpty, tyVoid, tyStub: discard + of tyNone, tyEmpty, tyVoid: discard of tyPointer, tySet, tyBool, tyChar, tyEnum, tyInt..tyUInt64, tyCstring, tyPtr, tyUncheckedArray, tyVar, tyLent: defaultOp(c, t, body, x, y) diff --git a/compiler/nifgen.nim b/compiler/nifgen.nim index c9d86b50f00b..cf267ef14bc5 100644 --- a/compiler/nifgen.nim +++ b/compiler/nifgen.nim @@ -666,7 +666,6 @@ proc toNifTag(s: TTypeKind): string = of tyConcept: "concept" of tyVoid: "void" of tyIterable: "iterable" - of tyStub: "stub" proc atom(t: PType; c: var TranslationContext) = c.b.withTree toNifTag(t.kind): @@ -925,7 +924,7 @@ proc toNifType(t: PType; parent: PNode; c: var TranslationContext) = atom t, c, "err" of tyCompositeTypeClass: toNifType t.last, parent, c of tyInferred: toNifType t.skipModifier, parent, c - of tyAnything, tyStub: atom t, c + of tyAnything: atom t, c of tyStatic: c.typeHead t: if t.hasElementType: diff --git a/compiler/typeallowed.nim b/compiler/typeallowed.nim index 0529a6671f6d..80b532371cb0 100644 --- a/compiler/typeallowed.nim +++ b/compiler/typeallowed.nim @@ -201,7 +201,7 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, result = typeAllowedNode(marker, t.n, kind, c, flags) of tyEmpty: if kind in {skVar, skLet}: result = t - of tyError, tyStub: + of tyError: # for now same as error node; we say it's a valid type as it should # prevent cascading errors: result = nil diff --git a/compiler/types.nim b/compiler/types.nim index b26a949407de..bd65c3f33137 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -491,7 +491,7 @@ const "BuiltInTypeClass", "UserTypeClass", "UserTypeClassInst", "CompositeTypeClass", "inferred", "and", "or", "not", "any", "static", "TypeFromExpr", "concept", # xxx bugfix - "void", "iterable", "stub"] + "void", "iterable"] const preferToResolveSymbols = {preferName, preferTypeName, preferModuleInfo, preferGenericArg, preferResolved, preferMixed, preferInlayHint, preferInferredEffects} @@ -1344,8 +1344,6 @@ proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool = result = a.id == b.id and sameFlags(a, b) of tyError: result = b.kind == tyError - of tyStub: - result = false of tyTuple: withoutShallowFlags: cycleCheck() diff --git a/compiler/vmdeps.nim b/compiler/vmdeps.nim index 882f030ee764..72eec34eadb6 100644 --- a/compiler/vmdeps.nim +++ b/compiler/vmdeps.nim @@ -97,7 +97,7 @@ proc mapTypeToAstX(cache: IdentCache; t: PType; info: TLineInfo; return atomicType(t.sym) case t.kind - of tyNone, tyStub: result = atomicType("none", mNone) + of tyNone: result = atomicType("none", mNone) of tyBool: result = atomicType("bool", mBool) of tyChar: result = atomicType("char", mChar) of tyNil: result = atomicType("nil", mNil)