diff --git a/.github/workflows/concourse-notify-pull-request.yml b/.github/workflows/concourse-notify-pull-request.yml index 0d763cc16..f7e29adfd 100644 --- a/.github/workflows/concourse-notify-pull-request.yml +++ b/.github/workflows/concourse-notify-pull-request.yml @@ -3,14 +3,14 @@ on: [ pull_request, workflow_dispatch ] env: HOST: https://ci.jitx.com TEAM: main - PIPELINE: lbstanza + PIPELINE: stanzapm INSTVARS: vars.branch="${{ github.base_ref }}" TOKEN: ${{ secrets.CONCOURSE_WEBHOOK_TOKEN }} jobs: trigger-pr: runs-on: ubuntu-latest env: - RESOURCE: github--lbstanza--pr + RESOURCE: github--stanzapm--pr steps: - name: Trigger Concourse Pull Request run: | diff --git a/.github/workflows/concourse-notify-push.yml b/.github/workflows/concourse-notify-push.yml index f8f4dcce7..0ee2e8795 100644 --- a/.github/workflows/concourse-notify-push.yml +++ b/.github/workflows/concourse-notify-push.yml @@ -3,14 +3,14 @@ on: [ push, workflow_dispatch ] env: HOST: https://ci.jitx.com TEAM: main - PIPELINE: lbstanza + PIPELINE: stanzapm INSTVARS: vars.branch="${{ github.ref_name }}" TOKEN: ${{ secrets.CONCOURSE_WEBHOOK_TOKEN }} jobs: trigger-push: runs-on: ubuntu-latest env: - RESOURCE: git--lbstanza + RESOURCE: git--stanzapm steps: - name: Trigger Concourse resource check run: | diff --git a/ci/build-stanza-version.txt b/ci/build-stanza-version.txt index 714348446..e641719cf 100644 --- a/ci/build-stanza-version.txt +++ b/ci/build-stanza-version.txt @@ -7,4 +7,5 @@ # like 1.23.45 # # Use version 0.17.56 to compile 0.18.0 -0.18.96 +0.19.0 + diff --git a/ci/build-stanza.sh b/ci/build-stanza.sh index 604f0d66c..829a7a7bb 100755 --- a/ci/build-stanza.sh +++ b/ci/build-stanza.sh @@ -11,7 +11,7 @@ USAGE="STANZA_CONFIG=/path $0" echo " STANZA_CONFIG:" "${STANZA_CONFIG:?Usage: ${USAGE}}" # directory where .stanza config file will be stored, as in normal stanza behavior # Defaulted env var inputs - can override if necessary -echo " REPODIR:" "${REPODIR:=lbstanza}" +echo " REPODIR:" "${REPODIR:=stanzapm}" echo " CONAN_USER_HOME:" "${CONAN_USER_HOME:=${REPODIR}}" echo " CREATE_ARCHIVE:" "${CREATE_ARCHIVE:=false}" echo " CREATE_PACKAGE:" "${CREATE_PACKAGE:=false}" @@ -79,7 +79,7 @@ esac cd "${REPODIR}" -echo "Building lbstanza version ${VER} in ${PWD}" +echo "Building stanzapm version ${VER} in ${PWD}" mkdir -p build mkdir -p bin diff --git a/ci/calc-stanza-version-bump.sh b/ci/calc-stanza-version-bump.sh index 305c5755d..b2ebac9a9 100755 --- a/ci/calc-stanza-version-bump.sh +++ b/ci/calc-stanza-version-bump.sh @@ -10,7 +10,7 @@ TOP="${PWD}" >&2 echo " BRANCH:" "${BRANCH:?Usage: BRANCH=foo $0}" # Defaulted env var inputs - can override if necessary ->&2 echo " REPODIR:" "${REPODIR:=lbstanza}" +>&2 echo " REPODIR:" "${REPODIR:=stanzapm}" ## By default, get the most recent previous tag on this branch from git ### note: if multiple tags exist on one commit, git doesn't always desecribe the most recent one ### ### maybe use: git tag --sort=committerdate --contains HEAD~ | tail -1 diff --git a/ci/install-stanza.sh b/ci/install-stanza.sh index eff3409e3..5afc24816 100755 --- a/ci/install-stanza.sh +++ b/ci/install-stanza.sh @@ -11,8 +11,8 @@ TOP="${PWD}" # the machine that you are building for (host) # and the machine that GCC will produce code for (target). -USAGE="STANZA_BUILD_PLATFORM={linux|macos|windows} STANZA_BUILD_VER=0.17.56 STANZA_CONFIG=/path STANZA_INSTALL_DIR=/path $0" -STANZA_DOWNLOAD_BASEURL="https://github.com/StanzaOrg/lbstanza/releases/download" +USAGE="STANZA_BUILD_PLATFORM={linux|macos|windows} STANZA_BUILD_VER=0.19.0 STANZA_CONFIG=/path STANZA_INSTALL_DIR=/path $0" +STANZA_DOWNLOAD_BASEURL="https://github.com/jitx-inc/stanzapm/releases/download" # Required env var inputs @@ -27,7 +27,7 @@ BSVTXT="${THISDIR}/build-stanza-version.txt" # extract the version from first non-comment line of the file BSTZVER=$(grep -v ^\# "${BSVTXT}" | head -1 | awk '{ print $1}') echo "Using existing stanza version $BSTZVER" -echo " STANZA_BUILD_VER:" "${STANZA_BUILD_VER:=${BSTZVER}}" # 0.17.56 +echo " STANZA_BUILD_VER:" "${STANZA_BUILD_VER:=${BSTZVER}}" # 0.19.0 echo "STANZA_BUILD_PLATFORM:" "${STANZA_BUILD_PLATFORM:=$(uname -s)}" # linux|macos|Darwin|windows|MINGW64 # var input validation diff --git a/compiler/codegen.stanza b/compiler/codegen.stanza index 96c140bcb..238c4c281 100644 --- a/compiler/codegen.stanza +++ b/compiler/codegen.stanza @@ -49,6 +49,10 @@ public defstruct AsmStubs : stack-limit:Int registers:Int system-registers-space:Int + profile-flag:Int + profile-buffer:Int + function-counters:Int + function-info:Int heap-start:Int heap-top:Int heap-limit:Int @@ -72,6 +76,7 @@ public defstruct AsmStubs : collect-garbage:Int saved-regs:Tuple saved-fregs:Tuple + heap-statistics:Int class-table:Int global-root-table:Int stackmap-table:Int @@ -105,6 +110,10 @@ public defn AsmStubs (backend:Backend) : next(id-counter) ;stack-limit:Int next(id-counter) ;registers:Int next(id-counter) ;system-registers-space:Int + next(id-counter) ;profile-flag:Int + next(id-counter) ;profile-buffer:Int + next(id-counter) ;function-counters:Int + next(id-counter) ;function-info:Int next(id-counter) ;heap-start:Int next(id-counter) ;heap-top:Int next(id-counter) ;heap-limit:Int @@ -128,6 +137,7 @@ public defn AsmStubs (backend:Backend) : next(id-counter) ;collect-garbage:Int saved-regs ;saved-regs:Tuple saved-fregs ;saved-fregs:Tuple + next(id-counter) ;heap-statistics:Int next(id-counter) ;class-table:Int next(id-counter) ;global-root-table:Int next(id-counter) ;stackmap-table:Int @@ -204,6 +214,10 @@ public defn compile-runtime-stubs (emitter:CodeEmitter, stubs:AsmStubs) : comment("stack-limit = %_" % [stack-limit(stubs)]) comment("registers = %_" % [registers(stubs)]) comment("system-registers-space = %_" % [system-registers-space(stubs)]) + comment("profile-flag = %_" % [profile-flag(stubs)]) + comment("profile-buffer = %_" % [profile-buffer(stubs)]) + comment("function-counters = %_" % [function-counters(stubs)]) + comment("function-info = %_" % [function-info(stubs)]) comment("heap-start = %_" % [heap-start(stubs)]) comment("heap-top = %_" % [heap-top(stubs)]) comment("heap-limit = %_" % [heap-limit(stubs)]) @@ -334,6 +348,12 @@ public defn compile-entry-function (emitter:CodeEmitter, stubs:AsmStubs) : #label(safepoint-table) ;safepoint-table:ptr #label(debug-table) ;debug-table:ptr #label(local-var-table) ;local-var-table:ptr + #label(heap-statistics) ;heap-statistics:ptr + #long() ;heap-dominator-tree:ptr + #label(profile-flag) ;profile-flag: ptr + #long() ;profile-buffer: ptr + #label(function-counters) ;function-counters: ptr + #label(function-info) ;function-info: ptr #label(class-table) ;class-table:ptr #label(global-root-table) ;global-root-table:ptr #label(stackmap-table) ;stackmap-table:ptr diff --git a/compiler/compiler-build-settings.stanza b/compiler/compiler-build-settings.stanza index d0bd22258..154fc36e5 100644 --- a/compiler/compiler-build-settings.stanza +++ b/compiler/compiler-build-settings.stanza @@ -22,6 +22,8 @@ public defstruct BuildSettings : build-from-source?: True|False optimize?: True|False debug?: True|False + profile?: True|False + coverage?: True|False ccfiles: Tuple ccflags: Tuple> flags: Tuple diff --git a/compiler/compiler-main.stanza b/compiler/compiler-main.stanza index cd5d68b5a..f0d770d05 100644 --- a/compiler/compiler-main.stanza +++ b/compiler/compiler-main.stanza @@ -158,6 +158,8 @@ public defn compile (proj-manager:ProjManager, backend:Backend, optimize?:True|False, debug?:True|False, + profile?:True|False, + coverage?:True|False, macro-plugins:Tuple, force-build-macros?:True|False) -> CompilationResult : defn driver () : @@ -224,7 +226,7 @@ public defn compile (proj-manager:ProjManager, `unoptimized-asm : val packages = for p in /packages(result) map : match(p) : - (p:EPackage) : compile(lower-unoptimized(p), debug?) + (p:EPackage) : compile(lower-unoptimized(p, profile?, coverage?), debug?) (p:StdPkg) : p compile-vmpackages(pkgsaver when write-out-pkgs?, packages, @@ -244,7 +246,7 @@ public defn compile (proj-manager:ProjManager, val epackages = for p in packages map : match(p:FastPkg) : EPackage(packageio(p), exps(p)) else : p as EPackage - compile(lower-optimized(epackages), debug?) + compile(lower-optimized(epackages, profile?, coverage?), debug?) ;- pkgsaver: False if we not need to write out any .pkg files to disk (either ; to pkg dir or to pkg cache.). @@ -309,7 +311,7 @@ public defn compile (proj-manager:ProjManager, for package in packages do : match(package) : (epackage:EPackage) : - val vmpackage = compile(lower-unoptimized(epackage), debug?) + val vmpackage = compile(lower-unoptimized(epackage, profile?, coverage?), debug?) val npkg = normalize(vmpackage, backend) val buffer = Vector() emit-normalized-package(npkg, buffer-emitter(buffer, stubs), stubs) @@ -399,3 +401,4 @@ defn collapse (p:NormVMPackage|StdPkg) : defmethod debug-table (this) : debug-table(vmp) defmethod safepoint-table (this) : safepoint-table(vmp) defmethod debug-name-table (this): debug-name-table(vmp) + defmethod function-info (this) : function-info(vmp) diff --git a/compiler/compiler.stanza b/compiler/compiler.stanza index 825d7765b..43297d1ab 100644 --- a/compiler/compiler.stanza +++ b/compiler/compiler.stanza @@ -69,6 +69,9 @@ defn compute-build-settings (projenv:StandardProjEnv, val build-optimization = optimize?(settings) or optimize(s) ;Compute build optimization level val build-debug = debug?(settings) + ;Compute build profile level + val build-profile = profile?(settings) or profile(s) + val build-coverage = coverage?(settings) or coverage(s) ;Compute assembly val build-asm = string-or?(original-asm(settings), value?(assembly(s))) ;Compute output @@ -110,6 +113,8 @@ defn compute-build-settings (projenv:StandardProjEnv, build-from-source?(settings) build-optimization, build-debug, + build-profile, + build-coverage, build-ccfiles, build-ccflags, to-tuple(build-flags), @@ -243,10 +248,12 @@ public defn compile (settings:BuildSettings, system:System) : println("Build target %~ is already up-to-date." % [build-target?(settings*)]) else : setup-system-flags(settings*) + add-flag(`HEAP-ANALYZER) val proj-params = ProjParams(compiler-flags(), optimize?(settings*), debug?(settings*), false, build-from-source?(settings), pkg-cache-dir(settings*)) val proj-manager = ProjManager(proj, proj-params, auxfile) val comp-result = compile(proj-manager, auxfile, build-inputs!(settings*), vm-packages(settings*), asm?(settings*), pkg-dir(settings*), - backend(platform(settings*) as Symbol), optimize?(settings*), debug?(settings*), + backend(platform(settings*) as Symbol), optimize?(settings*), debug?(settings*), + profile?(settings*), coverage?(settings*), macro-plugins(settings*), inputs(settings) is BuildTarget) save(auxfile) within delete-temporary-file-on-finish(settings*) : diff --git a/compiler/defs-db.stanza b/compiler/defs-db.stanza index 02e8fa147..7307c698a 100644 --- a/compiler/defs-db.stanza +++ b/compiler/defs-db.stanza @@ -99,6 +99,8 @@ protected-when(TESTING) defn analyze-input (input:DefsDbInput) -> DependencyResu true, ;build-from-source? optimize?(input), ;optimize? false, ;debug? + false, ;profile? + false, ;coverage? [], ;ccfiles [], ;ccflags flags(input) ;flags diff --git a/compiler/dl-ir.stanza b/compiler/dl-ir.stanza index 1383cc2f5..ae2bd5232 100644 --- a/compiler/dl-ir.stanza +++ b/compiler/dl-ir.stanza @@ -1192,6 +1192,7 @@ public val CORE-COLLECT-STACK-TRACE-ID = register $ core-fnid(`collect-stack-tra public val CORE-COLLECT-GARBAGE-ID = register $ core-fnid(`collect-garbage, [`long]) public val CORE-MAKE-STRING-ID = register $ core-fnid(`String, [DPtrT(DByte())]) public val CORE-EXECUTE-TOPLEVEL-COMMAND-ID = register $ core-fnid(`execute-toplevel-command, [DArrow([], core-type(`False))]) +public val CORE-PROFILE-STACK-TRACE-ID = register $ core-fnid(`profile-stack-trace) ;============================================================ ;====================== ID Environment ====================== diff --git a/compiler/el-infer.stanza b/compiler/el-infer.stanza index 4d2737bee..db206c926 100644 --- a/compiler/el-infer.stanza +++ b/compiler/el-infer.stanza @@ -360,7 +360,7 @@ defn analyze-single-block (emit:Ins -> ?, (ins:ETupleSet|EStruct|EPtr|ELabel|ETCall| EDump|EConv|EGoto|ECheckLength|ECheckSet| EBoxSet|EReturn|EObjectTGet|EClosureTGet| - ETDef|ETypeObject|EEnd|ELive|ECheckFail|ESafepoint) : gen-trivial-commands(ins) + ETDef|ETypeObject|EEnd|ELive|ECheckFail|ESafepoint|EProfile) : gen-trivial-commands(ins) ;Utility: For any uses of inferred variables generate ;them here. @@ -551,7 +551,7 @@ defn analyze-single-block (emit:Ins -> ?, defn gen-trivial-commands (ins:ETupleSet|EStruct|EPtr|ELabel|ETCall| EDump|EConv|EGoto|ECheckLength|ECheckSet| EBoxSet|EReturn|EObjectTGet|EClosureTGet| - ETDef|ETypeObject|EEnd|ELive|ECheckFail|ESafepoint) : + ETDef|ETypeObject|EEnd|ELive|ECheckFail|ESafepoint|EProfile) : gen-uses(ins) emit(Wrapped(ins)) for x in varlocs(ins) do : diff --git a/compiler/el-ir.stanza b/compiler/el-ir.stanza index f433f7d5a..515685ebe 100644 --- a/compiler/el-ir.stanza +++ b/compiler/el-ir.stanza @@ -46,6 +46,7 @@ public defstruct EBot <: HiEType public defstruct EPackage : packageio: PackageIO with: (updater => sub-packageio) exps: Tuple with: (updater => sub-exps) + function-info: False|Tuple with: (default => false, updater => sub-function-info) public defn name (e:EPackage) : package(packageio(e)) @@ -544,6 +545,10 @@ public defstruct DetupleContext <: CastErrorContext public defstruct GetVarContext <: CastErrorContext public defstruct BranchContext <: CastErrorContext +public defstruct EProfile <: EIns : + id: Int + kind: Int + ;============================================================ ;======================= Primitives ========================= ;============================================================ @@ -761,6 +766,7 @@ defmethod print (o:OutputStream, e:EIns) : (e:ETypeof) : P $ "goto L%_ when typeof(%_, %_) else L%_" % [n1(e), y(e), type(e), n2(e)] (e:ECheckFail) : P $ "check-fail(%_) %_ : %_" % [ctxt(e), y(e), type(e)] (e:ESafepoint) : P $ "safepoint %_ ;%_" % [group-id(e), info?(e)] + (e:EProfile) : P $ "profile(%_,%_)" % [id(e), kind(e)] defmethod print (o:OutputStream, t:CallType) : print{o, _} $ match(t) : @@ -1001,6 +1007,7 @@ defsyntax el-ir : defrule eins = (?x:#vloc : ?xt:#etype = load ?loc:#eloc) : ELoad(x, xt, loc) defrule eins = (?x:#vloc = call ?t:#calltype ?f:#eimm(?ys:#eimm ...)) : ECall(x, f, to-tuple(ys), t, add-base-dir(closest-info())) defrule eins = (?x:#vloc = ?y:#eimm) : EDef(x, y) + defrule eins = (profile(?id:#int, ?arg:#int)) : EProfile(id, arg) defproduction calltype : CallType defrule calltype = (guarded((?a1:#etype ...) -> ?a2:#etype)) : CallGuarded(to-tuple(a1), a2) @@ -1305,7 +1312,7 @@ defmethod map (f:ELItem -> ELItem, item:ELItem&?T) -> ELItem&T : (t:EBot) : t ;EPackage - (e:EPackage) : EPackage(packageio(e), h(exps(e))) + (e:EPackage) : EPackage(packageio(e), h(exps(e)), function-info(e)) ;ETExps (e:EDefGlobal) : EDefGlobal(n(e), h(type(e)), mutable?(e), lostanza?(e)) @@ -1420,6 +1427,9 @@ defmethod map (f:ELItem -> ELItem, item:ELItem&?T) -> ELItem&T : ;Closures (e:EClosure) : EClosure(n(e), h(targs(e)), h(ys(e))) + ;Profile + (e:EProfile) : EProfile(id(e), kind(e)) + ;============================================================ ;===================== Definitions ========================== ;============================================================ @@ -1478,6 +1488,7 @@ public defn varlocs (e:EIns) -> Tuple : (e:ETypeof) : [] (e:ECheckFail) : [] (e:ESafepoint) : [] + (e:EProfile) : [] ;Return the list of immediates that the instruction ;uses to perform its action. @@ -1532,6 +1543,7 @@ public defn uses (e:EIns) -> Seqable : (e:ETypeof) : [y(e)] (e:ECheckFail) : [y(e)] (e:ESafepoint) : [] + (e:EProfile) : [] ;Return the list of immediates that the instruction ;uses to retrieve its location. diff --git a/compiler/el-to-vm.stanza b/compiler/el-to-vm.stanza index 6c348cccf..d0a4790e1 100644 --- a/compiler/el-to-vm.stanza +++ b/compiler/el-to-vm.stanza @@ -61,7 +61,7 @@ public defn compile (epackage:EPackage, debug?:True|False) -> VMPackage : val init = compile-init(init-compiler, iotable, progbuffer) init - val vmp = to-vmpackage(progbuffer, io, init) + val vmp = to-vmpackage(progbuffer, io, init, function-info(epackage)) ;dump(vmp, "logs", "pre-analyze") val vmp* = within time-ms!("VM Analyze") : analyze(vmp) ;dump(vmp*, "logs", false) @@ -565,7 +565,7 @@ defmulti add (b:ProgBuffer, e:VMMethod) -> False defmulti add (b:ProgBuffer, e:VMDebugInfo) -> False defmulti add (b:ProgBuffer, e:SafepointEntry) -> SafepointIns defmulti add (b:ProgBuffer, e:VMDebugName) -> False -defmulti to-vmpackage (b:ProgBuffer, io:PackageIO, init:Int|False) -> VMPackage +defmulti to-vmpackage (b:ProgBuffer, io:PackageIO, init:Int|False, function-info:False|Tuple) -> VMPackage defn ProgBuffer () : val globals = Vector() @@ -591,7 +591,7 @@ defn ProgBuffer () : defmethod add (this, e:VMDebugInfo) : add(debug, e) defmethod add (this, e:SafepointEntry) : add(sptable, e) defmethod add (this, e:VMDebugName) : add(debugnames, e) - defmethod to-vmpackage (this, io:PackageIO, init:Int|False) : + defmethod to-vmpackage (this, io:PackageIO, init:Int|False, function-info:False|Tuple) : VMPackage(io, init, to-tuple(globals), to-tuple(datas), @@ -603,6 +603,7 @@ defn ProgBuffer () : to-tuple(extern-defns), VMDebugNameTable(to-tuple(debugnames)), VMDebugInfoTable(to-tuple(debug)), + function-info compile(sptable)) ;============================================================ @@ -1544,6 +1545,8 @@ defn Compiler (gt:GlobalTable, (info:StackTraceInfo) : make-safepoint(info) when /info(info) is-not False (f:False) : false + (ins:EProfile) : + emit(ProfileIns(id(ins), kind(ins))) ;--------------------------------------------------------- ;--------------- Create the Compiler Type ---------------- diff --git a/compiler/el.stanza b/compiler/el.stanza index 064d784cc..d6f3bdcab 100644 --- a/compiler/el.stanza +++ b/compiler/el.stanza @@ -31,17 +31,17 @@ defpackage stz/el : ;==================== Drivers =============================== ;============================================================ -public defn lower-optimized (epackages:Tuple) -> EPackage : +public defn lower-optimized (epackages:Tuple, profile?:True|False, coverage?:True|False) -> EPackage : vprintln("EL: Lower optimized packages: %," % [seq(name, epackages)]) val epackages* = map(fill-stack-trace-entries, epackages) ;do(dump{_, "logs", "input"}, epackages*) val collapsed = collapse(epackages*) ;dump(collapsed, "logs", "collapsed") - lower(collapsed, true) + lower(collapsed, true, profile?, coverage?) -public defn lower-unoptimized (epackage:EPackage) -> EPackage : +public defn lower-unoptimized (epackage:EPackage, profile?:True|False, coverage?:True|False) -> EPackage : vprintln("EL: Lower unoptimized package: %_" % [name(epackage)]) - lower(fill-stack-trace-entries(epackage), false) + lower(fill-stack-trace-entries(epackage), false, profile?, coverage?) ;============================================================ ;===================== Timers =============================== @@ -56,7 +56,7 @@ val EL-TIMERS = within pass-name = HashTable-init() : ;========================= Lowering ========================= ;============================================================ -defn lower (epackage:EPackage, optimize?:True|False) -> EPackage : +defn lower (epackage:EPackage, optimize?:True|False, profile?:True|False, coverage?:True|False) -> EPackage : val lower-timer-name = to-string("EL-LOWER -- %_" % [name(epackage)]) within log-time(EL-LOWER, lower-timer-name) : @@ -105,6 +105,8 @@ defn lower (epackage:EPackage, optimize?:True|False) -> EPackage : run-pass("Box Mutables", box-mutables, "boxed", false) run-pass("Detect Loops", detect-loops, "looped", false) run-pass("Simple Inline", simple-inline, "inlined0", false) + if profile? or coverage? : + run-pass("Profile", insert-profiling{_, coverage?}, "profile", true) run-pass("Within Package Inline", within-package-inline{_, true}, "wp-inlined0", false) run-pass("Cleanup Labels", cleanup-labels, "cleanup-labels", false) if optimize? : @@ -1754,6 +1756,8 @@ defn inline-call (buffer:BodyBuffer, emit(buffer, ESafepoint(group-id(i), info)) (call-info, instruction-info): emit(buffer, i) + (i:EProfile) : + false (i) : emit(buffer, i) @@ -2129,6 +2133,60 @@ defn rename-fn (f:EFn) -> EFn : create-definitions(f) rename(f) as EFn +;============================================================ +;=================== Insert Profiling ======================= +;============================================================ + +defn insert-profiling (epackage:EPackage, coverage?:True|False) : + val infos = Vector() + val iotable = IOTable(packageio(epackage)) + val num-uses = IntTable(0) + defn get-info (sinfo:False|TraceInfo, body:EBody) -> False|StackTraceInfo : + match(sinfo) : + (sinfo: StackTraceInfo) : sinfo + (sinfo) : + label return : + for i in ins(body) do : + match(info?(i)) : + (info:StackTraceInfo) : return(info) + (info) : false + + defn insert (e:EBody, sinfo:StackTraceInfo, insert?:True|False) -> EBody : + val buffer = BodyBuffer(e) + val id = length(infos) + emit(buffer, EProfile(id, 1 when coverage? else 0)) when insert? + for i in ins(e) do : + val i* = + match(info?(i)) : + (info:StackTraceInfo) : sub-info(i, sub-function(info, id)) + (info) : i + emit(buffer, i*) + if insert? and not coverage? : + match(i:ELabel) : + emit(buffer, EProfile(id, 0)) when (num-uses[n(i)] == 0) + else : + for v in label-uses(i) do : + update(num-uses, {_ + 1}, v) + add(infos, FunctionInfo(sinfo)) + to-body(buffer) + + defn insert-texp (t:ELBigItem, insert?:True|False) -> ELBigItem : + match(map(insert-texp{_, insert?}, t)) : + (e:EFn) : + match(get-info(info(e), body(e))) : + (i:StackTraceInfo) : sub-body(e, insert(body(e), i, insert?)) + (i:False) : e + (e) : e + + ;Inline all top-level expressions + val new-exps = for exp in exps(epackage) map : + match(exp) : + (exp:EDefn) : insert-texp(exp, not lostanza?(exp)) as ETExp + (exp:EDefClosure) : insert-texp(exp, true) as ETExp + (exp:EDefmethod) : insert-texp(exp, not lostanza?(exp)) as ETExp + (exp) : exp + sub-function-info(sub-exps(epackage, new-exps), to-tuple $ infos) + ;============================================================ ;==================== Label Cleanup ========================= ;============================================================ diff --git a/compiler/main.stanza b/compiler/main.stanza index 4538d4d73..d3be04c4c 100644 --- a/compiler/main.stanza +++ b/compiler/main.stanza @@ -6,6 +6,7 @@ defpackage stz/main : import reader import arg-parser import line-wrap + import profiler import stz/compiler import stz/params import stz/config @@ -63,6 +64,20 @@ defn run-with-timing-log (body:() -> ?T, cmd-args:CommandArgs) -> T : else : body() +;============================================================ +;=================== profile-compiler ======================= +;============================================================ + +;If -profile-compiler flag is provided, then execute the body +;with support for profiling. +defn run-with-profiler (body:() -> ?T, cmd-args:CommandArgs) -> T : + var res:T + if flag?(cmd-args, "profile-compiler") : + stack-collapse(profiling({ res = body() }, 100), cmd-args["profile-compiler"]) + else : + res = body() + res + ;============================================================ ;====================== Verbosity =========================== ;============================================================ @@ -229,6 +244,10 @@ val COMMON-STANZA-FLAGS = [ "Requests the compiler to compile in optimized mode.") Flag("debug", ZeroFlag, OptionalFlag, "Requests the compiler to compile for debugging.") + Flag("profile", ZeroFlag, OptionalFlag, + "Requests the compiler to add profile counters.") + Flag("coverage", ZeroFlag, OptionalFlag, + "Requests the compiler to add coverage counters.") Flag("link", OneFlag, OptionalFlag, "Provide the type of linking to use.") Flag("ccfiles", ZeroOrMoreFlag, OptionalFlag, @@ -247,6 +266,8 @@ val COMMON-STANZA-FLAGS = [ "The name of the output external dependencies file.") Flag("timing-log", OneFlag, OptionalFlag, "If provided, the filename of the timing log to generate.") + Flag("profile-compiler", OneFlag, OptionalFlag, + "If provided, the filename of the flame-graph to generate.") Flag("macros", AtLeastOneFlag, OptionalFlag, "If provided, the filename of the macro plugin.")] @@ -333,6 +354,8 @@ defn compile-command () : flag?(cmd-args, "build-from-source") flag?(cmd-args, "optimize") flag?(cmd-args, "debug") + flag?(cmd-args, "profile") + flag?(cmd-args, "coverage") get?(cmd-args, "ccfiles", []) ccflags(cmd-args) map(to-symbol, get?(cmd-args, "flags", [])) @@ -340,15 +363,17 @@ defn compile-command () : symbol?("link")) ;Launch! - within run-with-timing-log(cmd-args) : - within run-with-verbose-flag(cmd-args) : - compile(build-settings(), build-system()) + within run-with-profiler(cmd-args) : + within run-with-timing-log(cmd-args) : + within run-with-verbose-flag(cmd-args) : + compile(build-settings(), build-system()) ;Command Command("compile", AtLeastOneArg, "the .stanza/.proj input files or Stanza package names.", common-stanza-flags(["o" "s" "pkg" "pkg-cache" "build-from-source" "optimize" "debug" "ccfiles" "ccflags" "flags" - "verbose" "supported-vm-packages" "platform" "external-dependencies" "macros" "link" "timing-log"]), + "verbose" "supported-vm-packages" "platform" "external-dependencies" "macros" "link" "timing-log" "profile-compiler" + "profile" "coverage"]), compile-msg, false, verify-args, intercept-no-match-exceptions(compile-action)) @@ -383,6 +408,8 @@ defn build-command () : flag?(cmd-args, "build-from-source") flag?(cmd-args, "optimize") flag?(cmd-args, "debug") + flag?(cmd-args, "profile") + flag?(cmd-args, "coverage") [] ccflags(cmd-args) map(to-symbol, get?(cmd-args, "flags", [])) @@ -397,7 +424,8 @@ defn build-command () : ;Command definition Command("build", ZeroOrOneArg, "the name of the build target. If not supplied, the default build target is 'main'.", - common-stanza-flags(["s" "o" "external-dependencies" "pkg" "pkg-cache" "build-from-source" "flags" "optimize" "debug" "verbose" "ccflags" "macros" "link" "timing-log"]), + common-stanza-flags(["s" "o" "external-dependencies" "pkg" "pkg-cache" "build-from-source" "flags" "optimize" "debug" + "verbose" "ccflags" "macros" "link" "timing-log" "profile" "coverage"]), build-msg, false, verify-args, intercept-no-match-exceptions(build)) ;============================================================ @@ -449,6 +477,8 @@ defn extend-command () : false flag?(cmd-args, "optimize") false + false + false get?(cmd-args, "ccfiles", []) ccflags(cmd-args) map(to-symbol, get?(cmd-args, "flags", [])) @@ -518,6 +548,8 @@ defn compile-test-command () : flag?(cmd-args, "build-from-source") flag?(cmd-args, "optimize") flag?(cmd-args, "debug") + flag?(cmd-args, "profile") + flag?(cmd-args, "coverage") get?(cmd-args, "ccfiles", []) ccflags(cmd-args) new-flags @@ -531,7 +563,7 @@ defn compile-test-command () : Command("compile-test", AtLeastOneArg, "the .stanza/.proj input files or Stanza package names containing tests.", common-stanza-flags(["platform" "s" "o" "external-dependencies" "pkg" "pkg-cache" "build-from-source" "ccfiles" "ccflags" "flags" "optimize" "debug" "verbose" - "macros" "link" "timing-log"]) + "macros" "link" "timing-log" "profile" "coverage"]) compile-test-msg, false, verify-args, intercept-no-match-exceptions(compile-test)) ;============================================================ @@ -583,6 +615,8 @@ defn compile-macros-command () : flag?(cmd-args, "build-from-source") flag?(cmd-args, "optimize") false + false + false get?(cmd-args, "ccfiles", []) ccflags(cmd-args) get?(cmd-args, "flags", []) @@ -988,6 +1022,8 @@ defn analyze-dependencies-command () : flag?(cmd-args, "build-from-source") flag?(cmd-args, "optimize") false + false + false [] [] map(to-symbol, get?(cmd-args, "flags", [])) @@ -1062,6 +1098,8 @@ defn auto-doc-command () : false flag?(cmd-args, "optimize") false + false + false [] [] map(to-symbol, get?(cmd-args, "flags", [])) diff --git a/compiler/params.stanza b/compiler/params.stanza index ab93bbc41..a45044882 100644 --- a/compiler/params.stanza +++ b/compiler/params.stanza @@ -18,7 +18,7 @@ public defn compiler-flags () : to-tuple(COMPILE-FLAGS) ;========= Stanza Configuration ======== -public val STANZA-VERSION = [0 18 97] +public val STANZA-VERSION = [0 19 3] public var STANZA-INSTALL-DIR:String = "" public var OUTPUT-PLATFORM:Symbol = `platform public var STANZA-PKG-DIRS:List = List() diff --git a/compiler/pkg-serializer.stanza b/compiler/pkg-serializer.stanza index 34e750ee9..8c5c8bb8d 100644 --- a/compiler/pkg-serializer.stanza +++ b/compiler/pkg-serializer.stanza @@ -671,7 +671,7 @@ defserializer PkgSerializer (include-asm?:True|False) : DerefOp: () CRSPOp: () - deftype trace-entry (StackTraceInfo) : (package:symbol, signature:opt(string), info:opt(absinfo)) + deftype trace-entry (StackTraceInfo) : (package:symbol, signature:opt(string), info:opt(absinfo), function:opt(int)) ;========================================================== ;================== Assembly Types ======================== diff --git a/compiler/proj-field-types.stanza b/compiler/proj-field-types.stanza index c55685923..9d38ff594 100644 --- a/compiler/proj-field-types.stanza +++ b/compiler/proj-field-types.stanza @@ -161,7 +161,9 @@ defmethod map-fields (s:BuildStmtS0, mapper:Mapper) : map-field(mapper, ccfiles(s), MULTIPLE-PATHS) map-field(mapper, ccflags(s), MULTIPLE-FLAGS) map-field(mapper, flags(s), MULTIPLE-SYMBOLS) - optimize(s)) + optimize(s) + profile(s) + coverage(s)) defmethod convert (s:BuildStmtS0, extractor:Extractor) : BuildStmt( name(s) @@ -175,7 +177,9 @@ defmethod convert (s:BuildStmtS0, extractor:Extractor) : extract-tuple(extractor, ccfiles(s), MULTIPLE-PATHS) extract-tuple(extractor, ccflags(s), MULTIPLE-FLAGS) extract-tuple(extractor, flags(s), MULTIPLE-SYMBOLS) - optimize(s)) + optimize(s) + profile(s) + coverage(s)) defmethod map-fields (s:SyntaxPackagesDefinedInStmtS0, mapper:Mapper) : SyntaxPackagesDefinedInStmtS0( diff --git a/compiler/proj-ir.stanza b/compiler/proj-ir.stanza index 0b6588705..817b0b863 100644 --- a/compiler/proj-ir.stanza +++ b/compiler/proj-ir.stanza @@ -79,6 +79,8 @@ public defstruct BuildStmt <: ProjStmt : ccflags: Tuple> flags: Tuple optimize: True|False + profile: True|False + coverage: True|False with: printer => true diff --git a/compiler/proj-reader.stanza b/compiler/proj-reader.stanza index 2c3e50cab..e946f52a4 100644 --- a/compiler/proj-reader.stanza +++ b/compiler/proj-reader.stanza @@ -126,7 +126,10 @@ defsyntax stanza-projfile : entry?(bs, `ccfiles) entry?(bs, `ccflags) entry?(bs, `flags) - entry?(bs, `optimize, false)) + entry?(bs, `optimize, false) + entry?(bs, `profile, false) + entry?(bs, `coverage, false) + ) defrule projstmt = (var ?name:#symbol! = ?v:#projvalue!) : VarStmtS0(closest-info(), name, v) @@ -201,6 +204,8 @@ defsyntax stanza-projfile : defrule build-option! = (ccflags #:! ?v:#projvalue!) : `ccflags => v defrule build-option! = (flags #:! ?v:#projvalue!) : `flags => v defrule build-option! = (optimize) : `optimize => true + defrule build-option! = (profile) : `profile => true + defrule build-option! = (coverage) : `coverage => true ;---------------------------------------------------------- ;--------------------- Error Productions ------------------ diff --git a/compiler/proj-stage0.stanza b/compiler/proj-stage0.stanza index aaab50206..e2b2d7398 100644 --- a/compiler/proj-stage0.stanza +++ b/compiler/proj-stage0.stanza @@ -81,6 +81,8 @@ public defstruct BuildStmtS0 <: ProjStmt : ccflags: Maybe flags: Maybe optimize: True|False + profile: True|False + coverage: True|False with: (printer => true) ;Example: @@ -315,7 +317,7 @@ public defn map (f:ProjItem -> ProjItem, x:?T&ProjItem) -> T : h(dependencies(x)), h(foreign-packages(x)), h(commands(x))) (x:BuildStmtS0) : BuildStmtS0(info(x), name(x), type(x), h(inputs(x)), h(supported-vm-packages(x)), h(pkg(x)), h(output(x)), h(assembly(x)), h(external-dependencies(x)), - h(ccfiles(x)), h(ccflags(x)), h(flags(x)), optimize(x)) + h(ccfiles(x)), h(ccflags(x)), h(flags(x)), optimize(x), profile(x), coverage(x)) (x:SyntaxPackagesDefinedInStmtS0) : SyntaxPackagesDefinedInStmtS0(info(x), packages(x), h(filename(x))) (x:VarStmtS0) : VarStmtS0(info(x), name(x), h(value(x))) (x:ForeignPackageParamsStmtS0) : ForeignPackageParamsStmtS0(info(x), package-manager(x), h(projdir(x)), h(entries(x))) diff --git a/compiler/reg-alloc.stanza b/compiler/reg-alloc.stanza index 135bc8e10..cf5aa70e5 100644 --- a/compiler/reg-alloc.stanza +++ b/compiler/reg-alloc.stanza @@ -21,7 +21,8 @@ Constructs that must be handled, and their variants: Op - (AddOp / SubOp / MulOp / AndOp / OrOp / XorOp / NotOp / ShlOp / ShrOp / AshrOp / NegOp / EqOp / NeOp / LtOp / GtOp / LeOp / GeOp / UleOp / UltOp / UgtOp / UgeOp / FlushVMOp / LoadSpecialOp / DivModOp / NoOp / RecordLiveOp / LoadOp / - StoreOp / StoreArgOp / StoreSpecialOp / LoadArgOp / AllocOp / InstanceofOp) + StoreOp / StoreArgOp / StoreSpecialOp / LoadArgOp / AllocOp / InstanceofOp / + IncFunctionCounterOp / IsProfileOp) Branch - (EqOp / NeOp / LtOp / GtOp / LeOp / GeOp / UleOp / UltOp / UgtOp / UgeOp / HasHeapOp / HasStackOp / ArgEqOp) Set @@ -557,6 +558,10 @@ public defstruct FlipOp <: VMOp : (op:VMOp) defmethod print (o:OutputStream, x:FlipOp) : print(o, "flip(%_)" % [op(x)]) +public defstruct IncFunctionCounterOp <: VMOp : (id:Int) +defmethod print (o:OutputStream, x:IncFunctionCounterOp) : + print(o, "incfunctioncounter(%_)" % [id(x)]) + public defstruct SafepointOp <: VMOp : id:Int trace-entry:StackTraceInfo @@ -1014,6 +1019,8 @@ defn load-instructions (function:VMFunc, backend:Backend) : push(Op(SafepointOp(id(e), trace-entry!(e), []), List(), List())) (e:UnreachableIns) : false + (e:IncFunctionCounterIns) : + push(Op(IncFunctionCounterOp(id(e)), List(), List())) val next = to-list $ for s in succs(b) seq : index(blocks[s]) @@ -1412,6 +1419,7 @@ defn num-scratch-registers (i:Ins) : (i:Branch) : match(op(i)) : (op:HasHeapOp) : 4 + (op:IsProfileOp) : 4 (op) : 0 (i:Match) : 2 (i:MethodDispatch) : 0 @@ -1878,6 +1886,7 @@ defn register-assignment (blk:Block, b:Int, backend:Backend) : (e:Branch) : match(op(e)) : (op:HasHeapOp) : ensure-available(list-r0-r1, List()) + (op:IsProfileOp) : ensure-available(list-r0-r1, List()) (op) : false emit(Branch(op(e), map(annotate, xs(e)), killed?(e))) do(free-var, killed(e)) @@ -2837,12 +2846,18 @@ defn assemble (emitter:CodeEmitter, defn cmp-arity (arg:Int, value:Int, eq-op:asm-Op) : val arg-reg = R(call-regs(backend(stubs))[arg]) E $ BreakL(label-table[n(e)], eq-op, arg-reg, INT(value)) + defn cmp-profile-flag () : + val TMP = R0 + E $ LoadL(TMP, M(profile-flag(stubs))) + E $ BreakL(label-table[n(e)], asm-EqOp(), TMP, INT(1)) match(op(e)) : (op:HasStackOp) : cmp-stack-limit(asm-UleOp()) (op:HasHeapOp) : cmp-heap-limit(asm-UleOp()) + (op:IsProfileOp) : + cmp-profile-flag() (op:ArgEqOp) : cmp-arity(arg(op), value(op), asm-EqOp()) (fop:FlipOp) : diff --git a/compiler/repl.stanza b/compiler/repl.stanza index bd1a93ed1..e9cb92f6e 100644 --- a/compiler/repl.stanza +++ b/compiler/repl.stanza @@ -494,7 +494,7 @@ public defn REPL (macro-plugins:Tuple, vprintln("REPL: Compiling EPackage %_ to VMPackage." % [name(p)]) val vmp = within log-time(REPL-LOWERING-AND-COMPILATION, suffix(name(p))) : val lowered = within log-time(REPL-LOWER-TO-UNOPTIMIZED) : - lower-unoptimized(p) + lower-unoptimized(p, false, false) within log-time(REPL-COMPILE-LOWERED) : compile(lowered, false) vprintln("REPL: Done Compiling EPackage %_ to VMPackage." % [name(p)]) diff --git a/compiler/stitcher.stanza b/compiler/stitcher.stanza index 0c8bf64a0..8588a7bfb 100644 --- a/compiler/stitcher.stanza +++ b/compiler/stitcher.stanza @@ -856,6 +856,18 @@ public defn Stitcher (packages:Collection, bindings:Bindings|False, s E $ DefText() E $ Comment("End of Data Tables") + defn emit-heap-statistics-table (code-emitter:CodeEmitter) : + defn E (i:Ins) : emit(code-emitter, i) + E $ Comment("Heap Statistics Table") + E $ DefData() + E $ Label(/heap-statistics(stubs)) + E $ DefLong(to-long(num-concrete-classes)) + for i in 0 to num-concrete-classes do : + E $ DefLong(0L) ; num-uses + E $ DefLong(0L) ; num-bytes + E $ DefText() + E $ Comment("End of Heap Statistics Table") + ;Emit class table defn emit-class-table (code-emitter:CodeEmitter) : defn E (i:Ins) : emit(code-emitter, i) @@ -924,11 +936,13 @@ public defn Stitcher (packages:Collection, bindings:Bindings|False, s E $ DefLabel(string-lbls[filename(info)]) E $ DefInt(line(info)) E $ DefInt(column(info)) + E $ DefLong(-1L when (function(trace-entry) is False) else to-long(function(trace-entry) as Int)) (f:False) : E $ DefLong(0L) E $ DefLong(0L) E $ DefInt(0) E $ DefInt(0) + E $ DefLong(-1L) E $ DefText() E $ Comment("End of File Information Table") @@ -941,6 +955,47 @@ public defn Stitcher (packages:Collection, bindings:Bindings|False, s E $ DefText() E $ Comment("End of String Table for Filenames") + defn emit-profile-flag (code-emitter:CodeEmitter) : + defn E (i:Ins) : emit(code-emitter, i) + E $ Comment("Profile Flag") + E $ DefData() + E $ Label(profile-flag(stubs)) + E $ DefLong(0L) + + ;Emit function info table + defn emit-function-counters-table (code-emitter:CodeEmitter, num:Int) : + defn E (i:Ins) : emit(code-emitter, i) + E $ Comment("Function Counters for Profiling") + E $ DefData() + E $ Label(function-counters(stubs)) + for i in 0 to num do : + E $ DefLong(0L) + + ;Emit function info table + defn emit-function-info-table (code-emitter:CodeEmitter, function-info-table:Tuple, len:Int) : + defn E (i:Ins) : emit(code-emitter, i) + val string-lbls = HashTable-init(unique-id{stubs}) + E $ Comment("Function Info Table for Profiling") + E $ DefData() + E $ Label(function-info(stubs)) + E $ DefLong(to-long(length(function-info-table))) + for fun-info in function-info-table do : + E $ DefLabel(string-lbls[to-string(package(info(fun-info)))]) + match(signature(info(fun-info))) : + (s:String) : E $ DefLabel(string-lbls[s]) + (f:False) : E $ DefLong(0L) + val file-info = info(info(fun-info)) as AbsoluteFileInfo + E $ DefLabel(string-lbls[filename(file-info)]) + E $ DefInt(line(file-info)) + E $ DefInt(column(file-info)) + E $ DefText() + E $ DefData() + for entry in string-lbls do : + val [s, n] = [key(entry), value(entry)] + E $ Label(n) + E $ DefString(s) + E $ DefText() + defn emit-export-table (code-emitter: CodeEmitter) -> False: defn E (i:Ins) : emit(code-emitter, i) @@ -995,6 +1050,17 @@ public defn Stitcher (packages:Collection, bindings:Bindings|False, s ; so they can be dynamically imported by other executables if generate-export-directives-table?(backend(stubs)) : emit-export-table(code-emitter) + emit-heap-statistics-table(code-emitter) + val pkgs = to-tuple $ packages + val info = + if length(pkgs) == 1 : + match(function-info(pkgs[0])) : + (info:Tuple) : info + (info:False) : [] + else : [] + emit-profile-flag(code-emitter) + emit-function-counters-table(code-emitter, length(info)) + emit-function-info-table(code-emitter, info, length(info)) defmethod stubs (this) : stubs defmethod core-fn (this, id:FnId) : diff --git a/compiler/trace-info.stanza b/compiler/trace-info.stanza index 6c99fbbb6..739aeffe3 100644 --- a/compiler/trace-info.stanza +++ b/compiler/trace-info.stanza @@ -20,16 +20,18 @@ public defstruct StackTraceInfo : package:Symbol signature:String|False info:AbsoluteFileInfo|False + function:Int|False with: ( updater => sub-function ) with: constructor => #StackTraceInfo ;Sanity check: Ensure that we don't have signature and info as false. public defn StackTraceInfo (package:Symbol, signature:String|False, - info:AbsoluteFileInfo|False) : + info:AbsoluteFileInfo|False, + function:Int|False = false) : if signature is False and info is False : fatal("StackTraceInfo cannot have both signature and info be false.") - #StackTraceInfo(package, signature, info) + #StackTraceInfo(package, signature, info, function) ;Print in a human-readable way. ;e.g. function myfunction in mypackage at info diff --git a/compiler/utils.stanza b/compiler/utils.stanza index dde8ed207..7d18ba252 100644 --- a/compiler/utils.stanza +++ b/compiler/utils.stanza @@ -5,6 +5,7 @@ defpackage stz/utils : import collections import stz/algorithms import core/sha256 + import stz/trace-info ;============================================================ ;===================== Radix Conversions ==================== @@ -527,3 +528,6 @@ public defstruct MTItem : public defn EmptyMTItem () : MTItem(-1, List()) + +public defstruct FunctionInfo : + info : StackTraceInfo diff --git a/compiler/vm-analyze.stanza b/compiler/vm-analyze.stanza index 044d3dae9..ad7cf893a 100644 --- a/compiler/vm-analyze.stanza +++ b/compiler/vm-analyze.stanza @@ -336,6 +336,7 @@ deftype OperationIns : CommentIns <: OperationIns LiveIns <: OperationIns SafepointIns <: OperationIns + ProfileIns <: OperationIns ;Assumes that the results are loaded from registers in sequential order, ;and that we are sweeping through instructions backwards. @@ -373,6 +374,7 @@ defn do-results (func:Int -> ?, i:OperationIns) : (i:LiveIns) : false (i:CommentIns) : false (i:SafepointIns) : false + (i:ProfileIns) : false defn do-args (func:Int -> ?, i:OperationIns|DestinationIns) : defn f? (y:VMImm|False) : @@ -408,6 +410,7 @@ defn do-args (func:Int -> ?, i:OperationIns|DestinationIns) : (i:MethodDispatchIns) : (f?(ys(i)), f?(zs(i))) (i:UnreachableIns) : false (i:SafepointIns) : false + (i:ProfileIns) : false defn do-dest (f:Int -> ?, i:DestinationIns) : match(i) : diff --git a/compiler/vm-ir.stanza b/compiler/vm-ir.stanza index 62d220218..9566e65df 100644 --- a/compiler/vm-ir.stanza +++ b/compiler/vm-ir.stanza @@ -32,7 +32,8 @@ public val EXTEND-HEAP-FN = 0 public val EXTEND-STACK-FN = 1 public val INIT-CONSTS-FN = 2 public val EXECUTE-TOPLEVEL-COMMAND-FN = 3 -public val NUM-BUILTIN-FNS = 4 +public val PROFILE-STACK-TRACE-FN = 4 +public val NUM-BUILTIN-FNS = 5 ;============================================================ ;================== Design of Instructions ================== @@ -51,6 +52,7 @@ public defstruct VMPackage : extern-defns: Tuple debug-name-table: VMDebugNameTable debug-table: VMDebugInfoTable + function-info: False|Tuple with: (default => false) safepoint-table: VMSafepointTable ;A value that can be accessed directly within an instruction. @@ -312,6 +314,9 @@ public defstruct UnreachableIns <: VMIns public defstruct SafepointIns <: VMIns : id:Int trace-entry: StackTraceInfo with: (as-method => true) +public defstruct ProfileIns <: VMIns : + id : Int + kind : Int public defn trace-entry! (ins:SafepointIns) -> StackTraceInfo : trace-entry(ins) as StackTraceInfo @@ -879,6 +884,8 @@ defmethod print (o:OutputStream, i:VMIns) : val o2 = IndentedStream(o) lnprint(o2, "else : goto %_" % [default(i)]) lnprint(o2, "amb : goto %_" % [amb(i)]) + (i:ProfileIns) : + print(o, "profile(%_, %_)" % [id(i), kind(i)]) defmethod print (o:OutputStream, op:VMOp) : print{o, _} $ match(op) : diff --git a/compiler/vm-normalize.stanza b/compiler/vm-normalize.stanza index 19af3f70a..903800ccb 100644 --- a/compiler/vm-normalize.stanza +++ b/compiler/vm-normalize.stanza @@ -785,6 +785,19 @@ defn normalize (f0:VMFunction, databuffer:DataBuffer, backend:Backend, iotable:I emit(buffer, i) (i:SetIns) : emit(buffer, SetIns(x(i), #L(y(i)))) + (i:ProfileIns) : + if kind(i) == 0 : + val good-lbl = make-label(buffer) + val profile-lbl = make-label(buffer) + load-instruction(Branch0Ins(profile-lbl, good-lbl, IsProfileOp())) + load-instruction(LabelIns(profile-lbl)) + val profile-stack-trace = CodeId(n(iotable, CORE-PROFILE-STACK-TRACE-ID)) + val call-ins = CallIns([], profile-stack-trace, [false-marker(), NumConst(0)], false) + load-instruction(call-ins) + load-instruction(GotoIns(good-lbl)) + load-instruction(LabelIns(good-lbl)) + else : + load-instruction(IncFunctionCounterIns(id(i))) (i) : emit(buffer, i) @@ -916,6 +929,13 @@ with: (printer => true) public defstruct HasStackOp <: VMOp with: (printer => true) +public defstruct IsProfileOp <: VMOp +with: (printer => true) + +public defstruct IncFunctionCounterIns <: VMIns : + id: Int +with: (printer => true) + public defstruct ArgEqOp <: VMOp : arg: Int value: Int diff --git a/compiler/vm-structures.stanza b/compiler/vm-structures.stanza index 3bad507fa..ce034c65c 100644 --- a/compiler/vm-structures.stanza +++ b/compiler/vm-structures.stanza @@ -31,6 +31,12 @@ public lostanza deftype VMState : var safepoint-table: ptr ;(Permanent State) var debug-table: ptr ;(Permanent State) var local-var-table: ptr ;(Permanent State) + var heap-statistics: ptr ;(Permanent State) + var heap-dominator-tree: ptr ;(Variable State) + var profile-flag: ptr + var profile-buffer: ptr ; ptr + function-counters: ptr + function-info: ptr ; ptr var class-table: ptr ;(Permanent State) ;Interpreted Mode Tables var instructions: ptr ;(Permanent State) diff --git a/compiler/vm.stanza b/compiler/vm.stanza index 83517fa64..28a5d9798 100644 --- a/compiler/vm.stanza +++ b/compiler/vm.stanza @@ -371,6 +371,8 @@ public lostanza defn VirtualMachine (dylibs:ref, backend heap.iterate-roots = addr(vm-iterate-roots) heap.iterate-references-in-stack-frames = addr(vm-iterate-references-in-stack-frames) + vmstate.profile-flag = call-c clib/stz_malloc(8) + ;Initialize sighandler to false to indicate no handler on initialization. vmstate.sig-handler = false-marker() diff --git a/core/core.stanza b/core/core.stanza index a2a602acd..b03e7fe21 100644 --- a/core/core.stanza +++ b/core/core.stanza @@ -188,6 +188,17 @@ protected lostanza deftype StackTraceTableEntry : lbl: ptr record: StackTraceRecord +protected lostanza deftype FunctionInfoTable : + length: long + entries: FunctionInfoEntry ... + +protected lostanza deftype FunctionInfoEntry : + package: ptr + name: ptr + file: ptr + line: int + column: int + lostanza deftype ClassDescriptor : case:int num-base-bytes:int @@ -209,36 +220,101 @@ protected lostanza deftype ArrayRecord : num-item-roots:int roots:int ... +lostanza deftype HeapStatistics : + length: long + entries: HeapStatistic ... + +protected lostanza deftype HeapStatistic : + var num-uses:long + var num-bytes:long + ;The first fields in VMState are used by the core library ;in both compiled and interpreted mode. The last fields ;are used only in compiled mode. ;Permanent state changes in-between each code load. ;Variable state changes in-between each boundary change. -protected lostanza deftype VMState : - ;Compiled and Interpreted Mode - global-offsets: ptr ;(Permanent State) - global-mem: ptr ;(Permanent State) - var sig-handler: long ;(Permanent State) - var current-coroutine-ptr: ptr ;[TODO] Change to long to represent reference. - var stepping-coroutine-ptr: ptr ;[TODO] Change to long to represent reference. - const-table: ptr ;(Permanent State) - const-mem: ptr ;(Permanent State) - data-offsets: ptr ;(Permanent State) - data-mem: ptr ;(Permanent State) - code-offsets: ptr ;(Permanent State) - registers: ptr ;(Permanent State) - system-registers: ptr ;(Permanent State) - var heap: Heap ;(Variable State) - safepoint-table: ptr ;(Variable State) - debug-table: ptr ;(Variable State) - local-var-table: ptr ;(Variable State) - ;Compiled Mode Tables - class-table: ptr - global-root-table: ptr - stackmap-table: ptr> - stack-trace-table: ptr - extern-table: ptr - extern-defn-table: ptr +#if-defined(BOOTSTRAP) : + + protected lostanza deftype VMState : + ;Compiled and Interpreted Mode + global-offsets: ptr ;(Permanent State) + global-mem: ptr ;(Permanent State) + var sig-handler: long ;(Permanent State) + var current-coroutine-ptr: ptr ;[TODO] Change to long to represent reference. + var stepping-coroutine-ptr: ptr ;[TODO] Change to long to represent reference. + const-table: ptr ;(Permanent State) + const-mem: ptr ;(Permanent State) + data-offsets: ptr ;(Permanent State) + data-mem: ptr ;(Permanent State) + code-offsets: ptr ;(Permanent State) + registers: ptr ;(Permanent State) + system-registers: ptr ;(Permanent State) + var heap: Heap ;(Variable State) + safepoint-table: ptr ;(Variable State) + debug-table: ptr ;(Variable State) + local-var-table: ptr ;(Variable State) + ;Compiled Mode Tables + class-table: ptr + global-root-table: ptr + stackmap-table: ptr> + stack-trace-table: ptr + extern-table: ptr + extern-defn-table: ptr + + lostanza defn initialize-dominator-tree () -> ref : + return false + +#else: + + protected lostanza deftype HeapDominator : + var roots : ptr + var sizes : ptr + var addrs : ptr + var offs : ptr + var heap : ptr + + protected lostanza deftype VMState : + ;Compiled and Interpreted Mode + global-offsets: ptr ;(Permanent State) + global-mem: ptr ;(Permanent State) + var sig-handler: long ;(Permanent State) + var current-coroutine-ptr: ptr ;[TODO] Change to long to represent reference. + var stepping-coroutine-ptr: ptr ;[TODO] Change to long to represent reference. + const-table: ptr ;(Permanent State) + const-mem: ptr ;(Permanent State) + data-offsets: ptr ;(Permanent State) + data-mem: ptr ;(Permanent State) + code-offsets: ptr ;(Permanent State) + registers: ptr ;(Permanent State) + system-registers: ptr ;(Permanent State) + var heap: Heap ;(Variable State) + safepoint-table: ptr ;(Variable State) + debug-table: ptr ;(Variable State) + local-var-table: ptr ;(Variable State) + heap-statistics: ptr ;(Variable State) + var dom: ptr ;(Variable State) + profile-flag: ptr + var profile-buffer: ptr + function-counters: ptr + function-info: ptr + ;Compiled Mode Tables + class-table: ptr + global-root-table: ptr + stackmap-table: ptr> + stack-trace-table: ptr + extern-table: ptr + extern-defn-table: ptr + + lostanza defn initialize-dominator-tree () -> ref : + val vms:ptr = call-prim flush-vm() + val dom = (call-c clib/malloc(sizeof(HeapDominator))) as ptr + dom.roots = LSLongVector() + dom.sizes = LSLongVector() + dom.addrs = LSLongVector() + dom.offs = LSLongVector() + dom.heap = LSLongVector() + vms.dom = dom + return false lostanza deftype ExternTable : length: long @@ -518,12 +594,23 @@ lostanza defn print-stack-trace (stack:ref) -> ref : val frame = buffer.items[i] as ptr val entry = stack-trace-record(frame, stack-trace-table) if entry != null : + call-c clib/fprintf(current-err, "lbl 0x%lx: " frame.return) print-record(entry) ;Free the buffer free(buffer) return false +public lostanza defn print-stack-trace-table () -> ref : + val vms:ptr = call-prim flush-vm() + val trace-table = vms.stack-trace-table + for (var i:int = 0, i < trace-table.length, i = i + 1) : + val e = addr(trace-table.entries[i]) + call-c clib/fprintf(current-err, "lbl 0x%lx: " e.lbl) + val r = addr(e.record) + call-c clib/printf("P %s S %s B %s F %s L %d C %d F %d\n", r.package, r.signature, r.base, r.file, r.line, r.column, r.function) + return false + lostanza defn print-record (record:ptr) -> ref : call-c clib/fprintf(current-err, " in %s", record.package) if record.signature != null : @@ -1614,9 +1701,9 @@ lostanza defn iterate-roots (f:ptr<((ptr, ptr) -> ref)>, return [vms.heap.iterate-roots](f, vms) ;Call f on all references stored in the object pointed to by p. -lostanza defn iterate-references (p:ptr, - f:ptr<((ptr, ptr) -> ref)>, - vms:ptr) -> ref : +protected lostanza defn iterate-references (p:ptr, + f:ptr<((ptr, ptr) -> ref)>, + vms:ptr) -> ref : ;Retrieve the object's tag. val tag = get-tag(p) ;Fast path using fast descriptor table. @@ -2632,8 +2719,8 @@ public lostanza defn clear (start:ptr, size:long) -> ptr : return call-c clib/memset(start, 0, size) ;Call f on all root pointers. -lostanza defn core-iterate-roots (f:ptr<((ptr, ptr) -> ref)>, - vms:ptr) -> ref : +protected lostanza defn core-iterate-roots (f:ptr<((ptr, ptr) -> ref)>, + vms:ptr) -> ref : ;Scan globals val globals = vms.global-mem as ptr val roots = vms.global-root-table @@ -4192,6 +4279,20 @@ protected val uninitialized = new Uninitialized lostanza val LS-NL:ptr = "\n" val NL = "\n" +;============================================================ +;=================== Initialize Profiler ==================== +;============================================================ + +#if-defined(BOOTSTRAP) : + lostanza defn initialize-profiler () -> ref : + return false +#else : + lostanza defn initialize-profiler () -> ref : + val vms:ptr = call-prim flush-vm() + [vms.profile-flag] = 0L + vms.profile-buffer = LSLongVector(1 << 20) + return false + ;============================================================ ;=================== Initialize Remaining =================== ;============================================================ @@ -4203,6 +4304,8 @@ initialize-gc-notifiers() initialize-gc-statistics() initialize-liveness-handlers() initialize-symbol-table() +initialize-dominator-tree() +initialize-profiler() ;================================================================================ ;========================== End of Boot Sequence ================================ @@ -9538,14 +9641,16 @@ public defn all? (pred?: T -> True|False, xs:Seqable) -> True|False : for xs-seq in xs do-seq : let loop () : if empty?(xs-seq) : true - else : pred?(next(xs-seq)) and loop() + else if pred?(next(xs-seq)) : loop() + else : false public defn all? (pred?: (T,S) -> True|False, xs:Seqable, ys:Seqable) -> True|False : for xs-seq in xs do-seq : for ys-seq in ys do-seq : let loop () : if empty?(xs-seq) or empty?(ys-seq) : true - else : pred?(next(xs-seq), next(ys-seq)) and loop() + else if pred?(next(xs-seq), next(ys-seq)) : loop() + else : false public defn all? (pred?: (T,S,U) -> True|False, xs:Seqable, ys:Seqable, zs:Seqable) -> True|False : for xs-seq in xs do-seq : @@ -9553,7 +9658,8 @@ public defn all? (pred?: (T,S,U) -> True|False, xs:Seqable, ys:Seq for zs-seq in zs do-seq : let loop () : if empty?(xs-seq) or empty?(ys-seq) or empty?(zs-seq) : true - else : pred?(next(xs-seq), next(ys-seq), next(zs-seq)) and loop() + else if pred?(next(xs-seq), next(ys-seq), next(zs-seq)) : loop() + else : false public defn none? (xs:Seqable, pred?: T -> True|False) -> True|False : none?(pred?,xs) @@ -11147,6 +11253,45 @@ public deftype TypeObject public defmulti typeof? (x, t:TypeObject) -> True|False public defmulti name (t:TypeObject) -> String +;============================================================ +;====================== Profiling =========================== +;============================================================ + +#if-not-defined(BOOTSTRAP) : + + public var last-profiler-time:Long = 0L + + ;Called from app to record stack trace as return addresses in buffer + ;Demarcate end of each coroutine stack with -2 and end trace with -1 + lostanza defn profile-stack-trace () -> ref : + val vms:ptr = call-prim flush-vm() + [vms.profile-flag] = 2L + val buffer = vms.profile-buffer + add(buffer, call-c clib/current_time_ms()) + add(buffer, -2L) ; time marker + labels : + begin : + goto loop(current-coroutine) + loop (co:ref) : + val stack = co.stack + val end-sp = stack.stack-pointer + labels : + begin : goto loop-frame(stack.frames) + loop-frame (sp:ptr) : + if sp.return > 0L : + add(buffer, sp.return) + if sp < end-sp : ;Continue if we're not at the end of the stack + val map-index = sp.liveness-map + val stackmap = vms.stackmap-table[map-index] + goto loop-frame(sp + stackmap.size) + add(buffer, -2L) ;End coroutine stack marker + match(co.parent) : + (p:ref) : goto loop(p) + (p:ref) : () + add(buffer, -1L) ;End stack trace marker + [vms.profile-flag] = 0L + return false + ;############################################################ ;################## Math Package ############################ ;############################################################ diff --git a/core/heap-analyzer.stanza b/core/heap-analyzer.stanza new file mode 100644 index 000000000..93ea10f39 --- /dev/null +++ b/core/heap-analyzer.stanza @@ -0,0 +1,430 @@ +defpackage heap-analyzer : + import core + import collections + import core/long-vector + +;;; UTILITIES + +defn scatter (src:Seqable, idx:Tuple) -> Tuple : + val dst = Array(length(idx)) + for (x in src, i in 0 to false) do : dst[idx[i]] = x + to-tuple(dst) + +defn gather (src:Tuple, idx:Seqable) -> Seq : + seq({ src[_] }, idx) + +defn gather (src:IndexedCollection, idx:Seqable) -> Seq : + seq({ src[_] }, idx) + +lostanza defn clear (v:ptr) -> ref : + v.length = 0 + return false + +lostanza defn class-name (x:ref) -> ref : + var res:ref + if x.value == -1 : + res = String("root") + else : + res = String(class-name(x.value)) + return res + +#if-defined(BOOTSTRAP) : + + public defn heap-dominator-tree (filename:String, max-depth:Int = MAX-INT, min-size:Int = 0) : false + public defn analyze-heap () -> Long : 0L + +#else : + + public defstruct HeapStat : + tag : Int ; type + name : String ; name of type + num-uses : Long ; num references in heap + num-bytes : Long ; num bytes occupied in heap + with: + printer => true + + lostanza defn collect-heap-stats + (p:ptr, tag:int, size:long, vms:ptr) -> ref : + val stat = addr(vms.heap-statistics.entries[tag]) + stat.num-uses = stat.num-uses + 1L + stat.num-bytes = stat.num-bytes + size + return false + + ; run gc, collect heap stats while walking each object in nursery and heap + lostanza defn do-analyze-heap (stats:ref>) -> ref : + val vms:ptr = call-prim flush-vm() + run-garbage-collector() + val num-classes = vms.heap-statistics.length + clear(addr(vms.heap-statistics.entries), num-classes * sizeof(core/HeapStatistic)) + iterate-objects(vms.heap.start, vms.heap.old-objects-end, vms, addr(collect-heap-stats)) + val nursery = core/nursery-start(addr(vms.heap)) + iterate-objects(nursery, vms.heap.top, vms, addr(collect-heap-stats)) + var tot-size:long = 0L + for (var i:int = 0, i < num-classes, i = i + 1) : + val heap-stat = addr(vms.heap-statistics.entries[i]) + val num-uses = heap-stat.num-uses + if num-uses > 0L : + tot-size = tot-size + heap-stat.num-bytes + add(stats, HeapStat(new Int{i as int}, String(class-name(i)), new Long{num-uses}, new Long{heap-stat.num-bytes})) + return new Long{ tot-size } + + public defstruct HeapStats : + total-size : Long + entries : Tuple + + ; public interface to heap analyzer collecting and printing out stats + public defn analyze-heap () -> HeapStats : + val stats = Vector() + val size = do-analyze-heap(stats) + val res = reverse(to-list(lazy-qsort(num-bytes, stats))) + HeapStats(size, to-tuple(res)) + + public defn dump (stats:HeapStats) : + println("Heap size %_" % [total-size(stats)]) + var max-bytes-size = reduce(max, length("Size"), seq({ length(to-string(num-bytes(_))) }, entries(stats))) + var max-perc-size = 6 + var max-uses-size = reduce(max, length("Uses"), seq({ length(to-string(num-uses(_))) }, entries(stats))) + defn pad (s:String, n:Int) -> String : append-all(cat(repeatedly({ " " }, (n - length(s))), [s])) + defn pad0 (s:String, n:Int) -> String : append-all(cat([s], repeatedly({ "0" }, (n - length(s))))) + println(" %_ %_ %_: %_" % [pad("Size", max-bytes-size), pad("Perc", max-perc-size), pad("Use", max-uses-size), "Type"]) + for hc in entries(stats) do : + val pt = to-int(to-double(num-bytes(hc)) * 10000.0 / to-double(total-size(stats))) + val p = string-join $ [pt / 100, ".", pad0(to-string(pt - ((pt / 100) * 100)), 2)] + println(" %_ %_%% %_: %_" % [ + pad(to-string(num-bytes(hc)), max-bytes-size), pad(p, max-perc-size), pad(to-string(num-uses(hc)), max-uses-size), name(hc)]) + + ;;; INTERFACE TO STANZA MEMORY SYSTEM + + lostanza defn addrs (dom:ptr) -> ptr : + return dom.addrs as ptr + + lostanza defn heap (dom:ptr) -> ptr : + return dom.heap as ptr + + lostanza defn sizes (dom:ptr) -> ptr : + return dom.sizes as ptr + + lostanza defn roots (dom:ptr) -> ptr : + return dom.roots as ptr + + lostanza defn offs (dom:ptr) -> ptr : + return dom.offs as ptr + + lostanza defn collect-object-address-and-size + (p:ptr, tag:int, size:long, vms:ptr) -> ref : + add(addrs(vms.dom), p as long) + add(sizes(vms.dom), size as long) + return false + + lostanza defn collect-object-contents + (p:ptr, tag:int, size:long, vms:ptr) -> ref : + add(offs(vms.dom), heap(vms.dom).length) + add(heap(vms.dom), tag as long) + val idx = heap(vms.dom).length + add(heap(vms.dom), 0L) ; place holder + core/iterate-references(p, addr(do-collect-object-contents), vms) + heap(vms.dom).items[idx] = heap(vms.dom).length - idx - 1 + return false + + lostanza defn do-collect-object-contents (ref:ptr, vms:ptr) -> ref : + ;Retrieve the value at the given heap pointer. + val v = [ref] + ;Is this a reference to a Stanza heap object? + val tagbits = v & 7L + if tagbits == 1L : + ;Remove the tag bits to retrieve the object pointer. + val p = (v - 1) as ptr + add(heap(vms.dom), addr-to-id(addrs(vms.dom), p as long) + 1) + return false + + public lostanza defn register-all-roots (vms:ptr) -> ref : + core/core-iterate-roots(addr(register-root-reference), vms) + register-stack-roots(vms) + return false + + public lostanza defn register-stack-roots (vms:ptr) -> ref : + var stack:ptr = vms.heap.stacks + while stack != null : + iterate-references-in-stack-frames(stack, addr(register-root-reference), vms) + stack = stack.tail + return false + + public lostanza defn register-root-reference (ref:ptr, vms:ptr) -> ref : + val v = [ref] + val tagbits = v & 7L ; heap object? + if tagbits == 1L : + val p = (v - 1) as ptr ; remove tag bits to retrieve object pointer + add(roots(vms.dom), p as long) + return false + + lostanza defn iterate-objects + (pstart:ptr, pend:ptr, vms:ptr, + f:ptr<((ptr, int, long, ptr) -> ref)>) -> ref : + var p:ptr = pstart + while p < pend : + val tag = [p] as int + val class = vms.class-table[tag].record + var size:long = 0L + if class.item-size == 0 : + size = object-size-on-heap(class.size) + else : + val class = class as ptr + val array = p as ptr + val len = array.slots[0] + val base-size = class.base-size + val item-size = class.item-size + val my-size = base-size + item-size * len + size = object-size-on-heap(my-size) + [f](p, tag, size, vms) + p = p + size + return false + + ;; Look up offset into sorted list of object addresses using binary search + lostanza defn addr-to-id (xs:ptr, x:long) -> long : + var res:long = -1L + labels : + begin: goto loop(0L, xs.length) + loop (start:long, end:long) : + if end > start : + val center = (start + end) >> 1 + val xc = xs.items[center] + if x == xc : res = center + else if x < xc : goto loop(start, center) + else : goto loop(center + 1L, end) + return res + + ;;; LowFlatObject -- create flat and packed version of roots and objects + ;;; -- stores tag, num-refs, refs for each object + ;;; -- also has extra root root object with ref per root + + lostanza deftype LowFlatObjects : + var sizes : ptr ; static sizes of objects + var offs : ptr ; offsets to inlined objects in heap + var heap : ptr ; | type | len | ids ... | ... + + lostanza deftype FlatObjects <: IndexedCollection&Lengthable : + value : ptr + + lostanza defn FlatObjects + (sizes:ptr, offs:ptr, heap:ptr) -> ref : + val lfo = call-c clib/stz_malloc(sizeof(LowFlatObjects)) as ptr + lfo.sizes = sizes + lfo.offs = offs + lfo.heap = heap + return new FlatObjects{ lfo } + + lostanza defmethod length (xs:ref) -> ref : + return new Int{xs.value.offs.length} + + lostanza defn offset (xs:ref, id:ref) -> ref : + return new Int{xs.value.offs.items[id.value] as int} + + lostanza defmethod get (xs:ref, idx:ref) -> ref : + return new Int{xs.value.heap.items[idx.value] as int} + + ; for some reason can't name this method get like in stanza runtime + defn get-all (xs:FlatObjects, indices:Range) -> Seq : + seq({ xs[_] }, indices) + + defn tag-of (xs:FlatObjects, id:Int) -> Int : + xs[offset(xs, id)] + + lostanza defn size-of (xs:ref, id:ref) -> ref : + return new Int{ xs.value.sizes.items[id.value] as int } + + defn sizes (objs:FlatObjects) -> Seq : + seq(size-of{objs, _}, 0 to length(objs)) + + defn refs (objs:FlatObjects, id:Int) -> Seqable : + val off = offset(objs, id) ; base + val num-refs = objs[off + 1] + val refs-off = off + 2 + get-all(objs, refs-off to (refs-off + num-refs)) + + ;; Pack roots / heap into FlatObjects + lostanza defn FlatObjects () -> ref : + call-c clib/printf("GC...\n") + run-garbage-collector() + val vms:ptr = call-prim flush-vm() + val dom = vms.dom + clear(offs(dom)) + clear(sizes(dom)) + clear(heap(dom)) + ;; get all roots + register-all-roots(vms) + call-c clib/printf("FOUND %d ROOTS...\n", roots(dom).length) + ;; get sizes and addresses of objects on heap + add(sizes(dom), roots(dom).length as long) ; dummy root object + call-c clib/printf("COLLECT HEAP %lx OBJECT ADDRESSES AND SIZES...\n", vms.heap.start) + iterate-objects(vms.heap.start, vms.heap.old-objects-end, vms, addr(collect-object-address-and-size)) + val nursery = core/nursery-start(addr(vms.heap)) + call-c clib/printf("COLLECT NURSERY %lx OBJECT ADDRESSES AND SIZES...\n", nursery) + iterate-objects(nursery, vms.heap.top, vms, addr(collect-object-address-and-size)) + call-c clib/printf("FOUND %d OBJECTS...\n", addrs(dom).length) + ;; build heap data translated to object ids using addresses and binary search + add(offs(dom), 0L) ; first root object + add(heap(dom), -1L) ; dummy root object tag + add(heap(dom), roots(dom).length as long) + call-c clib/printf("CONVERTING ROOT ADDRESSES TO IDS...\n") + for (var i:int = 0, i < roots(dom).length, i = i + 1) : + add(heap(dom), addr-to-id(addrs(dom), roots(dom).items[i]) + 1) ; point to roots + call-c clib/printf("PACKING HEAP DATA...\n") + iterate-objects(vms.heap.start, vms.heap.old-objects-end, vms, addr(collect-object-contents)) + call-c clib/printf("PACKING NURSERY DATA...\n") + iterate-objects(nursery, vms.heap.top, vms, addr(collect-object-contents)) + clear(addrs(dom)) + clear(roots(dom)) + call-c clib/printf("DONE...\n") + return FlatObjects(sizes(dom), offs(dom), heap(dom)) + + ;;; FlatIdObjects + + ;; Permutation wrapper of flat-objects + defstruct FlatIdObjects : + order : Tuple + reorder : Tuple + objs : FlatObjects + with: + printer => true + + defn sizes (o:FlatIdObjects) -> Seq : + gather(to-tuple(sizes(objs(o))), order(o)) + + defn length (ios:FlatIdObjects) -> Int : + length(objs(ios)) + + defn nexts (fobjs:FlatIdObjects) -> Tuple> : + val objs = objs(fobjs) + to-tuple $ for id in order(fobjs) seq : + to-list $ seq({ reorder(fobjs)[_] }, refs(objs, id)) + + defn prevs (nexts:Tuple>) -> Tuple> : + val prevs = Array>(length(nexts), List()) + for (next in nexts, id in 0 to false) do : + for r in next do : + prevs[r] = cons(id, prevs[r]) + to-tuple $ prevs + + defn objects-to-id-objects (objs:FlatObjects) -> FlatIdObjects : + FlatIdObjects(to-tuple $ (0 to length(objs)), to-tuple $ (0 to length(objs)), objs) + + ;;; DOMINATORS + + ;; find depth first order of objects + defn depth-first (ios:FlatIdObjects) -> FlatIdObjects : + val nexts = nexts(ios) + val visited? = Array(length(ios), false) + val order0 = Vector() + let loop (idx:Int = 0) : + if not visited?[idx] : + visited?[idx] = true + for nidx in nexts[idx] do : loop(nidx) + add(order0, idx) + val missing = filter({ not visited?[_] }, 0 to length(visited?)) + val order = to-tuple $ cat(missing, order0) + FlatIdObjects(to-tuple $ order, scatter(0 to length(order), to-tuple(order)), objs(ios)) + + ; fast dominators algorithm assuming depth-first order + defn idom (num:Int, prevs:Tuple>) -> Tuple : + val doms = Array(num, -1) + val start-id = num - 1 + doms[start-id] = start-id + defn intersect (b1:Int, b2:Int) -> Int : + let loop (finger1:Int = b1, finger2:Int = b2) : + if finger1 != finger2 : + val finger1 = let iter (finger1:Int = finger1) : + if finger1 < finger2 : iter(doms[finger1]) + else : finger1 + val finger2 = let iter (finger2:Int = finger2) : + if finger2 < finger1 : iter(doms[finger2]) + else : finger2 + loop(finger1, finger2) + else : + finger1 + let loop () : + let iter (b : Int = start-id - 1, changed? : True|False = false) : + if b >= 0 : + val new-idom = let find (idom:Int = -1, ps:List = prevs[b]) : + if empty?(ps) : + idom + else : + val p = head(ps) + val nxt-idom = + if doms[p] != -1 : + if idom == -1 : p + else : intersect(p, idom) + else : idom + find(nxt-idom, tail(ps)) + val changed? = doms[b] != new-idom + doms[b] = new-idom + iter(b - 1, changed?) + else : + loop() when changed? + to-tuple $ doms + + defn calc-sizes (ios:FlatIdObjects, doms:Tuple) -> Array : + val tot-sizes = to-array $ sizes(ios) + val len = length(ios) + for i in 0 to (len - 1) do : + if doms[i] >= 0 : + tot-sizes[doms[i]] = tot-sizes[doms[i]] + tot-sizes[i] + tot-sizes + + defn print-xml + (s:FileOutputStream, id-objs:FlatIdObjects, sizes:Array, + nexts:Tuple>, doms:Tuple, max-depth:Int, min-size:Int) : + val objs = objs(id-objs) + defn children (doms:Tuple) -> Tuple> : + val children = to-tuple $ repeatedly({ Vector() }, length(nexts)) + for (dom in doms, id in 0 to false) do : + add(children[dom], id) when (dom >= 0 and dom != id) + map(to-tuple, children) + defn stringify (s:String) -> String : + replace(s, "&", "A") + defn P (n:Int, str:Printable) : + for i in 0 to (n * 2) do : print(s, " ") + println(s, str) + val kiddies = children(doms) + let walk (idx:Int = length(doms) - 1, depth:Int = 0) : + if depth < max-depth : + val id = order(id-objs)[idx] + val name = stringify(class-name(tag-of(objs, id))) + P(depth, "<%_ RETAINED=\"%_\" STATIC=\"%_\">" % [name, sizes[idx], size-of(objs, id)]) + val childs = reverse $ to-list $ qsort({ sizes[_] }, filter({ sizes[_] >= min-size }, kiddies[idx])) + for child in childs do : + walk(child, depth + 1) + P(depth, "" % [name]) + + public defn heap-dominator-tree (filename:String, max-depth:Int = MAX-INT, min-size:Int = 0) : + val objs = FlatObjects() + val id-objs0 = objects-to-id-objects(objs) + val id-objs = depth-first(id-objs0) + val nxts = nexts(id-objs) + val doms = idom(length(id-objs), prevs(nxts)) + val sizes = calc-sizes(id-objs, doms) + ; print-id-object-stats(objs, to-tuple $ gather(sizes, reorder(id-objs))) + val s = FileOutputStream(filename) + print-xml(s, id-objs, sizes, nxts, doms, max-depth, min-size) + close(s) + + ; heap-dominator-tree("sizes.xml") + + ; defn id-print-guts (id:Int, tag:Int, refs:Seqable) : + ; print("%_ = {%_ %_}" % [id, class-name(tag), to-tuple $ refs]) + ; + ; defn print-id-object-guts (objs:FlatObjects) -> False : + ; for id in 0 to length(objs) do : + ; id-print-guts(id, tag-of(objs, id), refs(objs, id)) + ; println("") + ; + ; defn id-print-stat (id:Int, tag:Int, tot-size:Int, size:Int) : + ; print("%_ = {%_ %_ %_}" % [id, class-name(tag), size, tot-size]) + ; + ; defn print-id-object-stats (objs:FlatObjects, tot-sizes:Tuple) -> False : + ; val ids = reverse $ to-list $ qsort({ tot-sizes[_] }, 0 to length(objs)) + ; for (id in ids, i in 0 to false) do : + ; val tot-size = tot-sizes[id] + ; if tot-size > 0 : + ; id-print-stat(id, tag-of(objs, id), tot-size, size-of(objs, id)) + ; println("") + diff --git a/core/long-vector.stanza b/core/long-vector.stanza index 062c15d49..5b8282276 100644 --- a/core/long-vector.stanza +++ b/core/long-vector.stanza @@ -1,6 +1,5 @@ defpackage core/long-vector : import core - import collections public lostanza deftype LSLongVector : var capacity: int @@ -39,4 +38,4 @@ lostanza defn ensure-capacity (v:ptr, new-capacity:int) -> int : while c < new-capacity : c = c << 1 v.capacity = c v.items = realloc(v.items, c * sizeof(long)) - return 0 \ No newline at end of file + return 0 diff --git a/core/profiler.stanza b/core/profiler.stanza new file mode 100644 index 000000000..c6d8cd42c --- /dev/null +++ b/core/profiler.stanza @@ -0,0 +1,340 @@ +;See License.txt for details about licensing. +#use-added-syntax(fastio-serializer) +defpackage profiler : + import core + import collections + import core/stack-trace + import core/long-vector + import stz/fastio-buffer + import stz/fastio-runtime + +public defstruct ProfileInfo : + id : Int + package : String + name : String + info : FileInfo + count : Long +with: + printer => true + +defstruct ProfileStackIdTrace : + msecs : Int + ids : Tuple + +public defstruct ProfileResult : + info : Tuple + id-traces : Tuple + msecs : Int +with: + printer => true + +defmethod print (o:OutputStream, c:ProfileInfo) : + print(o, "ProfileInfo(%_, %_, %_, %_)" % [id(c), package(c), name(c), info(c), count(c)]) + +public defn serialize-profile-result (filename:String, res:ProfileResult) -> False : + write-to-file(filename, + ProfileResultSerializer(), + serialize-profileresult, + res) + +public defn deserialize-profile-result (filename:String) -> ProfileResult : + read-from-file(filename, + ProfileResultSerializer(), + deserialize-profileresult) + +defserializer ProfileResultSerializer () : + entry-points: (profileresult) + + include "../compiler/serializer-primitives.spec" + + deftype profileinfo (ProfileInfo) : + id:int + package:string + name:string + info:info + count:long + + deftype profilestackidtrace (ProfileStackIdTrace) : + msecs:int + ids:tuple(int) + + deftype profileresult (ProfileResult) : + info:tuple(profileinfo) + id-traces:tuple(profilestackidtrace) + msecs:int + + + ;Fast stack trace record lookup +lostanza defn stack-trace-record (ret:long, trace-table:ptr, trace-table-table:ref>) -> ptr : + val idx = get?(trace-table-table, new Long{ret}, new Int{-1}) + if idx.value == -1 : + return null + else : + val entry = addr(trace-table.entries[idx.value]) + return addr(entry.record) + +;Fast stack trace record table construction +lostanza defn build-stack-trace-record-table (trace-table:ptr) -> ref> : + val tab = HashTable() + for (var i:int = 0, i < trace-table.length, i = i + 1) : + val entry = addr(trace-table.entries[i]) + set(tab, new Long{entry.lbl as long}, new Int{i}) + return tab + +#if-defined(BOOTSTRAP) : + + defn collect-profiling (msecs:Int) -> ProfileResult : ProfileResult([], [], 100) + defn collect-coverage () -> Tuple : [] + public defn clear-profiling () : false + public defn clear-coverage () : false + defn do-start-sample-profiling (num-usecs:Int) : false + defn do-stop-sample-profiling () -> Int : 100 + +#else : + + ;Remove empty stack traces + defn prune-traces (traces:Vector>) -> List> : + to-list $ seq(unique, filter({ not empty?(_) }, traces)) + + ;Count all functions references in stack traces where traces have function ids in them + defn count-traces (len:Int, traces:Vector) -> Array : + val counts = Array(len, 0L) + for trace in traces do : + for id in ids(trace) do : + counts[id] = counts[id] + 1L + counts + + ;Split stack trace buffer using delimiter + defn split-buffer (buffer:Tuple, delimiter:Long) -> Vector> : + val res = Vector>() + val chunk = Vector() + for id in buffer do : + if id == delimiter : + add(res, to-tuple(chunk)) + clear(chunk) + else : + add(chunk, id) + res + + defstruct ProfileRawStackTrace : + elapsed-time : Long + return-addresses : Tuple + + ;Reconsititue full stack traces by splitting up chunks and ordering them properly + defn build-stack-traces (buffer:Tuple) -> Tuple : + val buffers = split-buffer(buffer, -1L) + ;same as: + ; to-tuple $ for trace in buffers seq : + ; to-tuple $ reverse $ to-list $ cat-all $ seq(reverse, split-buffer(trace, -2L)) + to-tuple $ for (trace in buffers, k in 0 to false) seq : + val res = Vector() + val chunks = split-buffer(trace, -2L) ;split full stack trace into coroutine stack traces + val elapsed = chunks[0][0] + for i in (length(chunks) - 1) to 0 by -1 do : + val chunk = chunks[i] + for j in 0 to length(chunk) do : + add(res, chunk[j]) + ProfileRawStackTrace(elapsed, to-tuple(res)) + + ;Create function id traces and function info results after profiling + lostanza defn collect-profiling (specd-msecs:ref) -> ref : + val vms:ptr = call-prim flush-vm() + val info:ptr = vms.function-info + val counters:ptr = vms.function-counters + val buffer:ptr = vms.profile-buffer as ptr + val id-traces = Vector() + val infos = Vector() + + if info.length > 0L : + + val tab = build-stack-trace-record-table(vms.stack-trace-table) ;For fast lookup of return addresses + val buffer* = Vector() + + for (var i:long = 0, i < buffer.length, i = i + 1) : + add(buffer*, new Long{buffer.items[i]}) + + val traces = build-stack-traces(to-tuple(buffer*)) ;Order and concatenate stack trace chunks + + var last_global_elapsed_time:long = 0L + + ;Lookup stack trace records and then translate into function ids + for (var i:long = 0, i < length(traces).value, i = i + 1) : + val trace = get(traces, new Int{i as int}) + val global-elapsed = elapsed-time(trace) + var msecs:int = 0 + if last_global_elapsed_time == 0L : ; first trace? + msecs = specd-msecs.value + else : + msecs = (global-elapsed.value - last_global_elapsed_time) as int + last_global_elapsed_time = global-elapsed.value + val trace* = Vector() + val addrs = return-addresses(trace) + for (var j:long = 1, j < length(addrs).value, j = j + 1) : + val ret = get(addrs, new Int{j as int}) + val entry = stack-trace-record(ret.value, vms.stack-trace-table, tab) + if entry != null : + val id = entry.function + if id != -1L : + add(trace*, new Int{id as int}) + add(id-traces, ProfileStackIdTrace(new Int{msecs}, to-tuple(trace*))) + + val len:long = info.length + val counts = count-traces(new Int{len as int}, id-traces) + + ;Collect all FunctionInfos corresponding to ids that occurred on stack at least once + for (var i:long = 0L, i < len, i = i + 1) : + val entry = info.entries[i] + val fi = FileInfo(String(entry.file), new Int{entry.line}, new Int{entry.column}) + val count = get(counts, new Int{i as int}) + if count.value > 0L : + val package = String(entry.package) + val name = String(entry.name) + val ci = ProfileInfo(new Int{i as int}, package, name, fi, count) + add(infos, ci) + + return ProfileResult(to-tuple(infos), to-tuple(id-traces), specd-msecs) + + ;Collect all coverage information from all functions + lostanza defn collect-coverage () -> ref> : + val vms:ptr = call-prim flush-vm() + val info:ptr = vms.function-info + val counters:ptr = vms.function-counters + val len:long = info.length + val res = Vector() + for (var i:long = 0L, i < len, i = i + 1) : + val entry = info.entries[i] + val fi = FileInfo(String(entry.file), new Int{entry.line}, new Int{entry.column}) + val count = new Long{counters[i]} + val package = String(entry.package) + val name = String(entry.name) + val ci = ProfileInfo(new Int{i as int}, package, name, fi, count) + add(res, ci) + return res + + ;Clear profiling state ready for next run + public lostanza defn clear-profiling () -> ref : + val vms:ptr = call-prim flush-vm() + [vms.profile-flag] = 0L + vms.profile-buffer.length = 0 + return false + + ;Clear coverage state ready for next run + public lostanza defn clear-coverage () -> ref : + val vms:ptr = call-prim flush-vm() + val info:ptr = vms.function-info + val counters:ptr = vms.function-counters + val len:long = info.length + call-c clib/memset(counters, 0, len << 3L) + return false + + var PROFILING-MSECS:Int = 0 + + ;Start profiling by initializing state and starting profile thread + extern start_sample_profiling: (int, long, ptr, ptr) -> int + lostanza defn do-start-sample-profiling (msecs:ref) -> ref : + val vms:ptr = call-prim flush-vm() + [vms.profile-flag] = 0L + vms.profile-buffer.length = 0 + PROFILING-MSECS = msecs + val res = call-c start_sample_profiling(msecs.value, vms.function-info.length, vms.profile-flag, vms.function-counters) + if res == 0 : + return false + else : + return true + + ;Stop profiling by clearing state and stopping profile thread + extern stop_sample_profiling: () -> int + lostanza defn do-stop-sample-profiling () -> ref : + val vms:ptr = call-prim flush-vm() + [vms.profile-flag] = 0L + call-c stop_sample_profiling() + return PROFILING-MSECS + +;dumps coverage results to given file +; must be stanza compiled with -coverage flag as well +public defn dump-coverage (filename:String) : + val file = FileOutputStream(filename) + try : + val coverage = collect-coverage() + for elt in reverse(to-list(lazy-qsort(count, coverage))) do : + if count(elt) > 0L : + println(file, "%_: %_/%_ in %_ (%_)" % [count(elt), package(elt), name(elt), info(elt), id(elt)]) + finally: close(file) + +;Coverage wrapper to be called with within which +; runs thunk with coverage counting on and then dumps coverage report to file +public defn coverage (f:() -> ?, filename:String) : + clear-coverage() + f() + dump-coverage(filename) + +; starts profiling sampling at msecs period +public defn start-profiling (msecs:Int) : + clear-profiling() + do-start-sample-profiling(msecs) + false + +; dumps profiling results to given file +; must be stanza compiled with -profile flag as well +public defn stop-profiling () -> ProfileResult: + collect-profiling(do-stop-sample-profiling()) + +;Profiling wrapper to be called with within +public defn profiling (f:() -> ?, msecs:Int) -> ProfileResult : + start-profiling(msecs) + f() + stop-profiling() + +;;; convert to flame-graph format using traditional stack-collapse function + +;These functions clog up flame graph so filter them out +val default-silence-prefixes = [ + "core/do" + "core/seq" + "core/map" + "core/with-exception-handler" + "core/setup-coroutine" + "core/with-finally" + "core/dynamic-wind" + "core/wrap-length" + "core/repeat-while" + ] + +defn namestring (pi:ProfileInfo) -> String : + string-join $ [package(pi) "/" name(pi) ":" line(info(pi))] + +defn with-output-file-stream (f: FileOutputStream -> ?T, file:FileOutputStream) -> T : + try : f(file) + finally : close(file) + +; turn profile result into flame graph file format +public defn stack-collapse (res:ProfileResult, filename:String, top?:True|False = false, max-depth:Int = 1000, silence-prefixes:Tuple = default-silence-prefixes) : + val strings = IntTable() + val silence-ids = IntSet() + for elt in info(res) do : + val decl = namestring(elt) + strings[id(elt)] = decl + for p in silence-prefixes do : + add(silence-ids, id(elt)) when prefix?(decl, p) + within (file) = with-output-file-stream(FileOutputStream(filename)) : + if top? : + val tops = IntTable(0) + for trace in id-traces(res) do : + val ids = ids(trace) + val len = length(ids) + for off in 1 through 2 do : + val idx = len - off + if idx >= 0 : + val id = ids[idx] + tops[id] = tops[id] + 1 + val tops* = reverse $ to-list $ lazy-qsort(value, tops) + for top in tops* do : + println(file, "%_: %_" % [value(top) * msecs(res), strings[key(top)]]) + else : + for trace in id-traces(res) do : + for (record in filter({ not silence-ids[_] }, ids(trace)), i in 0 to max-depth) do : + print(file, ";") when i > 0 + print(file, strings[record]) + print(file, " ") + println(file, msecs(trace)) + diff --git a/core/stack-trace.stanza b/core/stack-trace.stanza index ae2423a25..a02532f3a 100644 --- a/core/stack-trace.stanza +++ b/core/stack-trace.stanza @@ -254,6 +254,7 @@ public lostanza deftype StackTraceRecord : file:ptr line:int column:int + function:long ;Add the given record as an entry in the stack trace. public lostanza defn add-entry (b:ref, diff --git a/core/stanza.proj b/core/stanza.proj index 80b055281..539892326 100644 --- a/core/stanza.proj +++ b/core/stanza.proj @@ -7,4 +7,6 @@ packages parser/* defined-in "parser" package reader defined-in "reader.stanza" package arg-parser defined-in "arg-parser.stanza" package line-wrap defined-in "line-wrap.stanza" +package heap-analyzer defined-in "heap-analyzer.stanza" +package profiler defined-in "profiler.stanza" packages core/* defined-in "." \ No newline at end of file diff --git a/runtime/driver.c b/runtime/driver.c index 2222b12a6..dc93e1b8d 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -592,6 +592,128 @@ void stz_memory_resize (void* p, stz_long old_size, stz_long new_size) { #endif +//============================================================ +//==================== Profiler ============================== +//============================================================ + +#if defined(PLATFORM_OS_X) || defined(PLATFORM_LINUX) + +#include +#include + +static pthread_t main_thread_handle; + +static pthread_t thread_create(int thread_routine(void *parameter), void *parameter) { + pthread_attr_t attr; + pthread_t os_thread; + + if (pthread_attr_init(&attr) != 0) { + return 0; + } + + // Don't leave thread/stack around after exit for join: + if (pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) != 0) { + return 0; + } + + // Apparently, it is a different type on Linux. + void* (*routine)(void*) = (void*(*)(void *))thread_routine; + + if (pthread_create(&os_thread, &attr, routine, NULL) != 0) { + return 0; + } + return os_thread; +} + +static bool ticker_stopped = false; +static bool ticker_running = false; +static bool ticker_stopping = false; +static bool ticker_created = false; +static bool EnableTicks = true; +static int TickInterval = 100; // mSec + +static uint64_t *profile_flag; +static uint64_t *function_counters; +static int num_functions; + +// Timer thread that sets flag every tickinterval msecs +static int ticker_thread_routine(void *parameter) { + ticker_stopped = false; + while (!ticker_stopping) { + usleep(TickInterval * 1000); + + if (ticker_running) { + if (*profile_flag != 2L) // Not still profiling stack trace? + *profile_flag = 1L; // Profile next stack trace + } + } + ticker_stopped = true; + return 0; +} + +bool start_ticks() { + if (!EnableTicks) { + return true; + } + ticker_running = true; + ticker_stopping = false; + if (!ticker_created) { + ticker_created = true; + if (thread_create(ticker_thread_routine, 0) == 0) { + return false; + } + } + return true; +} + +static void suspend_ticks() { + ticker_running = false; + sleep(1); // why is this necessary? +} + +static void resume_ticks() { + start_ticks(); +} + +static void stop_ticks() { + if (ticker_created) { + ticker_stopping = true; + if (ticker_running) { + for (int i=0; i<10 && !ticker_stopped; i++) { // Wait for thread to die + usleep(TickInterval * 1000); + } + } + ticker_created = false; + ticker_running = false; + ticker_stopped = false; + } +} + +int start_sample_profiling (int msecs, int num_functions_arg, uint64_t *profile_flag_arg, uint64_t *function_counters_arg) { + TickInterval = msecs; + num_functions = num_functions_arg; + profile_flag = profile_flag_arg; + function_counters = function_counters_arg; + start_ticks(); + return 1; +} + +int stop_sample_profiling() { + stop_ticks(); + return 1; +} + +#else + +int start_sample_profiling (int msecs, int num_functions_arg, uint64_t *profile_flag_arg, uint64_t *function_counters_arg) { + return 0; +} +int stop_sample_profiling () { + return 0; +} + +#endif + //============================================================ //================= Process Runtime ========================== //============================================================ diff --git a/scripts/make.sh b/scripts/make.sh index c36c2106c..47672095e 100755 --- a/scripts/make.sh +++ b/scripts/make.sh @@ -60,6 +60,8 @@ PKGFILES="math \ core/debug-table \ core/sighandler \ core/local-table \ + heap-analyzer \ + profiler \ arg-parser \ line-wrap \ stz/test-driver \