From 96e461a122cff410909361f9cfa3cd918018d91a Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Fri, 3 Jan 2025 13:26:12 -0800 Subject: [PATCH 01/38] collect bytes and uses from heap and print out sorted results --- compiler/codegen.stanza | 3 ++ compiler/stitcher.stanza | 14 +++++++ compiler/vm-structures.stanza | 1 + core/core.stanza | 76 +++++++++++++++++++++++++++++++++++ 4 files changed, 94 insertions(+) diff --git a/compiler/codegen.stanza b/compiler/codegen.stanza index 96c140bc..da9a5375 100644 --- a/compiler/codegen.stanza +++ b/compiler/codegen.stanza @@ -72,6 +72,7 @@ public defstruct AsmStubs : collect-garbage:Int saved-regs:Tuple saved-fregs:Tuple + heap-statistics:Int class-table:Int global-root-table:Int stackmap-table:Int @@ -128,6 +129,7 @@ public defn AsmStubs (backend:Backend) : next(id-counter) ;collect-garbage:Int saved-regs ;saved-regs:Tuple saved-fregs ;saved-fregs:Tuple + next(id-counter) ;heap-statistics:Int next(id-counter) ;class-table:Int next(id-counter) ;global-root-table:Int next(id-counter) ;stackmap-table:Int @@ -334,6 +336,7 @@ public defn compile-entry-function (emitter:CodeEmitter, stubs:AsmStubs) : #label(safepoint-table) ;safepoint-table:ptr #label(debug-table) ;debug-table:ptr #label(local-var-table) ;local-var-table:ptr + #label(heap-statistics) ;heap-statistics:ptr #label(class-table) ;class-table:ptr #label(global-root-table) ;global-root-table:ptr #label(stackmap-table) ;stackmap-table:ptr diff --git a/compiler/stitcher.stanza b/compiler/stitcher.stanza index 0c8bf64a..d14d6505 100644 --- a/compiler/stitcher.stanza +++ b/compiler/stitcher.stanza @@ -856,6 +856,19 @@ public defn Stitcher (packages:Collection, bindings:Bindings|False, s E $ DefText() E $ Comment("End of Data Tables") + defn emit-heap-statistics-table (code-emitter:CodeEmitter) : + defn E (i:Ins) : emit(code-emitter, i) + E $ Comment("Heap Statistics Table") + E $ DefData() + E $ Label(/heap-statistics(stubs)) + for i in 0 to num-concrete-classes do : + E $ DefLong(0L) ; num-uses + E $ DefLong(0L) ; num-bytes + E $ DefLong(to-long(num-concrete-classes)) + E $ DefLong(-1L) ; mark end of table + E $ DefText() + E $ Comment("End of Heap Statistics Table") + ;Emit class table defn emit-class-table (code-emitter:CodeEmitter) : defn E (i:Ins) : emit(code-emitter, i) @@ -995,6 +1008,7 @@ public defn Stitcher (packages:Collection, bindings:Bindings|False, s ; so they can be dynamically imported by other executables if generate-export-directives-table?(backend(stubs)) : emit-export-table(code-emitter) + emit-heap-statistics-table(code-emitter) defmethod stubs (this) : stubs defmethod core-fn (this, id:FnId) : diff --git a/compiler/vm-structures.stanza b/compiler/vm-structures.stanza index 3bad507f..746da736 100644 --- a/compiler/vm-structures.stanza +++ b/compiler/vm-structures.stanza @@ -31,6 +31,7 @@ public lostanza deftype VMState : var safepoint-table: ptr ;(Permanent State) var debug-table: ptr ;(Permanent State) var local-var-table: ptr ;(Permanent State) + var heap-statistics: ptr ;(Permanent State) var class-table: ptr ;(Permanent State) ;Interpreted Mode Tables var instructions: ptr ;(Permanent State) diff --git a/core/core.stanza b/core/core.stanza index a2a602ac..38aeb6c4 100644 --- a/core/core.stanza +++ b/core/core.stanza @@ -209,6 +209,10 @@ protected lostanza deftype ArrayRecord : num-item-roots:int roots:int ... +protected lostanza deftype HeapStatistics : + var num-uses:long + var num-bytes:long + ;The first fields in VMState are used by the core library ;in both compiled and interpreted mode. The last fields ;are used only in compiled mode. @@ -232,6 +236,7 @@ protected lostanza deftype VMState : safepoint-table: ptr ;(Variable State) debug-table: ptr ;(Variable State) local-var-table: ptr ;(Variable State) + heap-statistics: ptr ;(Variable State) ;Compiled Mode Tables class-table: ptr global-root-table: ptr @@ -2776,6 +2781,77 @@ public lostanza defn min (x:long, y:long) -> long : if x < y : return x else : return y +;============================================================ +;===================== Heap Analyzer ======================== +;============================================================ + +lostanza defn num-concrete-classes (vms:ptr) -> int : + labels: + begin: goto loop(0) + loop (i:int) : + val stat = vms.heap-statistics[i] + if stat.num-bytes < 0L : + return stat.num-uses as int + else : + goto loop(i + 1) + +public defstruct HeapStat : + tag : Int + name : String + num-uses : Long + num-bytes : Long +with: + printer => true + +public defn analyze-heap () -> Long : + val counts = Vector() + val size = do-analyze-heap(counts) + val res = reverse(to-list(lazy-qsort(num-bytes, counts))) + println("Heap size %_" % [size]) + var max-bytes-size = reduce(max, length("Size"), seq({ length(to-string(num-bytes(_))) }, res)) + var max-uses-size = reduce(max, length("Uses"), seq({ length(to-string(num-uses(_))) }, res)) + defn pad (s:String, n:Int) -> String : append-all(cat(repeatedly({ " " }, (n - length(s))), [s])) + println(" %_ %_: %_" % [pad("Size", max-bytes-size), pad("Use", max-uses-size), "Type"]) + for hc in res do : + println(" %_ %_: %_" % [pad(to-string(num-bytes(hc)), max-bytes-size), pad(to-string(num-uses(hc)), max-uses-size), name(hc)]) + size + +lostanza defn do-analyze-heap (counts:ref>) -> ref : + val vms:ptr = call-prim flush-vm() + run-garbage-collector() + val num-classes = num-concrete-classes(vms) + clear(vms.heap-statistics, num-classes * sizeof(HeapStatistics)) + val hsize = do-analyze-heap(vms.heap.start, vms.heap.old-objects-end, vms) + val nursery = nursery-start(addr(vms.heap)) + val nsize = do-analyze-heap(nursery, vms.heap.top, vms) + for (var i:int = 0, i < num-classes, i = i + 1) : + val heap-stat = vms.heap-statistics[i] + val num-uses = heap-stat.num-uses + if num-uses > 0L : + add(counts, HeapStat(new Int{i as int}, String(class-name(i)), new Long{num-uses}, new Long{heap-stat.num-bytes})) + return new Long{ hsize + nsize } + +lostanza defn do-analyze-heap (pstart:ptr, pend:ptr, vms:ptr) -> long : + var p:ptr = pstart + while p < pend : + val tag = [p] as int + val class = vms.class-table[tag].record + var size:long = 0L + if class.item-size == 0 : + size = object-size-on-heap(class.size) + else : + val class = class as ptr + val array = p as ptr + val len = array.slots[0] + val base-size = class.base-size + val item-size = class.item-size + val my-size = base-size + item-size * len + size = object-size-on-heap(my-size) + p = p + size + vms.heap-statistics[tag].num-uses = vms.heap-statistics[tag].num-uses + 1L + vms.heap-statistics[tag].num-bytes = vms.heap-statistics[tag].num-bytes + size + return (pend as long) - (pstart as long) + ;============================================================ ;===================== Debugging ============================ ;============================================================ From bade4c0c194e9366fe227feb27affda5c07228c2 Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Sun, 5 Jan 2025 20:20:29 -0800 Subject: [PATCH 02/38] doc, improve naming, improve stat report --- core/core.stanza | 66 +++++++++++++++++++++++++++--------------------- 1 file changed, 37 insertions(+), 29 deletions(-) diff --git a/core/core.stanza b/core/core.stanza index 38aeb6c4..8b34af70 100644 --- a/core/core.stanza +++ b/core/core.stanza @@ -209,7 +209,7 @@ protected lostanza deftype ArrayRecord : num-item-roots:int roots:int ... -protected lostanza deftype HeapStatistics : +protected lostanza deftype HeapStatistic : var num-uses:long var num-bytes:long @@ -236,7 +236,7 @@ protected lostanza deftype VMState : safepoint-table: ptr ;(Variable State) debug-table: ptr ;(Variable State) local-var-table: ptr ;(Variable State) - heap-statistics: ptr ;(Variable State) + heap-statistics: ptr ;(Variable State) ;Compiled Mode Tables class-table: ptr global-root-table: ptr @@ -2785,42 +2785,32 @@ public lostanza defn min (x:long, y:long) -> long : ;===================== Heap Analyzer ======================== ;============================================================ -lostanza defn num-concrete-classes (vms:ptr) -> int : +public defstruct HeapStat : + tag : Int ; type + name : String ; name of type + num-uses : Long ; num references in heap + num-bytes : Long ; num bytes occupied in heap +with: + printer => true + +; clear out statistics for each concrete class +lostanza defn clear-heap-statistics (vms:ptr) -> int : labels: begin: goto loop(0) loop (i:int) : val stat = vms.heap-statistics[i] - if stat.num-bytes < 0L : + if stat.num-bytes < 0L : ; sentinel return stat.num-uses as int else : + vms.heap-statistics[i].num-bytes = 0 + vms.heap-statistics[i].num-uses = 0 goto loop(i + 1) -public defstruct HeapStat : - tag : Int - name : String - num-uses : Long - num-bytes : Long -with: - printer => true - -public defn analyze-heap () -> Long : - val counts = Vector() - val size = do-analyze-heap(counts) - val res = reverse(to-list(lazy-qsort(num-bytes, counts))) - println("Heap size %_" % [size]) - var max-bytes-size = reduce(max, length("Size"), seq({ length(to-string(num-bytes(_))) }, res)) - var max-uses-size = reduce(max, length("Uses"), seq({ length(to-string(num-uses(_))) }, res)) - defn pad (s:String, n:Int) -> String : append-all(cat(repeatedly({ " " }, (n - length(s))), [s])) - println(" %_ %_: %_" % [pad("Size", max-bytes-size), pad("Use", max-uses-size), "Type"]) - for hc in res do : - println(" %_ %_: %_" % [pad(to-string(num-bytes(hc)), max-bytes-size), pad(to-string(num-uses(hc)), max-uses-size), name(hc)]) - size - -lostanza defn do-analyze-heap (counts:ref>) -> ref : +; run gc, collect heap stats while walking each object in nursery and heap +lostanza defn do-analyze-heap (stats:ref>) -> ref : val vms:ptr = call-prim flush-vm() run-garbage-collector() - val num-classes = num-concrete-classes(vms) - clear(vms.heap-statistics, num-classes * sizeof(HeapStatistics)) + val num-classes = clear-heap-statistics(vms) val hsize = do-analyze-heap(vms.heap.start, vms.heap.old-objects-end, vms) val nursery = nursery-start(addr(vms.heap)) val nsize = do-analyze-heap(nursery, vms.heap.top, vms) @@ -2828,9 +2818,10 @@ lostanza defn do-analyze-heap (counts:ref>) -> ref : val heap-stat = vms.heap-statistics[i] val num-uses = heap-stat.num-uses if num-uses > 0L : - add(counts, HeapStat(new Int{i as int}, String(class-name(i)), new Long{num-uses}, new Long{heap-stat.num-bytes})) + add(stats, HeapStat(new Int{i as int}, String(class-name(i)), new Long{num-uses}, new Long{heap-stat.num-bytes})) return new Long{ hsize + nsize } +; collect heap stats while walking each object in consecutive range of memory lostanza defn do-analyze-heap (pstart:ptr, pend:ptr, vms:ptr) -> long : var p:ptr = pstart while p < pend : @@ -2852,6 +2843,23 @@ lostanza defn do-analyze-heap (pstart:ptr, pend:ptr, vms:ptr Long : + val stats = Vector() + val size = do-analyze-heap(stats) + val res = reverse(to-list(lazy-qsort(num-bytes, stats))) + println("Heap size %_" % [size]) + var max-bytes-size = reduce(max, length("Size"), seq({ length(to-string(num-bytes(_))) }, res)) + var max-perc-size = 3 + var max-uses-size = reduce(max, length("Uses"), seq({ length(to-string(num-uses(_))) }, res)) + defn pad (s:String, n:Int) -> String : append-all(cat(repeatedly({ " " }, (n - length(s))), [s])) + println(" %_ %_ %_: %_" % [pad("Size", max-bytes-size), "Perc", pad("Use", max-uses-size), "Type"]) + for hc in res do : + val p = to-int(to-double(num-bytes(hc)) * 100.0 / to-double(size)) + println(" %_ %_%% %_: %_" % [ + pad(to-string(num-bytes(hc)), max-bytes-size), pad(to-string(p), max-perc-size), pad(to-string(num-uses(hc)), max-uses-size), name(hc)]) + size + ;============================================================ ;===================== Debugging ============================ ;============================================================ From 5a1c652f6ec4955d08338e1ddd7c7db40a62857c Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Mon, 6 Jan 2025 10:59:05 -0800 Subject: [PATCH 03/38] bootstrap changes to core --- ci/build-stanza-version.txt | 2 +- core/core.stanza | 232 +++++++++++++++++++++--------------- 2 files changed, 135 insertions(+), 99 deletions(-) diff --git a/ci/build-stanza-version.txt b/ci/build-stanza-version.txt index 71434844..bf293833 100644 --- a/ci/build-stanza-version.txt +++ b/ci/build-stanza-version.txt @@ -7,4 +7,4 @@ # like 1.23.45 # # Use version 0.17.56 to compile 0.18.0 -0.18.96 +0.18.97 diff --git a/core/core.stanza b/core/core.stanza index 8b34af70..ab4ad441 100644 --- a/core/core.stanza +++ b/core/core.stanza @@ -218,32 +218,62 @@ protected lostanza deftype HeapStatistic : ;are used only in compiled mode. ;Permanent state changes in-between each code load. ;Variable state changes in-between each boundary change. -protected lostanza deftype VMState : - ;Compiled and Interpreted Mode - global-offsets: ptr ;(Permanent State) - global-mem: ptr ;(Permanent State) - var sig-handler: long ;(Permanent State) - var current-coroutine-ptr: ptr ;[TODO] Change to long to represent reference. - var stepping-coroutine-ptr: ptr ;[TODO] Change to long to represent reference. - const-table: ptr ;(Permanent State) - const-mem: ptr ;(Permanent State) - data-offsets: ptr ;(Permanent State) - data-mem: ptr ;(Permanent State) - code-offsets: ptr ;(Permanent State) - registers: ptr ;(Permanent State) - system-registers: ptr ;(Permanent State) - var heap: Heap ;(Variable State) - safepoint-table: ptr ;(Variable State) - debug-table: ptr ;(Variable State) - local-var-table: ptr ;(Variable State) - heap-statistics: ptr ;(Variable State) - ;Compiled Mode Tables - class-table: ptr - global-root-table: ptr - stackmap-table: ptr> - stack-trace-table: ptr - extern-table: ptr - extern-defn-table: ptr +#if-defined(BOOTSTRAP) : + + protected lostanza deftype VMState : + ;Compiled and Interpreted Mode + global-offsets: ptr ;(Permanent State) + global-mem: ptr ;(Permanent State) + var sig-handler: long ;(Permanent State) + var current-coroutine-ptr: ptr ;[TODO] Change to long to represent reference. + var stepping-coroutine-ptr: ptr ;[TODO] Change to long to represent reference. + const-table: ptr ;(Permanent State) + const-mem: ptr ;(Permanent State) + data-offsets: ptr ;(Permanent State) + data-mem: ptr ;(Permanent State) + code-offsets: ptr ;(Permanent State) + registers: ptr ;(Permanent State) + system-registers: ptr ;(Permanent State) + var heap: Heap ;(Variable State) + safepoint-table: ptr ;(Variable State) + debug-table: ptr ;(Variable State) + local-var-table: ptr ;(Variable State) + ;Compiled Mode Tables + class-table: ptr + global-root-table: ptr + stackmap-table: ptr> + stack-trace-table: ptr + extern-table: ptr + extern-defn-table: ptr + +#else: + + protected lostanza deftype VMState : + ;Compiled and Interpreted Mode + global-offsets: ptr ;(Permanent State) + global-mem: ptr ;(Permanent State) + var sig-handler: long ;(Permanent State) + var current-coroutine-ptr: ptr ;[TODO] Change to long to represent reference. + var stepping-coroutine-ptr: ptr ;[TODO] Change to long to represent reference. + const-table: ptr ;(Permanent State) + const-mem: ptr ;(Permanent State) + data-offsets: ptr ;(Permanent State) + data-mem: ptr ;(Permanent State) + code-offsets: ptr ;(Permanent State) + registers: ptr ;(Permanent State) + system-registers: ptr ;(Permanent State) + var heap: Heap ;(Variable State) + safepoint-table: ptr ;(Variable State) + debug-table: ptr ;(Variable State) + local-var-table: ptr ;(Variable State) + heap-statistics: ptr ;(Variable State) + ;Compiled Mode Tables + class-table: ptr + global-root-table: ptr + stackmap-table: ptr> + stack-trace-table: ptr + extern-table: ptr + extern-defn-table: ptr lostanza deftype ExternTable : length: long @@ -2785,80 +2815,86 @@ public lostanza defn min (x:long, y:long) -> long : ;===================== Heap Analyzer ======================== ;============================================================ -public defstruct HeapStat : - tag : Int ; type - name : String ; name of type - num-uses : Long ; num references in heap - num-bytes : Long ; num bytes occupied in heap -with: - printer => true +#if-defined(BOOTSTRAP) : -; clear out statistics for each concrete class -lostanza defn clear-heap-statistics (vms:ptr) -> int : - labels: - begin: goto loop(0) - loop (i:int) : - val stat = vms.heap-statistics[i] - if stat.num-bytes < 0L : ; sentinel - return stat.num-uses as int - else : - vms.heap-statistics[i].num-bytes = 0 - vms.heap-statistics[i].num-uses = 0 - goto loop(i + 1) + public defn analyze-heap () -> Long : 0L -; run gc, collect heap stats while walking each object in nursery and heap -lostanza defn do-analyze-heap (stats:ref>) -> ref : - val vms:ptr = call-prim flush-vm() - run-garbage-collector() - val num-classes = clear-heap-statistics(vms) - val hsize = do-analyze-heap(vms.heap.start, vms.heap.old-objects-end, vms) - val nursery = nursery-start(addr(vms.heap)) - val nsize = do-analyze-heap(nursery, vms.heap.top, vms) - for (var i:int = 0, i < num-classes, i = i + 1) : - val heap-stat = vms.heap-statistics[i] - val num-uses = heap-stat.num-uses - if num-uses > 0L : - add(stats, HeapStat(new Int{i as int}, String(class-name(i)), new Long{num-uses}, new Long{heap-stat.num-bytes})) - return new Long{ hsize + nsize } - -; collect heap stats while walking each object in consecutive range of memory -lostanza defn do-analyze-heap (pstart:ptr, pend:ptr, vms:ptr) -> long : - var p:ptr = pstart - while p < pend : - val tag = [p] as int - val class = vms.class-table[tag].record - var size:long = 0L - if class.item-size == 0 : - size = object-size-on-heap(class.size) - else : - val class = class as ptr - val array = p as ptr - val len = array.slots[0] - val base-size = class.base-size - val item-size = class.item-size - val my-size = base-size + item-size * len - size = object-size-on-heap(my-size) - p = p + size - vms.heap-statistics[tag].num-uses = vms.heap-statistics[tag].num-uses + 1L - vms.heap-statistics[tag].num-bytes = vms.heap-statistics[tag].num-bytes + size - return (pend as long) - (pstart as long) - -; public interface to heap analyzer collecting and printing out stats -public defn analyze-heap () -> Long : - val stats = Vector() - val size = do-analyze-heap(stats) - val res = reverse(to-list(lazy-qsort(num-bytes, stats))) - println("Heap size %_" % [size]) - var max-bytes-size = reduce(max, length("Size"), seq({ length(to-string(num-bytes(_))) }, res)) - var max-perc-size = 3 - var max-uses-size = reduce(max, length("Uses"), seq({ length(to-string(num-uses(_))) }, res)) - defn pad (s:String, n:Int) -> String : append-all(cat(repeatedly({ " " }, (n - length(s))), [s])) - println(" %_ %_ %_: %_" % [pad("Size", max-bytes-size), "Perc", pad("Use", max-uses-size), "Type"]) - for hc in res do : - val p = to-int(to-double(num-bytes(hc)) * 100.0 / to-double(size)) - println(" %_ %_%% %_: %_" % [ - pad(to-string(num-bytes(hc)), max-bytes-size), pad(to-string(p), max-perc-size), pad(to-string(num-uses(hc)), max-uses-size), name(hc)]) - size +#else : + + public defstruct HeapStat : + tag : Int ; type + name : String ; name of type + num-uses : Long ; num references in heap + num-bytes : Long ; num bytes occupied in heap + with: + printer => true + + ; clear out statistics for each concrete class + lostanza defn clear-heap-statistics (vms:ptr) -> int : + labels: + begin: goto loop(0) + loop (i:int) : + val stat = vms.heap-statistics[i] + if stat.num-bytes < 0L : ; sentinel + return stat.num-uses as int + else : + vms.heap-statistics[i].num-bytes = 0 + vms.heap-statistics[i].num-uses = 0 + goto loop(i + 1) + + ; run gc, collect heap stats while walking each object in nursery and heap + lostanza defn do-analyze-heap (stats:ref>) -> ref : + val vms:ptr = call-prim flush-vm() + run-garbage-collector() + val num-classes = clear-heap-statistics(vms) + val hsize = do-analyze-heap(vms.heap.start, vms.heap.old-objects-end, vms) + val nursery = nursery-start(addr(vms.heap)) + val nsize = do-analyze-heap(nursery, vms.heap.top, vms) + for (var i:int = 0, i < num-classes, i = i + 1) : + val heap-stat = vms.heap-statistics[i] + val num-uses = heap-stat.num-uses + if num-uses > 0L : + add(stats, HeapStat(new Int{i as int}, String(class-name(i)), new Long{num-uses}, new Long{heap-stat.num-bytes})) + return new Long{ hsize + nsize } + + ; collect heap stats while walking each object in consecutive range of memory + lostanza defn do-analyze-heap (pstart:ptr, pend:ptr, vms:ptr) -> long : + var p:ptr = pstart + while p < pend : + val tag = [p] as int + val class = vms.class-table[tag].record + var size:long = 0L + if class.item-size == 0 : + size = object-size-on-heap(class.size) + else : + val class = class as ptr + val array = p as ptr + val len = array.slots[0] + val base-size = class.base-size + val item-size = class.item-size + val my-size = base-size + item-size * len + size = object-size-on-heap(my-size) + p = p + size + vms.heap-statistics[tag].num-uses = vms.heap-statistics[tag].num-uses + 1L + vms.heap-statistics[tag].num-bytes = vms.heap-statistics[tag].num-bytes + size + return (pend as long) - (pstart as long) + + ; public interface to heap analyzer collecting and printing out stats + public defn analyze-heap () -> Long : + val stats = Vector() + val size = do-analyze-heap(stats) + val res = reverse(to-list(lazy-qsort(num-bytes, stats))) + println("Heap size %_" % [size]) + var max-bytes-size = reduce(max, length("Size"), seq({ length(to-string(num-bytes(_))) }, res)) + var max-perc-size = 3 + var max-uses-size = reduce(max, length("Uses"), seq({ length(to-string(num-uses(_))) }, res)) + defn pad (s:String, n:Int) -> String : append-all(cat(repeatedly({ " " }, (n - length(s))), [s])) + println(" %_ %_ %_: %_" % [pad("Size", max-bytes-size), "Perc", pad("Use", max-uses-size), "Type"]) + for hc in res do : + val p = to-int(to-double(num-bytes(hc)) * 100.0 / to-double(size)) + println(" %_ %_%% %_: %_" % [ + pad(to-string(num-bytes(hc)), max-bytes-size), pad(to-string(p), max-perc-size), pad(to-string(num-uses(hc)), max-uses-size), name(hc)]) + size ;============================================================ ;===================== Debugging ============================ From 351c4a0400fc8e0fc1fca7c85b74b4c88ff4b329 Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Mon, 6 Jan 2025 22:06:26 -0800 Subject: [PATCH 04/38] minimal profiling --- compiler/codegen.stanza | 16 ++ compiler/compiler-main.stanza | 1 + compiler/el-ir.stanza | 1 + compiler/stitcher.stanza | 51 +++++ compiler/utils.stanza | 4 + compiler/vm-ir.stanza | 1 + compiler/vm-structures.stanza | 4 + compiler/vm.stanza | 2 + core/core.stanza | 389 +++++++++++++++++++++++++++++++--- runtime/driver.c | 122 +++++++++++ 10 files changed, 566 insertions(+), 25 deletions(-) diff --git a/compiler/codegen.stanza b/compiler/codegen.stanza index 96c140bc..e2ca4543 100644 --- a/compiler/codegen.stanza +++ b/compiler/codegen.stanza @@ -49,6 +49,10 @@ public defstruct AsmStubs : stack-limit:Int registers:Int system-registers-space:Int + profile-flag:Int + profile-buffer:Int + function-counters:Int + function-info:Int heap-start:Int heap-top:Int heap-limit:Int @@ -105,6 +109,10 @@ public defn AsmStubs (backend:Backend) : next(id-counter) ;stack-limit:Int next(id-counter) ;registers:Int next(id-counter) ;system-registers-space:Int + next(id-counter) ;profile-flag:Int + next(id-counter) ;profile-buffer:Int + next(id-counter) ;function-counters:Int + next(id-counter) ;function-info:Int next(id-counter) ;heap-start:Int next(id-counter) ;heap-top:Int next(id-counter) ;heap-limit:Int @@ -204,6 +212,10 @@ public defn compile-runtime-stubs (emitter:CodeEmitter, stubs:AsmStubs) : comment("stack-limit = %_" % [stack-limit(stubs)]) comment("registers = %_" % [registers(stubs)]) comment("system-registers-space = %_" % [system-registers-space(stubs)]) + comment("profile-flag = %_" % [profile-flag(stubs)]) + comment("profile-buffer = %_" % [profile-buffer(stubs)]) + comment("function-counters = %_" % [function-counters(stubs)]) + comment("function-info = %_" % [function-info(stubs)]) comment("heap-start = %_" % [heap-start(stubs)]) comment("heap-top = %_" % [heap-top(stubs)]) comment("heap-limit = %_" % [heap-limit(stubs)]) @@ -334,6 +346,10 @@ public defn compile-entry-function (emitter:CodeEmitter, stubs:AsmStubs) : #label(safepoint-table) ;safepoint-table:ptr #label(debug-table) ;debug-table:ptr #label(local-var-table) ;local-var-table:ptr + #label(profile-flag) ;profile-flag: ptr + #long() ;profile-buffer: ptr + #label(function-counters) ;function-counters: ptr + #label(function-info) ;function-info: ptr #label(class-table) ;class-table:ptr #label(global-root-table) ;global-root-table:ptr #label(stackmap-table) ;stackmap-table:ptr diff --git a/compiler/compiler-main.stanza b/compiler/compiler-main.stanza index cd5d68b5..9a746ee5 100644 --- a/compiler/compiler-main.stanza +++ b/compiler/compiler-main.stanza @@ -399,3 +399,4 @@ defn collapse (p:NormVMPackage|StdPkg) : defmethod debug-table (this) : debug-table(vmp) defmethod safepoint-table (this) : safepoint-table(vmp) defmethod debug-name-table (this): debug-name-table(vmp) + defmethod function-info (this) : function-info(vmp) diff --git a/compiler/el-ir.stanza b/compiler/el-ir.stanza index f433f7d5..b5c3e446 100644 --- a/compiler/el-ir.stanza +++ b/compiler/el-ir.stanza @@ -46,6 +46,7 @@ public defstruct EBot <: HiEType public defstruct EPackage : packageio: PackageIO with: (updater => sub-packageio) exps: Tuple with: (updater => sub-exps) + function-info: False|Tuple with: (default => false, updater => sub-function-info) public defn name (e:EPackage) : package(packageio(e)) diff --git a/compiler/stitcher.stanza b/compiler/stitcher.stanza index 0c8bf64a..8b511832 100644 --- a/compiler/stitcher.stanza +++ b/compiler/stitcher.stanza @@ -941,6 +941,47 @@ public defn Stitcher (packages:Collection, bindings:Bindings|False, s E $ DefText() E $ Comment("End of String Table for Filenames") + defn emit-profile-flag (code-emitter:CodeEmitter) : + defn E (i:Ins) : emit(code-emitter, i) + E $ Comment("Profile Flag") + E $ DefData() + E $ Label(profile-flag(stubs)) + E $ DefLong(0L) + + ;Emit function info table + defn emit-function-counters-table (code-emitter:CodeEmitter, num:Int) : + defn E (i:Ins) : emit(code-emitter, i) + E $ Comment("Function Counters for Profiling") + E $ DefData() + E $ Label(function-counters(stubs)) + for i in 0 to num do : + E $ DefLong(0L) + + ;Emit function info table + defn emit-function-info-table (code-emitter:CodeEmitter, function-info-table:Tuple, len:Int) : + defn E (i:Ins) : emit(code-emitter, i) + val string-lbls = HashTable-init(unique-id{stubs}) + E $ Comment("Function Info Table for Profiling") + E $ DefData() + E $ Label(function-info(stubs)) + E $ DefLong(to-long(length(function-info-table))) + for fun-info in function-info-table do : + E $ DefLabel(string-lbls[to-string(package(info(fun-info)))]) + match(signature(info(fun-info))) : + (s:String) : E $ DefLabel(string-lbls[s]) + (f:False) : E $ DefLong(0L) + val file-info = info(info(fun-info)) as AbsoluteFileInfo + E $ DefLabel(string-lbls[filename(file-info)]) + E $ DefInt(line(file-info)) + E $ DefInt(column(file-info)) + E $ DefText() + E $ DefData() + for entry in string-lbls do : + val [s, n] = [key(entry), value(entry)] + E $ Label(n) + E $ DefString(s) + E $ DefText() + defn emit-export-table (code-emitter: CodeEmitter) -> False: defn E (i:Ins) : emit(code-emitter, i) @@ -995,6 +1036,16 @@ public defn Stitcher (packages:Collection, bindings:Bindings|False, s ; so they can be dynamically imported by other executables if generate-export-directives-table?(backend(stubs)) : emit-export-table(code-emitter) + val pkgs = to-tuple $ packages + val info = + if length(pkgs) == 1 : + match(function-info(pkgs[0])) : + (info:Tuple) : info + (info:False) : [] + else : [] + emit-profile-flag(code-emitter) + emit-function-counters-table(code-emitter, length(info)) + emit-function-info-table(code-emitter, info, length(info)) defmethod stubs (this) : stubs defmethod core-fn (this, id:FnId) : diff --git a/compiler/utils.stanza b/compiler/utils.stanza index dde8ed20..7d18ba25 100644 --- a/compiler/utils.stanza +++ b/compiler/utils.stanza @@ -5,6 +5,7 @@ defpackage stz/utils : import collections import stz/algorithms import core/sha256 + import stz/trace-info ;============================================================ ;===================== Radix Conversions ==================== @@ -527,3 +528,6 @@ public defstruct MTItem : public defn EmptyMTItem () : MTItem(-1, List()) + +public defstruct FunctionInfo : + info : StackTraceInfo diff --git a/compiler/vm-ir.stanza b/compiler/vm-ir.stanza index 62d22021..e8664371 100644 --- a/compiler/vm-ir.stanza +++ b/compiler/vm-ir.stanza @@ -50,6 +50,7 @@ public defstruct VMPackage : externs: Tuple extern-defns: Tuple debug-name-table: VMDebugNameTable + function-info: False|Tuple with: (default => false) debug-table: VMDebugInfoTable safepoint-table: VMSafepointTable diff --git a/compiler/vm-structures.stanza b/compiler/vm-structures.stanza index 3bad507f..c52300e5 100644 --- a/compiler/vm-structures.stanza +++ b/compiler/vm-structures.stanza @@ -31,6 +31,10 @@ public lostanza deftype VMState : var safepoint-table: ptr ;(Permanent State) var debug-table: ptr ;(Permanent State) var local-var-table: ptr ;(Permanent State) + var profile-flag: ptr + var profile-buffer: ptr ; ptr + function-counters: ptr + function-info: ptr ; ptr var class-table: ptr ;(Permanent State) ;Interpreted Mode Tables var instructions: ptr ;(Permanent State) diff --git a/compiler/vm.stanza b/compiler/vm.stanza index 83517fa6..28a5d979 100644 --- a/compiler/vm.stanza +++ b/compiler/vm.stanza @@ -371,6 +371,8 @@ public lostanza defn VirtualMachine (dylibs:ref, backend heap.iterate-roots = addr(vm-iterate-roots) heap.iterate-references-in-stack-frames = addr(vm-iterate-references-in-stack-frames) + vmstate.profile-flag = call-c clib/stz_malloc(8) + ;Initialize sighandler to false to indicate no handler on initialization. vmstate.sig-handler = false-marker() diff --git a/core/core.stanza b/core/core.stanza index a2a602ac..298b18f2 100644 --- a/core/core.stanza +++ b/core/core.stanza @@ -188,6 +188,17 @@ protected lostanza deftype StackTraceTableEntry : lbl: ptr record: StackTraceRecord +lostanza deftype FunctionInfoTable : + length: long + entries: FunctionInfoEntry ... + +lostanza deftype FunctionInfoEntry : + package: ptr + name: ptr + file: ptr + line: int + column: int + lostanza deftype ClassDescriptor : case:int num-base-bytes:int @@ -214,31 +225,62 @@ protected lostanza deftype ArrayRecord : ;are used only in compiled mode. ;Permanent state changes in-between each code load. ;Variable state changes in-between each boundary change. -protected lostanza deftype VMState : - ;Compiled and Interpreted Mode - global-offsets: ptr ;(Permanent State) - global-mem: ptr ;(Permanent State) - var sig-handler: long ;(Permanent State) - var current-coroutine-ptr: ptr ;[TODO] Change to long to represent reference. - var stepping-coroutine-ptr: ptr ;[TODO] Change to long to represent reference. - const-table: ptr ;(Permanent State) - const-mem: ptr ;(Permanent State) - data-offsets: ptr ;(Permanent State) - data-mem: ptr ;(Permanent State) - code-offsets: ptr ;(Permanent State) - registers: ptr ;(Permanent State) - system-registers: ptr ;(Permanent State) - var heap: Heap ;(Variable State) - safepoint-table: ptr ;(Variable State) - debug-table: ptr ;(Variable State) - local-var-table: ptr ;(Variable State) - ;Compiled Mode Tables - class-table: ptr - global-root-table: ptr - stackmap-table: ptr> - stack-trace-table: ptr - extern-table: ptr - extern-defn-table: ptr +#if-defined(BOOTSTRAP) : + protected lostanza deftype VMState : + ;Compiled and Interpreted Mode + global-offsets: ptr ;(Permanent State) + global-mem: ptr ;(Permanent State) + var sig-handler: long ;(Permanent State) + var current-coroutine-ptr: ptr ;[TODO] Change to long to represent reference. + var stepping-coroutine-ptr: ptr ;[TODO] Change to long to represent reference. + const-table: ptr ;(Permanent State) + const-mem: ptr ;(Permanent State) + data-offsets: ptr ;(Permanent State) + data-mem: ptr ;(Permanent State) + code-offsets: ptr ;(Permanent State) + registers: ptr ;(Permanent State) + system-registers: ptr ;(Permanent State) + var heap: Heap ;(Variable State) + safepoint-table: ptr ;(Variable State) + debug-table: ptr ;(Variable State) + local-var-table: ptr ;(Variable State) + ;Compiled Mode Tables + class-table: ptr + global-root-table: ptr + stackmap-table: ptr> + stack-trace-table: ptr + extern-table: ptr + extern-defn-table: ptr +#else: + protected lostanza deftype VMState : + ;Compiled and Interpreted Mode + global-offsets: ptr ;(Permanent State) + global-mem: ptr ;(Permanent State) + var sig-handler: long ;(Permanent State) + var current-coroutine-ptr: ptr ;[TODO] Change to long to represent reference. + var stepping-coroutine-ptr: ptr ;[TODO] Change to long to represent reference. + const-table: ptr ;(Permanent State) + const-mem: ptr ;(Permanent State) + data-offsets: ptr ;(Permanent State) + data-mem: ptr ;(Permanent State) + code-offsets: ptr ;(Permanent State) + registers: ptr ;(Permanent State) + system-registers: ptr ;(Permanent State) + var heap: Heap ;(Variable State) + safepoint-table: ptr ;(Variable State) + debug-table: ptr ;(Variable State) + local-var-table: ptr ;(Variable State) + profile-flag: ptr + var profile-buffer: ptr + function-counters: ptr + function-info: ptr + ;Compiled Mode Tables + class-table: ptr + global-root-table: ptr + stackmap-table: ptr> + stack-trace-table: ptr + extern-table: ptr + extern-defn-table: ptr lostanza deftype ExternTable : length: long @@ -11147,6 +11189,303 @@ public deftype TypeObject public defmulti typeof? (x, t:TypeObject) -> True|False public defmulti name (t:TypeObject) -> String +;============================================================ +;=================== Initialize Profiler ==================== +;============================================================ + +#if-defined(BOOTSTRAP) : + lostanza defn initialize-profiler () -> ref : + return false +#else : + lostanza defn initialize-profiler () -> ref : + val vms:ptr = call-prim flush-vm() + [vms.profile-flag] = 0L + vms.profile-buffer = LSLongVector(1 << 20) + return false + +;============================================================ +;====================== Profiling =========================== +;============================================================ + +public defstruct ProfileInfo : + id : Int + package : String + name : String + info : FileInfo + count : Long + +public defstruct ProfileResult : + info : Tuple + id-traces : Tuple> + +defmethod print (o:OutputStream, c:ProfileInfo) : + print(o, "ProfileInfo(%_, %_, %_, %_)" % [id(c), package(c), name(c), info(c), count(c)]) + +defn namestring (p:ProfileInfo) -> String : + to-string("%_/%_ %_ (%_)" % [package(p), name(p), info(p), id(p)]) + +;Fast stack trace record lookup +lostanza defn stack-trace-record (ret:long, trace-table:ptr, trace-table-table:ref>) -> ptr : + val idx = get?(trace-table-table, new Long{ret}, new Int{-1}) + if idx.value == -1 : + return null + else : + val entry = addr(trace-table.entries[idx.value]) + return addr(entry.record) + +;Fast stack trace record table construction +lostanza defn build-stack-trace-record-table (trace-table:ptr) -> ref> : + val tab = HashTable() + for (var i:int = 0, i < trace-table.length, i = i + 1) : + val entry = addr(trace-table.entries[i]) + set(tab, new Long{entry.lbl as long}, new Int{i}) + return tab + +#if-defined(BOOTSTRAP) : + + defn collect-profiling () -> ProfileResult : ProfileResult([], []) + defn collect-coverage () -> Tuple : [] + public defn clear-profiling () : false + public defn clear-coverage () : false + defn do-start-sample-profiling (num-usecs:Int) : false + defn do-stop-sample-profiling () : false + +#else : + + ;Called from app to record stack trace as return addresses in buffer + ;Demarcate end of each coroutine stack with -2 and end trace with -1 + lostanza defn profile-stack-trace () -> ref : + val vms:ptr = call-prim flush-vm() + [vms.profile-flag] = 2L + val buffer = vms.profile-buffer + labels : + begin : + goto loop(current-coroutine) + loop (co:ref) : + val stack = co.stack + val end-sp = stack.stack-pointer + labels : + begin : goto loop-frame(stack.frames) + loop-frame (sp:ptr) : + if sp.return > 0L : + add(buffer, sp.return) + if sp < end-sp : ;Continue if we're not at the end of the stack + val map-index = sp.liveness-map + val stackmap = vms.stackmap-table[map-index] + goto loop-frame(sp + stackmap.size) + add(buffer, -2L) ;End coroutine stack marker + match(co.parent) : + (p:ref) : goto loop(p) + (p:ref) : () + add(buffer, -1L) ;End stack trace marker + [vms.profile-flag] = 0L + return false + + ;Remove empty stack traces + defn prune-traces (traces:Vector>) -> List> : + to-list $ seq(unique, filter({ not empty?(_) }, traces)) + + ;Count all functions references in stack traces where traces have function ids in them + defn count-traces (len:Int, traces:Vector>) -> Array : + val counts = Array(len, 0L) + for trace in traces do : + for id in trace do : + counts[id] = counts[id] + 1L + counts + + ;Split stack trace buffer using delimiter + defn split-buffer (buffer:List, delimiter:Long) -> List> : + let loop (i:Int = 0, buffers:List> = List(), ids:List = List()) : + if i < length(buffer) : + val id = buffer[i] + if id == delimiter : + loop(i + 1, cons(reverse(ids), buffers), List()) + else : + loop(i + 1, buffers, cons(id, ids)) + else : + reverse(buffers) + + ;Split stack trace buffer using delimiter + defn split-buffer (buffer:Tuple, delimiter:Long) -> Vector> : + val res = Vector>() + val chunk = Vector() + for id in buffer do : + if id == delimiter : + add(res, to-tuple(chunk)) + clear(chunk) + else : + add(chunk, id) + res + + ;Reconsititue full stack traces by splitting up chunks and ordering them properly + defn build-stack-traces (buffer:Tuple) -> Tuple> : + val buffers = split-buffer(buffer, -1L) + ;same as: + ; to-tuple $ for trace in buffers seq : + ; to-tuple $ reverse $ to-list $ cat-all $ seq(reverse, split-buffer(trace, -2L)) + to-tuple $ for (trace in buffers, k in 0 to false) seq : + val res = Vector() + val chunks = split-buffer(trace, -2L) ;split full stack trace into coroutine stack traces + for i in (length(chunks) - 1) through 0 by -1 do : + val chunk = chunks[i] + for j in 0 to length(chunk) do : + add(res, chunk[j]) + to-tuple(res) + + ;Create function id traces and function info results after profiling + lostanza defn collect-profiling () -> ref : + val vms:ptr = call-prim flush-vm() + val info:ptr = vms.function-info + val counters:ptr = vms.function-counters + val buffer:ptr = vms.profile-buffer + val tab = build-stack-trace-record-table(vms.stack-trace-table) ;For fast lookup of return addresses + val buffer* = Vector() + + for (var i:long = 0, i < buffer.length, i = i + 1) : + add(buffer*, new Long{buffer.items[i]}) + + val traces = build-stack-traces(to-tuple(buffer*)) ;Order and concatenate stack trace chunks + + ;Lookup stack trace records and then translate into function ids + val id-traces = Vector>() + ; for (var i:long = 0, i < length(traces).value, i = i + 1) : + ; val trace = get(traces, new Int{i as int}) + ; val trace* = Vector() + ; for (var j:long = 0, j < length(trace).value, j = j + 1) : + ; val ret = get(trace, new Int{j as int}) + ; val entry = stack-trace-record(ret.value, vms.stack-trace-table, tab) + ; if entry != null : + ; val id = entry.function + ; if id != -1L : + ; add(trace*, new Int{id as int}) + ; add(id-traces, to-tuple(trace*)) + + val len:long = info.length + val counts = count-traces(new Int{len as int}, id-traces) + val infos = Vector() + + ;Collect all FunctionInfos corresponding to ids that occurred on stack at least once + for (var i:long = 0L, i < len, i = i + 1) : + val entry = info.entries[i] + val fi = FileInfo(String(entry.file), new Int{entry.line}, new Int{entry.column}) + val count = get(counts, new Int{i as int}) + val package = String(entry.package) + val name = String(entry.name) + val ci = ProfileInfo(new Int{i as int}, package, name, fi, count) + add(infos, ci) + + return ProfileResult(to-tuple(infos), to-tuple(id-traces)) + + ;Collect all coverage information from all functions + lostanza defn collect-coverage () -> ref> : + val vms:ptr = call-prim flush-vm() + val info:ptr = vms.function-info + val counters:ptr = vms.function-counters + val len:long = info.length + val res = Vector() + for (var i:long = 0L, i < len, i = i + 1) : + val entry = info.entries[i] + val fi = FileInfo(String(entry.file), new Int{entry.line}, new Int{entry.column}) + val count = new Long{counters[i]} + val package = String(entry.package) + val name = String(entry.name) + val ci = ProfileInfo(new Int{i as int}, package, name, fi, count) + add(res, ci) + return res + + ;Clear profiling state ready for next run + public lostanza defn clear-profiling () -> ref : + val vms:ptr = call-prim flush-vm() + [vms.profile-flag] = 0L + vms.profile-buffer.length = 0 + return false + + ;Clear coverage state ready for next run + public lostanza defn clear-coverage () -> ref : + val vms:ptr = call-prim flush-vm() + val info:ptr = vms.function-info + val counters:ptr = vms.function-counters + val len:long = info.length + call-c clib/memset(counters, 0, len << 3L) + return false + + var PROFILING-MSECS:Int = 0 + + ;Start profiling by initializing state and starting profile thread + ; extern start_sample_profiling: (int, long, ptr, ptr) -> int + lostanza defn do-start-sample-profiling (msecs:ref) -> ref : + call-c clib/printf("DO START PROFILING\n") + val vms:ptr = call-prim flush-vm() + [vms.profile-flag] = 0L + vms.profile-buffer.length = 0 + PROFILING-MSECS = msecs + ; val res = call-c start_sample_profiling(msecs.value, vms.function-info.length, vms.profile-flag, vms.function-counters) + val res = 0 + if res == 0 : + return false + else : + return true + + ;Stop profiling by clearing state and stopping profile thread + ; extern stop_sample_profiling: () -> int + lostanza defn do-stop-sample-profiling () -> ref : + call-c clib/printf("DO STOP PROFILING\n") + val vms:ptr = call-prim flush-vm() + [vms.profile-flag] = 0L + ; call-c stop_sample_profiling() + return PROFILING-MSECS + +;dumps coverage results to given file +; must be stanza compiled with -coverage flag as well +public defn dump-coverage (filename:String) : + val file = FileOutputStream(filename) + try : + val coverage = collect-coverage() + for elt in reverse(to-list(lazy-qsort(count, coverage))) do : + if count(elt) > 0L : + println(file, "%_: %_/%_ in %_ (%_)" % [count(elt), package(elt), name(elt), info(elt), id(elt)]) + finally: close(file) + +;Coverage wrapper to be called with within which +; runs thunk with coverage counting on and then dumps coverage report to file +public defn coverage (f:() -> ?, filename:String) : + clear-coverage() + f() + dump-coverage(filename) + +; starts profiling sampling at msecs period +public defn start-profiling (msecs:Int) : + clear-profiling() + do-start-sample-profiling(msecs) + false + +; dumps profiling results to given file +; must be stanza compiled with -profile flag as well +public defn stop-profiling (filename:String) : + val msecs = do-stop-sample-profiling() + val profiling = collect-profiling() + if not empty?(info(profiling)) : + ; within (file) = with-output-file(filename) : + val file = FileOutputStream(filename) + try : + println(file, msecs) + println(file, "(") + do(println{file, _}, id-traces(profiling)) + println(file, ")") + println(file, "(") + for (elt in info(profiling), i in 0 to false) do : + if count(elt) > 0L : + println(file, "(%_ %~)" % [id(elt), string-join $ [package(elt) "/" name(elt) ":" line(info(elt))]]) + println(file, ")") + finally: close(file) + false + +;Profiling wrapper to be called with within +public defn profiling (f:() -> ?, msecs:Int, filename:String) : + start-profiling(msecs) + f() + stop-profiling(filename) + ;############################################################ ;################## Math Package ############################ ;############################################################ diff --git a/runtime/driver.c b/runtime/driver.c index 2222b12a..f67b49eb 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -592,6 +592,128 @@ void stz_memory_resize (void* p, stz_long old_size, stz_long new_size) { #endif +//============================================================ +//==================== Profiler ============================== +//============================================================ + +#if defined(PLATFORM_OS_X) || defined(PLATFORM_LINUX) + +#include +#include + +static pthread_t main_thread_handle; + +static pthread_t thread_create(int thread_routine(void *parameter), void *parameter) { + pthread_attr_t attr; + pthread_t os_thread; + + if (pthread_attr_init(&attr) != 0) { + return 0; + } + + // Don't leave thread/stack around after exit for join: + if (pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) != 0) { + return 0; + } + + // Apparently, it is a different type on Linux. + void* (*routine)(void*) = (void*(*)(void *))thread_routine; + + if (pthread_create(&os_thread, &attr, routine, NULL) != 0) { + return 0; + } + return os_thread; +} + +static bool ticker_stopped = false; +static bool ticker_running = false; +static bool ticker_stopping = false; +static bool ticker_created = false; +static bool EnableTicks = true; +static int TickInterval = 100; // mSec + +static uint64_t *profile_flag; +static uint64_t *function_counters; +static int num_functions; + +// Timer thread that sets flag every tickinterval msecs +static int ticker_thread_routine(void *parameter) { + ticker_stopped = false; + while (!ticker_stopping) { + usleep(TickInterval * 1000); + + if (ticker_running) { + if (*profile_flag != 2L) // Not still profiling stack trace? + *profile_flag = 1L; // Profile next stack trace + } + } + ticker_stopped = true; + return 0; +} + +bool start_ticks() { + if (!EnableTicks) { + return true; + } + ticker_running = true; + ticker_stopping = false; + if (!ticker_created) { + ticker_created = true; + if (thread_create(ticker_thread_routine, 0) == 0) { + return false; + } + } + return true; +} + +static void suspend_ticks() { + ticker_running = false; + sleep(1); // why is this necessary? +} + +static void resume_ticks() { + start_ticks(); +} + +static void stop_ticks() { + if (ticker_created) { + ticker_stopping = true; + if (ticker_running) { + for (int i=0; i<10 && !ticker_stopped; i++) { // Wait for thread to die + usleep(TickInterval * 1000); + } + } + ticker_created = false; + ticker_running = false; + ticker_stopped = false; + } +} + +int start_sample_profiling (int msecs, int num_functions_arg, uint64_t *profile_flag_arg, uint64_t *function_counters_arg) { + TickInterval = msecs; + num_functions = num_functions_arg; + profile_flag = profile_flag_arg; + function_counters = function_counters_arg; + start_ticks(); + return 1; +} + +int stop_sample_profiling() { + stop_ticks(); + return 1; +} + +#else + +int start_profile_sampling (void *handler, int usecs) { + return 0; +} +int stop_sample_profiling () { + return 0; +} + +#endif + //============================================================ //================= Process Runtime ========================== //============================================================ From 8efeaf1cafddd52c5a36f135d310b69500aa639a Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Tue, 7 Jan 2025 08:58:22 -0800 Subject: [PATCH 05/38] profile/coverage function/build interfaces in place --- compiler/compiler-build-settings.stanza | 2 ++ compiler/compiler-main.stanza | 8 +++++--- compiler/compiler.stanza | 7 ++++++- compiler/defs-db.stanza | 2 ++ compiler/el.stanza | 10 +++++----- compiler/main.stanza | 26 ++++++++++++++++++++++--- compiler/proj-field-types.stanza | 8 ++++++-- compiler/proj-ir.stanza | 2 ++ compiler/proj-reader.stanza | 6 +++++- compiler/proj-stage0.stanza | 4 +++- compiler/repl.stanza | 2 +- 11 files changed, 60 insertions(+), 17 deletions(-) diff --git a/compiler/compiler-build-settings.stanza b/compiler/compiler-build-settings.stanza index d0bd2225..154fc36e 100644 --- a/compiler/compiler-build-settings.stanza +++ b/compiler/compiler-build-settings.stanza @@ -22,6 +22,8 @@ public defstruct BuildSettings : build-from-source?: True|False optimize?: True|False debug?: True|False + profile?: True|False + coverage?: True|False ccfiles: Tuple ccflags: Tuple> flags: Tuple diff --git a/compiler/compiler-main.stanza b/compiler/compiler-main.stanza index 9a746ee5..f0d770d0 100644 --- a/compiler/compiler-main.stanza +++ b/compiler/compiler-main.stanza @@ -158,6 +158,8 @@ public defn compile (proj-manager:ProjManager, backend:Backend, optimize?:True|False, debug?:True|False, + profile?:True|False, + coverage?:True|False, macro-plugins:Tuple, force-build-macros?:True|False) -> CompilationResult : defn driver () : @@ -224,7 +226,7 @@ public defn compile (proj-manager:ProjManager, `unoptimized-asm : val packages = for p in /packages(result) map : match(p) : - (p:EPackage) : compile(lower-unoptimized(p), debug?) + (p:EPackage) : compile(lower-unoptimized(p, profile?, coverage?), debug?) (p:StdPkg) : p compile-vmpackages(pkgsaver when write-out-pkgs?, packages, @@ -244,7 +246,7 @@ public defn compile (proj-manager:ProjManager, val epackages = for p in packages map : match(p:FastPkg) : EPackage(packageio(p), exps(p)) else : p as EPackage - compile(lower-optimized(epackages), debug?) + compile(lower-optimized(epackages, profile?, coverage?), debug?) ;- pkgsaver: False if we not need to write out any .pkg files to disk (either ; to pkg dir or to pkg cache.). @@ -309,7 +311,7 @@ public defn compile (proj-manager:ProjManager, for package in packages do : match(package) : (epackage:EPackage) : - val vmpackage = compile(lower-unoptimized(epackage), debug?) + val vmpackage = compile(lower-unoptimized(epackage, profile?, coverage?), debug?) val npkg = normalize(vmpackage, backend) val buffer = Vector() emit-normalized-package(npkg, buffer-emitter(buffer, stubs), stubs) diff --git a/compiler/compiler.stanza b/compiler/compiler.stanza index 825d7765..1abf5ba4 100644 --- a/compiler/compiler.stanza +++ b/compiler/compiler.stanza @@ -69,6 +69,9 @@ defn compute-build-settings (projenv:StandardProjEnv, val build-optimization = optimize?(settings) or optimize(s) ;Compute build optimization level val build-debug = debug?(settings) + ;Compute build profile level + val build-profile = profile?(settings) or profile(s) + val build-coverage = coverage?(settings) or coverage(s) ;Compute assembly val build-asm = string-or?(original-asm(settings), value?(assembly(s))) ;Compute output @@ -110,6 +113,8 @@ defn compute-build-settings (projenv:StandardProjEnv, build-from-source?(settings) build-optimization, build-debug, + build-profile, + build-coverage, build-ccfiles, build-ccflags, to-tuple(build-flags), @@ -246,7 +251,7 @@ public defn compile (settings:BuildSettings, system:System) : val proj-params = ProjParams(compiler-flags(), optimize?(settings*), debug?(settings*), false, build-from-source?(settings), pkg-cache-dir(settings*)) val proj-manager = ProjManager(proj, proj-params, auxfile) val comp-result = compile(proj-manager, auxfile, build-inputs!(settings*), vm-packages(settings*), asm?(settings*), pkg-dir(settings*), - backend(platform(settings*) as Symbol), optimize?(settings*), debug?(settings*), + backend(platform(settings*) as Symbol), optimize?(settings*), debug?(settings*), profile?(settings*), coverage?(settings*), macro-plugins(settings*), inputs(settings) is BuildTarget) save(auxfile) within delete-temporary-file-on-finish(settings*) : diff --git a/compiler/defs-db.stanza b/compiler/defs-db.stanza index 02e8fa14..7307c698 100644 --- a/compiler/defs-db.stanza +++ b/compiler/defs-db.stanza @@ -99,6 +99,8 @@ protected-when(TESTING) defn analyze-input (input:DefsDbInput) -> DependencyResu true, ;build-from-source? optimize?(input), ;optimize? false, ;debug? + false, ;profile? + false, ;coverage? [], ;ccfiles [], ;ccflags flags(input) ;flags diff --git a/compiler/el.stanza b/compiler/el.stanza index 064d784c..830c9394 100644 --- a/compiler/el.stanza +++ b/compiler/el.stanza @@ -31,17 +31,17 @@ defpackage stz/el : ;==================== Drivers =============================== ;============================================================ -public defn lower-optimized (epackages:Tuple) -> EPackage : +public defn lower-optimized (epackages:Tuple, profile?:True|False, coverage?:True|False) -> EPackage : vprintln("EL: Lower optimized packages: %," % [seq(name, epackages)]) val epackages* = map(fill-stack-trace-entries, epackages) ;do(dump{_, "logs", "input"}, epackages*) val collapsed = collapse(epackages*) ;dump(collapsed, "logs", "collapsed") - lower(collapsed, true) + lower(collapsed, true, profile?, coverage?) -public defn lower-unoptimized (epackage:EPackage) -> EPackage : +public defn lower-unoptimized (epackage:EPackage, profile?:True|False, coverage?:True|False) -> EPackage : vprintln("EL: Lower unoptimized package: %_" % [name(epackage)]) - lower(fill-stack-trace-entries(epackage), false) + lower(fill-stack-trace-entries(epackage), false, profile?, coverage?) ;============================================================ ;===================== Timers =============================== @@ -56,7 +56,7 @@ val EL-TIMERS = within pass-name = HashTable-init() : ;========================= Lowering ========================= ;============================================================ -defn lower (epackage:EPackage, optimize?:True|False) -> EPackage : +defn lower (epackage:EPackage, optimize?:True|False, profile?:True|False, coverage?:True|False) -> EPackage : val lower-timer-name = to-string("EL-LOWER -- %_" % [name(epackage)]) within log-time(EL-LOWER, lower-timer-name) : diff --git a/compiler/main.stanza b/compiler/main.stanza index 4538d4d7..7ee62eec 100644 --- a/compiler/main.stanza +++ b/compiler/main.stanza @@ -229,6 +229,10 @@ val COMMON-STANZA-FLAGS = [ "Requests the compiler to compile in optimized mode.") Flag("debug", ZeroFlag, OptionalFlag, "Requests the compiler to compile for debugging.") + Flag("profile", ZeroFlag, OptionalFlag, + "Requests the compiler to add profile counters.") + Flag("coverage", ZeroFlag, OptionalFlag, + "Requests the compiler to add coverage counters.") Flag("link", OneFlag, OptionalFlag, "Provide the type of linking to use.") Flag("ccfiles", ZeroOrMoreFlag, OptionalFlag, @@ -333,6 +337,8 @@ defn compile-command () : flag?(cmd-args, "build-from-source") flag?(cmd-args, "optimize") flag?(cmd-args, "debug") + flag?(cmd-args, "profile") + flag?(cmd-args, "coverage") get?(cmd-args, "ccfiles", []) ccflags(cmd-args) map(to-symbol, get?(cmd-args, "flags", [])) @@ -348,7 +354,8 @@ defn compile-command () : Command("compile", AtLeastOneArg, "the .stanza/.proj input files or Stanza package names.", common-stanza-flags(["o" "s" "pkg" "pkg-cache" "build-from-source" "optimize" "debug" "ccfiles" "ccflags" "flags" - "verbose" "supported-vm-packages" "platform" "external-dependencies" "macros" "link" "timing-log"]), + "verbose" "supported-vm-packages" "platform" "external-dependencies" "macros" "link" "timing-log" + "profile" "coverage"]), compile-msg, false, verify-args, intercept-no-match-exceptions(compile-action)) @@ -383,6 +390,8 @@ defn build-command () : flag?(cmd-args, "build-from-source") flag?(cmd-args, "optimize") flag?(cmd-args, "debug") + flag?(cmd-args, "profile") + flag?(cmd-args, "coverage") [] ccflags(cmd-args) map(to-symbol, get?(cmd-args, "flags", [])) @@ -397,7 +406,8 @@ defn build-command () : ;Command definition Command("build", ZeroOrOneArg, "the name of the build target. If not supplied, the default build target is 'main'.", - common-stanza-flags(["s" "o" "external-dependencies" "pkg" "pkg-cache" "build-from-source" "flags" "optimize" "debug" "verbose" "ccflags" "macros" "link" "timing-log"]), + common-stanza-flags(["s" "o" "external-dependencies" "pkg" "pkg-cache" "build-from-source" "flags" "optimize" "debug" + "verbose" "ccflags" "macros" "link" "timing-log" "profile" "coverage"]), build-msg, false, verify-args, intercept-no-match-exceptions(build)) ;============================================================ @@ -449,6 +459,8 @@ defn extend-command () : false flag?(cmd-args, "optimize") false + false + false get?(cmd-args, "ccfiles", []) ccflags(cmd-args) map(to-symbol, get?(cmd-args, "flags", [])) @@ -518,6 +530,8 @@ defn compile-test-command () : flag?(cmd-args, "build-from-source") flag?(cmd-args, "optimize") flag?(cmd-args, "debug") + flag?(cmd-args, "profile") + flag?(cmd-args, "coverage") get?(cmd-args, "ccfiles", []) ccflags(cmd-args) new-flags @@ -531,7 +545,7 @@ defn compile-test-command () : Command("compile-test", AtLeastOneArg, "the .stanza/.proj input files or Stanza package names containing tests.", common-stanza-flags(["platform" "s" "o" "external-dependencies" "pkg" "pkg-cache" "build-from-source" "ccfiles" "ccflags" "flags" "optimize" "debug" "verbose" - "macros" "link" "timing-log"]) + "macros" "link" "timing-log" "profile" "coverage"]) compile-test-msg, false, verify-args, intercept-no-match-exceptions(compile-test)) ;============================================================ @@ -583,6 +597,8 @@ defn compile-macros-command () : flag?(cmd-args, "build-from-source") flag?(cmd-args, "optimize") false + false + false get?(cmd-args, "ccfiles", []) ccflags(cmd-args) get?(cmd-args, "flags", []) @@ -988,6 +1004,8 @@ defn analyze-dependencies-command () : flag?(cmd-args, "build-from-source") flag?(cmd-args, "optimize") false + false + false [] [] map(to-symbol, get?(cmd-args, "flags", [])) @@ -1062,6 +1080,8 @@ defn auto-doc-command () : false flag?(cmd-args, "optimize") false + false + false [] [] map(to-symbol, get?(cmd-args, "flags", [])) diff --git a/compiler/proj-field-types.stanza b/compiler/proj-field-types.stanza index c5568592..9d38ff59 100644 --- a/compiler/proj-field-types.stanza +++ b/compiler/proj-field-types.stanza @@ -161,7 +161,9 @@ defmethod map-fields (s:BuildStmtS0, mapper:Mapper) : map-field(mapper, ccfiles(s), MULTIPLE-PATHS) map-field(mapper, ccflags(s), MULTIPLE-FLAGS) map-field(mapper, flags(s), MULTIPLE-SYMBOLS) - optimize(s)) + optimize(s) + profile(s) + coverage(s)) defmethod convert (s:BuildStmtS0, extractor:Extractor) : BuildStmt( name(s) @@ -175,7 +177,9 @@ defmethod convert (s:BuildStmtS0, extractor:Extractor) : extract-tuple(extractor, ccfiles(s), MULTIPLE-PATHS) extract-tuple(extractor, ccflags(s), MULTIPLE-FLAGS) extract-tuple(extractor, flags(s), MULTIPLE-SYMBOLS) - optimize(s)) + optimize(s) + profile(s) + coverage(s)) defmethod map-fields (s:SyntaxPackagesDefinedInStmtS0, mapper:Mapper) : SyntaxPackagesDefinedInStmtS0( diff --git a/compiler/proj-ir.stanza b/compiler/proj-ir.stanza index 0b658870..817b0b86 100644 --- a/compiler/proj-ir.stanza +++ b/compiler/proj-ir.stanza @@ -79,6 +79,8 @@ public defstruct BuildStmt <: ProjStmt : ccflags: Tuple> flags: Tuple optimize: True|False + profile: True|False + coverage: True|False with: printer => true diff --git a/compiler/proj-reader.stanza b/compiler/proj-reader.stanza index 2c3e50ca..c7d93b27 100644 --- a/compiler/proj-reader.stanza +++ b/compiler/proj-reader.stanza @@ -126,7 +126,9 @@ defsyntax stanza-projfile : entry?(bs, `ccfiles) entry?(bs, `ccflags) entry?(bs, `flags) - entry?(bs, `optimize, false)) + entry?(bs, `optimize, false) + entry?(bs, `profile, false) + entry?(bs, `coverage, false)) defrule projstmt = (var ?name:#symbol! = ?v:#projvalue!) : VarStmtS0(closest-info(), name, v) @@ -201,6 +203,8 @@ defsyntax stanza-projfile : defrule build-option! = (ccflags #:! ?v:#projvalue!) : `ccflags => v defrule build-option! = (flags #:! ?v:#projvalue!) : `flags => v defrule build-option! = (optimize) : `optimize => true + defrule build-option! = (profile) : `profile => true + defrule build-option! = (coverage) : `coverage => true ;---------------------------------------------------------- ;--------------------- Error Productions ------------------ diff --git a/compiler/proj-stage0.stanza b/compiler/proj-stage0.stanza index aaab5020..e2b2d739 100644 --- a/compiler/proj-stage0.stanza +++ b/compiler/proj-stage0.stanza @@ -81,6 +81,8 @@ public defstruct BuildStmtS0 <: ProjStmt : ccflags: Maybe flags: Maybe optimize: True|False + profile: True|False + coverage: True|False with: (printer => true) ;Example: @@ -315,7 +317,7 @@ public defn map (f:ProjItem -> ProjItem, x:?T&ProjItem) -> T : h(dependencies(x)), h(foreign-packages(x)), h(commands(x))) (x:BuildStmtS0) : BuildStmtS0(info(x), name(x), type(x), h(inputs(x)), h(supported-vm-packages(x)), h(pkg(x)), h(output(x)), h(assembly(x)), h(external-dependencies(x)), - h(ccfiles(x)), h(ccflags(x)), h(flags(x)), optimize(x)) + h(ccfiles(x)), h(ccflags(x)), h(flags(x)), optimize(x), profile(x), coverage(x)) (x:SyntaxPackagesDefinedInStmtS0) : SyntaxPackagesDefinedInStmtS0(info(x), packages(x), h(filename(x))) (x:VarStmtS0) : VarStmtS0(info(x), name(x), h(value(x))) (x:ForeignPackageParamsStmtS0) : ForeignPackageParamsStmtS0(info(x), package-manager(x), h(projdir(x)), h(entries(x))) diff --git a/compiler/repl.stanza b/compiler/repl.stanza index bd1a93ed..e9cb92f6 100644 --- a/compiler/repl.stanza +++ b/compiler/repl.stanza @@ -494,7 +494,7 @@ public defn REPL (macro-plugins:Tuple, vprintln("REPL: Compiling EPackage %_ to VMPackage." % [name(p)]) val vmp = within log-time(REPL-LOWERING-AND-COMPILATION, suffix(name(p))) : val lowered = within log-time(REPL-LOWER-TO-UNOPTIMIZED) : - lower-unoptimized(p) + lower-unoptimized(p, false, false) within log-time(REPL-COMPILE-LOWERED) : compile(lowered, false) vprintln("REPL: Done Compiling EPackage %_ to VMPackage." % [name(p)]) From 95566812e0c79af8870c4ca8280d43a1de64adc1 Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Tue, 7 Jan 2025 09:20:16 -0800 Subject: [PATCH 06/38] add profiling instructions --- compiler/dl-ir.stanza | 1 + compiler/el-ir.stanza | 11 +++++++ compiler/el.stanza | 59 ++++++++++++++++++++++++++++++++++++ compiler/proj-reader.stanza | 3 +- compiler/reg-alloc.stanza | 17 ++++++++++- compiler/vm-ir.stanza | 5 +++ compiler/vm-normalize.stanza | 20 ++++++++++++ 7 files changed, 114 insertions(+), 2 deletions(-) diff --git a/compiler/dl-ir.stanza b/compiler/dl-ir.stanza index 1383cc2f..ae2bd523 100644 --- a/compiler/dl-ir.stanza +++ b/compiler/dl-ir.stanza @@ -1192,6 +1192,7 @@ public val CORE-COLLECT-STACK-TRACE-ID = register $ core-fnid(`collect-stack-tra public val CORE-COLLECT-GARBAGE-ID = register $ core-fnid(`collect-garbage, [`long]) public val CORE-MAKE-STRING-ID = register $ core-fnid(`String, [DPtrT(DByte())]) public val CORE-EXECUTE-TOPLEVEL-COMMAND-ID = register $ core-fnid(`execute-toplevel-command, [DArrow([], core-type(`False))]) +public val CORE-PROFILE-STACK-TRACE-ID = register $ core-fnid(`profile-stack-trace) ;============================================================ ;====================== ID Environment ====================== diff --git a/compiler/el-ir.stanza b/compiler/el-ir.stanza index b5c3e446..6ed1a3a2 100644 --- a/compiler/el-ir.stanza +++ b/compiler/el-ir.stanza @@ -545,6 +545,10 @@ public defstruct DetupleContext <: CastErrorContext public defstruct GetVarContext <: CastErrorContext public defstruct BranchContext <: CastErrorContext +public defstruct EProfile <: EIns : + id: Int + kind: Int + ;============================================================ ;======================= Primitives ========================= ;============================================================ @@ -762,6 +766,7 @@ defmethod print (o:OutputStream, e:EIns) : (e:ETypeof) : P $ "goto L%_ when typeof(%_, %_) else L%_" % [n1(e), y(e), type(e), n2(e)] (e:ECheckFail) : P $ "check-fail(%_) %_ : %_" % [ctxt(e), y(e), type(e)] (e:ESafepoint) : P $ "safepoint %_ ;%_" % [group-id(e), info?(e)] + (e:EProfile) : P $ "profile(%_,%_)" % [id(e), kind(e)] defmethod print (o:OutputStream, t:CallType) : print{o, _} $ match(t) : @@ -1002,6 +1007,7 @@ defsyntax el-ir : defrule eins = (?x:#vloc : ?xt:#etype = load ?loc:#eloc) : ELoad(x, xt, loc) defrule eins = (?x:#vloc = call ?t:#calltype ?f:#eimm(?ys:#eimm ...)) : ECall(x, f, to-tuple(ys), t, add-base-dir(closest-info())) defrule eins = (?x:#vloc = ?y:#eimm) : EDef(x, y) + defrule eins = (profile(?id:#int, ?arg:#int)) : EProfile(id, arg) defproduction calltype : CallType defrule calltype = (guarded((?a1:#etype ...) -> ?a2:#etype)) : CallGuarded(to-tuple(a1), a2) @@ -1421,6 +1427,9 @@ defmethod map (f:ELItem -> ELItem, item:ELItem&?T) -> ELItem&T : ;Closures (e:EClosure) : EClosure(n(e), h(targs(e)), h(ys(e))) + ;Profile + (e:EProfile) : EProfile(id(e), kind(e)) + ;============================================================ ;===================== Definitions ========================== ;============================================================ @@ -1479,6 +1488,7 @@ public defn varlocs (e:EIns) -> Tuple : (e:ETypeof) : [] (e:ECheckFail) : [] (e:ESafepoint) : [] + (e:EProfile) : [] ;Return the list of immediates that the instruction ;uses to perform its action. @@ -1533,6 +1543,7 @@ public defn uses (e:EIns) -> Seqable : (e:ETypeof) : [y(e)] (e:ECheckFail) : [y(e)] (e:ESafepoint) : [] + (e:EProfile) : [] ;Return the list of immediates that the instruction ;uses to retrieve its location. diff --git a/compiler/el.stanza b/compiler/el.stanza index 830c9394..535fa350 100644 --- a/compiler/el.stanza +++ b/compiler/el.stanza @@ -105,6 +105,8 @@ defn lower (epackage:EPackage, optimize?:True|False, profile?:True|False, covera run-pass("Box Mutables", box-mutables, "boxed", false) run-pass("Detect Loops", detect-loops, "looped", false) run-pass("Simple Inline", simple-inline, "inlined0", false) + if profile? or coverage? : + run-pass("Profile", insert-profiling{_, coverage?}, "profile", true) run-pass("Within Package Inline", within-package-inline{_, true}, "wp-inlined0", false) run-pass("Cleanup Labels", cleanup-labels, "cleanup-labels", false) if optimize? : @@ -1754,6 +1756,8 @@ defn inline-call (buffer:BodyBuffer, emit(buffer, ESafepoint(group-id(i), info)) (call-info, instruction-info): emit(buffer, i) + (i:EProfile) : + false (i) : emit(buffer, i) @@ -2129,6 +2133,61 @@ defn rename-fn (f:EFn) -> EFn : create-definitions(f) rename(f) as EFn +;============================================================ +;=================== Insert Profiling ======================= +;============================================================ + +defn insert-profiling (epackage:EPackage, coverage?:True|False) : + val infos = Vector() + val iotable = IOTable(packageio(epackage)) + val num-uses = IntTable(0) + defn get-info (sinfo:False|TraceInfo, body:EBody) -> False|StackTraceInfo : + match(sinfo) : + (sinfo: StackTraceInfo) : sinfo + (sinfo) : + label return : + for i in ins(body) do : + match(info?(i)) : + (info:StackTraceInfo) : return(info) + (info) : false + + defn insert (e:EBody, sinfo:StackTraceInfo, insert?:True|False) -> EBody : + val buffer = BodyBuffer(e) + val id = length(infos) + emit(buffer, EProfile(id, 1 when coverage? else 0)) when insert? + for i in ins(e) do : + val i* = + match(info?(i)) : + ; (info:StackTraceInfo) : sub-info(i, sub-function(info, id)) + (info:StackTraceInfo) : i + (info) : i + emit(buffer, i*) + if insert? and not coverage? : + match(i:ELabel) : + emit(buffer, EProfile(id, 0)) when (num-uses[n(i)] == 0) + else : + for v in label-uses(i) do : + update(num-uses, {_ + 1}, v) + add(infos, FunctionInfo(sinfo)) + to-body(buffer) + + defn insert-texp (t:ELBigItem, insert?:True|False) -> ELBigItem : + match(map(insert-texp{_, insert?}, t)) : + (e:EFn) : + ; println("%_ INFO %_" % [length(infos), get-info(info(e), body(e))]) + match(get-info(info(e), body(e))) : + (i:StackTraceInfo) : sub-body(e, insert(body(e), i, insert?)) + (i:False) : e + (e) : e + + ;Inline all top-level expressions + val new-exps = for exp in exps(epackage) map : + match(exp) : + (exp:EDefn) : insert-texp(exp, n(exp) != n(iotable,CORE-PROFILE-STACK-TRACE-ID)) as ETExp + (exp:EDefClosure|EDefmethod) : insert-texp(exp, true) as ETExp + (exp) : exp + sub-function-info(sub-exps(epackage, new-exps), to-tuple $ infos) + ;============================================================ ;==================== Label Cleanup ========================= ;============================================================ diff --git a/compiler/proj-reader.stanza b/compiler/proj-reader.stanza index c7d93b27..e946f52a 100644 --- a/compiler/proj-reader.stanza +++ b/compiler/proj-reader.stanza @@ -128,7 +128,8 @@ defsyntax stanza-projfile : entry?(bs, `flags) entry?(bs, `optimize, false) entry?(bs, `profile, false) - entry?(bs, `coverage, false)) + entry?(bs, `coverage, false) + ) defrule projstmt = (var ?name:#symbol! = ?v:#projvalue!) : VarStmtS0(closest-info(), name, v) diff --git a/compiler/reg-alloc.stanza b/compiler/reg-alloc.stanza index 135bc8e1..cf5aa70e 100644 --- a/compiler/reg-alloc.stanza +++ b/compiler/reg-alloc.stanza @@ -21,7 +21,8 @@ Constructs that must be handled, and their variants: Op - (AddOp / SubOp / MulOp / AndOp / OrOp / XorOp / NotOp / ShlOp / ShrOp / AshrOp / NegOp / EqOp / NeOp / LtOp / GtOp / LeOp / GeOp / UleOp / UltOp / UgtOp / UgeOp / FlushVMOp / LoadSpecialOp / DivModOp / NoOp / RecordLiveOp / LoadOp / - StoreOp / StoreArgOp / StoreSpecialOp / LoadArgOp / AllocOp / InstanceofOp) + StoreOp / StoreArgOp / StoreSpecialOp / LoadArgOp / AllocOp / InstanceofOp / + IncFunctionCounterOp / IsProfileOp) Branch - (EqOp / NeOp / LtOp / GtOp / LeOp / GeOp / UleOp / UltOp / UgtOp / UgeOp / HasHeapOp / HasStackOp / ArgEqOp) Set @@ -557,6 +558,10 @@ public defstruct FlipOp <: VMOp : (op:VMOp) defmethod print (o:OutputStream, x:FlipOp) : print(o, "flip(%_)" % [op(x)]) +public defstruct IncFunctionCounterOp <: VMOp : (id:Int) +defmethod print (o:OutputStream, x:IncFunctionCounterOp) : + print(o, "incfunctioncounter(%_)" % [id(x)]) + public defstruct SafepointOp <: VMOp : id:Int trace-entry:StackTraceInfo @@ -1014,6 +1019,8 @@ defn load-instructions (function:VMFunc, backend:Backend) : push(Op(SafepointOp(id(e), trace-entry!(e), []), List(), List())) (e:UnreachableIns) : false + (e:IncFunctionCounterIns) : + push(Op(IncFunctionCounterOp(id(e)), List(), List())) val next = to-list $ for s in succs(b) seq : index(blocks[s]) @@ -1412,6 +1419,7 @@ defn num-scratch-registers (i:Ins) : (i:Branch) : match(op(i)) : (op:HasHeapOp) : 4 + (op:IsProfileOp) : 4 (op) : 0 (i:Match) : 2 (i:MethodDispatch) : 0 @@ -1878,6 +1886,7 @@ defn register-assignment (blk:Block, b:Int, backend:Backend) : (e:Branch) : match(op(e)) : (op:HasHeapOp) : ensure-available(list-r0-r1, List()) + (op:IsProfileOp) : ensure-available(list-r0-r1, List()) (op) : false emit(Branch(op(e), map(annotate, xs(e)), killed?(e))) do(free-var, killed(e)) @@ -2837,12 +2846,18 @@ defn assemble (emitter:CodeEmitter, defn cmp-arity (arg:Int, value:Int, eq-op:asm-Op) : val arg-reg = R(call-regs(backend(stubs))[arg]) E $ BreakL(label-table[n(e)], eq-op, arg-reg, INT(value)) + defn cmp-profile-flag () : + val TMP = R0 + E $ LoadL(TMP, M(profile-flag(stubs))) + E $ BreakL(label-table[n(e)], asm-EqOp(), TMP, INT(1)) match(op(e)) : (op:HasStackOp) : cmp-stack-limit(asm-UleOp()) (op:HasHeapOp) : cmp-heap-limit(asm-UleOp()) + (op:IsProfileOp) : + cmp-profile-flag() (op:ArgEqOp) : cmp-arity(arg(op), value(op), asm-EqOp()) (fop:FlipOp) : diff --git a/compiler/vm-ir.stanza b/compiler/vm-ir.stanza index e8664371..f4c0db8e 100644 --- a/compiler/vm-ir.stanza +++ b/compiler/vm-ir.stanza @@ -313,6 +313,9 @@ public defstruct UnreachableIns <: VMIns public defstruct SafepointIns <: VMIns : id:Int trace-entry: StackTraceInfo with: (as-method => true) +public defstruct ProfileIns <: VMIns : + id : Int + kind : Int public defn trace-entry! (ins:SafepointIns) -> StackTraceInfo : trace-entry(ins) as StackTraceInfo @@ -880,6 +883,8 @@ defmethod print (o:OutputStream, i:VMIns) : val o2 = IndentedStream(o) lnprint(o2, "else : goto %_" % [default(i)]) lnprint(o2, "amb : goto %_" % [amb(i)]) + (i:ProfileIns) : + print(o, "profile(%_, %_)" % [id(i), kind(i)]) defmethod print (o:OutputStream, op:VMOp) : print{o, _} $ match(op) : diff --git a/compiler/vm-normalize.stanza b/compiler/vm-normalize.stanza index 19af3f70..903800cc 100644 --- a/compiler/vm-normalize.stanza +++ b/compiler/vm-normalize.stanza @@ -785,6 +785,19 @@ defn normalize (f0:VMFunction, databuffer:DataBuffer, backend:Backend, iotable:I emit(buffer, i) (i:SetIns) : emit(buffer, SetIns(x(i), #L(y(i)))) + (i:ProfileIns) : + if kind(i) == 0 : + val good-lbl = make-label(buffer) + val profile-lbl = make-label(buffer) + load-instruction(Branch0Ins(profile-lbl, good-lbl, IsProfileOp())) + load-instruction(LabelIns(profile-lbl)) + val profile-stack-trace = CodeId(n(iotable, CORE-PROFILE-STACK-TRACE-ID)) + val call-ins = CallIns([], profile-stack-trace, [false-marker(), NumConst(0)], false) + load-instruction(call-ins) + load-instruction(GotoIns(good-lbl)) + load-instruction(LabelIns(good-lbl)) + else : + load-instruction(IncFunctionCounterIns(id(i))) (i) : emit(buffer, i) @@ -916,6 +929,13 @@ with: (printer => true) public defstruct HasStackOp <: VMOp with: (printer => true) +public defstruct IsProfileOp <: VMOp +with: (printer => true) + +public defstruct IncFunctionCounterIns <: VMIns : + id: Int +with: (printer => true) + public defstruct ArgEqOp <: VMOp : arg: Int value: Int From 0e06a5467d72b3a04307bf5168221a7875152008 Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Tue, 7 Jan 2025 14:22:54 -0800 Subject: [PATCH 07/38] backtraces working now --- compiler/compiler.stanza | 3 ++- compiler/el.stanza | 3 +-- compiler/pkg-serializer.stanza | 2 +- compiler/stitcher.stanza | 2 ++ compiler/trace-info.stanza | 6 ++++-- compiler/vm-ir.stanza | 3 ++- compiler/vm.stanza | 2 +- core/core.stanza | 11 +++++++++++ core/stack-trace.stanza | 1 + 9 files changed, 25 insertions(+), 8 deletions(-) diff --git a/compiler/compiler.stanza b/compiler/compiler.stanza index 1abf5ba4..36606bd3 100644 --- a/compiler/compiler.stanza +++ b/compiler/compiler.stanza @@ -251,7 +251,8 @@ public defn compile (settings:BuildSettings, system:System) : val proj-params = ProjParams(compiler-flags(), optimize?(settings*), debug?(settings*), false, build-from-source?(settings), pkg-cache-dir(settings*)) val proj-manager = ProjManager(proj, proj-params, auxfile) val comp-result = compile(proj-manager, auxfile, build-inputs!(settings*), vm-packages(settings*), asm?(settings*), pkg-dir(settings*), - backend(platform(settings*) as Symbol), optimize?(settings*), debug?(settings*), profile?(settings*), coverage?(settings*), + backend(platform(settings*) as Symbol), optimize?(settings*), debug?(settings*), + profile?(settings*), coverage?(settings*), macro-plugins(settings*), inputs(settings) is BuildTarget) save(auxfile) within delete-temporary-file-on-finish(settings*) : diff --git a/compiler/el.stanza b/compiler/el.stanza index 535fa350..5dfc39c9 100644 --- a/compiler/el.stanza +++ b/compiler/el.stanza @@ -2158,8 +2158,7 @@ defn insert-profiling (epackage:EPackage, coverage?:True|False) : for i in ins(e) do : val i* = match(info?(i)) : - ; (info:StackTraceInfo) : sub-info(i, sub-function(info, id)) - (info:StackTraceInfo) : i + (info:StackTraceInfo) : sub-info(i, sub-function(info, id)) (info) : i emit(buffer, i*) if insert? and not coverage? : diff --git a/compiler/pkg-serializer.stanza b/compiler/pkg-serializer.stanza index 34e750ee..8c5c8bb8 100644 --- a/compiler/pkg-serializer.stanza +++ b/compiler/pkg-serializer.stanza @@ -671,7 +671,7 @@ defserializer PkgSerializer (include-asm?:True|False) : DerefOp: () CRSPOp: () - deftype trace-entry (StackTraceInfo) : (package:symbol, signature:opt(string), info:opt(absinfo)) + deftype trace-entry (StackTraceInfo) : (package:symbol, signature:opt(string), info:opt(absinfo), function:opt(int)) ;========================================================== ;================== Assembly Types ======================== diff --git a/compiler/stitcher.stanza b/compiler/stitcher.stanza index 8b511832..e49b977a 100644 --- a/compiler/stitcher.stanza +++ b/compiler/stitcher.stanza @@ -924,11 +924,13 @@ public defn Stitcher (packages:Collection, bindings:Bindings|False, s E $ DefLabel(string-lbls[filename(info)]) E $ DefInt(line(info)) E $ DefInt(column(info)) + E $ DefLong(-1L when (function(trace-entry) is False) else to-long(function(trace-entry) as Int)) (f:False) : E $ DefLong(0L) E $ DefLong(0L) E $ DefInt(0) E $ DefInt(0) + E $ DefLong(-1L) E $ DefText() E $ Comment("End of File Information Table") diff --git a/compiler/trace-info.stanza b/compiler/trace-info.stanza index 6c99fbbb..739aeffe 100644 --- a/compiler/trace-info.stanza +++ b/compiler/trace-info.stanza @@ -20,16 +20,18 @@ public defstruct StackTraceInfo : package:Symbol signature:String|False info:AbsoluteFileInfo|False + function:Int|False with: ( updater => sub-function ) with: constructor => #StackTraceInfo ;Sanity check: Ensure that we don't have signature and info as false. public defn StackTraceInfo (package:Symbol, signature:String|False, - info:AbsoluteFileInfo|False) : + info:AbsoluteFileInfo|False, + function:Int|False = false) : if signature is False and info is False : fatal("StackTraceInfo cannot have both signature and info be false.") - #StackTraceInfo(package, signature, info) + #StackTraceInfo(package, signature, info, function) ;Print in a human-readable way. ;e.g. function myfunction in mypackage at info diff --git a/compiler/vm-ir.stanza b/compiler/vm-ir.stanza index f4c0db8e..a1a907d2 100644 --- a/compiler/vm-ir.stanza +++ b/compiler/vm-ir.stanza @@ -32,7 +32,8 @@ public val EXTEND-HEAP-FN = 0 public val EXTEND-STACK-FN = 1 public val INIT-CONSTS-FN = 2 public val EXECUTE-TOPLEVEL-COMMAND-FN = 3 -public val NUM-BUILTIN-FNS = 4 +public val PROFILE-STACK-TRACE-FN = 4 +public val NUM-BUILTIN-FNS = 5 ;============================================================ ;================== Design of Instructions ================== diff --git a/compiler/vm.stanza b/compiler/vm.stanza index 28a5d979..42ab6e76 100644 --- a/compiler/vm.stanza +++ b/compiler/vm.stanza @@ -574,7 +574,7 @@ defn add-entries (b:StackTraceBuilder, es:Seqable) : ;------------------------------------------------------------ defn make-entry (ip:Long, sp:Long, st-info:StackTraceInfo) : - StackTraceEntry(ip, sp, package(st-info), signature(st-info), info(st-info)) + StackTraceEntry(ip, sp, package(st-info), signature(st-info), info(st-info), function(st-info)) lostanza defn collect-stack-trace-entries (stack:ptr, vmtable:ref, diff --git a/core/core.stanza b/core/core.stanza index 298b18f2..176ac9db 100644 --- a/core/core.stanza +++ b/core/core.stanza @@ -560,12 +560,23 @@ lostanza defn print-stack-trace (stack:ref) -> ref : val frame = buffer.items[i] as ptr val entry = stack-trace-record(frame, stack-trace-table) if entry != null : + call-c clib/fprintf(current-err, "lbl 0x%lx: " frame.return) print-record(entry) ;Free the buffer free(buffer) return false +public lostanza defn print-stack-trace-table () -> ref : + val vms:ptr = call-prim flush-vm() + val trace-table = vms.stack-trace-table + for (var i:int = 0, i < trace-table.length, i = i + 1) : + val e = addr(trace-table.entries[i]) + call-c clib/fprintf(current-err, "lbl 0x%lx: " e.lbl) + val r = addr(e.record) + call-c clib/printf("P %s S %s B %s F %s L %d C %d F %d\n", r.package, r.signature, r.base, r.file, r.line, r.column, r.function) + return false + lostanza defn print-record (record:ptr) -> ref : call-c clib/fprintf(current-err, " in %s", record.package) if record.signature != null : diff --git a/core/stack-trace.stanza b/core/stack-trace.stanza index ae2423a2..a02532f3 100644 --- a/core/stack-trace.stanza +++ b/core/stack-trace.stanza @@ -254,6 +254,7 @@ public lostanza deftype StackTraceRecord : file:ptr line:int column:int + function:long ;Add the given record as an entry in the stack trace. public lostanza defn add-entry (b:ref, From 7567ae852363f8df631869302f82d09390fdce5a Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Tue, 7 Jan 2025 17:48:32 -0800 Subject: [PATCH 08/38] turn on profiler in core and fix compilation issue with vm / make-entry --- compiler/vm.stanza | 2 +- core/core.stanza | 31 +++++++++++++++---------------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/compiler/vm.stanza b/compiler/vm.stanza index 42ab6e76..28a5d979 100644 --- a/compiler/vm.stanza +++ b/compiler/vm.stanza @@ -574,7 +574,7 @@ defn add-entries (b:StackTraceBuilder, es:Seqable) : ;------------------------------------------------------------ defn make-entry (ip:Long, sp:Long, st-info:StackTraceInfo) : - StackTraceEntry(ip, sp, package(st-info), signature(st-info), info(st-info), function(st-info)) + StackTraceEntry(ip, sp, package(st-info), signature(st-info), info(st-info)) lostanza defn collect-stack-trace-entries (stack:ptr, vmtable:ref, diff --git a/core/core.stanza b/core/core.stanza index 176ac9db..39d086b0 100644 --- a/core/core.stanza +++ b/core/core.stanza @@ -11359,17 +11359,17 @@ lostanza defn build-stack-trace-record-table (trace-table:ptr) ;Lookup stack trace records and then translate into function ids val id-traces = Vector>() - ; for (var i:long = 0, i < length(traces).value, i = i + 1) : - ; val trace = get(traces, new Int{i as int}) - ; val trace* = Vector() - ; for (var j:long = 0, j < length(trace).value, j = j + 1) : - ; val ret = get(trace, new Int{j as int}) - ; val entry = stack-trace-record(ret.value, vms.stack-trace-table, tab) - ; if entry != null : - ; val id = entry.function - ; if id != -1L : - ; add(trace*, new Int{id as int}) - ; add(id-traces, to-tuple(trace*)) + for (var i:long = 0, i < length(traces).value, i = i + 1) : + val trace = get(traces, new Int{i as int}) + val trace* = Vector() + for (var j:long = 0, j < length(trace).value, j = j + 1) : + val ret = get(trace, new Int{j as int}) + val entry = stack-trace-record(ret.value, vms.stack-trace-table, tab) + if entry != null : + val id = entry.function + if id != -1L : + add(trace*, new Int{id as int}) + add(id-traces, to-tuple(trace*)) val len:long = info.length val counts = count-traces(new Int{len as int}, id-traces) @@ -11423,27 +11423,26 @@ lostanza defn build-stack-trace-record-table (trace-table:ptr) var PROFILING-MSECS:Int = 0 ;Start profiling by initializing state and starting profile thread - ; extern start_sample_profiling: (int, long, ptr, ptr) -> int + extern start_sample_profiling: (int, long, ptr, ptr) -> int lostanza defn do-start-sample-profiling (msecs:ref) -> ref : call-c clib/printf("DO START PROFILING\n") val vms:ptr = call-prim flush-vm() [vms.profile-flag] = 0L vms.profile-buffer.length = 0 PROFILING-MSECS = msecs - ; val res = call-c start_sample_profiling(msecs.value, vms.function-info.length, vms.profile-flag, vms.function-counters) - val res = 0 + val res = call-c start_sample_profiling(msecs.value, vms.function-info.length, vms.profile-flag, vms.function-counters) if res == 0 : return false else : return true ;Stop profiling by clearing state and stopping profile thread - ; extern stop_sample_profiling: () -> int + extern stop_sample_profiling: () -> int lostanza defn do-stop-sample-profiling () -> ref : call-c clib/printf("DO STOP PROFILING\n") val vms:ptr = call-prim flush-vm() [vms.profile-flag] = 0L - ; call-c stop_sample_profiling() + call-c stop_sample_profiling() return PROFILING-MSECS ;dumps coverage results to given file From 33085a6c5d3094207e167df9e5ec923af3141da5 Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Wed, 8 Jan 2025 13:30:38 -0800 Subject: [PATCH 09/38] get profiling working at least for optimized mode only --- compiler/el-infer.stanza | 4 ++-- compiler/el-ir.stanza | 2 +- compiler/el-to-vm.stanza | 9 ++++++--- compiler/el.stanza | 1 - compiler/vm-analyze.stanza | 3 +++ compiler/vm-ir.stanza | 2 +- core/core.stanza | 31 +++++++++++++++---------------- 7 files changed, 28 insertions(+), 24 deletions(-) diff --git a/compiler/el-infer.stanza b/compiler/el-infer.stanza index 4d2737be..db206c92 100644 --- a/compiler/el-infer.stanza +++ b/compiler/el-infer.stanza @@ -360,7 +360,7 @@ defn analyze-single-block (emit:Ins -> ?, (ins:ETupleSet|EStruct|EPtr|ELabel|ETCall| EDump|EConv|EGoto|ECheckLength|ECheckSet| EBoxSet|EReturn|EObjectTGet|EClosureTGet| - ETDef|ETypeObject|EEnd|ELive|ECheckFail|ESafepoint) : gen-trivial-commands(ins) + ETDef|ETypeObject|EEnd|ELive|ECheckFail|ESafepoint|EProfile) : gen-trivial-commands(ins) ;Utility: For any uses of inferred variables generate ;them here. @@ -551,7 +551,7 @@ defn analyze-single-block (emit:Ins -> ?, defn gen-trivial-commands (ins:ETupleSet|EStruct|EPtr|ELabel|ETCall| EDump|EConv|EGoto|ECheckLength|ECheckSet| EBoxSet|EReturn|EObjectTGet|EClosureTGet| - ETDef|ETypeObject|EEnd|ELive|ECheckFail|ESafepoint) : + ETDef|ETypeObject|EEnd|ELive|ECheckFail|ESafepoint|EProfile) : gen-uses(ins) emit(Wrapped(ins)) for x in varlocs(ins) do : diff --git a/compiler/el-ir.stanza b/compiler/el-ir.stanza index 6ed1a3a2..515685eb 100644 --- a/compiler/el-ir.stanza +++ b/compiler/el-ir.stanza @@ -1312,7 +1312,7 @@ defmethod map (f:ELItem -> ELItem, item:ELItem&?T) -> ELItem&T : (t:EBot) : t ;EPackage - (e:EPackage) : EPackage(packageio(e), h(exps(e))) + (e:EPackage) : EPackage(packageio(e), h(exps(e)), function-info(e)) ;ETExps (e:EDefGlobal) : EDefGlobal(n(e), h(type(e)), mutable?(e), lostanza?(e)) diff --git a/compiler/el-to-vm.stanza b/compiler/el-to-vm.stanza index 6c348ccc..d0a4790e 100644 --- a/compiler/el-to-vm.stanza +++ b/compiler/el-to-vm.stanza @@ -61,7 +61,7 @@ public defn compile (epackage:EPackage, debug?:True|False) -> VMPackage : val init = compile-init(init-compiler, iotable, progbuffer) init - val vmp = to-vmpackage(progbuffer, io, init) + val vmp = to-vmpackage(progbuffer, io, init, function-info(epackage)) ;dump(vmp, "logs", "pre-analyze") val vmp* = within time-ms!("VM Analyze") : analyze(vmp) ;dump(vmp*, "logs", false) @@ -565,7 +565,7 @@ defmulti add (b:ProgBuffer, e:VMMethod) -> False defmulti add (b:ProgBuffer, e:VMDebugInfo) -> False defmulti add (b:ProgBuffer, e:SafepointEntry) -> SafepointIns defmulti add (b:ProgBuffer, e:VMDebugName) -> False -defmulti to-vmpackage (b:ProgBuffer, io:PackageIO, init:Int|False) -> VMPackage +defmulti to-vmpackage (b:ProgBuffer, io:PackageIO, init:Int|False, function-info:False|Tuple) -> VMPackage defn ProgBuffer () : val globals = Vector() @@ -591,7 +591,7 @@ defn ProgBuffer () : defmethod add (this, e:VMDebugInfo) : add(debug, e) defmethod add (this, e:SafepointEntry) : add(sptable, e) defmethod add (this, e:VMDebugName) : add(debugnames, e) - defmethod to-vmpackage (this, io:PackageIO, init:Int|False) : + defmethod to-vmpackage (this, io:PackageIO, init:Int|False, function-info:False|Tuple) : VMPackage(io, init, to-tuple(globals), to-tuple(datas), @@ -603,6 +603,7 @@ defn ProgBuffer () : to-tuple(extern-defns), VMDebugNameTable(to-tuple(debugnames)), VMDebugInfoTable(to-tuple(debug)), + function-info compile(sptable)) ;============================================================ @@ -1544,6 +1545,8 @@ defn Compiler (gt:GlobalTable, (info:StackTraceInfo) : make-safepoint(info) when /info(info) is-not False (f:False) : false + (ins:EProfile) : + emit(ProfileIns(id(ins), kind(ins))) ;--------------------------------------------------------- ;--------------- Create the Compiler Type ---------------- diff --git a/compiler/el.stanza b/compiler/el.stanza index 5dfc39c9..5776ee2b 100644 --- a/compiler/el.stanza +++ b/compiler/el.stanza @@ -2173,7 +2173,6 @@ defn insert-profiling (epackage:EPackage, coverage?:True|False) : defn insert-texp (t:ELBigItem, insert?:True|False) -> ELBigItem : match(map(insert-texp{_, insert?}, t)) : (e:EFn) : - ; println("%_ INFO %_" % [length(infos), get-info(info(e), body(e))]) match(get-info(info(e), body(e))) : (i:StackTraceInfo) : sub-body(e, insert(body(e), i, insert?)) (i:False) : e diff --git a/compiler/vm-analyze.stanza b/compiler/vm-analyze.stanza index 044d3dae..ad7cf893 100644 --- a/compiler/vm-analyze.stanza +++ b/compiler/vm-analyze.stanza @@ -336,6 +336,7 @@ deftype OperationIns : CommentIns <: OperationIns LiveIns <: OperationIns SafepointIns <: OperationIns + ProfileIns <: OperationIns ;Assumes that the results are loaded from registers in sequential order, ;and that we are sweeping through instructions backwards. @@ -373,6 +374,7 @@ defn do-results (func:Int -> ?, i:OperationIns) : (i:LiveIns) : false (i:CommentIns) : false (i:SafepointIns) : false + (i:ProfileIns) : false defn do-args (func:Int -> ?, i:OperationIns|DestinationIns) : defn f? (y:VMImm|False) : @@ -408,6 +410,7 @@ defn do-args (func:Int -> ?, i:OperationIns|DestinationIns) : (i:MethodDispatchIns) : (f?(ys(i)), f?(zs(i))) (i:UnreachableIns) : false (i:SafepointIns) : false + (i:ProfileIns) : false defn do-dest (f:Int -> ?, i:DestinationIns) : match(i) : diff --git a/compiler/vm-ir.stanza b/compiler/vm-ir.stanza index a1a907d2..9566e65d 100644 --- a/compiler/vm-ir.stanza +++ b/compiler/vm-ir.stanza @@ -51,8 +51,8 @@ public defstruct VMPackage : externs: Tuple extern-defns: Tuple debug-name-table: VMDebugNameTable - function-info: False|Tuple with: (default => false) debug-table: VMDebugInfoTable + function-info: False|Tuple with: (default => false) safepoint-table: VMSafepointTable ;A value that can be accessed directly within an instruction. diff --git a/core/core.stanza b/core/core.stanza index 39d086b0..ba0c0624 100644 --- a/core/core.stanza +++ b/core/core.stanza @@ -4245,6 +4245,20 @@ protected val uninitialized = new Uninitialized lostanza val LS-NL:ptr = "\n" val NL = "\n" +;============================================================ +;=================== Initialize Profiler ==================== +;============================================================ + +#if-defined(BOOTSTRAP) : + lostanza defn initialize-profiler () -> ref : + return false +#else : + lostanza defn initialize-profiler () -> ref : + val vms:ptr = call-prim flush-vm() + [vms.profile-flag] = 0L + vms.profile-buffer = LSLongVector(1 << 20) + return false + ;============================================================ ;=================== Initialize Remaining =================== ;============================================================ @@ -4256,6 +4270,7 @@ initialize-gc-notifiers() initialize-gc-statistics() initialize-liveness-handlers() initialize-symbol-table() +initialize-profiler() ;================================================================================ ;========================== End of Boot Sequence ================================ @@ -11200,20 +11215,6 @@ public deftype TypeObject public defmulti typeof? (x, t:TypeObject) -> True|False public defmulti name (t:TypeObject) -> String -;============================================================ -;=================== Initialize Profiler ==================== -;============================================================ - -#if-defined(BOOTSTRAP) : - lostanza defn initialize-profiler () -> ref : - return false -#else : - lostanza defn initialize-profiler () -> ref : - val vms:ptr = call-prim flush-vm() - [vms.profile-flag] = 0L - vms.profile-buffer = LSLongVector(1 << 20) - return false - ;============================================================ ;====================== Profiling =========================== ;============================================================ @@ -11425,7 +11426,6 @@ lostanza defn build-stack-trace-record-table (trace-table:ptr) ;Start profiling by initializing state and starting profile thread extern start_sample_profiling: (int, long, ptr, ptr) -> int lostanza defn do-start-sample-profiling (msecs:ref) -> ref : - call-c clib/printf("DO START PROFILING\n") val vms:ptr = call-prim flush-vm() [vms.profile-flag] = 0L vms.profile-buffer.length = 0 @@ -11439,7 +11439,6 @@ lostanza defn build-stack-trace-record-table (trace-table:ptr) ;Stop profiling by clearing state and stopping profile thread extern stop_sample_profiling: () -> int lostanza defn do-stop-sample-profiling () -> ref : - call-c clib/printf("DO STOP PROFILING\n") val vms:ptr = call-prim flush-vm() [vms.profile-flag] = 0L call-c stop_sample_profiling() From 88b579cb374fb74fef077a84aa5b80a12d054977 Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Thu, 9 Jan 2025 12:54:09 -0800 Subject: [PATCH 10/38] split out profiler, add serializer, and refactor collapse stack --- core/core.stanza | 251 +------------------------------- core/profiler.stanza | 330 +++++++++++++++++++++++++++++++++++++++++++ core/stanza.proj | 1 + 3 files changed, 334 insertions(+), 248 deletions(-) create mode 100644 core/profiler.stanza diff --git a/core/core.stanza b/core/core.stanza index ba0c0624..0f645b6a 100644 --- a/core/core.stanza +++ b/core/core.stanza @@ -188,11 +188,11 @@ protected lostanza deftype StackTraceTableEntry : lbl: ptr record: StackTraceRecord -lostanza deftype FunctionInfoTable : +protected lostanza deftype FunctionInfoTable : length: long entries: FunctionInfoEntry ... -lostanza deftype FunctionInfoEntry : +protected lostanza deftype FunctionInfoEntry : package: ptr name: ptr file: ptr @@ -11219,50 +11219,7 @@ public defmulti name (t:TypeObject) -> String ;====================== Profiling =========================== ;============================================================ -public defstruct ProfileInfo : - id : Int - package : String - name : String - info : FileInfo - count : Long - -public defstruct ProfileResult : - info : Tuple - id-traces : Tuple> - -defmethod print (o:OutputStream, c:ProfileInfo) : - print(o, "ProfileInfo(%_, %_, %_, %_)" % [id(c), package(c), name(c), info(c), count(c)]) - -defn namestring (p:ProfileInfo) -> String : - to-string("%_/%_ %_ (%_)" % [package(p), name(p), info(p), id(p)]) - -;Fast stack trace record lookup -lostanza defn stack-trace-record (ret:long, trace-table:ptr, trace-table-table:ref>) -> ptr : - val idx = get?(trace-table-table, new Long{ret}, new Int{-1}) - if idx.value == -1 : - return null - else : - val entry = addr(trace-table.entries[idx.value]) - return addr(entry.record) - -;Fast stack trace record table construction -lostanza defn build-stack-trace-record-table (trace-table:ptr) -> ref> : - val tab = HashTable() - for (var i:int = 0, i < trace-table.length, i = i + 1) : - val entry = addr(trace-table.entries[i]) - set(tab, new Long{entry.lbl as long}, new Int{i}) - return tab - -#if-defined(BOOTSTRAP) : - - defn collect-profiling () -> ProfileResult : ProfileResult([], []) - defn collect-coverage () -> Tuple : [] - public defn clear-profiling () : false - public defn clear-coverage () : false - defn do-start-sample-profiling (num-usecs:Int) : false - defn do-stop-sample-profiling () : false - -#else : +#if-not-defined(BOOTSTRAP) : ;Called from app to record stack trace as return addresses in buffer ;Demarcate end of each coroutine stack with -2 and end trace with -1 @@ -11293,208 +11250,6 @@ lostanza defn build-stack-trace-record-table (trace-table:ptr) [vms.profile-flag] = 0L return false - ;Remove empty stack traces - defn prune-traces (traces:Vector>) -> List> : - to-list $ seq(unique, filter({ not empty?(_) }, traces)) - - ;Count all functions references in stack traces where traces have function ids in them - defn count-traces (len:Int, traces:Vector>) -> Array : - val counts = Array(len, 0L) - for trace in traces do : - for id in trace do : - counts[id] = counts[id] + 1L - counts - - ;Split stack trace buffer using delimiter - defn split-buffer (buffer:List, delimiter:Long) -> List> : - let loop (i:Int = 0, buffers:List> = List(), ids:List = List()) : - if i < length(buffer) : - val id = buffer[i] - if id == delimiter : - loop(i + 1, cons(reverse(ids), buffers), List()) - else : - loop(i + 1, buffers, cons(id, ids)) - else : - reverse(buffers) - - ;Split stack trace buffer using delimiter - defn split-buffer (buffer:Tuple, delimiter:Long) -> Vector> : - val res = Vector>() - val chunk = Vector() - for id in buffer do : - if id == delimiter : - add(res, to-tuple(chunk)) - clear(chunk) - else : - add(chunk, id) - res - - ;Reconsititue full stack traces by splitting up chunks and ordering them properly - defn build-stack-traces (buffer:Tuple) -> Tuple> : - val buffers = split-buffer(buffer, -1L) - ;same as: - ; to-tuple $ for trace in buffers seq : - ; to-tuple $ reverse $ to-list $ cat-all $ seq(reverse, split-buffer(trace, -2L)) - to-tuple $ for (trace in buffers, k in 0 to false) seq : - val res = Vector() - val chunks = split-buffer(trace, -2L) ;split full stack trace into coroutine stack traces - for i in (length(chunks) - 1) through 0 by -1 do : - val chunk = chunks[i] - for j in 0 to length(chunk) do : - add(res, chunk[j]) - to-tuple(res) - - ;Create function id traces and function info results after profiling - lostanza defn collect-profiling () -> ref : - val vms:ptr = call-prim flush-vm() - val info:ptr = vms.function-info - val counters:ptr = vms.function-counters - val buffer:ptr = vms.profile-buffer - val tab = build-stack-trace-record-table(vms.stack-trace-table) ;For fast lookup of return addresses - val buffer* = Vector() - - for (var i:long = 0, i < buffer.length, i = i + 1) : - add(buffer*, new Long{buffer.items[i]}) - - val traces = build-stack-traces(to-tuple(buffer*)) ;Order and concatenate stack trace chunks - - ;Lookup stack trace records and then translate into function ids - val id-traces = Vector>() - for (var i:long = 0, i < length(traces).value, i = i + 1) : - val trace = get(traces, new Int{i as int}) - val trace* = Vector() - for (var j:long = 0, j < length(trace).value, j = j + 1) : - val ret = get(trace, new Int{j as int}) - val entry = stack-trace-record(ret.value, vms.stack-trace-table, tab) - if entry != null : - val id = entry.function - if id != -1L : - add(trace*, new Int{id as int}) - add(id-traces, to-tuple(trace*)) - - val len:long = info.length - val counts = count-traces(new Int{len as int}, id-traces) - val infos = Vector() - - ;Collect all FunctionInfos corresponding to ids that occurred on stack at least once - for (var i:long = 0L, i < len, i = i + 1) : - val entry = info.entries[i] - val fi = FileInfo(String(entry.file), new Int{entry.line}, new Int{entry.column}) - val count = get(counts, new Int{i as int}) - val package = String(entry.package) - val name = String(entry.name) - val ci = ProfileInfo(new Int{i as int}, package, name, fi, count) - add(infos, ci) - - return ProfileResult(to-tuple(infos), to-tuple(id-traces)) - - ;Collect all coverage information from all functions - lostanza defn collect-coverage () -> ref> : - val vms:ptr = call-prim flush-vm() - val info:ptr = vms.function-info - val counters:ptr = vms.function-counters - val len:long = info.length - val res = Vector() - for (var i:long = 0L, i < len, i = i + 1) : - val entry = info.entries[i] - val fi = FileInfo(String(entry.file), new Int{entry.line}, new Int{entry.column}) - val count = new Long{counters[i]} - val package = String(entry.package) - val name = String(entry.name) - val ci = ProfileInfo(new Int{i as int}, package, name, fi, count) - add(res, ci) - return res - - ;Clear profiling state ready for next run - public lostanza defn clear-profiling () -> ref : - val vms:ptr = call-prim flush-vm() - [vms.profile-flag] = 0L - vms.profile-buffer.length = 0 - return false - - ;Clear coverage state ready for next run - public lostanza defn clear-coverage () -> ref : - val vms:ptr = call-prim flush-vm() - val info:ptr = vms.function-info - val counters:ptr = vms.function-counters - val len:long = info.length - call-c clib/memset(counters, 0, len << 3L) - return false - - var PROFILING-MSECS:Int = 0 - - ;Start profiling by initializing state and starting profile thread - extern start_sample_profiling: (int, long, ptr, ptr) -> int - lostanza defn do-start-sample-profiling (msecs:ref) -> ref : - val vms:ptr = call-prim flush-vm() - [vms.profile-flag] = 0L - vms.profile-buffer.length = 0 - PROFILING-MSECS = msecs - val res = call-c start_sample_profiling(msecs.value, vms.function-info.length, vms.profile-flag, vms.function-counters) - if res == 0 : - return false - else : - return true - - ;Stop profiling by clearing state and stopping profile thread - extern stop_sample_profiling: () -> int - lostanza defn do-stop-sample-profiling () -> ref : - val vms:ptr = call-prim flush-vm() - [vms.profile-flag] = 0L - call-c stop_sample_profiling() - return PROFILING-MSECS - -;dumps coverage results to given file -; must be stanza compiled with -coverage flag as well -public defn dump-coverage (filename:String) : - val file = FileOutputStream(filename) - try : - val coverage = collect-coverage() - for elt in reverse(to-list(lazy-qsort(count, coverage))) do : - if count(elt) > 0L : - println(file, "%_: %_/%_ in %_ (%_)" % [count(elt), package(elt), name(elt), info(elt), id(elt)]) - finally: close(file) - -;Coverage wrapper to be called with within which -; runs thunk with coverage counting on and then dumps coverage report to file -public defn coverage (f:() -> ?, filename:String) : - clear-coverage() - f() - dump-coverage(filename) - -; starts profiling sampling at msecs period -public defn start-profiling (msecs:Int) : - clear-profiling() - do-start-sample-profiling(msecs) - false - -; dumps profiling results to given file -; must be stanza compiled with -profile flag as well -public defn stop-profiling (filename:String) : - val msecs = do-stop-sample-profiling() - val profiling = collect-profiling() - if not empty?(info(profiling)) : - ; within (file) = with-output-file(filename) : - val file = FileOutputStream(filename) - try : - println(file, msecs) - println(file, "(") - do(println{file, _}, id-traces(profiling)) - println(file, ")") - println(file, "(") - for (elt in info(profiling), i in 0 to false) do : - if count(elt) > 0L : - println(file, "(%_ %~)" % [id(elt), string-join $ [package(elt) "/" name(elt) ":" line(info(elt))]]) - println(file, ")") - finally: close(file) - false - -;Profiling wrapper to be called with within -public defn profiling (f:() -> ?, msecs:Int, filename:String) : - start-profiling(msecs) - f() - stop-profiling(filename) - ;############################################################ ;################## Math Package ############################ ;############################################################ diff --git a/core/profiler.stanza b/core/profiler.stanza new file mode 100644 index 00000000..f6a2b847 --- /dev/null +++ b/core/profiler.stanza @@ -0,0 +1,330 @@ +;See License.txt for details about licensing. +#use-added-syntax(fastio-serializer) +defpackage profiler : + import core + import collections + import core/stack-trace + import core/long-vector + import stz/fastio-buffer + import stz/fastio-runtime + +public defstruct ProfileInfo : + id : Int + package : String + name : String + info : FileInfo + count : Long +with: + printer => true + +public defstruct ProfileResult : + info : Tuple + id-traces : Tuple> + msecs : Int +with: + printer => true + +defmethod print (o:OutputStream, c:ProfileInfo) : + print(o, "ProfileInfo(%_, %_, %_, %_)" % [id(c), package(c), name(c), info(c), count(c)]) + +public defn serialize-profile-result (filename:String, res:ProfileResult) -> False : + write-to-file(filename, + ProfileResultSerializer(), + serialize-profileresult, + res) + +public defn deserialize-profile-result (filename:String) -> ProfileResult : + read-from-file(filename, + ProfileResultSerializer(), + deserialize-profileresult) + +defserializer ProfileResultSerializer () : + entry-points: (profileresult) + + include "../compiler/serializer-primitives.spec" + + deftype profileinfo (ProfileInfo) : + id:int + package:string + name:string + info:info + count:long + + deftype profileresult (ProfileResult) : + info:tuple(profileinfo) + id-traces:tuple(tuple(int)) + msecs:int + + + ;Fast stack trace record lookup +lostanza defn stack-trace-record (ret:long, trace-table:ptr, trace-table-table:ref>) -> ptr : + val idx = get?(trace-table-table, new Long{ret}, new Int{-1}) + if idx.value == -1 : + return null + else : + val entry = addr(trace-table.entries[idx.value]) + return addr(entry.record) + +;Fast stack trace record table construction +lostanza defn build-stack-trace-record-table (trace-table:ptr) -> ref> : + val tab = HashTable() + for (var i:int = 0, i < trace-table.length, i = i + 1) : + val entry = addr(trace-table.entries[i]) + set(tab, new Long{entry.lbl as long}, new Int{i}) + return tab + +#if-defined(BOOTSTRAP) : + + defn collect-profiling () -> ProfileResult : ProfileResult([], []) + defn collect-coverage () -> Tuple : [] + public defn clear-profiling () : false + public defn clear-coverage () : false + defn do-start-sample-profiling (num-usecs:Int) : false + defn do-stop-sample-profiling () : false + +#else : + + ;Remove empty stack traces + defn prune-traces (traces:Vector>) -> List> : + to-list $ seq(unique, filter({ not empty?(_) }, traces)) + + ;Count all functions references in stack traces where traces have function ids in them + defn count-traces (len:Int, traces:Vector>) -> Array : + val counts = Array(len, 0L) + for trace in traces do : + for id in trace do : + counts[id] = counts[id] + 1L + counts + + ;Split stack trace buffer using delimiter + defn split-buffer (buffer:List, delimiter:Long) -> List> : + let loop (i:Int = 0, buffers:List> = List(), ids:List = List()) : + if i < length(buffer) : + val id = buffer[i] + if id == delimiter : + loop(i + 1, cons(reverse(ids), buffers), List()) + else : + loop(i + 1, buffers, cons(id, ids)) + else : + reverse(buffers) + + ;Split stack trace buffer using delimiter + defn split-buffer (buffer:Tuple, delimiter:Long) -> Vector> : + val res = Vector>() + val chunk = Vector() + for id in buffer do : + if id == delimiter : + add(res, to-tuple(chunk)) + clear(chunk) + else : + add(chunk, id) + res + + ;Reconsititue full stack traces by splitting up chunks and ordering them properly + defn build-stack-traces (buffer:Tuple) -> Tuple> : + val buffers = split-buffer(buffer, -1L) + ;same as: + ; to-tuple $ for trace in buffers seq : + ; to-tuple $ reverse $ to-list $ cat-all $ seq(reverse, split-buffer(trace, -2L)) + to-tuple $ for (trace in buffers, k in 0 to false) seq : + val res = Vector() + val chunks = split-buffer(trace, -2L) ;split full stack trace into coroutine stack traces + for i in (length(chunks) - 1) through 0 by -1 do : + val chunk = chunks[i] + for j in 0 to length(chunk) do : + add(res, chunk[j]) + to-tuple(res) + + ;Create function id traces and function info results after profiling + lostanza defn collect-profiling (msecs:ref) -> ref : + val vms:ptr = call-prim flush-vm() + val info:ptr = vms.function-info + val counters:ptr = vms.function-counters + val buffer:ptr = vms.profile-buffer as ptr + val id-traces = Vector>() + val infos = Vector() + + if info.length > 0L : + + val tab = build-stack-trace-record-table(vms.stack-trace-table) ;For fast lookup of return addresses + val buffer* = Vector() + + for (var i:long = 0, i < buffer.length, i = i + 1) : + add(buffer*, new Long{buffer.items[i]}) + + val traces = build-stack-traces(to-tuple(buffer*)) ;Order and concatenate stack trace chunks + + ;Lookup stack trace records and then translate into function ids + for (var i:long = 0, i < length(traces).value, i = i + 1) : + val trace = get(traces, new Int{i as int}) + val trace* = Vector() + for (var j:long = 0, j < length(trace).value, j = j + 1) : + val ret = get(trace, new Int{j as int}) + val entry = stack-trace-record(ret.value, vms.stack-trace-table, tab) + if entry != null : + val id = entry.function + if id != -1L : + add(trace*, new Int{id as int}) + add(id-traces, to-tuple(trace*)) + + val len:long = info.length + val counts = count-traces(new Int{len as int}, id-traces) + + ;Collect all FunctionInfos corresponding to ids that occurred on stack at least once + for (var i:long = 0L, i < len, i = i + 1) : + val entry = info.entries[i] + val fi = FileInfo(String(entry.file), new Int{entry.line}, new Int{entry.column}) + val count = get(counts, new Int{i as int}) + if count.value > 0L : + val package = String(entry.package) + val name = String(entry.name) + val ci = ProfileInfo(new Int{i as int}, package, name, fi, count) + add(infos, ci) + + return ProfileResult(to-tuple(infos), to-tuple(id-traces), msecs) + + ;Collect all coverage information from all functions + lostanza defn collect-coverage () -> ref> : + val vms:ptr = call-prim flush-vm() + val info:ptr = vms.function-info + val counters:ptr = vms.function-counters + val len:long = info.length + val res = Vector() + for (var i:long = 0L, i < len, i = i + 1) : + val entry = info.entries[i] + val fi = FileInfo(String(entry.file), new Int{entry.line}, new Int{entry.column}) + val count = new Long{counters[i]} + val package = String(entry.package) + val name = String(entry.name) + val ci = ProfileInfo(new Int{i as int}, package, name, fi, count) + add(res, ci) + return res + + ;Clear profiling state ready for next run + public lostanza defn clear-profiling () -> ref : + val vms:ptr = call-prim flush-vm() + [vms.profile-flag] = 0L + vms.profile-buffer.length = 0 + return false + + ;Clear coverage state ready for next run + public lostanza defn clear-coverage () -> ref : + val vms:ptr = call-prim flush-vm() + val info:ptr = vms.function-info + val counters:ptr = vms.function-counters + val len:long = info.length + call-c clib/memset(counters, 0, len << 3L) + return false + + var PROFILING-MSECS:Int = 0 + + ;Start profiling by initializing state and starting profile thread + extern start_sample_profiling: (int, long, ptr, ptr) -> int + lostanza defn do-start-sample-profiling (msecs:ref) -> ref : + val vms:ptr = call-prim flush-vm() + [vms.profile-flag] = 0L + vms.profile-buffer.length = 0 + PROFILING-MSECS = msecs + val res = call-c start_sample_profiling(msecs.value, vms.function-info.length, vms.profile-flag, vms.function-counters) + if res == 0 : + return false + else : + return true + + ;Stop profiling by clearing state and stopping profile thread + extern stop_sample_profiling: () -> int + lostanza defn do-stop-sample-profiling () -> ref : + val vms:ptr = call-prim flush-vm() + [vms.profile-flag] = 0L + call-c stop_sample_profiling() + return PROFILING-MSECS + +;dumps coverage results to given file +; must be stanza compiled with -coverage flag as well +public defn dump-coverage (filename:String) : + val file = FileOutputStream(filename) + try : + val coverage = collect-coverage() + for elt in reverse(to-list(lazy-qsort(count, coverage))) do : + if count(elt) > 0L : + println(file, "%_: %_/%_ in %_ (%_)" % [count(elt), package(elt), name(elt), info(elt), id(elt)]) + finally: close(file) + +;Coverage wrapper to be called with within which +; runs thunk with coverage counting on and then dumps coverage report to file +public defn coverage (f:() -> ?, filename:String) : + clear-coverage() + f() + dump-coverage(filename) + +; starts profiling sampling at msecs period +public defn start-profiling (msecs:Int) : + clear-profiling() + do-start-sample-profiling(msecs) + false + +; dumps profiling results to given file +; must be stanza compiled with -profile flag as well +public defn stop-profiling () -> ProfileResult: + collect-profiling(do-stop-sample-profiling()) + +;Profiling wrapper to be called with within +public defn profiling (f:() -> ?, msecs:Int) : + start-profiling(msecs) + f() + stop-profiling() + +;;; convert to flame-graph format using traditional stack-collapse function + +;These functions clog up flame graph so filter them out +val default-silence-prefixes = [ + "core/do" + "core/seq" + "core/map" + "core/with-exception-handler" + "core/setup-coroutine" + "core/with-finally" + "core/dynamic-wind" + "core/wrap-length" + "core/repeat-while" + ] + +defn namestring (pi:ProfileInfo) -> String : + string-join $ [package(pi) "/" name(pi) ":" line(info(pi))] + +defn with-output-file-stream (f: FileOutputStream -> ?T, file:FileOutputStream) -> T : + try : f(file) + finally : close(file) + +public defn stack-collapse (res:ProfileResult, filename:String, top?:True|False = false, depth:Int = 1, silence-prefixes:Tuple = default-silence-prefixes) : + val strings = IntTable() + val silence-ids = IntSet() + for elt in info(res) do : + val decl = namestring(elt) + strings[id(elt)] = decl + for p in silence-prefixes do : + add(silence-ids, id(elt)) when prefix?(decl, p) + within (file) = with-output-file-stream(FileOutputStream(filename)) : + if top? : + val tops = IntTable(0) + for trace in id-traces(res) do : + val len = length(trace) + for off in 1 through 2 do : + val idx = len - off + if idx >= 0 : + val id = trace[idx] + tops[id] = tops[id] + 1 + val tops* = reverse $ to-list $ lazy-qsort(value, tops) + for top in tops* do : + println(file, "%_: %_" % [value(top) * msecs(res), strings[key(top)]]) + else : + for trace in id-traces(res) do : + var first?:True|False = true + for record in trace do : + if not silence-ids[record] : + print(file, ";") when not first? + print(file, strings[record]) + first? = false + print(file, " ") + println(file, msecs(res)) + diff --git a/core/stanza.proj b/core/stanza.proj index 80b05528..cd23bade 100644 --- a/core/stanza.proj +++ b/core/stanza.proj @@ -7,4 +7,5 @@ packages parser/* defined-in "parser" package reader defined-in "reader.stanza" package arg-parser defined-in "arg-parser.stanza" package line-wrap defined-in "line-wrap.stanza" +package profiler defined-in "profiler.stanza" packages core/* defined-in "." \ No newline at end of file From 67670fa30ac8e30bb633f28a00c2b18c2cf009a5 Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Thu, 9 Jan 2025 14:14:40 -0800 Subject: [PATCH 11/38] move length to front of heapstatistics and split out dumping from collecting of stats --- compiler/stitcher.stanza | 3 +-- core/core.stanza | 58 +++++++++++++++++++++------------------- 2 files changed, 31 insertions(+), 30 deletions(-) diff --git a/compiler/stitcher.stanza b/compiler/stitcher.stanza index d14d6505..7b1a922f 100644 --- a/compiler/stitcher.stanza +++ b/compiler/stitcher.stanza @@ -861,11 +861,10 @@ public defn Stitcher (packages:Collection, bindings:Bindings|False, s E $ Comment("Heap Statistics Table") E $ DefData() E $ Label(/heap-statistics(stubs)) + E $ DefLong(to-long(num-concrete-classes)) for i in 0 to num-concrete-classes do : E $ DefLong(0L) ; num-uses E $ DefLong(0L) ; num-bytes - E $ DefLong(to-long(num-concrete-classes)) - E $ DefLong(-1L) ; mark end of table E $ DefText() E $ Comment("End of Heap Statistics Table") diff --git a/core/core.stanza b/core/core.stanza index ab4ad441..6fb7c819 100644 --- a/core/core.stanza +++ b/core/core.stanza @@ -209,6 +209,10 @@ protected lostanza deftype ArrayRecord : num-item-roots:int roots:int ... +lostanza deftype HeapStatistics : + length: long + entries: HeapStatistic ... + protected lostanza deftype HeapStatistic : var num-uses:long var num-bytes:long @@ -266,7 +270,7 @@ protected lostanza deftype HeapStatistic : safepoint-table: ptr ;(Variable State) debug-table: ptr ;(Variable State) local-var-table: ptr ;(Variable State) - heap-statistics: ptr ;(Variable State) + heap-statistics: ptr ;(Variable State) ;Compiled Mode Tables class-table: ptr global-root-table: ptr @@ -2829,29 +2833,17 @@ public lostanza defn min (x:long, y:long) -> long : with: printer => true - ; clear out statistics for each concrete class - lostanza defn clear-heap-statistics (vms:ptr) -> int : - labels: - begin: goto loop(0) - loop (i:int) : - val stat = vms.heap-statistics[i] - if stat.num-bytes < 0L : ; sentinel - return stat.num-uses as int - else : - vms.heap-statistics[i].num-bytes = 0 - vms.heap-statistics[i].num-uses = 0 - goto loop(i + 1) - ; run gc, collect heap stats while walking each object in nursery and heap lostanza defn do-analyze-heap (stats:ref>) -> ref : val vms:ptr = call-prim flush-vm() run-garbage-collector() - val num-classes = clear-heap-statistics(vms) + val num-classes = vms.heap-statistics.length + clear(addr(vms.heap-statistics.entries), num-classes * sizeof(HeapStatistic)) val hsize = do-analyze-heap(vms.heap.start, vms.heap.old-objects-end, vms) val nursery = nursery-start(addr(vms.heap)) val nsize = do-analyze-heap(nursery, vms.heap.top, vms) for (var i:int = 0, i < num-classes, i = i + 1) : - val heap-stat = vms.heap-statistics[i] + val heap-stat = addr(vms.heap-statistics.entries[i]) val num-uses = heap-stat.num-uses if num-uses > 0L : add(stats, HeapStat(new Int{i as int}, String(class-name(i)), new Long{num-uses}, new Long{heap-stat.num-bytes})) @@ -2875,26 +2867,36 @@ public lostanza defn min (x:long, y:long) -> long : val my-size = base-size + item-size * len size = object-size-on-heap(my-size) p = p + size - vms.heap-statistics[tag].num-uses = vms.heap-statistics[tag].num-uses + 1L - vms.heap-statistics[tag].num-bytes = vms.heap-statistics[tag].num-bytes + size + val stat = addr(vms.heap-statistics.entries[tag]) + stat.num-uses = stat.num-uses + 1L + stat.num-bytes = stat.num-bytes + size return (pend as long) - (pstart as long) + public defstruct HeapStats : + total-size : Long + entries : Tuple + ; public interface to heap analyzer collecting and printing out stats - public defn analyze-heap () -> Long : + public defn analyze-heap () -> HeapStats : val stats = Vector() val size = do-analyze-heap(stats) val res = reverse(to-list(lazy-qsort(num-bytes, stats))) - println("Heap size %_" % [size]) - var max-bytes-size = reduce(max, length("Size"), seq({ length(to-string(num-bytes(_))) }, res)) - var max-perc-size = 3 - var max-uses-size = reduce(max, length("Uses"), seq({ length(to-string(num-uses(_))) }, res)) + HeapStats(size, to-tuple(res)) + + public defn dump (stats:HeapStats) : + println("Heap size %_" % [total-size(stats)]) + var max-bytes-size = reduce(max, length("Size"), seq({ length(to-string(num-bytes(_))) }, entries(stats))) + var max-perc-size = 6 + var max-uses-size = reduce(max, length("Uses"), seq({ length(to-string(num-uses(_))) }, entries(stats))) defn pad (s:String, n:Int) -> String : append-all(cat(repeatedly({ " " }, (n - length(s))), [s])) - println(" %_ %_ %_: %_" % [pad("Size", max-bytes-size), "Perc", pad("Use", max-uses-size), "Type"]) - for hc in res do : - val p = to-int(to-double(num-bytes(hc)) * 100.0 / to-double(size)) + defn pad0 (s:String, n:Int) -> String : append-all(cat([s], repeatedly({ "0" }, (n - length(s))))) + println(" %_ %_ %_: %_" % [pad("Size", max-bytes-size), pad("Perc", max-perc-size), pad("Use", max-uses-size), "Type"]) + for hc in entries(stats) do : + val pt = to-int(to-double(num-bytes(hc)) * 10000.0 / to-double(total-size(stats))) + val p = string-join $ [pt / 100, ".", pad0(to-string(pt - ((pt / 100) * 100)), 2)] println(" %_ %_%% %_: %_" % [ - pad(to-string(num-bytes(hc)), max-bytes-size), pad(to-string(p), max-perc-size), pad(to-string(num-uses(hc)), max-uses-size), name(hc)]) - size + pad(to-string(num-bytes(hc)), max-bytes-size), pad(p, max-perc-size), pad(to-string(num-uses(hc)), max-uses-size), name(hc)]) + ;============================================================ ;===================== Debugging ============================ From aad63965c35ddf23a9aee357a97d3075d8fa78f4 Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Fri, 17 Jan 2025 09:40:48 -0800 Subject: [PATCH 12/38] add profiling support in compiler --- compiler/main.stanza | 24 ++++++++++++++++++++---- core/profiler.stanza | 11 ++++++----- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/compiler/main.stanza b/compiler/main.stanza index 7ee62eec..d544fce5 100644 --- a/compiler/main.stanza +++ b/compiler/main.stanza @@ -6,6 +6,7 @@ defpackage stz/main : import reader import arg-parser import line-wrap + import profiler import stz/compiler import stz/params import stz/config @@ -63,6 +64,18 @@ defn run-with-timing-log (body:() -> ?T, cmd-args:CommandArgs) -> T : else : body() +;============================================================ +;=================== profile-compiler ======================= +;============================================================ + +;If -profile-compiler flag is provided, then execute the body +;with support for profiling. +defn run-with-profiler (body:() -> ?T, cmd-args:CommandArgs) -> T : + if flag?(cmd-args, "profile-compiler") : + profiling(body, 100, { stack-collapse(_, cmd-args["profile-compiler"]) }) + else : + body() + ;============================================================ ;====================== Verbosity =========================== ;============================================================ @@ -251,6 +264,8 @@ val COMMON-STANZA-FLAGS = [ "The name of the output external dependencies file.") Flag("timing-log", OneFlag, OptionalFlag, "If provided, the filename of the timing log to generate.") + Flag("profile-compiler", OneFlag, OptionalFlag, + "If provided, the filename of the flame-graph to generate.") Flag("macros", AtLeastOneFlag, OptionalFlag, "If provided, the filename of the macro plugin.")] @@ -346,15 +361,16 @@ defn compile-command () : symbol?("link")) ;Launch! - within run-with-timing-log(cmd-args) : - within run-with-verbose-flag(cmd-args) : - compile(build-settings(), build-system()) + within run-with-profiler(cmd-args) : + within run-with-timing-log(cmd-args) : + within run-with-verbose-flag(cmd-args) : + compile(build-settings(), build-system()) ;Command Command("compile", AtLeastOneArg, "the .stanza/.proj input files or Stanza package names.", common-stanza-flags(["o" "s" "pkg" "pkg-cache" "build-from-source" "optimize" "debug" "ccfiles" "ccflags" "flags" - "verbose" "supported-vm-packages" "platform" "external-dependencies" "macros" "link" "timing-log" + "verbose" "supported-vm-packages" "platform" "external-dependencies" "macros" "link" "timing-log" "profile-compiler" "profile" "coverage"]), compile-msg, false, verify-args, intercept-no-match-exceptions(compile-action)) diff --git a/core/profiler.stanza b/core/profiler.stanza index f6a2b847..112a10f9 100644 --- a/core/profiler.stanza +++ b/core/profiler.stanza @@ -75,12 +75,12 @@ lostanza defn build-stack-trace-record-table (trace-table:ptr ProfileResult : ProfileResult([], []) + defn collect-profiling (msecs:Int) -> ProfileResult : ProfileResult([], [], 100) defn collect-coverage () -> Tuple : [] public defn clear-profiling () : false public defn clear-coverage () : false defn do-start-sample-profiling (num-usecs:Int) : false - defn do-stop-sample-profiling () : false + defn do-stop-sample-profiling () -> Int : 100 #else : @@ -269,10 +269,11 @@ public defn stop-profiling () -> ProfileResult: collect-profiling(do-stop-sample-profiling()) ;Profiling wrapper to be called with within -public defn profiling (f:() -> ?, msecs:Int) : +public defn profiling (f:() -> ?T, msecs:Int, g:ProfileResult -> ? = { false }) -> T : start-profiling(msecs) - f() - stop-profiling() + val res = f() + g(stop-profiling()) + res ;;; convert to flame-graph format using traditional stack-collapse function From 9415cd6ab5ab78fda3efcd2975c97b1f5f1d0bb5 Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Fri, 17 Jan 2025 09:47:40 -0800 Subject: [PATCH 13/38] add HEAP-ANALYZER flag so we can write code ahead of PR review --- compiler/compiler.stanza | 1 + 1 file changed, 1 insertion(+) diff --git a/compiler/compiler.stanza b/compiler/compiler.stanza index 825d7765..26e27dfc 100644 --- a/compiler/compiler.stanza +++ b/compiler/compiler.stanza @@ -243,6 +243,7 @@ public defn compile (settings:BuildSettings, system:System) : println("Build target %~ is already up-to-date." % [build-target?(settings*)]) else : setup-system-flags(settings*) + add-flag(`HEAP-ANALYZER) val proj-params = ProjParams(compiler-flags(), optimize?(settings*), debug?(settings*), false, build-from-source?(settings), pkg-cache-dir(settings*)) val proj-manager = ProjManager(proj, proj-params, auxfile) val comp-result = compile(proj-manager, auxfile, build-inputs!(settings*), vm-packages(settings*), asm?(settings*), pkg-dir(settings*), From 062be1e17ebca9d687c6dbed0b0c93c792ff93f8 Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Tue, 21 Jan 2025 14:33:15 -0800 Subject: [PATCH 14/38] dont profile check in lostanza functions --- compiler/el.stanza | 5 +++-- core/profiler.stanza | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/compiler/el.stanza b/compiler/el.stanza index 5776ee2b..d6f3bdca 100644 --- a/compiler/el.stanza +++ b/compiler/el.stanza @@ -2181,8 +2181,9 @@ defn insert-profiling (epackage:EPackage, coverage?:True|False) : ;Inline all top-level expressions val new-exps = for exp in exps(epackage) map : match(exp) : - (exp:EDefn) : insert-texp(exp, n(exp) != n(iotable,CORE-PROFILE-STACK-TRACE-ID)) as ETExp - (exp:EDefClosure|EDefmethod) : insert-texp(exp, true) as ETExp + (exp:EDefn) : insert-texp(exp, not lostanza?(exp)) as ETExp + (exp:EDefClosure) : insert-texp(exp, true) as ETExp + (exp:EDefmethod) : insert-texp(exp, not lostanza?(exp)) as ETExp (exp) : exp sub-function-info(sub-exps(epackage, new-exps), to-tuple $ infos) diff --git a/core/profiler.stanza b/core/profiler.stanza index 112a10f9..44a561d6 100644 --- a/core/profiler.stanza +++ b/core/profiler.stanza @@ -297,6 +297,7 @@ defn with-output-file-stream (f: FileOutputStream -> ?T, file:FileOutputStre try : f(file) finally : close(file) +; turn profile result into flame graph file format public defn stack-collapse (res:ProfileResult, filename:String, top?:True|False = false, depth:Int = 1, silence-prefixes:Tuple = default-silence-prefixes) : val strings = IntTable() val silence-ids = IntSet() From b0aa5761085fd637f8cf65b75640022d98ef04c4 Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Wed, 22 Jan 2025 13:21:35 -0800 Subject: [PATCH 15/38] add sample size in msec to profiler --- core/core.stanza | 13 +++++-- core/profiler.stanza | 91 ++++++++++++++++++++++++-------------------- 2 files changed, 60 insertions(+), 44 deletions(-) diff --git a/core/core.stanza b/core/core.stanza index 0f645b6a..472e7f30 100644 --- a/core/core.stanza +++ b/core/core.stanza @@ -9606,14 +9606,16 @@ public defn all? (pred?: T -> True|False, xs:Seqable) -> True|False : for xs-seq in xs do-seq : let loop () : if empty?(xs-seq) : true - else : pred?(next(xs-seq)) and loop() + else if pred?(next(xs-seq)) : loop() + else : false public defn all? (pred?: (T,S) -> True|False, xs:Seqable, ys:Seqable) -> True|False : for xs-seq in xs do-seq : for ys-seq in ys do-seq : let loop () : if empty?(xs-seq) or empty?(ys-seq) : true - else : pred?(next(xs-seq), next(ys-seq)) and loop() + else if pred?(next(xs-seq), next(ys-seq)) : loop() + else : false public defn all? (pred?: (T,S,U) -> True|False, xs:Seqable, ys:Seqable, zs:Seqable) -> True|False : for xs-seq in xs do-seq : @@ -9621,7 +9623,8 @@ public defn all? (pred?: (T,S,U) -> True|False, xs:Seqable, ys:Seq for zs-seq in zs do-seq : let loop () : if empty?(xs-seq) or empty?(ys-seq) or empty?(zs-seq) : true - else : pred?(next(xs-seq), next(ys-seq), next(zs-seq)) and loop() + else if pred?(next(xs-seq), next(ys-seq), next(zs-seq)) : loop() + else : false public defn none? (xs:Seqable, pred?: T -> True|False) -> True|False : none?(pred?,xs) @@ -11221,12 +11224,16 @@ public defmulti name (t:TypeObject) -> String #if-not-defined(BOOTSTRAP) : + public var last-profiler-time:Long = 0L + ;Called from app to record stack trace as return addresses in buffer ;Demarcate end of each coroutine stack with -2 and end trace with -1 lostanza defn profile-stack-trace () -> ref : val vms:ptr = call-prim flush-vm() [vms.profile-flag] = 2L val buffer = vms.profile-buffer + add(buffer, call-c clib/current_time_ms()) + add(buffer, -2L) ; time marker labels : begin : goto loop(current-coroutine) diff --git a/core/profiler.stanza b/core/profiler.stanza index 44a561d6..6bc43f9b 100644 --- a/core/profiler.stanza +++ b/core/profiler.stanza @@ -17,9 +17,13 @@ public defstruct ProfileInfo : with: printer => true +defstruct ProfileStackIdTrace : + msecs : Int + ids : Tuple + public defstruct ProfileResult : info : Tuple - id-traces : Tuple> + id-traces : Tuple msecs : Int with: printer => true @@ -50,9 +54,13 @@ defserializer ProfileResultSerializer () : info:info count:long + deftype profilestackidtrace (ProfileStackIdTrace) : + msecs:int + ids:tuple(int) + deftype profileresult (ProfileResult) : info:tuple(profileinfo) - id-traces:tuple(tuple(int)) + id-traces:tuple(profilestackidtrace) msecs:int @@ -89,25 +97,13 @@ lostanza defn build-stack-trace-record-table (trace-table:ptr>) -> Array : + defn count-traces (len:Int, traces:Vector) -> Array : val counts = Array(len, 0L) for trace in traces do : - for id in trace do : + for id in ids(trace) do : counts[id] = counts[id] + 1L counts - ;Split stack trace buffer using delimiter - defn split-buffer (buffer:List, delimiter:Long) -> List> : - let loop (i:Int = 0, buffers:List> = List(), ids:List = List()) : - if i < length(buffer) : - val id = buffer[i] - if id == delimiter : - loop(i + 1, cons(reverse(ids), buffers), List()) - else : - loop(i + 1, buffers, cons(id, ids)) - else : - reverse(buffers) - ;Split stack trace buffer using delimiter defn split-buffer (buffer:Tuple, delimiter:Long) -> Vector> : val res = Vector>() @@ -120,8 +116,12 @@ lostanza defn build-stack-trace-record-table (trace-table:ptr + ;Reconsititue full stack traces by splitting up chunks and ordering them properly - defn build-stack-traces (buffer:Tuple) -> Tuple> : + defn build-stack-traces (buffer:Tuple) -> Tuple : val buffers = split-buffer(buffer, -1L) ;same as: ; to-tuple $ for trace in buffers seq : @@ -129,47 +129,58 @@ lostanza defn build-stack-trace-record-table (trace-table:ptr() val chunks = split-buffer(trace, -2L) ;split full stack trace into coroutine stack traces - for i in (length(chunks) - 1) through 0 by -1 do : + val elapsed = chunks[0][0] + for i in (length(chunks) - 1) to 0 by -1 do : val chunk = chunks[i] for j in 0 to length(chunk) do : add(res, chunk[j]) - to-tuple(res) + ProfileRawStackTrace(elapsed, to-tuple(res)) ;Create function id traces and function info results after profiling - lostanza defn collect-profiling (msecs:ref) -> ref : + lostanza defn collect-profiling (specd-msecs:ref) -> ref : val vms:ptr = call-prim flush-vm() val info:ptr = vms.function-info val counters:ptr = vms.function-counters val buffer:ptr = vms.profile-buffer as ptr - val id-traces = Vector>() + val id-traces = Vector() val infos = Vector() if info.length > 0L : - + val tab = build-stack-trace-record-table(vms.stack-trace-table) ;For fast lookup of return addresses val buffer* = Vector() - + for (var i:long = 0, i < buffer.length, i = i + 1) : add(buffer*, new Long{buffer.items[i]}) - + val traces = build-stack-traces(to-tuple(buffer*)) ;Order and concatenate stack trace chunks - + + var last_global_elapsed_time:long = 0L + ;Lookup stack trace records and then translate into function ids for (var i:long = 0, i < length(traces).value, i = i + 1) : val trace = get(traces, new Int{i as int}) + val global-elapsed = elapsed-time(trace) + var msecs:int = 0 + if last_global_elapsed_time == 0L : ; first trace? + msecs = specd-msecs.value + else : + msecs = (global-elapsed.value - last_global_elapsed_time) as int + last_global_elapsed_time = global-elapsed.value val trace* = Vector() - for (var j:long = 0, j < length(trace).value, j = j + 1) : - val ret = get(trace, new Int{j as int}) + val addrs = return-addresses(trace) + for (var j:long = 1, j < length(addrs).value, j = j + 1) : + val ret = get(addrs, new Int{j as int}) val entry = stack-trace-record(ret.value, vms.stack-trace-table, tab) if entry != null : val id = entry.function if id != -1L : add(trace*, new Int{id as int}) - add(id-traces, to-tuple(trace*)) - + add(id-traces, ProfileStackIdTrace(new Int{msecs}, to-tuple(trace*))) + val len:long = info.length val counts = count-traces(new Int{len as int}, id-traces) - + ;Collect all FunctionInfos corresponding to ids that occurred on stack at least once for (var i:long = 0L, i < len, i = i + 1) : val entry = info.entries[i] @@ -181,7 +192,7 @@ lostanza defn build-stack-trace-record-table (trace-table:ptr ref> : @@ -298,7 +309,7 @@ defn with-output-file-stream (f: FileOutputStream -> ?T, file:FileOutputStre finally : close(file) ; turn profile result into flame graph file format -public defn stack-collapse (res:ProfileResult, filename:String, top?:True|False = false, depth:Int = 1, silence-prefixes:Tuple = default-silence-prefixes) : +public defn stack-collapse (res:ProfileResult, filename:String, top?:True|False = false, max-depth:Int = 1000, silence-prefixes:Tuple = default-silence-prefixes) : val strings = IntTable() val silence-ids = IntSet() for elt in info(res) do : @@ -310,23 +321,21 @@ public defn stack-collapse (res:ProfileResult, filename:String, top?:True|False if top? : val tops = IntTable(0) for trace in id-traces(res) do : - val len = length(trace) + val ids = ids(trace) + val len = length(ids) for off in 1 through 2 do : val idx = len - off if idx >= 0 : - val id = trace[idx] + val id = ids[idx] tops[id] = tops[id] + 1 val tops* = reverse $ to-list $ lazy-qsort(value, tops) for top in tops* do : println(file, "%_: %_" % [value(top) * msecs(res), strings[key(top)]]) else : for trace in id-traces(res) do : - var first?:True|False = true - for record in trace do : - if not silence-ids[record] : - print(file, ";") when not first? - print(file, strings[record]) - first? = false + for (record in filter({ not silence-ids[_] }, ids(trace)), i in 0 to max-depth) do : + print(file, ";") when i > 0 + print(file, strings[record]) print(file, " ") - println(file, msecs(res)) + println(file, msecs(trace)) From 7fd1535531ab5f10acadb0688ca30d0c78696e52 Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Thu, 23 Jan 2025 16:24:15 -0800 Subject: [PATCH 16/38] improve profiler interface --- compiler/main.stanza | 9 +++++---- core/profiler.stanza | 7 +++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/compiler/main.stanza b/compiler/main.stanza index d544fce5..78671b47 100644 --- a/compiler/main.stanza +++ b/compiler/main.stanza @@ -71,10 +71,11 @@ defn run-with-timing-log (body:() -> ?T, cmd-args:CommandArgs) -> T : ;If -profile-compiler flag is provided, then execute the body ;with support for profiling. defn run-with-profiler (body:() -> ?T, cmd-args:CommandArgs) -> T : - if flag?(cmd-args, "profile-compiler") : - profiling(body, 100, { stack-collapse(_, cmd-args["profile-compiler"]) }) - else : - body() + generate : + if flag?(cmd-args, "profile-compiler") : + stack-collapse(profiling({ yield(body()) }, 100), cmd-args["profile-compiler"]) + else : + body() ;============================================================ ;====================== Verbosity =========================== diff --git a/core/profiler.stanza b/core/profiler.stanza index 6bc43f9b..c6d8cd42 100644 --- a/core/profiler.stanza +++ b/core/profiler.stanza @@ -280,11 +280,10 @@ public defn stop-profiling () -> ProfileResult: collect-profiling(do-stop-sample-profiling()) ;Profiling wrapper to be called with within -public defn profiling (f:() -> ?T, msecs:Int, g:ProfileResult -> ? = { false }) -> T : +public defn profiling (f:() -> ?, msecs:Int) -> ProfileResult : start-profiling(msecs) - val res = f() - g(stop-profiling()) - res + f() + stop-profiling() ;;; convert to flame-graph format using traditional stack-collapse function From b259c1c34f6dbe1e591a3542e580a3b6b439b3c3 Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Tue, 28 Jan 2025 09:06:35 -0800 Subject: [PATCH 17/38] get initial version of retained heap analysis going --- compiler/codegen.stanza | 1 + compiler/vm-structures.stanza | 1 + core/core.stanza | 113 +++++--- core/heap-analysis.stanza | 476 ++++++++++++++++++++++++++++++++++ core/long-vector.stanza | 3 +- 5 files changed, 562 insertions(+), 32 deletions(-) create mode 100644 core/heap-analysis.stanza diff --git a/compiler/codegen.stanza b/compiler/codegen.stanza index 96c140bc..71f706c8 100644 --- a/compiler/codegen.stanza +++ b/compiler/codegen.stanza @@ -334,6 +334,7 @@ public defn compile-entry-function (emitter:CodeEmitter, stubs:AsmStubs) : #label(safepoint-table) ;safepoint-table:ptr #label(debug-table) ;debug-table:ptr #label(local-var-table) ;local-var-table:ptr + #long() ;heap-dominator-tree:ptr #label(class-table) ;class-table:ptr #label(global-root-table) ;global-root-table:ptr #label(stackmap-table) ;stackmap-table:ptr diff --git a/compiler/vm-structures.stanza b/compiler/vm-structures.stanza index 3bad507f..6fef0c40 100644 --- a/compiler/vm-structures.stanza +++ b/compiler/vm-structures.stanza @@ -31,6 +31,7 @@ public lostanza deftype VMState : var safepoint-table: ptr ;(Permanent State) var debug-table: ptr ;(Permanent State) var local-var-table: ptr ;(Permanent State) + var heap-dominator-tree: ptr ;(Variable State) var class-table: ptr ;(Permanent State) ;Interpreted Mode Tables var instructions: ptr ;(Permanent State) diff --git a/core/core.stanza b/core/core.stanza index a2a602ac..77d487b7 100644 --- a/core/core.stanza +++ b/core/core.stanza @@ -214,31 +214,83 @@ protected lostanza deftype ArrayRecord : ;are used only in compiled mode. ;Permanent state changes in-between each code load. ;Variable state changes in-between each boundary change. -protected lostanza deftype VMState : - ;Compiled and Interpreted Mode - global-offsets: ptr ;(Permanent State) - global-mem: ptr ;(Permanent State) - var sig-handler: long ;(Permanent State) - var current-coroutine-ptr: ptr ;[TODO] Change to long to represent reference. - var stepping-coroutine-ptr: ptr ;[TODO] Change to long to represent reference. - const-table: ptr ;(Permanent State) - const-mem: ptr ;(Permanent State) - data-offsets: ptr ;(Permanent State) - data-mem: ptr ;(Permanent State) - code-offsets: ptr ;(Permanent State) - registers: ptr ;(Permanent State) - system-registers: ptr ;(Permanent State) - var heap: Heap ;(Variable State) - safepoint-table: ptr ;(Variable State) - debug-table: ptr ;(Variable State) - local-var-table: ptr ;(Variable State) - ;Compiled Mode Tables - class-table: ptr - global-root-table: ptr - stackmap-table: ptr> - stack-trace-table: ptr - extern-table: ptr - extern-defn-table: ptr +#if-defined(BOOTSTRAP) : + + protected lostanza deftype VMState : + ;Compiled and Interpreted Mode + global-offsets: ptr ;(Permanent State) + global-mem: ptr ;(Permanent State) + var sig-handler: long ;(Permanent State) + var current-coroutine-ptr: ptr ;[TODO] Change to long to represent reference. + var stepping-coroutine-ptr: ptr ;[TODO] Change to long to represent reference. + const-table: ptr ;(Permanent State) + const-mem: ptr ;(Permanent State) + data-offsets: ptr ;(Permanent State) + data-mem: ptr ;(Permanent State) + code-offsets: ptr ;(Permanent State) + registers: ptr ;(Permanent State) + system-registers: ptr ;(Permanent State) + var heap: Heap ;(Variable State) + safepoint-table: ptr ;(Variable State) + debug-table: ptr ;(Variable State) + local-var-table: ptr ;(Variable State) + ;Compiled Mode Tables + class-table: ptr + global-root-table: ptr + stackmap-table: ptr> + stack-trace-table: ptr + extern-table: ptr + extern-defn-table: ptr + + lostanza defn initialize-dominator-tree () -> ref : + return false + +#else: + + protected lostanza deftype HeapDominator : + var roots : ptr + var sizes : ptr + var addrs : ptr + var offs : ptr + var heap : ptr + + protected lostanza deftype VMState : + ;Compiled and Interpreted Mode + global-offsets: ptr ;(Permanent State) + global-mem: ptr ;(Permanent State) + var sig-handler: long ;(Permanent State) + var current-coroutine-ptr: ptr ;[TODO] Change to long to represent reference. + var stepping-coroutine-ptr: ptr ;[TODO] Change to long to represent reference. + const-table: ptr ;(Permanent State) + const-mem: ptr ;(Permanent State) + data-offsets: ptr ;(Permanent State) + data-mem: ptr ;(Permanent State) + code-offsets: ptr ;(Permanent State) + registers: ptr ;(Permanent State) + system-registers: ptr ;(Permanent State) + var heap: Heap ;(Variable State) + safepoint-table: ptr ;(Variable State) + debug-table: ptr ;(Variable State) + local-var-table: ptr ;(Variable State) + var dom: ptr ;(Variable State) + ;Compiled Mode Tables + class-table: ptr + global-root-table: ptr + stackmap-table: ptr> + stack-trace-table: ptr + extern-table: ptr + extern-defn-table: ptr + + lostanza defn initialize-dominator-tree () -> ref : + val vms:ptr = call-prim flush-vm() + val dom = (call-c clib/malloc(sizeof(HeapDominator))) as ptr + dom.roots = LSLongVector() + dom.sizes = LSLongVector() + dom.addrs = LSLongVector() + dom.offs = LSLongVector() + dom.heap = LSLongVector() + vms.dom = dom + return false lostanza deftype ExternTable : length: long @@ -1614,9 +1666,9 @@ lostanza defn iterate-roots (f:ptr<((ptr, ptr) -> ref)>, return [vms.heap.iterate-roots](f, vms) ;Call f on all references stored in the object pointed to by p. -lostanza defn iterate-references (p:ptr, - f:ptr<((ptr, ptr) -> ref)>, - vms:ptr) -> ref : +protected lostanza defn iterate-references (p:ptr, + f:ptr<((ptr, ptr) -> ref)>, + vms:ptr) -> ref : ;Retrieve the object's tag. val tag = get-tag(p) ;Fast path using fast descriptor table. @@ -2632,8 +2684,8 @@ public lostanza defn clear (start:ptr, size:long) -> ptr : return call-c clib/memset(start, 0, size) ;Call f on all root pointers. -lostanza defn core-iterate-roots (f:ptr<((ptr, ptr) -> ref)>, - vms:ptr) -> ref : +protected lostanza defn core-iterate-roots (f:ptr<((ptr, ptr) -> ref)>, + vms:ptr) -> ref : ;Scan globals val globals = vms.global-mem as ptr val roots = vms.global-root-table @@ -4203,6 +4255,7 @@ initialize-gc-notifiers() initialize-gc-statistics() initialize-liveness-handlers() initialize-symbol-table() +initialize-dominator-tree() ;================================================================================ ;========================== End of Boot Sequence ================================ diff --git a/core/heap-analysis.stanza b/core/heap-analysis.stanza new file mode 100644 index 00000000..9a222d2f --- /dev/null +++ b/core/heap-analysis.stanza @@ -0,0 +1,476 @@ +defpackage core/heap-analysis : + import core + import collections + import core/long-vector + +; TODO: +; remove unique-id +; flatten prevs/nexts ??? -- perhaps adaptive structure like a list + +defn scatter (src:Seqable, idx:Tuple) -> Tuple : + val dst = Array(length(idx)) + for (x in src, i in 0 to false) do : dst[idx[i]] = x + to-tuple(dst) + +defn gather (src:Tuple, idx:Seqable) -> Seq : + seq({ src[_] }, idx) + +defn gather (src:IndexedCollection, idx:Seqable) -> Seq : + seq({ src[_] }, idx) + +lostanza defn clear (v:ptr) -> ref : + v.length = 0 + return false + +;;; INTERFACE TO STANZA MEMORY SYSTEM + +lostanza defn addrs (dom:ptr) -> ptr : + return dom.addrs as ptr + +lostanza defn heap (dom:ptr) -> ptr : + return dom.heap as ptr + +lostanza defn sizes (dom:ptr) -> ptr : + return dom.sizes as ptr + +lostanza defn roots (dom:ptr) -> ptr : + return dom.roots as ptr + +lostanza defn offs (dom:ptr) -> ptr : + return dom.offs as ptr + +lostanza defn collect-object-address-and-size (p:ptr, tag:int, size:long, vms:ptr) -> ref : + add(addrs(vms.dom), p as long) + ; call-c clib/printf("ADDR %lx\n", p) + add(sizes(vms.dom), size as long) + return false + +lostanza var unique-id:long = 1000L + +lostanza defn collect-object-contents (p:ptr, tag:int, size:long, vms:ptr) -> ref : + add(offs(vms.dom), heap(vms.dom).length) + add(heap(vms.dom), tag as long) + add(heap(vms.dom), unique-id) + unique-id = unique-id + 1L + val idx = heap(vms.dom).length + add(heap(vms.dom), 0L) ; place holder + core/iterate-references(p, addr(do-collect-object-contents), vms) + heap(vms.dom).items[idx] = heap(vms.dom).length - idx - 1 + return false + +lostanza defn do-collect-object-contents (ref:ptr, vms:ptr) -> ref : + ;Retrieve the value at the given heap pointer. + val v = [ref] + ;Is this a reference to a Stanza heap object? + val tagbits = v & 7L + if tagbits == 1L : + ;Remove the tag bits to retrieve the object pointer. + val p = (v - 1) as ptr + add(heap(vms.dom), addr-to-id(addrs(vms.dom), p as long) + 1) + return false + +public lostanza defn register-all-roots (vms:ptr) -> ref : + core/core-iterate-roots(addr(register-root-reference), vms) + register-stack-roots(vms) + return false + +public lostanza defn register-stack-roots (vms:ptr) -> ref : + var stack:ptr = vms.heap.stacks + while stack != null : + iterate-references-in-stack-frames(stack, addr(register-root-reference), vms) + stack = stack.tail + return false + +public lostanza defn register-root-reference (ref:ptr, vms:ptr) -> ref : + val v = [ref] + val tagbits = v & 7L ; heap object? + if tagbits == 1L : + val p = (v - 1) as ptr ; remove tag bits to retrieve object pointer + add(roots(vms.dom), p as long) + return false + +lostanza defn iterate-objects + (pstart:ptr, pend:ptr, vms:ptr, f:ptr<((ptr, int, long, ptr) -> ref)>) -> ref : + var p:ptr = pstart + while p < pend : + val tag = [p] as int + val class = vms.class-table[tag].record + var size:long = 0L + if class.item-size == 0 : + size = object-size-on-heap(class.size) + else : + val class = class as ptr + val array = p as ptr + val len = array.slots[0] + val base-size = class.base-size + val item-size = class.item-size + val my-size = base-size + item-size * len + size = object-size-on-heap(my-size) + [f](p, tag, size, vms) + p = p + size + return false + +lostanza defn addr-to-id (xs:ptr, x:long) -> long : + var res:long = -1L + labels : + begin: goto loop(0L, xs.length) + loop (start:long, end:long) : + if end > start : + val center = (start + end) >> 1 + val xc = xs.items[center] + if x == xc : res = center + else if x < xc : goto loop(start, center) + else : goto loop(center + 1L, end) + return res + +lostanza deftype LowFlatObjects : + var sizes : ptr ; static sizes of objects + var offs : ptr ; offsets to inlined objects in heap + var heap : ptr ; | type | len | ids ... | ... + +lostanza deftype FlatObjects <: IndexedCollection&Lengthable : + value : ptr + +lostanza defn FlatObjects (sizes:ptr, offs:ptr, heap:ptr) -> ref : + val lfo = call-c clib/stz_malloc(sizeof(LowFlatObjects)) as ptr + lfo.sizes = sizes + lfo.offs = offs + lfo.heap = heap + return new FlatObjects{ lfo } + +lostanza defmethod length (xs:ref) -> ref : + return new Int{xs.value.offs.length} + +lostanza defn offset (xs:ref, id:ref) -> ref : + return new Int{xs.value.offs.items[id.value] as int} + +lostanza defmethod get (xs:ref, idx:ref) -> ref : + return new Int{xs.value.heap.items[idx.value] as int} + +defn get-all (xs:FlatObjects, indices:Range) -> Seq : + seq({ xs[_] }, indices) + +defn tag-of (xs:FlatObjects, id:Int) -> Int : + xs[offset(xs, id)] + +defn unique-of (xs:FlatObjects, id:Int) -> Int : + xs[offset(xs, id) + 1] + +lostanza defn size-of (xs:ref, id:ref) -> ref : + return new Int{ xs.value.sizes.items[id.value] as int } + +defn sizes (objs:FlatObjects) -> Seq : + seq(size-of{objs, _}, 0 to length(objs)) + +defn refs (objs:FlatObjects, id:Int) -> Seqable : + val off = offset(objs, id) + val len = objs[off + 2] + val refs-off = off + 3 + get-all(objs, refs-off to (refs-off + len)) + +lostanza defn do-dominator-tree () -> ref : + call-c clib/printf("GC...\n") + run-garbage-collector() + val vms:ptr = call-prim flush-vm() + val dom = vms.dom + clear(offs(dom)) + clear(sizes(dom)) + clear(heap(dom)) + ;; get all roots + call-c clib/printf("REG ROOTS...\n") + register-all-roots(vms) + call-c clib/printf("FOUND %d ROOTS...\n", roots(dom).length) + ;; get sizes and addresses of objects on heap + add(sizes(dom), roots(dom).length as long) ; dummy root object + call-c clib/printf("COLLECT HEAP %lx OBJECT ADDRESSES AND SIZES...\n", vms.heap.start) + iterate-objects(vms.heap.start, vms.heap.old-objects-end, vms, addr(collect-object-address-and-size)) + val nursery = core/nursery-start(addr(vms.heap)) + call-c clib/printf("COLLECT NURSERY %lx OBJECT ADDRESSES AND SIZES...\n", nursery) + iterate-objects(nursery, vms.heap.top, vms, addr(collect-object-address-and-size)) + call-c clib/printf("DONE %d OBJECTS...\n", addrs(dom).length) + ;; build heap data translated to object ids using addresses and binary search + add(offs(dom), 0L) ; first root object + add(heap(dom), -1L) ; dummy root object tag + add(heap(dom), unique-id) + unique-id = unique-id + 1L + add(heap(dom), roots(dom).length as long) + for (var i:int = 0, i < roots(dom).length, i = i + 1) : + add(heap(dom), addr-to-id(addrs(dom), roots(dom).items[i]) + 1) ; point to roots + iterate-objects(vms.heap.start, vms.heap.old-objects-end, vms, addr(collect-object-contents)) + iterate-objects(nursery, vms.heap.top, vms, addr(collect-object-contents)) + clear(addrs(dom)) + clear(roots(dom)) + call-c clib/printf("DONE... %d OFFS\n", offs(dom).length) + return FlatObjects(sizes(dom), offs(dom), heap(dom)) + +;;; FlatIdObjects + +defstruct FlatIdObjects : + order : Tuple + reorder : Tuple + objs : FlatObjects +with: + printer => true + +defn sizes (o:FlatIdObjects) -> Seq : + gather(to-tuple(sizes(objs(o))), order(o)) + +defn length (ios:FlatIdObjects) -> Int : + length(objs(ios)) + +lostanza defn class-name (x:ref) -> ref : + var res:ref + if x.value == -1 : + res = String("root") + else : + res = String(class-name(x.value)) + return res + +defn nexts (fobjs:FlatIdObjects) -> Tuple> : + val objs = objs(fobjs) + to-tuple $ for id in order(fobjs) seq : + to-list $ seq({ reorder(fobjs)[_] }, refs(objs, id)) + +defn prevs (nexts:Tuple>) -> Tuple> : + val prevs = Array>(length(nexts), List()) + for (next in nexts, id in 0 to false) do : + for r in next do : + prevs[r] = cons(id, prevs[r]) + to-tuple $ prevs + +defn id-print-guts (idx:Int, id:Int, unique:Int, tag:Int, refs:Seqable) : + print("%_: %_ = {%_ %_ %_}" % [idx, id, class-name(tag), unique, to-tuple $ refs]) + +defn print-id-object-guts (objs:FlatObjects) -> False : + for id in 0 to length(objs) do : + id-print-guts(id, id, unique-of(objs, id), tag-of(objs, id), refs(objs, id)) + println("") + +defn id-print-stat (idx:Int, id:Int, unique:Int, tag:Int, tot-size:Int, size:Int) : + print("%_: %_ = {%_ %_ %_ %_}" % [idx, id, class-name(tag), unique, size, tot-size]) + +defn print-id-object-stats (objs:FlatObjects, tot-sizes:Tuple) -> False : + val ids = reverse $ to-list $ qsort({ tot-sizes[_] }, 0 to length(objs)) + for (id in ids, i in 0 to false) do : + val tot-size = tot-sizes[id] + if tot-size > 0 : + id-print-stat(i, id, unique-of(objs, id), tag-of(objs, id), tot-size, size-of(objs, id)) + println("") + +defn objects-to-id-objects (objs:FlatObjects) -> FlatIdObjects : + ; print-id-object-guts(objs) + FlatIdObjects(to-tuple $ (0 to length(objs)), to-tuple $ (0 to length(objs)), objs) + +;;; DOMINATORS + +;; find depth first order of objects +defn depth-first (ios:FlatIdObjects) -> FlatIdObjects : + val nexts = nexts(ios) + val visited? = Array(length(ios), false) + val order0 = Vector() + let loop (idx:Int = 0) : + if not visited?[idx] : + visited?[idx] = true + for nidx in nexts[idx] do : loop(nidx) + add(order0, idx) + val missing = filter({ not visited?[_] }, 0 to length(visited?)) + val order = to-tuple $ cat(missing, order0) + ; println("DFS %_ %_" % [length(order), order]) + FlatIdObjects(to-tuple $ order, scatter(0 to length(order), to-tuple(order)), objs(ios)) + +; fast dominators algorithm assuming depth-first order +defn idom (num:Int, prevs:Tuple>) -> Tuple : + ; println("IDOM NUM %_ PREVS %_" % [num, prevs]) + val doms = Array(num, -1) + val start-id = num - 1 + doms[start-id] = start-id + defn intersect (b1:Int, b2:Int) -> Int : + let loop (finger1:Int = b1, finger2:Int = b2) : + if finger1 != finger2 : + val finger1 = let iter (finger1:Int = finger1) : + if finger1 < finger2 : iter(doms[finger1]) + else : finger1 + val finger2 = let iter (finger2:Int = finger2) : + if finger2 < finger1 : iter(doms[finger2]) + else : finger2 + loop(finger1, finger2) + else : + finger1 + let loop () : + let iter (b : Int = start-id - 1, changed? : True|False = false) : + if b >= 0 : + val new-idom = let find (idom:Int = -1, ps:List = prevs[b]) : + if empty?(ps) : + idom + else : + val p = head(ps) + val nxt-idom = + if doms[p] != -1 : + if idom == -1 : p + else : intersect(p, idom) + else : idom + find(nxt-idom, tail(ps)) + val changed? = doms[b] != new-idom + doms[b] = new-idom + iter(b - 1, changed?) + else : + loop() when changed? + to-tuple $ doms + +defn calc-sizes (ios:FlatIdObjects, doms:Tuple) -> Array : + val tot-sizes = to-array $ sizes(ios) + ; println("%_: %_" % [0, tot-sizes]) + val len = length(ios) + for i in 0 to (len - 1) do : + if doms[i] >= 0 : + tot-sizes[doms[i]] = tot-sizes[doms[i]] + tot-sizes[i] + ; println("%_: %_" % [i + 1, tot-sizes]) + tot-sizes + +defn print-xml (s:FileOutputStream, id-objs:FlatIdObjects, sizes:Array, nexts:Tuple>, doms:Tuple, threshold:Int = 0) : + val objs = objs(id-objs) + defn children (doms:Tuple) -> Tuple> : + val children = to-tuple $ repeatedly({ Vector() }, length(nexts)) + for (dom in doms, id in 0 to false) do : + add(children[dom], id) when (dom >= 0 and dom != id) + map(to-tuple, children) + defn stringify (s:String) -> String : + replace(s, "&", "A") + defn indent (n:Int) : + for i in 0 to n do : print(s, " ") + val kiddies = children(doms) + let walk (idx:Int = length(doms) - 1, depth:Int = 0) : + val id = order(id-objs)[idx] + val name = stringify(class-name(tag-of(objs, id))) + indent(depth * 2) println(s, "<%_ RETAINED=\"%_\" STATIC=\"%_\">" % [name, sizes[idx], size-of(objs, id)]) + val childs = reverse $ to-list $ qsort({ sizes[_] }, filter({ sizes[_] > threshold }, kiddies[idx])) + for child in childs do : + walk(child, depth + 1) + indent(depth * 2) println(s, "" % [name]) + +public defn heap-dominator-tree (filename:String) -> FlatIdObjects : + val objs = do-dominator-tree() + ; val objs = tst-dominator-tree() + ; dump-heap(objs) + val id-objs0 = objects-to-id-objects(objs) + ; val nxts0 = nexts(id-objs0) + ; val prvs0 = prevs(nxts0) + ; for (id in order(id-objs0), i in 0 to false) do : + ; id-print-guts(i, id, unique-of(objs, id), tag-of(objs, id), refs(objs, id)) + ; print(" NEXTS %_ PREVS %_" % [nxts0[i], prvs0[i]]) + ; println("") + val id-objs = depth-first(id-objs0) + ; val nxts = nexts(id-objs) + ; val prvs = prevs(nxts) + ; for (id in order(id-objs), i in 0 to false) do : + ; id-print-guts(i, id, unique-of(objs, id), tag-of(objs, id), refs(objs, id)) + ; print(" NEXTS %_ PREVS %_" % [nxts[i], prvs[i]]) + ; println("") + val nxts = nexts(id-objs) + val doms = idom(length(id-objs), prevs(nxts)) + ; println("IDOM DONE %_" % [doms]) + val sizes = calc-sizes(id-objs, doms) + ; println("SIZES %_" % [sizes]) + print-id-object-stats(objs, to-tuple $ gather(sizes, reorder(id-objs))) + val s = FileOutputStream(filename) + print-xml(s, id-objs, sizes, nxts, doms) + close(s) + id-objs + +; defstruct Tup : +; value : Tuple +; +; val tup3 = Tup([ 0 1 2 3 4 5 6 7 8 9 ]) +; val tup2 = Tup([ tup3 tup3 ]) +; val tup1 = Tup([ tup2 tup2 ]) +; val tup0 = Tup([ tup1 tup1 ]) + +defstruct BinTree : + left : BinTree|Int + right : BinTree|Int + +defn bin-tree (n:Int) -> BinTree : + if n <= 0 : + BinTree(0, 1) + else : + BinTree(bin-tree(n - 1), bin-tree(n - 1)) + +val tup = bin-tree(6) + +heap-dominator-tree("sizes.xml") + +; val tst-2 = [ 1 ] +; val tst-1 = [ tst-2 ] +; val tst-0 = [ tst-1, tst-2 ] +; +; lostanza defn object-type-id (x:ref) -> int : +; val ref = x as long +; val p = (ref - 1L) as ptr +; return [p] as int +; +; lostanza defn tst-dominator-tree () -> ref : +; val vms:ptr = call-prim flush-vm() +; val dom = vms.dom +; clear(offs(dom)) +; clear(sizes(dom)) +; clear(heap(dom)) +; ;0 +; add(offs(dom), 0L) +; add(heap(dom), -1L) +; add(heap(dom), unique-id) +; unique-id = unique-id + 1L +; add(heap(dom), 1L) +; add(heap(dom), 2L) +; add(sizes(dom), 2L * 8L) +; ;1 +; add(offs(dom), heap(dom).length) +; add(heap(dom), object-type-id(tst-1)) +; add(heap(dom), unique-id) +; unique-id = unique-id + 1L +; add(heap(dom), 1L) +; add(heap(dom), 3L) +; add(sizes(dom), 8L + 1L * 8L) +; ;2 +; add(offs(dom), heap(dom).length) +; add(heap(dom), object-type-id(tst-0)) +; add(heap(dom), unique-id) +; unique-id = unique-id + 1L +; add(heap(dom), 2L) +; add(heap(dom), 1L) +; add(heap(dom), 3L) +; add(sizes(dom), 8L + 2L * 8L) +; ;3 +; add(offs(dom), heap(dom).length) +; add(heap(dom), object-type-id(tst-1)) +; add(heap(dom), unique-id) +; unique-id = unique-id + 1L +; add(heap(dom), 0L) +; add(sizes(dom), 8L) +; return FlatObjects(sizes(dom), offs(dom), heap(dom)) + +lostanza defn dump-heap (objs:ref) -> ref : + call-c clib/printf("OFFS:\n") + for (var i:int = 0, i < objs.value.offs.length, i = i + 1) : + call-c clib/printf("%d : %ld\n", i, objs.value.offs.items[i]) + call-c clib/printf("HEAP:\n") + for (var i:int = 0, i < objs.value.heap.length, i = i + 1) : + call-c clib/printf("%d : %ld\n", i, objs.value.heap.items[i]) + return false + +defstruct IntArrayPow2 : + sizes : IntArray + offs : IntArray + items : IntArray + +defn accum (xs:Seqable, init:Int) -> Seq : + var accum:Int = init + for x in xs seq : (val r = accum, accum = accum + x, r) + +defn IntArrayPow2 (sizes:IntArray) -> IntArrayPow2 : + val len = length(sizes) + val offs = to-intarray $ accum(sizes, 0) + val items = IntArray(offs[len - 1] + sizes[len - 1]) + IntArrayPow2(sizes, offs, items) + +defn get (vv:IntArrayPow2, idx:Int) -> Collection : + items(vv)[offs(vv)[idx] to (offs(vv)[idx] + sizes(vv)[idx])] \ No newline at end of file diff --git a/core/long-vector.stanza b/core/long-vector.stanza index 062c15d4..5b828227 100644 --- a/core/long-vector.stanza +++ b/core/long-vector.stanza @@ -1,6 +1,5 @@ defpackage core/long-vector : import core - import collections public lostanza deftype LSLongVector : var capacity: int @@ -39,4 +38,4 @@ lostanza defn ensure-capacity (v:ptr, new-capacity:int) -> int : while c < new-capacity : c = c << 1 v.capacity = c v.items = realloc(v.items, c * sizeof(long)) - return 0 \ No newline at end of file + return 0 From 6bee7f81c289851aaaea629f93cf8f22c05114f3 Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Tue, 28 Jan 2025 09:15:52 -0800 Subject: [PATCH 18/38] remove unique-ids and clean up print outs --- core/heap-analysis.stanza | 43 +++++++++------------------------------ 1 file changed, 10 insertions(+), 33 deletions(-) diff --git a/core/heap-analysis.stanza b/core/heap-analysis.stanza index 9a222d2f..f354372b 100644 --- a/core/heap-analysis.stanza +++ b/core/heap-analysis.stanza @@ -3,10 +3,6 @@ defpackage core/heap-analysis : import collections import core/long-vector -; TODO: -; remove unique-id -; flatten prevs/nexts ??? -- perhaps adaptive structure like a list - defn scatter (src:Seqable, idx:Tuple) -> Tuple : val dst = Array(length(idx)) for (x in src, i in 0 to false) do : dst[idx[i]] = x @@ -41,17 +37,12 @@ lostanza defn offs (dom:ptr) -> ptr : lostanza defn collect-object-address-and-size (p:ptr, tag:int, size:long, vms:ptr) -> ref : add(addrs(vms.dom), p as long) - ; call-c clib/printf("ADDR %lx\n", p) add(sizes(vms.dom), size as long) return false -lostanza var unique-id:long = 1000L - lostanza defn collect-object-contents (p:ptr, tag:int, size:long, vms:ptr) -> ref : add(offs(vms.dom), heap(vms.dom).length) add(heap(vms.dom), tag as long) - add(heap(vms.dom), unique-id) - unique-id = unique-id + 1L val idx = heap(vms.dom).length add(heap(vms.dom), 0L) ; place holder core/iterate-references(p, addr(do-collect-object-contents), vms) @@ -153,9 +144,6 @@ defn get-all (xs:FlatObjects, indices:Range) -> Seq : defn tag-of (xs:FlatObjects, id:Int) -> Int : xs[offset(xs, id)] -defn unique-of (xs:FlatObjects, id:Int) -> Int : - xs[offset(xs, id) + 1] - lostanza defn size-of (xs:ref, id:ref) -> ref : return new Int{ xs.value.sizes.items[id.value] as int } @@ -164,8 +152,8 @@ defn sizes (objs:FlatObjects) -> Seq : defn refs (objs:FlatObjects, id:Int) -> Seqable : val off = offset(objs, id) - val len = objs[off + 2] - val refs-off = off + 3 + val len = objs[off + 1] + val refs-off = off + 2 get-all(objs, refs-off to (refs-off + len)) lostanza defn do-dominator-tree () -> ref : @@ -177,7 +165,6 @@ lostanza defn do-dominator-tree () -> ref : clear(sizes(dom)) clear(heap(dom)) ;; get all roots - call-c clib/printf("REG ROOTS...\n") register-all-roots(vms) call-c clib/printf("FOUND %d ROOTS...\n", roots(dom).length) ;; get sizes and addresses of objects on heap @@ -191,8 +178,6 @@ lostanza defn do-dominator-tree () -> ref : ;; build heap data translated to object ids using addresses and binary search add(offs(dom), 0L) ; first root object add(heap(dom), -1L) ; dummy root object tag - add(heap(dom), unique-id) - unique-id = unique-id + 1L add(heap(dom), roots(dom).length as long) for (var i:int = 0, i < roots(dom).length, i = i + 1) : add(heap(dom), addr-to-id(addrs(dom), roots(dom).items[i]) + 1) ; point to roots @@ -238,23 +223,23 @@ defn prevs (nexts:Tuple>) -> Tuple> : prevs[r] = cons(id, prevs[r]) to-tuple $ prevs -defn id-print-guts (idx:Int, id:Int, unique:Int, tag:Int, refs:Seqable) : - print("%_: %_ = {%_ %_ %_}" % [idx, id, class-name(tag), unique, to-tuple $ refs]) +defn id-print-guts (id:Int, tag:Int, refs:Seqable) : + print("%_ = {%_ %_}" % [id, class-name(tag), to-tuple $ refs]) defn print-id-object-guts (objs:FlatObjects) -> False : for id in 0 to length(objs) do : - id-print-guts(id, id, unique-of(objs, id), tag-of(objs, id), refs(objs, id)) + id-print-guts(id, tag-of(objs, id), refs(objs, id)) println("") -defn id-print-stat (idx:Int, id:Int, unique:Int, tag:Int, tot-size:Int, size:Int) : - print("%_: %_ = {%_ %_ %_ %_}" % [idx, id, class-name(tag), unique, size, tot-size]) +defn id-print-stat (id:Int, tag:Int, tot-size:Int, size:Int) : + print("%_ = {%_ %_ %_}" % [id, class-name(tag), size, tot-size]) defn print-id-object-stats (objs:FlatObjects, tot-sizes:Tuple) -> False : val ids = reverse $ to-list $ qsort({ tot-sizes[_] }, 0 to length(objs)) for (id in ids, i in 0 to false) do : val tot-size = tot-sizes[id] if tot-size > 0 : - id-print-stat(i, id, unique-of(objs, id), tag-of(objs, id), tot-size, size-of(objs, id)) + id-print-stat(id, tag-of(objs, id), tot-size, size-of(objs, id)) println("") defn objects-to-id-objects (objs:FlatObjects) -> FlatIdObjects : @@ -356,14 +341,14 @@ public defn heap-dominator-tree (filename:String) -> FlatIdObjects : ; val nxts0 = nexts(id-objs0) ; val prvs0 = prevs(nxts0) ; for (id in order(id-objs0), i in 0 to false) do : - ; id-print-guts(i, id, unique-of(objs, id), tag-of(objs, id), refs(objs, id)) + ; id-print-guts(i, id, tag-of(objs, id), refs(objs, id)) ; print(" NEXTS %_ PREVS %_" % [nxts0[i], prvs0[i]]) ; println("") val id-objs = depth-first(id-objs0) ; val nxts = nexts(id-objs) ; val prvs = prevs(nxts) ; for (id in order(id-objs), i in 0 to false) do : - ; id-print-guts(i, id, unique-of(objs, id), tag-of(objs, id), refs(objs, id)) + ; id-print-guts(i, id, tag-of(objs, id), refs(objs, id)) ; print(" NEXTS %_ PREVS %_" % [nxts[i], prvs[i]]) ; println("") val nxts = nexts(id-objs) @@ -417,24 +402,18 @@ heap-dominator-tree("sizes.xml") ; ;0 ; add(offs(dom), 0L) ; add(heap(dom), -1L) -; add(heap(dom), unique-id) -; unique-id = unique-id + 1L ; add(heap(dom), 1L) ; add(heap(dom), 2L) ; add(sizes(dom), 2L * 8L) ; ;1 ; add(offs(dom), heap(dom).length) ; add(heap(dom), object-type-id(tst-1)) -; add(heap(dom), unique-id) -; unique-id = unique-id + 1L ; add(heap(dom), 1L) ; add(heap(dom), 3L) ; add(sizes(dom), 8L + 1L * 8L) ; ;2 ; add(offs(dom), heap(dom).length) ; add(heap(dom), object-type-id(tst-0)) -; add(heap(dom), unique-id) -; unique-id = unique-id + 1L ; add(heap(dom), 2L) ; add(heap(dom), 1L) ; add(heap(dom), 3L) @@ -442,8 +421,6 @@ heap-dominator-tree("sizes.xml") ; ;3 ; add(offs(dom), heap(dom).length) ; add(heap(dom), object-type-id(tst-1)) -; add(heap(dom), unique-id) -; unique-id = unique-id + 1L ; add(heap(dom), 0L) ; add(sizes(dom), 8L) ; return FlatObjects(sizes(dom), offs(dom), heap(dom)) From f3e6f8286f4d130ec89b2313d7a57a1e4f07f89c Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Tue, 28 Jan 2025 09:27:28 -0800 Subject: [PATCH 19/38] clean up more --- core/heap-analysis.stanza | 136 +++++--------------------------------- 1 file changed, 17 insertions(+), 119 deletions(-) diff --git a/core/heap-analysis.stanza b/core/heap-analysis.stanza index f354372b..37f757c6 100644 --- a/core/heap-analysis.stanza +++ b/core/heap-analysis.stanza @@ -35,12 +35,14 @@ lostanza defn roots (dom:ptr) -> ptr : lostanza defn offs (dom:ptr) -> ptr : return dom.offs as ptr -lostanza defn collect-object-address-and-size (p:ptr, tag:int, size:long, vms:ptr) -> ref : +lostanza defn collect-object-address-and-size + (p:ptr, tag:int, size:long, vms:ptr) -> ref : add(addrs(vms.dom), p as long) add(sizes(vms.dom), size as long) return false -lostanza defn collect-object-contents (p:ptr, tag:int, size:long, vms:ptr) -> ref : +lostanza defn collect-object-contents + (p:ptr, tag:int, size:long, vms:ptr) -> ref : add(offs(vms.dom), heap(vms.dom).length) add(heap(vms.dom), tag as long) val idx = heap(vms.dom).length @@ -81,7 +83,8 @@ public lostanza defn register-root-reference (ref:ptr, vms:ptr, pend:ptr, vms:ptr, f:ptr<((ptr, int, long, ptr) -> ref)>) -> ref : + (pstart:ptr, pend:ptr, vms:ptr, + f:ptr<((ptr, int, long, ptr) -> ref)>) -> ref : var p:ptr = pstart while p < pend : val tag = [p] as int @@ -122,7 +125,8 @@ lostanza deftype LowFlatObjects : lostanza deftype FlatObjects <: IndexedCollection&Lengthable : value : ptr -lostanza defn FlatObjects (sizes:ptr, offs:ptr, heap:ptr) -> ref : +lostanza defn FlatObjects + (sizes:ptr, offs:ptr, heap:ptr) -> ref : val lfo = call-c clib/stz_malloc(sizeof(LowFlatObjects)) as ptr lfo.sizes = sizes lfo.offs = offs @@ -243,7 +247,6 @@ defn print-id-object-stats (objs:FlatObjects, tot-sizes:Tuple) -> False : println("") defn objects-to-id-objects (objs:FlatObjects) -> FlatIdObjects : - ; print-id-object-guts(objs) FlatIdObjects(to-tuple $ (0 to length(objs)), to-tuple $ (0 to length(objs)), objs) ;;; DOMINATORS @@ -260,12 +263,10 @@ defn depth-first (ios:FlatIdObjects) -> FlatIdObjects : add(order0, idx) val missing = filter({ not visited?[_] }, 0 to length(visited?)) val order = to-tuple $ cat(missing, order0) - ; println("DFS %_ %_" % [length(order), order]) FlatIdObjects(to-tuple $ order, scatter(0 to length(order), to-tuple(order)), objs(ios)) ; fast dominators algorithm assuming depth-first order defn idom (num:Int, prevs:Tuple>) -> Tuple : - ; println("IDOM NUM %_ PREVS %_" % [num, prevs]) val doms = Array(num, -1) val start-id = num - 1 doms[start-id] = start-id @@ -304,15 +305,15 @@ defn idom (num:Int, prevs:Tuple>) -> Tuple : defn calc-sizes (ios:FlatIdObjects, doms:Tuple) -> Array : val tot-sizes = to-array $ sizes(ios) - ; println("%_: %_" % [0, tot-sizes]) val len = length(ios) for i in 0 to (len - 1) do : if doms[i] >= 0 : tot-sizes[doms[i]] = tot-sizes[doms[i]] + tot-sizes[i] - ; println("%_: %_" % [i + 1, tot-sizes]) tot-sizes -defn print-xml (s:FileOutputStream, id-objs:FlatIdObjects, sizes:Array, nexts:Tuple>, doms:Tuple, threshold:Int = 0) : +defn print-xml + (s:FileOutputStream, id-objs:FlatIdObjects, sizes:Array, + nexts:Tuple>, doms:Tuple, threshold:Int = 0) : val objs = objs(id-objs) defn children (doms:Tuple) -> Tuple> : val children = to-tuple $ repeatedly({ Vector() }, length(nexts)) @@ -321,133 +322,30 @@ defn print-xml (s:FileOutputStream, id-objs:FlatIdObjects, sizes:Array, nex map(to-tuple, children) defn stringify (s:String) -> String : replace(s, "&", "A") - defn indent (n:Int) : - for i in 0 to n do : print(s, " ") + defn P (n:Int, str:Printable) : + for i in 0 to (n * 2) do : print(s, " ") + println(s, str) val kiddies = children(doms) let walk (idx:Int = length(doms) - 1, depth:Int = 0) : val id = order(id-objs)[idx] val name = stringify(class-name(tag-of(objs, id))) - indent(depth * 2) println(s, "<%_ RETAINED=\"%_\" STATIC=\"%_\">" % [name, sizes[idx], size-of(objs, id)]) + P(depth, "<%_ RETAINED=\"%_\" STATIC=\"%_\">" % [name, sizes[idx], size-of(objs, id)]) val childs = reverse $ to-list $ qsort({ sizes[_] }, filter({ sizes[_] > threshold }, kiddies[idx])) for child in childs do : walk(child, depth + 1) - indent(depth * 2) println(s, "" % [name]) + P(depth, "" % [name]) public defn heap-dominator-tree (filename:String) -> FlatIdObjects : val objs = do-dominator-tree() - ; val objs = tst-dominator-tree() - ; dump-heap(objs) val id-objs0 = objects-to-id-objects(objs) - ; val nxts0 = nexts(id-objs0) - ; val prvs0 = prevs(nxts0) - ; for (id in order(id-objs0), i in 0 to false) do : - ; id-print-guts(i, id, tag-of(objs, id), refs(objs, id)) - ; print(" NEXTS %_ PREVS %_" % [nxts0[i], prvs0[i]]) - ; println("") val id-objs = depth-first(id-objs0) - ; val nxts = nexts(id-objs) - ; val prvs = prevs(nxts) - ; for (id in order(id-objs), i in 0 to false) do : - ; id-print-guts(i, id, tag-of(objs, id), refs(objs, id)) - ; print(" NEXTS %_ PREVS %_" % [nxts[i], prvs[i]]) - ; println("") val nxts = nexts(id-objs) val doms = idom(length(id-objs), prevs(nxts)) - ; println("IDOM DONE %_" % [doms]) val sizes = calc-sizes(id-objs, doms) - ; println("SIZES %_" % [sizes]) print-id-object-stats(objs, to-tuple $ gather(sizes, reorder(id-objs))) val s = FileOutputStream(filename) print-xml(s, id-objs, sizes, nxts, doms) close(s) id-objs -; defstruct Tup : -; value : Tuple -; -; val tup3 = Tup([ 0 1 2 3 4 5 6 7 8 9 ]) -; val tup2 = Tup([ tup3 tup3 ]) -; val tup1 = Tup([ tup2 tup2 ]) -; val tup0 = Tup([ tup1 tup1 ]) - -defstruct BinTree : - left : BinTree|Int - right : BinTree|Int - -defn bin-tree (n:Int) -> BinTree : - if n <= 0 : - BinTree(0, 1) - else : - BinTree(bin-tree(n - 1), bin-tree(n - 1)) - -val tup = bin-tree(6) - -heap-dominator-tree("sizes.xml") - -; val tst-2 = [ 1 ] -; val tst-1 = [ tst-2 ] -; val tst-0 = [ tst-1, tst-2 ] -; -; lostanza defn object-type-id (x:ref) -> int : -; val ref = x as long -; val p = (ref - 1L) as ptr -; return [p] as int -; -; lostanza defn tst-dominator-tree () -> ref : -; val vms:ptr = call-prim flush-vm() -; val dom = vms.dom -; clear(offs(dom)) -; clear(sizes(dom)) -; clear(heap(dom)) -; ;0 -; add(offs(dom), 0L) -; add(heap(dom), -1L) -; add(heap(dom), 1L) -; add(heap(dom), 2L) -; add(sizes(dom), 2L * 8L) -; ;1 -; add(offs(dom), heap(dom).length) -; add(heap(dom), object-type-id(tst-1)) -; add(heap(dom), 1L) -; add(heap(dom), 3L) -; add(sizes(dom), 8L + 1L * 8L) -; ;2 -; add(offs(dom), heap(dom).length) -; add(heap(dom), object-type-id(tst-0)) -; add(heap(dom), 2L) -; add(heap(dom), 1L) -; add(heap(dom), 3L) -; add(sizes(dom), 8L + 2L * 8L) -; ;3 -; add(offs(dom), heap(dom).length) -; add(heap(dom), object-type-id(tst-1)) -; add(heap(dom), 0L) -; add(sizes(dom), 8L) -; return FlatObjects(sizes(dom), offs(dom), heap(dom)) - -lostanza defn dump-heap (objs:ref) -> ref : - call-c clib/printf("OFFS:\n") - for (var i:int = 0, i < objs.value.offs.length, i = i + 1) : - call-c clib/printf("%d : %ld\n", i, objs.value.offs.items[i]) - call-c clib/printf("HEAP:\n") - for (var i:int = 0, i < objs.value.heap.length, i = i + 1) : - call-c clib/printf("%d : %ld\n", i, objs.value.heap.items[i]) - return false - -defstruct IntArrayPow2 : - sizes : IntArray - offs : IntArray - items : IntArray - -defn accum (xs:Seqable, init:Int) -> Seq : - var accum:Int = init - for x in xs seq : (val r = accum, accum = accum + x, r) - -defn IntArrayPow2 (sizes:IntArray) -> IntArrayPow2 : - val len = length(sizes) - val offs = to-intarray $ accum(sizes, 0) - val items = IntArray(offs[len - 1] + sizes[len - 1]) - IntArrayPow2(sizes, offs, items) - -defn get (vv:IntArrayPow2, idx:Int) -> Collection : - items(vv)[offs(vv)[idx] to (offs(vv)[idx] + sizes(vv)[idx])] \ No newline at end of file +; heap-dominator-tree("sizes.xml") \ No newline at end of file From eadcf8ba4b4c85ae90b0c1f2be6bb4aed0e44798 Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Tue, 28 Jan 2025 09:46:41 -0800 Subject: [PATCH 20/38] more comments and cleanups --- core/heap-analysis.stanza | 90 ++++++++++++++++++++++----------------- 1 file changed, 52 insertions(+), 38 deletions(-) diff --git a/core/heap-analysis.stanza b/core/heap-analysis.stanza index 37f757c6..0d49fe6e 100644 --- a/core/heap-analysis.stanza +++ b/core/heap-analysis.stanza @@ -3,6 +3,8 @@ defpackage core/heap-analysis : import collections import core/long-vector +;;; UTILITIES + defn scatter (src:Seqable, idx:Tuple) -> Tuple : val dst = Array(length(idx)) for (x in src, i in 0 to false) do : dst[idx[i]] = x @@ -18,6 +20,14 @@ lostanza defn clear (v:ptr) -> ref : v.length = 0 return false +lostanza defn class-name (x:ref) -> ref : + var res:ref + if x.value == -1 : + res = String("root") + else : + res = String(class-name(x.value)) + return res + ;;; INTERFACE TO STANZA MEMORY SYSTEM lostanza defn addrs (dom:ptr) -> ptr : @@ -104,6 +114,7 @@ lostanza defn iterate-objects p = p + size return false +;; Look up offset into sorted list of object addresses using binary search lostanza defn addr-to-id (xs:ptr, x:long) -> long : var res:long = -1L labels : @@ -117,6 +128,10 @@ lostanza defn addr-to-id (xs:ptr, x:long) -> long : else : goto loop(center + 1L, end) return res +;;; LowFlatObject -- create flat and packed version of roots and objects +;;; -- stores tag, num-refs, refs for each object +;;; -- also has extra root root object with ref per root + lostanza deftype LowFlatObjects : var sizes : ptr ; static sizes of objects var offs : ptr ; offsets to inlined objects in heap @@ -129,8 +144,8 @@ lostanza defn FlatObjects (sizes:ptr, offs:ptr, heap:ptr) -> ref : val lfo = call-c clib/stz_malloc(sizeof(LowFlatObjects)) as ptr lfo.sizes = sizes - lfo.offs = offs - lfo.heap = heap + lfo.offs = offs + lfo.heap = heap return new FlatObjects{ lfo } lostanza defmethod length (xs:ref) -> ref : @@ -142,6 +157,7 @@ lostanza defn offset (xs:ref, id:ref) -> ref : lostanza defmethod get (xs:ref, idx:ref) -> ref : return new Int{xs.value.heap.items[idx.value] as int} +; for some reason can't name this method get like in stanza runtime defn get-all (xs:FlatObjects, indices:Range) -> Seq : seq({ xs[_] }, indices) @@ -155,12 +171,13 @@ defn sizes (objs:FlatObjects) -> Seq : seq(size-of{objs, _}, 0 to length(objs)) defn refs (objs:FlatObjects, id:Int) -> Seqable : - val off = offset(objs, id) - val len = objs[off + 1] + val off = offset(objs, id) ; base + val num-refs = objs[off + 1] val refs-off = off + 2 - get-all(objs, refs-off to (refs-off + len)) + get-all(objs, refs-off to (refs-off + num-refs)) -lostanza defn do-dominator-tree () -> ref : +;; Pack roots / heap into FlatObjects +lostanza defn FlatObjects () -> ref : call-c clib/printf("GC...\n") run-garbage-collector() val vms:ptr = call-prim flush-vm() @@ -178,22 +195,26 @@ lostanza defn do-dominator-tree () -> ref : val nursery = core/nursery-start(addr(vms.heap)) call-c clib/printf("COLLECT NURSERY %lx OBJECT ADDRESSES AND SIZES...\n", nursery) iterate-objects(nursery, vms.heap.top, vms, addr(collect-object-address-and-size)) - call-c clib/printf("DONE %d OBJECTS...\n", addrs(dom).length) + call-c clib/printf("FOUND %d OBJECTS...\n", addrs(dom).length) ;; build heap data translated to object ids using addresses and binary search add(offs(dom), 0L) ; first root object add(heap(dom), -1L) ; dummy root object tag add(heap(dom), roots(dom).length as long) + call-c clib/printf("CONVERTING ROOT ADDRESSES TO IDS...\n") for (var i:int = 0, i < roots(dom).length, i = i + 1) : add(heap(dom), addr-to-id(addrs(dom), roots(dom).items[i]) + 1) ; point to roots + call-c clib/printf("PACKING HEAP DATA...\n") iterate-objects(vms.heap.start, vms.heap.old-objects-end, vms, addr(collect-object-contents)) + call-c clib/printf("PACKING NURSERY DATA...\n") iterate-objects(nursery, vms.heap.top, vms, addr(collect-object-contents)) clear(addrs(dom)) clear(roots(dom)) - call-c clib/printf("DONE... %d OFFS\n", offs(dom).length) + call-c clib/printf("DONE...\n") return FlatObjects(sizes(dom), offs(dom), heap(dom)) ;;; FlatIdObjects +;; Permutation wrapper of flat-objects defstruct FlatIdObjects : order : Tuple reorder : Tuple @@ -207,14 +228,6 @@ defn sizes (o:FlatIdObjects) -> Seq : defn length (ios:FlatIdObjects) -> Int : length(objs(ios)) -lostanza defn class-name (x:ref) -> ref : - var res:ref - if x.value == -1 : - res = String("root") - else : - res = String(class-name(x.value)) - return res - defn nexts (fobjs:FlatIdObjects) -> Tuple> : val objs = objs(fobjs) to-tuple $ for id in order(fobjs) seq : @@ -227,25 +240,6 @@ defn prevs (nexts:Tuple>) -> Tuple> : prevs[r] = cons(id, prevs[r]) to-tuple $ prevs -defn id-print-guts (id:Int, tag:Int, refs:Seqable) : - print("%_ = {%_ %_}" % [id, class-name(tag), to-tuple $ refs]) - -defn print-id-object-guts (objs:FlatObjects) -> False : - for id in 0 to length(objs) do : - id-print-guts(id, tag-of(objs, id), refs(objs, id)) - println("") - -defn id-print-stat (id:Int, tag:Int, tot-size:Int, size:Int) : - print("%_ = {%_ %_ %_}" % [id, class-name(tag), size, tot-size]) - -defn print-id-object-stats (objs:FlatObjects, tot-sizes:Tuple) -> False : - val ids = reverse $ to-list $ qsort({ tot-sizes[_] }, 0 to length(objs)) - for (id in ids, i in 0 to false) do : - val tot-size = tot-sizes[id] - if tot-size > 0 : - id-print-stat(id, tag-of(objs, id), tot-size, size-of(objs, id)) - println("") - defn objects-to-id-objects (objs:FlatObjects) -> FlatIdObjects : FlatIdObjects(to-tuple $ (0 to length(objs)), to-tuple $ (0 to length(objs)), objs) @@ -336,16 +330,36 @@ defn print-xml P(depth, "" % [name]) public defn heap-dominator-tree (filename:String) -> FlatIdObjects : - val objs = do-dominator-tree() + val objs = FlatObjects() val id-objs0 = objects-to-id-objects(objs) val id-objs = depth-first(id-objs0) val nxts = nexts(id-objs) val doms = idom(length(id-objs), prevs(nxts)) val sizes = calc-sizes(id-objs, doms) - print-id-object-stats(objs, to-tuple $ gather(sizes, reorder(id-objs))) + ; print-id-object-stats(objs, to-tuple $ gather(sizes, reorder(id-objs))) val s = FileOutputStream(filename) print-xml(s, id-objs, sizes, nxts, doms) close(s) id-objs -; heap-dominator-tree("sizes.xml") \ No newline at end of file +heap-dominator-tree("sizes.xml") + +; defn id-print-guts (id:Int, tag:Int, refs:Seqable) : +; print("%_ = {%_ %_}" % [id, class-name(tag), to-tuple $ refs]) +; +; defn print-id-object-guts (objs:FlatObjects) -> False : +; for id in 0 to length(objs) do : +; id-print-guts(id, tag-of(objs, id), refs(objs, id)) +; println("") +; +; defn id-print-stat (id:Int, tag:Int, tot-size:Int, size:Int) : +; print("%_ = {%_ %_ %_}" % [id, class-name(tag), size, tot-size]) +; +; defn print-id-object-stats (objs:FlatObjects, tot-sizes:Tuple) -> False : +; val ids = reverse $ to-list $ qsort({ tot-sizes[_] }, 0 to length(objs)) +; for (id in ids, i in 0 to false) do : +; val tot-size = tot-sizes[id] +; if tot-size > 0 : +; id-print-stat(id, tag-of(objs, id), tot-size, size-of(objs, id)) +; println("") + From e267344e92205179b6943626186e24c7a4934cac Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Tue, 28 Jan 2025 09:57:14 -0800 Subject: [PATCH 21/38] remove top-level call to run it --- core/heap-analysis.stanza | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/heap-analysis.stanza b/core/heap-analysis.stanza index 0d49fe6e..34fc1155 100644 --- a/core/heap-analysis.stanza +++ b/core/heap-analysis.stanza @@ -342,7 +342,7 @@ public defn heap-dominator-tree (filename:String) -> FlatIdObjects : close(s) id-objs -heap-dominator-tree("sizes.xml") +; heap-dominator-tree("sizes.xml") ; defn id-print-guts (id:Int, tag:Int, refs:Seqable) : ; print("%_ = {%_ %_}" % [id, class-name(tag), to-tuple $ refs]) From e72f91f773c552842a60e16705d74486e6b9999c Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Tue, 28 Jan 2025 11:46:41 -0800 Subject: [PATCH 22/38] prepare for PR --- core/heap-analysis.stanza | 671 +++++++++++++++++++------------------- scripts/make.sh | 1 + 2 files changed, 339 insertions(+), 333 deletions(-) diff --git a/core/heap-analysis.stanza b/core/heap-analysis.stanza index 34fc1155..a1253107 100644 --- a/core/heap-analysis.stanza +++ b/core/heap-analysis.stanza @@ -28,338 +28,343 @@ lostanza defn class-name (x:ref) -> ref : res = String(class-name(x.value)) return res -;;; INTERFACE TO STANZA MEMORY SYSTEM - -lostanza defn addrs (dom:ptr) -> ptr : - return dom.addrs as ptr - -lostanza defn heap (dom:ptr) -> ptr : - return dom.heap as ptr - -lostanza defn sizes (dom:ptr) -> ptr : - return dom.sizes as ptr - -lostanza defn roots (dom:ptr) -> ptr : - return dom.roots as ptr - -lostanza defn offs (dom:ptr) -> ptr : - return dom.offs as ptr - -lostanza defn collect-object-address-and-size - (p:ptr, tag:int, size:long, vms:ptr) -> ref : - add(addrs(vms.dom), p as long) - add(sizes(vms.dom), size as long) - return false - -lostanza defn collect-object-contents - (p:ptr, tag:int, size:long, vms:ptr) -> ref : - add(offs(vms.dom), heap(vms.dom).length) - add(heap(vms.dom), tag as long) - val idx = heap(vms.dom).length - add(heap(vms.dom), 0L) ; place holder - core/iterate-references(p, addr(do-collect-object-contents), vms) - heap(vms.dom).items[idx] = heap(vms.dom).length - idx - 1 - return false - -lostanza defn do-collect-object-contents (ref:ptr, vms:ptr) -> ref : - ;Retrieve the value at the given heap pointer. - val v = [ref] - ;Is this a reference to a Stanza heap object? - val tagbits = v & 7L - if tagbits == 1L : - ;Remove the tag bits to retrieve the object pointer. - val p = (v - 1) as ptr - add(heap(vms.dom), addr-to-id(addrs(vms.dom), p as long) + 1) - return false - -public lostanza defn register-all-roots (vms:ptr) -> ref : - core/core-iterate-roots(addr(register-root-reference), vms) - register-stack-roots(vms) - return false - -public lostanza defn register-stack-roots (vms:ptr) -> ref : - var stack:ptr = vms.heap.stacks - while stack != null : - iterate-references-in-stack-frames(stack, addr(register-root-reference), vms) - stack = stack.tail - return false - -public lostanza defn register-root-reference (ref:ptr, vms:ptr) -> ref : - val v = [ref] - val tagbits = v & 7L ; heap object? - if tagbits == 1L : - val p = (v - 1) as ptr ; remove tag bits to retrieve object pointer - add(roots(vms.dom), p as long) - return false - -lostanza defn iterate-objects - (pstart:ptr, pend:ptr, vms:ptr, - f:ptr<((ptr, int, long, ptr) -> ref)>) -> ref : - var p:ptr = pstart - while p < pend : - val tag = [p] as int - val class = vms.class-table[tag].record - var size:long = 0L - if class.item-size == 0 : - size = object-size-on-heap(class.size) - else : - val class = class as ptr - val array = p as ptr - val len = array.slots[0] - val base-size = class.base-size - val item-size = class.item-size - val my-size = base-size + item-size * len - size = object-size-on-heap(my-size) - [f](p, tag, size, vms) - p = p + size - return false - -;; Look up offset into sorted list of object addresses using binary search -lostanza defn addr-to-id (xs:ptr, x:long) -> long : - var res:long = -1L - labels : - begin: goto loop(0L, xs.length) - loop (start:long, end:long) : - if end > start : - val center = (start + end) >> 1 - val xc = xs.items[center] - if x == xc : res = center - else if x < xc : goto loop(start, center) - else : goto loop(center + 1L, end) - return res - -;;; LowFlatObject -- create flat and packed version of roots and objects -;;; -- stores tag, num-refs, refs for each object -;;; -- also has extra root root object with ref per root - -lostanza deftype LowFlatObjects : - var sizes : ptr ; static sizes of objects - var offs : ptr ; offsets to inlined objects in heap - var heap : ptr ; | type | len | ids ... | ... - -lostanza deftype FlatObjects <: IndexedCollection&Lengthable : - value : ptr - -lostanza defn FlatObjects - (sizes:ptr, offs:ptr, heap:ptr) -> ref : - val lfo = call-c clib/stz_malloc(sizeof(LowFlatObjects)) as ptr - lfo.sizes = sizes - lfo.offs = offs - lfo.heap = heap - return new FlatObjects{ lfo } - -lostanza defmethod length (xs:ref) -> ref : - return new Int{xs.value.offs.length} - -lostanza defn offset (xs:ref, id:ref) -> ref : - return new Int{xs.value.offs.items[id.value] as int} - -lostanza defmethod get (xs:ref, idx:ref) -> ref : - return new Int{xs.value.heap.items[idx.value] as int} - -; for some reason can't name this method get like in stanza runtime -defn get-all (xs:FlatObjects, indices:Range) -> Seq : - seq({ xs[_] }, indices) - -defn tag-of (xs:FlatObjects, id:Int) -> Int : - xs[offset(xs, id)] - -lostanza defn size-of (xs:ref, id:ref) -> ref : - return new Int{ xs.value.sizes.items[id.value] as int } - -defn sizes (objs:FlatObjects) -> Seq : - seq(size-of{objs, _}, 0 to length(objs)) - -defn refs (objs:FlatObjects, id:Int) -> Seqable : - val off = offset(objs, id) ; base - val num-refs = objs[off + 1] - val refs-off = off + 2 - get-all(objs, refs-off to (refs-off + num-refs)) - -;; Pack roots / heap into FlatObjects -lostanza defn FlatObjects () -> ref : - call-c clib/printf("GC...\n") - run-garbage-collector() - val vms:ptr = call-prim flush-vm() - val dom = vms.dom - clear(offs(dom)) - clear(sizes(dom)) - clear(heap(dom)) - ;; get all roots - register-all-roots(vms) - call-c clib/printf("FOUND %d ROOTS...\n", roots(dom).length) - ;; get sizes and addresses of objects on heap - add(sizes(dom), roots(dom).length as long) ; dummy root object - call-c clib/printf("COLLECT HEAP %lx OBJECT ADDRESSES AND SIZES...\n", vms.heap.start) - iterate-objects(vms.heap.start, vms.heap.old-objects-end, vms, addr(collect-object-address-and-size)) - val nursery = core/nursery-start(addr(vms.heap)) - call-c clib/printf("COLLECT NURSERY %lx OBJECT ADDRESSES AND SIZES...\n", nursery) - iterate-objects(nursery, vms.heap.top, vms, addr(collect-object-address-and-size)) - call-c clib/printf("FOUND %d OBJECTS...\n", addrs(dom).length) - ;; build heap data translated to object ids using addresses and binary search - add(offs(dom), 0L) ; first root object - add(heap(dom), -1L) ; dummy root object tag - add(heap(dom), roots(dom).length as long) - call-c clib/printf("CONVERTING ROOT ADDRESSES TO IDS...\n") - for (var i:int = 0, i < roots(dom).length, i = i + 1) : - add(heap(dom), addr-to-id(addrs(dom), roots(dom).items[i]) + 1) ; point to roots - call-c clib/printf("PACKING HEAP DATA...\n") - iterate-objects(vms.heap.start, vms.heap.old-objects-end, vms, addr(collect-object-contents)) - call-c clib/printf("PACKING NURSERY DATA...\n") - iterate-objects(nursery, vms.heap.top, vms, addr(collect-object-contents)) - clear(addrs(dom)) - clear(roots(dom)) - call-c clib/printf("DONE...\n") - return FlatObjects(sizes(dom), offs(dom), heap(dom)) - -;;; FlatIdObjects - -;; Permutation wrapper of flat-objects -defstruct FlatIdObjects : - order : Tuple - reorder : Tuple - objs : FlatObjects -with: - printer => true - -defn sizes (o:FlatIdObjects) -> Seq : - gather(to-tuple(sizes(objs(o))), order(o)) - -defn length (ios:FlatIdObjects) -> Int : - length(objs(ios)) - -defn nexts (fobjs:FlatIdObjects) -> Tuple> : - val objs = objs(fobjs) - to-tuple $ for id in order(fobjs) seq : - to-list $ seq({ reorder(fobjs)[_] }, refs(objs, id)) - -defn prevs (nexts:Tuple>) -> Tuple> : - val prevs = Array>(length(nexts), List()) - for (next in nexts, id in 0 to false) do : - for r in next do : - prevs[r] = cons(id, prevs[r]) - to-tuple $ prevs - -defn objects-to-id-objects (objs:FlatObjects) -> FlatIdObjects : - FlatIdObjects(to-tuple $ (0 to length(objs)), to-tuple $ (0 to length(objs)), objs) - -;;; DOMINATORS - -;; find depth first order of objects -defn depth-first (ios:FlatIdObjects) -> FlatIdObjects : - val nexts = nexts(ios) - val visited? = Array(length(ios), false) - val order0 = Vector() - let loop (idx:Int = 0) : - if not visited?[idx] : - visited?[idx] = true - for nidx in nexts[idx] do : loop(nidx) - add(order0, idx) - val missing = filter({ not visited?[_] }, 0 to length(visited?)) - val order = to-tuple $ cat(missing, order0) - FlatIdObjects(to-tuple $ order, scatter(0 to length(order), to-tuple(order)), objs(ios)) - -; fast dominators algorithm assuming depth-first order -defn idom (num:Int, prevs:Tuple>) -> Tuple : - val doms = Array(num, -1) - val start-id = num - 1 - doms[start-id] = start-id - defn intersect (b1:Int, b2:Int) -> Int : - let loop (finger1:Int = b1, finger2:Int = b2) : - if finger1 != finger2 : - val finger1 = let iter (finger1:Int = finger1) : - if finger1 < finger2 : iter(doms[finger1]) - else : finger1 - val finger2 = let iter (finger2:Int = finger2) : - if finger2 < finger1 : iter(doms[finger2]) - else : finger2 - loop(finger1, finger2) - else : - finger1 - let loop () : - let iter (b : Int = start-id - 1, changed? : True|False = false) : - if b >= 0 : - val new-idom = let find (idom:Int = -1, ps:List = prevs[b]) : - if empty?(ps) : - idom - else : - val p = head(ps) - val nxt-idom = - if doms[p] != -1 : - if idom == -1 : p - else : intersect(p, idom) - else : idom - find(nxt-idom, tail(ps)) - val changed? = doms[b] != new-idom - doms[b] = new-idom - iter(b - 1, changed?) +#if-defined(BOOTSTRAP) : + + ;;; INTERFACE TO STANZA MEMORY SYSTEM + + lostanza defn addrs (dom:ptr) -> ptr : + return dom.addrs as ptr + + lostanza defn heap (dom:ptr) -> ptr : + return dom.heap as ptr + + lostanza defn sizes (dom:ptr) -> ptr : + return dom.sizes as ptr + + lostanza defn roots (dom:ptr) -> ptr : + return dom.roots as ptr + + lostanza defn offs (dom:ptr) -> ptr : + return dom.offs as ptr + + lostanza defn collect-object-address-and-size + (p:ptr, tag:int, size:long, vms:ptr) -> ref : + add(addrs(vms.dom), p as long) + add(sizes(vms.dom), size as long) + return false + + lostanza defn collect-object-contents + (p:ptr, tag:int, size:long, vms:ptr) -> ref : + add(offs(vms.dom), heap(vms.dom).length) + add(heap(vms.dom), tag as long) + val idx = heap(vms.dom).length + add(heap(vms.dom), 0L) ; place holder + core/iterate-references(p, addr(do-collect-object-contents), vms) + heap(vms.dom).items[idx] = heap(vms.dom).length - idx - 1 + return false + + lostanza defn do-collect-object-contents (ref:ptr, vms:ptr) -> ref : + ;Retrieve the value at the given heap pointer. + val v = [ref] + ;Is this a reference to a Stanza heap object? + val tagbits = v & 7L + if tagbits == 1L : + ;Remove the tag bits to retrieve the object pointer. + val p = (v - 1) as ptr + add(heap(vms.dom), addr-to-id(addrs(vms.dom), p as long) + 1) + return false + + public lostanza defn register-all-roots (vms:ptr) -> ref : + core/core-iterate-roots(addr(register-root-reference), vms) + register-stack-roots(vms) + return false + + public lostanza defn register-stack-roots (vms:ptr) -> ref : + var stack:ptr = vms.heap.stacks + while stack != null : + iterate-references-in-stack-frames(stack, addr(register-root-reference), vms) + stack = stack.tail + return false + + public lostanza defn register-root-reference (ref:ptr, vms:ptr) -> ref : + val v = [ref] + val tagbits = v & 7L ; heap object? + if tagbits == 1L : + val p = (v - 1) as ptr ; remove tag bits to retrieve object pointer + add(roots(vms.dom), p as long) + return false + + lostanza defn iterate-objects + (pstart:ptr, pend:ptr, vms:ptr, + f:ptr<((ptr, int, long, ptr) -> ref)>) -> ref : + var p:ptr = pstart + while p < pend : + val tag = [p] as int + val class = vms.class-table[tag].record + var size:long = 0L + if class.item-size == 0 : + size = object-size-on-heap(class.size) else : - loop() when changed? - to-tuple $ doms - -defn calc-sizes (ios:FlatIdObjects, doms:Tuple) -> Array : - val tot-sizes = to-array $ sizes(ios) - val len = length(ios) - for i in 0 to (len - 1) do : - if doms[i] >= 0 : - tot-sizes[doms[i]] = tot-sizes[doms[i]] + tot-sizes[i] - tot-sizes - -defn print-xml - (s:FileOutputStream, id-objs:FlatIdObjects, sizes:Array, - nexts:Tuple>, doms:Tuple, threshold:Int = 0) : - val objs = objs(id-objs) - defn children (doms:Tuple) -> Tuple> : - val children = to-tuple $ repeatedly({ Vector() }, length(nexts)) - for (dom in doms, id in 0 to false) do : - add(children[dom], id) when (dom >= 0 and dom != id) - map(to-tuple, children) - defn stringify (s:String) -> String : - replace(s, "&", "A") - defn P (n:Int, str:Printable) : - for i in 0 to (n * 2) do : print(s, " ") - println(s, str) - val kiddies = children(doms) - let walk (idx:Int = length(doms) - 1, depth:Int = 0) : - val id = order(id-objs)[idx] - val name = stringify(class-name(tag-of(objs, id))) - P(depth, "<%_ RETAINED=\"%_\" STATIC=\"%_\">" % [name, sizes[idx], size-of(objs, id)]) - val childs = reverse $ to-list $ qsort({ sizes[_] }, filter({ sizes[_] > threshold }, kiddies[idx])) - for child in childs do : - walk(child, depth + 1) - P(depth, "" % [name]) - -public defn heap-dominator-tree (filename:String) -> FlatIdObjects : - val objs = FlatObjects() - val id-objs0 = objects-to-id-objects(objs) - val id-objs = depth-first(id-objs0) - val nxts = nexts(id-objs) - val doms = idom(length(id-objs), prevs(nxts)) - val sizes = calc-sizes(id-objs, doms) - ; print-id-object-stats(objs, to-tuple $ gather(sizes, reorder(id-objs))) - val s = FileOutputStream(filename) - print-xml(s, id-objs, sizes, nxts, doms) - close(s) - id-objs - -; heap-dominator-tree("sizes.xml") - -; defn id-print-guts (id:Int, tag:Int, refs:Seqable) : -; print("%_ = {%_ %_}" % [id, class-name(tag), to-tuple $ refs]) -; -; defn print-id-object-guts (objs:FlatObjects) -> False : -; for id in 0 to length(objs) do : -; id-print-guts(id, tag-of(objs, id), refs(objs, id)) -; println("") -; -; defn id-print-stat (id:Int, tag:Int, tot-size:Int, size:Int) : -; print("%_ = {%_ %_ %_}" % [id, class-name(tag), size, tot-size]) -; -; defn print-id-object-stats (objs:FlatObjects, tot-sizes:Tuple) -> False : -; val ids = reverse $ to-list $ qsort({ tot-sizes[_] }, 0 to length(objs)) -; for (id in ids, i in 0 to false) do : -; val tot-size = tot-sizes[id] -; if tot-size > 0 : -; id-print-stat(id, tag-of(objs, id), tot-size, size-of(objs, id)) -; println("") + val class = class as ptr + val array = p as ptr + val len = array.slots[0] + val base-size = class.base-size + val item-size = class.item-size + val my-size = base-size + item-size * len + size = object-size-on-heap(my-size) + [f](p, tag, size, vms) + p = p + size + return false + + ;; Look up offset into sorted list of object addresses using binary search + lostanza defn addr-to-id (xs:ptr, x:long) -> long : + var res:long = -1L + labels : + begin: goto loop(0L, xs.length) + loop (start:long, end:long) : + if end > start : + val center = (start + end) >> 1 + val xc = xs.items[center] + if x == xc : res = center + else if x < xc : goto loop(start, center) + else : goto loop(center + 1L, end) + return res + + ;;; LowFlatObject -- create flat and packed version of roots and objects + ;;; -- stores tag, num-refs, refs for each object + ;;; -- also has extra root root object with ref per root + + lostanza deftype LowFlatObjects : + var sizes : ptr ; static sizes of objects + var offs : ptr ; offsets to inlined objects in heap + var heap : ptr ; | type | len | ids ... | ... + + lostanza deftype FlatObjects <: IndexedCollection&Lengthable : + value : ptr + + lostanza defn FlatObjects + (sizes:ptr, offs:ptr, heap:ptr) -> ref : + val lfo = call-c clib/stz_malloc(sizeof(LowFlatObjects)) as ptr + lfo.sizes = sizes + lfo.offs = offs + lfo.heap = heap + return new FlatObjects{ lfo } + + lostanza defmethod length (xs:ref) -> ref : + return new Int{xs.value.offs.length} + + lostanza defn offset (xs:ref, id:ref) -> ref : + return new Int{xs.value.offs.items[id.value] as int} + + lostanza defmethod get (xs:ref, idx:ref) -> ref : + return new Int{xs.value.heap.items[idx.value] as int} + + ; for some reason can't name this method get like in stanza runtime + defn get-all (xs:FlatObjects, indices:Range) -> Seq : + seq({ xs[_] }, indices) + + defn tag-of (xs:FlatObjects, id:Int) -> Int : + xs[offset(xs, id)] + + lostanza defn size-of (xs:ref, id:ref) -> ref : + return new Int{ xs.value.sizes.items[id.value] as int } + + defn sizes (objs:FlatObjects) -> Seq : + seq(size-of{objs, _}, 0 to length(objs)) + + defn refs (objs:FlatObjects, id:Int) -> Seqable : + val off = offset(objs, id) ; base + val num-refs = objs[off + 1] + val refs-off = off + 2 + get-all(objs, refs-off to (refs-off + num-refs)) + + ;; Pack roots / heap into FlatObjects + lostanza defn FlatObjects () -> ref : + call-c clib/printf("GC...\n") + run-garbage-collector() + val vms:ptr = call-prim flush-vm() + val dom = vms.dom + clear(offs(dom)) + clear(sizes(dom)) + clear(heap(dom)) + ;; get all roots + register-all-roots(vms) + call-c clib/printf("FOUND %d ROOTS...\n", roots(dom).length) + ;; get sizes and addresses of objects on heap + add(sizes(dom), roots(dom).length as long) ; dummy root object + call-c clib/printf("COLLECT HEAP %lx OBJECT ADDRESSES AND SIZES...\n", vms.heap.start) + iterate-objects(vms.heap.start, vms.heap.old-objects-end, vms, addr(collect-object-address-and-size)) + val nursery = core/nursery-start(addr(vms.heap)) + call-c clib/printf("COLLECT NURSERY %lx OBJECT ADDRESSES AND SIZES...\n", nursery) + iterate-objects(nursery, vms.heap.top, vms, addr(collect-object-address-and-size)) + call-c clib/printf("FOUND %d OBJECTS...\n", addrs(dom).length) + ;; build heap data translated to object ids using addresses and binary search + add(offs(dom), 0L) ; first root object + add(heap(dom), -1L) ; dummy root object tag + add(heap(dom), roots(dom).length as long) + call-c clib/printf("CONVERTING ROOT ADDRESSES TO IDS...\n") + for (var i:int = 0, i < roots(dom).length, i = i + 1) : + add(heap(dom), addr-to-id(addrs(dom), roots(dom).items[i]) + 1) ; point to roots + call-c clib/printf("PACKING HEAP DATA...\n") + iterate-objects(vms.heap.start, vms.heap.old-objects-end, vms, addr(collect-object-contents)) + call-c clib/printf("PACKING NURSERY DATA...\n") + iterate-objects(nursery, vms.heap.top, vms, addr(collect-object-contents)) + clear(addrs(dom)) + clear(roots(dom)) + call-c clib/printf("DONE...\n") + return FlatObjects(sizes(dom), offs(dom), heap(dom)) + + ;;; FlatIdObjects + + ;; Permutation wrapper of flat-objects + defstruct FlatIdObjects : + order : Tuple + reorder : Tuple + objs : FlatObjects + with: + printer => true + + defn sizes (o:FlatIdObjects) -> Seq : + gather(to-tuple(sizes(objs(o))), order(o)) + + defn length (ios:FlatIdObjects) -> Int : + length(objs(ios)) + + defn nexts (fobjs:FlatIdObjects) -> Tuple> : + val objs = objs(fobjs) + to-tuple $ for id in order(fobjs) seq : + to-list $ seq({ reorder(fobjs)[_] }, refs(objs, id)) + + defn prevs (nexts:Tuple>) -> Tuple> : + val prevs = Array>(length(nexts), List()) + for (next in nexts, id in 0 to false) do : + for r in next do : + prevs[r] = cons(id, prevs[r]) + to-tuple $ prevs + + defn objects-to-id-objects (objs:FlatObjects) -> FlatIdObjects : + FlatIdObjects(to-tuple $ (0 to length(objs)), to-tuple $ (0 to length(objs)), objs) + + ;;; DOMINATORS + + ;; find depth first order of objects + defn depth-first (ios:FlatIdObjects) -> FlatIdObjects : + val nexts = nexts(ios) + val visited? = Array(length(ios), false) + val order0 = Vector() + let loop (idx:Int = 0) : + if not visited?[idx] : + visited?[idx] = true + for nidx in nexts[idx] do : loop(nidx) + add(order0, idx) + val missing = filter({ not visited?[_] }, 0 to length(visited?)) + val order = to-tuple $ cat(missing, order0) + FlatIdObjects(to-tuple $ order, scatter(0 to length(order), to-tuple(order)), objs(ios)) + + ; fast dominators algorithm assuming depth-first order + defn idom (num:Int, prevs:Tuple>) -> Tuple : + val doms = Array(num, -1) + val start-id = num - 1 + doms[start-id] = start-id + defn intersect (b1:Int, b2:Int) -> Int : + let loop (finger1:Int = b1, finger2:Int = b2) : + if finger1 != finger2 : + val finger1 = let iter (finger1:Int = finger1) : + if finger1 < finger2 : iter(doms[finger1]) + else : finger1 + val finger2 = let iter (finger2:Int = finger2) : + if finger2 < finger1 : iter(doms[finger2]) + else : finger2 + loop(finger1, finger2) + else : + finger1 + let loop () : + let iter (b : Int = start-id - 1, changed? : True|False = false) : + if b >= 0 : + val new-idom = let find (idom:Int = -1, ps:List = prevs[b]) : + if empty?(ps) : + idom + else : + val p = head(ps) + val nxt-idom = + if doms[p] != -1 : + if idom == -1 : p + else : intersect(p, idom) + else : idom + find(nxt-idom, tail(ps)) + val changed? = doms[b] != new-idom + doms[b] = new-idom + iter(b - 1, changed?) + else : + loop() when changed? + to-tuple $ doms + + defn calc-sizes (ios:FlatIdObjects, doms:Tuple) -> Array : + val tot-sizes = to-array $ sizes(ios) + val len = length(ios) + for i in 0 to (len - 1) do : + if doms[i] >= 0 : + tot-sizes[doms[i]] = tot-sizes[doms[i]] + tot-sizes[i] + tot-sizes + + defn print-xml + (s:FileOutputStream, id-objs:FlatIdObjects, sizes:Array, + nexts:Tuple>, doms:Tuple, threshold:Int = 0) : + val objs = objs(id-objs) + defn children (doms:Tuple) -> Tuple> : + val children = to-tuple $ repeatedly({ Vector() }, length(nexts)) + for (dom in doms, id in 0 to false) do : + add(children[dom], id) when (dom >= 0 and dom != id) + map(to-tuple, children) + defn stringify (s:String) -> String : + replace(s, "&", "A") + defn P (n:Int, str:Printable) : + for i in 0 to (n * 2) do : print(s, " ") + println(s, str) + val kiddies = children(doms) + let walk (idx:Int = length(doms) - 1, depth:Int = 0) : + val id = order(id-objs)[idx] + val name = stringify(class-name(tag-of(objs, id))) + P(depth, "<%_ RETAINED=\"%_\" STATIC=\"%_\">" % [name, sizes[idx], size-of(objs, id)]) + val childs = reverse $ to-list $ qsort({ sizes[_] }, filter({ sizes[_] > threshold }, kiddies[idx])) + for child in childs do : + walk(child, depth + 1) + P(depth, "" % [name]) + + public defn heap-dominator-tree (filename:String) : + val objs = FlatObjects() + val id-objs0 = objects-to-id-objects(objs) + val id-objs = depth-first(id-objs0) + val nxts = nexts(id-objs) + val doms = idom(length(id-objs), prevs(nxts)) + val sizes = calc-sizes(id-objs, doms) + ; print-id-object-stats(objs, to-tuple $ gather(sizes, reorder(id-objs))) + val s = FileOutputStream(filename) + print-xml(s, id-objs, sizes, nxts, doms) + close(s) + + ; heap-dominator-tree("sizes.xml") + + ; defn id-print-guts (id:Int, tag:Int, refs:Seqable) : + ; print("%_ = {%_ %_}" % [id, class-name(tag), to-tuple $ refs]) + ; + ; defn print-id-object-guts (objs:FlatObjects) -> False : + ; for id in 0 to length(objs) do : + ; id-print-guts(id, tag-of(objs, id), refs(objs, id)) + ; println("") + ; + ; defn id-print-stat (id:Int, tag:Int, tot-size:Int, size:Int) : + ; print("%_ = {%_ %_ %_}" % [id, class-name(tag), size, tot-size]) + ; + ; defn print-id-object-stats (objs:FlatObjects, tot-sizes:Tuple) -> False : + ; val ids = reverse $ to-list $ qsort({ tot-sizes[_] }, 0 to length(objs)) + ; for (id in ids, i in 0 to false) do : + ; val tot-size = tot-sizes[id] + ; if tot-size > 0 : + ; id-print-stat(id, tag-of(objs, id), tot-size, size-of(objs, id)) + ; println("") + +#else : + + public defn heap-dominator-tree (filename:String) : false diff --git a/scripts/make.sh b/scripts/make.sh index c36c2106..d44e8b34 100755 --- a/scripts/make.sh +++ b/scripts/make.sh @@ -60,6 +60,7 @@ PKGFILES="math \ core/debug-table \ core/sighandler \ core/local-table \ + core/heap-analysis \ arg-parser \ line-wrap \ stz/test-driver \ From ca854d2fd9debc10cd2b986dc1a98ca6f8f75071 Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Tue, 28 Jan 2025 12:10:54 -0800 Subject: [PATCH 23/38] change arms of if bootstrap --- core/heap-analysis.stanza | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/heap-analysis.stanza b/core/heap-analysis.stanza index a1253107..61cefd95 100644 --- a/core/heap-analysis.stanza +++ b/core/heap-analysis.stanza @@ -30,6 +30,10 @@ lostanza defn class-name (x:ref) -> ref : #if-defined(BOOTSTRAP) : + public defn heap-dominator-tree (filename:String) : false + +#else : + ;;; INTERFACE TO STANZA MEMORY SYSTEM lostanza defn addrs (dom:ptr) -> ptr : @@ -364,7 +368,3 @@ lostanza defn class-name (x:ref) -> ref : ; id-print-stat(id, tag-of(objs, id), tot-size, size-of(objs, id)) ; println("") -#else : - - public defn heap-dominator-tree (filename:String) : false - From 7fd0a738802ba09d6217500a88494d46e7731d2e Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Tue, 28 Jan 2025 14:11:00 -0800 Subject: [PATCH 24/38] get both heap analyses running merged together and cleaned up --- core/core.stanza | 4 ++-- core/heap-analysis.stanza | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/core.stanza b/core/core.stanza index aacf47c8..aff516df 100644 --- a/core/core.stanza +++ b/core/core.stanza @@ -250,11 +250,11 @@ protected lostanza deftype HeapStatistic : extern-table: ptr extern-defn-table: ptr -#else: - lostanza defn initialize-dominator-tree () -> ref : return false +#else: + protected lostanza deftype HeapDominator : var roots : ptr var sizes : ptr diff --git a/core/heap-analysis.stanza b/core/heap-analysis.stanza index 5ce0fd82..db51d8e7 100644 --- a/core/heap-analysis.stanza +++ b/core/heap-analysis.stanza @@ -55,10 +55,10 @@ lostanza defn class-name (x:ref) -> ref : ; run gc, collect heap stats while walking each object in nursery and heap lostanza defn do-analyze-heap (stats:ref>) -> ref : - val vms:ptr = call-prim flush-vm() + val vms:ptr = call-prim flush-vm() run-garbage-collector() val num-classes = vms.heap-statistics.length - clear(addr(vms.heap-statistics.entries), num-classes * sizeof(HeapStatistic)) + clear(addr(vms.heap-statistics.entries), num-classes * sizeof(core/HeapStatistic)) iterate-objects(vms.heap.start, vms.heap.old-objects-end, vms, addr(collect-heap-stats)) val nursery = core/nursery-start(addr(vms.heap)) iterate-objects(nursery, vms.heap.top, vms, addr(collect-heap-stats)) From 9ea5e6c19e50e84682c40f899e261ac8912d97d9 Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Tue, 28 Jan 2025 15:08:25 -0800 Subject: [PATCH 25/38] fix bug in call to profiler --- compiler/main.stanza | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/compiler/main.stanza b/compiler/main.stanza index 78671b47..d3be04c4 100644 --- a/compiler/main.stanza +++ b/compiler/main.stanza @@ -71,11 +71,12 @@ defn run-with-timing-log (body:() -> ?T, cmd-args:CommandArgs) -> T : ;If -profile-compiler flag is provided, then execute the body ;with support for profiling. defn run-with-profiler (body:() -> ?T, cmd-args:CommandArgs) -> T : - generate : - if flag?(cmd-args, "profile-compiler") : - stack-collapse(profiling({ yield(body()) }, 100), cmd-args["profile-compiler"]) - else : - body() + var res:T + if flag?(cmd-args, "profile-compiler") : + stack-collapse(profiling({ res = body() }, 100), cmd-args["profile-compiler"]) + else : + res = body() + res ;============================================================ ;====================== Verbosity =========================== From e7efa60196c0d6c1505568236944b8c999f97e43 Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Tue, 28 Jan 2025 15:32:11 -0800 Subject: [PATCH 26/38] fix typo --- runtime/driver.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/driver.c b/runtime/driver.c index f67b49eb..dacc3f7a 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -705,7 +705,7 @@ int stop_sample_profiling() { #else -int start_profile_sampling (void *handler, int usecs) { +int start_sample_profiling (void *handler, int usecs) { return 0; } int stop_sample_profiling () { From 207e2d409c46221e29b19f13485407d4c8b476a0 Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Tue, 28 Jan 2025 15:51:57 -0800 Subject: [PATCH 27/38] rename profiler package, add core/profiler to build list --- core/profiler.stanza | 2 +- scripts/make.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/core/profiler.stanza b/core/profiler.stanza index c6d8cd42..c5622016 100644 --- a/core/profiler.stanza +++ b/core/profiler.stanza @@ -1,6 +1,6 @@ ;See License.txt for details about licensing. #use-added-syntax(fastio-serializer) -defpackage profiler : +defpackage core/profiler : import core import collections import core/stack-trace diff --git a/scripts/make.sh b/scripts/make.sh index d44e8b34..efb32b6f 100755 --- a/scripts/make.sh +++ b/scripts/make.sh @@ -61,6 +61,7 @@ PKGFILES="math \ core/sighandler \ core/local-table \ core/heap-analysis \ + core/profiler \ arg-parser \ line-wrap \ stz/test-driver \ From 33f4b8ea784cf7324714fc34249a9bd97b7f9a34 Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Tue, 28 Jan 2025 16:01:09 -0800 Subject: [PATCH 28/38] rename packages --- core/heap-analysis.stanza | 2 +- core/profiler.stanza | 2 +- scripts/make.sh | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/heap-analysis.stanza b/core/heap-analysis.stanza index db51d8e7..558a12d7 100644 --- a/core/heap-analysis.stanza +++ b/core/heap-analysis.stanza @@ -1,4 +1,4 @@ -defpackage core/heap-analysis : +defpackage heap-analyzer : import core import collections import core/long-vector diff --git a/core/profiler.stanza b/core/profiler.stanza index c5622016..c6d8cd42 100644 --- a/core/profiler.stanza +++ b/core/profiler.stanza @@ -1,6 +1,6 @@ ;See License.txt for details about licensing. #use-added-syntax(fastio-serializer) -defpackage core/profiler : +defpackage profiler : import core import collections import core/stack-trace diff --git a/scripts/make.sh b/scripts/make.sh index efb32b6f..47672095 100755 --- a/scripts/make.sh +++ b/scripts/make.sh @@ -60,8 +60,8 @@ PKGFILES="math \ core/debug-table \ core/sighandler \ core/local-table \ - core/heap-analysis \ - core/profiler \ + heap-analyzer \ + profiler \ arg-parser \ line-wrap \ stz/test-driver \ From 6aeabd7a7d9bb4eea9303dd86f854499180a7fb6 Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Tue, 28 Jan 2025 16:01:57 -0800 Subject: [PATCH 29/38] rename heap-analysis to heap-analyzer --- core/{heap-analysis.stanza => heap-analyzer.stanza} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename core/{heap-analysis.stanza => heap-analyzer.stanza} (100%) diff --git a/core/heap-analysis.stanza b/core/heap-analyzer.stanza similarity index 100% rename from core/heap-analysis.stanza rename to core/heap-analyzer.stanza From ebcaca31647f84d236830ca9574a2959bb03d84a Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Tue, 28 Jan 2025 16:28:58 -0800 Subject: [PATCH 30/38] get package naming right in files and stanza.proj etc --- core/heap-analyzer.stanza | 3 --- core/stanza.proj | 1 + 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/core/heap-analyzer.stanza b/core/heap-analyzer.stanza index 558a12d7..56a4460b 100644 --- a/core/heap-analyzer.stanza +++ b/core/heap-analyzer.stanza @@ -31,9 +31,6 @@ lostanza defn class-name (x:ref) -> ref : #if-defined(BOOTSTRAP) : public defn heap-dominator-tree (filename:String) : false - -#if-defined(BOOTSTRAP) : - public defn analyze-heap () -> Long : 0L #else : diff --git a/core/stanza.proj b/core/stanza.proj index cd23bade..53989232 100644 --- a/core/stanza.proj +++ b/core/stanza.proj @@ -7,5 +7,6 @@ packages parser/* defined-in "parser" package reader defined-in "reader.stanza" package arg-parser defined-in "arg-parser.stanza" package line-wrap defined-in "line-wrap.stanza" +package heap-analyzer defined-in "heap-analyzer.stanza" package profiler defined-in "profiler.stanza" packages core/* defined-in "." \ No newline at end of file From c308e2b80915a0cb61e54fb100c8cb2d5bebf69a Mon Sep 17 00:00:00 2001 From: Jason Watson Date: Thu, 6 Feb 2025 18:28:03 -0500 Subject: [PATCH 31/38] bump stanzapm to 0.19.0 --- compiler/params.stanza | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/params.stanza b/compiler/params.stanza index ab93bbc4..fb23f7db 100644 --- a/compiler/params.stanza +++ b/compiler/params.stanza @@ -18,7 +18,7 @@ public defn compiler-flags () : to-tuple(COMPILE-FLAGS) ;========= Stanza Configuration ======== -public val STANZA-VERSION = [0 18 97] +public val STANZA-VERSION = [0 19 0] public var STANZA-INSTALL-DIR:String = "" public var OUTPUT-PLATFORM:Symbol = `platform public var STANZA-PKG-DIRS:List = List() From ae78ab04924599d05523a5cb715a72adb532d309 Mon Sep 17 00:00:00 2001 From: Jason Watson Date: Thu, 6 Feb 2025 18:29:39 -0500 Subject: [PATCH 32/38] ci: point to stanzapm pipelines --- .github/workflows/concourse-notify-pull-request.yml | 2 +- .github/workflows/concourse-notify-push.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/concourse-notify-pull-request.yml b/.github/workflows/concourse-notify-pull-request.yml index 0d763cc1..335969f9 100644 --- a/.github/workflows/concourse-notify-pull-request.yml +++ b/.github/workflows/concourse-notify-pull-request.yml @@ -3,7 +3,7 @@ on: [ pull_request, workflow_dispatch ] env: HOST: https://ci.jitx.com TEAM: main - PIPELINE: lbstanza + PIPELINE: stanzapm INSTVARS: vars.branch="${{ github.base_ref }}" TOKEN: ${{ secrets.CONCOURSE_WEBHOOK_TOKEN }} jobs: diff --git a/.github/workflows/concourse-notify-push.yml b/.github/workflows/concourse-notify-push.yml index f8f4dcce..edc0a98c 100644 --- a/.github/workflows/concourse-notify-push.yml +++ b/.github/workflows/concourse-notify-push.yml @@ -3,7 +3,7 @@ on: [ push, workflow_dispatch ] env: HOST: https://ci.jitx.com TEAM: main - PIPELINE: lbstanza + PIPELINE: stanzapm INSTVARS: vars.branch="${{ github.ref_name }}" TOKEN: ${{ secrets.CONCOURSE_WEBHOOK_TOKEN }} jobs: From 7f2a205cf30da54fd0226fd8245e0094b7a65075 Mon Sep 17 00:00:00 2001 From: Jason Watson Date: Thu, 6 Feb 2025 18:55:33 -0500 Subject: [PATCH 33/38] ci: point to stanzapm pipelines (2) --- ci/build-stanza.sh | 4 ++-- ci/calc-stanza-version-bump.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/build-stanza.sh b/ci/build-stanza.sh index 604f0d66..829a7a7b 100755 --- a/ci/build-stanza.sh +++ b/ci/build-stanza.sh @@ -11,7 +11,7 @@ USAGE="STANZA_CONFIG=/path $0" echo " STANZA_CONFIG:" "${STANZA_CONFIG:?Usage: ${USAGE}}" # directory where .stanza config file will be stored, as in normal stanza behavior # Defaulted env var inputs - can override if necessary -echo " REPODIR:" "${REPODIR:=lbstanza}" +echo " REPODIR:" "${REPODIR:=stanzapm}" echo " CONAN_USER_HOME:" "${CONAN_USER_HOME:=${REPODIR}}" echo " CREATE_ARCHIVE:" "${CREATE_ARCHIVE:=false}" echo " CREATE_PACKAGE:" "${CREATE_PACKAGE:=false}" @@ -79,7 +79,7 @@ esac cd "${REPODIR}" -echo "Building lbstanza version ${VER} in ${PWD}" +echo "Building stanzapm version ${VER} in ${PWD}" mkdir -p build mkdir -p bin diff --git a/ci/calc-stanza-version-bump.sh b/ci/calc-stanza-version-bump.sh index 305c5755..b2ebac9a 100755 --- a/ci/calc-stanza-version-bump.sh +++ b/ci/calc-stanza-version-bump.sh @@ -10,7 +10,7 @@ TOP="${PWD}" >&2 echo " BRANCH:" "${BRANCH:?Usage: BRANCH=foo $0}" # Defaulted env var inputs - can override if necessary ->&2 echo " REPODIR:" "${REPODIR:=lbstanza}" +>&2 echo " REPODIR:" "${REPODIR:=stanzapm}" ## By default, get the most recent previous tag on this branch from git ### note: if multiple tags exist on one commit, git doesn't always desecribe the most recent one ### ### maybe use: git tag --sort=committerdate --contains HEAD~ | tail -1 From 9c1cbe8cb584e8db21cc4e8e51b98c72114ab669 Mon Sep 17 00:00:00 2001 From: Jason Watson Date: Thu, 6 Feb 2025 19:02:47 -0500 Subject: [PATCH 34/38] ci: point to stanzapm pipelines (3) --- .github/workflows/concourse-notify-pull-request.yml | 2 +- .github/workflows/concourse-notify-push.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/concourse-notify-pull-request.yml b/.github/workflows/concourse-notify-pull-request.yml index 335969f9..f7e29adf 100644 --- a/.github/workflows/concourse-notify-pull-request.yml +++ b/.github/workflows/concourse-notify-pull-request.yml @@ -10,7 +10,7 @@ jobs: trigger-pr: runs-on: ubuntu-latest env: - RESOURCE: github--lbstanza--pr + RESOURCE: github--stanzapm--pr steps: - name: Trigger Concourse Pull Request run: | diff --git a/.github/workflows/concourse-notify-push.yml b/.github/workflows/concourse-notify-push.yml index edc0a98c..0ee2e879 100644 --- a/.github/workflows/concourse-notify-push.yml +++ b/.github/workflows/concourse-notify-push.yml @@ -10,7 +10,7 @@ jobs: trigger-push: runs-on: ubuntu-latest env: - RESOURCE: git--lbstanza + RESOURCE: git--stanzapm steps: - name: Trigger Concourse resource check run: | From b17face8cf060c62ae410809f2ad47c7a0866331 Mon Sep 17 00:00:00 2001 From: Jason Watson Date: Thu, 6 Feb 2025 20:51:32 -0500 Subject: [PATCH 35/38] ci: build using stanzapm release --- ci/build-stanza-version.txt | 2 +- ci/install-stanza.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/build-stanza-version.txt b/ci/build-stanza-version.txt index 71434844..53753a14 100644 --- a/ci/build-stanza-version.txt +++ b/ci/build-stanza-version.txt @@ -7,4 +7,4 @@ # like 1.23.45 # # Use version 0.17.56 to compile 0.18.0 -0.18.96 +0.19.0 diff --git a/ci/install-stanza.sh b/ci/install-stanza.sh index eff3409e..5afc2481 100755 --- a/ci/install-stanza.sh +++ b/ci/install-stanza.sh @@ -11,8 +11,8 @@ TOP="${PWD}" # the machine that you are building for (host) # and the machine that GCC will produce code for (target). -USAGE="STANZA_BUILD_PLATFORM={linux|macos|windows} STANZA_BUILD_VER=0.17.56 STANZA_CONFIG=/path STANZA_INSTALL_DIR=/path $0" -STANZA_DOWNLOAD_BASEURL="https://github.com/StanzaOrg/lbstanza/releases/download" +USAGE="STANZA_BUILD_PLATFORM={linux|macos|windows} STANZA_BUILD_VER=0.19.0 STANZA_CONFIG=/path STANZA_INSTALL_DIR=/path $0" +STANZA_DOWNLOAD_BASEURL="https://github.com/jitx-inc/stanzapm/releases/download" # Required env var inputs @@ -27,7 +27,7 @@ BSVTXT="${THISDIR}/build-stanza-version.txt" # extract the version from first non-comment line of the file BSTZVER=$(grep -v ^\# "${BSVTXT}" | head -1 | awk '{ print $1}') echo "Using existing stanza version $BSTZVER" -echo " STANZA_BUILD_VER:" "${STANZA_BUILD_VER:=${BSTZVER}}" # 0.17.56 +echo " STANZA_BUILD_VER:" "${STANZA_BUILD_VER:=${BSTZVER}}" # 0.19.0 echo "STANZA_BUILD_PLATFORM:" "${STANZA_BUILD_PLATFORM:=$(uname -s)}" # linux|macos|Darwin|windows|MINGW64 # var input validation From 09bbc2af67f8815b216ac7af98d803d5c02e14b4 Mon Sep 17 00:00:00 2001 From: Jason Watson Date: Fri, 7 Feb 2025 12:36:47 -0500 Subject: [PATCH 36/38] Bump version to 0.19.1 --- compiler/params.stanza | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/params.stanza b/compiler/params.stanza index fb23f7db..cb5e6ae8 100644 --- a/compiler/params.stanza +++ b/compiler/params.stanza @@ -18,7 +18,7 @@ public defn compiler-flags () : to-tuple(COMPILE-FLAGS) ;========= Stanza Configuration ======== -public val STANZA-VERSION = [0 19 0] +public val STANZA-VERSION = [0 19 1] public var STANZA-INSTALL-DIR:String = "" public var OUTPUT-PLATFORM:Symbol = `platform public var STANZA-PKG-DIRS:List = List() From 5dbd8f48e6f85cce338ef19d12c0f81fe275f81a Mon Sep 17 00:00:00 2001 From: Jason Watson Date: Fri, 7 Feb 2025 17:56:43 -0500 Subject: [PATCH 37/38] Bump version to 0.19.2 --- compiler/params.stanza | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/params.stanza b/compiler/params.stanza index cb5e6ae8..6b548bfb 100644 --- a/compiler/params.stanza +++ b/compiler/params.stanza @@ -18,7 +18,7 @@ public defn compiler-flags () : to-tuple(COMPILE-FLAGS) ;========= Stanza Configuration ======== -public val STANZA-VERSION = [0 19 1] +public val STANZA-VERSION = [0 19 2] public var STANZA-INSTALL-DIR:String = "" public var OUTPUT-PLATFORM:Symbol = `platform public var STANZA-PKG-DIRS:List = List() From e693e2181daf58f600ae0688c6c79d37b824c041 Mon Sep 17 00:00:00 2001 From: jackbackrack Date: Mon, 24 Feb 2025 17:14:03 -0800 Subject: [PATCH 38/38] fix args for windows on start_sample_profiling --- runtime/driver.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/driver.c b/runtime/driver.c index dacc3f7a..dc93e1b8 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -705,7 +705,7 @@ int stop_sample_profiling() { #else -int start_sample_profiling (void *handler, int usecs) { +int start_sample_profiling (int msecs, int num_functions_arg, uint64_t *profile_flag_arg, uint64_t *function_counters_arg) { return 0; } int stop_sample_profiling () {