diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..5ace4600 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/compiler/Makefile b/compiler/Makefile index 216554ec..47df99ca 100644 --- a/compiler/Makefile +++ b/compiler/Makefile @@ -1,9 +1,20 @@ .PHONY: test -all: - stack -v build $(STACK_OPTS) +all: build install + +build: VERBOSITY_FLAG = +build: + stack $(VERBOSITY_FLAG) build $(STACK_OPTS) +build/verbose: + $(MAKE) $(MAKE_FLAGS) build VERBOSITY_FLAG="-v" + +install: VERBOSITY_FLAG = +install: + $(MAKE) $(MAKE_FLAGS) build mkdir -p ./../bin - stack -v install $(STACK_OPTS) --local-bin-path ./../bin/ + stack $(VERBOSITY_FLAG) install $(STACK_OPTS) --local-bin-path ./../bin/ +install/verbose: + $(MAKE) $(MAKE_FLAGS) install VERBOSITY_FLAG="-v" clean: rm *.cabal @@ -11,14 +22,14 @@ clean: rm -rf ../bin # If problems still persist after this, remove all GHC compilers in ~/.stack/programs/**/ -ghci-irtester: - stack ghci --main-is Troupe-compiler:exe:irtester --no-load - -ghci-troupec: - stack ghci --main-is Troupe-compiler:exe:troupec --no-load - test: stack test $(STACK_OPTS) parser-info: stack exec happy -- -i src/Parser.y + +ghci/irtester: + stack ghci --main-is Troupe-compiler:exe:irtester --no-load + +ghci/troupec: + stack ghci --main-is Troupe-compiler:exe:troupec --no-load diff --git a/compiler/app/Main.hs b/compiler/app/Main.hs index fd007e2b..400fa6f5 100644 --- a/compiler/app/Main.hs +++ b/compiler/app/Main.hs @@ -21,9 +21,8 @@ import qualified Raw2Stack import qualified Stack2JS import qualified RawOpt -- import System.IO (isEOF) -import qualified Data.ByteString as BS +import qualified Data.ByteString.Char8 as BS import Data.ByteString.Base64 (decode) -import qualified Data.ByteString.Char8 as BSChar8 import qualified Data.ByteString.Lazy.Char8 as BSLazyChar8 import System.IO import System.Exit @@ -220,7 +219,7 @@ fromStdinIR = do input <- BS.getLine if BS.isPrefixOf "!ECHO " input then let response = BS.drop 6 input - in do BSChar8.putStrLn response + in do BS.putStrLn response -- debugOut "echo" else case decode input of @@ -244,7 +243,7 @@ fromStdinIRJson = do input <- BS.getLine if BS.isPrefixOf "!ECHO " input then let response = BS.drop 6 input - in BSChar8.putStrLn response + in BS.putStrLn response else case decode input of Right bs -> diff --git a/compiler/src/ClosureConv.hs b/compiler/src/ClosureConv.hs index d92d4024..4b212f1c 100644 --- a/compiler/src/ClosureConv.hs +++ b/compiler/src/ClosureConv.hs @@ -201,7 +201,7 @@ cpsToIR (CPS.LetSimple vname@(VN ident) st kt) = do cpsToIR (CPS.LetRet (CPS.Cont arg kt') kt) = do t <- cpsToIR kt t' <- local (insVar arg) (cpsToIR kt') - return $ CCIR.BB [] $ Call arg t t' + return $ CCIR.BB [] $ StackExpand arg t t' cpsToIR (CPS.LetFun fdefs kt) = do let vnames_orig = map (\(CPS.Fun fname _) -> fname) fdefs let localExt = local (insVars vnames_orig) diff --git a/compiler/src/IR.hs b/compiler/src/IR.hs index 8621c088..c4836153 100644 --- a/compiler/src/IR.hs +++ b/compiler/src/IR.hs @@ -91,7 +91,7 @@ data IRTerminator -- and then execute the second BB, which can refer to this variable and -- where PC is reset to the level before entering the first BB. -- Represents a "let x = ... in ..." format. - | Call VarName IRBBTree IRBBTree + | StackExpand VarName IRBBTree IRBBTree deriving (Eq,Show,Generic) @@ -147,7 +147,7 @@ instance ComputesDependencies IRBBTree where instance ComputesDependencies IRTerminator where dependencies (If _ bb1 bb2) = mapM_ dependencies [bb1, bb2] dependencies (AssertElseError _ bb1 _ _) = dependencies bb1 - dependencies (Call _ t1 t2) = dependencies t1 >> dependencies t2 + dependencies (StackExpand _ t1 t2) = dependencies t1 >> dependencies t2 dependencies _ = return () instance ComputesDependencies FunDef where @@ -231,15 +231,15 @@ instance WellFormedIRCheck IRInst where wfir (Assign (VN x) e) = do checkId x wfir e wfir (MkFunClosures _ fdefs) = mapM_ (\((VN x), _) -> checkId x) fdefs - + instance WellFormedIRCheck IRTerminator where wfir (If _ bb1 bb2) = do wfir bb1 wfir bb2 wfir (AssertElseError _ bb _ _) = wfir bb - wfir (Call (VN x) bb1 bb2 ) = do - checkId x + wfir (StackExpand (VN x) bb1 bb2 ) = do + checkId x wfir bb1 wfir bb2 @@ -442,7 +442,8 @@ ppIR (MkFunClosures varmap fdefs) = -ppTr (Call vn bb1 bb2) = (ppId vn <+> text "= call" $$ nest 2 (ppBB bb1)) $$ (ppBB bb2) + +ppTr (StackExpand vn bb1 bb2) = (ppId vn <+> text "= call" $$ nest 2 (ppBB bb1)) $$ (ppBB bb2) ppTr (AssertElseError va ir va2 _) diff --git a/compiler/src/IR2Raw.hs b/compiler/src/IR2Raw.hs index 7f663c17..6bc633c9 100644 --- a/compiler/src/IR2Raw.hs +++ b/compiler/src/IR2Raw.hs @@ -699,7 +699,7 @@ tr2raw = \case return $ If r bb1' bb2' -- Revision 2023-08: Equivalent, only way of modifying bb2 changed. - IR.Call v irBB1 irBB2 -> do + IR.StackExpand v irBB1 irBB2 -> do bb1 <- tree2raw irBB1 BB insts2 tr2 <- tree2raw irBB2 -- Prepend before insts2 instructions to store in variable v the result @@ -711,7 +711,7 @@ tr2raw = \case -- generally using Sequence (faster concatenation) for instructions -- might improve performance let bb2 = BB insts2' tr2 - return $ Call bb1 bb2 + return $ StackExpand bb1 bb2 -- Note: This is translated into branching and Error for throwing RT exception -- Revision 2023-08: More fine-grained raising of blocking label, see below. diff --git a/compiler/src/IROpt.hs b/compiler/src/IROpt.hs index 610c1f24..f0676ef2 100644 --- a/compiler/src/IROpt.hs +++ b/compiler/src/IROpt.hs @@ -67,7 +67,7 @@ instance Substitutable IRTerminator where AssertElseError (apply subst x) (apply subst bb) (apply subst y) pos LibExport x -> LibExport (apply subst x) Error x pos -> Error (apply subst x) pos - Call decVar bb1 bb2 -> Call decVar (apply subst bb1) (apply subst bb2) + StackExpand decVar bb1 bb2 -> StackExpand decVar (apply subst bb1) (apply subst bb2) instance Substitutable IRBBTree where apply subst (BB insts tr) = @@ -462,7 +462,7 @@ trPeval (AssertElseError x bb y_err pos) = do return $ BB [] (AssertElseError x bb' y_err pos) -trPeval (Call x bb1 bb2) = do +trPeval (StackExpand x bb1 bb2) = do bb1' <- peval bb1 bb2' <- peval bb2 @@ -473,7 +473,7 @@ trPeval (Call x bb1 bb2) = do setChangeFlag return $ BB (insts1 ++ insts2) tr2 _ -> - return $ BB [] (Call x bb1' bb2') + return $ BB [] (StackExpand x bb1' bb2') trPeval tr@(Ret x) = do markUsed' x diff --git a/compiler/src/Raw.hs b/compiler/src/Raw.hs index a9a17046..2f7a5ff9 100644 --- a/compiler/src/Raw.hs +++ b/compiler/src/Raw.hs @@ -158,7 +158,7 @@ data RawTerminator | Error RawVar PosInf -- | Execute the first BB and then execute the second BB where -- PC is reset to the level before entering the first BB. - | Call RawBBTree RawBBTree + | StackExpand RawBBTree RawBBTree deriving (Eq, Show) @@ -341,7 +341,7 @@ ppIR (MkFunClosures varmap fdefs) = -- ppIR (LevelOperations _ insts) = -- text "level operation" $$ nest 2 (vcat (map ppIR insts)) -ppTr (Call bb1 bb2) = (text "call" $$ nest 4 (ppBB bb1)) $$ (ppBB bb2) +ppTr (StackExpand bb1 bb2) = (text "call" $$ nest 4 (ppBB bb1)) $$ (ppBB bb2) -- ppTr (AssertElseError va ir va2 _) diff --git a/compiler/src/Raw2Stack.hs b/compiler/src/Raw2Stack.hs index caf87c3b..b4e892a7 100644 --- a/compiler/src/Raw2Stack.hs +++ b/compiler/src/Raw2Stack.hs @@ -188,7 +188,7 @@ trTr (Raw.LibExport v) = do return $ Stack.LibExport v trTr (Raw.Error r1 p) = do return $ Stack.Error r1 p -trTr (Raw.Call bb1 bb2) = do +trTr (Raw.StackExpand bb1 bb2) = do __callDepth <- localCallDepth <$> ask bb1' <- local (\tenv -> tenv { localCallDepth = __callDepth + 1 } ) $ trBB bb1 n <- getBlockNumber @@ -205,7 +205,7 @@ trTr (Raw.Call bb1 bb2) = do | x <- filter filterConsts (Set.elems varsToLoad) ] bb2'@(Stack.BB inst_2 tr_2) <- trBB bb2 - return $ Stack.Call bb1' (Stack.BB (loads ++ inst_2) tr_2) + return $ Stack.StackExpand bb1' (Stack.BB (loads ++ inst_2) tr_2) trBB :: Raw.RawBBTree -> Tr Stack.StackBBTree diff --git a/compiler/src/RawDefUse.hs b/compiler/src/RawDefUse.hs index c6b7314f..e987b917 100644 --- a/compiler/src/RawDefUse.hs +++ b/compiler/src/RawDefUse.hs @@ -233,7 +233,7 @@ instance Trav RawTerminator where trav bb2 LibExport v -> use v Error r _ -> use r - Call bb1 bb2 -> do + StackExpand bb1 bb2 -> do trav bb1 modify (\s -> let (c, _) = locInfo s diff --git a/compiler/src/RawOpt.hs b/compiler/src/RawOpt.hs index 937dc8be..e7253b77 100644 --- a/compiler/src/RawOpt.hs +++ b/compiler/src/RawOpt.hs @@ -78,7 +78,7 @@ instance Substitutable RawTerminator where If r bb1 bb2 -> If (apply subst r) (apply subst bb1) (apply subst bb2) Error r p -> Error (apply subst r) p - Call bb1 bb2 -> Call (apply subst bb1) (apply subst bb2) + StackExpand bb1 bb2 -> StackExpand (apply subst bb1) (apply subst bb2) _ -> tr instance Substitutable RawBBTree where @@ -420,7 +420,7 @@ instance PEval RawTerminator where } bb2' <- peval bb2 return $ If x bb1' bb2' - Call bb1 bb2 -> do + StackExpand bb1 bb2 -> do s <- get bb1' <- peval bb1 put $ s { stateMon = Map.empty @@ -428,7 +428,7 @@ instance PEval RawTerminator where , stateJoins = stateJoins s } -- reset the monitor state bb2' <- peval bb2 - return $ Call bb1' bb2' + return $ StackExpand bb1' bb2' Ret -> do return tr' TailCall x -> do @@ -470,14 +470,15 @@ filterInstBwd ls = f (Nothing, Nothing) (reverse ls) [] --- | This optimization for 'Call' moves instructions from the continuation to before the 'Call'. --- This can result in a 'Call' which just contains a 'Ret', which is then optimized away. --- The optimization compensates for redundant assignments introduced by the translation. -hoistCalls :: RawBBTree -> RawBBTree -hoistCalls bb@(BB insts tr) = +-- | This optimization for 'StackExpand' moves instructions from the continuation to before the +-- 'StackExpand'. This can result in a 'StackExpand' which just contains a 'Ret', which is then +-- optimized away. The optimization compensates for redundant assignments introduced by the +-- translation. +hoistStackExpand :: RawBBTree -> RawBBTree +hoistStackExpand bb@(BB insts tr) = case tr of -- Here we check which instructions from ii_1 can be moved to before the call - Call (BB ii_1 tr_1) bb2 -> + StackExpand (BB ii_1 tr_1) bb2 -> let isFrameSpecific i = case i of SetBranchFlag -> True @@ -487,7 +488,7 @@ hoistCalls bb@(BB insts tr) = -- jx_1: non-frame-specific instructions, are moved to before the call -- jx_2: frame-specific instructions, stay under the call's instructions (jx_1, jx_2) = Data.List.break isFrameSpecific ii_1 - in BB (insts ++ jx_1) (Call (BB jx_2 tr_1) bb2) + in BB (insts ++ jx_1) (StackExpand (BB jx_2 tr_1) bb2) -- If returning, the current frame will be removed, and thus all PC set instructions -- are redundant and can be removed. Ret -> @@ -537,7 +538,7 @@ instance PEval RawBBTree where If x (BB (set_pc_bl ++ i_then) tr_then) (BB (set_pc_bl ++ i_else) tr_else) - _ -> hoistCalls $ BB (insts_no_ret ++ set_pc_bl) tr'' + _ -> hoistStackExpand $ BB (insts_no_ret ++ set_pc_bl) tr'' let insts_sorted = instOrder insts_ return $ BB insts_sorted bb_ diff --git a/compiler/src/Stack.hs b/compiler/src/Stack.hs index 6427a452..91f3e4f9 100644 --- a/compiler/src/Stack.hs +++ b/compiler/src/Stack.hs @@ -47,7 +47,7 @@ data StackTerminator | If RawVar StackBBTree StackBBTree | LibExport VarAccess | Error RawVar PosInf - | Call StackBBTree StackBBTree + | StackExpand StackBBTree StackBBTree deriving (Eq, Show) @@ -150,7 +150,7 @@ ppIR (MkFunClosures varmap fdefs) = ppIR (LabelGroup insts) = text "group" $$ nest 2 (vcat (map ppIR insts)) -ppTr (Call bb1 bb2) = (text "= call" $$ nest 2 (ppBB bb1)) $$ (ppBB bb2) +ppTr (StackExpand bb1 bb2) = (text "= call" $$ nest 2 (ppBB bb1)) $$ (ppBB bb2) -- ppTr (AssertElseError va ir va2 _) diff --git a/compiler/src/Stack2JS.hs b/compiler/src/Stack2JS.hs index 5717b99f..0a11bedd 100644 --- a/compiler/src/Stack2JS.hs +++ b/compiler/src/Stack2JS.hs @@ -452,7 +452,7 @@ ir2js InvalidateSparseBit = return $ {-- TERMINATORS --} -tr2js (Call bb bb2) = do +tr2js (StackExpand bb bb2) = do _frameSize <- gets frameSize _sparseSlot <- gets sparseSlot _consts <- gets consts diff --git a/compiler/test/ir2raw-test/testcases/TR.hs b/compiler/test/ir2raw-test/testcases/TR.hs index 4800b478..f330a8e0 100644 --- a/compiler/test/ir2raw-test/testcases/TR.hs +++ b/compiler/test/ir2raw-test/testcases/TR.hs @@ -30,8 +30,8 @@ tcs = map (second mkP) (BB [Assign (VN "b1") (Base "v1") ] (LibExport (mkV "b1"))) (BB [Assign (VN "b2") (Base "v2") ] (LibExport (mkV "b2"))) ), - ( "Call" - , Call (VN "x") + ( "StackExpand" + , StackExpand (VN "x") (BB [Assign (VN "b1") (Base "v1") ] (LibExport (mkV "b1"))) (BB [Assign (VN "b2") (Base "v2") ] (LibExport (mkV "b2"))) ), diff --git a/lib/Hash.trp b/lib/Hash.trp index 5a4b0d90..f10ec7b0 100644 --- a/lib/Hash.trp +++ b/lib/Hash.trp @@ -68,15 +68,13 @@ let (*--- Module ---*) val Hash = { - hashString = hashString, - hashMultiplyShift = hashMultiplyShift, - hashInt = hashInt, - hashNumber = hashNumber, - hashList = hashList, - hash = hash + hashString, + hashMultiplyShift, + hashInt, + hashNumber, + hashList, + hash } -in [ ("Hash", Hash) - , ("hash", hash) - ] +in [ ("Hash", Hash), ("hash", hash) ] end diff --git a/lib/HashMap.trp b/lib/HashMap.trp index 43358544..a8e25072 100644 --- a/lib/HashMap.trp +++ b/lib/HashMap.trp @@ -202,24 +202,20 @@ let (* NOTE: The map is implemented as a Hash Array Mapped Trie (HAMT), i.e. a p (*--- Module ---*) val HashMap = { - (* Construction *) - empty = empty, - singleton = singleton, - insert = insert, - remove = remove, - (* Queries *) - null = null, - size = size, - findOpt = findOpt, - find = find, - mem = mem, - (* Manipulation *) - fold = fold, - (* List Conversion*) - keys = keys, - values = values, - toList = toList, - fromList = fromList + empty, + singleton, + insert, + remove, + null, + size, + findOpt, + find, + mem, + fold, + keys, + values, + toList, + fromList } in [ ("HashMap", HashMap) ] diff --git a/lib/HashSet.trp b/lib/HashSet.trp index 0ffccbc5..ccad42d0 100644 --- a/lib/HashSet.trp +++ b/lib/HashSet.trp @@ -47,21 +47,17 @@ let (* NOTE: The set is implemented as a HashMap with dummy values, `()`. This i (*--- Module ---*) val HashSet = { - (* Construction *) - empty = empty, - singleton = singleton, - insert = insert, - remove = remove, - (* Queries *) - null = null, - size = size, - mem = mem, - (* Manipulation *) - fold = fold, - (* List Conversion*) - elems = elems, - toList = toList, - fromList = fromList + empty, + singleton, + insert, + remove, + null, + size, + mem, + fold, + elems, + toList, + fromList } in [ ("HashSet", HashSet) ] diff --git a/lib/List.trp b/lib/List.trp index 872936e9..775007e3 100644 --- a/lib/List.trp +++ b/lib/List.trp @@ -169,33 +169,26 @@ let (* -- List Access -- *) (*--- Module ---*) val List = { - head = head, - tail = tail, - nth = nth, - - null = null, - elem = elem, - length = length, - - reverse = reverse, - append = append, - revAppend = revAppend, - appendAt = appendAt, - sublist = sublist, - - map = map, - mapi = mapi, - foldl = foldl, - filter = filter, - filteri = filteri, - partition = partition, - - range = range, - - sort = sort + head, + tail, + nth, + null, + elem, + length, + reverse, + append, + revAppend, + appendAt, + sublist, + map, + mapi, + foldl, + filter, + filteri, + partition, + range, + sort } -in [ ("List", List), - ("length", length) - ] +in [ ("List", List), ("length", length) ] end diff --git a/lib/ListPair.trp b/lib/ListPair.trp index 20d03ca6..94b54eed 100644 --- a/lib/ListPair.trp +++ b/lib/ListPair.trp @@ -64,22 +64,19 @@ let (* -- ListPair Generation -- *) (*--- Module ---*) val ListPair = { - zip = zip, - unzip = unzip, - - null = null, - length = length, - - reverse = reverse, - append = append, - revAppend = revAppend, - - findOpt = findOpt, - find = find, - mem = mem, - - map = map, - foldl = foldl + zip, + unzip, + null, + length, + reverse, + append, + revAppend, + findOpt, + find, + mem, + map, + foldl } -in [ ("ListPair", ListPair) ] end +in [ ("ListPair", ListPair) ] +end diff --git a/lib/Number.trp b/lib/Number.trp index ad9b7527..a8867220 100644 --- a/lib/Number.trp +++ b/lib/Number.trp @@ -93,25 +93,26 @@ let (** Largest (safe) possible integral value. Anything larger than this cannot (*--- Module ---*) val Number = { - maxInt = maxInt, - minInt = minInt, - precision = precision, - maxInt32 = maxInt32, - minInt32 = minInt32, - maxNum = maxNum, - minNum = minNum, - abs = abs, - min = min, - max = max, - ceil = ceil, - floor = floor, - round = round, - sqrt = sqrt, - isInt = isInt, - toInt = toInt, - toInt32 = toInt32, - toString = toString, - fromString = fromString + maxInt, + minInt, + precision, + maxInt32, + minInt32, + maxNum, + minNum, + abs, + min, + max, + ceil, + floor, + round, + sqrt, + isInt, + toInt, + toInt32, + toString, + fromString } + in [("Number", Number)] end diff --git a/lib/README.md b/lib/README.md index ea43f188..44119947 100644 --- a/lib/README.md +++ b/lib/README.md @@ -21,13 +21,19 @@ reviewed rigorously rather than depend on the monitor. To compile a module as part of the standard library, add it to the list of files in the `lib` target of the *makefile*. +## Design Principles + +- File names are written in `CamelCase`. This makes them conform to the Standard ML Basis Library. +- It is more important to match the function names and signatures in the Standard ML library than to + improve on them. For example, `String.sub` would make more sense with the type `[Char] -> Int -> + Char` but to match the SML library, we will stick with `[Char] * Int -> Char`. +- Each module exports a single *record* with the same name as the file. This (1) makes it closer to + the SML module system and (2) allows for name resolution, e.g. `HashMap.findOpt` and + `ListPair.findOpt` can be used in the same file. +- Each function that is exported has to be documented (`(** *)`). In the long run, we will + auto-generate documentation for the Standard Library. + ## TODO -- To conform with the Standard ML Basis Library, we should have the files conform to a `CamelCase` - style. -- To fake namespaced import, e.g. `List.length`, the library should export a struct instead. Only - certain functions should "pollute" the global namespace. -- Quite a lot of the standard library is not documented in any way. What is the purpose of each - function and each module? The [modules](#modules) above are the ones that have been updated and - documented. -- There are a lot of things in here - some of it dead. Can we merge/remove some things? +The [modules](#modules) mentioned above already follow the [design principles](#design-principles). +The remaining files either need to be updated or to be removed. diff --git a/lib/StencilVector.trp b/lib/StencilVector.trp index a272bc91..f73701cc 100644 --- a/lib/StencilVector.trp +++ b/lib/StencilVector.trp @@ -146,26 +146,24 @@ let (*--- Constants ---*) (* TODO: Lift list functions `mapi`, `find` and `filter`? *) + (*--- Module ---*) val StencilVector = { - (* Constants *) - maskBits = maskBits, - maskMax = maskMax, - (* Functions *) - empty = empty, - singleton = singleton, - get = get, - getOrDefault = getOrDefault, - set = set, - unset = unset, - mem = mem, - valid = valid, - null = null, - mask = mask, - length = length, - map = map, - fold = fold + maskBits, + maskMax, + empty, + singleton, + get, + getOrDefault, + set, + unset, + mem, + valid, + null, + mask, + length, + map, + fold } -in (* Export public functions *) - [ ("StencilVector", StencilVector) - ] + +in [ ("StencilVector", StencilVector) ] end diff --git a/lib/String.trp b/lib/String.trp index b275f776..2dfe068e 100644 --- a/lib/String.trp +++ b/lib/String.trp @@ -70,17 +70,18 @@ let (** The maximum length of a string. (*--- Module ---*) val String = { - maxSize = maxSize, - size = size, - sub = sub, - subCode = subCode, - substring = substring, - concat = concat, - concatWith = concatWith, - implode = implode, - explode = explode, - map = map, - translate = translate + maxSize, + size, + sub, + subCode, + substring, + concat, + concatWith, + implode, + explode, + map, + translate } + in [("String", String)] end diff --git a/lib/Unit.trp b/lib/Unit.trp index 483d32ac..f4b49eba 100644 --- a/lib/Unit.trp +++ b/lib/Unit.trp @@ -112,13 +112,13 @@ let (*--- Module ---*) val Unit = { - group = group, - it = it, - isEq = isEq, - isTrue = isTrue, - isFalse = isFalse, - isNeq = isNeq, - run = run + group, + it, + isEq, + isTrue, + isFalse, + isNeq, + run } in [ ("Unit", Unit) ] diff --git a/rt/src/Asserts.mts b/rt/src/Asserts.mts index 88ed0081..a72df5f3 100644 --- a/rt/src/Asserts.mts +++ b/rt/src/Asserts.mts @@ -16,7 +16,7 @@ import { TroupeAggregateRawValue, TroupeRawValue } from './TroupeRawValue.mjs'; // import { LVal } from './Lval'; function _thread() { - return getRuntimeObject().__sched.__currentThread + return getRuntimeObject().__sched.getCurrentThread() } function __stringRep (v) { diff --git a/rt/src/MailboxProcessor.mts b/rt/src/MailboxProcessor.mts index 8c7bd239..65a7f152 100644 --- a/rt/src/MailboxProcessor.mts +++ b/rt/src/MailboxProcessor.mts @@ -105,7 +105,7 @@ export class MailboxProcessor implements MailboxInterface { peek(lev: Level, index: number, lowb: Level, highb: Level) { - let theThread = this.sched.__currentThread + let theThread = this.sched.getCurrentThread() let mb = theThread.mailbox; debug (`peek index: ${index}`) debug (`peek interval: [${lowb.stringRep()}, ${highb.stringRep()}]`) @@ -138,7 +138,7 @@ export class MailboxProcessor implements MailboxInterface { } consume(lev: Level, index: number, lowb: Level, highb: Level) { - let theThread = this.sched.__currentThread + let theThread = this.sched.getCurrentThread() let mb = theThread.mailbox; debug (`consume index: ${index}`) debug (`consume interval: [${lowb.stringRep()} to ${highb.stringRep()}]`) diff --git a/rt/src/RuntimeInterface.mts b/rt/src/RuntimeInterface.mts index f212b8a2..c7b559c2 100644 --- a/rt/src/RuntimeInterface.mts +++ b/rt/src/RuntimeInterface.mts @@ -11,9 +11,10 @@ export interface RuntimeInterface { $t: Thread; $service: any; // todo 2021-06-13; identify what the right interface here should be debug(arg0: string); + __userRuntime: any __sched: SchedulerInterface __mbox : MailboxInterface - sendMessageNoChecks(toPid: any, message: import("./Lval.mjs").LVal, arg2?: boolean): any; + sendMessageNoChecks(toPid: any, message: LVal, arg2?: boolean): any; ret(arg0: any); // ret_raw () // tailcall(funclos: any, __unit: any); diff --git a/rt/src/Scheduler.mts b/rt/src/Scheduler.mts index e580e714..8e089f96 100644 --- a/rt/src/Scheduler.mts +++ b/rt/src/Scheduler.mts @@ -1,321 +1,295 @@ 'use strict'; + import { v4 as uuidv4} from 'uuid' import { Thread } from './Thread.mjs'; import runId from './runId.mjs'; -import { __unit } from './UnitVal.mjs'; import { mkTuple } from './ValuesUtil.mjs'; import { SchedulerInterface } from './SchedulerInterface.mjs'; import { RuntimeInterface } from './RuntimeInterface.mjs'; import { LVal } from './Lval.mjs' +import { Level } from "./Level.mjs"; import {ProcessID, pid_equals} from './process.mjs' import SandboxStatus from './SandboxStatus.mjs' import {ThreadError, TroupeError} from './TroupeError.mjs' import {lub} from './Level.mjs' -import { getCliArgs, TroupeCliArg } from './TroupeCliArgs.mjs'; import {SYSTEM_PROCESS_STRING} from './Constants.mjs' -const argv = getCliArgs(); -const showStack = argv[TroupeCliArg.ShowStack] -import { mkLogger } from './logger.mjs' -const logger = mkLogger('scheduler'); -const info = x => logger.info(x) -const debug = x => logger.debug(x) +import { getCliArgs, TroupeCliArg } from './TroupeCliArgs.mjs'; +const argv = getCliArgs(); -const STACKDEPTH = 150; +/** Enum for termination statuses. */ +export enum ThreadType { + /** System service thread. */ + System = -1, + /** Main thread. */ + Main = 0, + /** Other threads, spawned from 'Main' or 'System'. */ + Other = 1 +} -let TerminationStatus = { - OK: 0, - ERR: 1 +/** Enum for termination statuses. */ +enum TerminationStatus { + /** Thread finished its computation. */ + OK = 0, + /** Thread stopped early due to an error. */ + ERR = 1 } export class Scheduler implements SchedulerInterface { - rt_uuid: any; - __funloop: Thread[]; - __blocked: any[]; - __alive: {}; - __currentThread: Thread; - stackcounter: number; - __unit: any; - rtObj : RuntimeInterface - __node: any; - __stopWhenAllThreadsAreDone: boolean; - __stopRuntime: () => void; - constructor(rtObj:RuntimeInterface) { - this.rt_uuid = runId; - this.rtObj = rtObj - this.__funloop = new Array() - this.__blocked = new Array() - this.__alive = {} // new Set(); - - this.__currentThread = null; // current thread object - - this.stackcounter = 0; - - // the unit value - this.__unit = __unit - } + // Current thread state + /** Current thread alive */ + __currentThread: Thread; - resetScheduler() { - // console.log (`The current length of __funloop is ${this.__funloop.length}`) - // console.log (`The number of active threads is ${Object.keys(this.__alive).length}`) - for (let x in this.__alive) { - if (this.currentThreadId.val.toString() == x) { - // console.log (x, "ACTIVE") - } else { - // console.log (x, "KILLING"); - delete this.__alive[x] - } - } - this.__blocked = [] - this.__funloop = [] - // console.log (`The number of active threads is ${Object.keys(this.__alive).length}`) - // console.log (`The number of blocked threads is ${this.__blocked.length}`) - } + /** FIFO queue of all threads to evaluate */ + __funloop: Thread[]; - done () { - this.notifyMonitors(); - // console.log (this.__currentThread.processDebuggingName, this.currentThreadId.val.toString(), "done") - delete this.__alive [this.currentThreadId.val.toString()]; - } + /** Queue of blocked threads. */ + __blocked: { [tid in string]: Thread }; + /** Map of alive threads from their stringified identifier, `tid`. */ + __alive: { [tid in string]: Thread }; - halt (persist=null) { - this.raiseCurrentThreadPCToBlockingLev(); - let retVal = new LVal (this.__currentThread.r0_val, - lub(this.__currentThread.bl, this.__currentThread.r0_lev), - lub(this.__currentThread.bl, this.__currentThread.r0_tlev)) + // Dependencies for unique thread identifier creation. + rt_uuid: any; + __node: any; - this.notifyMonitors (); + // Runtime dependencies + rtObj : RuntimeInterface + __stopWhenAllThreadsAreDone: boolean; + __stopRuntime: () => void; - delete this.__alive[this.currentThreadId.val.toString()]; - console.log(">>> Main thread finished with value:", retVal.stringRep()); - if (persist) { - this.rtObj.persist (retVal, persist ) - console.log ("Saved the result value in file", persist) - } - return null; - } - - notifyMonitors (status = TerminationStatus.OK, errstr = null) { - let mkVal = this.__currentThread.mkVal - let ids = Object.keys (this.__currentThread.monitors); - for ( let i = 0; i < ids.length; i ++ ) { - let id = ids[i]; - let toPid = this.__currentThread.monitors[id].pid; - let refUUID = this.__currentThread.monitors[id].uuid; - let thisPid = this.__currentThread.tid; - let statusVal = this.__currentThread.mkVal ( status ) ; - let reason = TerminationStatus.OK == status ? statusVal : - mkTuple ( [statusVal, mkVal (errstr)] ); - let message = mkVal (mkTuple ([ mkVal("DONE"), refUUID, thisPid, reason])) - this.rtObj.sendMessageNoChecks ( toPid, message , false) // false flag means no need to return in the process - } - } + /*************************************************************************************************\ + Scheduler state + \*************************************************************************************************/ - raiseCurrentThreadPC (l) { - this.__currentThread.raiseCurrentThreadPC(l); - } - - raiseCurrentThreadPCToBlockingLev () { - this.__currentThread.raiseCurrentThreadPCToBlockingLev() + /** */ + constructor(rtObj: RuntimeInterface) { + this.rt_uuid = runId; + this.rtObj = rtObj; + this.__funloop = []; + this.__blocked = {}; + this.__alive = {}; + this.__currentThread = null; } - - raiseBlockingThreadLev (l) { - this.__currentThread.raiseBlockingThreadLev(l); + /** Initialisation of the scheduler based on the p2p layer, e.g. the `node` identifier and + * the scheduler should proceed despite all threads being done. */ + initScheduler(node, stopWhenAllThreadsAreDone, stopRuntime) { + this.__node = node; + this.__stopWhenAllThreadsAreDone = stopWhenAllThreadsAreDone; + this.__stopRuntime = stopRuntime; } - - pinipush (l, cap) { - this.__currentThread.pcpinipush(l, cap) + /** Kill all threads except the current one, staying ready for spawning new threads. + * + * @note This does not notify the monitors. */ + resetScheduler() { + for (let x in this.__alive) { + if (this.__currentThread.tid.val.toString() !== x) { + delete this.__alive[x]; + } + } + this.__blocked = {}; + this.__funloop = []; } - pinipop (cap) { - return this.__currentThread.pinipop(cap); - } + /*************************************************************************************************\ + Thread creation + \*************************************************************************************************/ - mkVal(x) { - return this.__currentThread.mkVal (x); - } - - mkValPos (x,p) { - return this.__currentThread.mkValPos (x,p); - } - - mkCopy (x) { - return this.__currentThread.mkCopy (x); + /** Add a thread `t` to the active function loop. */ + scheduleThread(t: Thread) { + this.__funloop.push(t); } + /** Create a new thread `t` for the given function to be evaluated and schedule it. + * + * NOTE (20-10-2025; SS): A hypothesis about the Javascript event loop: + * + * It would be a more clean design to return the thread identifier of type `LVal`, as we + * do right now, together with a `Promise` of the final returned value. But, since + * the Javascript event loop is a LIFO queue, i.e. a stack, this would bury resolving the + * termination of each thread (especially the *main* thread) beneath everything else. + */ + scheduleNewThread(f: () => any, + arg: any, + pc: Level, + block: Level, + tType: ThreadType = ThreadType.Other, + cb: (LVal) => void = (_) => {}) + { + // Create a new process ID at the given level. + const pid = tType === ThreadType.System ? SYSTEM_PROCESS_STRING : uuidv4(); + const tid = new LVal(new ProcessID(this.rt_uuid, pid, this.__node), pc); + + const halt = () => { + this.__currentThread.raiseCurrentThreadPCToBlockingLev(); + this.notifyMonitors(); + + const currT = this.__currentThread; + const retVal = new LVal (currT.r0_val, lub(currT.bl, currT.r0_lev), lub(currT.bl, currT.r0_tlev)); + + delete this.__alive[this.__currentThread.tid.val.toString()]; + + cb(retVal); + } - initScheduler(node, stopWhenAllThreadsAreDone = false, stopRuntime = () => {}) { - this.__node = node; - this.__stopWhenAllThreadsAreDone = stopWhenAllThreadsAreDone; - this.__stopRuntime = stopRuntime - } + // New thread + const sStatus = new SandboxStatus.NORMAL(); + const t = new Thread(tid, halt, f, arg, pc, block, sStatus, this.rtObj, this); + this.__alive[tid.val.toString()] = t; + this.scheduleThread(t); - - get currentThreadId() { - return this.__currentThread.tid; + return tid as LVal; } - set handlerState (st) { - this.__currentThread.handlerState = st; - } + /*************************************************************************************************\ + Thread access + \*************************************************************************************************/ - get handlerState () { - return this.__currentThread.handlerState; + /** Whether the thread with identifier, `tid`, is alive. */ + isAlive(tid: LVal) { + return (this.__alive[tid.val.toString()] != null); } - resumeLoopAsync() { - setImmediate(() => {this.loop()}); + /** The thread object with the given identifier, `tid`. */ + getThread (tid: LVal) { + return this.__alive[tid.val.toString()]; } - - - scheduleThread(t) { - this.__funloop.push(t) + /** The currently scheduled thread */ + getCurrentThread() { + return this.__currentThread; } - - createNewProcessIDAtLevel(pcArg, isSystem = false) { - let pid = isSystem ? SYSTEM_PROCESS_STRING : uuidv4(); - let pidObj = new ProcessID(this.rt_uuid, pid, this.__node); - return new LVal(pidObj, pcArg); + /** Overwrites the current thread; the previously current thread is returned. */ + setCurrentThread(t: Thread) { + const prev = this.__currentThread + this.__currentThread = t; + return prev; } + /*************************************************************************************************\ + Thread blocking/unblocking + \*************************************************************************************************/ - - scheduleNewThreadAtLevel (thefun, arg, levpc, levblock, ismain = false, persist=null, isSystem = false) { - let newPid = this.createNewProcessIDAtLevel(levpc, isSystem); - - let halt = ismain ? ()=> { this.halt (persist) } : - () => { this.done () }; - - - let t = new Thread - ( newPid - , halt - , thefun - , arg - , levpc - , levblock - , new SandboxStatus.NORMAL() - , this.rtObj - , this ); - - - this.__alive[newPid.val.toString()] = t; - this.scheduleThread (t) - return newPid; - } - - schedule(thefun, args, nm) { - this.__currentThread.runNext (thefun, args, nm); - this.scheduleThread(this.__currentThread) + /** Block thread object `t`. */ + blockThread(t: Thread) { + this.__blocked[t.tid.val.toString()] = t; } + /** Unblock the thread with the given identifier, `pid`. */ + unblockThread(tid: LVal) { + if (!this.__blocked[tid.val.toString()]) { return; } - blockThread(t) { - this.__blocked.push(t) + this.scheduleThread(this.__blocked[tid.val.toString()]); + delete this.__blocked[tid.val.toString()]; } + /*************************************************************************************************\ + Thread Termination + \*************************************************************************************************/ - unblockThread(pid) { - for (let i = 0; i < this.__blocked.length; i++) { - if (pid_equals(this.__blocked[i].tid, pid)) { - this.scheduleThread(this.__blocked[i]); - this.__blocked.splice(i, 1); - break; - } + /** Notify monitors about thread termination. */ + notifyMonitors (status = TerminationStatus.OK, errstr = null) { + let mkVal = this.__currentThread.mkVal; + let ids = Object.keys(this.__currentThread.monitors); + for (let i = 0; i < ids.length; i++) { + let id = ids[i]; + let toPid = this.__currentThread.monitors[id].pid; + let refUUID = this.__currentThread.monitors[id].uuid; + let thisPid = this.__currentThread.tid; + let statusVal = this.__currentThread.mkVal( status ); + let reason = TerminationStatus.OK == status + ? statusVal + : mkTuple ([statusVal, mkVal (errstr)]); + let message = mkVal (mkTuple([ mkVal("DONE"), refUUID, thisPid, reason])); + // false flag means no need to return in the process + this.rtObj.sendMessageNoChecks( toPid, message, false); } } - - isAlive(tid) { - return (this.__alive[tid.val.toString()] != null); - } - - getThread (tid) { - return this.__alive[tid.val.toString()]; - } - - - stopThreadWithErrorMessage (t:Thread, s:string ) { - this.notifyMonitors(TerminationStatus.ERR, s) ; + /** Kill thread `t` with the error message `s` sent to its monitors. */ + stopThreadWithErrorMessage (t: Thread, errMsg: string) { + this.notifyMonitors(TerminationStatus.ERR, errMsg); delete this.__alive [t.tid.val.toString()]; } - /*****************************************************************************\ - - 2018-02-18: AA: a hypothesis about memory management in V8 - - It appears that V8's memory management is not very well suited for infinitely - running functions. In other words, functions are expected to eventually - terminate, and all long-running computations are expected to run through the - event loop. This is not surprising given the application where V8 is used. - This is why we periodically yield to the event loop; this hack appears to let - GC claim the objects allocated throughout the runtime of this function. Note - that without this hack, we are observing memory leaks for many "server"-like - programs; with the hack, we get a waivy memory consumption profile that reaches - around 50M on the low points of the wave. + /*************************************************************************************************\ + Scheduler loop + \*************************************************************************************************/ + + /** Start the main scheduler loop. + * + * HACK (2018-02-18: AA): a hypothesis about memory management in V8: + * + * It appears that V8's memory management is not very well suited for infinitely running + * functions. In other words, functions are expected to eventually terminate, and all + * long-running computations are expected to run through the event loop. This is not + * surprising given the application where V8 is used. This is why we periodically yield to + * the event loop; this hack appears to let GC claim the objects allocated throughout the + * runtime of this function. Note that without this hack, we are observing memory leaks for + * many "server"-like programs; with the hack, we get a waivy memory consumption profile + * that reaches around 50M on the low points of the wave. + */ + loop() { + const maxThreadsPerLoop = 500000; + const maxKontsPerThread = 1000; - \*****************************************************************************/ + let dest: () => any; + try { + for (let i = 0; i < maxThreadsPerLoop && this.__funloop.length > 0; ++i) { + // Pop front of function queue and set it to be the next thread. + this.__currentThread = this.__funloop.shift(); + if (!this.__alive[this.__currentThread.tid.val.toString()]) { continue; } + dest = this.__currentThread.next; - loop() { - const $$LOOPBOUND = 500000; - let _FUNLOOP = this.__funloop - let _curThread: Thread; - let dest; - try { - for (let $$loopiter = 0; $$loopiter < $$LOOPBOUND && _FUNLOOP.length > 0; $$loopiter ++ ) { - _curThread = _FUNLOOP.shift(); - this.__currentThread = _curThread; - dest = _curThread.next - let ttl = 1000; // magic constant; 2021-04-29 - while (dest && ttl -- ) { - // if (showStack) { // 2021-04-24; AA; TODO: profile the addition of this conditional in this tight loop - // this.__currentThread.showStack() - // } - // console.log (">>>>>>>>>>") - // console.log (dest.toString()) - // console.log ("<<<<<<<<<<") - // if (dest.debugname ) { - // console.log (" -- ", dest.debugname) - // } - dest = dest () + // Run thread for `maxKontsPerThread` continuations. + for (let j = 0; dest && j < maxKontsPerThread; ++j) { + dest = dest(); } + // If not done, push it back into the queue. if (dest) { - _curThread.handlerState.checkGuard() - - _curThread.next = dest ; - _FUNLOOP.push (_curThread); + this.__currentThread.handlerState.checkGuard(); + this.__currentThread.next = dest; + this.__funloop.push(this.__currentThread); } - } + } } catch (e) { if (e instanceof TroupeError) { e.handleError(this); } else { - console.log ("--- Schedule module caught an internal exception ---") - console.log ("--- The following output may help identify a bug in the runtime ---") - console.log ("Destination function\n" , dest) - this.__currentThread.showStack() + console.log("--- Schedule module caught an internal exception ---"); + console.log("--- The following output may help identify a bug in the runtime ---"); + console.log("Destination function\n", dest); + + if (argv[TroupeCliArg.ShowStack]) { + this.__currentThread.showStack(); + } throw e; } } - if (_FUNLOOP.length > 0) { - // we are not really done, but are just hacking around the V8's memory management + // If more work is to be done, then resume `loop` after the Javascript runtime has been able + // to run other tasks, e.g. garbage collection. + if (this.__funloop.length > 0) { this.resumeLoopAsync(); } - - if (this.__stopWhenAllThreadsAreDone && Object.keys(this.__alive).length == 0 ) { + + // If everything is done, and the node should not persist, then terminate. + if (this.__stopWhenAllThreadsAreDone && Object.keys(this.__alive).length == 0) { this.__stopRuntime(); } } - -} \ No newline at end of file + + /** Add continuation of the main Troupe execution loop to the Javascript queue. In the meantime + * other code, e.g. the p2p and deserialization layers can run. */ + resumeLoopAsync() { + setImmediate(() => { this.loop(); }); + } +} diff --git a/rt/src/SchedulerInterface.mts b/rt/src/SchedulerInterface.mts index 742f3b81..b1f7475f 100644 --- a/rt/src/SchedulerInterface.mts +++ b/rt/src/SchedulerInterface.mts @@ -1,20 +1,22 @@ import { Thread } from "./Thread.mjs"; +import { LVal } from './Lval.mjs' +import { Level } from "./Level.mjs"; export interface SchedulerInterface { - // tailToTroupeFun(f: any, arg:any) - // tailToTroupeFun_raw(f: any) - // stepThread(); - resetScheduler(); - __alive: any; - scheduleNewThreadAtLevel(fun: any, arg: any, pc: any, blockingTopLev: any); - scheduleThread(theThread: any); - resumeLoopAsync(); - blockThread(__currentThread: Thread); - isAlive(toPid: any); - getThread(toPid: any); - unblockThread(toPid: any); - schedule(fun: any, args: any[], namespace: any); - __currentThread: Thread; - stopThreadWithErrorMessage (t:Thread, s:string) - -} \ No newline at end of file + resetScheduler(): void; + + scheduleNewThread(fun: () => any, arg: any, pc: Level, block: Level): LVal; + scheduleThread(t: Thread): void; + + blockThread(t: Thread): void; + unblockThread(tid: LVal): void; + + isAlive(tid: LVal): boolean; + getThread(tid: LVal): Thread; + getCurrentThread(): Thread; + setCurrentThread(t: Thread): Thread; + + stopThreadWithErrorMessage (t: Thread, errMsg: string): void + + resumeLoopAsync(): void; +} diff --git a/rt/src/builtins/UserRuntimeZero.mts b/rt/src/builtins/UserRuntimeZero.mts index 1f34839c..001888bf 100644 --- a/rt/src/builtins/UserRuntimeZero.mts +++ b/rt/src/builtins/UserRuntimeZero.mts @@ -259,13 +259,12 @@ export class UserRuntimeZero { libLoadingPseudoThread = new Thread(null, null, null, __unit, levels.BOT, levels.BOT, null, this, null); - savedThread = null ;// this.runtime.__sched.__currentThread; + savedThread = null ;// this.runtime.__sched.getCurrentThread(); setLibloadMode() { this.mkVal = (x) => new LVal(x, levels.BOT); this.mkValPos = (x, pos) => new LVal(x, levels.BOT, levels.BOT, pos); this.Env = LibEnv; - this.savedThread = this.runtime.__sched.__currentThread; - this.runtime.__sched.__currentThread = this.libLoadingPseudoThread; + this.savedThread = this.runtime.__sched.setCurrentThread(this.libLoadingPseudoThread); } @@ -273,7 +272,7 @@ export class UserRuntimeZero { this.mkVal = this.default_mkVal; this.mkValPos = this.default_mkValPos this.Env = RtEnv; - this.runtime.__sched.__currentThread = this.savedThread; + this.runtime.__sched.setCurrentThread(this.savedThread); } // tailcall(lff, arg) { diff --git a/rt/src/builtins/monitor.mts b/rt/src/builtins/monitor.mts index 2daf5d4a..9a0ff26d 100644 --- a/rt/src/builtins/monitor.mts +++ b/rt/src/builtins/monitor.mts @@ -14,12 +14,12 @@ export function BuiltinMonitors > (Ba // 1. find the thread corresponding to that tid - let t = this.runtime.__sched.__alive[tid.toString()]; + let t = this.runtime.__sched.getThread(tid); // 2. update the monitor state of that thread let r = this.runtime.rt_mkuuid(); if (t) { - t.addMonitor(this.runtime.__sched.__currentThread.tid, r); + t.addMonitor(this.runtime.__sched.getCurrentThread().tid, r); } return this.runtime.ret(r); diff --git a/rt/src/builtins/receive.mts b/rt/src/builtins/receive.mts index 0bbbc459..39796d89 100644 --- a/rt/src/builtins/receive.mts +++ b/rt/src/builtins/receive.mts @@ -133,7 +133,7 @@ export function BuiltinReceive>(Base: _blockThread = mkBase ((arg) => { assertIsUnit(arg) - this.runtime.__sched.blockThread(this.runtime.__sched.__currentThread); + this.runtime.__sched.blockThread(this.runtime.__sched.getCurrentThread()); return null; }) diff --git a/rt/src/builtins/self.mts b/rt/src/builtins/self.mts index 6b098337..5ff6d8b9 100644 --- a/rt/src/builtins/self.mts +++ b/rt/src/builtins/self.mts @@ -8,7 +8,7 @@ import { UserRuntimeZero, Constructor, mkBase } from './UserRuntimeZero.mjs' export function BuiltinSelf>(Base: TBase) { return class extends Base { self = mkBase((arg) => { - return this.runtime.ret(this.runtime.__sched.__currentThread.tid); + return this.runtime.ret(this.runtime.__sched.getCurrentThread().tid); }, "self"); } } \ No newline at end of file diff --git a/rt/src/builtins/spawn.mts b/rt/src/builtins/spawn.mts index 930b5244..b5750015 100644 --- a/rt/src/builtins/spawn.mts +++ b/rt/src/builtins/spawn.mts @@ -21,22 +21,13 @@ export function BuiltinSpawn>(Base: T // console.log ("SPAWN ARGS", larg) this.runtime.$t.raiseCurrentThreadPC(larg.lev); let arg = larg.val; - let __sched = this.runtime.__sched - - let spawnLocal = (arg) => { - // debug ("scheduled rt_spawn ", arg.fun); - - let newPid = __sched.scheduleNewThreadAtLevel( - arg, - __unit, // [arg.env, __unit], - // arg.namespace, - this.runtime.$t.pc, - this.runtime.$t.bl) - return this.runtime.$t.returnImmediateLValue(newPid) ; + const spawnLocal = (func) => { + const tid = this.runtime.__sched.scheduleNewThread( + func, __unit, this.runtime.$t.pc, this.runtime.$t.bl); + return this.runtime.$t.returnImmediateLValue(tid); } - if (Array.isArray(arg)) { if (__nodeManager.isLocalNode(arg[0].val)) { // check if we are at the same node or note // debug ("SAME NODE") @@ -55,4 +46,4 @@ export function BuiltinSpawn>(Base: T } }, "spawn"); } -} \ No newline at end of file +} diff --git a/rt/src/deserialize.mts b/rt/src/deserialize.mts index 2c194875..8ea8a6e2 100644 --- a/rt/src/deserialize.mts +++ b/rt/src/deserialize.mts @@ -15,28 +15,37 @@ import { Record } from './Record.mjs'; import { RawClosure } from './RawClosure.mjs'; import * as levels from './Level.mjs'; +// OBS: The variables below are all global! This is because the callback and deserializedJson +// changes all the time while the compiler process has been started. + +/** We spawn an instance of the Troupe compiler in its interactive IR mode. Through this, we + * pass the IR provided by other nodes. + * + * Since there is only one compiler process which is accessed via the lock below, we can guarantee + * a FIFO ordering on the compilation input/output pairs. + */ let __compilerOsProcess = null; -let __rtObj = null; - -// obs: these are global... -let __isCurrentlyUsingCompiler = false; // simple flag to make sure we handle one deserialization at a time -let __currentCallback = null; // a callback for synchronizing with the caller -let __currentDeserializedJson = null; -let __trustLevel = null; +/** Simple flag to make sure we handle one deserialization at a time. */ +let __isCurrentlyUsingCompiler = false; +/** The runtime object to which we should be deserializing. */ +let __rtObj = null; export function setRuntimeObj(rt: RuntimeInterface) { - __rtObj = rt; + __rtObj = rt; } -const HEADER:string = - "this.libSet = new Set () \n\ - this.libs = [] \n\ - this.addLib = function (lib, decl)\ - { if (!this.libSet.has (lib +'.'+decl)) { \ - this.libSet.add (lib +'.'+decl);\ - this.libs.push ({lib:lib, decl:decl})} }\n" +/** A callback for synchronizing with the caller. */ +let __currentCallback = null; + +/** The JSON with the context for deserialization. */ +let __currentDeserializedJson = null; + +/** The trust level of the sender, i.e. implicit declassification based on the (lack of) trust. */ +let __trustLevel = null; + +const MARKER = "/*-----*/"; function startCompiler() { __compilerOsProcess = spawn(process.env.TROUPE + '/bin/troupec', ['--json']); @@ -44,12 +53,14 @@ function startCompiler() { process.exit(code); }); - let marker = "/*-----*/\n\n" + let marker = MARKER + "\n\n"; // accumulator of communication with the compiler; reset after // each deserialization; needed because we have no guarantees about // how the data coming back from the compiler is chunked - + // + // TODO: Switch to an array of strings which are `join`ed at the end. + // This is ~4-10x faster. let accum = ""; __compilerOsProcess.stdout.on('data', (data: string) => { @@ -61,141 +72,121 @@ function startCompiler() { } }); } - startCompiler(); export function stopCompiler() { __compilerOsProcess.stdin.end(); } - -// -------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // some rudimentary debugging mechanisms; probably should be rewritten -function debuglog(...s) { - let spaces = ""; - for (let j = 0; j < indentcounter; j++) { - spaces = " " + spaces; - } - - s.unshift("DEBUG:" + spaces) - console.log.apply(null, s) -} - -var indentcounter = 0; +var indentCounter = 0; function indent() { - indentcounter++; + indentCounter++; } function unindent() { - indentcounter--; + indentCounter--; } +function debuglog(...s) { + let spaces = ""; + for (let j = 0; j < indentCounter; j++) { + spaces = " " + spaces; + } - -function deserializationError() { - console.log("DESERIALIZATION ERROR HANDLING IS NOT IMPLEMENTED") - process.exit(1); + s.unshift("DEBUG:" + spaces); + console.log.apply(null, s); } -function constructCurrent(compilerOutput: string) { - // debuglog (deserializationObject) +// ------------------------------------------------------------------------------------------------- +const HEADER : string = ` +this.libSet = new Set () +this.libs = [] +this.addLib = function (lib, decl) { + if (!this.libSet.has (lib +'.'+decl)) { + this.libSet.add (lib +'.'+decl); + this.libs.push ({lib:lib, decl:decl}); + } +} +`; + +function constructCurrent(compilerOutput: string | null) { __isCurrentlyUsingCompiler = false; - let serobj = __currentDeserializedJson; - let desercb = __currentCallback; + const serobj = __currentDeserializedJson; + const desercb = __currentCallback; // 1. reconstruct the namespaces - let snippets = compilerOutput.split("\n\n"); - let k = 0; - - - let ctxt = { // deserialization context + let ctxt = { // deserialization context namespaces : new Array (serobj.namespaces.length), closures : new Array (serobj.closures.length), - envs : new Array (serobj.envs.length) + envs : new Array (serobj.envs.length), } + const snippets = compilerOutput ? compilerOutput.split("\n\n") : []; + let k = 0; for (let i = 0; i < serobj.namespaces.length; i++) { - let ns = serobj.namespaces[i] - let nsFun = HEADER + let ns = serobj.namespaces[i] + let nsFun = HEADER; let atomSet = new Set() - // nsFun += "this.libSet = new Set () \n" - // nsFun += "this.libs = [] \n" - // nsFun += "this.addLib = function (lib, decl) " + - // " { if (!this.libSet.has (lib +'.'+decl)) { " + - // " this.libSet.add (lib +'.'+decl); " + - // " this.libs.push ({lib:lib, decl:decl})} } \n" - // nsFun += "this.loadlibs = function (cb) { rt.linkLibs (this.libs, this, cb) } \n" - - for (let j = 0; j < ns.length; j++) { if (j > 0) { nsFun += "\n\n" // looks neater this way } let snippetJson = JSON.parse(snippets[k++]); - // console.log (snippetJson.libs); - // console.log (snippetJson.fname); nsFun += snippetJson.code; for (let atom of snippetJson.atoms) { - atomSet.add(atom) + atomSet.add(atom); } - // console.log (snippetJson.atoms) } - let argNames = Array.from(atomSet); - let argValues = argNames.map( argName => {return new Atom(argName)}) - argNames.unshift('rt') - argNames.push(nsFun) - // Observe that there is some serious level of - // reflection going on in here - // Arguments to Function are - // 'rt', ATOM1, ..., ATOMk, nsFun - // - // - let NS: any = Reflect.construct (Function, argNames) + let argNames = Array.from(atomSet); + let argValues = argNames.map(argName => {return new Atom(argName)}) + argNames.unshift('rt'); + argNames.push(nsFun); + // Observe that there is some serious level of reflection going on in here. + // The arguments to `Function` are: 'rt', ATOM1, ..., ATOMk, nsFun + const NS: any = Reflect.construct (Function, argNames); // We now construct an instance of the newly constructed object // that takes the runtime object + atoms as its arguments - - // console.log (NS.toString()); // debugging - argValues.unshift(__rtObj) - ctxt.namespaces[i] = Reflect.construct (NS, argValues) - + argValues.unshift(__rtObj); + ctxt.namespaces[i] = Reflect.construct (NS, argValues); } // 2. reconstruct the closures and environments - let sercloss = serobj.closures; - - let serenvs = serobj.envs; + const sercloss = serobj.closures; + const serenvs = serobj.envs; function mkClosure(i: number) { - if (!ctxt.closures[i]) { - let nm = ctxt.namespaces[sercloss[i].namespacePtr.NamespaceID] - let fn = nm[sercloss[i].fun]; - let env = mkEnv(sercloss[i].envptr.EnvID, (env) => { + if (!ctxt.closures[i]) { + const nm = ctxt.namespaces[sercloss[i].namespacePtr.NamespaceID] + const fn = nm[sercloss[i].fun]; + const env = mkEnv(sercloss[i].envptr.EnvID, (env) => { ctxt.closures[i] = RawClosure(env, nm, fn); - }) - ctxt.closures[i].__dataLevel = env.__dataLevel; + }) + ctxt.closures[i].__dataLevel = env.__dataLevel; } return ctxt.closures[i]; } function mkEnv(i: number, post_init?: (any)=>void ) { - if (!ctxt.envs[i]) { + if (!ctxt.envs[i]) { let env = {__dataLevel : levels.BOT}; if (post_init) { - post_init (env) + post_init (env); } ctxt.envs[i] = env; for (var field in serenvs[i]) { - let v = mkValue(serenvs[i][field]); - env[field] = v + const v = mkValue(serenvs[i][field]); + env[field] = v; env.__dataLevel = levels.lub (env.__dataLevel, v.dataLevel) - } + } } else { if (post_init) { post_init (ctxt.envs[i]); @@ -210,68 +201,53 @@ function constructCurrent(compilerOutput: string) { for (let i = 0; i < x.length; i++) { a.push(mkValue(x[i])); } - return a + return a; } - /* - # # - # # # # # # ## # # # ###### - ## ## # # # # # # # # # # - # ## # #### # # # # # # # ##### - # # # # # # ###### # # # # - # # # # # # # # # # # # - # # # # # # # ###### #### ###### - - */ - function mkValue(arg: { val: any; lev: any; tlev: any; troupeType: Ty.TroupeType; }) { - // debuglog ("*** mkValue", arg); assert(Ty.isLVal(arg)); - let obj = arg.val; - let lev = mkLevel(arg.lev); - let tlev = mkLevel(arg.tlev); + const obj = arg.val; + const lev = mkLevel(arg.lev); + const tlev = mkLevel(arg.tlev); function _trustGLB(x: Level) { - return (glb(x, __trustLevel)) + return glb(x, __trustLevel); } - let _tt = arg.troupeType - - - function value() { - switch (_tt) { + function value() { + switch (arg.troupeType) { case Ty.TroupeType.RECORD: - // for reords, the serialization format is [[key, value_json], ...] + // for records, the serialization format is [[key, value_json], ...] let a = []; for (let i = 0; i < obj.length; i++) { - a.push ([ obj[i][0], mkValue(obj[i][1]) ]) + a.push ([ obj[i][0], mkValue(obj[i][1]) ]); } return Record.mkRecord(a); case Ty.TroupeType.LIST: - return mkList(deserializeArray(obj)) + return mkList(deserializeArray(obj)); case Ty.TroupeType.TUPLE: - return mkTuple(deserializeArray(obj)) + return mkTuple(deserializeArray(obj)); case Ty.TroupeType.CLOSURE: - return mkClosure(obj.ClosureID) - case Ty.TroupeType.NUMBER: - case Ty.TroupeType.BOOLEAN: + return mkClosure(obj.ClosureID); + case Ty.TroupeType.NUMBER: + case Ty.TroupeType.BOOLEAN: case Ty.TroupeType.STRING: return obj; case Ty.TroupeType.PROCESS_ID: return new ProcessID(obj.uuid, obj.pid, obj.node) case Ty.TroupeType.AUTHORITY: - // 2018-10-18: AA: authority attenuation based on the trust level of the sender - return new Authority(_trustGLB(mkLevel(obj.authorityLevel))) + // Attenuate authority based on the trust level of the sender + return new Authority(_trustGLB(mkLevel(obj.authorityLevel))); case Ty.TroupeType.LEVEL: - return mkLevel(obj.lev) + return mkLevel(obj.lev); case Ty.TroupeType.LVAL: - return mkValue(obj) + return mkValue(obj); case Ty.TroupeType.ATOM: - return new Atom(obj.atom, obj.creation_uuid) + return new Atom(obj.atom, obj.creation_uuid); case Ty.TroupeType.UNIT: - return __unitbase + return __unitbase; default: - return obj; + return obj; } } @@ -288,12 +264,10 @@ function constructCurrent(compilerOutput: string) { let v = mkValue(serobj.value); - // go over the namespaces we have generated - // and load all libraries before calling the last callback - + // For each namespace we have generated, load all libraries before calling the last callback. function loadLib(i: number, cb) { if (i < ctxt.namespaces.length) { - __rtObj.linkLibs(ctxt.namespaces[i]).then(() => loadLib(i + 1, cb)) + __rtObj.linkLibs(ctxt.namespaces[i]).then(() => loadLib(i + 1, cb)); } else { cb(); } @@ -302,46 +276,38 @@ function constructCurrent(compilerOutput: string) { loadLib(0, () => desercb(v)); } -// 2018-11-30: AA: TODO: implement a proper deserialization queue instead of -// the coarse-grained piggybacking on the event loop - +// TODO: Implement a proper deserialization queue instead of the coarse-grained piggybacking on the +// event loop below. function deserializeCb(lev: Level, jsonObj: any, cb: (body: LVal) => void) { if (__isCurrentlyUsingCompiler) { - setImmediate(deserializeCb, lev, jsonObj, cb) // postpone; 2018-03-04;aa + // Other thread is currently deserializing, postpone execution. + setImmediate(deserializeCb, lev, jsonObj, cb); } else { - __isCurrentlyUsingCompiler = true // prevent parallel deserialization attempts; important! -- leads to nasty - // race conditions otherwise; 2018-11-30; AA + // Prevent parallel deserialization attempts (abuses that JavaScript is a singly threaded + // language). Be wary when messing with the variables below, they are all global! + __isCurrentlyUsingCompiler = true; __trustLevel = lev; - __currentCallback = cb; // obs: this is a global for this module; - // the access to it should be carefully controlled - - // we need to share this object with the callbacks - - __currentDeserializedJson = jsonObj; // obs: another global that we must be careful with + __currentCallback = cb; + __currentDeserializedJson = jsonObj; if (jsonObj.namespaces.length > 0) { for (let i = 0; i < jsonObj.namespaces.length; i++) { let ns = jsonObj.namespaces[i]; for (let j = 0; j < ns.length; j++) { - // debuglog("*s deserialize", ns[j]); __compilerOsProcess.stdin.write(ns[j][1]); - __compilerOsProcess.stdin.write("\n") - // debuglog ("data out") + __compilerOsProcess.stdin.write("\n"); } } - __compilerOsProcess.stdin.write("!ECHO /*-----*/\n") + __compilerOsProcess.stdin.write("!ECHO " + MARKER + "\n"); } else { - // shortcutting the unnecessary interaction with the compiler - // 2018-09-20: AA - constructCurrent(""); + // Unnecessary interaction with the compiler: skip it! + constructCurrent(null); } } } export function deserialize(lev: Level, jsonObj: any): Promise { return new Promise((resolve, reject) => { - deserializeCb(lev, jsonObj, (body: LVal) => { - resolve(body) - }) + deserializeCb(lev, jsonObj, (body: LVal) => resolve(body)); }); } diff --git a/rt/src/loadLibs.mts b/rt/src/loadLibs.mts deleted file mode 100644 index 9ba07192..00000000 --- a/rt/src/loadLibs.mts +++ /dev/null @@ -1,101 +0,0 @@ -/* 2020-05-19: AA This code is deprecated */ - -'use strict' - -import * as fs from 'node:fs' - -import { mkLogger } from './logger.mjs' -const logger = mkLogger('lib') - -const info = x => logger.info(x) -const debug = x => logger.debug(x) - -const __libcache = {} - -export function loadLibsAsync(obj, rtObj, cb) { - let libs = obj.libs - obj.libs = {} - function iterateAsync(n) { - if (n < libs.length) { - let lib = libs[n].lib; - let decl = libs[n].decl; - - const key = lib +"." + decl - if (__libcache[key]) { - debug ('lib cache hit on: ' + key) - obj.libs[key]=__libcache[key]; - setImmediate(iterateAsync, n + 1); - return; - } - - // 1. Find the file -- note that we load all the libs from a default - // location - - let filename = process.env.TROUPE + "/lib/out/" + lib + ".js" - - - - // 2. Load the file -- note that this is an asynchronous operation - fs.readFile(filename, 'utf8', (err, input) => { - - // File read operation finished; we are now in the callbacak that has - // been asynchronously called by the node runtime - - // TODO: check for error! 2018-07-03: aa - - // 3. Create a JS class (function) from it - let Lib:any = new Function('rt', input); - - // 4. We create a "new" instance of the resulting class - - let libinstance = new Lib(rtObj); - - - // load dependent libraries?? - - // libinstance.loadlibs (() => - loadLibsAsync(libinstance, rtObj, () => { - // 5. Execute .export() function to obtain the table note - this is a - // regular JS function (generated by the compiler) that we just call - // here - - rtObj.setLibloadMode(); // 2019-01-03: AA; Hack - let table = libinstance.export().val.toArray(); - rtObj.setNormalMode(); // 2019-01-03: AA; EOH - - // 6. Lookup in the resulting table - - for (let i = 0; i < table.length; i++) { - let name = table[i].val[0].val; - let libf = table[i].val[1].val - if (name == decl) { - // We store the resulting function in the object that was provided - // to us as an argument - obj.libs[key] = libf; - __libcache [key] = libf; - break; - } - } - - // Next iteration - iterateAsync (n + 1); - }) - }) - - } else { - // We are done processing the lib files. Transferring control back to the - // callback. The callback is either - // - // a. The next thing in the initialization, if this is the first time we - // are loading libraries -- typically scheduler init, etc (see `start` - // function in the runtime), OR - // - // b. The next iteration in deserialization, which is more library loading - // when we have several namespaces, or whatever is the deserialization - // callback (see `mkValue` function in the serialize module). - - cb(); - } - } - iterateAsync (0); -} diff --git a/rt/src/loadLibsAsync.mts b/rt/src/loadLibsAsync.mts index 12adc2ca..54616a4b 100644 --- a/rt/src/loadLibsAsync.mts +++ b/rt/src/loadLibsAsync.mts @@ -1,72 +1,71 @@ 'use strict' import * as fs from 'node:fs' import * as levels from './Level.mjs'; -const { readFile } = fs.promises +const { readFile } = fs.promises; -import { mkLogger } from './logger.mjs' -const logger = mkLogger('lib') +import { mkLogger } from './logger.mjs'; +const logger = mkLogger('lib'); -const info = x => logger.info(x) -const debug = x => logger.debug(x) +const info = x => logger.info(x); +const debug = x => logger.debug(x); -const __libcache = {} +const __libcache = {}; export async function loadLibsAsync(obj, rtObj) { - let libs = obj.libs - obj.libs = {} + let libs = obj.libs; + obj.libs = {}; for (let n = 0; n < libs.length; n++) { let lib = libs[n].lib; let decl = libs[n].decl; - const key = lib + "." + decl + const key = lib + "." + decl; if (__libcache[key]) { - debug('lib cache hit on: ' + key) + debug('lib cache hit on: ' + key); obj.libs[key] = __libcache[key]; - continue + continue; } - // 1. Find the file -- note that we load all the libs from a default - // location + // 1. Find the file. Note, that we load all the libs from a default + // location. + let filename = process.env.TROUPE + "/lib/out/" + lib + ".js"; - let filename = process.env.TROUPE + "/lib/out/" + lib + ".js" - - // 2. Load the file -- note that this is an asynchronous operation - let input = await readFile(filename, 'utf8') + // 2. Load the file. Note, this is an asynchronous operation + let input = await readFile(filename, 'utf8'); // File read operation finished; we are now in the callbacak that has - // been asynchronously called by the node runtime + // been asynchronously called by the node runtime. - // TODO: check for error! 2018-07-03: aa + // TODO: check for error! 2018-07-03: AA // 3. Create a JS class (function) from it let Lib: any = new Function('rt', input); // 4. We create a "new" instance of the resulting class - let libinstance = new Lib(rtObj); + // load dependent libraries + await loadLibsAsync(libinstance, rtObj); - // load dependent libraries?? - - // libinstance.loadlibs (() => - await loadLibsAsync(libinstance, rtObj) - - // 5. Execute .export() function to obtain the table note - this is a + // 5. Execute `.export()` function to obtain the table note - this is a // regular JS function (generated by the compiler) that we just call - // here - - rtObj.setLibloadMode(); // 2019-01-03: AA; Hack + // here. + // + // 2019-01-03: AA; HACK + // We assume that the library merely exports values that require no + // computations. Hence, there are no continuations to be resolved and + // we can immediately extract the list of functions/values returned + // from the given function. + rtObj.setLibloadMode(); let table = libinstance.export({__dataLevel:levels.BOT}).val.toArray(); - rtObj.setNormalMode(); // 2019-01-03: AA; EOH - - // 6. Lookup in the resulting table + rtObj.setNormalMode(); + // 6. Lookup the desired value in the resulting table for (let i = 0; i < table.length; i++) { let name = table[i].val[0].val; - let libf = table[i].val[1].val + let libf = table[i].val[1].val; if (name == decl) { // We store the resulting function in the object that was provided - // to us as an argument + // to us as an argument. obj.libs[key] = libf; __libcache[key] = libf; break; diff --git a/rt/src/runId.mts b/rt/src/runId.mts index 0dbb125f..40c1f5f6 100644 --- a/rt/src/runId.mts +++ b/rt/src/runId.mts @@ -1,3 +1,3 @@ -import { v4 as uuidv4} from 'uuid' -let runId = uuidv4() +import { v4 as uuidv4 } from 'uuid' +const runId = uuidv4() export default runId diff --git a/rt/src/runtimeMonitored.mts b/rt/src/runtimeMonitored.mts index 1c54578c..bed187ff 100644 --- a/rt/src/runtimeMonitored.mts +++ b/rt/src/runtimeMonitored.mts @@ -4,13 +4,14 @@ import { v4 as uuidv4 } from 'uuid' import AggregateError from 'aggregate-error'; import { __unit } from './UnitVal.mjs' import { Authority } from './Authority.mjs' -import { Scheduler } from './Scheduler.mjs' +import { Scheduler, ThreadType } from './Scheduler.mjs' import { MailboxProcessor } from './MailboxProcessor.mjs' import { RuntimeInterface } from './RuntimeInterface.mjs' import { LVal, MbVal } from './Lval.mjs' import { ProcessID } from './process.mjs'; import { UserRuntime } from './UserRuntime.mjs' import * as levels from './Level.mjs' +const { flowsTo, lub, glb } = levels import * as DS from './deserialize.mjs' import { p2p } from './p2p/p2p.mjs' import { closeReadline } from './builtins/stdio.mjs'; @@ -22,10 +23,7 @@ import { setRuntimeObject } from './SysState.mjs'; import { initTrustMap, nodeTrustLevel, _trustMap } from './TrustManager.mjs'; import { serialize } from './serialize.mjs'; import { Thread } from './Thread.mjs'; - import { Console } from 'node:console' - -const { flowsTo, lub, glb } = levels import { getCliArgs, TroupeCliArg } from './TroupeCliArgs.mjs'; import { configureColors, isColorEnabled } from './colorConfig.mjs'; import { mkLogger } from './logger.mjs' @@ -33,38 +31,32 @@ import { Record } from './Record.mjs'; import { level } from 'winston'; const readFile = fs.promises.readFile -const rt_uuid = runId const argv = getCliArgs(); // Configure colors before any chalk or logger usage configureColors(); -let logLevel = argv[TroupeCliArg.Debug] ? 'debug': 'info' +const logLevel = argv[TroupeCliArg.Debug] ? 'debug': 'info' const logger = mkLogger('RTM', logLevel); -const info = x => logger.info(x) -const debug = x => logger.debug(x) -const error = x => logger.error(x) - let __p2pRunning = false; - -let rt_xconsole = +const rt_xconsole = new Console({ stdout: process.stdout , stderr: process.stderr , colorMode: isColorEnabled() }); -function $t():Thread { return __sched.__currentThread }; // returns the current thread +/** Returns the current thread */ +function $t():Thread { return __sched.getCurrentThread() }; // -------------------------------------------------- async function spawnAtNode(nodeid, f) { - debug (`* rt spawnAtNode ${nodeid}`); + logger.debug(`* rt spawnAtNode ${nodeid}`); let node = __nodeManager.getNode(nodeid.val); - // debug ("XX", node); - // TODO: 2018-09-24: AA: do the information flow check + // TODO (2018-09-24: AA): do the information flow check let { data, level } = serialize(f, lub($t().pc, nodeid.lev)); @@ -74,27 +66,24 @@ async function spawnAtNode(nodeid, f) { if (!flowsTo(level, trustLevel)) { theThread.throwInSuspended("Illegal trust flow when spawning on a remote node\n" + ` | the trust level of the recepient node: ${trustLevel.stringRep()}\n` + - ` | the level of the information in spawn: ${level.stringRep()}`) + ` | the level of the information in spawn: ${level.stringRep()}`); __sched.scheduleThread(theThread); - __sched.resumeLoopAsync(); + __sched.resumeLoopAsync(); return; } // 0. we assume that the node is different from // the local node - // 1. we make a connection to the remote node // 2. we send the serialized version of f // 3. we wait for the reply (should be a pid) // 4. we return the obtained pid //-------------------------------------------------- - - try { let body1 = await p2p.spawnp2p(node.nodeId, data); - let body = await DS.deserialize(nodeTrustLevel(node.nodeId), body1) + let body = await DS.deserialize(nodeTrustLevel(node.nodeId), body1); let pid = new ProcessID(body.val.uuid, body.val.pid, body.val.node); theThread.returnSuspended(new LVal(pid, body.lev)); @@ -102,13 +91,13 @@ async function spawnAtNode(nodeid, f) { __sched.resumeLoopAsync(); } catch (err) { - error("error spawning remotely; this blocks current thread") + logger.error("error spawning remotely; this blocks current thread") if (err instanceof AggregateError) { for (let ie in err) { - error(`${ie}`) + logger.error(`${ie}`); } } else { - error(`${err}`) + logger.error(`${err}`); } } } @@ -136,16 +125,16 @@ function remoteSpawnOK() { * The identity of the node that initiates the spawning. */ async function spawnFromRemote(jsonObj, fromNode) { - debug ("spawn from remote") + logger.debug("spawn from remote"); // 2018-05-17: AA; note that this _only_ uses the lf.lev and // is completely independent of the current thread's pc; - let nodeLev = nodeTrustLevel(fromNode); + const nodeLev = nodeTrustLevel(fromNode); - let lf = await DS.deserialize(nodeLev, jsonObj) - let f = lf.val; - let newPid = - __sched.scheduleNewThreadAtLevel( + const lf = await DS.deserialize(nodeLev, jsonObj); + const f = lf.val; + const tid = + __sched.scheduleNewThread( f , __unit //[f.env, __unit] // , f.namespace @@ -156,9 +145,9 @@ async function spawnFromRemote(jsonObj, fromNode) { // 2018-09-19: AA: because we need to send some info back, we have to invoke // serialization. - let serObj = serialize(newPid, levels.BOT).data + const serObj = serialize(tid, levels.BOT).data __sched.resumeLoopAsync(); - return (serObj); + return serObj; } @@ -173,18 +162,18 @@ async function spawnFromRemote(jsonObj, fromNode) { * The node identity of the sender node */ async function receiveFromRemote(pid, jsonObj, fromNode) { - debug(`* rt receiveFromremote * ${JSON.stringify(jsonObj)}`) - let data = await DS.deserialize(nodeTrustLevel(fromNode), jsonObj) - debug(`* rt receiveFromremote * ${fromNode} ${data.stringRep()}`); + // Deserialize the data to runtime values, either directly or via the `troupec` compiler + logger.debug(`* rt receiveFromRemote * ${JSON.stringify(jsonObj)}`); + const data = await DS.deserialize(nodeTrustLevel(fromNode), jsonObj); + logger.debug(`* rt receiveFromRemote * ${fromNode} ${data.stringRep()}`); - // TODO: 2018-07-23: do we need to do some more raising - // about the level of the fromNode?; AA + // TODO (AA; 2018-07-23): do we need to do some more reasoning about the level of the fromNode? - let fromNodeId = __sched.mkVal(fromNode); - let toPid = new LVal(new ProcessID(rt_uuid, pid, __nodeManager.getLocalNode()), data.lev); + // If successful, add the deserialized message to the mailbox of said process. + const fromNodeId = $t().mkVal(fromNode); + const toPid = new LVal(new ProcessID(runId, pid, __nodeManager.getLocalNode()), data.lev); __theMailbox.addMessage(fromNodeId, toPid, data.val, data.lev); __sched.resumeLoopAsync(); - } @@ -198,33 +187,39 @@ async function receiveFromRemote(pid, jsonObj, fromNode) { * */ function sendMessageToRemote(toPid, message) { - let node = toPid.node.nodeId; - let pid = toPid.pid; - // debug (`* rt * ${toPid} ${message.stringRep()}`); - - let { data, level } = serialize(new MbVal(message, $t().pc), $t().pc); + const node = toPid.node.nodeId; + const pid = toPid.pid; - // debug (`* rt * ${JSON.stringify(data)}`); - let trustLevel = nodeTrustLevel(node); + const { data, level } = serialize(new MbVal(message, $t().pc), $t().pc); - // debug ("data level: " + level.stringRep()); - // debug ("remote trust level: " + trustLevel.stringRep()); + const trustLevel = nodeTrustLevel(node); if (!flowsTo(level, trustLevel)) { - threadError("Illegal trust flow when sending information to a remote node\n" + - ` | the trust level of the recepient node: ${trustLevel.stringRep()}\n` + - ` | the level of the information to send: ${level.stringRep()}`); + $t().threadError("Illegal trust flow when sending information to a remote node\n" + + ` | the trust level of the recepient node: ${trustLevel.stringRep()}\n` + + ` | the level of the information to send: ${level.stringRep()}`, + false); } else { + // we return unit to the call site at the thread level p2p.sendp2p(node, pid, data) - return $t().returnImmediateLValue(__unit); // we return unit to the call site at the thread level + return $t().returnImmediateLValue(__unit); + } +} + + +async function whereisFromRemote(k) { + __sched.resumeLoopAsync() + // TODO (AA; 2018-10-20): Make use of the levels as they were + // recorded during the registration (instead of the bottom here) + if (__theRegister[k]) { + return serialize(__theRegister[k], levels.BOT).data; } } -// TODO: AA; 2020-05-19; consider moving these two functions somewhere else +// TODO (AA; 2020-05-19): consider moving these two functions somewhere else function isLocalPid(pid) { - let x = pid.uuid.toString() == rt_uuid.toString(); - return (x); + return pid.uuid.toString() == runId.toString();; } function rt_mkuuid() { @@ -235,25 +230,22 @@ function rt_mkuuid() { function rt_sendMessageNochecks(lRecipientPid, message, ret = true) { let recipientPid = lRecipientPid.val; - // debug (`rt_sendMessageNoChecks ${message.stringRep()}`) if (isLocalPid(recipientPid)) { - let nodeId = __sched.mkVal(__nodeManager.getNodeId()); + let nodeId = $t().mkVal(__nodeManager.getNodeId()); __theMailbox.addMessage(nodeId, lRecipientPid, message, $t().pc); if (ret) { - return $t().returnImmediateLValue(__unit); + return $t().returnImmediateLValue(__unit); } } else { - debug ("* rt rt_send remote *"/*, recipientPid, message*/); + logger.debug ("* rt rt_send remote *"/*, recipientPid, message*/); return sendMessageToRemote(recipientPid, message) } } - - -let rt_debug = function (s) { +function rt_debug (s) { function formatToN(s, n) { if (s.length < n) { let j = s.length; @@ -264,168 +256,107 @@ let rt_debug = function (s) { return s; } - let tid = $t().tidErrorStringRep() - let pc = $t().pc.stringRep() - let bl = $t().bl.stringRep() - let handler_state = __sched.handlerState.toString() + const tid = $t().tidErrorStringRep(); + const pc = $t().pc.stringRep(); + const bl = $t().bl.stringRep(); + const handler_state = $t().handlerState.toString(); rt_xconsole.log( chalk.red(formatToN("PID:" + tid, 50)), chalk.red(formatToN("PC:" + pc, 20)), chalk.red(formatToN("BL:" + bl, 20)), chalk.red(formatToN("HN" + handler_state, 20)), chalk.red(formatToN("_sp:" + $t()._sp, 20)), - s + s ); } - - -async function whereisFromRemote(k) { - __sched.resumeLoopAsync() - // TODO: 2018-10-20: make use of the levels as they were - // recorded during the registration (instead of the bottom here ) - if (__theRegister[k]) { - let serObj = serialize(__theRegister[k], levels.BOT).data - return serObj - } -} - - - function rt_mkLabel(x) { - // debug ("mkLabel", x, x === "secret"); - - return new LVal(levels.fromSingleTag(x), $t().pc); - } - - - -function threadError(s, internal = false) { - return $t().threadError(s, internal); -} - -let rt_threadError = threadError; - -function rt_error(x) { - threadError(x.val); -} - -function rt_errorPos(x, pos) { - if (pos != '') { - threadError(x.val + " at " + pos); - } else { - threadError(x.val); - } +function rt_ret (arg) { + return $t().returnImmediateLValue(arg); } - -let rt_ret = (arg) => { return $t().returnImmediateLValue(arg); } -// let rt_ret_raw = () => __sched.returnInThread_raw(); - -// function tailcall(lff, arg) { -// assertIsFunction(lff); -// $t().raiseCurrentThreadPC(lff.lev); -// __sched.tailToTroupeFun(lff.val, arg); -// } - +// TODO: Clean up the mess below... let __sched: Scheduler let __theMailbox: MailboxProcessor let __userRuntime: any let __service:any = {} class RuntimeObject implements RuntimeInterface { - // tailcall = tailcall - xconsole = rt_xconsole - ret = rt_ret - // ret_raw = rt_ret_raw - debug = rt_debug - spawnAtNode = spawnAtNode - rt_mkuuid = rt_mkuuid - mkLabel = rt_mkLabel + xconsole = rt_xconsole; + ret = rt_ret; + debug = rt_debug; + spawnAtNode = spawnAtNode; + rt_mkuuid = rt_mkuuid; + mkLabel = rt_mkLabel; sendMessageNoChecks = rt_sendMessageNochecks; - cleanup = cleanupAsync + cleanup = cleanupAsync; persist(obj, path) { let jsonObj = serialize(obj, $t().pc).data; fs.writeFileSync(path, JSON.stringify(jsonObj)); } get $service () { - return __service + return __service; } - + get $t() { - return $t() + return $t(); } get __sched() { - return __sched + return __sched; } get __mbox() { - return __theMailbox + return __theMailbox; } get __userRuntime() { - return __userRuntime + return __userRuntime; } constructor() { - __sched = new Scheduler(this) - __theMailbox = new MailboxProcessor(this) - __userRuntime = new UserRuntime(this) + __sched = new Scheduler(this); + __theMailbox = new MailboxProcessor(this); + __userRuntime = new UserRuntime(this); } - } - let __rtObj = new RuntimeObject(); DS.setRuntimeObj(__rtObj.__userRuntime); setRuntimeObject(__rtObj) - - async function cleanupAsync() { closeReadline() DS.stopCompiler(); if (__p2pRunning) { try { - debug("stopping p2p") + logger.debug("stopping p2p") await p2p.stopp2p() - debug("p2p stop OK") + logger.debug("p2p stop OK") } catch (err) { - debug(`p2p stop failed ${err}`) + logger.debug(`p2p stop failed ${err}`) } } } - // 2020-02-09; AA; ugly ugly hack function bulletProofSigint() { - let listeners = process.listeners("SIGINT"); - // console.log (util.inspect(listeners)) - // for (let i = 0; i < listeners.length; i ++ ) { - // console.log (listeners[i].toString()); - // } - - // process.stdin.removeAllListeners("on"); process.removeAllListeners("SIGINT"); - // console.log ("sigint bulletproofing") process.on('SIGINT', () => { - debug("SIGINT"); + logger.debug("SIGINT"); (async () => { await cleanupAsync() process.exit(0); })() }) - // setTimeout (bulletProofSigint, 1000) } bulletProofSigint(); - async function loadServiceCode() { let input = await fs.promises.readFile(process.env.TROUPE + '/trp-rt/out/service.js', 'utf8') let S: any = new Function('rt', input) @@ -445,15 +376,12 @@ async function loadServiceCode() { } - async function getNetworkPeerId(rtHandlers) { const nodeIdFile = argv[TroupeCliArg.Id] as string; if (nodeIdFile) { try { let nodeIdObj = await readFile(nodeIdFile, 'utf-8') process.on('unhandledRejection', (e) => p2p.processExpectedNetworkErrors(e, "unhandledRejection")) - // process.on ('unhandledRejection', up => {console.log ("Unhandled rejection"); console.error (up)}) - // process.on ('uncaughtException', up => {console.log ("Uncaught exception"); console.error (up)}) process.on('uncaughtException', (e) => p2p.processExpectedNetworkErrors(e, "uncaughtException")) return await p2p.startp2p(JSON.parse(nodeIdObj), rtHandlers); } catch (err) { @@ -463,73 +391,95 @@ async function getNetworkPeerId(rtHandlers) { } else { try { if (argv[TroupeCliArg.LocalOnly] || argv[TroupeCliArg.Persist]) { - info("Skipping network creation. Observe that all external IO operations will yield a runtime error.") + logger.info("Skipping network creation. Observe that all external IO operations will yield a runtime error."); if (argv[TroupeCliArg.Persist]) { - info("Running with persist flag.") + logger.info("Running with persist flag."); } - return null// OBS: 2018-07-22: we are jumping over the network creation + // OBS: 2018-07-22: we are jumping over the network creation + return null; } else { return await p2p.startp2p(null, rtHandlers); } } catch (err) { - logger.error("uncaught exception in the runtime") - console.error(err.stack);; + logger.error("uncaught exception in the runtime"); + console.error(err.stack); process.exit(1); } } } + export async function start(f) { - await initTrustMap() + // Set up p2p network + await initTrustMap(); let peerid = await getNetworkPeerId({ remoteSpawnOK, spawnFromRemote, receiveFromRemote, whereisFromRemote - }) + }); if (peerid) { - __p2pRunning = true - debug("network ready") + __p2pRunning = true; + logger.debug("network ready"); } else { - debug("network not initialized") + logger.debug("network not initialized"); } __nodeManager.setLocalPeerId(peerid); - let stopWhenAllThreadsAreDone = !__p2pRunning - __sched.initScheduler(__nodeManager.getLocalNode() - , stopWhenAllThreadsAreDone - , cleanupAsync); + // --------------------------------------------------------------------------- + // Initialise 'scheduler' for Troupe code execution + __sched.initScheduler(__nodeManager.getLocalNode() , !__p2pRunning, cleanupAsync); - await loadServiceCode() - await __userRuntime.linkLibs(f) - let mainAuthority = new LVal(new Authority(levels.ROOT), levels.BOT); + // --------------------------------------------------------------------------- + // Set up 'service' thread + + // HACK: Despite the fact that service code is only spawned, if `__p2pRunning`, + // we need to populate the runtime.$service object. + // + // TODO: Instead, treat these fields as nullable in `builtins/receive.mts` and + // elsewhere. Best is to also put this into the typesystem. + await loadServiceCode(); if (__p2pRunning) { - let service_arg = - new LVal ( new Record([ ["authority", mainAuthority], - ["options", __unit]]), + const serviceAuthority = new LVal(new Authority(levels.ROOT), levels.BOT); + + let service_arg = + new LVal ( new Record([ ["authority", serviceAuthority], + ["options", __unit]]), levels.BOT); - __sched.scheduleNewThreadAtLevel(__service['service'] + __sched.scheduleNewThread(__service['service'] , service_arg , levels.TOP , levels.BOT - , false - , null - , true); + , ThreadType.System); } - __sched.scheduleNewThreadAtLevel( - () => f.main ({__dataLevel:levels.BOT}) + // Set up 'main' thread + const mainAuthority = new LVal(new Authority(levels.ROOT), levels.BOT); + + await __userRuntime.linkLibs(f); + + const onTerminate = (retVal: LVal) => { + console.log(`>>> Main thread finished with value: ${retVal.stringRep()}`); + if (argv[TroupeCliArg.Persist]) { + this.rtObj.persist(retVal, argv[TroupeCliArg.Persist]) + console.log("Saved the result value in file", argv[TroupeCliArg.Persist]) + } + }; + + __sched.scheduleNewThread( + () => f.main({__dataLevel:levels.BOT}) , mainAuthority - // , f , levels.BOT , levels.BOT - , true - , argv[TroupeCliArg.Persist] - ) - __sched.loop() -} + , ThreadType.Main + , onTerminate + ); + // --------------------------------------------------------------------------- + // Start code execution + __sched.resumeLoopAsync(); +} diff --git a/rt/src/troupe.mts b/rt/src/troupe.mts index 981ac8c6..82079eee 100644 --- a/rt/src/troupe.mts +++ b/rt/src/troupe.mts @@ -19,9 +19,8 @@ if (!fs.existsSync(p)) { } (async () => { let d = await import (p); - let Top = d.default - let __userRuntime = (getRuntimeObject() as any).__userRuntime; - let top = new Top(__userRuntime); + let Top = d.default; + let top = new Top(getRuntimeObject().__userRuntime); start(top); }) ()